== conf-sanity test 102: obdclass module cleanup upon error ========================================================== 07:59:34 (1688471974)
umount lustre on /mnt/lustre.....
stop ost1 service on oleg310-server
stop mds service on oleg310-server
stop mds service on oleg310-server
unloading modules on: 'oleg310-server'
oleg310-server: oleg310-server.virtnet: executing unload_modules_local
modules unloaded.
++ uname -r
+ do_facet mds1 ls -ali /usr/lib/modules/3.10.0-7.9-debug/extra/kernel/net/lustre
+ local verbose
+ local quiet
+ [[ mds1 == \-\-\v\e\r\b\o\s\e ]]
+ [[ mds1 == \-\-\q\u\i\e\t ]]
+ [[ mds1 == \-\q ]]
+ local facet=mds1
+ shift
++ facet_active_host mds1
+++ facet_active mds1
+++ local facet=mds1
+++ local activevar=mds1active
+++ '[' -f /tmp/mds1active ']'
+++ active=
+++ '[' -z '' ']'
+++ echo -n mds1
++ facet_host mds1
++ local facet=mds1
++ local varname
++ '[' mds1 == client ']'
++ varname=mds1_HOST
++ '[' -z '' ']'
++ '[' mds == ost ']'
++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
++ local temp
++ '[' mds1 == mgsfailover ']'
++ temp=oleg310-server
++ eval export mds1_HOST=oleg310-server
+++ export mds1_HOST=oleg310-server
+++ mds1_HOST=oleg310-server
++ echo -n oleg310-server
+ local host=oleg310-server
+ '[' -z oleg310-server ']'
+ do_node oleg310-server ls -ali /usr/lib/modules/3.10.0-7.9-debug/extra/kernel/net/lustre
+ local verbose
+ local quiet
+ [[ oleg310-server == \-\-\v\e\r\b\o\s\e ]]
+ [[ oleg310-server == \-\-\q\u\i\e\t ]]
+ [[ oleg310-server == \-\q ]]
+ local HOST=oleg310-server
+ shift
+ local 'myPDSH=pdsh -S -Rssh -w'
+ '[' oleg310-server = oleg310-client.virtnet ']'
+ '[' -z 'pdsh -S -Rssh -w' -o 'pdsh -S -Rssh -w' = no_dsh ']'
+ false
+ [[ pdsh -S -Rssh -w == \r\s\h ]]
+ [[ pdsh -S -Rssh -w == *pdsh* ]]
+ [[ pdsh -S -Rssh -w != *-S* ]]
+ [[ -n '' ]]
+ pdsh -S -Rssh -w oleg310-server '(PATH=$PATH:/home/green/git/lustre-release/lustre/utils:/home/green/git/lustre-release/lustre/tests:/sbin:/usr/sbin;		cd /root; LUSTRE="/home/green/git/lustre-release/lustre" bash -c "ls -ali /usr/lib/modules/3.10.0-7.9-debug/extra/kernel/net/lustre")'
+ sed 's/^oleg310-server: //'
oleg310-server: ls: cannot access /usr/lib/modules/3.10.0-7.9-debug/extra/kernel/net/lustre: No such file or directory
pdsh@oleg310-client: oleg310-server: ssh exited with exit code 2
+ return 2
+ do_facet mds1 depmod -a
+ local verbose
+ local quiet
+ [[ mds1 == \-\-\v\e\r\b\o\s\e ]]
+ [[ mds1 == \-\-\q\u\i\e\t ]]
+ [[ mds1 == \-\q ]]
+ local facet=mds1
+ shift
++ facet_active_host mds1
+++ facet_active mds1
+++ local facet=mds1
+++ local activevar=mds1active
+++ '[' -f /tmp/mds1active ']'
+++ active=
+++ '[' -z '' ']'
+++ echo -n mds1
++ facet_host mds1
++ local facet=mds1
++ local varname
++ '[' mds1 == client ']'
++ varname=mds1_HOST
++ '[' -z '' ']'
++ '[' mds == ost ']'
++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
++ local temp
++ '[' mds1 == mgsfailover ']'
++ temp=oleg310-server
++ eval export mds1_HOST=oleg310-server
+++ export mds1_HOST=oleg310-server
+++ mds1_HOST=oleg310-server
++ echo -n oleg310-server
+ local host=oleg310-server
+ '[' -z oleg310-server ']'
+ do_node oleg310-server depmod -a
+ local verbose
+ local quiet
+ [[ oleg310-server == \-\-\v\e\r\b\o\s\e ]]
+ [[ oleg310-server == \-\-\q\u\i\e\t ]]
+ [[ oleg310-server == \-\q ]]
+ local HOST=oleg310-server
+ shift
+ local 'myPDSH=pdsh -S -Rssh -w'
+ '[' oleg310-server = oleg310-client.virtnet ']'
+ '[' -z 'pdsh -S -Rssh -w' -o 'pdsh -S -Rssh -w' = no_dsh ']'
+ false
+ [[ pdsh -S -Rssh -w == \r\s\h ]]
+ [[ pdsh -S -Rssh -w == *pdsh* ]]
+ [[ pdsh -S -Rssh -w != *-S* ]]
+ [[ -n '' ]]
+ pdsh -S -Rssh -w oleg310-server '(PATH=$PATH:/home/green/git/lustre-release/lustre/utils:/home/green/git/lustre-release/lustre/tests:/sbin:/usr/sbin;		cd /root; LUSTRE="/home/green/git/lustre-release/lustre" bash -c "depmod -a")'
+ sed 's/^oleg310-server: //'
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.dep.tmp, 1101, 644): Read-only file system
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.dep.bin.tmp, 1101, 644): Read-only file system
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.alias.tmp, 1101, 644): Read-only file system
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.alias.bin.tmp, 1101, 644): Read-only file system
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.softdep.tmp, 1101, 644): Read-only file system
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.symbols.tmp, 1101, 644): Read-only file system
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.symbols.bin.tmp, 1101, 644): Read-only file system
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.builtin.bin.tmp, 1101, 644): Read-only file system
oleg310-server: depmod: ERROR: openat(/lib/modules/3.10.0-7.9-debug, modules.devname.tmp, 1101, 644): Read-only file system
+ return 0
++ uname -r
+ do_facet mds1 ls -ali /usr/lib/modules/3.10.0-7.9-debug/extra/kernel/net/lustre
+ local verbose
+ local quiet
+ [[ mds1 == \-\-\v\e\r\b\o\s\e ]]
+ [[ mds1 == \-\-\q\u\i\e\t ]]
+ [[ mds1 == \-\q ]]
+ local facet=mds1
+ shift
++ facet_active_host mds1
+++ facet_active mds1
+++ local facet=mds1
+++ local activevar=mds1active
+++ '[' -f /tmp/mds1active ']'
+++ active=
+++ '[' -z '' ']'
+++ echo -n mds1
++ facet_host mds1
++ local facet=mds1
++ local varname
++ '[' mds1 == client ']'
++ varname=mds1_HOST
++ '[' -z '' ']'
++ '[' mds == ost ']'
++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
++ local temp
++ '[' mds1 == mgsfailover ']'
++ temp=oleg310-server
++ eval export mds1_HOST=oleg310-server
+++ export mds1_HOST=oleg310-server
+++ mds1_HOST=oleg310-server
++ echo -n oleg310-server
+ local host=oleg310-server
+ '[' -z oleg310-server ']'
+ do_node oleg310-server ls -ali /usr/lib/modules/3.10.0-7.9-debug/extra/kernel/net/lustre
+ local verbose
+ local quiet
+ [[ oleg310-server == \-\-\v\e\r\b\o\s\e ]]
+ [[ oleg310-server == \-\-\q\u\i\e\t ]]
+ [[ oleg310-server == \-\q ]]
+ local HOST=oleg310-server
+ shift
+ local 'myPDSH=pdsh -S -Rssh -w'
+ '[' oleg310-server = oleg310-client.virtnet ']'
+ '[' -z 'pdsh -S -Rssh -w' -o 'pdsh -S -Rssh -w' = no_dsh ']'
+ false
+ [[ pdsh -S -Rssh -w == \r\s\h ]]
+ [[ pdsh -S -Rssh -w == *pdsh* ]]
+ [[ pdsh -S -Rssh -w != *-S* ]]
+ [[ -n '' ]]
+ pdsh -S -Rssh -w oleg310-server '(PATH=$PATH:/home/green/git/lustre-release/lustre/utils:/home/green/git/lustre-release/lustre/tests:/sbin:/usr/sbin;		cd /root; LUSTRE="/home/green/git/lustre-release/lustre" bash -c "ls -ali /usr/lib/modules/3.10.0-7.9-debug/extra/kernel/net/lustre")'
+ sed 's/^oleg310-server: //'
oleg310-server: ls: cannot access /usr/lib/modules/3.10.0-7.9-debug/extra/kernel/net/lustre: No such file or directory
pdsh@oleg310-client: oleg310-server: ssh exited with exit code 2
+ return 2
+ do_facet mds1 modprobe libcfs
+ local verbose
+ local quiet
+ [[ mds1 == \-\-\v\e\r\b\o\s\e ]]
+ [[ mds1 == \-\-\q\u\i\e\t ]]
+ [[ mds1 == \-\q ]]
+ local facet=mds1
+ shift
++ facet_active_host mds1
+++ facet_active mds1
+++ local facet=mds1
+++ local activevar=mds1active
+++ '[' -f /tmp/mds1active ']'
+++ active=
+++ '[' -z '' ']'
+++ echo -n mds1
++ facet_host mds1
++ local facet=mds1
++ local varname
++ '[' mds1 == client ']'
++ varname=mds1_HOST
++ '[' -z '' ']'
++ '[' mds == ost ']'
++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
++ local temp
++ '[' mds1 == mgsfailover ']'
++ temp=oleg310-server
++ eval export mds1_HOST=oleg310-server
+++ export mds1_HOST=oleg310-server
+++ mds1_HOST=oleg310-server
++ echo -n oleg310-server
+ local host=oleg310-server
+ '[' -z oleg310-server ']'
+ do_node oleg310-server modprobe libcfs
+ local verbose
+ local quiet
+ [[ oleg310-server == \-\-\v\e\r\b\o\s\e ]]
+ [[ oleg310-server == \-\-\q\u\i\e\t ]]
+ [[ oleg310-server == \-\q ]]
+ local HOST=oleg310-server
+ shift
+ local 'myPDSH=pdsh -S -Rssh -w'
+ '[' oleg310-server = oleg310-client.virtnet ']'
+ '[' -z 'pdsh -S -Rssh -w' -o 'pdsh -S -Rssh -w' = no_dsh ']'
+ false
+ [[ pdsh -S -Rssh -w == \r\s\h ]]
+ [[ pdsh -S -Rssh -w == *pdsh* ]]
+ [[ pdsh -S -Rssh -w != *-S* ]]
+ [[ -n '' ]]
+ pdsh -S -Rssh -w oleg310-server '(PATH=$PATH:/home/green/git/lustre-release/lustre/utils:/home/green/git/lustre-release/lustre/tests:/sbin:/usr/sbin;		cd /root; LUSTRE="/home/green/git/lustre-release/lustre" bash -c "modprobe libcfs")'
+ sed 's/^oleg310-server: //'
oleg310-server: modprobe: FATAL: Module libcfs not found.
pdsh@oleg310-client: oleg310-server: ssh exited with exit code 1
+ return 1
+ error 'libcfs not loaded'
+ report_error 'libcfs not loaded'
+ local TYPE=FAIL
+ local dump=true
+ '[' 'xlibcfs not loaded' = xfalse ']'
+ log ' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ echo ' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
 conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded 
+ load_module ../libcfs/libcfs/libcfs
+ local module=../libcfs/libcfs/libcfs
+ shift
+ local ext=.ko
++ basename ../libcfs/libcfs/libcfs .ko
+ local base=libcfs
+ local path
+ local -A module_is_loaded_aa
+ local optvar
+ local mod
++ lsmod
++ awk '{ print $1; }'
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ for mod in '$(lsmod | awk '\''{ print $1; }'\'')'
+ module_is_loaded_aa[${mod//-/_}]=true
+ module_is_loaded libcfs
+ false
+ [[ -f /home/green/git/lustre-release/lustre/../libcfs/libcfs/libcfs.ko ]]
+ path=/home/green/git/lustre-release/lustre/../libcfs/libcfs/libcfs.ko
+ [[ -n /home/green/git/lustre-release/lustre/../libcfs/libcfs/libcfs.ko ]]
++ modinfo --field=depends /home/green/git/lustre-release/lustre/../libcfs/libcfs/libcfs.ko
++ tr , ' '
+ '[' 0 -eq 0 ']'
++ basename ../libcfs/libcfs/libcfs
++ tr a-z A-Z
+ optvar=MODOPTS_LIBCFS
+ eval set -- '$MODOPTS_LIBCFS'
++ set --
+ '[' 0 -eq 0 -a -n '' ']'
+ '[' 0 -gt 0 ']'
+ [[ -n /home/green/git/lustre-release/lustre/../libcfs/libcfs/libcfs.ko ]]
+ lustre_insmod /home/green/git/lustre-release/lustre/../libcfs/libcfs/libcfs.ko
+ local module=/home/green/git/lustre-release/lustre/../libcfs/libcfs/libcfs.ko
+ shift
+ local args=
+ local msg
+ local rc=0
+ false
++ insmod /home/green/git/lustre-release/lustre/../libcfs/libcfs/libcfs.ko
+ msg=
+ return 0
+ local 'MSG= conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ MSG=' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+++ nodes_list
+++ local nodes=oleg310-client.virtnet
+++ local nodes_sort
+++ local i
+++ '[' -n oleg310-client.virtnet ']'
+++ nodes=oleg310-client.virtnet
+++ '[' 'pdsh -S -Rssh -w' -a 'pdsh -S -Rssh -w' '!=' no_dsh ']'
+++++ get_facets
+++++ local 'types=OST MDS MGS'
+++++ local list=
+++++ for entry in '$types'
++++++ echo OST
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=ost
++++++ echo OST
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=OST
+++++ case $type in
+++++ local count=OSTCOUNT
+++++ (( i=1 ))
+++++ (( i<=2 ))
+++++ list=' ost1'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ for entry in '$types'
++++++ echo MDS
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=mds
++++++ echo MDS
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=MDS
+++++ case $type in
+++++ local count=MDSCOUNT
+++++ (( i=1 ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2 mds1'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2 mds1 mds2'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ for entry in '$types'
++++++ echo MGS
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=mgs
++++++ echo MGS
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=MGS
+++++ case $type in
+++++ list=' ost1 ost2 mds1 mds2 mgs'
++++++ comma_list ost1 ost2 mds1 mds2 mgs
++++++ tr ' ' ,
+++++++ tr -s ' ' '\n'
+++++++ sort -b -u
++++++ echo mds1 mds2 mgs ost1 ost2
+++++ echo mds1,mds2,mgs,ost1,ost2
++++ facets_nodes mds1,mds2,mgs,ost1,ost2
++++ local facets=mds1,mds2,mgs,ost1,ost2
++++ local facet
++++ local nodes
++++ local nodes_sort
++++ local i
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mds1
++++++ facet_active mds1
++++++ local facet=mds1
++++++ local activevar=mds1active
++++++ '[' -f /tmp/mds1active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mds1
+++++ facet_host mds1
+++++ local facet=mds1
+++++ local varname
+++++ '[' mds1 == client ']'
+++++ varname=mds1_HOST
+++++ '[' -z '' ']'
+++++ '[' mds == ost ']'
+++++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
+++++ local temp
+++++ '[' mds1 == mgsfailover ']'
+++++ temp=oleg310-server
+++++ eval export mds1_HOST=oleg310-server
++++++ export mds1_HOST=oleg310-server
++++++ mds1_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mds2
++++++ facet_active mds2
++++++ local facet=mds2
++++++ local activevar=mds2active
++++++ '[' -f /tmp/mds2active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mds2
+++++ facet_host mds2
+++++ local facet=mds2
+++++ local varname
+++++ '[' mds2 == client ']'
+++++ varname=mds2_HOST
+++++ '[' -z '' ']'
+++++ '[' mds == ost ']'
+++++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
+++++ local temp
+++++ '[' mds2 == mgsfailover ']'
+++++ temp=oleg310-server
+++++ eval export mds2_HOST=oleg310-server
++++++ export mds2_HOST=oleg310-server
++++++ mds2_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mgs
++++++ facet_active mgs
++++++ local facet=mgs
++++++ local activevar=mgsactive
++++++ '[' -f /tmp/mgsactive ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mgs
+++++ facet_host mgs
+++++ local facet=mgs
+++++ local varname
+++++ '[' mgs == client ']'
+++++ varname=mgs_HOST
+++++ '[' -z oleg310-server ']'
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host ost1
++++++ facet_active ost1
++++++ local facet=ost1
++++++ local activevar=ost1active
++++++ '[' -f /tmp/ost1active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n ost1
+++++ facet_host ost1
+++++ local facet=ost1
+++++ local varname
+++++ '[' ost1 == client ']'
+++++ varname=ost1_HOST
+++++ '[' -z '' ']'
+++++ '[' ost == ost ']'
+++++ local fh=ost1_HOST
+++++ eval export ost1_HOST=
++++++ export ost1_HOST=
++++++ ost1_HOST=
+++++ '[' -z '' ']'
+++++ eval export ost1_HOST=oleg310-server
++++++ export ost1_HOST=oleg310-server
++++++ ost1_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host ost2
++++++ facet_active ost2
++++++ local facet=ost2
++++++ local activevar=ost2active
++++++ '[' -f /tmp/ost2active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n ost2
+++++ facet_host ost2
+++++ local facet=ost2
+++++ local varname
+++++ '[' ost2 == client ']'
+++++ varname=ost2_HOST
+++++ '[' -z '' ']'
+++++ '[' ost == ost ']'
+++++ local fh=ost2_HOST
+++++ eval export ost2_HOST=
++++++ export ost2_HOST=
++++++ ost2_HOST=
+++++ '[' -z '' ']'
+++++ eval export ost2_HOST=oleg310-server
++++++ export ost2_HOST=oleg310-server
++++++ ost2_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server oleg310-server oleg310-server'
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ sort -u
++++ nodes_sort=oleg310-server
++++ echo -n oleg310-server
+++ nodes='oleg310-client.virtnet oleg310-server'
++++ for i in '$nodes'
++++ echo oleg310-client.virtnet
++++ for i in '$nodes'
++++ echo oleg310-server
++++ sort -u
+++ nodes_sort='oleg310-client.virtnet
oleg310-server'
+++ echo -n oleg310-client.virtnet oleg310-server
++ comma_list oleg310-client.virtnet oleg310-server
++ tr ' ' ,
+++ tr -s ' ' '\n'
+++ sort -b -u
++ echo oleg310-client.virtnet oleg310-server
+ do_nodes oleg310-client.virtnet,oleg310-server /home/green/git/lustre-release/lustre/utils/lctl mark ' conf-sanity test_102: @@@@@@ FAIL: libcfs not loaded '
+ true
+ print_stack_trace 2
+ local skip=2
+ echo '  Trace dump:'
  Trace dump:
+ (( i=2 ))
+ (( i < 8  ))
+ local src=/home/green/git/lustre-release/lustre/tests/test-framework.sh
+ local lineno=6639
+ local funcname=error
+ echo '  = /home/green/git/lustre-release/lustre/tests/test-framework.sh:6639:error()'
  = /home/green/git/lustre-release/lustre/tests/test-framework.sh:6639:error()
+ (( i++  ))
+ (( i < 8  ))
+ local src=/home/green/git/lustre-release/lustre/tests/conf-sanity.sh
+ local lineno=8120
+ local funcname=test_102
+ echo '  = /home/green/git/lustre-release/lustre/tests/conf-sanity.sh:8120:test_102()'
  = /home/green/git/lustre-release/lustre/tests/conf-sanity.sh:8120:test_102()
+ (( i++  ))
+ (( i < 8  ))
+ local src=/home/green/git/lustre-release/lustre/tests/test-framework.sh
+ local lineno=6979
+ local funcname=run_one
+ echo '  = /home/green/git/lustre-release/lustre/tests/test-framework.sh:6979:run_one()'
  = /home/green/git/lustre-release/lustre/tests/test-framework.sh:6979:run_one()
+ (( i++  ))
+ (( i < 8  ))
+ local src=/home/green/git/lustre-release/lustre/tests/test-framework.sh
+ local lineno=7028
+ local funcname=run_one_logged
+ echo '  = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7028:run_one_logged()'
  = /home/green/git/lustre-release/lustre/tests/test-framework.sh:7028:run_one_logged()
+ (( i++  ))
+ (( i < 8  ))
+ local src=/home/green/git/lustre-release/lustre/tests/test-framework.sh
+ local lineno=6850
+ local funcname=run_test
+ echo '  = /home/green/git/lustre-release/lustre/tests/test-framework.sh:6850:run_test()'
  = /home/green/git/lustre-release/lustre/tests/test-framework.sh:6850:run_test()
+ (( i++  ))
+ (( i < 8  ))
+ local src=/home/green/git/lustre-release/lustre/tests/conf-sanity.sh
+ local lineno=8138
+ local funcname=main
+ echo '  = /home/green/git/lustre-release/lustre/tests/conf-sanity.sh:8138:main()'
  = /home/green/git/lustre-release/lustre/tests/conf-sanity.sh:8138:main()
+ (( i++  ))
+ (( i < 8  ))
+ mkdir -p /tmp/testlogs/
+ true
+++ nodes_list
+++ local nodes=oleg310-client.virtnet
+++ local nodes_sort
+++ local i
+++ '[' -n oleg310-client.virtnet ']'
+++ nodes=oleg310-client.virtnet
+++ '[' 'pdsh -S -Rssh -w' -a 'pdsh -S -Rssh -w' '!=' no_dsh ']'
+++++ get_facets
+++++ local 'types=OST MDS MGS'
+++++ local list=
+++++ for entry in '$types'
++++++ echo OST
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=ost
++++++ echo OST
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=OST
+++++ case $type in
+++++ local count=OSTCOUNT
+++++ (( i=1 ))
+++++ (( i<=2 ))
+++++ list=' ost1'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ for entry in '$types'
++++++ echo MDS
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=mds
++++++ echo MDS
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=MDS
+++++ case $type in
+++++ local count=MDSCOUNT
+++++ (( i=1 ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2 mds1'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2 mds1 mds2'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ for entry in '$types'
++++++ echo MGS
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=mgs
++++++ echo MGS
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=MGS
+++++ case $type in
+++++ list=' ost1 ost2 mds1 mds2 mgs'
++++++ comma_list ost1 ost2 mds1 mds2 mgs
++++++ tr ' ' ,
+++++++ sort -b -u
+++++++ tr -s ' ' '\n'
++++++ echo mds1 mds2 mgs ost1 ost2
+++++ echo mds1,mds2,mgs,ost1,ost2
++++ facets_nodes mds1,mds2,mgs,ost1,ost2
++++ local facets=mds1,mds2,mgs,ost1,ost2
++++ local facet
++++ local nodes
++++ local nodes_sort
++++ local i
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mds1
++++++ facet_active mds1
++++++ local facet=mds1
++++++ local activevar=mds1active
++++++ '[' -f /tmp/mds1active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mds1
+++++ facet_host mds1
+++++ local facet=mds1
+++++ local varname
+++++ '[' mds1 == client ']'
+++++ varname=mds1_HOST
+++++ '[' -z '' ']'
+++++ '[' mds == ost ']'
+++++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
+++++ local temp
+++++ '[' mds1 == mgsfailover ']'
+++++ temp=oleg310-server
+++++ eval export mds1_HOST=oleg310-server
++++++ export mds1_HOST=oleg310-server
++++++ mds1_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mds2
++++++ facet_active mds2
++++++ local facet=mds2
++++++ local activevar=mds2active
++++++ '[' -f /tmp/mds2active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mds2
+++++ facet_host mds2
+++++ local facet=mds2
+++++ local varname
+++++ '[' mds2 == client ']'
+++++ varname=mds2_HOST
+++++ '[' -z '' ']'
+++++ '[' mds == ost ']'
+++++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
+++++ local temp
+++++ '[' mds2 == mgsfailover ']'
+++++ temp=oleg310-server
+++++ eval export mds2_HOST=oleg310-server
++++++ export mds2_HOST=oleg310-server
++++++ mds2_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mgs
++++++ facet_active mgs
++++++ local facet=mgs
++++++ local activevar=mgsactive
++++++ '[' -f /tmp/mgsactive ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mgs
+++++ facet_host mgs
+++++ local facet=mgs
+++++ local varname
+++++ '[' mgs == client ']'
+++++ varname=mgs_HOST
+++++ '[' -z oleg310-server ']'
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host ost1
++++++ facet_active ost1
++++++ local facet=ost1
++++++ local activevar=ost1active
++++++ '[' -f /tmp/ost1active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n ost1
+++++ facet_host ost1
+++++ local facet=ost1
+++++ local varname
+++++ '[' ost1 == client ']'
+++++ varname=ost1_HOST
+++++ '[' -z '' ']'
+++++ '[' ost == ost ']'
+++++ local fh=ost1_HOST
+++++ eval export ost1_HOST=
++++++ export ost1_HOST=
++++++ ost1_HOST=
+++++ '[' -z '' ']'
+++++ eval export ost1_HOST=oleg310-server
++++++ export ost1_HOST=oleg310-server
++++++ ost1_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host ost2
++++++ facet_active ost2
++++++ local facet=ost2
++++++ local activevar=ost2active
++++++ '[' -f /tmp/ost2active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n ost2
+++++ facet_host ost2
+++++ local facet=ost2
+++++ local varname
+++++ '[' ost2 == client ']'
+++++ varname=ost2_HOST
+++++ '[' -z '' ']'
+++++ '[' ost == ost ']'
+++++ local fh=ost2_HOST
+++++ eval export ost2_HOST=
++++++ export ost2_HOST=
++++++ ost2_HOST=
+++++ '[' -z '' ']'
+++++ eval export ost2_HOST=oleg310-server
++++++ export ost2_HOST=oleg310-server
++++++ ost2_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server oleg310-server oleg310-server'
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ sort -u
++++ nodes_sort=oleg310-server
++++ echo -n oleg310-server
+++ nodes='oleg310-client.virtnet oleg310-server'
++++ for i in '$nodes'
++++ echo oleg310-client.virtnet
++++ for i in '$nodes'
++++ echo oleg310-server
++++ sort -u
+++ nodes_sort='oleg310-client.virtnet
oleg310-server'
+++ echo -n oleg310-client.virtnet oleg310-server
++ comma_list oleg310-client.virtnet oleg310-server
++ tr ' ' ,
+++ tr -s ' ' '\n'
+++ sort -b -u
++ echo oleg310-client.virtnet oleg310-server
+ gather_logs oleg310-client.virtnet,oleg310-server
+ local list=oleg310-client.virtnet,oleg310-server
++ date +%s
+ local ts=1688471984
+ local docp=true
+ [[ ! -f /tmp/testlogs//results.yml ]]
+ '[' -f /tmp/testlogs//shared ']'
+ prefix=/tmp/testlogs//conf-sanity.test_102
+ suffix=1688471984.log
+ echo 'Dumping lctl log to /tmp/testlogs//conf-sanity.test_102.*.1688471984.log'
Dumping lctl log to /tmp/testlogs//conf-sanity.test_102.*.1688471984.log
+ '[' -n '' -o 'pdsh -S -Rssh -w' == no_dsh ']'
+ do_nodesv oleg310-client.virtnet,oleg310-server '/home/green/git/lustre-release/lustre/utils/lctl dk > /tmp/testlogs//conf-sanity.test_102.debug_log.\$(hostname -s).1688471984.log;
		dmesg > /tmp/testlogs//conf-sanity.test_102.dmesg.\$(hostname -s).1688471984.log'
+ do_nodes --verbose oleg310-client.virtnet,oleg310-server '/home/green/git/lustre-release/lustre/utils/lctl dk > /tmp/testlogs//conf-sanity.test_102.debug_log.\$(hostname -s).1688471984.log;
		dmesg > /tmp/testlogs//conf-sanity.test_102.dmesg.\$(hostname -s).1688471984.log'
+ local verbose
+ local quiet
+ [[ --verbose == \-\-\v\e\r\b\o\s\e ]]
+ verbose=--verbose
+ shift
+ [[ oleg310-client.virtnet,oleg310-server == \-\-\q\u\i\e\t ]]
+ [[ oleg310-client.virtnet,oleg310-server == \-\q ]]
+ local rnodes=oleg310-client.virtnet,oleg310-server
+ shift
+ single_local_node oleg310-client.virtnet,oleg310-server
+ '[' oleg310-client.virtnet,oleg310-server = oleg310-client.virtnet ']'
+ local 'myPDSH=pdsh -S -Rssh -w'
+ '[' -z 'pdsh -S -Rssh -w' -o 'pdsh -S -Rssh -w' = no_dsh -o 'pdsh -S -Rssh -w' = rsh ']'
++ get_node_count 'oleg310-client.virtnet oleg310-server'
++ local 'nodes=oleg310-client.virtnet oleg310-server'
++ echo oleg310-client.virtnet oleg310-server
++ wc -w
+ export FANOUT=2
+ FANOUT=2
+ false
+ [[ -n --verbose ]]
++ get_env_vars
++ local var
++ local value
+++ get_facets
+++ local 'types=OST MDS MGS'
+++ local list=
+++ for entry in '$types'
++++ echo OST
++++ tr '[:upper:]' '[:lower:]'
+++ local name=ost
++++ echo OST
++++ tr '[:lower:]' '[:upper:]'
+++ local type=OST
+++ case $type in
+++ local count=OSTCOUNT
+++ (( i=1 ))
+++ (( i<=2 ))
+++ list=' ost1'
+++ (( i++ ))
+++ (( i<=2 ))
+++ list=' ost1 ost2'
+++ (( i++ ))
+++ (( i<=2 ))
+++ for entry in '$types'
++++ echo MDS
++++ tr '[:upper:]' '[:lower:]'
+++ local name=mds
++++ echo MDS
++++ tr '[:lower:]' '[:upper:]'
+++ local type=MDS
+++ case $type in
+++ local count=MDSCOUNT
+++ (( i=1 ))
+++ (( i<=2 ))
+++ list=' ost1 ost2 mds1'
+++ (( i++ ))
+++ (( i<=2 ))
+++ list=' ost1 ost2 mds1 mds2'
+++ (( i++ ))
+++ (( i<=2 ))
+++ for entry in '$types'
++++ echo MGS
++++ tr '[:upper:]' '[:lower:]'
+++ local name=mgs
++++ echo MGS
++++ tr '[:lower:]' '[:upper:]'
+++ local type=MGS
+++ case $type in
+++ list=' ost1 ost2 mds1 mds2 mgs'
++++ comma_list ost1 ost2 mds1 mds2 mgs
++++ tr ' ' ,
+++++ tr -s ' ' '\n'
+++++ sort -b -u
++++ echo mds1 mds2 mgs ost1 ost2
+++ echo mds1,mds2,mgs,ost1,ost2
++ local facets=mds1,mds2,mgs,ost1,ost2
++ local facet
++ for var in '${!MODOPTS_*}'
++ value=lbug_on_grant_miscount=1
++ echo -n ' MODOPTS_PTLRPC="lbug_on_grant_miscount=1"'
++ for facet in '${facets//,/ }'
++ var=mds1_FSTYPE
++ '[' -n zfs ']'
++ echo -n ' mds1_FSTYPE=zfs'
++ for facet in '${facets//,/ }'
++ var=mds2_FSTYPE
++ '[' -n '' ']'
++ for facet in '${facets//,/ }'
++ var=mgs_FSTYPE
++ '[' -n '' ']'
++ for facet in '${facets//,/ }'
++ var=ost1_FSTYPE
++ '[' -n zfs ']'
++ echo -n ' ost1_FSTYPE=zfs'
++ for facet in '${facets//,/ }'
++ var=ost2_FSTYPE
++ '[' -n '' ']'
++ for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE
++ '[' -n '' ']'
++ for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE
++ '[' -n '' ']'
++ for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE
++ '[' -n '' ']'
++ for var in VERBOSE
++ '[' -n false ']'
++ echo -n ' VERBOSE=false'
++ '[' -n zfs ']'
++ echo -n ' FSTYPE=zfs'
++ for var in LNETLND NETTYPE
++ '[' -n socklnd/ksocklnd ']'
++ echo -n ' LNETLND=socklnd/ksocklnd'
++ for var in LNETLND NETTYPE
++ '[' -n tcp ']'
++ echo -n ' NETTYPE=tcp'
+ pdsh -S -Rssh -w oleg310-client.virtnet,oleg310-server '(PATH=$PATH:/home/green/git/lustre-release/lustre/utils:/home/green/git/lustre-release/lustre/tests:/sbin:/usr/sbin; cd /root; LUSTRE="/home/green/git/lustre-release/lustre"  MODOPTS_PTLRPC="lbug_on_grant_miscount=1" mds1_FSTYPE=zfs ost1_FSTYPE=zfs VERBOSE=false FSTYPE=zfs LNETLND=socklnd/ksocklnd NETTYPE=tcp bash -c "/home/green/git/lustre-release/lustre/utils/lctl dk > /tmp/testlogs//conf-sanity.test_102.debug_log.\$(hostname -s).1688471984.log;
		dmesg > /tmp/testlogs//conf-sanity.test_102.dmesg.\$(hostname -s).1688471984.log")'
oleg310-server: invalid parameter 'dump_kernel'
oleg310-server: open(dump_kernel) failed: No such file or directory
+ return 0
+ '[' false = true ']'
+ '[' false = true ']'
+ '[' '!' -f /tmp/testlogs//shared ']'
++ exclude_items_from_list oleg310-client.virtnet,oleg310-server oleg310-client.virtnet
++ local list=oleg310-client.virtnet,oleg310-server
++ local excluded=oleg310-client.virtnet
++ local item
++ list='oleg310-client.virtnet oleg310-server'
++ for item in '${excluded//,/ }'
+++ echo ' oleg310-client.virtnet oleg310-server '
+++ sed -re 's/\s+oleg310-client.virtnet\s+/ /g'
++ list=' oleg310-server '
+++ comma_list oleg310-server
+++ tr ' ' ,
++++ tr -s ' ' '\n'
++++ sort -b -u
+++ echo oleg310-server
++ echo oleg310-server
+ local remote_nodes=oleg310-server
+ for node in '${remote_nodes//,/ }'
+ debugrestore
+ rsync -az -e ssh 'oleg310-server:/tmp/testlogs//conf-sanity.test_102.*.1688471984.log' /tmp/testlogs/
+ '[' -n '' ']'
+ true
+ DEBUGSAVE=
+ '[' -n '' ']'
+ true
+ DEBUGSAVE_SERVER=
+ '[' /tmp/conf-sanity.log ']'
+ echo 'conf-sanity: FAIL: test_102 libcfs not loaded'
+ '[' -z 'libcfs not loaded' ']'
++ echo FAIL
++ grep '^IGNORE'
+ [[ -n '' ]]
+ echo 'libcfs not loaded'
+ reset_fail_loc
+++ nodes_list
+++ local nodes=oleg310-client.virtnet
+++ local nodes_sort
+++ local i
+++ '[' -n oleg310-client.virtnet ']'
+++ nodes=oleg310-client.virtnet
+++ '[' 'pdsh -S -Rssh -w' -a 'pdsh -S -Rssh -w' '!=' no_dsh ']'
+++++ get_facets
+++++ local 'types=OST MDS MGS'
+++++ local list=
+++++ for entry in '$types'
++++++ echo OST
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=ost
++++++ echo OST
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=OST
+++++ case $type in
+++++ local count=OSTCOUNT
+++++ (( i=1 ))
+++++ (( i<=2 ))
+++++ list=' ost1'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ for entry in '$types'
++++++ echo MDS
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=mds
++++++ echo MDS
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=MDS
+++++ case $type in
+++++ local count=MDSCOUNT
+++++ (( i=1 ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2 mds1'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ list=' ost1 ost2 mds1 mds2'
+++++ (( i++ ))
+++++ (( i<=2 ))
+++++ for entry in '$types'
++++++ echo MGS
++++++ tr '[:upper:]' '[:lower:]'
+++++ local name=mgs
++++++ echo MGS
++++++ tr '[:lower:]' '[:upper:]'
+++++ local type=MGS
+++++ case $type in
+++++ list=' ost1 ost2 mds1 mds2 mgs'
++++++ comma_list ost1 ost2 mds1 mds2 mgs
++++++ tr ' ' ,
+++++++ tr -s ' ' '\n'
+++++++ sort -b -u
++++++ echo mds1 mds2 mgs ost1 ost2
+++++ echo mds1,mds2,mgs,ost1,ost2
++++ facets_nodes mds1,mds2,mgs,ost1,ost2
++++ local facets=mds1,mds2,mgs,ost1,ost2
++++ local facet
++++ local nodes
++++ local nodes_sort
++++ local i
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mds1
++++++ facet_active mds1
++++++ local facet=mds1
++++++ local activevar=mds1active
++++++ '[' -f /tmp/mds1active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mds1
+++++ facet_host mds1
+++++ local facet=mds1
+++++ local varname
+++++ '[' mds1 == client ']'
+++++ varname=mds1_HOST
+++++ '[' -z '' ']'
+++++ '[' mds == ost ']'
+++++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
+++++ local temp
+++++ '[' mds1 == mgsfailover ']'
+++++ temp=oleg310-server
+++++ eval export mds1_HOST=oleg310-server
++++++ export mds1_HOST=oleg310-server
++++++ mds1_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mds2
++++++ facet_active mds2
++++++ local facet=mds2
++++++ local activevar=mds2active
++++++ '[' -f /tmp/mds2active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mds2
+++++ facet_host mds2
+++++ local facet=mds2
+++++ local varname
+++++ '[' mds2 == client ']'
+++++ varname=mds2_HOST
+++++ '[' -z '' ']'
+++++ '[' mds == ost ']'
+++++ '[' mds == mdt -o mds == mds -o mds == mgs ']'
+++++ local temp
+++++ '[' mds2 == mgsfailover ']'
+++++ temp=oleg310-server
+++++ eval export mds2_HOST=oleg310-server
++++++ export mds2_HOST=oleg310-server
++++++ mds2_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host mgs
++++++ facet_active mgs
++++++ local facet=mgs
++++++ local activevar=mgsactive
++++++ '[' -f /tmp/mgsactive ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n mgs
+++++ facet_host mgs
+++++ local facet=mgs
+++++ local varname
+++++ '[' mgs == client ']'
+++++ varname=mgs_HOST
+++++ '[' -z oleg310-server ']'
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host ost1
++++++ facet_active ost1
++++++ local facet=ost1
++++++ local activevar=ost1active
++++++ '[' -f /tmp/ost1active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n ost1
+++++ facet_host ost1
+++++ local facet=ost1
+++++ local varname
+++++ '[' ost1 == client ']'
+++++ varname=ost1_HOST
+++++ '[' -z '' ']'
+++++ '[' ost == ost ']'
+++++ local fh=ost1_HOST
+++++ eval export ost1_HOST=
++++++ export ost1_HOST=
++++++ ost1_HOST=
+++++ '[' -z '' ']'
+++++ eval export ost1_HOST=oleg310-server
++++++ export ost1_HOST=oleg310-server
++++++ ost1_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server oleg310-server'
++++ for facet in '${facets//,/ }'
+++++ facet_active_host ost2
++++++ facet_active ost2
++++++ local facet=ost2
++++++ local activevar=ost2active
++++++ '[' -f /tmp/ost2active ']'
++++++ active=
++++++ '[' -z '' ']'
++++++ echo -n ost2
+++++ facet_host ost2
+++++ local facet=ost2
+++++ local varname
+++++ '[' ost2 == client ']'
+++++ varname=ost2_HOST
+++++ '[' -z '' ']'
+++++ '[' ost == ost ']'
+++++ local fh=ost2_HOST
+++++ eval export ost2_HOST=
++++++ export ost2_HOST=
++++++ ost2_HOST=
+++++ '[' -z '' ']'
+++++ eval export ost2_HOST=oleg310-server
++++++ export ost2_HOST=oleg310-server
++++++ ost2_HOST=oleg310-server
+++++ echo -n oleg310-server
++++ nodes=' oleg310-server oleg310-server oleg310-server oleg310-server oleg310-server'
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ for i in '$nodes'
+++++ echo oleg310-server
+++++ sort -u
++++ nodes_sort=oleg310-server
++++ echo -n oleg310-server
+++ nodes='oleg310-client.virtnet oleg310-server'
++++ for i in '$nodes'
++++ echo oleg310-client.virtnet
++++ for i in '$nodes'
++++ echo oleg310-server
++++ sort -u
+++ nodes_sort='oleg310-client.virtnet
oleg310-server'
+++ echo -n oleg310-client.virtnet oleg310-server
++ comma_list oleg310-client.virtnet oleg310-server
++ tr ' ' ,
+++ tr -s ' ' '\n'
+++ sort -b -u
++ echo oleg310-client.virtnet oleg310-server
+ do_nodes --quiet oleg310-client.virtnet,oleg310-server 'lctl set_param -n fail_loc=0 fail_val=0 2>/dev/null'
+ local verbose
+ local quiet
+ [[ --quiet == \-\-\v\e\r\b\o\s\e ]]
+ [[ --quiet == \-\-\q\u\i\e\t ]]
+ quiet=--quiet
+ shift
+ local rnodes=oleg310-client.virtnet,oleg310-server
+ shift
+ single_local_node oleg310-client.virtnet,oleg310-server
+ '[' oleg310-client.virtnet,oleg310-server = oleg310-client.virtnet ']'
+ local 'myPDSH=pdsh -S -Rssh -w'
+ '[' -z 'pdsh -S -Rssh -w' -o 'pdsh -S -Rssh -w' = no_dsh -o 'pdsh -S -Rssh -w' = rsh ']'
++ get_node_count 'oleg310-client.virtnet oleg310-server'
++ local 'nodes=oleg310-client.virtnet oleg310-server'
++ echo oleg310-client.virtnet oleg310-server
++ wc -w
+ export FANOUT=2
+ FANOUT=2
+ false
+ [[ -n '' ]]
+ [[ pdsh -S -Rssh -w = *-N* ]]
+ sed -re 's/^[^:]*: //g'
++ get_env_vars
++ local var
++ local value
+++ get_facets
+++ local 'types=OST MDS MGS'
+++ local list=
+++ for entry in '$types'
++++ echo OST
++++ tr '[:upper:]' '[:lower:]'
+++ local name=ost
++++ echo OST
++++ tr '[:lower:]' '[:upper:]'
+++ local type=OST
+++ case $type in
+++ local count=OSTCOUNT
+++ (( i=1 ))
+++ (( i<=2 ))
+++ list=' ost1'
+++ (( i++ ))
+++ (( i<=2 ))
+++ list=' ost1 ost2'
+++ (( i++ ))
+++ (( i<=2 ))
+++ for entry in '$types'
++++ echo MDS
++++ tr '[:upper:]' '[:lower:]'
+++ local name=mds
++++ echo MDS
++++ tr '[:lower:]' '[:upper:]'
+++ local type=MDS
+++ case $type in
+++ local count=MDSCOUNT
+++ (( i=1 ))
+++ (( i<=2 ))
+++ list=' ost1 ost2 mds1'
+++ (( i++ ))
+++ (( i<=2 ))
+++ list=' ost1 ost2 mds1 mds2'
+++ (( i++ ))
+++ (( i<=2 ))
+++ for entry in '$types'
++++ echo MGS
++++ tr '[:upper:]' '[:lower:]'
+++ local name=mgs
++++ echo MGS
++++ tr '[:lower:]' '[:upper:]'
+++ local type=MGS
+++ case $type in
+++ list=' ost1 ost2 mds1 mds2 mgs'
++++ comma_list ost1 ost2 mds1 mds2 mgs
++++ tr ' ' ,
+++++ tr -s ' ' '\n'
+++++ sort -b -u
++++ echo mds1 mds2 mgs ost1 ost2
+++ echo mds1,mds2,mgs,ost1,ost2
++ local facets=mds1,mds2,mgs,ost1,ost2
++ local facet
++ for var in '${!MODOPTS_*}'
++ value=lbug_on_grant_miscount=1
++ echo -n ' MODOPTS_PTLRPC="lbug_on_grant_miscount=1"'
++ for facet in '${facets//,/ }'
++ var=mds1_FSTYPE
++ '[' -n zfs ']'
++ echo -n ' mds1_FSTYPE=zfs'
++ for facet in '${facets//,/ }'
++ var=mds2_FSTYPE
++ '[' -n '' ']'
++ for facet in '${facets//,/ }'
++ var=mgs_FSTYPE
++ '[' -n '' ']'
++ for facet in '${facets//,/ }'
++ var=ost1_FSTYPE
++ '[' -n zfs ']'
++ echo -n ' ost1_FSTYPE=zfs'
++ for facet in '${facets//,/ }'
++ var=ost2_FSTYPE
++ '[' -n '' ']'
++ for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE
++ '[' -n '' ']'
++ for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE
++ '[' -n '' ']'
++ for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE
++ '[' -n '' ']'
++ for var in VERBOSE
++ '[' -n false ']'
++ echo -n ' VERBOSE=false'
++ '[' -n zfs ']'
++ echo -n ' FSTYPE=zfs'
++ for var in LNETLND NETTYPE
++ '[' -n socklnd/ksocklnd ']'
++ echo -n ' LNETLND=socklnd/ksocklnd'
++ for var in LNETLND NETTYPE
++ '[' -n tcp ']'
++ echo -n ' NETTYPE=tcp'
+ pdsh -S -Rssh -w oleg310-client.virtnet,oleg310-server '(PATH=$PATH:/home/green/git/lustre-release/lustre/utils:/home/green/git/lustre-release/lustre/tests:/sbin:/usr/sbin; cd /root; LUSTRE="/home/green/git/lustre-release/lustre"  MODOPTS_PTLRPC="lbug_on_grant_miscount=1" mds1_FSTYPE=zfs ost1_FSTYPE=zfs VERBOSE=false FSTYPE=zfs LNETLND=socklnd/ksocklnd NETTYPE=tcp bash -c "lctl set_param -n fail_loc=0 fail_val=0 2>/dev/null")'
rsync: chown "/tmp/testlogs/.conf-sanity.test_102.debug_log.oleg310-server.1688471984.log.AU8UkA" failed: Operation not permitted (1)
rsync: chown "/tmp/testlogs/.conf-sanity.test_102.dmesg.oleg310-server.1688471984.log.K9fRmZ" failed: Operation not permitted (1)
rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1651) [generator=3.1.2]
pdsh@oleg310-client: oleg310-server: ssh exited with exit code 2
+ return 2
+ true
+ exit 1