#!/usr/bin/bash
#
# client failure does not affect other clients

# Start load on clients (each client works on it's own directory).
# At defined (5-10 minutes) interval fail one random client and then fail mds.
# Reintegrate failed client after recovery completed,
# application errors are allowed for that client but not on other clients.
# 10 minute intervals and verify that no application errors occur.

# Test runs one of CLIENT_LOAD progs on remote clients.
set -e

ONLY=${ONLY:-"$*"}

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env "$@"
init_logging
. $LUSTRE/tests/recovery-scale-lib.sh

# bug number for skipped test:
ALWAYS_EXCEPT="$RECOVERY_RANDOM_SCALE_EXCEPT "
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!

build_test_filter

remote_mds_nodsh && skip_env "remote MDS with nodsh"
remote_ost_nodsh && skip_env "remote OST with nodsh"

if (( CLIENTCOUNT < 3 )); then
	skip_env "need three or more clients"
fi

if [[ -z "$SHARED_DIRECTORY" ]] || ! check_shared_dir "$SHARED_DIRECTORY"; then
	skip_env "SHARED_DIRECTORY should be specified with a shared directory \
which is accessible on all of the nodes"
fi

if [[ "$FAILURE_MODE" == "SOFT" ]]; then
	log "WARNING: $0 is not functional with FAILURE_MODE = SOFT, bz22797"
fi

# Application failures are allowed for the failed client
# but not for other clients.
ERRORS_OK="yes"

init_stripe_dir_params RECOVERY_SCALE_ENABLE_REMOTE_DIRS \
	RECOVERY_SCALE_ENABLE_STRIPED_DIRS

numfailovers () {
	local facet
	local var

	for facet in ${MDTS//,/ } ${FAILED_CLIENTS//,/ }; do
		var=$(node_var_name $facet)_nums
		val=${!var}
		if [ "$val" ] ; then
			echo "$facet failed over $val times"
		fi
	done
}

check_and_setup_lustre
rm -rf $DIR/[Rdfs][0-9]*

insulate_clients
check_progs_installed $NODES_TO_USE "${CLIENT_LOADS[@]}"

MAX_RECOV_TIME=$(max_recovery_time)
MDTS=$(get_facets MDS)
OSTS=$(get_facets OST)

# Fail a random client and then failover a random MDS.
test_fail_client_mds() {
	local fail_client
	local serverfacet
	local client_var
	local var

	stack_trap summary_and_cleanup EXIT INT

	# start vmstat on OSS nodes
	[[ -z "$VMSTAT" ]] || start_vmstat $(osts_nodes) $VMSTAT_PID_FILE

	# start client loads
	rm -f $END_RUN_FILE
	start_client_loads $NODES_TO_USE

	echo client loads pids:
	do_nodesv $NODES_TO_USE "cat $LOAD_PID_FILE" || exit 3

	ELAPSED=0
	local it_time_start
	local sleep=0
	local reqfail=0
	local start_ts=$(date +%s)
	local current_ts=$start_ts

	while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
		# In order to perform the
		# expected number of failovers, we need to account the
		# following:
		# 1) the time that has elapsed during the client load checking
		# 2) time takes for failover
		it_time_start=$(date +%s)

		fail_client=$(get_random_entry $NODES_TO_USE)
		client_var=$(node_var_name $fail_client)_nums

		# store the list of failed clients
		# lists are comma separated
		FAILED_CLIENTS=$(expand_list $FAILED_CLIENTS $fail_client)

		serverfacet=$(get_random_entry $MDTS)
		var=$(node_var_name $serverfacet)_nums

		# Check that our client loads are still running. If any have
		# died, that means they have died outside of recovery, which
		# is unacceptable.
		log "==== Checking clients loads BEFORE failover -- failure NOT OK \
		     ELAPSED=$ELAPSED DURATION=$DURATION \
		     PERIOD=$SERVER_FAILOVER_PERIOD"
		check_client_loads $NODES_TO_USE || exit 4

		log "FAIL CLIENT $fail_client..."
		shutdown_client $fail_client

		log "Starting failover on $serverfacet"
		facet_failover "$serverfacet" || exit 1

		if ! wait_recovery_complete $serverfacet; then
			echo "$serverfacet recovery is not completed!"
			exit 7
		fi

		boot_node $fail_client
		echo "Reintegrating $fail_client"
		zconf_mount $fail_client $MOUNT || exit $?
		client_up $fail_client || exit $?

		# Increment the number of failovers
		val=$((${!var} + 1))
		eval $var=$val
		val=$((${!client_var} + 1))
		eval $client_var=$val

		# load script on failed clients could create END_RUN_FILE
		# We shuold remove it and ignore the failure if this
		# file contains the failed client only.
		# We can not use ERRORS_OK when start all loads at the start of
		# this script because the application errors allowed for random
		# failed client only, but not for all clients.
		if [ -e $END_RUN_FILE ]; then
			local end_run_node
			read end_run_node < $END_RUN_FILE
			if [[ $end_run_node = $fail_client ]]; then
				rm -f $END_RUN_FILE
			else
				echo "failure is expected on FAIL CLIENT \
					$fail_client, not on $end_run_node"
				exit 13
			fi
		fi

		restart_client_loads $fail_client $ERRORS_OK || exit $?

		# Check that not failed clients loads are still running.
		# No application failures should occur on clients that were
		# not failed.
		log "==== Checking clients loads AFTER failed client reintegrated \
			-- failure NOT OK"
		if ! ERRORS_OK= check_client_loads \
			$(exclude_items_from_list $NODES_TO_USE $fail_client); then
			log "Client load failed. Exiting..."
			exit 5
		fi

		current_ts=$(date +%s)
		ELAPSED=$((current_ts - start_ts))
		sleep=$((SERVER_FAILOVER_PERIOD - (current_ts - it_time_start)))

		# Keep counting the number of iterations when
		# time spent to failover and two client loads check exceeded
		# the value ( SERVER_FAILOVER_PERIOD - MINSLEEP ).
		if [ $sleep -lt $MINSLEEP ]; then
			reqfail=$((reqfail + 1))
			log "WARNING: failover, client reintegration and \
check_client_loads time exceeded SERVER_FAILOVER_PERIOD - MINSLEEP!
Failed to load the filesystem with I/O for a minimum period of \
$MINSLEEP $reqfail times ( REQFAIL=$REQFAIL ).
This iteration, the load was only applied for sleep=$sleep seconds.
Estimated max recovery time : $MAX_RECOV_TIME
Probably the hardware is taking excessively long time to boot.
Try to increase SERVER_FAILOVER_PERIOD (current is $SERVER_FAILOVER_PERIOD), \
bug 20918"
			[ $reqfail -gt $REQFAIL ] && exit 6
		fi

		log "Number of failovers:
$(numfailovers)                and counting..."

		[ $((ELAPSED + sleep)) -ge $DURATION ] && break

		if [ $sleep -gt 0 ]; then
			echo "sleeping $sleep seconds... "
			sleep $sleep
		fi
	done
	exit 0
}
run_test fail_client_mds "fail client, then failover MDS"

zconf_mount $HOSTNAME $MOUNT || error "mount $MOUNT on $HOSTNAME failed"
client_up || error "start client on $HOSTNAME failed"

complete_test $SECONDS
check_and_cleanup_lustre
exit_status
