Merge pull request openucx#34 from mike-dubman/topic/iodemo_launch_ad…

…dons

Add support for individual client/server start/stop/status

test/apps/iodemo/README.md

@@ -0,0 +1,75 @@
+
+# Running iodemo
+
+
+## Generate launch script per host
+
+The run_io_demo.sh will generate launch script with name ```iodemo_commands_$(hostname).sh```
+
+``` bash
+%./run_io_demo.sh --dry-run -i eth0 -H $(hostname),$(hostname) --num-clients 5 \
+	--num-servers 5 --tasks-per-node 10 --duration $((10*60)) $PWD/tester
+Launch configuration:
+               host_list : 'mtr-vdi-326,mtr-vdi-326'
+          tasks_per_node : '10'
+                  map_by : 'node'
+             num_clients : '5'
+             num_servers : '5'
+              iodemo_exe : '/labhome/miked/workspace/git/forks/ucx/test/apps/iodemo/tester'
+      iodemo_client_args : ''
+                  net_if : 'eth0'
+           base_port_num : '20000'
+                duration : '600'
+        client_wait_time : '2'
+                launcher : 'pdsh -b -w'
+                 dry_run : '1'
+iodemo_mtr-vdi-326_server_00.log
+iodemo_mtr-vdi-326_server_01.log
+iodemo_mtr-vdi-326_client_00.log
+iodemo_mtr-vdi-326_client_01.log
+iodemo_mtr-vdi-326_server_00.log
+iodemo_mtr-vdi-326_server_01.log
+iodemo_mtr-vdi-326_client_00.log
+iodemo_mtr-vdi-326_client_01.log
+
+% ls -1 iodemo_commands_mtr-vdi-326.sh
+iodemo_commands_mtr-vdi-326.sh
+
+```
+
+## Check what tags are provided by script for start/stop/status operations
+
+``` bash
+%./iodemo_commands_mtr-vdi-326.sh -show-tags
+Showing tags
+==== Servers:
+server_0
+server_1
+server_2
+server_3
+server_4
+==== Clients:
+client_0
+client_1
+client_2
+client_3
+client_4
+```
+
+## Running individual processes by tag (start/stop/status)
+
+
+
+``` bash
+%./iodemo_commands_mtr-vdi-326.sh -tag server_0 -status
+%./iodemo_commands_mtr-vdi-326.sh -tag server_0 -stop
+%./iodemo_commands_mtr-vdi-326.sh -tag server_0 -start
+
+```
+
+## Running all processes
+
+``` bash
+%./iodemo_commands_mtr-vdi-326.sh
+```
+

test/apps/iodemo/run_io_demo.sh

@@ -427,16 +427,111 @@ make_scripts()
 			    kill_iodemo
 			}
 
+
+
 			# kill existing processes and trap signals
 			trap signal_handler INT TERM
-			kill_iodemo
 
 			# Use same work dir as launch script
 			cd $PWD
 
+			num_servers_per_host=${num_servers}
+			num_clients_per_host=${num_clients}
+			iodemo_exe=${iodemo_exe}
+
 			# UCX inherited environment variables
 			EOF
 
+		cat >>${command_file} <<-'EOF'
+			parse_args() {
+				while [[ $# -gt 0 ]]
+				do
+					key="$1"
+					case $key in
+					-tag)
+						tag="$2"
+						shift
+						;;
+					-start)
+						oper="start"
+						;;
+					-stop)
+						oper="stop"
+						;;
+					-status)
+						oper="status"
+						;;
+					-show-tags)
+						echo "Available tags"
+						echo ==== Servers:
+						for ((i=0;i<${num_servers_per_host[${host}]};++i))
+						do
+							echo server_${i}
+						done 
+						echo ==== Clients:
+						for ((i=0;i<${num_clients_per_host[${host}]};++i))
+						do
+							echo client_${i}
+						done
+						echo
+						exit 0
+						;;
+					*)
+						error "Invalid parameter '${key}'"
+						;;
+					esac
+					shift
+				done
+
+
+
+				case $oper in
+				start)
+					func="${oper}_${tag}"
+					if [ -n "${func}" ]; then
+						if type $func &>/dev/null; then
+							echo Starting $tag
+						else
+							echo No method defined to start $tag
+							exit 1
+						fi
+					fi
+					eval "${func}"
+					exit 0
+				;;
+				stop)
+					if [ -z "$tag" ]; then
+						echo Error: specify -tag param
+						exit 1
+					fi
+					for pid in $(pgrep -f $iodemo_exe)
+					do
+						if xargs --null --max-args=1 echo < /proc/$pid/environ |grep -q IODEMO_ROLE=$tag; then
+								echo Stopping $iodemo_exe with role=$tag pid=$pid
+								kill -9 $pid
+						fi
+					done
+					exit 0
+				;;
+				status)
+					if [ -z "$tag" ]; then
+						echo Error: specify -tag param
+						exit 1
+					fi
+					for pid in $(pgrep -f $iodemo_exe)
+					do
+						if xargs --null --max-args=1 echo < /proc/$pid/environ |grep -q IODEMO_ROLE=$tag; then
+								echo Found $iodemo_exe with role=$tag pid=$pid
+						fi
+					done
+					exit 0
+				;;
+				*)
+				;;
+				esac
+			}
+			EOF
+
 		# Add all relevant env vars which start with UCX_ to the command file
 		env | grep -P '^UCX_.*=|^PATH=|^LD_LIBRARY_PATH=' | \
 			xargs -L 1 echo export >>${command_file}
@@ -452,21 +547,18 @@ make_scripts()
 		do
 			port_num=$((base_port_num + i))
 			log_file=$(printf "iodemo_%s_server_%02d.log" ${host} $i)
+			cmd_prefix+=" env IODEMO_ROLE=server_${i} "
 			echo ${log_file}
 			cat >>${command_file} <<-EOF
+			function start_server_${i}() {
 				${cmd_prefix} \\
 				    ${iodemo_exe} \\
 				        ${iodemo_server_args} -p ${port_num} \\
 				        ${log_redirect} ${log_file} &
+			}
 				EOF
 		done
 
-		# Add short sleep to let servers start running
-		cat >>${command_file} <<-EOF
-
-			# Wait for servers to start
-			sleep ${client_wait_time}
-			EOF
 
 		# Add client commands
 		cat >>${command_file} <<-EOF
@@ -478,12 +570,38 @@ make_scripts()
 		for ((i=0;i<num_clients_per_host[${host}];++i))
 		do
 			log_file=$(printf "iodemo_%s_client_%02d.log" ${host} $i)
+			cmd_prefix+=" env IODEMO_ROLE=client_${i} "
 			echo ${log_file}
 			cat >>${command_file} <<-EOF
+			function start_client_${i}() {
 				${cmd_prefix} \\
 				    ${iodemo_exe} \\
 				        ${iodemo_client_args} ${client_connect_list} \\
 				        ${log_redirect} ${log_file} &
+			}
+				EOF
+		done
+
+		cat >>${command_file} <<-EOF
+		main() {
+		EOF
+		for ((i=0;i<${num_servers_per_host[${host}]};++i))
+		do
+			cat >>${command_file} <<-EOF
+			start_server_${i}
+				EOF
+		done
+
+		# Add short sleep to let servers start running
+		cat >>${command_file} <<-EOF
+			# Wait for servers to start
+			sleep ${client_wait_time}
+		EOF
+
+		for ((i=0;i<num_clients_per_host[${host}];++i))
+		do
+			cat >>${command_file} <<-EOF
+			start_client_${i}
 				EOF
 		done
 
@@ -493,6 +611,12 @@ make_scripts()
 			# Wait for background processes
 			wait ${wait_redirect}
 			echo "Test finished"
+		}
+			EOF
+		cat >>${command_file} <<-'EOF'
+		parse_args "$@"
+		kill_iodemo
+		main
 			EOF
 		chmod a+x ${command_file}
 	done