Update CI roles

tests/roles/custom-test-fonts/files/run-family-test

@@ -83,13 +83,13 @@ debug "Family: $OPT_FAMILY"
 debug "Lang: $OPT_LANG"
 debug "Artifacts dir: $OPT_ARTIFACTS_DIR"
 debug "Package name: $OPT_PACKAGE"
-STR_TEST_DASHED=$(echo "${OPT_PACKAGE}_${OPT_ALIAS}" | sed -e 's/\//-/g' -e 's/ /-/g')
+STR_TEST_DASHED=$(echo "${OPT_PACKAGE}_${OPT_ALIAS}_${OPT_LANG}" | sed -e 's/\//-/g' -e 's/ /-/g')
 debug "Log file: $STR_TEST_DASHED.log"
 
 clean_exit() {
 	rc=$?;
 	trap - SIGINT SIGTERM SIGABRT EXIT
-	echo "Run test $OPT_ALIAS: done."
+	echo "Run test $OPT_ALIAS: done. Test's exit code: $rc"
 	for pid in $(ps -o pid --no-headers --ppid $$); do
 		if [ -n "$(ps -p $pid -o pid=)" ]; then
 			kill -s HUP $pid
@@ -114,11 +114,16 @@ clean_exit() {
 		status="ERROR"
 	fi
 	echo "$status $OPT_ALIAS" >> "$OPT_ARTIFACTS_DIR/test.log"
-	mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}_${log_file_name}"
+	mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}"
 	local results="$OPT_ARTIFACTS_DIR/results.yml"
 	local result=$(echo $status | tr '[:upper:]' '[:lower:]')
 	test -f "$results" || echo 'results:' > "$results"
-	echo "- {result: $result, test: $OPT_ALIAS}" >> "$results"
+	printf '%s\n' '' \
+	       "- test: $OPT_ALIAS" \
+	       "  result: $result" \
+	       "  logs:" \
+	       "  - ${status}_${log_file_name}" \
+	       >> "$results"
 	exit 0
 }
 trap clean_exit SIGINT SIGTERM SIGABRT EXIT
@@ -139,15 +144,19 @@ debug "Config: `cat $tmpconfd/fonts.conf`"
 
 mkdir -p "$OPT_ARTIFACTS_DIR"
 export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log"
-logfile_stdout="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log"
+logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log"
-logfile_stderr="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED-err.log"
+logfile="$(realpath "$logfile")"
-exec 3>&1 4>&2 1> >(tee -a "$logfile_stdout" >&3) 2> >(tee -a "$logfile_stderr" >&4)
+exec > >(tee -a "$logfile") 2>&1
 
 debug "Check family assignment"
 res=`FONTCONFIG_FILE=$tmpconfd/fonts.conf fc-match -f "%{family[0]}" :family=$OPT_ALIAS:lang=$OPT_LANG`
+ret=0
 if [ "x$res" = "x$OPT_FAMILY" ]; then
 	echo "RESULT: PASS: $OPT_FAMILY was assigned to $OPT_ALIAS as expected"
 else
-	echo "RESULT: FAIL: $OPT_FAMILY wasn't assigned to $OPT_ALIAS (actual result: $res)"
+    echo "RESULT: FAIL: $OPT_FAMILY wasn't assigned to $OPT_ALIAS (actual result: $res)"
+    ret=1
 fi
 rm -rf $tmpconfd
+
+exit $ret

tests/roles/custom-test-fonts/files/run-lang-coverage-test

@@ -1,13 +1,13 @@
 #! /bin/bash -efu
 
 debug() {
-	if [ -n "$DEBUG" ]; then
+    if [ -n "$DEBUG" ]; then
-		echo "$*" >&2
+	echo "$*" >&2
-	fi
+    fi
 }
 
 msg_usage() {
-	cat <<_EOF_
+    cat <<_EOF_
 Run language coverage test.
 
 Usage:
@@ -28,7 +28,7 @@ _EOF_
 PROG="${PROG:-${0##*/}}"
 DEBUG="${DEBUG:-}"
 OPT_LANG="${OPT_LANG:-en}"
-OPT_PATH="${OPT_PATH:-}"
+OPT_PATH=()
 OPT_ARTIFACTS_DIR="${OPT_ARTIFACTS_DIR:-}"
 OPT_EXCLUDE=()
 OPT_INCLUDE=()
@@ -37,161 +37,197 @@ OPT_NAME="${OPT_NAME:-}"
 opt=$(getopt -n "$0" --options "hvl:p:n:a:e:i:" --longoptions "help,verbose,lang:,path:,name:,artifactsdir:,exclude:,include:" -- "$@")
 eval set -- "$opt"
 while [[ $# -gt 0 ]]; do
-	case "$1" in
+    case "$1" in
 	-n|--name)
-		OPT_NAME="$2"
+	    OPT_NAME="$2"
-		shift 2
+	    shift 2
-		;;
+	    ;;
 	-i|--include)
-		OPT_INCLUDE+=("$2")
+	    OPT_INCLUDE+=("$2")
-		shift 2
+	    shift 2
-		;;
+	    ;;
 	-e|--exclude)
-		OPT_EXCLUDE+=("$2")
+	    OPT_EXCLUDE+=("$2")
-		shift 2
+	    shift 2
-		;;
+	    ;;
 	-a|--artifactsdir)
-		OPT_ARTIFACTS_DIR="$2"
+	    OPT_ARTIFACTS_DIR="$2"
-		shift 2
+	    shift 2
-		;;
+	    ;;
 	-p|--path)
-		OPT_PATH="$2"
+	    OPT_PATH+=("$2")
-		shift 2
+	    shift 2
-		;;
+	    ;;
 	-l|--lang)
-		OPT_LANG="$2"
+	    OPT_LANG="$2"
-		shift 2
+	    shift 2
-		;;
+	    ;;
 	-v|--verbose)
-		DEBUG="-v"
+	    DEBUG="-v"
-		shift
+	    shift
-		;;
+	    ;;
 	-h|--help)
-		msg_usage
+	    msg_usage
-		exit 0
+	    exit 0
-		;;
+	    ;;
 	--)
-		shift
+	    shift
-		;;
+	    ;;
 	*)
-		msg_usage
+	    msg_usage
-		exit 1
+	    exit 1
-	esac
+    esac
 done
 
 if [ -z "$OPT_ARTIFACTS_DIR" ] || [ -z "$OPT_LANG" ] || [ -z "$OPT_PATH" ]; then
-	echo "Use: $PROG -h for help."
+    echo "Use: $PROG -h for help."
-	exit 0
+    exit 0
 fi
 
-expand_regex() {
+STR_TEST_DASHED=$(echo "${OPT_NAME:-$OPT_LANG}" | sed -e 's/\//-/g')
-	local e ret=()
-	local path="$1"
-	shift
-	(cd $path;
-	for e; do
-		debug "$e"
-		set +f
-		local x=$(find -regextype posix-egrep -regex "./$e" -print|sed -e 's,^\./,,g')
-		ret+=($x)
-		set -f
-		echo ${ret[@]}
-	done)
-	echo ${ret[@]}
-}
-
-OPT_INCLUDE=($(expand_regex $OPT_PATH ${OPT_INCLUDE[@]}))
-OPT_EXCLUDE=($(expand_regex $OPT_PATH ${OPT_EXCLUDE[@]}))
-
-debug "Path: $OPT_PATH"
-debug "Lang: $OPT_LANG"
-debug "Artifacts dir: $OPT_ARTIFACTS_DIR"
-debug "Exclude: ${OPT_EXCLUDE[@]}"
-debug "Include: ${OPT_INCLUDE[@]}"
-STR_TEST_DASHED=$(echo "${OPT_NAME:-${OPT_LANG}_${OPT_PATH}}" | sed -e 's/\//-/g')
 
 clean_exit() {
-	rc=$?;
+    rc=$?;
-	trap - SIGINT SIGTERM SIGABRT EXIT
+    trap - SIGINT SIGTERM SIGABRT EXIT
-	echo "Run test $OPT_PATH: done."
+    echo "Run test $OPT_LANG: done. Test's exit code: $rc"
-	for pid in $(ps -o pid --no-headers --ppid $$); do
+    for pid in $(ps -o pid --no-headers --ppid $$); do
-		if [ -n "$(ps -p $pid -o pid=)" ]; then
+	if [ -n "$(ps -p $pid -o pid=)" ]; then
-			kill -s HUP $pid
+	    kill -s HUP $pid
-		fi
-	done
-	local log_file_name="$STR_TEST_DASHED.log"
-	local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name"
-	local status
-	if [[ $rc -eq 127 ]]; then
-		status="ERROR"
-	elif grep -q "RESULT: WARN" "$log_file_path"; then
-		status="ERROR"
-	elif grep -q "RESULT: FAIL" "$log_file_path"; then
-		status="FAIL"
-	elif grep -q "RESULT: PASS" "$log_file_path"; then
-		status="PASS"
-	elif grep -q "FAIL" "$log_file_path"; then
-		status="FAIL"
-	elif grep -q "PASS" "$log_file_path"; then
-		status="PASS"
-	else
-		status="ERROR"
 	fi
-	echo "$status $OPT_PATH" >> "$OPT_ARTIFACTS_DIR/test.log"
+    done
-	mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}_${log_file_name}"
+    local log_file_name="$STR_TEST_DASHED.log"
-	local results="$OPT_ARTIFACTS_DIR/results.yml"
+    local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name"
-	local result=$(echo $status | tr '[:upper:]' '[:lower:]')
+    local status
-	test -f "$results" || echo 'results:' > "$results"
+    if [[ $rc -eq 127 ]]; then
-	echo "- {result: $result, test: $OPT_PATH}" >> "$results"
+	status="ERROR"
-	exit 0
+    elif grep -q "RESULT: WARN" "$log_file_path"; then
+	status="ERROR"
+    elif grep -q "RESULT: FAIL" "$log_file_path"; then
+	status="FAIL"
+    elif grep -q "RESULT: PASS" "$log_file_path"; then
+	status="PASS"
+    elif grep -q "WARN" "$log_file_path"; then
+	status="ERROR"
+    elif grep -q "FAIL" "$log_file_path"; then
+	status="FAIL"
+    elif grep -q "PASS" "$log_file_path"; then
+	status="PASS"
+    else
+	status="ERROR"
+    fi
+    echo "$status $OPT_LANG" >> "$OPT_ARTIFACTS_DIR/test.log"
+    mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}"
+    local results="$OPT_ARTIFACTS_DIR/results.yml"
+    local result=$(echo $status | tr '[:upper:]' '[:lower:]')
+    test -f "$results" || echo 'results:' > "$results"
+    printf '%s\n' '' \
+	   "- test: $OPT_LANG" \
+	   "  result: $result" \
+	   "  logs:" \
+	   "  - ${status}_${log_file_name}" \
+	   >> "$results"
+    exit 0
 }
 trap clean_exit SIGINT SIGTERM SIGABRT EXIT
 
 mkdir -p "$OPT_ARTIFACTS_DIR"
 export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log"
-logfile_stdout="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log"
+logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log"
-logfile_stderr="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED-err.log"
+logfile="$(realpath "$logfile")"
-exec 3>&1 4>&2 1> >(tee -a "$logfile_stdout" >&3) 2> >(tee -a "$logfile_stderr" >&4)
+exec > >(tee -a "$logfile") 2>&1
+
+expand_regex() {
+    local p ret=()
+    local regex="$1"
+    shift
+    debug "Expanding $regex"
+    for p; do
+	set +f
+	debug "$p: $regex"
+	 (cd $p;
+	  local x=$(find -regextype posix-egrep -regex "./$regex" -print|sed -e 's,^\./,,g')
+	  debug "$x"
+	  ret+=($x)
+	  set -f
+	  echo -n ${ret[@]}
+	 )
+    done
+    echo -n ${ret[@]}
+}
+
+iv=()
+ev=()
+x=()
+for p in ${OPT_INCLUDE[@]}; do
+    x=$(expand_regex $p ${OPT_PATH[@]})
+    if [ "x$x" == "x" ]; then
+	echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?"
+	continue
+    fi
+    iv=("${iv[@]}" "${x[@]}")
+done
+for p in ${OPT_EXCLUDE[@]}; do
+    x=$(expand_regex $p ${OPT_PATH[@]})
+    if [ "x$x" == "x" ]; then
+	echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?"
+	continue
+    fi
+    ev=("${ev[@]}" "${x[@]}")
+done
+OPT_EXCLUDE=(${ev[@]})
+OPT_INCLUDE=(${iv[@]})
+
+debug "Path: ${OPT_PATH[@]}"
+debug "Lang: $OPT_LANG"
+debug "Artifacts dir: $OPT_ARTIFACTS_DIR"
+debug "Exclude: ${#OPT_EXCLUDE[@]}: ${OPT_EXCLUDE[@]}"
+debug "Include: ${#OPT_INCLUDE[@]}: ${OPT_INCLUDE[@]}"
 
 contains() {
-	local e match="$1"
+    local e match="$1"
-	shift
+    shift
-	for e; do [[ "$e" == "$match" ]] && return 1; done
+    for e; do [[ "$e" == "$match" ]] && return 1; done
-	return 0
+    return 0
 }
 
 debug "Check language coverage"
+ret=0
 set +f
-for i in `find $OPT_PATH -regex '.*/*\.\(t1\)?\(ttf\)?\(otf\)?\(ttc\)?\(pcf.*\)?\(pfa\)?'`; do
+for p in ${OPT_PATH[@]}; do
-set -f
+    for i in `find $p -regex '.*/*\.\(t1\)?\(ttf\)?\(otf\)?\(ttc\)?\(pcf.*\)?\(pfa\)?'`; do
+	set -f
 	debug "$i"
 	if test -f $i; then
-		n=`basename $i`
+	    n=`basename $i`
+	    set +e
+	    contains "$n" "${OPT_EXCLUDE[@]}"
+	    ret=$?
+	    set -e
+	    if [ $ret -eq 1 ]; then
+		debug "ignoring $i"
+		continue
+	    fi
+	    if [ ${#OPT_INCLUDE[@]} -ne 0 ]; then
 		set +e
-		contains "$n" "${OPT_EXCLUDE[@]}"
+		contains "$n" "${OPT_INCLUDE[@]}"
 		ret=$?
 		set -e
-		if [ $ret -eq 1 ]; then
+		if [ $ret -eq 0 ]; then
-			debug "ignoring $i"
+		    debug "$i isn't targeted file"
-			continue
+		    continue
-		fi
-		if [ ${#OPT_INCLUDE[@]} -ne 0 ]; then
-			set +e
-			contains "$n" "${OPT_INCLUDE[@]}"
-			ret=$?
-			set -e
-			if [ $ret -eq 0 ]; then
-				debug "$i isn't targeted file"
-				continue
-			fi
-		fi
-		debug "  $i"
-		res=`fc-validate -l $OPT_LANG $i`
-		if echo $res | grep -q Missing; then
-			echo "RESULT: FAIL: $i doesn't satisfy $OPT_LANG language coverage."
-		else
-			echo "RESULT: PASS: $i satisfy $OPT_LANG language coverage."
 		fi
+		NOT_MATCHED=("${NOT_MATCHED[@]/$n}")
+	    fi
+	    debug "  $i"
+	    res=`fc-validate -l $OPT_LANG $i || :`
+	    if echo $res | grep -q Missing; then
+		echo "RESULT: FAIL: $i doesn't satisfy $OPT_LANG language coverage."
+		ret=1
+	    else
+		echo "RESULT: PASS: $i satisfy $OPT_LANG language coverage."
+	    fi
 	fi
+    done
 done
 
+exit $ret

tests/roles/custom-test-fonts/tasks/main.yml

@@ -2,15 +2,23 @@
 
 - block:
   - name: language coverage
-    script: run-lang-coverage-test --lang "{{ item }}" --path "{{ path_prefix }}" --artifactsdir "{{ remote_artifacts }}" {{ "" if coverage.values is not defined or coverage[item].name is not defined else "--name " + coverage[item].name }} {{ "" if coverage.values is not defined or coverage[item].exclude is not defined else '--exclude "' + (coverage[item].exclude | join('" --exclude "')) + '"'}} {{ "" if coverage.values is not defined or coverage[item].include is not defined else '--include "' + (coverage[item].include | join('" --include "')) + '"'}}
+    environment:
+      LANG: "en_US.UTF-8"
+    script: run-lang-coverage-test --lang "{{ item }}" {% if coverage.values is not defined or coverage[item].path_prefix is not defined %} --path {{ path_prefix }} {% else %} {{ '--path "' + (coverage[item].path_prefix | join('" --path "')) + '"' }} {% endif %} --artifactsdir "{{ remote_artifacts }}" {{ "" if coverage.values is not defined or coverage[item].name is not defined else "--name " + coverage[item].name }} {{ "" if coverage.values is not defined or coverage[item].exclude is not defined else '--exclude "' + (coverage[item].exclude | join('" --exclude "')) + '"'}} {{ "" if coverage.values is not defined or coverage[item].include is not defined else '--include "' + (coverage[item].include | join('" --include "')) + '"'}}
     with_items:
     - "{{ coverage if coverage.keys is not defined else coverage.keys()|list }}"
   - name: generic family assignment
+    environment:
+      LANG: "en_US.UTF-8"
     when: families is defined
     script: run-family-test --lang {{ item.lang }} --family '{{ item.family }}' --alias {{ item.alias }} --artifactsdir {{ remote_artifacts }} --package {{ package if item.package is not defined else item.package }}
     with_items:
     - "{{ families }}"
 
+  - name: Collect all of failures into one file
+    shell: |
+      grep -v PASS "{{ remote_artifacts }}"/FAIL_*.log > "{{ remote_artifacts }}"/FAIL-all.log || :
+
   - name: Check the results
     shell: |
       log="{{ remote_artifacts }}/test.log"
@@ -38,3 +46,7 @@
 
   - include_role:
       name: str-common-final
+
+  - name: Validate the result
+    shell: echo "test_results.stdout"
+    failed_when: test_results.stdout != 'PASS'