From 2a3865591faca21507a8e2332cf52ffd50cfb148 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 21 Jan 2020 19:02:24 +0000
Subject: [PATCH] Add ability to blacklist sytests

---
 docs/sytest.md              |  2 +-
 show-expected-fail-tests.sh | 62 +++++++++++++++++++++++++++++++------
 2 files changed, 54 insertions(+), 10 deletions(-)

diff --git a/docs/sytest.md b/docs/sytest.md
index e936dc493..a5998dd36 100644
--- a/docs/sytest.md
+++ b/docs/sytest.md
@@ -59,7 +59,7 @@ Once the tests are complete, run the helper script to see if you need to add
 any newly passing test names to `testfile` in the project's root directory:
 
 ```sh
-../dendrite/show-expected-fail-tests.sh results.tap ../dendrite/testfile
+../dendrite/show-expected-fail-tests.sh results.tap ../dendrite/testfile ../dendrite/sytest-blacklist
 ```
 
 If the script prints nothing/exits with 0, then you're good to go.
diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh
index 80b842ab1..b2f17c495 100755
--- a/show-expected-fail-tests.sh
+++ b/show-expected-fail-tests.sh
@@ -1,34 +1,78 @@
 #! /bin/bash
+#
+# Parses a results.tap file from SyTest output and a file containing test names (a test whitelist)
+# and checks whether a test name that exists in the whitelist (that should pass), failed or not.
+#
+# An optional blacklist file can be added, also containing test names, where if a test name is
+# present, the script will not error even if the test is in the whitelist file and failed
+#
+# For each of these files, lines starting with '#' are ignored.
+#
+# Usage ./show-expected-fail-tests.sh results.tap testfile [blacklist_file]
 
 results_file=$1
-testfile=$2
+whitelist_file=$2
+blacklist_file=$3
 
 fail_build=0
 
+if [ $# -lt 2 ]; then
+	echo "Usage: $0 results.tap testfile [blacklist]"
+	exit 1
+fi
+
 if [ ! -f "$results_file" ]; then
-	echo "ERROR: Specified results file ${results_file} doesn't exist."
+	echo "ERROR: Specified results file '${results_file}' doesn't exist."
 	fail_build=1
 fi
 
-if [ ! -f "$testfile" ]; then
-	echo "ERROR: Specified testfile ${testfile} doesn't exist."
+if [ ! -f "$whitelist_file" ]; then
+	echo "ERROR: Specified test whitelist '${whitelist_file}' doesn't exist."
 	fail_build=1
 fi
 
+blacklisted_tests=()
+
+# Check if a blacklist file was provided
+if [ $# -eq 3 ]; then
+	# Read test blacklist file
+	if [ ! -f "$blacklist_file" ]; then
+		echo "ERROR: Specified test blacklist file '${blacklist_file}' doesn't exist."
+		fail_build=1
+	fi
+
+	# Read each line, ignoring those that start with '#'
+	blacklisted_tests=""
+	search_non_comments=$(grep -v '^#' ${blacklist_file})
+	while read -r line ; do
+		# Record the blacklisted test name
+		blacklisted_tests+=("${line}")
+	done <<< "${search_non_comments}"  # This allows us to edit blacklisted_tests in the while loop
+fi
+
 [ "$fail_build" = 0 ] || exit 1
 
 passed_but_expected_fail=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//')
 tests_to_add=""
 already_in_testfile=""
 
-while read -r test_id; do
-	[ "${test_id}" = "" ] && continue
-	grep "${test_id}" "${testfile}" > /dev/null 2>&1
+while read -r test_name; do
+	# Ignore empty lines
+	[ "${test_name}" = "" ] && continue
+
+	grep "${test_name}" "${testfile}" > /dev/null 2>&1
 	if [ "$?" != "0" ]; then
-		tests_to_add="${tests_to_add}${test_id}\n"
+		# Check if this test name is blacklisted
+		if printf '%s\n' "${blacklisted_tests[@]}" | grep -q -P "^${test_name}$"; then
+			# Don't notify about this test
+			continue
+		fi
+
+		# Append this test_name to the existing list
+		tests_to_add="${tests_to_add}${test_name}\n"
 		fail_build=1
 	else
-		already_in_testfile="${already_in_testfile}${test_id}\n"
+		already_in_testfile="${already_in_testfile}${test_name}\n"
 	fi
 done <<< "${passed_but_expected_fail}"