Add Are We Synapse Yet to GHA (#2321)

* Add Are We Synapse Yet to GHA

* Better output & add comments

.github/workflows/dendrite.yml

@@ -233,7 +233,14 @@ jobs:
       - name: Summarise results.tap
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-
+      - name: Sytest List Maintenance
+        if: ${{ always() }}
+        run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
+        continue-on-error: true # not fatal
+      - name: Are We Synapse Yet?
+        if: ${{ always() }}
+        run: /src/are-we-synapse-yet.py /logs/results.tap -v
+        continue-on-error: true # not fatal
       - name: Upload Sytest logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}

are-we-synapse-yet.py

@@ -3,7 +3,7 @@
 from __future__ import division
 import argparse
 import re
-import sys
+import os
 
 # Usage: $ ./are-we-synapse-yet.py [-v] results.tap
 # This script scans a results.tap file from Dendrite's CI process and spits out
@@ -156,6 +156,7 @@ def parse_test_line(line):
 #    ✓ POST /register downcases capitals in usernames
 #    ...
 def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
+    ci = os.getenv("CI") # When running from GHA, this groups the subsections
     subsections = [] # Registration: 100% (13/13 tests)
     subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"]
     total_passing = 0
@@ -169,7 +170,7 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
         for name, passing in tests.items():
             if passing:
                 group_passing += 1
-            test_names_and_marks.append(f"{'✓' if passing else '×'} {name}")
+            test_names_and_marks.append(f"{'✅' if passing else '❌'} {name}")
             
         total_tests += group_total
         total_passing += group_passing
@@ -186,11 +187,11 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
     print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests))
     print("-" * (len(header_name)+1))
     for line in subsections:
-        print("  %s" % (line,))
+        print("%s%s" % ("::group::" if ci and verbose else "", line,))
         if verbose:
             for test_name_and_pass_mark in subsection_test_names[line]:
                 print("    %s" % (test_name_and_pass_mark,))
-            print("")
+            print("%s" % ("::endgroup::" if ci else ""))
     print("")
 
 def main(results_tap_path, verbose):

show-expected-fail-tests.sh

@@ -89,17 +89,17 @@ if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then
 fi
 
 if [ -n "${tests_to_add}" ]; then
-	echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:"
-    echo "\`\`\`"
-    echo -e "${tests_to_add}"
-    echo "\`\`\`"
+	echo "::error::The following tests passed but are not present in \`$2\`. Please append them to the file:"
+	echo "::group::Passing tests"
+  echo -e "${tests_to_add}"
+  echo "::endgroup::"
 fi
 
 if [ -n "${already_in_whitelist}" ]; then
-	echo "**WARN**: Tests in the whitelist still marked as **expected fail**:"
-    echo "\`\`\`"
-    echo -e "${already_in_whitelist}"
-    echo "\`\`\`"
+	echo "::warning::Tests in the whitelist still marked as **expected fail**:"
+	echo "::group::Still marked as expected fail"
+  echo -e "${already_in_whitelist}"
+  echo "::endgroup::"
 fi
 
 exit ${fail_build}