diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8b6f4db02..13053a31a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -8,10 +8,10 @@ jobs:
       - checkout
 
       # Set up dendrite
-      - run: 
+      - run:
           name: Build Dendrite
           command: ./build.sh
-      - run: 
+      - run:
           name: Copy dummy keys to root
           command: |
             mv .circleci/matrix_key.pem .
@@ -19,6 +19,9 @@ jobs:
       - run:
           name: Run sytest with whitelisted tests
           command: /dendrite_sytest.sh
+      - run:
+          name: Check for new tests to be added to testfile
+          command: /show-expected-fail-tests.sh /logs/results.tap
 
       - store_artifacts:
           path: /logs
diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh
new file mode 100755
index 000000000..7641b4722
--- /dev/null
+++ b/show-expected-fail-tests.sh
@@ -0,0 +1,17 @@
+#! /bin/bash
+
+results_file=$1
+tests_to_add=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//')
+
+fail_build=0
+while read -r test_id; do
+	grep "${test_id}" testfile > /dev/null 2>&1
+	if [ "$?" != "0" ]; then
+		echo "ERROR: Passed test not present in testfile: ${test_id}"
+		fail_build=1
+	else
+		echo "WARN: Test in testfile still marked as expected fail: ${test_id}"
+	fi
+done <<< "${tests_to_add}"
+
+exit ${fail_build}