diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go
index 61663be31..2a70f7ed3 100644
--- a/federationsender/internal/api.go
+++ b/federationsender/internal/api.go
@@ -70,7 +70,10 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti
 	if !ok {
 		return stats.Failure()
 	}
-	if mxerr.Code >= 500 && mxerr.Code < 600 {
+	if mxerr.Code == 401 { // invalid signature in X-Matrix header
+		return stats.Failure()
+	}
+	if mxerr.Code >= 500 && mxerr.Code < 600 { // internal server errors
 		return stats.Failure()
 	}
 	return