diff --git a/appservice/README.md b/appservice/README.md
deleted file mode 100644
index d75557448..000000000
--- a/appservice/README.md
+++ /dev/null
@@ -1,10 +0,0 @@
-# Application Service
-
-This component interfaces with external [Application
-Services](https://matrix.org/docs/spec/application_service/unstable.html).
-This includes any HTTP endpoints that application services call, as well as talking
-to any HTTP endpoints that application services provide themselves.
-
-## Consumers
-
-This component consumes and filters events from the Roomserver Kafka stream, passing on any necessary events to subscribing application services.
\ No newline at end of file
diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index 60522f6d7..a73a6c671 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -207,7 +207,7 @@ func (s *appserviceState) backoffAndPause(err error) error {
 		s.backoff++
 	}
 	duration := time.Second * time.Duration(math.Pow(2, float64(s.backoff)))
-	log.WithField("appservice", s.ID).WithError(err).Warnf("Unable to send transaction to appservice successfully, backing off for %s", duration.String())
+	log.WithField("appservice", s.ID).WithError(err).Errorf("Unable to send transaction to appservice, backing off for %s", duration.String())
 	time.Sleep(duration)
 	return err
 }
diff --git a/setup/jetstream/helpers.go b/setup/jetstream/helpers.go
index f47637c69..99a0d2d4a 100644
--- a/setup/jetstream/helpers.go
+++ b/setup/jetstream/helpers.go
@@ -3,6 +3,7 @@ package jetstream
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/getsentry/sentry-go"
 	"github.com/nats-io/nats.go"
@@ -101,7 +102,7 @@ func JetStreamConsumer(
 					sentry.CaptureException(err)
 				}
 			} else {
-				if err = msg.Nak(nats.Context(ctx)); err != nil {
+				if err = msg.NakWithDelay(time.Second*15, nats.Context(ctx)); err != nil {
 					logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.Nak: %w", err))
 					sentry.CaptureException(err)
 				}