From 3da182212e86daca1d7019efbd424b2bd38b8b3c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 13:02:41 +0100
Subject: [PATCH] Track reasons why the process is in a degraded state

---
 setup/base/base.go       |  8 +++++++-
 setup/jetstream/nats.go  |  6 +++---
 setup/process/process.go | 35 +++++++++++++++++++++++------------
 3 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/setup/base/base.go b/setup/base/base.go
index 0636c7b8d..2e3a3a195 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"database/sql"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -467,8 +468,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		w.WriteHeader(200)
 	})
 	b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) {
-		if b.ProcessContext.IsDegraded() {
+		if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded {
 			w.WriteHeader(503)
+			_ = json.NewEncoder(w).Encode(struct {
+				Warnings []string `json:"warnings"`
+			}{
+				Warnings: reasons,
+			})
 			return
 		}
 		w.WriteHeader(200)
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 7409fd6c8..af4eb2949 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -169,9 +169,9 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
 					// We've managed to add the stream in memory.  What's on the
 					// disk will be left alone, but our ability to recover from a
 					// future crash will be limited. Yell about it.
-					sentry.CaptureException(fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible", namespaced.Name))
-					logrus.Warn("Stream is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible")
-					process.Degraded()
+					err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name)
+					sentry.CaptureException(err)
+					process.Degraded(err)
 				}
 			}
 		}
diff --git a/setup/process/process.go b/setup/process/process.go
index 06ef60217..b2d2844a8 100644
--- a/setup/process/process.go
+++ b/setup/process/process.go
@@ -2,19 +2,18 @@ package process
 
 import (
 	"context"
-	"fmt"
 	"sync"
 
 	"github.com/getsentry/sentry-go"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 )
 
 type ProcessContext struct {
-	wg       *sync.WaitGroup    // used to wait for components to shutdown
-	ctx      context.Context    // cancelled when Stop is called
-	shutdown context.CancelFunc // shut down Dendrite
-	degraded atomic.Bool
+	mu       sync.RWMutex
+	wg       *sync.WaitGroup     // used to wait for components to shutdown
+	ctx      context.Context     // cancelled when Stop is called
+	shutdown context.CancelFunc  // shut down Dendrite
+	degraded map[string]struct{} // reasons why the process is degraded
 }
 
 func NewProcessContext() *ProcessContext {
@@ -50,13 +49,25 @@ func (b *ProcessContext) WaitForComponentsToFinish() {
 	b.wg.Wait()
 }
 
-func (b *ProcessContext) Degraded() {
-	if b.degraded.CompareAndSwap(false, true) {
-		logrus.Warn("Dendrite is running in a degraded state")
-		sentry.CaptureException(fmt.Errorf("Process is running in a degraded state"))
+func (b *ProcessContext) Degraded(err error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if _, ok := b.degraded[err.Error()]; !ok {
+		logrus.WithError(err).Warn("Dendrite has entered a degraded state")
+		sentry.CaptureException(err)
+		b.degraded[err.Error()] = struct{}{}
 	}
 }
 
-func (b *ProcessContext) IsDegraded() bool {
-	return b.degraded.Load()
+func (b *ProcessContext) IsDegraded() (bool, []string) {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	if len(b.degraded) == 0 {
+		return false, nil
+	}
+	reasons := make([]string, 0, len(b.degraded))
+	for reason := range b.degraded {
+		reasons = append(reasons, reason)
+	}
+	return true, reasons
 }