diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index 932a9ba6b..cf121a4bd 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -3,24 +3,87 @@ package main
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
 	"path"
+	"regexp"
+	"runtime"
+	"sort"
 	"strings"
+	"sync"
 	"time"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/codeclysm/extract"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
+	"github.com/docker/go-connections/nat"
 )
 
 var (
-	flagTempDir = flag.String("tmp", "tmp", "Path to temporary directory to dump tarballs to")
+	flagTempDir          = flag.String("tmp", "tmp", "Path to temporary directory to dump tarballs to")
+	flagFrom             = flag.String("from", "0.1.0", "The version to start from e.g '0.3.1'.")
+	flagTo               = flag.String("to", "HEAD", "The version to end on e.g '0.3.3'.")
+	flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images")
+	alphaNumerics        = regexp.MustCompile("[^a-zA-Z0-9]+")
 )
 
+// Embed the Dockerfile to use when building dendrite versions.
+// We cannot use the dockerfile associated with the repo with each version sadly due to changes in
+// Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
+// due to the error:
+//   When using COPY with more than one source file, the destination must be a directory and end with a /
+// We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
+const Dockerfile = `FROM golang:1.13-stretch as build
+RUN apt-get update && apt-get install -y postgresql
+WORKDIR /build
+
+# Copy the build context to the repo as this is the right dendrite code. This is different to the
+# Complement Dockerfile which wgets a branch.
+COPY . .
+
+RUN go build ./cmd/dendrite-monolith-server
+RUN go build ./cmd/generate-keys
+RUN go build ./cmd/generate-config
+RUN ./generate-config --ci > dendrite.yaml
+RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
+
+# Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password
+RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml 
+# No password when connecting over localhost
+RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/9.6/main/pg_hba.conf
+# Bump up max conns for moar concurrency
+RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf
+RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml
+
+# This entry script starts postgres, waits for it to be up then starts dendrite
+RUN echo '\
+#!/bin/bash -eu \n\
+pg_lsclusters \n\
+pg_ctlcluster 9.6 main start \n\
+ \n\
+until pg_isready \n\
+do \n\
+  echo "Waiting for postgres"; \n\
+  sleep 1; \n\
+done \n\
+ \n\
+sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
+./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+' > run_dendrite.sh && chmod +x run_dendrite.sh
+
+ENV SERVER_NAME=localhost
+EXPOSE 8008 8448
+CMD /build/run_dendrite.sh `
+
 func ptr(in string) *string {
 	return &in
 }
@@ -31,8 +94,7 @@ func ptr(in string) *string {
 // the contents of `dockerfile` as a root file `Dockerfile` in the re-tarballed directory such that
 // you can directly feed the retarballed archive to `ImageBuild` to have it run said dockerfile.
 // Returns the tarball buffer on success.
-func downloadArchive(cli *http.Client, archiveURL string, dockerfile []byte) (*bytes.Buffer, error) {
-	tmpDir := *flagTempDir
+func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []byte) (*bytes.Buffer, error) {
 	resp, err := cli.Get(archiveURL)
 	if err != nil {
 		return nil, err
@@ -59,42 +121,264 @@ func downloadArchive(cli *http.Client, archiveURL string, dockerfile []byte) (*b
 	}
 	// now re-tarball it :/
 	var tarball bytes.Buffer
-	err = compress(*flagTempDir, &tarball)
+	err = compress(tmpDir, &tarball)
 	if err != nil {
 		return nil, err
 	}
 	return &tarball, nil
 }
 
+// buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
+func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) {
+	log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir)
+	// pull an archive, this contains a top-level directory which screws with the build context
+	// which we need to fix up post download
+	u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
+	tarball, err := downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
+	if err != nil {
+		return "", fmt.Errorf("failed to download archive %s: %w", u, err)
+	}
+	log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len())
+	log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName)
+	res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
+		Tags: []string{"dendrite-upgrade"},
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to start building image: %s", err)
+	}
+	defer res.Body.Close()
+	decoder := json.NewDecoder(res.Body)
+	// {"aux":{"ID":"sha256:247082c717963bc2639fc2daed08838d67811ea12356cd4fda43e1ffef94f2eb"}}
+	var imageID string
+	for decoder.More() {
+		var dl struct {
+			Stream string                 `json:"stream"`
+			Aux    map[string]interface{} `json:"aux"`
+		}
+		if err := decoder.Decode(&dl); err != nil {
+			return "", fmt.Errorf("failed to decode build image output line: %w", err)
+		}
+		log.Printf("%s: %s", branchOrTagName, dl.Stream)
+		if dl.Aux != nil {
+			imgID, ok := dl.Aux["ID"]
+			if ok {
+				imageID = imgID.(string)
+			}
+		}
+	}
+	return imageID, nil
+}
+
+func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Version, err error) {
+	u := "https://api.github.com/repos/matrix-org/dendrite/tags"
+	res, err := httpClient.Get(u)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("%s returned HTTP %d", u, res.StatusCode)
+	}
+	resp := []struct {
+		Name string `json:"name"`
+	}{}
+	if err = json.NewDecoder(res.Body).Decode(&resp); err != nil {
+		return nil, err
+	}
+	for _, r := range resp {
+		v, err := semver.NewVersion(r.Name)
+		if err != nil {
+			continue // not a semver, that's ok and isn't an error, we allow tags that aren't semvers
+		}
+		semVers = append(semVers, v)
+	}
+	sort.Sort(semver.Collection(semVers))
+	return semVers, nil
+}
+
+func calculateVersions(cli *http.Client, from, to string) ([]string, error) {
+	semvers, err := getAndSortVersionsFromGithub(cli)
+	if err != nil {
+		log.Fatalf("failed to collect semvers from github: %s", err)
+	}
+	// snip the lower bound depending on --from
+	if from != "" {
+		fromVer, err := semver.NewVersion(from)
+		if err != nil {
+			log.Fatalf("invalid --from: %s", err)
+		}
+		i := 0
+		for i = 0; i < len(semvers); i++ {
+			if semvers[i].LessThan(fromVer) {
+				continue
+			}
+			break
+		}
+		semvers = semvers[i:]
+	}
+	if to != "" && to != "HEAD" {
+		toVer, err := semver.NewVersion(to)
+		if err != nil {
+			log.Fatalf("invalid --to: %s", err)
+		}
+		i := len(semvers) - 1
+		for i = len(semvers) - 1; i >= 0; i-- {
+			if semvers[i].GreaterThan(toVer) {
+				continue
+			}
+			break
+		}
+		semvers = semvers[:i+1]
+	}
+	var versions []string
+	for _, sv := range semvers {
+		versions = append(versions, sv.Original())
+	}
+	if *flagTo == "HEAD" {
+		versions = append(versions, "HEAD")
+	}
+	return versions, nil
+}
+
+func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, branchOrTagNames []string) map[string]string {
+	// concurrently build all versions, this can be done in any order. The mutex protects the map
+	branchToImageID := make(map[string]string)
+	var mu sync.Mutex
+
+	var wg sync.WaitGroup
+	wg.Add(concurrency)
+	ch := make(chan string, len(branchOrTagNames))
+	for _, branchName := range branchOrTagNames {
+		ch <- branchName
+	}
+	close(ch)
+
+	for i := 0; i < concurrency; i++ {
+		go func() {
+			defer wg.Done()
+			for branchName := range ch {
+				tmpDir := baseTempDir + alphaNumerics.ReplaceAllString(branchName, "")
+				imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName)
+				if err != nil {
+					log.Fatalf("%s: failed to build dendrite image: %s", branchName, err)
+				}
+				mu.Lock()
+				branchToImageID[branchName] = imgID
+				mu.Unlock()
+			}
+		}()
+	}
+	wg.Wait()
+	return branchToImageID
+}
+
+func runImage(dockerClient *client.Client, volumeName, version, imageID string) (csAPIURL, containerID string, err error) {
+	log.Printf("%s: running image %s\n", version, imageID)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+	body, err := dockerClient.ContainerCreate(ctx, &container.Config{
+		Image: imageID,
+		Env:   []string{"SERVER_NAME=hs1"},
+	}, &container.HostConfig{
+		PublishAllPorts: true,
+		Mounts: []mount.Mount{
+			{
+				Type:   mount.TypeVolume,
+				Source: volumeName,
+				Target: "/usr/local/pgsql/data",
+			},
+		},
+	}, nil, nil, "dendrite_upgrade_test_"+version)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to ContainerCreate: %s", err)
+	}
+	containerID = body.ID
+
+	err = dockerClient.ContainerStart(ctx, containerID, types.ContainerStartOptions{})
+	if err != nil {
+		return "", "", fmt.Errorf("failed to ContainerStart: %s", err)
+	}
+	inspect, err := dockerClient.ContainerInspect(ctx, containerID)
+	if err != nil {
+		return "", "", err
+	}
+	csapiPortInfo, ok := inspect.NetworkSettings.Ports[nat.Port("8008/tcp")]
+	if !ok {
+		return "", "", fmt.Errorf("port 8008 not exposed - exposed ports: %v", inspect.NetworkSettings.Ports)
+	}
+	baseURL := fmt.Sprintf("http://localhost:%s", csapiPortInfo[0].HostPort)
+	versionsURL := fmt.Sprintf("%s/_matrix/client/versions", baseURL)
+	// hit /versions to check it is up
+	var lastErr error
+	for i := 0; i < 100; i++ {
+		res, err := http.Get(versionsURL)
+		if err != nil {
+			lastErr = fmt.Errorf("GET %s => error: %s", versionsURL, err)
+			time.Sleep(50 * time.Millisecond)
+			continue
+		}
+		if res.StatusCode != 200 {
+			lastErr = fmt.Errorf("GET %s => HTTP %s", versionsURL, res.Status)
+			time.Sleep(50 * time.Millisecond)
+			continue
+		}
+		lastErr = nil
+		break
+	}
+	return baseURL, containerID, lastErr
+}
+
+func destroyContainer(dockerClient *client.Client, containerID string) error {
+	return dockerClient.ContainerRemove(context.TODO(), containerID, types.ContainerRemoveOptions{
+		Force: true,
+	})
+}
+
+func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchToImageID map[string]string) error {
+	csAPIURL, containerID, err := runImage(dockerClient, volumeName, v, branchToImageID[v])
+	if err != nil {
+		return fmt.Errorf("failed to run container for branch %v: %v", v, err)
+	}
+	defer destroyContainer(dockerClient, containerID)
+	log.Printf("URL %s -> %s \n", csAPIURL, containerID)
+	if err = runTests(csAPIURL, v); err != nil {
+		return fmt.Errorf("failed to run tests on version %s: %s", v, err)
+	}
+	return nil
+}
+
 func main() {
 	flag.Parse()
 	httpClient := &http.Client{
 		Timeout: 60 * time.Second,
 	}
-	// pull an archive, this contains a top-level directory which screws with the build context
-	// which we need to fix up post download
-	u := "https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz"
-	tarball, err := downloadArchive(httpClient, u, []byte("FROM golang:1.15\nRUN echo 'hello world'"))
+	dockerClient, err := client.NewEnvClient()
 	if err != nil {
-		panic(err)
+		log.Fatalf("failed to make docker client: %s", err)
 	}
-	fmt.Printf("%s => %d bytes\n", u, tarball.Len())
+	if *flagFrom == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+	versions, err := calculateVersions(httpClient, *flagFrom, *flagTo)
+	if err != nil {
+		log.Fatalf("failed to calculate versions to build: %s", err)
+	}
+	log.Printf("Testing dendrite versions: %v\n", versions)
 
-	cli, err := client.NewEnvClient()
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("building...")
-	res, err := cli.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
-		Tags: []string{"dendrite-upgrade"},
+	branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions)
+
+	// make a shared postgres volume
+	volume, err := dockerClient.VolumeCreate(context.Background(), volume.VolumeCreateBody{
+		Name: "dendrite_upgrade_test",
 	})
 	if err != nil {
-		panic(err)
+		log.Fatalf("failed to make docker volume: %s", err)
 	}
-	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		panic(err)
+
+	// run through images sequentially
+	for _, v := range versions {
+		if err = loadAndRunTests(dockerClient, volume.Name, v, branchToImageID); err != nil {
+			log.Fatalf("failed to run tests for %v: %s", v, err)
+		}
 	}
-	fmt.Println(string(body))
 }
diff --git a/cmd/dendrite-upgrade-tests/tar.go b/cmd/dendrite-upgrade-tests/tar.go
index 2a412945a..c05dfefa0 100644
--- a/cmd/dendrite-upgrade-tests/tar.go
+++ b/cmd/dendrite-upgrade-tests/tar.go
@@ -3,7 +3,6 @@ package main
 import (
 	"archive/tar"
 	"compress/gzip"
-	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -28,7 +27,6 @@ func compress(src string, buf io.Writer) error {
 		// must provide real name
 		// (see https://golang.org/src/archive/tar/common.go?#L626)
 		header.Name = strings.TrimPrefix(filepath.ToSlash(file), src+"/")
-		fmt.Println(header.Name)
 		// write header
 		if err := tw.WriteHeader(header); err != nil {
 			return err
diff --git a/cmd/dendrite-upgrade-tests/tests.go b/cmd/dendrite-upgrade-tests/tests.go
new file mode 100644
index 000000000..f9879df27
--- /dev/null
+++ b/cmd/dendrite-upgrade-tests/tests.go
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+	"time"
+)
+
+func runTests(baseURL, branchName string) error {
+	httpClient := &http.Client{
+		Timeout: 60 * time.Second,
+	}
+	err := assertSuccess(httpClient.Post(
+		fmt.Sprintf("%s/_matrix/client/r0/register", baseURL),
+		"application/json",
+		bytes.NewBufferString(fmt.Sprintf(`{"username":"%s","password":"%s","auth":{"type":"m.login.dummy"}}`, "alice", "this_is_a_long_password")),
+	))
+	if err != nil {
+		return fmt.Errorf("failed to /register: %s", err)
+	}
+	return nil
+}
+
+func assertSuccess(res *http.Response, err error) error {
+	if err != nil {
+		return fmt.Errorf("response returned error: %s", err)
+	}
+	if res.StatusCode != 200 {
+		return fmt.Errorf("response returned HTTP %d - %s", res.StatusCode, dumpResponse(res))
+	}
+	return nil
+}
+
+func dumpResponse(res *http.Response) string {
+	d, _ := httputil.DumpResponse(res, true)
+	return string(d)
+}
diff --git a/go.mod b/go.mod
index 24f8c3857..8b6f51730 100644
--- a/go.mod
+++ b/go.mod
@@ -3,11 +3,13 @@ module github.com/matrix-org/dendrite
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
+	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/Shopify/sarama v1.28.0
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/containerd/containerd v1.5.2 // indirect
+	github.com/coreos/go-semver v0.3.0
 	github.com/docker/docker v20.10.7+incompatible
-	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.10.0
 	github.com/gologme/log v1.2.0
 	github.com/gorilla/mux v1.8.0
diff --git a/go.sum b/go.sum
index a0b50f751..20ad7fa1f 100644
--- a/go.sum
+++ b/go.sum
@@ -58,6 +58,9 @@ github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnl
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
+github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=