Add WIP binary for testing dendrite version upgrades

cmd/dendrite-upgrade-tests/main.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/codeclysm/extract"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+)
+
+var (
+	flagTempDir = flag.String("tmp", "tmp", "Path to temporary directory to dump tarballs to")
+)
+
+func ptr(in string) *string {
+	return &in
+}
+
+// downloadArchive downloads an arbitrary github archive of the form:
+//   https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz
+// and re-tarballs it without the top-level directory which contains branch information. It inserts
+// the contents of `dockerfile` as a root file `Dockerfile` in the re-tarballed directory such that
+// you can directly feed the retarballed archive to `ImageBuild` to have it run said dockerfile.
+// Returns the tarball buffer on success.
+func downloadArchive(cli *http.Client, archiveURL string, dockerfile []byte) (*bytes.Buffer, error) {
+	tmpDir := *flagTempDir
+	resp, err := cli.Get(archiveURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("got HTTP %d", resp.StatusCode)
+	}
+	os.Mkdir(tmpDir, os.ModePerm)
+	defer os.RemoveAll(tmpDir)
+	// dump the tarball temporarily, stripping the top-level directory
+	err = extract.Archive(context.Background(), resp.Body, tmpDir, func(inPath string) string {
+		// remove top level
+		segments := strings.Split(inPath, "/")
+		return strings.Join(segments[1:], "/")
+	})
+	if err != nil {
+		return nil, err
+	}
+	// add top level Dockerfile
+	err = ioutil.WriteFile(path.Join(tmpDir, "Dockerfile"), dockerfile, os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to inject /Dockerfile: %w", err)
+	}
+	// now re-tarball it :/
+	var tarball bytes.Buffer
+	err = compress(*flagTempDir, &tarball)
+	if err != nil {
+		return nil, err
+	}
+	return &tarball, nil
+}
+
+func main() {
+	flag.Parse()
+	httpClient := &http.Client{
+		Timeout: 60 * time.Second,
+	}
+	// pull an archive, this contains a top-level directory which screws with the build context
+	// which we need to fix up post download
+	u := "https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz"
+	tarball, err := downloadArchive(httpClient, u, []byte("FROM golang:1.15\nRUN echo 'hello world'"))
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("%s => %d bytes\n", u, tarball.Len())
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println("building...")
+	res, err := cli.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
+		Tags: []string{"dendrite-upgrade"},
+	})
+	if err != nil {
+		panic(err)
+	}
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(string(body))
+}

cmd/dendrite-upgrade-tests/tar.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// From https://gist.github.com/mimoo/25fc9716e0f1353791f5908f94d6e726
+// Modified to strip off top-level when compressing
+func compress(src string, buf io.Writer) error {
+	// tar > gzip > buf
+	zr := gzip.NewWriter(buf)
+	tw := tar.NewWriter(zr)
+
+	// walk through every file in the folder
+	filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {
+		// generate tar header
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			return err
+		}
+
+		// must provide real name
+		// (see https://golang.org/src/archive/tar/common.go?#L626)
+		header.Name = strings.TrimPrefix(filepath.ToSlash(file), src+"/")
+		fmt.Println(header.Name)
+		// write header
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+		// if not a dir, write file content
+		if !fi.IsDir() {
+			data, err := os.Open(file)
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(tw, data); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	// produce tar
+	if err := tw.Close(); err != nil {
+		return err
+	}
+	// produce gzip
+	if err := zr.Close(); err != nil {
+		return err
+	}
+	//
+	return nil
+}

go.mod

@@ -4,11 +4,17 @@ require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
 	github.com/Shopify/sarama v1.28.0
+	github.com/codeclysm/extract v2.2.0+incompatible
+	github.com/containerd/containerd v1.5.2 // indirect
+	github.com/docker/docker v20.10.7+incompatible
+	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/getsentry/sentry-go v0.10.0
 	github.com/gologme/log v1.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
+	github.com/h2non/filetype v1.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4
+	github.com/juju/testing v0.0.0-20210324180055-18c50b0c2098 // indirect
 	github.com/lib/pq v1.9.0
 	github.com/libp2p/go-libp2p v0.13.0
 	github.com/libp2p/go-libp2p-circuit v0.4.0
@@ -28,6 +34,7 @@ require (
 	github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6
 	github.com/opentracing/opentracing-go v1.2.0
@@ -41,7 +48,7 @@ require (
 	github.com/uber/jaeger-lib v2.4.0+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20210218094457-e77ca8019daa
 	go.uber.org/atomic v1.7.0
-	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1