Merge branch 'master' of https://github.com/matrix-org/dendrite into add-presence

cmd/dendrite-upgrade-tests/main.go

@@ -35,9 +35,13 @@ var (
 	flagFrom             = flag.String("from", "HEAD-1", "The version to start from e.g '0.3.1'. If 'HEAD-N' then starts N versions behind HEAD.")
 	flagTo               = flag.String("to", "HEAD", "The version to end on e.g '0.3.3'.")
 	flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images")
+	flagHead             = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github")
+	flagDockerHost       = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
 	alphaNumerics        = regexp.MustCompile("[^a-zA-Z0-9]+")
 )
 
+const HEAD = "HEAD"
+
 // Embed the Dockerfile to use when building dendrite versions.
 // We cannot use the dockerfile associated with the repo with each version sadly due to changes in
 // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
@@ -104,6 +108,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
 	if resp.StatusCode != 200 {
 		return nil, fmt.Errorf("got HTTP %d", resp.StatusCode)
 	}
+	_ = os.RemoveAll(tmpDir)
 	if err = os.Mkdir(tmpDir, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("failed to make temporary directory: %s", err)
 	}
@@ -134,15 +139,36 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
 
 // buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
 func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) {
-	log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir)
-	// pull an archive, this contains a top-level directory which screws with the build context
-	// which we need to fix up post download
-	u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
-	tarball, err := downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
-	if err != nil {
-		return "", fmt.Errorf("failed to download archive %s: %w", u, err)
+	var tarball *bytes.Buffer
+	var err error
+	// If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
+	// where we want to use the working directory.
+	if branchOrTagName == HEAD && *flagHead != "" {
+		log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead)
+		// add top level Dockerfile
+		err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
+		if err != nil {
+			return "", fmt.Errorf("Custom HEAD: failed to inject /Dockerfile: %w", err)
+		}
+		// now tarball it
+		var buffer bytes.Buffer
+		err = compress(*flagHead, &buffer)
+		if err != nil {
+			return "", fmt.Errorf("failed to tarball custom HEAD %s : %s", *flagHead, err)
+		}
+		tarball = &buffer
+	} else {
+		log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir)
+		// pull an archive, this contains a top-level directory which screws with the build context
+		// which we need to fix up post download
+		u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
+		tarball, err = downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
+		if err != nil {
+			return "", fmt.Errorf("failed to download archive %s: %w", u, err)
+		}
+		log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len())
 	}
-	log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len())
+
 	log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName)
 	res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
 		Tags: []string{"dendrite-upgrade"},
@@ -234,7 +260,7 @@ func calculateVersions(cli *http.Client, from, to string) []string {
 			semvers = semvers[i:]
 		}
 	}
-	if to != "" && to != "HEAD" {
+	if to != "" && to != HEAD {
 		toVer, err := semver.NewVersion(to)
 		if err != nil {
 			log.Fatalf("invalid --to: %s", err)
@@ -252,8 +278,8 @@ func calculateVersions(cli *http.Client, from, to string) []string {
 	for _, sv := range semvers {
 		versions = append(versions, sv.Original())
 	}
-	if to == "HEAD" {
-		versions = append(versions, "HEAD")
+	if to == HEAD {
+		versions = append(versions, HEAD)
 	}
 	return versions
 }
@@ -327,7 +353,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
 	if !ok {
 		return "", "", fmt.Errorf("port 8008 not exposed - exposed ports: %v", inspect.NetworkSettings.Ports)
 	}
-	baseURL := fmt.Sprintf("http://localhost:%s", csapiPortInfo[0].HostPort)
+	baseURL := fmt.Sprintf("http://%s:%s", *flagDockerHost, csapiPortInfo[0].HostPort)
 	versionsURL := fmt.Sprintf("%s/_matrix/client/versions", baseURL)
 	// hit /versions to check it is up
 	var lastErr error

cmd/dendrite-upgrade-tests/tar.go

@@ -17,7 +17,7 @@ func compress(src string, buf io.Writer) error {
 	tw := tar.NewWriter(zr)
 
 	// walk through every file in the folder
-	_ = filepath.Walk(src, func(file string, fi os.FileInfo, e error) error {
+	err := filepath.Walk(src, func(file string, fi os.FileInfo, e error) error {
 		// generate tar header
 		header, err := tar.FileInfoHeader(fi, file)
 		if err != nil {
@@ -37,12 +37,18 @@ func compress(src string, buf io.Writer) error {
 			if err != nil {
 				return err
 			}
-			if _, err := io.Copy(tw, data); err != nil {
+			if _, err = io.Copy(tw, data); err != nil {
+				return err
+			}
+			if err = data.Close(); err != nil {
 				return err
 			}
 		}
 		return nil
 	})
+	if err != nil {
+		return err
+	}
 
 	// produce tar
 	if err := tw.Close(); err != nil {

internal/log.go

@@ -73,9 +73,9 @@ func callerPrettyfier(f *runtime.Frame) (string, string) {
 	// Append a newline + tab to it to move the actual log content to its own line
 	funcname += "\n\t"
 
-	// Surround the filepath in brackets and append line number so IDEs can quickly
-	// navigate
-	filename := fmt.Sprintf(" [%s:%d]", f.File, f.Line)
+	// Use a shortened file path which just has the filename to avoid having lots of redundant
+	// directories which contribute significantly to overall log sizes!
+	filename := fmt.Sprintf(" [%s:%d]", path.Base(f.File), f.Line)
 
 	return funcname, filename
 }

roomserver/internal/perform/perform_leave.go

@@ -64,7 +64,14 @@ func (r *Leaver) performLeaveRoomByID(
 	// that.
 	isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID)
 	if err == nil && isInvitePending {
-		return r.performRejectInvite(ctx, req, res, senderUser, eventID)
+		var host gomatrixserverlib.ServerName
+		_, host, err = gomatrixserverlib.SplitID('@', senderUser)
+		if err != nil {
+			return nil, fmt.Errorf("Sender %q is invalid", senderUser)
+		}
+		if host != r.Cfg.Matrix.ServerName {
+			return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID)
+		}
 	}
 
 	// There's no invite pending, so first of all we want to find out
@@ -94,9 +101,7 @@ func (r *Leaver) performLeaveRoomByID(
 	if err != nil {
 		return nil, fmt.Errorf("Error getting membership: %w", err)
 	}
-	if membership != gomatrixserverlib.Join {
-		// TODO: should be able to handle "invite" in this case too, if
-		// it's a case of kicking or banning or such
+	if membership != gomatrixserverlib.Join && membership != gomatrixserverlib.Invite {
 		return nil, fmt.Errorf("User %q is not joined to the room (membership is %q)", req.UserID, membership)
 	}
 
@@ -147,7 +152,7 @@ func (r *Leaver) performLeaveRoomByID(
 	return nil, nil
 }
 
-func (r *Leaver) performRejectInvite(
+func (r *Leaver) performFederatedRejectInvite(
 	ctx context.Context,
 	req *api.PerformLeaveRequest,
 	res *api.PerformLeaveResponse, // nolint:unparam

roomserver/storage/postgres/state_block_table.go

@@ -64,7 +64,7 @@ const insertStateDataSQL = "" +
 
 const bulkSelectStateBlockEntriesSQL = "" +
 	"SELECT state_block_nid, event_nids" +
-	" FROM roomserver_state_block WHERE state_block_nid = ANY($1)"
+	" FROM roomserver_state_block WHERE state_block_nid = ANY($1) ORDER BY state_block_nid ASC"
 
 type stateBlockStatements struct {
 	insertStateDataStmt             *sql.Stmt

roomserver/storage/sqlite3/state_block_table.go

@@ -57,7 +57,7 @@ const insertStateDataSQL = `
 
 const bulkSelectStateBlockEntriesSQL = "" +
 	"SELECT state_block_nid, event_nids" +
-	" FROM roomserver_state_block WHERE state_block_nid IN ($1)"
+	" FROM roomserver_state_block WHERE state_block_nid IN ($1) ORDER BY state_block_nid ASC"
 
 type stateBlockStatements struct {
 	db                              *sql.DB