From 2626525c65e5e7f3cf446d7ab23c26c6c345320a Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 15 Jan 2021 11:44:36 +0000 Subject: [PATCH 01/11] MSC2946: Allow redactions/updates for space state events (#1712) --- setup/mscs/msc2946/msc2946.go | 7 ++++++- setup/mscs/msc2946/msc2946_test.go | 19 +++++++++++++++++++ setup/mscs/msc2946/storage.go | 4 ++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index a2d801e4d..accdbd395 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -333,7 +333,12 @@ func (w *walker) references(roomID string) (eventLookup, error) { } el := make(eventLookup) for _, ev := range events { - el.set(ev) + // only return events that have a `via` key as per MSC1772 + // else we'll incorrectly walk redacted events (as the link + // is in the state_key) + if gjson.GetBytes(ev.Content(), "via").Exists() { + el.set(ev) + } } return el, nil } diff --git a/setup/mscs/msc2946/msc2946_test.go b/setup/mscs/msc2946/msc2946_test.go index 017319dc5..a7650f0ba 100644 --- a/setup/mscs/msc2946/msc2946_test.go +++ b/setup/mscs/msc2946/msc2946_test.go @@ -254,7 +254,26 @@ func TestMSC2946(t *testing.T) { if len(res.Rooms) != len(allRooms) { t.Errorf("got %d rooms, want %d", len(res.Rooms), len(allRooms)) } + }) + t.Run("can update the graph", func(t *testing.T) { + // remove R3 from the graph + rmS1ToR3 := mustCreateEvent(t, fledglingEvent{ + RoomID: subSpaceS1, + Sender: alice, + Type: msc2946.ConstSpaceChildEventType, + StateKey: &room3, + Content: map[string]interface{}{}, // redacted + }) + nopRsAPI.events[rmS1ToR3.EventID()] = rmS1ToR3 + hooks.Run(hooks.KindNewEventPersisted, rmS1ToR3) + res := postSpaces(t, 200, "alice", rootSpace, newReq(t, map[string]interface{}{})) + if len(res.Events) != 6 { // one less since we don't return redacted events + t.Errorf("got %d events, want 6", len(res.Events)) + } + if len(res.Rooms) != (len(allRooms) - 1) { // one less due to lack of R3 + t.Errorf("got %d rooms, want %d", len(res.Rooms), len(allRooms)-1) + } }) } diff --git a/setup/mscs/msc2946/storage.go b/setup/mscs/msc2946/storage.go index eb4a5efb9..5798310a6 100644 --- a/setup/mscs/msc2946/storage.go +++ b/setup/mscs/msc2946/storage.go @@ -81,7 +81,7 @@ func newPostgresDatabase(dbOpts *config.DatabaseOptions) (Database, error) { if d.insertEdgeStmt, err = d.db.Prepare(` INSERT INTO msc2946_edges(room_version, source_room_id, dest_room_id, rel_type, event_json) VALUES($1, $2, $3, $4, $5) - ON CONFLICT DO NOTHING + ON CONFLICT ON CONSTRAINT msc2946_edges_uniq DO UPDATE SET event_json = $5 `); err != nil { return nil, err } @@ -121,7 +121,7 @@ func newSQLiteDatabase(dbOpts *config.DatabaseOptions) (Database, error) { if d.insertEdgeStmt, err = d.db.Prepare(` INSERT INTO msc2946_edges(room_version, source_room_id, dest_room_id, rel_type, event_json) VALUES($1, $2, $3, $4, $5) - ON CONFLICT DO NOTHING + ON CONFLICT (source_room_id, dest_room_id, rel_type) DO UPDATE SET event_json = $5 `); err != nil { return nil, err } From 5366c338851dac8a3d64381ef8ef301c85a3c66d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 15 Jan 2021 12:30:41 +0000 Subject: [PATCH 02/11] MSC2946: Use new MSC1772 format for space parents (#1713) --- setup/mscs/msc2946/msc2946.go | 10 +++++----- setup/mscs/msc2946/msc2946_test.go | 24 ++++++++---------------- setup/mscs/msc2946/storage.go | 3 +-- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index accdbd395..3405ebe7b 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -36,7 +36,7 @@ import ( const ( ConstCreateEventContentKey = "org.matrix.msc1772.type" ConstSpaceChildEventType = "org.matrix.msc1772.space.child" - ConstSpaceParentEventType = "org.matrix.msc1772.room.parent" + ConstSpaceParentEventType = "org.matrix.msc1772.space.parent" ) // SpacesRequest is the request body to POST /_matrix/client/r0/rooms/{roomID}/spaces @@ -57,7 +57,7 @@ type SpacesResponse struct { NextBatch string `json:"next_batch"` // Rooms are nodes on the space graph. Rooms []Room `json:"rooms"` - // Events are edges on the space graph, exclusively m.space.child or m.room.parent events + // Events are edges on the space graph, exclusively m.space.child or m.space.parent events Events []gomatrixserverlib.ClientEvent `json:"events"` } @@ -182,8 +182,8 @@ func (w *walker) walk() *SpacesResponse { if !w.authorised(roomID) { continue } - // Get all `m.space.child` and `m.room.parent` state events for the room. *In addition*, get - // all `m.space.child` and `m.room.parent` state events which *point to* (via `state_key` or `content.room_id`) + // Get all `m.space.child` and `m.space.parent` state events for the room. *In addition*, get + // all `m.space.child` and `m.space.parent` state events which *point to* (via `state_key` or `content.room_id`) // this room. This requires servers to store reverse lookups. refs, err := w.references(roomID) if err != nil { @@ -196,7 +196,7 @@ func (w *walker) walk() *SpacesResponse { if !w.alreadySent(roomID) { pubRoom := w.publicRoomsChunk(roomID) roomType := "" - create := w.stateEvent(roomID, "m.room.create", "") + create := w.stateEvent(roomID, gomatrixserverlib.MRoomCreate, "") if create != nil { roomType = gjson.GetBytes(create.Content(), ConstCreateEventContentKey).Str } diff --git a/setup/mscs/msc2946/msc2946_test.go b/setup/mscs/msc2946/msc2946_test.go index a7650f0ba..d2d935e86 100644 --- a/setup/mscs/msc2946/msc2946_test.go +++ b/setup/mscs/msc2946/msc2946_test.go @@ -86,8 +86,7 @@ func TestMSC2946(t *testing.T) { Type: msc2946.ConstSpaceChildEventType, StateKey: &room1, Content: map[string]interface{}{ - "via": []string{"localhost"}, - "present": true, + "via": []string{"localhost"}, }, }) rootToR2 := mustCreateEvent(t, fledglingEvent{ @@ -96,8 +95,7 @@ func TestMSC2946(t *testing.T) { Type: msc2946.ConstSpaceChildEventType, StateKey: &room2, Content: map[string]interface{}{ - "via": []string{"localhost"}, - "present": true, + "via": []string{"localhost"}, }, }) rootToS1 := mustCreateEvent(t, fledglingEvent{ @@ -106,8 +104,7 @@ func TestMSC2946(t *testing.T) { Type: msc2946.ConstSpaceChildEventType, StateKey: &subSpaceS1, Content: map[string]interface{}{ - "via": []string{"localhost"}, - "present": true, + "via": []string{"localhost"}, }, }) s1ToR3 := mustCreateEvent(t, fledglingEvent{ @@ -116,8 +113,7 @@ func TestMSC2946(t *testing.T) { Type: msc2946.ConstSpaceChildEventType, StateKey: &room3, Content: map[string]interface{}{ - "via": []string{"localhost"}, - "present": true, + "via": []string{"localhost"}, }, }) s1ToR4 := mustCreateEvent(t, fledglingEvent{ @@ -126,8 +122,7 @@ func TestMSC2946(t *testing.T) { Type: msc2946.ConstSpaceChildEventType, StateKey: &room4, Content: map[string]interface{}{ - "via": []string{"localhost"}, - "present": true, + "via": []string{"localhost"}, }, }) s1ToS2 := mustCreateEvent(t, fledglingEvent{ @@ -136,8 +131,7 @@ func TestMSC2946(t *testing.T) { Type: msc2946.ConstSpaceChildEventType, StateKey: &subSpaceS2, Content: map[string]interface{}{ - "via": []string{"localhost"}, - "present": true, + "via": []string{"localhost"}, }, }) // This is a parent link only @@ -145,11 +139,9 @@ func TestMSC2946(t *testing.T) { RoomID: room5, Sender: alice, Type: msc2946.ConstSpaceParentEventType, - StateKey: &empty, + StateKey: &subSpaceS2, Content: map[string]interface{}{ - "room_id": subSpaceS2, - "via": []string{"localhost"}, - "present": true, + "via": []string{"localhost"}, }, }) // history visibility for R4 diff --git a/setup/mscs/msc2946/storage.go b/setup/mscs/msc2946/storage.go index 5798310a6..20db18594 100644 --- a/setup/mscs/msc2946/storage.go +++ b/setup/mscs/msc2946/storage.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" - "github.com/tidwall/gjson" ) var ( @@ -175,7 +174,7 @@ func SpaceTarget(he *gomatrixserverlib.HeaderedEvent) string { } switch he.Type() { case ConstSpaceParentEventType: - return gjson.GetBytes(he.Content(), "room_id").Str + return *he.StateKey() case ConstSpaceChildEventType: return *he.StateKey() } From 2113da53326cec2f3f9c1d7a641e6768ecddbfbf Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 15 Jan 2021 17:56:50 +0000 Subject: [PATCH 03/11] Enable MSC 2946 in CI --- cmd/generate-config/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index c7ad1b37c..a07b72ebd 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -63,7 +63,7 @@ func main() { if *defaultsForCI { cfg.ClientAPI.RateLimiting.Enabled = false cfg.FederationSender.DisableTLSValidation = true - cfg.MSCs.MSCs = []string{"msc2836"} + cfg.MSCs.MSCs = []string{"msc2836","msc2946"} cfg.Logging[0].Level = "trace" // don't hit matrix.org when running tests!!! cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{} From fdd534f86a493c26f6935cf31e721b1a2605de8c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 15 Jan 2021 18:49:21 +0000 Subject: [PATCH 04/11] Escape the create event content key so it can be extracted correctly --- setup/mscs/msc2946/msc2946.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index 3405ebe7b..2b5477376 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "net/http" + "strings" "sync" "github.com/gorilla/mux" @@ -198,7 +199,8 @@ func (w *walker) walk() *SpacesResponse { roomType := "" create := w.stateEvent(roomID, gomatrixserverlib.MRoomCreate, "") if create != nil { - roomType = gjson.GetBytes(create.Content(), ConstCreateEventContentKey).Str + // escape the `.`s so gjson doesn't think it's nested + roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str } // Add the total number of events to `PublicRoomsChunk` under `num_refs`. Add `PublicRoomsChunk` to `rooms`. From 6dadb1c06b6bfc23d3741d5611492b0d3dadddbc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 09:34:49 +0000 Subject: [PATCH 05/11] Fix lint error in generate-keys --- cmd/generate-config/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index a07b72ebd..ff0b311aa 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -63,7 +63,7 @@ func main() { if *defaultsForCI { cfg.ClientAPI.RateLimiting.Enabled = false cfg.FederationSender.DisableTLSValidation = true - cfg.MSCs.MSCs = []string{"msc2836","msc2946"} + cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} cfg.Logging[0].Level = "trace" // don't hit matrix.org when running tests!!! cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{} From ff94490384facc0a891a853ec10d7247587fcce9 Mon Sep 17 00:00:00 2001 From: TR_SLimey <37966924+TR-SLimey@users.noreply.github.com> Date: Mon, 18 Jan 2021 10:23:53 +0000 Subject: [PATCH 06/11] GitHub action for automatic multiarch Docker build (#1613) * Create docker-build-and-push.yml * Switched to using official Docker buildx action * Added comment to docker-build-and-push.yml In case something needs to be tweaked in the future, the link contains some examples and explanations which would be useful * Run only on release (and produce release tags) As this workflow takes quite a lot of time, and [pushing to master happens frequently](https://github.com/matrix-org/dendrite/pull/1613#issuecomment-746086980), the container will now only be built when a release is created, and the builds will also be correctly tagged. * Add latest tag, test at neilalexander/dendrite* Co-authored-by: Neil Alexander --- .github/workflows/docker-build-and-push.yml | 73 +++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/docker-build-and-push.yml diff --git a/.github/workflows/docker-build-and-push.yml b/.github/workflows/docker-build-and-push.yml new file mode 100644 index 000000000..2d67b210e --- /dev/null +++ b/.github/workflows/docker-build-and-push.yml @@ -0,0 +1,73 @@ +# Based on https://github.com/docker/build-push-action + +name: "Docker Multiarch Build & Push" + +on: + release: +# types: [published] + branches: [master] + +env: + DOCKER_HUB_USER: neilalexander + +jobs: + BuildAndPush: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Get release tag + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + - + name: Build temporary (builder) image + id: docker_build_temporary + uses: docker/build-push-action@v2 + with: + context: . + file: ./build/docker/Dockerfile + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: false + tags: ${{ env.DOCKER_HUB_USER }}/dendrite:latest + - + name: Build monolith image + id: docker_build_monolith + uses: docker/build-push-action@v2 + with: + context: . + file: ./build/docker/Dockerfile.monolith + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: true + tags: | + ${{ env.DOCKER_HUB_USER }}/dendrite-monolith:latest + ${{ env.DOCKER_HUB_USER }}/dendrite-monolith:${{ env.RELEASE_VERSION }} + - + name: Build polylith image + id: docker_build_polylith + uses: docker/build-push-action@v2 + with: + context: . + file: ./build/docker/Dockerfile.polylith + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: true + tags: | + ${{ env.DOCKER_HUB_USER }}/dendrite-polylith:latest + ${{ env.DOCKER_HUB_USER }}/dendrite-polylith:${{ env.RELEASE_VERSION }} + - + name: Image digest + run: | + echo Monolith ( ${{ env.RELEASE_VERSION }} ) image digest - ${{ steps.docker_build_monolith.outputs.digest }} + echo Polylith ( ${{ env.RELEASE_VERSION }} ) image digest - ${{ steps.docker_build_polylith.outputs.digest }} From 833e01a6122776b61da4a9ab67d9a53c1dbe93e4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 10:26:36 +0000 Subject: [PATCH 07/11] Update docker-build-and-push.yml --- .github/workflows/docker-build-and-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-and-push.yml b/.github/workflows/docker-build-and-push.yml index 2d67b210e..f9a7fa694 100644 --- a/.github/workflows/docker-build-and-push.yml +++ b/.github/workflows/docker-build-and-push.yml @@ -3,7 +3,7 @@ name: "Docker Multiarch Build & Push" on: - release: + push: # types: [published] branches: [master] From df4386f764268b410657ab8c6b2bb4fc40819823 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 11:03:50 +0000 Subject: [PATCH 08/11] Build docker branch only --- .github/workflows/docker-build-and-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-and-push.yml b/.github/workflows/docker-build-and-push.yml index f9a7fa694..85aa0ab53 100644 --- a/.github/workflows/docker-build-and-push.yml +++ b/.github/workflows/docker-build-and-push.yml @@ -5,7 +5,7 @@ name: "Docker Multiarch Build & Push" on: push: # types: [published] - branches: [master] + branches: [docker] env: DOCKER_HUB_USER: neilalexander From cf82e08096f6b0035baf377c6dc4243c20ecadef Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 12:24:23 +0000 Subject: [PATCH 09/11] Update GHA Docker Hub builds Squashed commit of the following: commit 4a61aa711473deece2adf415cfd65501dbca63b2 Author: Neil Alexander Date: Mon Jan 18 12:19:24 2021 +0000 Set back to matrixdotorg on published releases commit 6d1ac53f2c0c9b30e1e70c0bb1559e1b8ec874a2 Author: Neil Alexander Date: Mon Jan 18 11:55:28 2021 +0000 Rename commit 258999f7fb7b655b3a02a06a7ea05e66fb7740fb Author: Neil Alexander Date: Mon Jan 18 11:52:26 2021 +0000 Refactor multi-stage builds commit c7ab8e476939899571e7b5668860dec372b9b60f Author: Neil Alexander Date: Mon Jan 18 11:13:19 2021 +0000 Let's try this again --- ...cker-build-and-push.yml => docker-hub.yml} | 71 +++++++++---------- build/docker/Dockerfile | 10 --- build/docker/Dockerfile.monolith | 19 +++-- build/docker/Dockerfile.polylith | 19 +++-- build/docker/images-build.sh | 2 - 5 files changed, 62 insertions(+), 59 deletions(-) rename .github/workflows/{docker-build-and-push.yml => docker-hub.yml} (50%) delete mode 100644 build/docker/Dockerfile diff --git a/.github/workflows/docker-build-and-push.yml b/.github/workflows/docker-hub.yml similarity index 50% rename from .github/workflows/docker-build-and-push.yml rename to .github/workflows/docker-hub.yml index 85aa0ab53..84745f7b2 100644 --- a/.github/workflows/docker-build-and-push.yml +++ b/.github/workflows/docker-hub.yml @@ -1,73 +1,70 @@ # Based on https://github.com/docker/build-push-action -name: "Docker Multiarch Build & Push" +name: "Docker Hub" on: - push: -# types: [published] - branches: [docker] + release: + types: [published] env: - DOCKER_HUB_USER: neilalexander + DOCKER_HUB_USER: matrixdotorg + PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 jobs: - BuildAndPush: + Monolith: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v2 - - - name: Get release tag + - name: Get release tag run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - - name: Set up QEMU + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub + - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build temporary (builder) image - id: docker_build_temporary - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/docker/Dockerfile - platforms: linux/amd64,linux/arm64,linux/arm/v7 - push: false - tags: ${{ env.DOCKER_HUB_USER }}/dendrite:latest - - - name: Build monolith image + + - name: Build monolith image id: docker_build_monolith uses: docker/build-push-action@v2 with: context: . file: ./build/docker/Dockerfile.monolith - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_HUB_USER }}/dendrite-monolith:latest ${{ env.DOCKER_HUB_USER }}/dendrite-monolith:${{ env.RELEASE_VERSION }} - - - name: Build polylith image + + Polylith: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get release tag + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build polylith image id: docker_build_polylith uses: docker/build-push-action@v2 with: context: . file: ./build/docker/Dockerfile.polylith - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_HUB_USER }}/dendrite-polylith:latest ${{ env.DOCKER_HUB_USER }}/dendrite-polylith:${{ env.RELEASE_VERSION }} - - - name: Image digest - run: | - echo Monolith ( ${{ env.RELEASE_VERSION }} ) image digest - ${{ steps.docker_build_monolith.outputs.digest }} - echo Polylith ( ${{ env.RELEASE_VERSION }} ) image digest - ${{ steps.docker_build_polylith.outputs.digest }} diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile deleted file mode 100644 index 5cab0530f..000000000 --- a/build/docker/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM docker.io/golang:1.15-alpine AS builder - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN sh ./build.sh \ No newline at end of file diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith index 3e9d0cba4..eb099c4cc 100644 --- a/build/docker/Dockerfile.monolith +++ b/build/docker/Dockerfile.monolith @@ -1,11 +1,20 @@ -FROM matrixdotorg/dendrite:latest AS base +FROM docker.io/golang:1.15-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server +RUN go build -trimpath -o bin/ ./cmd/goose +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys FROM alpine:latest -COPY --from=base /build/bin/dendrite-monolith-server /usr/bin -COPY --from=base /build/bin/goose /usr/bin -COPY --from=base /build/bin/create-account /usr/bin -COPY --from=base /build/bin/generate-keys /usr/bin +COPY --from=base /build/bin/* /usr/bin VOLUME /etc/dendrite WORKDIR /etc/dendrite diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith index dd4cbd38f..1a7ba193e 100644 --- a/build/docker/Dockerfile.polylith +++ b/build/docker/Dockerfile.polylith @@ -1,11 +1,20 @@ -FROM matrixdotorg/dendrite:latest AS base +FROM docker.io/golang:1.15-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi +RUN go build -trimpath -o bin/ ./cmd/goose +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys FROM alpine:latest -COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin -COPY --from=base /build/bin/goose /usr/bin -COPY --from=base /build/bin/create-account /usr/bin -COPY --from=base /build/bin/generate-keys /usr/bin +COPY --from=base /build/bin/* /usr/bin VOLUME /etc/dendrite WORKDIR /etc/dendrite diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index f80f6bed2..eaed5f6dc 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,7 +6,5 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" -docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} . - docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file From 940577cd3c9df97d1616476dbd8498863a9a9632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFck=20Bonniot?= Date: Mon, 18 Jan 2021 13:43:15 +0100 Subject: [PATCH 10/11] Fix integer overflow in device_list_update.go (#1717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #1511 On 32-bits systems, int(hash.Sum32()) can be negative. This makes the computation of array indices using modulo invalid, crashing dendrite. Signed-off-by: Loïck Bonniot --- keyserver/internal/device_list_update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index ff57be00e..c4950a119 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -245,7 +245,7 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) { } hash := fnv.New32a() _, _ = hash.Write([]byte(remoteServer)) - index := int(hash.Sum32()) % len(u.workerChans) + index := int(int64(hash.Sum32()) % int64(len(u.workerChans))) ch := u.assignChannel(userID) u.workerChans[index] <- remoteServer From efc91146f0476239ac737c37ae8822346c415014 Mon Sep 17 00:00:00 2001 From: Caleb Xavier Berger Date: Mon, 18 Jan 2021 07:44:44 -0500 Subject: [PATCH 11/11] Use go build instead of go install (#1716) * Use go build instead of go install go install doesn't like to cross-compile things. (Try running build.sh with GOARCH set to something other than what it "should" be.) With go build, it appears that cross-compilation is really, really straightforward. Simply install a compiler for your target platform and set `GOARCH` and `CC` accordingly. * Use shell expansion instead of loop Co-authored-by: Neil Alexander --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index e5e7fe8f2..09ecb61ca 100755 --- a/build.sh +++ b/build.sh @@ -17,6 +17,6 @@ else export FLAGS="" fi -go install -trimpath -ldflags "$FLAGS" -v $PWD/`dirname $0`/cmd/... +CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/... -GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs +CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs