From 6948d1652742ae83cf7729f1cfaa0d1ea796b703 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:12:06 +0100 Subject: [PATCH] Update Go, use go tool covdata for coverage files? --- .github/workflows/schedules.yaml | 19 +++++++++---------- build/scripts/Complement.Dockerfile | 4 ++-- build/scripts/ComplementPostgres.Dockerfile | 10 +++++----- build/scripts/complement-cmd.sh | 5 +++-- internal/fulltext/bleve.go | 12 +++++++----- internal/fulltext/bleve_test.go | 2 +- syncapi/syncapi.go | 3 +-- test/testrig/base.go | 3 ++- 8 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index dff9b34c9..b6a15bab0 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -12,6 +12,7 @@ concurrency: jobs: # run Sytest in different variations sytest: + if: ${{ false }} timeout-minutes: 60 name: "Sytest (${{ matrix.label }})" runs-on: ubuntu-latest @@ -75,7 +76,7 @@ jobs: name: "Sytest Coverage" runs-on: ubuntu-latest needs: sytest # only run once Sytest is done - if: ${{ always() }} + if: ${{ false }} steps: - uses: actions/checkout@v3 - name: Install Go @@ -167,7 +168,7 @@ jobs: cat < /tmp/posttest.sh #!/bin/bash mkdir -p /tmp/Complement/logs/\$2/\$1/ - docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/ + docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/ EOF chmod +x /tmp/posttest.sh @@ -190,7 +191,7 @@ jobs: with: name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }}) path: | - /tmp/Complement/**/complementcover.log + /tmp/Complement/logs/** complement-coverage: timeout-minutes: 5 @@ -199,7 +200,6 @@ jobs: needs: complement # only run once Complement is done if: ${{ always() }} steps: - - uses: actions/checkout@v3 - name: Install Go uses: actions/setup-go@v3 with: @@ -207,20 +207,19 @@ jobs: cache: true - name: Download all artifacts uses: actions/download-artifact@v3 - - name: Install gocovmerge - run: go install github.com/wadey/gocovmerge@latest - - name: Run gocovmerge + - name: Collect coverage run: | - find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov - go tool cover -func=complement.cov + go tool covdata textfmt -i=$(find . -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o complement.cov + grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - files: ./complement.cov + files: ./final.cov flags: complement fail_ci_if_error: true element_web: + if: ${{ false }} timeout-minutes: 120 runs-on: ubuntu-latest steps: diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 70bbe8f95..453d89765 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -1,6 +1,6 @@ #syntax=docker/dockerfile:1.2 -FROM golang:1.18-stretch as build +FROM golang:1.20-bullseye as build RUN apt-get update && apt-get install -y sqlite3 WORKDIR /build @@ -17,7 +17,7 @@ RUN --mount=target=. \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \ CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \ - CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ + CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ cp build/scripts/complement-cmd.sh /complement-cmd.sh WORKDIR /dendrite diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index d4b6d3f75..77071b450 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -1,19 +1,19 @@ #syntax=docker/dockerfile:1.2 -FROM golang:1.18-stretch as build +FROM golang:1.20-bullseye as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build # No password when connecting over localhost -RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/9.6/main/pg_hba.conf && \ +RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/13/main/pg_hba.conf && \ # Bump up max conns for moar concurrency - sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf + sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/13/main/postgresql.conf # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ #!/bin/bash -eu \n\ pg_lsclusters \n\ - pg_ctlcluster 9.6 main start \n\ + pg_ctlcluster 13 main start \n\ \n\ until pg_isready \n\ do \n\ @@ -35,7 +35,7 @@ RUN --mount=target=. \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \ CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \ - CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ + CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ cp build/scripts/complement-cmd.sh /complement-cmd.sh WORKDIR /dendrite diff --git a/build/scripts/complement-cmd.sh b/build/scripts/complement-cmd.sh index 52b063d01..3f6ed070c 100755 --- a/build/scripts/complement-cmd.sh +++ b/build/scripts/complement-cmd.sh @@ -2,14 +2,15 @@ # This script is intended to be used inside a docker container for Complement +export GOCOVERDIR=/tmp/covdatafiles +mkdir -p "${GOCOVERDIR}" if [[ "${COVER}" -eq 1 ]]; then echo "Running with coverage" exec /dendrite/dendrite-cover \ --really-enable-open-registration \ --tls-cert server.crt \ --tls-key server.key \ - --config dendrite.yaml \ - --test.coverprofile=complementcover.log + --config dendrite.yaml else echo "Not running with coverage" exec /dendrite/dendrite \ diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go index 08cef8a97..dea7c504c 100644 --- a/internal/fulltext/bleve.go +++ b/internal/fulltext/bleve.go @@ -18,10 +18,11 @@ package fulltext import ( - "context" "strings" "github.com/blevesearch/bleve/v2" + "github.com/matrix-org/dendrite/setup/process" + // side effect imports to allow all possible languages _ "github.com/blevesearch/bleve/v2/analysis/lang/ar" _ "github.com/blevesearch/bleve/v2/analysis/lang/cjk" @@ -84,17 +85,18 @@ func (i *IndexElement) SetContentType(v string) { } // New opens a new/existing fulltext index -func New(ctx context.Context, cfg config.Fulltext) (fts *Search, err error) { +func New(processCtx *process.ProcessContext, cfg config.Fulltext) (fts *Search, err error) { fts = &Search{} fts.FulltextIndex, err = openIndex(cfg) if err != nil { return nil, err } go func() { - // Wait for the context (should be from process.ProcessContext) to be - // done, indicating that Dendrite is shutting down. - <-ctx.Done() + processCtx.ComponentStarted() + // Wait for the processContext to be done, indicating that Dendrite is shutting down. + <-processCtx.WaitForShutdown() _ = fts.Close() + processCtx.ComponentFinished() }() return fts, nil } diff --git a/internal/fulltext/bleve_test.go b/internal/fulltext/bleve_test.go index 6ce691454..bd8289d58 100644 --- a/internal/fulltext/bleve_test.go +++ b/internal/fulltext/bleve_test.go @@ -38,7 +38,7 @@ func mustOpenIndex(t *testing.T, tempDir string) (*fulltext.Search, *process.Pro cfg.InMemory = false } ctx := process.NewProcessContext() - fts, err := fulltext.New(ctx.Context(), cfg) + fts, err := fulltext.New(ctx, cfg) if err != nil { t.Fatal("failed to open fulltext index:", err) } diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 9a27b954a..ecbe05dd8 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -69,11 +69,10 @@ func AddPublicRoutes( var fts *fulltext.Search if dendriteCfg.SyncAPI.Fulltext.Enabled { - fts, err = fulltext.New(processContext.Context(), dendriteCfg.SyncAPI.Fulltext) + fts, err = fulltext.New(processContext, dendriteCfg.SyncAPI.Fulltext) if err != nil { logrus.WithError(err).Panicf("failed to create full text") } - processContext.ComponentStarted() } federationPresenceProducer := &producers.FederationAPIPresenceProducer{ diff --git a/test/testrig/base.go b/test/testrig/base.go index fd85578ee..953704595 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -51,6 +51,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process. // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) + cfg.SyncAPI.Fulltext.InMemory = true connStr, closeDb := test.PrepareDBConnectionString(t, dbType) cfg.Global.DatabaseOptions = config.DatabaseOptions{ @@ -70,7 +71,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process. SingleDatabase: false, }) cfg.Global.ServerName = "test" - + cfg.SyncAPI.Fulltext.InMemory = true // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)