mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-16 18:43:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/devicelistcleanup

Browse source
This commit is contained in:
Till Faelligen 2022-12-08 14:00:12 +01:00
parent 0234b0c466 8846de7312
commit fe9ac2511b
No known key found for this signature in database
GPG key ID: ACCDC9606D472758
68 changed files with 1417 additions and 450 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
.github/codecov.yaml vendored Normal file
View file

@ -0,0 +1,20 @@
flag_management:
default_rules:
carryforward: true
coverage:
status:
project:
default:
target: auto
threshold: 0%
base: auto
flags:
- unittests
patch:
default:
target: 75%
threshold: 0%
base: auto
flags:
- unittests

75
.github/workflows/dendrite.yml vendored
View file

@ -68,7 +68,7 @@ jobs:
# run go test with different go versions
test:
timeout-minutes: 5
timeout-minutes: 10
name: Unit tests (Go ${{ matrix.go }})
runs-on: ubuntu-latest
# Service containers to run with `container-job`
@ -94,14 +94,22 @@ jobs:
strategy:
fail-fast: false
matrix:
go: ["1.18", "1.19"]
go: ["1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
cache: true
- uses: actions/cache@v3
# manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-unit-
- name: Set up gotestfmt
uses: gotesttools/gotestfmt-action@v2
with:
@ -194,6 +202,66 @@ jobs:
with:
jobs: ${{ toJSON(needs) }}
# run go test with different go versions
integration:
timeout-minutes: 20
needs: initial-tests-done
name: Integration tests (Go ${{ matrix.go }})
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
# Label used to access the service container
postgres:
# Docker Hub image
image: postgres:13-alpine
# Provide the password for postgres
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
go: ["1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Set up gotestfmt
uses: gotesttools/gotestfmt-action@v2
with:
# Optional: pass GITHUB_TOKEN to avoid rate limiting.
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-test-race-
- run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
env:
POSTGRES_HOST: localhost
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
flags: unittests
# run database upgrade tests
upgrade_test:
name: Upgrade tests
@ -404,6 +472,7 @@ jobs:
upgrade_test_direct,
sytest,
complement,
integration
]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped

48
.github/workflows/docker.yml vendored
View file

@ -68,18 +68,6 @@ jobs:
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
- name: Build release monolith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_monolith_release
@ -98,6 +86,18 @@ jobs:
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
polylith:
name: Polylith image
runs-on: ubuntu-latest
@ -148,18 +148,6 @@ jobs:
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
- name: Build release polylith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_polylith_release
@ -178,6 +166,18 @@ jobs:
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
demo-pinecone:
name: Pinecone demo image
runs-on: ubuntu-latest

82
.github/workflows/schedules.yaml vendored
View file

@ -10,79 +10,9 @@ concurrency:
cancel-in-progress: true
jobs:
# run go test with different go versions
test:
timeout-minutes: 20
name: Unit tests (Go ${{ matrix.go }})
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
# Label used to access the service container
postgres:
# Docker Hub image
image: postgres:13-alpine
# Provide the password for postgres
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
go: ["1.18", "1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Set up gotestfmt
uses: gotesttools/gotestfmt-action@v2
with:
# Optional: pass GITHUB_TOKEN to avoid rate limiting.
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-test-race-
- run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
env:
POSTGRES_HOST: localhost
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
# Dummy step to gate other tests on without repeating the whole list
initial-tests-done:
name: Initial tests passed
needs: [test]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
steps:
- name: Check initial tests passed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
# run Sytest in different variations
sytest:
timeout-minutes: 60
needs: initial-tests-done
name: "Sytest (${{ matrix.label }})"
runs-on: ubuntu-latest
strategy:
@ -104,13 +34,23 @@ jobs:
image: matrixdotorg/sytest-dendrite:latest
volumes:
- ${{ github.workspace }}:/src
- /root/.cache/go-build:/github/home/.cache/go-build
- /root/.cache/go-mod:/gopath/pkg/mod
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }}
RACE_DETECTION: 1
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
/gopath/pkg/mod
key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-sytest-
- name: Run Sytest
run: /bootstrap.sh dendrite
working-directory: /src

23
CHANGES.md
View file

@ -1,5 +1,28 @@
# Changelog
## Dendrite 0.10.8 (2022-11-29)
### Features
* The built-in NATS Server has been updated to version 2.9.8
* A number of under-the-hood changes have been merged for future virtual hosting support in Dendrite (running multiple domain names on the same Dendrite deployment)
### Fixes
* Event auth handling of invites has been refactored, which should fix some edge cases being handled incorrectly
* Fix a bug when returning an empty protocol list, which could cause Element to display "The homeserver may be too old to support third party networks" when opening the public room directory
* The sync API will no longer filter out the user's own membership when using lazy-loading
* Dendrite will now correctly detect JetStream consumers being deleted, stopping the consumer goroutine as needed
* A panic in the federation API where the server list could go out of bounds has been fixed
* Blacklisted servers will now be excluded when querying joined servers, which improves CPU usage and performs less unnecessary outbound requests
* A database writer will now be used to assign state key NIDs when requesting NIDs that may not exist yet
* Dendrite will now correctly move local aliases for an upgraded room when the room is upgraded remotely
* Dendrite will now correctly move account data for an upgraded room when the room is upgraded remotely
* Missing state key NIDs will now be allocated on request rather than returning an error
* Guest access is now correctly denied on a number of endpoints
* Presence information will now be correctly sent for new private chats
* A number of unspecced fields have been removed from outbound `/send` transactions
## Dendrite 0.10.7 (2022-11-04)
### Features

7
appservice/appservice.go
View file

@ -24,6 +24,8 @@ import (
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/matrix-org/gomatrixserverlib"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/appservice/consumers"
"github.com/matrix-org/dendrite/appservice/inthttp"
@ -32,12 +34,11 @@ import (
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
)
// AddInternalRoutes registers HTTP handlers for internal API calls
func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI) {
inthttp.AddRoutes(queryAPI, router)
func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI, enableMetrics bool) {
inthttp.AddRoutes(queryAPI, router, enableMetrics)
}
// NewInternalAPI returns a concerete implementation of the internal API. Callers

223
appservice/appservice_test.go Normal file
View file

@ -0,0 +1,223 @@
package appservice_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"regexp"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/appservice/inthttp"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/dendrite/test/testrig"
)
func TestAppserviceInternalAPI(t *testing.T) {
// Set expected results
existingProtocol := "irc"
wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
wantProtocolResult := map[string]api.ASProtocolResponse{
existingProtocol: wantProtocolResponse,
}
// create a dummy AS url, handling some cases
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.Contains(r.URL.Path, "location"):
// Check if we've got an existing protocol, if so, return a proper response.
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
case strings.Contains(r.URL.Path, "user"):
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
case strings.Contains(r.URL.Path, "protocol"):
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode(nil); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
default:
t.Logf("hit location: %s", r.URL.Path)
}
}))
// The test cases to run
runCases := func(t *testing.T, testAPI api.AppServiceInternalAPI) {
t.Run("UserIDExists", func(t *testing.T) {
testUserIDExists(t, testAPI, "@as-testing:test", true)
testUserIDExists(t, testAPI, "@as1-testing:test", false)
})
t.Run("AliasExists", func(t *testing.T) {
testAliasExists(t, testAPI, "@asroom-testing:test", true)
testAliasExists(t, testAPI, "@asroom1-testing:test", false)
})
t.Run("Locations", func(t *testing.T) {
testLocations(t, testAPI, existingProtocol, wantLocationResponse)
testLocations(t, testAPI, "abc", nil)
})
t.Run("User", func(t *testing.T) {
testUser(t, testAPI, existingProtocol, wantUserResponse)
testUser(t, testAPI, "abc", nil)
})
t.Run("Protocols", func(t *testing.T) {
testProtocol(t, testAPI, existingProtocol, wantProtocolResult)
testProtocol(t, testAPI, existingProtocol, wantProtocolResult) // tests the cache
testProtocol(t, testAPI, "", wantProtocolResult) // tests getting all protocols
testProtocol(t, testAPI, "abc", nil)
})
}
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, closeBase := testrig.CreateBaseDendrite(t, dbType)
defer closeBase()
// Create a dummy application service
base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
{
ID: "someID",
URL: srv.URL,
ASToken: "",
HSToken: "",
SenderLocalpart: "senderLocalPart",
NamespaceMap: map[string][]config.ApplicationServiceNamespace{
"users": {{RegexpObject: regexp.MustCompile("as-.*")}},
"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
},
Protocols: []string{existingProtocol},
},
}
// Create required internal APIs
rsAPI := roomserver.NewInternalAPI(base)
usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil)
asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
// Finally execute the tests
t.Run("HTTP API", func(t *testing.T) {
router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
appservice.AddInternalRoutes(router, asAPI, base.EnableMetrics)
apiURL, cancel := test.ListenAndServe(t, router, false)
defer cancel()
asHTTPApi, err := inthttp.NewAppserviceClient(apiURL, &http.Client{})
if err != nil {
t.Fatalf("failed to create HTTP client: %s", err)
}
runCases(t, asHTTPApi)
})
t.Run("Monolith", func(t *testing.T) {
runCases(t, asAPI)
})
})
}
func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) {
ctx := context.Background()
userResp := &api.UserIDExistsResponse{}
if err := asAPI.UserIDExists(ctx, &api.UserIDExistsRequest{
UserID: userID,
}, userResp); err != nil {
t.Errorf("failed to get userID: %s", err)
}
if userResp.UserIDExists != wantExists {
t.Errorf("unexpected result for UserIDExists(%s): %v, expected %v", userID, userResp.UserIDExists, wantExists)
}
}
func testAliasExists(t *testing.T, asAPI api.AppServiceInternalAPI, alias string, wantExists bool) {
ctx := context.Background()
aliasResp := &api.RoomAliasExistsResponse{}
if err := asAPI.RoomAliasExists(ctx, &api.RoomAliasExistsRequest{
Alias: alias,
}, aliasResp); err != nil {
t.Errorf("failed to get alias: %s", err)
}
if aliasResp.AliasExists != wantExists {
t.Errorf("unexpected result for RoomAliasExists(%s): %v, expected %v", alias, aliasResp.AliasExists, wantExists)
}
}
func testLocations(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASLocationResponse) {
ctx := context.Background()
locationResp := &api.LocationResponse{}
if err := asAPI.Locations(ctx, &api.LocationRequest{
Protocol: proto,
}, locationResp); err != nil {
t.Errorf("failed to get locations: %s", err)
}
if !reflect.DeepEqual(locationResp.Locations, wantResult) {
t.Errorf("unexpected result for Locations(%s): %+v, expected %+v", proto, locationResp.Locations, wantResult)
}
}
func testUser(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASUserResponse) {
ctx := context.Background()
userResp := &api.UserResponse{}
if err := asAPI.User(ctx, &api.UserRequest{
Protocol: proto,
}, userResp); err != nil {
t.Errorf("failed to get user: %s", err)
}
if !reflect.DeepEqual(userResp.Users, wantResult) {
t.Errorf("unexpected result for User(%s): %+v, expected %+v", proto, userResp.Users, wantResult)
}
}
func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult map[string]api.ASProtocolResponse) {
ctx := context.Background()
protoResp := &api.ProtocolResponse{}
if err := asAPI.Protocols(ctx, &api.ProtocolRequest{
Protocol: proto,
}, protoResp); err != nil {
t.Errorf("failed to get Protocols: %s", err)
}
if !reflect.DeepEqual(protoResp.Protocols, wantResult) {
t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
}
}

12
appservice/inthttp/server.go
View file

@ -8,29 +8,29 @@ import (
)
// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
internalAPIMux.Handle(
AppServiceRoomAliasExistsPath,
httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", enableMetrics, a.RoomAliasExists),
)
internalAPIMux.Handle(
AppServiceUserIDExistsPath,
httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
httputil.MakeInternalRPCAPI("AppserviceUserIDExists", enableMetrics, a.UserIDExists),
)
internalAPIMux.Handle(
AppServiceProtocolsPath,
httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
httputil.MakeInternalRPCAPI("AppserviceProtocols", enableMetrics, a.Protocols),
)
internalAPIMux.Handle(
AppServiceLocationsPath,
httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
httputil.MakeInternalRPCAPI("AppserviceLocations", enableMetrics, a.Locations),
)
internalAPIMux.Handle(
AppServiceUserPath,
httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
httputil.MakeInternalRPCAPI("AppserviceUser", enableMetrics, a.User),
)
}

3
build/gobind-pinecone/monolith.go
View file

@ -336,6 +336,7 @@ func (m *DendriteMonolith) Start() {
}
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
defer base.Close() // nolint: errcheck
federation := conn.CreateFederationClient(base, m.PineconeQUIC)
@ -382,6 +383,8 @@ func (m *DendriteMonolith) Start() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()

3
build/gobind-yggdrasil/monolith.go
View file

@ -150,6 +150,7 @@ func (m *DendriteMonolith) Start() {
}
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
m.processContext = base.ProcessContext
defer base.Close() // nolint: errcheck
@ -196,6 +197,8 @@ func (m *DendriteMonolith) Start() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
yggRouter := mux.NewRouter()
yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)

10
clientapi/routing/thirdparty.go
View file

@ -36,9 +36,15 @@ func Protocols(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, dev
return jsonerror.InternalServerError()
}
if !resp.Exists {
if protocol != "" {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The protocol is unknown."),
}
}
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The protocol is unknown."),
Code: http.StatusOK,
JSON: struct{}{},
}
}
if protocol != "" {

2
cmd/create-account/main.go
View file

@ -177,7 +177,7 @@ func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, a
defer regResp.Body.Close() // nolint: errcheck
if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
body, _ = io.ReadAll(regResp.Body)
return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
return "", fmt.Errorf("got HTTP %d error from server: %s", regResp.StatusCode, string(body))
}
r, err := io.ReadAll(regResp.Body)
if err != nil {

3
cmd/dendrite-demo-pinecone/main.go
View file

@ -155,6 +155,7 @@ func main() {
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
defer base.Close() // nolint: errcheck
pineconeEventChannel := make(chan pineconeEvents.Event)
@ -248,6 +249,8 @@ func main() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
httpRouter.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
c, err := wsUpgrader.Upgrade(w, r, nil)
if err != nil {

3
cmd/dendrite-demo-yggdrasil/main.go
View file

@ -144,6 +144,7 @@ func main() {
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
defer base.Close() // nolint: errcheck
ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
@ -199,6 +200,8 @@ func main() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()

13
cmd/dendrite-monolith-server/main.go
View file

@ -18,6 +18,8 @@ import (
"flag"
"os"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/keyserver"
@ -29,7 +31,6 @@ import (
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/sirupsen/logrus"
)
var (
@ -75,7 +76,7 @@ func main() {
// call functions directly on the impl unless running in HTTP mode
rsAPI := rsImpl
if base.UseHTTPAPIs {
roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl)
roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl, base.EnableMetrics)
rsAPI = base.RoomserverHTTPClient()
}
if traceInternal {
@ -89,7 +90,7 @@ func main() {
)
fsImplAPI := fsAPI
if base.UseHTTPAPIs {
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
fsAPI = base.FederationAPIHTTPClient()
}
keyRing := fsAPI.KeyRing()
@ -97,7 +98,7 @@ func main() {
keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
keyAPI := keyImpl
if base.UseHTTPAPIs {
keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI, base.EnableMetrics)
keyAPI = base.KeyServerHTTPClient()
}
@ -105,7 +106,7 @@ func main() {
userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
userAPI := userImpl
if base.UseHTTPAPIs {
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
userAPI = base.UserAPIClient()
}
if traceInternal {
@ -119,7 +120,7 @@ func main() {
// before the listeners are up.
asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
if base.UseHTTPAPIs {
appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
appservice.AddInternalRoutes(base.InternalAPIMux, asAPI, base.EnableMetrics)
asAPI = base.AppserviceHTTPClient()
}

2
cmd/dendrite-polylith-multi/personalities/appservice.go
View file

@ -26,7 +26,7 @@ func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) {
rsAPI := base.RoomserverHTTPClient()
intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
appservice.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener

2
cmd/dendrite-polylith-multi/personalities/federationapi.go
View file

@ -34,7 +34,7 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
rsAPI, fsAPI, keyAPI, nil,
)
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.FederationAPI.InternalAPI.Listen,

2
cmd/dendrite-polylith-multi/personalities/keyserver.go
View file

@ -26,7 +26,7 @@ func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
intAPI.SetUserAPI(base.UserAPIClient())
keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.KeyServer.InternalAPI.Listen, // internal listener

2
cmd/dendrite-polylith-multi/personalities/roomserver.go
View file

@ -26,7 +26,7 @@ func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
rsAPI := roomserver.NewInternalAPI(base)
rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing())
rsAPI.SetAppserviceAPI(asAPI)
roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.RoomServer.InternalAPI.Listen, // internal listener

2
cmd/dendrite-polylith-multi/personalities/userapi.go
View file

@ -27,7 +27,7 @@ func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
base.PushGatewayHTTPClient(),
)
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.UserAPI.InternalAPI.Listen, // internal listener

43
cmd/dendrite-upgrade-tests/main.go
View file

@ -7,6 +7,7 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
@ -61,6 +62,7 @@ COPY . .
RUN go build ./cmd/dendrite-monolith-server
RUN go build ./cmd/generate-keys
RUN go build ./cmd/generate-config
RUN go build ./cmd/create-account
RUN ./generate-config --ci > dendrite.yaml
RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
@ -104,6 +106,7 @@ COPY . .
RUN go build ./cmd/dendrite-monolith-server
RUN go build ./cmd/generate-keys
RUN go build ./cmd/generate-config
RUN go build ./cmd/create-account
RUN ./generate-config --ci > dendrite.yaml
RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
@ -458,6 +461,46 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo
if err = runTests(csAPIURL, v); err != nil {
return fmt.Errorf("failed to run tests on version %s: %s", v, err)
}
err = testCreateAccount(dockerClient, v, containerID)
if err != nil {
return err
}
return nil
}
// test that create-account is working
func testCreateAccount(dockerClient *client.Client, v string, containerID string) error {
createUser := strings.ToLower("createaccountuser-" + v)
log.Printf("%s: Creating account %s with create-account\n", v, createUser)
respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
AttachStderr: true,
AttachStdout: true,
Cmd: []string{
"/build/create-account",
"-username", createUser,
"-password", "someRandomPassword",
},
})
if err != nil {
return fmt.Errorf("failed to ContainerExecCreate: %w", err)
}
response, err := dockerClient.ContainerExecAttach(context.Background(), respID.ID, types.ExecStartCheck{})
if err != nil {
return fmt.Errorf("failed to attach to container: %w", err)
}
defer response.Close()
data, err := ioutil.ReadAll(response.Reader)
if err != nil {
return err
}
if !bytes.Contains(data, []byte("AccessToken")) {
return fmt.Errorf("failed to create-account: %s", string(data))
}
return nil
}

37
docs/CONTRIBUTING.md
View file

@ -9,6 +9,28 @@ permalink: /development/contributing
Everyone is welcome to contribute to Dendrite! We aim to make it as easy as
possible to get started.
## Contribution types
We are a small team maintaining a large project. As a result, we cannot merge every feature, even if it
is bug-free and useful, because we then commit to maintaining it indefinitely. We will always accept:
- bug fixes
- security fixes (please responsibly disclose via security@matrix.org *before* creating pull requests)
We will accept the following with caveats:
- documentation fixes, provided they do not add additional instructions which can end up going out-of-date,
e.g example configs, shell commands.
- performance fixes, provided they do not add significantly more maintenance burden.
- additional functionality on existing features, provided the functionality is small and maintainable.
- additional functionality that, in its absence, would impact the ecosystem e.g spam and abuse mitigations
- test-only changes, provided they help improve coverage or test tricky code.
The following items are at risk of not being accepted:
- Configuration or CLI changes, particularly ones which increase the overall configuration surface.
The following items are unlikely to be accepted into a main Dendrite release for now:
- New MSC implementations.
- New features which are not in the specification.
## Sign off
We require that everyone who contributes to the project signs off their contributions
@ -75,7 +97,20 @@ comment. Please avoid doing this if you can.
We also have unit tests which we run via:
```bash
go test --race ./...
DENDRITE_TEST_SKIP_NODB=1 go test --race ./...
```
This only runs SQLite database tests. If you wish to execute Postgres tests as well, you'll either need to
have Postgres installed locally (`createdb` will be used) or have a remote/containerized Postgres instance
available.
To configure the connection to a remote Postgres, you can use the following enviroment variables:
```bash
POSTGRES_USER=postgres
POSTGERS_PASSWORD=yourPostgresPassword
POSTGRES_HOST=localhost
POSTGRES_DB=postgres # the superuser database to use
```
In general, we like submissions that come with tests. Anything that proves that the

6
docs/Gemfile.lock
View file

@ -231,9 +231,9 @@ GEM
jekyll-seo-tag (~> 2.1)
minitest (5.15.0)
multipart-post (2.1.1)
nokogiri (1.13.9-arm64-darwin)
nokogiri (1.13.10-arm64-darwin)
racc (~> 1.4)
nokogiri (1.13.9-x86_64-linux)
nokogiri (1.13.10-x86_64-linux)
racc (~> 1.4)
octokit (4.22.0)
faraday (>= 0.9)
@ -241,7 +241,7 @@ GEM
pathutil (0.16.2)
forwardable-extended (~> 2.6)
public_suffix (4.0.7)
racc (1.6.0)
racc (1.6.1)
rb-fsevent (0.11.1)
rb-inotify (0.10.1)
ffi (~> 1.0)

2
federationapi/consumers/roomserver.go
View file

@ -232,7 +232,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
}
func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []types.JoinedHost) {
joined := make([]gomatrixserverlib.ServerName, len(addedJoined))
joined := make([]gomatrixserverlib.ServerName, 0, len(addedJoined))
for _, added := range addedJoined {
joined = append(joined, added.ServerName)
}

4
federationapi/federationapi.go
View file

@ -43,8 +43,8 @@ import (
// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
// on the given input API.
func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) {
inthttp.AddRoutes(intAPI, router)
func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI, enableMetrics bool) {
inthttp.AddRoutes(intAPI, router, enableMetrics)
}
// AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.

46
federationapi/inthttp/server.go
View file

@ -17,41 +17,41 @@ import (
// AddRoutes adds the FederationInternalAPI handlers to the http.ServeMux.
// nolint:gocyclo
func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
internalAPIMux.Handle(
FederationAPIQueryJoinedHostServerNamesInRoomPath,
httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", intAPI.QueryJoinedHostServerNamesInRoom),
httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", enableMetrics, intAPI.QueryJoinedHostServerNamesInRoom),
)
internalAPIMux.Handle(
FederationAPIPerformInviteRequestPath,
httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", intAPI.PerformInvite),
httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", enableMetrics, intAPI.PerformInvite),
)
internalAPIMux.Handle(
FederationAPIPerformLeaveRequestPath,
httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", intAPI.PerformLeave),
httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", enableMetrics, intAPI.PerformLeave),
)
internalAPIMux.Handle(
FederationAPIPerformDirectoryLookupRequestPath,
httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", intAPI.PerformDirectoryLookup),
httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", enableMetrics, intAPI.PerformDirectoryLookup),
)
internalAPIMux.Handle(
FederationAPIPerformBroadcastEDUPath,
httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", intAPI.PerformBroadcastEDU),
httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", enableMetrics, intAPI.PerformBroadcastEDU),
)
internalAPIMux.Handle(
FederationAPIPerformWakeupServers,
httputil.MakeInternalRPCAPI("FederationAPIPerformWakeupServers", intAPI.PerformWakeupServers),
httputil.MakeInternalRPCAPI("FederationAPIPerformWakeupServers", enableMetrics, intAPI.PerformWakeupServers),
)
internalAPIMux.Handle(
FederationAPIPerformJoinRequestPath,
httputil.MakeInternalRPCAPI(
"FederationAPIPerformJoinRequest",
"FederationAPIPerformJoinRequest", enableMetrics,
func(ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse) error {
intAPI.PerformJoin(ctx, req, res)
return nil
@ -62,7 +62,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPIGetUserDevicesPath,
httputil.MakeInternalProxyAPI(
"FederationAPIGetUserDevices",
"FederationAPIGetUserDevices", enableMetrics,
func(ctx context.Context, req *getUserDevices) (*gomatrixserverlib.RespUserDevices, error) {
res, err := intAPI.GetUserDevices(ctx, req.Origin, req.S, req.UserID)
return &res, federationClientError(err)
@ -73,7 +73,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPIClaimKeysPath,
httputil.MakeInternalProxyAPI(
"FederationAPIClaimKeys",
"FederationAPIClaimKeys", enableMetrics,
func(ctx context.Context, req *claimKeys) (*gomatrixserverlib.RespClaimKeys, error) {
res, err := intAPI.ClaimKeys(ctx, req.Origin, req.S, req.OneTimeKeys)
return &res, federationClientError(err)
@ -84,7 +84,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPIQueryKeysPath,
httputil.MakeInternalProxyAPI(
"FederationAPIQueryKeys",
"FederationAPIQueryKeys", enableMetrics,
func(ctx context.Context, req *queryKeys) (*gomatrixserverlib.RespQueryKeys, error) {
res, err := intAPI.QueryKeys(ctx, req.Origin, req.S, req.Keys)
return &res, federationClientError(err)
@ -95,7 +95,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPIBackfillPath,
httputil.MakeInternalProxyAPI(
"FederationAPIBackfill",
"FederationAPIBackfill", enableMetrics,
func(ctx context.Context, req *backfill) (*gomatrixserverlib.Transaction, error) {
res, err := intAPI.Backfill(ctx, req.Origin, req.S, req.RoomID, req.Limit, req.EventIDs)
return &res, federationClientError(err)
@ -106,7 +106,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPILookupStatePath,
httputil.MakeInternalProxyAPI(
"FederationAPILookupState",
"FederationAPILookupState", enableMetrics,
func(ctx context.Context, req *lookupState) (*gomatrixserverlib.RespState, error) {
res, err := intAPI.LookupState(ctx, req.Origin, req.S, req.RoomID, req.EventID, req.RoomVersion)
return &res, federationClientError(err)
@ -117,7 +117,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPILookupStateIDsPath,
httputil.MakeInternalProxyAPI(
"FederationAPILookupStateIDs",
"FederationAPILookupStateIDs", enableMetrics,
func(ctx context.Context, req *lookupStateIDs) (*gomatrixserverlib.RespStateIDs, error) {
res, err := intAPI.LookupStateIDs(ctx, req.Origin, req.S, req.RoomID, req.EventID)
return &res, federationClientError(err)
@ -128,7 +128,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPILookupMissingEventsPath,
httputil.MakeInternalProxyAPI(
"FederationAPILookupMissingEvents",
"FederationAPILookupMissingEvents", enableMetrics,
func(ctx context.Context, req *lookupMissingEvents) (*gomatrixserverlib.RespMissingEvents, error) {
res, err := intAPI.LookupMissingEvents(ctx, req.Origin, req.S, req.RoomID, req.Missing, req.RoomVersion)
return &res, federationClientError(err)
@ -139,7 +139,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPIGetEventPath,
httputil.MakeInternalProxyAPI(
"FederationAPIGetEvent",
"FederationAPIGetEvent", enableMetrics,
func(ctx context.Context, req *getEvent) (*gomatrixserverlib.Transaction, error) {
res, err := intAPI.GetEvent(ctx, req.Origin, req.S, req.EventID)
return &res, federationClientError(err)
@ -150,7 +150,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPIGetEventAuthPath,
httputil.MakeInternalProxyAPI(
"FederationAPIGetEventAuth",
"FederationAPIGetEventAuth", enableMetrics,
func(ctx context.Context, req *getEventAuth) (*gomatrixserverlib.RespEventAuth, error) {
res, err := intAPI.GetEventAuth(ctx, req.Origin, req.S, req.RoomVersion, req.RoomID, req.EventID)
return &res, federationClientError(err)
@ -160,13 +160,13 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPIQueryServerKeysPath,
httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", intAPI.QueryServerKeys),
httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", enableMetrics, intAPI.QueryServerKeys),
)
internalAPIMux.Handle(
FederationAPILookupServerKeysPath,
httputil.MakeInternalProxyAPI(
"FederationAPILookupServerKeys",
"FederationAPILookupServerKeys", enableMetrics,
func(ctx context.Context, req *lookupServerKeys) (*[]gomatrixserverlib.ServerKeys, error) {
res, err := intAPI.LookupServerKeys(ctx, req.S, req.KeyRequests)
return &res, federationClientError(err)
@ -177,7 +177,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPIEventRelationshipsPath,
httputil.MakeInternalProxyAPI(
"FederationAPIMSC2836EventRelationships",
"FederationAPIMSC2836EventRelationships", enableMetrics,
func(ctx context.Context, req *eventRelationships) (*gomatrixserverlib.MSC2836EventRelationshipsResponse, error) {
res, err := intAPI.MSC2836EventRelationships(ctx, req.Origin, req.S, req.Req, req.RoomVer)
return &res, federationClientError(err)
@ -188,7 +188,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
FederationAPISpacesSummaryPath,
httputil.MakeInternalProxyAPI(
"FederationAPIMSC2946SpacesSummary",
"FederationAPIMSC2946SpacesSummary", enableMetrics,
func(ctx context.Context, req *spacesReq) (*gomatrixserverlib.MSC2946SpacesResponse, error) {
res, err := intAPI.MSC2946Spaces(ctx, req.Origin, req.S, req.RoomID, req.SuggestedOnly)
return &res, federationClientError(err)
@ -198,7 +198,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
// TODO: Look at this shape
internalAPIMux.Handle(FederationAPIQueryPublicKeyPath,
httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", func(req *http.Request) util.JSONResponse {
httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse {
request := api.QueryPublicKeysRequest{}
response := api.QueryPublicKeysResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@ -215,7 +215,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
// TODO: Look at this shape
internalAPIMux.Handle(FederationAPIInputPublicKeyPath,
httputil.MakeInternalAPI("FederationAPIInputPublicKeys", func(req *http.Request) util.JSONResponse {
httputil.MakeInternalAPI("FederationAPIInputPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse {
request := api.InputPublicKeysRequest{}
response := api.InputPublicKeysResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {

4
go.mod
View file

@ -22,11 +22,11 @@ require (
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
github.com/matrix-org/gomatrixserverlib v0.0.0-20221118122129-9b9340bf29d7
github.com/matrix-org/gomatrixserverlib v0.0.0-20221129095800-8835f6db16b8
github.com/matrix-org/pinecone v0.0.0-20221118192051-fef26631b847
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/mattn/go-sqlite3 v1.14.15
github.com/nats-io/nats-server/v2 v2.9.6
github.com/nats-io/nats-server/v2 v2.9.8
github.com/nats-io/nats.go v1.20.0
github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646

8
go.sum
View file

@ -348,8 +348,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
github.com/matrix-org/gomatrixserverlib v0.0.0-20221118122129-9b9340bf29d7 h1:S2TNN7C00CZlE1Af31LzxkOsAEkFt0RYZ7/3VdR1D5U=
github.com/matrix-org/gomatrixserverlib v0.0.0-20221118122129-9b9340bf29d7/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
github.com/matrix-org/gomatrixserverlib v0.0.0-20221129095800-8835f6db16b8 h1:jVvlCGs6OosCdvw9MkfiVnTVnIt7vKMHg/F6th9BtSo=
github.com/matrix-org/gomatrixserverlib v0.0.0-20221129095800-8835f6db16b8/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
github.com/matrix-org/pinecone v0.0.0-20221118192051-fef26631b847 h1:auIBCi7gfZuvztD0aPr1G/J5Ya5vWr79M/+TJqwD/JM=
github.com/matrix-org/pinecone v0.0.0-20221118192051-fef26631b847/go.mod h1:F3GHppRuHCTDeoOmmgjZMeJdbql91+RSGGsATWfC7oc=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
@ -385,8 +385,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
github.com/nats-io/nats-server/v2 v2.9.6 h1:RTtK+rv/4CcliOuqGsy58g7MuWkBaWmF5TUNwuUo9Uw=
github.com/nats-io/nats-server/v2 v2.9.6/go.mod h1:AB6hAnGZDlYfqb7CTAm66ZKMZy9DpfierY1/PbpvI2g=
github.com/nats-io/nats-server/v2 v2.9.8 h1:jgxZsv+A3Reb3MgwxaINcNq/za8xZInKhDg9Q0cGN1o=
github.com/nats-io/nats-server/v2 v2.9.8/go.mod h1:AB6hAnGZDlYfqb7CTAm66ZKMZy9DpfierY1/PbpvI2g=
github.com/nats-io/nats.go v1.20.0 h1:T8JJnQfVSdh1CzGiwAOv5hEobYCBho/0EupGznYw0oM=
github.com/nats-io/nats.go v1.20.0/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA=
github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=

15
internal/httputil/httpapi.go
View file

@ -24,16 +24,17 @@ import (
"strings"
"github.com/getsentry/sentry-go"
"github.com/matrix-org/dendrite/clientapi/auth"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
opentracing "github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/clientapi/auth"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
userapi "github.com/matrix-org/dendrite/userapi/api"
)
// BasicAuth is used for authorization on /metrics handlers
@ -227,7 +228,7 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request)
// This is used for APIs that are internal to dendrite.
// If we are passed a tracing context in the request headers then we use that
// as the parent of any tracing spans we create.
func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler {
func MakeInternalAPI(metricsName string, enableMetrics bool, f func(*http.Request) util.JSONResponse) http.Handler {
h := util.MakeJSONAPI(util.NewJSONRequestHandler(f))
withSpan := func(w http.ResponseWriter, req *http.Request) {
carrier := opentracing.HTTPHeadersCarrier(req.Header)
@ -246,6 +247,10 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse
h.ServeHTTP(w, req)
}
if !enableMetrics {
return http.HandlerFunc(withSpan)
}
return promhttp.InstrumentHandlerCounter(
promauto.NewCounterVec(
prometheus.CounterOpts{

10
internal/httputil/internalapi.go
View file

@ -22,7 +22,7 @@ import (
"reflect"
"github.com/matrix-org/util"
opentracing "github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go"
)
type InternalAPIError struct {
@ -34,8 +34,8 @@ func (e InternalAPIError) Error() string {
return fmt.Sprintf("internal API returned %q error: %s", e.Type, e.Message)
}
func MakeInternalRPCAPI[reqtype, restype any](metricsName string, f func(context.Context, *reqtype, *restype) error) http.Handler {
return MakeInternalAPI(metricsName, func(req *http.Request) util.JSONResponse {
func MakeInternalRPCAPI[reqtype, restype any](metricsName string, enableMetrics bool, f func(context.Context, *reqtype, *restype) error) http.Handler {
return MakeInternalAPI(metricsName, enableMetrics, func(req *http.Request) util.JSONResponse {
var request reqtype
var response restype
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@ -57,8 +57,8 @@ func MakeInternalRPCAPI[reqtype, restype any](metricsName string, f func(context
})
}
func MakeInternalProxyAPI[reqtype, restype any](metricsName string, f func(context.Context, *reqtype) (*restype, error)) http.Handler {
return MakeInternalAPI(metricsName, func(req *http.Request) util.JSONResponse {
func MakeInternalProxyAPI[reqtype, restype any](metricsName string, enableMetrics bool, f func(context.Context, *reqtype) (*restype, error)) http.Handler {
return MakeInternalAPI(metricsName, enableMetrics, func(req *http.Request) util.JSONResponse {
var request reqtype
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())

5
internal/log.go
View file

@ -33,6 +33,11 @@ import (
"github.com/matrix-org/dendrite/setup/config"
)
// logrus is using a global variable when we're using `logrus.AddHook`
// this unfortunately results in us adding the same hook multiple times.
// This map ensures we only ever add one level hook.
var stdLevelLogAdded = make(map[logrus.Level]bool)
type utcFormatter struct {
logrus.Formatter
}

13
internal/log_unix.go
View file

@ -22,16 +22,16 @@ import (
"log/syslog"
"github.com/MFAshby/stdemuxerhook"
"github.com/matrix-org/dendrite/setup/config"
"github.com/sirupsen/logrus"
lSyslog "github.com/sirupsen/logrus/hooks/syslog"
"github.com/matrix-org/dendrite/setup/config"
)
// SetupHookLogging configures the logging hooks defined in the configuration.
// If something fails here it means that the logging was improperly configured,
// so we just exit with the error
func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
stdLogAdded := false
for _, hook := range hooks {
// Check we received a proper logging level
level, err := logrus.ParseLevel(hook.Level)
@ -54,14 +54,11 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
setupSyslogHook(hook, level, componentName)
case "std":
setupStdLogHook(level)
stdLogAdded = true
default:
logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
}
}
if !stdLogAdded {
setupStdLogHook(logrus.InfoLevel)
}
setupStdLogHook(logrus.InfoLevel)
// Hooks are now configured for stdout/err, so throw away the default logger output
logrus.SetOutput(io.Discard)
}
@ -88,7 +85,11 @@ func checkSyslogHookParams(params map[string]interface{}) {
}
func setupStdLogHook(level logrus.Level) {
if stdLevelLogAdded[level] {
return
}
logrus.AddHook(&logLevelHook{level, stdemuxerhook.New(logrus.StandardLogger())})
stdLevelLogAdded[level] = true
}
func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) {

5
internal/pushgateway/client.go
View file

@ -9,6 +9,8 @@ import (
"net/http"
"time"
"github.com/matrix-org/dendrite/internal"
"github.com/opentracing/opentracing-go"
)
@ -50,8 +52,7 @@ func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest,
return err
}
//nolint:errcheck
defer hresp.Body.Close()
defer internal.CloseAndLogIfError(ctx, hresp.Body, "failed to close response body")
if hresp.StatusCode == http.StatusOK {
return json.NewDecoder(hresp.Body).Decode(resp)

54
internal/pushgateway/client_test.go Normal file
View file

@ -0,0 +1,54 @@
package pushgateway
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"testing"
)
func TestNotify(t *testing.T) {
wantResponse := NotifyResponse{
Rejected: []string{"testing"},
}
var i = 0
svr := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// /notify only accepts POST requests
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusNotImplemented)
return
}
if i != 0 { // error path
w.WriteHeader(http.StatusBadRequest)
return
}
// happy path
json.NewEncoder(w).Encode(wantResponse)
}))
defer svr.Close()
cl := NewHTTPClient(true)
gotResponse := NotifyResponse{}
// Test happy path
err := cl.Notify(context.Background(), svr.URL, &NotifyRequest{}, &gotResponse)
if err != nil {
t.Errorf("failed to notify client")
}
if !reflect.DeepEqual(gotResponse, wantResponse) {
t.Errorf("expected response %+v, got %+v", wantResponse, gotResponse)
}
// Test error path
i++
err = cl.Notify(context.Background(), svr.URL, &NotifyRequest{}, &gotResponse)
if err == nil {
t.Errorf("expected notifying the pushgateway to fail, but it succeeded")
}
}

13
internal/pushrules/evaluate.go
View file

@ -145,6 +145,11 @@ func conditionMatches(cond *Condition, event *gomatrixserverlib.Event, ec Evalua
}
func patternMatches(key, pattern string, event *gomatrixserverlib.Event) (bool, error) {
// It doesn't make sense for an empty pattern to match anything.
if pattern == "" {
return false, nil
}
re, err := globToRegexp(pattern)
if err != nil {
return false, err
@ -154,12 +159,20 @@ func patternMatches(key, pattern string, event *gomatrixserverlib.Event) (bool,
if err = json.Unmarshal(event.JSON(), &eventMap); err != nil {
return false, fmt.Errorf("parsing event: %w", err)
}
// From the spec:
// "If the property specified by key is completely absent from
// the event, or does not have a string value, then the condition
// will not match, even if pattern is *."
v, err := lookupMapPath(strings.Split(key, "."), eventMap)
if err != nil {
// An unknown path is a benign error that shouldn't stop rule
// processing. It's just a non-match.
return false, nil
}
if _, ok := v.(string); !ok {
// A non-string never matches.
return false, nil
}
return re.MatchString(fmt.Sprint(v)), nil
}

13
internal/pushrules/evaluate_test.go
View file

@ -111,7 +111,10 @@ func TestConditionMatches(t *testing.T) {
{"empty", Condition{}, `{}`, false},
{"empty", Condition{Kind: "unknownstring"}, `{}`, false},
{"eventMatch", Condition{Kind: EventMatchCondition, Key: "content"}, `{"content":{}}`, true},
// Neither of these should match because `content` is not a full string match,
// and `content.body` is not a string value.
{"eventMatch", Condition{Kind: EventMatchCondition, Key: "content"}, `{"content":{}}`, false},
{"eventBodyMatch", Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3"}, `{"content":{"body": 3}}`, false},
{"displayNameNoMatch", Condition{Kind: ContainsDisplayNameCondition}, `{"content":{"body":"something without displayname"}}`, false},
{"displayNameMatch", Condition{Kind: ContainsDisplayNameCondition}, `{"content":{"body":"hello Dear User, how are you?"}}`, true},
@ -137,7 +140,7 @@ func TestConditionMatches(t *testing.T) {
t.Fatalf("conditionMatches failed: %v", err)
}
if got != tst.Want {
t.Errorf("conditionMatches: got %v, want %v", got, tst.Want)
t.Errorf("conditionMatches: got %v, want %v on %s", got, tst.Want, tst.Name)
}
})
}
@ -161,9 +164,7 @@ func TestPatternMatches(t *testing.T) {
}{
{"empty", "", "", `{}`, false},
// Note that an empty pattern contains no wildcard characters,
// which implicitly means "*".
{"patternEmpty", "content", "", `{"content":{}}`, true},
{"patternEmpty", "content", "", `{"content":{}}`, false},
{"literal", "content.creator", "acreator", `{"content":{"creator":"acreator"}}`, true},
{"substring", "content.creator", "reat", `{"content":{"creator":"acreator"}}`, true},
@ -178,7 +179,7 @@ func TestPatternMatches(t *testing.T) {
t.Fatalf("patternMatches failed: %v", err)
}
if got != tst.Want {
t.Errorf("patternMatches: got %v, want %v", got, tst.Want)
t.Errorf("patternMatches: got %v, want %v on %s", got, tst.Want, tst.Name)
}
})
}

31
internal/pushrules/util.go
View file

@ -11,22 +11,27 @@ import (
// kind and a tweaks map. Returns a nil map if it would have been
// empty.
func ActionsToTweaks(as []*Action) (ActionKind, map[string]interface{}, error) {
kind := UnknownAction
tweaks := map[string]interface{}{}
var kind ActionKind
var tweaks map[string]interface{}
for _, a := range as {
if a.Kind == SetTweakAction {
tweaks[string(a.Tweak)] = a.Value
continue
}
if kind != UnknownAction {
return UnknownAction, nil, fmt.Errorf("got multiple primary actions: already had %q, got %s", kind, a.Kind)
}
kind = a.Kind
}
switch a.Kind {
case DontNotifyAction:
// Don't bother processing any further
return DontNotifyAction, nil, nil
if len(tweaks) == 0 {
tweaks = nil
case SetTweakAction:
if tweaks == nil {
tweaks = map[string]interface{}{}
}
tweaks[string(a.Tweak)] = a.Value
default:
if kind != UnknownAction {
return UnknownAction, nil, fmt.Errorf("got multiple primary actions: already had %q, got %s", kind, a.Kind)
}
kind = a.Kind
}
}
return kind, tweaks, nil

1
internal/pushrules/util_test.go
View file

@ -17,6 +17,7 @@ func TestActionsToTweaks(t *testing.T) {
{"empty", nil, UnknownAction, nil},
{"zero", []*Action{{}}, UnknownAction, nil},
{"onlyPrimary", []*Action{{Kind: NotifyAction}}, NotifyAction, nil},
{"onlyPrimaryDontNotify", []*Action{{Kind: DontNotifyAction}}, DontNotifyAction, nil},
{"onlyTweak", []*Action{{Kind: SetTweakAction, Tweak: HighlightTweak}}, UnknownAction, map[string]interface{}{"highlight": nil}},
{"onlyTweakWithValue", []*Action{{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, UnknownAction, map[string]interface{}{"sound": "default"}},
{

2
internal/version.go
View file

@ -17,7 +17,7 @@ var build string
const (
VersionMajor = 0
VersionMinor = 10
VersionPatch = 7
VersionPatch = 8
VersionTag = "" // example: "rc1"
)

24
keyserver/inthttp/server.go
View file

@ -21,59 +21,59 @@ import (
"github.com/matrix-org/dendrite/keyserver/api"
)
func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) {
func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI, enableMetrics bool) {
internalAPIMux.Handle(
PerformClaimKeysPath,
httputil.MakeInternalRPCAPI("KeyserverPerformClaimKeys", s.PerformClaimKeys),
httputil.MakeInternalRPCAPI("KeyserverPerformClaimKeys", enableMetrics, s.PerformClaimKeys),
)
internalAPIMux.Handle(
PerformDeleteKeysPath,
httputil.MakeInternalRPCAPI("KeyserverPerformDeleteKeys", s.PerformDeleteKeys),
httputil.MakeInternalRPCAPI("KeyserverPerformDeleteKeys", enableMetrics, s.PerformDeleteKeys),
)
internalAPIMux.Handle(
PerformUploadKeysPath,
httputil.MakeInternalRPCAPI("KeyserverPerformUploadKeys", s.PerformUploadKeys),
httputil.MakeInternalRPCAPI("KeyserverPerformUploadKeys", enableMetrics, s.PerformUploadKeys),
)
internalAPIMux.Handle(
PerformUploadDeviceKeysPath,
httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceKeys", s.PerformUploadDeviceKeys),
httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceKeys", enableMetrics, s.PerformUploadDeviceKeys),
)
internalAPIMux.Handle(
PerformUploadDeviceSignaturesPath,
httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceSignatures", s.PerformUploadDeviceSignatures),
httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceSignatures", enableMetrics, s.PerformUploadDeviceSignatures),
)
internalAPIMux.Handle(
QueryKeysPath,
httputil.MakeInternalRPCAPI("KeyserverQueryKeys", s.QueryKeys),
httputil.MakeInternalRPCAPI("KeyserverQueryKeys", enableMetrics, s.QueryKeys),
)
internalAPIMux.Handle(
QueryOneTimeKeysPath,
httputil.MakeInternalRPCAPI("KeyserverQueryOneTimeKeys", s.QueryOneTimeKeys),
httputil.MakeInternalRPCAPI("KeyserverQueryOneTimeKeys", enableMetrics, s.QueryOneTimeKeys),
)
internalAPIMux.Handle(
QueryDeviceMessagesPath,
httputil.MakeInternalRPCAPI("KeyserverQueryDeviceMessages", s.QueryDeviceMessages),
httputil.MakeInternalRPCAPI("KeyserverQueryDeviceMessages", enableMetrics, s.QueryDeviceMessages),
)
internalAPIMux.Handle(
QueryKeyChangesPath,
httputil.MakeInternalRPCAPI("KeyserverQueryKeyChanges", s.QueryKeyChanges),
httputil.MakeInternalRPCAPI("KeyserverQueryKeyChanges", enableMetrics, s.QueryKeyChanges),
)
internalAPIMux.Handle(
QuerySignaturesPath,
httputil.MakeInternalRPCAPI("KeyserverQuerySignatures", s.QuerySignatures),
httputil.MakeInternalRPCAPI("KeyserverQuerySignatures", enableMetrics, s.QuerySignatures),
)
internalAPIMux.Handle(
PerformMarkAsStalePath,
httputil.MakeInternalRPCAPI("KeyserverMarkAsStale", s.PerformMarkAsStaleIfNeeded),
httputil.MakeInternalRPCAPI("KeyserverMarkAsStale", enableMetrics, s.PerformMarkAsStaleIfNeeded),
)
}

4
keyserver/keyserver.go
View file

@ -33,8 +33,8 @@ import (
// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
// on the given input API.
func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) {
inthttp.AddRoutes(router, intAPI)
func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI, enableMetrics bool) {
inthttp.AddRoutes(router, intAPI, enableMetrics)
}
// NewInternalAPI returns a concerete implementation of the internal API. Callers

80
roomserver/inthttp/server.go
View file

@ -9,199 +9,199 @@ import (
// AddRoutes adds the RoomserverInternalAPI handlers to the http.ServeMux.
// nolint: gocyclo
func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
internalAPIMux.Handle(
RoomserverInputRoomEventsPath,
httputil.MakeInternalRPCAPI("RoomserverInputRoomEvents", r.InputRoomEvents),
httputil.MakeInternalRPCAPI("RoomserverInputRoomEvents", enableMetrics, r.InputRoomEvents),
)
internalAPIMux.Handle(
RoomserverPerformInvitePath,
httputil.MakeInternalRPCAPI("RoomserverPerformInvite", r.PerformInvite),
httputil.MakeInternalRPCAPI("RoomserverPerformInvite", enableMetrics, r.PerformInvite),
)
internalAPIMux.Handle(
RoomserverPerformJoinPath,
httputil.MakeInternalRPCAPI("RoomserverPerformJoin", r.PerformJoin),
httputil.MakeInternalRPCAPI("RoomserverPerformJoin", enableMetrics, r.PerformJoin),
)
internalAPIMux.Handle(
RoomserverPerformLeavePath,
httputil.MakeInternalRPCAPI("RoomserverPerformLeave", r.PerformLeave),
httputil.MakeInternalRPCAPI("RoomserverPerformLeave", enableMetrics, r.PerformLeave),
)
internalAPIMux.Handle(
RoomserverPerformPeekPath,
httputil.MakeInternalRPCAPI("RoomserverPerformPeek", r.PerformPeek),
httputil.MakeInternalRPCAPI("RoomserverPerformPeek", enableMetrics, r.PerformPeek),
)
internalAPIMux.Handle(
RoomserverPerformInboundPeekPath,
httputil.MakeInternalRPCAPI("RoomserverPerformInboundPeek", r.PerformInboundPeek),
httputil.MakeInternalRPCAPI("RoomserverPerformInboundPeek", enableMetrics, r.PerformInboundPeek),
)
internalAPIMux.Handle(
RoomserverPerformUnpeekPath,
httputil.MakeInternalRPCAPI("RoomserverPerformUnpeek", r.PerformUnpeek),
httputil.MakeInternalRPCAPI("RoomserverPerformUnpeek", enableMetrics, r.PerformUnpeek),
)
internalAPIMux.Handle(
RoomserverPerformRoomUpgradePath,
httputil.MakeInternalRPCAPI("RoomserverPerformRoomUpgrade", r.PerformRoomUpgrade),
httputil.MakeInternalRPCAPI("RoomserverPerformRoomUpgrade", enableMetrics, r.PerformRoomUpgrade),
)
internalAPIMux.Handle(
RoomserverPerformPublishPath,
httputil.MakeInternalRPCAPI("RoomserverPerformPublish", r.PerformPublish),
httputil.MakeInternalRPCAPI("RoomserverPerformPublish", enableMetrics, r.PerformPublish),
)
internalAPIMux.Handle(
RoomserverPerformAdminEvacuateRoomPath,
httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateRoom", r.PerformAdminEvacuateRoom),
httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateRoom", enableMetrics, r.PerformAdminEvacuateRoom),
)
internalAPIMux.Handle(
RoomserverPerformAdminEvacuateUserPath,
httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", r.PerformAdminEvacuateUser),
httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", enableMetrics, r.PerformAdminEvacuateUser),
)
internalAPIMux.Handle(
RoomserverPerformAdminDownloadStatePath,
httputil.MakeInternalRPCAPI("RoomserverPerformAdminDownloadState", r.PerformAdminDownloadState),
httputil.MakeInternalRPCAPI("RoomserverPerformAdminDownloadState", enableMetrics, r.PerformAdminDownloadState),
)
internalAPIMux.Handle(
RoomserverQueryPublishedRoomsPath,
httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", r.QueryPublishedRooms),
httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", enableMetrics, r.QueryPublishedRooms),
)
internalAPIMux.Handle(
RoomserverQueryLatestEventsAndStatePath,
httputil.MakeInternalRPCAPI("RoomserverQueryLatestEventsAndState", r.QueryLatestEventsAndState),
httputil.MakeInternalRPCAPI("RoomserverQueryLatestEventsAndState", enableMetrics, r.QueryLatestEventsAndState),
)
internalAPIMux.Handle(
RoomserverQueryStateAfterEventsPath,
httputil.MakeInternalRPCAPI("RoomserverQueryStateAfterEvents", r.QueryStateAfterEvents),
httputil.MakeInternalRPCAPI("RoomserverQueryStateAfterEvents", enableMetrics, r.QueryStateAfterEvents),
)
internalAPIMux.Handle(
RoomserverQueryEventsByIDPath,
httputil.MakeInternalRPCAPI("RoomserverQueryEventsByID", r.QueryEventsByID),
httputil.MakeInternalRPCAPI("RoomserverQueryEventsByID", enableMetrics, r.QueryEventsByID),
)
internalAPIMux.Handle(
RoomserverQueryMembershipForUserPath,
httputil.MakeInternalRPCAPI("RoomserverQueryMembershipForUser", r.QueryMembershipForUser),
httputil.MakeInternalRPCAPI("RoomserverQueryMembershipForUser", enableMetrics, r.QueryMembershipForUser),
)
internalAPIMux.Handle(
RoomserverQueryMembershipsForRoomPath,
httputil.MakeInternalRPCAPI("RoomserverQueryMembershipsForRoom", r.QueryMembershipsForRoom),
httputil.MakeInternalRPCAPI("RoomserverQueryMembershipsForRoom", enableMetrics, r.QueryMembershipsForRoom),
)
internalAPIMux.Handle(
RoomserverQueryServerJoinedToRoomPath,
httputil.MakeInternalRPCAPI("RoomserverQueryServerJoinedToRoom", r.QueryServerJoinedToRoom),
httputil.MakeInternalRPCAPI("RoomserverQueryServerJoinedToRoom", enableMetrics, r.QueryServerJoinedToRoom),
)
internalAPIMux.Handle(
RoomserverQueryServerAllowedToSeeEventPath,
httputil.MakeInternalRPCAPI("RoomserverQueryServerAllowedToSeeEvent", r.QueryServerAllowedToSeeEvent),
httputil.MakeInternalRPCAPI("RoomserverQueryServerAllowedToSeeEvent", enableMetrics, r.QueryServerAllowedToSeeEvent),
)
internalAPIMux.Handle(
RoomserverQueryMissingEventsPath,
httputil.MakeInternalRPCAPI("RoomserverQueryMissingEvents", r.QueryMissingEvents),
httputil.MakeInternalRPCAPI("RoomserverQueryMissingEvents", enableMetrics, r.QueryMissingEvents),
)
internalAPIMux.Handle(
RoomserverQueryStateAndAuthChainPath,
httputil.MakeInternalRPCAPI("RoomserverQueryStateAndAuthChain", r.QueryStateAndAuthChain),
httputil.MakeInternalRPCAPI("RoomserverQueryStateAndAuthChain", enableMetrics, r.QueryStateAndAuthChain),
)
internalAPIMux.Handle(
RoomserverPerformBackfillPath,
httputil.MakeInternalRPCAPI("RoomserverPerformBackfill", r.PerformBackfill),
httputil.MakeInternalRPCAPI("RoomserverPerformBackfill", enableMetrics, r.PerformBackfill),
)
internalAPIMux.Handle(
RoomserverPerformForgetPath,
httputil.MakeInternalRPCAPI("RoomserverPerformForget", r.PerformForget),
httputil.MakeInternalRPCAPI("RoomserverPerformForget", enableMetrics, r.PerformForget),
)
internalAPIMux.Handle(
RoomserverQueryRoomVersionCapabilitiesPath,
httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionCapabilities", r.QueryRoomVersionCapabilities),
httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionCapabilities", enableMetrics, r.QueryRoomVersionCapabilities),
)
internalAPIMux.Handle(
RoomserverQueryRoomVersionForRoomPath,
httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionForRoom", r.QueryRoomVersionForRoom),
httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionForRoom", enableMetrics, r.QueryRoomVersionForRoom),
)
internalAPIMux.Handle(
RoomserverSetRoomAliasPath,
httputil.MakeInternalRPCAPI("RoomserverSetRoomAlias", r.SetRoomAlias),
httputil.MakeInternalRPCAPI("RoomserverSetRoomAlias", enableMetrics, r.SetRoomAlias),
)
internalAPIMux.Handle(
RoomserverGetRoomIDForAliasPath,
httputil.MakeInternalRPCAPI("RoomserverGetRoomIDForAlias", r.GetRoomIDForAlias),
httputil.MakeInternalRPCAPI("RoomserverGetRoomIDForAlias", enableMetrics, r.GetRoomIDForAlias),
)
internalAPIMux.Handle(
RoomserverGetAliasesForRoomIDPath,
httputil.MakeInternalRPCAPI("RoomserverGetAliasesForRoomID", r.GetAliasesForRoomID),
httputil.MakeInternalRPCAPI("RoomserverGetAliasesForRoomID", enableMetrics, r.GetAliasesForRoomID),
)
internalAPIMux.Handle(
RoomserverRemoveRoomAliasPath,
httputil.MakeInternalRPCAPI("RoomserverRemoveRoomAlias", r.RemoveRoomAlias),
httputil.MakeInternalRPCAPI("RoomserverRemoveRoomAlias", enableMetrics, r.RemoveRoomAlias),
)
internalAPIMux.Handle(
RoomserverQueryCurrentStatePath,
httputil.MakeInternalRPCAPI("RoomserverQueryCurrentState", r.QueryCurrentState),
httputil.MakeInternalRPCAPI("RoomserverQueryCurrentState", enableMetrics, r.QueryCurrentState),
)
internalAPIMux.Handle(
RoomserverQueryRoomsForUserPath,
httputil.MakeInternalRPCAPI("RoomserverQueryRoomsForUser", r.QueryRoomsForUser),
httputil.MakeInternalRPCAPI("RoomserverQueryRoomsForUser", enableMetrics, r.QueryRoomsForUser),
)
internalAPIMux.Handle(
RoomserverQueryBulkStateContentPath,
httputil.MakeInternalRPCAPI("RoomserverQueryBulkStateContent", r.QueryBulkStateContent),
httputil.MakeInternalRPCAPI("RoomserverQueryBulkStateContent", enableMetrics, r.QueryBulkStateContent),
)
internalAPIMux.Handle(
RoomserverQuerySharedUsersPath,
httputil.MakeInternalRPCAPI("RoomserverQuerySharedUsers", r.QuerySharedUsers),
httputil.MakeInternalRPCAPI("RoomserverQuerySharedUsers", enableMetrics, r.QuerySharedUsers),
)
internalAPIMux.Handle(
RoomserverQueryKnownUsersPath,
httputil.MakeInternalRPCAPI("RoomserverQueryKnownUsers", r.QueryKnownUsers),
httputil.MakeInternalRPCAPI("RoomserverQueryKnownUsers", enableMetrics, r.QueryKnownUsers),
)
internalAPIMux.Handle(
RoomserverQueryServerBannedFromRoomPath,
httputil.MakeInternalRPCAPI("RoomserverQueryServerBannedFromRoom", r.QueryServerBannedFromRoom),
httputil.MakeInternalRPCAPI("RoomserverQueryServerBannedFromRoom", enableMetrics, r.QueryServerBannedFromRoom),
)
internalAPIMux.Handle(
RoomserverQueryAuthChainPath,
httputil.MakeInternalRPCAPI("RoomserverQueryAuthChain", r.QueryAuthChain),
httputil.MakeInternalRPCAPI("RoomserverQueryAuthChain", enableMetrics, r.QueryAuthChain),
)
internalAPIMux.Handle(
RoomserverQueryRestrictedJoinAllowed,
httputil.MakeInternalRPCAPI("RoomserverQueryRestrictedJoinAllowed", r.QueryRestrictedJoinAllowed),
httputil.MakeInternalRPCAPI("RoomserverQueryRestrictedJoinAllowed", enableMetrics, r.QueryRestrictedJoinAllowed),
)
internalAPIMux.Handle(
RoomserverQueryMembershipAtEventPath,
httputil.MakeInternalRPCAPI("RoomserverQueryMembershipAtEventPath", r.QueryMembershipAtEvent),
httputil.MakeInternalRPCAPI("RoomserverQueryMembershipAtEventPath", enableMetrics, r.QueryMembershipAtEvent),
)
internalAPIMux.Handle(

7
roomserver/roomserver.go
View file

@ -16,18 +16,19 @@ package roomserver
import (
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/internal"
"github.com/matrix-org/dendrite/roomserver/inthttp"
"github.com/matrix-org/dendrite/roomserver/storage"
"github.com/matrix-org/dendrite/setup/base"
"github.com/sirupsen/logrus"
)
// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
// on the given input API.
func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI) {
inthttp.AddRoutes(intAPI, router)
func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI, enableMetrics bool) {
inthttp.AddRoutes(intAPI, router, enableMetrics)
}
// NewInternalAPI returns a concerete implementation of the internal API. Callers

34
setup/base/base.go
View file

@ -413,6 +413,24 @@ func (b *BaseDendrite) configureHTTPErrors() {
b.PublicClientAPIMux.MethodNotAllowedHandler = http.HandlerFunc(clientNotFoundHandler)
}
func (b *BaseDendrite) ConfigureAdminEndpoints() {
b.DendriteAdminMux.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
})
b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) {
if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded {
w.WriteHeader(503)
_ = json.NewEncoder(w).Encode(struct {
Warnings []string `json:"warnings"`
}{
Warnings: reasons,
})
return
}
w.WriteHeader(200)
})
}
// SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on
// ApiMux under /api/ and adds a prometheus handler under /metrics.
func (b *BaseDendrite) SetupAndServeHTTP(
@ -463,21 +481,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(
internalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth))
}
b.DendriteAdminMux.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
})
b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) {
if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded {
w.WriteHeader(503)
_ = json.NewEncoder(w).Encode(struct {
Warnings []string `json:"warnings"`
}{
Warnings: reasons,
})
return
}
w.WriteHeader(200)
})
b.ConfigureAdminEndpoints()
var clientHandler http.Handler
clientHandler = b.PublicClientAPIMux

2
setup/config/config_global.go
View file

@ -17,7 +17,7 @@ type Global struct {
gomatrixserverlib.SigningIdentity `yaml:",inline"`
// The secondary server names, used for virtual hosting.
VirtualHosts []*VirtualHost `yaml:"virtual_hosts"`
VirtualHosts []*VirtualHost `yaml:"-"`
// Path to the private key which will be used to sign requests and events.
PrivateKeyPath Path `yaml:"private_key"`

12
syncapi/consumers/presence.go
View file

@ -78,7 +78,7 @@ func (s *PresenceConsumer) Start() error {
// Normal NATS subscription, used by Request/Reply
_, err := s.nats.Subscribe(s.requestTopic, func(msg *nats.Msg) {
userID := msg.Header.Get(jetstream.UserID)
presence, err := s.db.GetPresence(context.Background(), userID)
presences, err := s.db.GetPresences(context.Background(), []string{userID})
m := &nats.Msg{
Header: nats.Header{},
}
@ -89,10 +89,12 @@ func (s *PresenceConsumer) Start() error {
}
return
}
if presence == nil {
presence = &types.PresenceInternal{
UserID: userID,
}
presence := &types.PresenceInternal{
UserID: userID,
}
if len(presences) > 0 {
presence = presences[0]
}
deviceRes := api.QueryDevicesResponse{}

4
syncapi/storage/interface.go
View file

@ -106,7 +106,7 @@ type DatabaseTransaction interface {
SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error)
// getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms
GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error)
GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
GetPresences(ctx context.Context, userID []string) ([]*types.PresenceInternal, error)
PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error)
}
@ -186,7 +186,7 @@ type Database interface {
}
type Presence interface {
GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
GetPresences(ctx context.Context, userIDs []string) ([]*types.PresenceInternal, error)
UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error)
}

38
syncapi/storage/postgres/presence_table.go
View file

@ -19,10 +19,12 @@ import (
"database/sql"
"time"
"github.com/lib/pq"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
)
const presenceSchema = `
@ -63,9 +65,9 @@ const upsertPresenceFromSyncSQL = "" +
" RETURNING id"
const selectPresenceForUserSQL = "" +
"SELECT presence, status_msg, last_active_ts" +
"SELECT user_id, presence, status_msg, last_active_ts" +
" FROM syncapi_presence" +
" WHERE user_id = $1 LIMIT 1"
" WHERE user_id = ANY($1)"
const selectMaxPresenceSQL = "" +
"SELECT COALESCE(MAX(id), 0) FROM syncapi_presence"
@ -119,20 +121,28 @@ func (p *presenceStatements) UpsertPresence(
return
}
// GetPresenceForUser returns the current presence of a user.
func (p *presenceStatements) GetPresenceForUser(
// GetPresenceForUsers returns the current presence for a list of users.
// If the user doesn't have a presence status yet, it is omitted from the response.
func (p *presenceStatements) GetPresenceForUsers(
ctx context.Context, txn *sql.Tx,
userID string,
) (*types.PresenceInternal, error) {
result := &types.PresenceInternal{
UserID: userID,
}
userIDs []string,
) ([]*types.PresenceInternal, error) {
result := make([]*types.PresenceInternal, 0, len(userIDs))
stmt := sqlutil.TxStmt(txn, p.selectPresenceForUsersStmt)
err := stmt.QueryRowContext(ctx, userID).Scan(&result.Presence, &result.ClientFields.StatusMsg, &result.LastActiveTS)
if err == sql.ErrNoRows {
return nil, nil
rows, err := stmt.QueryContext(ctx, pq.Array(userIDs))
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "GetPresenceForUsers: rows.close() failed")
for rows.Next() {
presence := &types.PresenceInternal{}
if err = rows.Scan(&presence.UserID, &presence.Presence, &presence.ClientFields.StatusMsg, &presence.LastActiveTS); err != nil {
return nil, err
}
presence.ClientFields.Presence = presence.Presence.String()
result = append(result, presence)
}
result.ClientFields.Presence = result.Presence.String()
return result, err
}

46
syncapi/storage/shared/storage_consumer.go
View file

@ -57,31 +57,23 @@ type Database struct {
}
func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) {
return d.NewDatabaseTransaction(ctx)
/*
TODO: Repeatable read is probably the right thing to do here,
but it seems to cause some problems with the invite tests, so
need to investigate that further.
txn, err := d.DB.BeginTx(ctx, &sql.TxOptions{
// Set the isolation level so that we see a snapshot of the database.
// In PostgreSQL repeatable read transactions will see a snapshot taken
// at the first query, and since the transaction is read-only it can't
// run into any serialisation errors.
// https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ
Isolation: sql.LevelRepeatableRead,
ReadOnly: true,
})
if err != nil {
return nil, err
}
return &DatabaseTransaction{
Database: d,
ctx: ctx,
txn: txn,
}, nil
*/
txn, err := d.DB.BeginTx(ctx, &sql.TxOptions{
// Set the isolation level so that we see a snapshot of the database.
// In PostgreSQL repeatable read transactions will see a snapshot taken
// at the first query, and since the transaction is read-only it can't
// run into any serialisation errors.
// https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ
Isolation: sql.LevelRepeatableRead,
ReadOnly: true,
})
if err != nil {
return nil, err
}
return &DatabaseTransaction{
Database: d,
ctx: ctx,
txn: txn,
}, nil
}
func (d *Database) NewDatabaseTransaction(ctx context.Context) (*DatabaseTransaction, error) {
@ -572,8 +564,8 @@ func (d *Database) UpdatePresence(ctx context.Context, userID string, presence t
return pos, err
}
func (d *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
return d.Presence.GetPresenceForUser(ctx, nil, userID)
func (d *Database) GetPresences(ctx context.Context, userIDs []string) ([]*types.PresenceInternal, error) {
return d.Presence.GetPresenceForUsers(ctx, nil, userIDs)
}
func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) {

4
syncapi/storage/shared/storage_sync.go
View file

@ -596,8 +596,8 @@ func (d *DatabaseTransaction) GetUserUnreadNotificationCountsForRooms(ctx contex
return d.NotificationData.SelectUserUnreadCountsForRooms(ctx, d.txn, userID, roomIDs)
}
func (d *DatabaseTransaction) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
return d.Presence.GetPresenceForUser(ctx, d.txn, userID)
func (d *DatabaseTransaction) GetPresences(ctx context.Context, userIDs []string) ([]*types.PresenceInternal, error) {
return d.Presence.GetPresenceForUsers(ctx, d.txn, userIDs)
}
func (d *DatabaseTransaction) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {

48
syncapi/storage/sqlite3/presence_table.go
View file

@ -17,12 +17,14 @@ package sqlite3
import (
"context"
"database/sql"
"strings"
"time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
)
const presenceSchema = `
@ -62,9 +64,9 @@ const upsertPresenceFromSyncSQL = "" +
" RETURNING id"
const selectPresenceForUserSQL = "" +
"SELECT presence, status_msg, last_active_ts" +
"SELECT user_id, presence, status_msg, last_active_ts" +
" FROM syncapi_presence" +
" WHERE user_id = $1 LIMIT 1"
" WHERE user_id IN ($1)"
const selectMaxPresenceSQL = "" +
"SELECT COALESCE(MAX(id), 0) FROM syncapi_presence"
@ -134,20 +136,38 @@ func (p *presenceStatements) UpsertPresence(
return
}
// GetPresenceForUser returns the current presence of a user.
func (p *presenceStatements) GetPresenceForUser(
// GetPresenceForUsers returns the current presence for a list of users.
// If the user doesn't have a presence status yet, it is omitted from the response.
func (p *presenceStatements) GetPresenceForUsers(
ctx context.Context, txn *sql.Tx,
userID string,
) (*types.PresenceInternal, error) {
result := &types.PresenceInternal{
UserID: userID,
userIDs []string,
) ([]*types.PresenceInternal, error) {
qry := strings.Replace(selectPresenceForUserSQL, "($1)", sqlutil.QueryVariadic(len(userIDs)), 1)
prepStmt, err := p.db.Prepare(qry)
if err != nil {
return nil, err
}
stmt := sqlutil.TxStmt(txn, p.selectPresenceForUsersStmt)
err := stmt.QueryRowContext(ctx, userID).Scan(&result.Presence, &result.ClientFields.StatusMsg, &result.LastActiveTS)
if err == sql.ErrNoRows {
return nil, nil
defer internal.CloseAndLogIfError(ctx, prepStmt, "GetPresenceForUsers: stmt.close() failed")
params := make([]interface{}, len(userIDs))
for i := range userIDs {
params[i] = userIDs[i]
}
rows, err := sqlutil.TxStmt(txn, prepStmt).QueryContext(ctx, params...)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "GetPresenceForUsers: rows.close() failed")
result := make([]*types.PresenceInternal, 0, len(userIDs))
for rows.Next() {
presence := &types.PresenceInternal{}
if err = rows.Scan(&presence.UserID, &presence.Presence, &presence.ClientFields.StatusMsg, &presence.LastActiveTS); err != nil {
return nil, err
}
presence.ClientFields.Presence = presence.Presence.String()
result = append(result, presence)
}
result.ClientFields.Presence = result.Presence.String()
return result, err
}

2
syncapi/storage/tables/interface.go
View file

@ -207,7 +207,7 @@ type Ignores interface {
type Presence interface {
UpsertPresence(ctx context.Context, txn *sql.Tx, userID string, statusMsg *string, presence types.Presence, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (pos types.StreamPosition, err error)
GetPresenceForUser(ctx context.Context, txn *sql.Tx, userID string) (presence *types.PresenceInternal, err error)
GetPresenceForUsers(ctx context.Context, txn *sql.Tx, userIDs []string) (presence []*types.PresenceInternal, err error)
GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error)
GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error)
}

136
syncapi/storage/tables/presence_table_test.go Normal file
View file

@ -0,0 +1,136 @@
package tables_test
import (
"context"
"database/sql"
"reflect"
"testing"
"time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/syncapi/storage/postgres"
"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
"github.com/matrix-org/dendrite/syncapi/storage/tables"
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/dendrite/test"
)
func mustPresenceTable(t *testing.T, dbType test.DBType) (tables.Presence, func()) {
t.Helper()
connStr, close := test.PrepareDBConnectionString(t, dbType)
db, err := sqlutil.Open(&config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, sqlutil.NewExclusiveWriter())
if err != nil {
t.Fatalf("failed to open db: %s", err)
}
var tab tables.Presence
switch dbType {
case test.DBTypePostgres:
tab, err = postgres.NewPostgresPresenceTable(db)
case test.DBTypeSQLite:
var stream sqlite3.StreamIDStatements
if err = stream.Prepare(db); err != nil {
t.Fatalf("failed to prepare stream stmts: %s", err)
}
tab, err = sqlite3.NewSqlitePresenceTable(db, &stream)
}
if err != nil {
t.Fatalf("failed to make new table: %s", err)
}
return tab, close
}
func TestPresence(t *testing.T) {
alice := test.NewUser(t)
bob := test.NewUser(t)
ctx := context.Background()
statusMsg := "Hello World!"
timestamp := gomatrixserverlib.AsTimestamp(time.Now())
var txn *sql.Tx
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, closeDB := mustPresenceTable(t, dbType)
defer closeDB()
// Insert some presences
pos, err := tab.UpsertPresence(ctx, txn, alice.ID, &statusMsg, types.PresenceOnline, timestamp, false)
if err != nil {
t.Error(err)
}
wantPos := types.StreamPosition(1)
if pos != wantPos {
t.Errorf("expected pos to be %d, got %d", wantPos, pos)
}
pos, err = tab.UpsertPresence(ctx, txn, bob.ID, &statusMsg, types.PresenceOnline, timestamp, false)
if err != nil {
t.Error(err)
}
wantPos = 2
if pos != wantPos {
t.Errorf("expected pos to be %d, got %d", wantPos, pos)
}
// verify the expected max presence ID
maxPos, err := tab.GetMaxPresenceID(ctx, txn)
if err != nil {
t.Error(err)
}
if maxPos != wantPos {
t.Errorf("expected max pos to be %d, got %d", wantPos, maxPos)
}
// This should increment the position
pos, err = tab.UpsertPresence(ctx, txn, bob.ID, &statusMsg, types.PresenceOnline, timestamp, true)
if err != nil {
t.Error(err)
}
wantPos = pos
if wantPos <= maxPos {
t.Errorf("expected pos to be %d incremented, got %d", wantPos, pos)
}
// This should return only Bobs status
presences, err := tab.GetPresenceAfter(ctx, txn, maxPos, gomatrixserverlib.EventFilter{Limit: 10})
if err != nil {
t.Error(err)
}
if c := len(presences); c > 1 {
t.Errorf("expected only one presence, got %d", c)
}
// Validate the response
wantPresence := &types.PresenceInternal{
UserID: bob.ID,
Presence: types.PresenceOnline,
StreamPos: wantPos,
LastActiveTS: timestamp,
ClientFields: types.PresenceClientResponse{
LastActiveAgo: 0,
Presence: types.PresenceOnline.String(),
StatusMsg: &statusMsg,
},
}
if !reflect.DeepEqual(wantPresence, presences[bob.ID]) {
t.Errorf("unexpected presence result:\n%+v, want\n%+v", presences[bob.ID], wantPresence)
}
// Try getting presences for existing and non-existing users
getUsers := []string{alice.ID, bob.ID, "@doesntexist:test"}
presencesForUsers, err := tab.GetPresenceForUsers(ctx, nil, getUsers)
if err != nil {
t.Error(err)
}
if len(presencesForUsers) >= len(getUsers) {
t.Errorf("expected less presences, but they are the same/more as requested: %d >= %d", len(presencesForUsers), len(getUsers))
}
})
}

80
syncapi/streams/stream_presence.go
View file

@ -17,6 +17,7 @@ package streams
import (
"context"
"encoding/json"
"fmt"
"sync"
"github.com/matrix-org/gomatrixserverlib"
@ -70,39 +71,25 @@ func (p *PresenceStreamProvider) IncrementalSync(
return from
}
if len(presences) == 0 {
getPresenceForUsers, err := p.getNeededUsersFromRequest(ctx, req, presences)
if err != nil {
req.Log.WithError(err).Error("getNeededUsersFromRequest failed")
return from
}
// Got no presence between range and no presence to get from the database
if len(getPresenceForUsers) == 0 && len(presences) == 0 {
return to
}
// add newly joined rooms user presences
newlyJoined := joinedRooms(req.Response, req.Device.UserID)
if len(newlyJoined) > 0 {
// TODO: Check if this is working better than before.
if err = p.notifier.LoadRooms(ctx, p.DB, newlyJoined); err != nil {
req.Log.WithError(err).Error("unable to refresh notifier lists")
return from
}
NewlyJoinedLoop:
for _, roomID := range newlyJoined {
roomUsers := p.notifier.JoinedUsers(roomID)
for i := range roomUsers {
// we already got a presence from this user
if _, ok := presences[roomUsers[i]]; ok {
continue
}
// Bear in mind that this might return nil, but at least populating
// a nil means that there's a map entry so we won't repeat this call.
presences[roomUsers[i]], err = snapshot.GetPresence(ctx, roomUsers[i])
if err != nil {
req.Log.WithError(err).Error("unable to query presence for user")
_ = snapshot.Rollback()
return from
}
if len(presences) > req.Filter.Presence.Limit {
break NewlyJoinedLoop
}
}
}
dbPresences, err := snapshot.GetPresences(ctx, getPresenceForUsers)
if err != nil {
req.Log.WithError(err).Error("unable to query presence for user")
_ = snapshot.Rollback()
return from
}
for _, presence := range dbPresences {
presences[presence.UserID] = presence
}
lastPos := from
@ -164,6 +151,39 @@ func (p *PresenceStreamProvider) IncrementalSync(
return lastPos
}
func (p *PresenceStreamProvider) getNeededUsersFromRequest(ctx context.Context, req *types.SyncRequest, presences map[string]*types.PresenceInternal) ([]string, error) {
getPresenceForUsers := []string{}
// Add presence for users which newly joined a room
for userID := range req.MembershipChanges {
if _, ok := presences[userID]; ok {
continue
}
getPresenceForUsers = append(getPresenceForUsers, userID)
}
// add newly joined rooms user presences
newlyJoined := joinedRooms(req.Response, req.Device.UserID)
if len(newlyJoined) == 0 {
return getPresenceForUsers, nil
}
// TODO: Check if this is working better than before.
if err := p.notifier.LoadRooms(ctx, p.DB, newlyJoined); err != nil {
return getPresenceForUsers, fmt.Errorf("unable to refresh notifier lists: %w", err)
}
for _, roomID := range newlyJoined {
roomUsers := p.notifier.JoinedUsers(roomID)
for i := range roomUsers {
// we already got a presence from this user
if _, ok := presences[roomUsers[i]]; ok {
continue
}
getPresenceForUsers = append(getPresenceForUsers, roomUsers[i])
}
}
return getPresenceForUsers, nil
}
func joinedRooms(res *types.Response, userID string) []string {
var roomIDs []string
for roomID, join := range res.Rooms.Join {

3
syncapi/streams/stream_receipt.go
View file

@ -87,8 +87,7 @@ func (p *ReceiptStreamProvider) IncrementalSync(
}
ev := gomatrixserverlib.ClientEvent{
Type: gomatrixserverlib.MReceipt,
RoomID: roomID,
Type: gomatrixserverlib.MReceipt,
}
content := make(map[string]ReceiptMRead)
for _, receipt := range receipts {

6
syncapi/sync/requestpool.go
View file

@ -145,12 +145,12 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
}
// ensure we also send the current status_msg to federated servers and not nil
dbPresence, err := db.GetPresence(context.Background(), userID)
dbPresence, err := db.GetPresences(context.Background(), []string{userID})
if err != nil && err != sql.ErrNoRows {
return
}
if dbPresence != nil {
newPresence.ClientFields = dbPresence.ClientFields
if len(dbPresence) > 0 && dbPresence[0] != nil {
newPresence.ClientFields = dbPresence[0].ClientFields
}
newPresence.ClientFields.Presence = presenceID.String()

4
syncapi/sync/requestpool_test.go
View file

@ -29,8 +29,8 @@ func (d dummyDB) UpdatePresence(ctx context.Context, userID string, presence typ
return 0, nil
}
func (d dummyDB) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
return &types.PresenceInternal{}, nil
func (d dummyDB) GetPresences(ctx context.Context, userID []string) ([]*types.PresenceInternal, error) {
return []*types.PresenceInternal{}, nil
}
func (d dummyDB) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {

7
syncapi/types/types.go
View file

@ -480,6 +480,13 @@ func (jr JoinResponse) MarshalJSON() ([]byte, error) {
if jr.Ephemeral != nil && len(jr.Ephemeral.Events) == 0 {
a.Ephemeral = nil
}
if jr.Ephemeral != nil {
// Remove the room_id from EDUs, as this seems to cause Element Web
// to trigger notifications - https://github.com/vector-im/element-web/issues/17263
for i := range jr.Ephemeral.Events {
jr.Ephemeral.Events[i].RoomID = ""
}
}
if jr.AccountData != nil && len(jr.AccountData.Events) == 0 {
a.AccountData = nil
}

100
syncapi/types/types_test.go
View file

@ -2,6 +2,7 @@ package types
import (
"encoding/json"
"reflect"
"testing"
"github.com/matrix-org/gomatrixserverlib"
@ -63,3 +64,102 @@ func TestNewInviteResponse(t *testing.T) {
t.Fatalf("Invite response didn't contain correct info")
}
}
func TestJoinResponse_MarshalJSON(t *testing.T) {
type fields struct {
Summary *Summary
State *ClientEvents
Timeline *Timeline
Ephemeral *ClientEvents
AccountData *ClientEvents
UnreadNotifications *UnreadNotifications
}
tests := []struct {
name string
fields fields
want []byte
wantErr bool
}{
{
name: "empty state is removed",
fields: fields{
State: &ClientEvents{},
},
want: []byte("{}"),
},
{
name: "empty accountdata is removed",
fields: fields{
AccountData: &ClientEvents{},
},
want: []byte("{}"),
},
{
name: "empty ephemeral is removed",
fields: fields{
Ephemeral: &ClientEvents{},
},
want: []byte("{}"),
},
{
name: "empty timeline is removed",
fields: fields{
Timeline: &Timeline{},
},
want: []byte("{}"),
},
{
name: "empty summary is removed",
fields: fields{
Summary: &Summary{},
},
want: []byte("{}"),
},
{
name: "unread notifications are removed, if everything else is empty",
fields: fields{
UnreadNotifications: &UnreadNotifications{},
},
want: []byte("{}"),
},
{
name: "unread notifications are NOT removed, if state is set",
fields: fields{
State: &ClientEvents{Events: []gomatrixserverlib.ClientEvent{{Content: []byte("{}")}}},
UnreadNotifications: &UnreadNotifications{NotificationCount: 1},
},
want: []byte(`{"state":{"events":[{"content":{},"type":""}]},"unread_notifications":{"highlight_count":0,"notification_count":1}}`),
},
{
name: "roomID is removed from EDUs",
fields: fields{
Ephemeral: &ClientEvents{
Events: []gomatrixserverlib.ClientEvent{
{RoomID: "!someRandomRoomID:test", Content: []byte("{}")},
},
},
},
want: []byte(`{"ephemeral":{"events":[{"content":{},"type":""}]}}`),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
jr := JoinResponse{
Summary: tt.fields.Summary,
State: tt.fields.State,
Timeline: tt.fields.Timeline,
Ephemeral: tt.fields.Ephemeral,
AccountData: tt.fields.AccountData,
UnreadNotifications: tt.fields.UnreadNotifications,
}
got, err := jr.MarshalJSON()
if (err != nil) != tt.wantErr {
t.Errorf("MarshalJSON() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("MarshalJSON() got = %v, want %v", string(got), string(tt.want))
}
})
}
}

2
test/testrig/base.go
View file

@ -108,7 +108,7 @@ func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nat
cfg.Global.JetStream.InMemory = true
cfg.SyncAPI.Fulltext.InMemory = true
cfg.FederationAPI.KeyPerspectives = nil
base := base.NewBaseDendrite(cfg, "Tests")
base := base.NewBaseDendrite(cfg, "Tests", base.DisableMetrics)
js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream)
return base, js, jc
}

23
userapi/consumers/roomserver.go
View file

@ -385,7 +385,6 @@ func (s *OutputRoomEventConsumer) localRoomMembers(ctx context.Context, roomID s
req := &rsapi.QueryMembershipsForRoomRequest{
RoomID: roomID,
JoinedOnly: true,
LocalOnly: true,
}
var res rsapi.QueryMembershipsForRoomResponse
@ -396,8 +395,23 @@ func (s *OutputRoomEventConsumer) localRoomMembers(ctx context.Context, roomID s
}
var members []*localMembership
var ntotal int
for _, event := range res.JoinEvents {
// Filter out invalid join events
if event.StateKey == nil {
continue
}
if *event.StateKey == "" {
continue
}
_, serverName, err := gomatrixserverlib.SplitID('@', *event.StateKey)
if err != nil {
log.WithError(err).Error("failed to get servername from statekey")
continue
}
// Only get memberships for our server
if serverName != s.serverName {
continue
}
member, err := newLocalMembership(&event)
if err != nil {
log.WithError(err).Errorf("Parsing MemberContent")
@ -410,11 +424,10 @@ func (s *OutputRoomEventConsumer) localRoomMembers(ctx context.Context, roomID s
continue
}
ntotal++
members = append(members, member)
}
return members, ntotal, nil
return members, len(res.JoinEvents), nil
}
// roomName returns the name in the event (if type==m.room.name), or
@ -641,7 +654,7 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event *
if rule == nil {
// SPEC: If no rules match an event, the homeserver MUST NOT
// notify the Push Gateway for that event.
return nil, err
return nil, nil
}
log.WithFields(log.Fields{

71
userapi/inthttp/server.go
View file

@ -16,176 +16,177 @@ package inthttp
import (
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/userapi/api"
)
// nolint: gocyclo
func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
addRoutesLoginToken(internalAPIMux, s)
func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI, enableMetrics bool) {
addRoutesLoginToken(internalAPIMux, s, enableMetrics)
internalAPIMux.Handle(
PerformAccountCreationPath,
httputil.MakeInternalRPCAPI("UserAPIPerformAccountCreation", s.PerformAccountCreation),
httputil.MakeInternalRPCAPI("UserAPIPerformAccountCreation", enableMetrics, s.PerformAccountCreation),
)
internalAPIMux.Handle(
PerformPasswordUpdatePath,
httputil.MakeInternalRPCAPI("UserAPIPerformPasswordUpdate", s.PerformPasswordUpdate),
httputil.MakeInternalRPCAPI("UserAPIPerformPasswordUpdate", enableMetrics, s.PerformPasswordUpdate),
)
internalAPIMux.Handle(
PerformDeviceCreationPath,
httputil.MakeInternalRPCAPI("UserAPIPerformDeviceCreation", s.PerformDeviceCreation),
httputil.MakeInternalRPCAPI("UserAPIPerformDeviceCreation", enableMetrics, s.PerformDeviceCreation),
)
internalAPIMux.Handle(
PerformLastSeenUpdatePath,
httputil.MakeInternalRPCAPI("UserAPIPerformLastSeenUpdate", s.PerformLastSeenUpdate),
httputil.MakeInternalRPCAPI("UserAPIPerformLastSeenUpdate", enableMetrics, s.PerformLastSeenUpdate),
)
internalAPIMux.Handle(
PerformDeviceUpdatePath,
httputil.MakeInternalRPCAPI("UserAPIPerformDeviceUpdate", s.PerformDeviceUpdate),
httputil.MakeInternalRPCAPI("UserAPIPerformDeviceUpdate", enableMetrics, s.PerformDeviceUpdate),
)
internalAPIMux.Handle(
PerformDeviceDeletionPath,
httputil.MakeInternalRPCAPI("UserAPIPerformDeviceDeletion", s.PerformDeviceDeletion),
httputil.MakeInternalRPCAPI("UserAPIPerformDeviceDeletion", enableMetrics, s.PerformDeviceDeletion),
)
internalAPIMux.Handle(
PerformAccountDeactivationPath,
httputil.MakeInternalRPCAPI("UserAPIPerformAccountDeactivation", s.PerformAccountDeactivation),
httputil.MakeInternalRPCAPI("UserAPIPerformAccountDeactivation", enableMetrics, s.PerformAccountDeactivation),
)
internalAPIMux.Handle(
PerformOpenIDTokenCreationPath,
httputil.MakeInternalRPCAPI("UserAPIPerformOpenIDTokenCreation", s.PerformOpenIDTokenCreation),
httputil.MakeInternalRPCAPI("UserAPIPerformOpenIDTokenCreation", enableMetrics, s.PerformOpenIDTokenCreation),
)
internalAPIMux.Handle(
QueryProfilePath,
httputil.MakeInternalRPCAPI("UserAPIQueryProfile", s.QueryProfile),
httputil.MakeInternalRPCAPI("UserAPIQueryProfile", enableMetrics, s.QueryProfile),
)
internalAPIMux.Handle(
QueryAccessTokenPath,
httputil.MakeInternalRPCAPI("UserAPIQueryAccessToken", s.QueryAccessToken),
httputil.MakeInternalRPCAPI("UserAPIQueryAccessToken", enableMetrics, s.QueryAccessToken),
)
internalAPIMux.Handle(
QueryDevicesPath,
httputil.MakeInternalRPCAPI("UserAPIQueryDevices", s.QueryDevices),
httputil.MakeInternalRPCAPI("UserAPIQueryDevices", enableMetrics, s.QueryDevices),
)
internalAPIMux.Handle(
QueryAccountDataPath,
httputil.MakeInternalRPCAPI("UserAPIQueryAccountData", s.QueryAccountData),
httputil.MakeInternalRPCAPI("UserAPIQueryAccountData", enableMetrics, s.QueryAccountData),
)
internalAPIMux.Handle(
QueryDeviceInfosPath,
httputil.MakeInternalRPCAPI("UserAPIQueryDeviceInfos", s.QueryDeviceInfos),
httputil.MakeInternalRPCAPI("UserAPIQueryDeviceInfos", enableMetrics, s.QueryDeviceInfos),
)
internalAPIMux.Handle(
QuerySearchProfilesPath,
httputil.MakeInternalRPCAPI("UserAPIQuerySearchProfiles", s.QuerySearchProfiles),
httputil.MakeInternalRPCAPI("UserAPIQuerySearchProfiles", enableMetrics, s.QuerySearchProfiles),
)
internalAPIMux.Handle(
QueryOpenIDTokenPath,
httputil.MakeInternalRPCAPI("UserAPIQueryOpenIDToken", s.QueryOpenIDToken),
httputil.MakeInternalRPCAPI("UserAPIQueryOpenIDToken", enableMetrics, s.QueryOpenIDToken),
)
internalAPIMux.Handle(
InputAccountDataPath,
httputil.MakeInternalRPCAPI("UserAPIInputAccountData", s.InputAccountData),
httputil.MakeInternalRPCAPI("UserAPIInputAccountData", enableMetrics, s.InputAccountData),
)
internalAPIMux.Handle(
QueryKeyBackupPath,
httputil.MakeInternalRPCAPI("UserAPIQueryKeyBackup", s.QueryKeyBackup),
httputil.MakeInternalRPCAPI("UserAPIQueryKeyBackup", enableMetrics, s.QueryKeyBackup),
)
internalAPIMux.Handle(
PerformKeyBackupPath,
httputil.MakeInternalRPCAPI("UserAPIPerformKeyBackup", s.PerformKeyBackup),
httputil.MakeInternalRPCAPI("UserAPIPerformKeyBackup", enableMetrics, s.PerformKeyBackup),
)
internalAPIMux.Handle(
QueryNotificationsPath,
httputil.MakeInternalRPCAPI("UserAPIQueryNotifications", s.QueryNotifications),
httputil.MakeInternalRPCAPI("UserAPIQueryNotifications", enableMetrics, s.QueryNotifications),
)
internalAPIMux.Handle(
PerformPusherSetPath,
httputil.MakeInternalRPCAPI("UserAPIPerformPusherSet", s.PerformPusherSet),
httputil.MakeInternalRPCAPI("UserAPIPerformPusherSet", enableMetrics, s.PerformPusherSet),
)
internalAPIMux.Handle(
PerformPusherDeletionPath,
httputil.MakeInternalRPCAPI("UserAPIPerformPusherDeletion", s.PerformPusherDeletion),
httputil.MakeInternalRPCAPI("UserAPIPerformPusherDeletion", enableMetrics, s.PerformPusherDeletion),
)
internalAPIMux.Handle(
QueryPushersPath,
httputil.MakeInternalRPCAPI("UserAPIQueryPushers", s.QueryPushers),
httputil.MakeInternalRPCAPI("UserAPIQueryPushers", enableMetrics, s.QueryPushers),
)
internalAPIMux.Handle(
PerformPushRulesPutPath,
httputil.MakeInternalRPCAPI("UserAPIPerformPushRulesPut", s.PerformPushRulesPut),
httputil.MakeInternalRPCAPI("UserAPIPerformPushRulesPut", enableMetrics, s.PerformPushRulesPut),
)
internalAPIMux.Handle(
QueryPushRulesPath,
httputil.MakeInternalRPCAPI("UserAPIQueryPushRules", s.QueryPushRules),
httputil.MakeInternalRPCAPI("UserAPIQueryPushRules", enableMetrics, s.QueryPushRules),
)
internalAPIMux.Handle(
PerformSetAvatarURLPath,
httputil.MakeInternalRPCAPI("UserAPIPerformSetAvatarURL", s.SetAvatarURL),
httputil.MakeInternalRPCAPI("UserAPIPerformSetAvatarURL", enableMetrics, s.SetAvatarURL),
)
internalAPIMux.Handle(
QueryNumericLocalpartPath,
httputil.MakeInternalRPCAPI("UserAPIQueryNumericLocalpart", s.QueryNumericLocalpart),
httputil.MakeInternalRPCAPI("UserAPIQueryNumericLocalpart", enableMetrics, s.QueryNumericLocalpart),
)
internalAPIMux.Handle(
QueryAccountAvailabilityPath,
httputil.MakeInternalRPCAPI("UserAPIQueryAccountAvailability", s.QueryAccountAvailability),
httputil.MakeInternalRPCAPI("UserAPIQueryAccountAvailability", enableMetrics, s.QueryAccountAvailability),
)
internalAPIMux.Handle(
QueryAccountByPasswordPath,
httputil.MakeInternalRPCAPI("UserAPIQueryAccountByPassword", s.QueryAccountByPassword),
httputil.MakeInternalRPCAPI("UserAPIQueryAccountByPassword", enableMetrics, s.QueryAccountByPassword),
)
internalAPIMux.Handle(
PerformSetDisplayNamePath,
httputil.MakeInternalRPCAPI("UserAPISetDisplayName", s.SetDisplayName),
httputil.MakeInternalRPCAPI("UserAPISetDisplayName", enableMetrics, s.SetDisplayName),
)
internalAPIMux.Handle(
QueryLocalpartForThreePIDPath,
httputil.MakeInternalRPCAPI("UserAPIQueryLocalpartForThreePID", s.QueryLocalpartForThreePID),
httputil.MakeInternalRPCAPI("UserAPIQueryLocalpartForThreePID", enableMetrics, s.QueryLocalpartForThreePID),
)
internalAPIMux.Handle(
QueryThreePIDsForLocalpartPath,
httputil.MakeInternalRPCAPI("UserAPIQueryThreePIDsForLocalpart", s.QueryThreePIDsForLocalpart),
httputil.MakeInternalRPCAPI("UserAPIQueryThreePIDsForLocalpart", enableMetrics, s.QueryThreePIDsForLocalpart),
)
internalAPIMux.Handle(
PerformForgetThreePIDPath,
httputil.MakeInternalRPCAPI("UserAPIPerformForgetThreePID", s.PerformForgetThreePID),
httputil.MakeInternalRPCAPI("UserAPIPerformForgetThreePID", enableMetrics, s.PerformForgetThreePID),
)
internalAPIMux.Handle(
PerformSaveThreePIDAssociationPath,
httputil.MakeInternalRPCAPI("UserAPIPerformSaveThreePIDAssociation", s.PerformSaveThreePIDAssociation),
httputil.MakeInternalRPCAPI("UserAPIPerformSaveThreePIDAssociation", enableMetrics, s.PerformSaveThreePIDAssociation),
)
}

9
userapi/inthttp/server_logintoken.go
View file

@ -16,24 +16,25 @@ package inthttp
import (
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/userapi/api"
)
// addRoutesLoginToken adds routes for all login token API calls.
func addRoutesLoginToken(internalAPIMux *mux.Router, s api.UserInternalAPI) {
func addRoutesLoginToken(internalAPIMux *mux.Router, s api.UserInternalAPI, enableMetrics bool) {
internalAPIMux.Handle(
PerformLoginTokenCreationPath,
httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenCreation", s.PerformLoginTokenCreation),
httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenCreation", enableMetrics, s.PerformLoginTokenCreation),
)
internalAPIMux.Handle(
PerformLoginTokenDeletionPath,
httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenDeletion", s.PerformLoginTokenDeletion),
httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenDeletion", enableMetrics, s.PerformLoginTokenDeletion),
)
internalAPIMux.Handle(
QueryLoginTokenPath,
httputil.MakeInternalRPCAPI("UserAPIQueryLoginToken", s.QueryLoginToken),
httputil.MakeInternalRPCAPI("UserAPIQueryLoginToken", enableMetrics, s.QueryLoginToken),
)
}

4
userapi/userapi.go
View file

@ -37,8 +37,8 @@ import (
// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
// on the given input API.
func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI) {
inthttp.AddRoutes(router, intAPI)
func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI, enableMetrics bool) {
inthttp.AddRoutes(router, intAPI, enableMetrics)
}
// NewInternalAPI returns a concerete implementation of the internal API. Callers

55
userapi/userapi_test.go
View file

@ -27,14 +27,13 @@ import (
"golang.org/x/crypto/bcrypt"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/dendrite/userapi/inthttp"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/internal"
"github.com/matrix-org/dendrite/userapi/inthttp"
"github.com/matrix-org/dendrite/userapi/storage"
)
@ -79,19 +78,6 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType) (ap
func TestQueryProfile(t *testing.T) {
aliceAvatarURL := "mxc://example.com/alice"
aliceDisplayName := "Alice"
// only one DBType, since userapi.AddInternalRoutes complains about multiple prometheus counters added
userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, test.DBTypeSQLite)
defer close()
_, err := accountDB.CreateAccount(context.TODO(), "alice", serverName, "foobar", "", api.AccountTypeUser)
if err != nil {
t.Fatalf("failed to make account: %s", err)
}
if _, _, err := accountDB.SetAvatarURL(context.TODO(), "alice", serverName, aliceAvatarURL); err != nil {
t.Fatalf("failed to set avatar url: %s", err)
}
if _, _, err := accountDB.SetDisplayName(context.TODO(), "alice", serverName, aliceDisplayName); err != nil {
t.Fatalf("failed to set display name: %s", err)
}
testCases := []struct {
req api.QueryProfileRequest
@ -142,19 +128,34 @@ func TestQueryProfile(t *testing.T) {
}
}
t.Run("HTTP API", func(t *testing.T) {
router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
userapi.AddInternalRoutes(router, userAPI)
apiURL, cancel := test.ListenAndServe(t, router, false)
defer cancel()
httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{})
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType)
defer close()
_, err := accountDB.CreateAccount(context.TODO(), "alice", serverName, "foobar", "", api.AccountTypeUser)
if err != nil {
t.Fatalf("failed to create HTTP client")
t.Fatalf("failed to make account: %s", err)
}
runCases(httpAPI, true)
})
t.Run("Monolith", func(t *testing.T) {
runCases(userAPI, false)
if _, _, err := accountDB.SetAvatarURL(context.TODO(), "alice", serverName, aliceAvatarURL); err != nil {
t.Fatalf("failed to set avatar url: %s", err)
}
if _, _, err := accountDB.SetDisplayName(context.TODO(), "alice", serverName, aliceDisplayName); err != nil {
t.Fatalf("failed to set display name: %s", err)
}
t.Run("HTTP API", func(t *testing.T) {
router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
userapi.AddInternalRoutes(router, userAPI, false)
apiURL, cancel := test.ListenAndServe(t, router, false)
defer cancel()
httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{})
if err != nil {
t.Fatalf("failed to create HTTP client")
}
runCases(httpAPI, true)
})
t.Run("Monolith", func(t *testing.T) {
runCases(userAPI, false)
})
})
}

119
userapi/util/notify_test.go Normal file
View file

@ -0,0 +1,119 @@
package util_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"golang.org/x/crypto/bcrypt"
"github.com/matrix-org/dendrite/internal/pushgateway"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage"
userUtil "github.com/matrix-org/dendrite/userapi/util"
)
func TestNotifyUserCountsAsync(t *testing.T) {
alice := test.NewUser(t)
aliceLocalpart, serverName, err := gomatrixserverlib.SplitID('@', alice.ID)
if err != nil {
t.Error(err)
}
ctx := context.Background()
// Create a test room, just used to provide events
room := test.NewRoom(t, alice)
dummyEvent := room.Events()[len(room.Events())-1]
appID := util.RandomString(8)
pushKey := util.RandomString(8)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
receivedRequest := make(chan bool, 1)
// create a test server which responds to our /notify call
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var data pushgateway.NotifyRequest
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
t.Error(err)
}
notification := data.Notification
// Validate the request
if notification.Counts == nil {
t.Fatal("no unread notification counts in request")
}
if unread := notification.Counts.Unread; unread != 1 {
t.Errorf("expected one unread notification, got %d", unread)
}
if len(notification.Devices) == 0 {
t.Fatal("expected devices in request")
}
// We only created one push device, so access it directly
device := notification.Devices[0]
if device.AppID != appID {
t.Errorf("unexpected app_id: %s, want %s", device.AppID, appID)
}
if device.PushKey != pushKey {
t.Errorf("unexpected push_key: %s, want %s", device.PushKey, pushKey)
}
// Return empty result, otherwise the call is handled as failed
if _, err := w.Write([]byte("{}")); err != nil {
t.Error(err)
}
close(receivedRequest)
}))
defer srv.Close()
// Create DB and Dendrite base
connStr, close := test.PrepareDBConnectionString(t, dbType)
defer close()
base, _, _ := testrig.Base(nil)
defer base.Close()
db, err := storage.NewUserAPIDatabase(base, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, "test", bcrypt.MinCost, 0, 0, "")
if err != nil {
t.Error(err)
}
// Prepare pusher with our test server URL
if err := db.UpsertPusher(ctx, api.Pusher{
Kind: api.HTTPKind,
AppID: appID,
PushKey: pushKey,
Data: map[string]interface{}{
"url": srv.URL,
},
}, aliceLocalpart, serverName); err != nil {
t.Error(err)
}
// Insert a dummy event
if err := db.InsertNotification(ctx, aliceLocalpart, serverName, dummyEvent.EventID(), 0, nil, &api.Notification{
Event: gomatrixserverlib.HeaderedToClientEvent(dummyEvent, gomatrixserverlib.FormatAll),
}); err != nil {
t.Error(err)
}
// Notify the user about a new notification
if err := userUtil.NotifyUserCountsAsync(ctx, pushgateway.NewHTTPClient(true), aliceLocalpart, serverName, db); err != nil {
t.Error(err)
}
select {
case <-time.After(time.Second * 5):
t.Error("timed out waiting for response")
case <-receivedRequest:
}
})
}

10
userapi/util/phonehomestats.go
View file

@ -97,12 +97,10 @@ func (p *phoneHomeStats) collect() {
// configuration information
p.stats["federation_disabled"] = p.cfg.Global.DisableFederation
p.stats["nats_embedded"] = true
p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory
if len(p.cfg.Global.JetStream.Addresses) > 0 {
p.stats["nats_embedded"] = false
p.stats["nats_in_memory"] = false // probably
}
natsEmbedded := len(p.cfg.Global.JetStream.Addresses) == 0
p.stats["nats_embedded"] = natsEmbedded
p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory && natsEmbedded
if len(p.cfg.Logging) > 0 {
p.stats["log_level"] = p.cfg.Logging[0].Level
} else {

84
userapi/util/phonehomestats_test.go Normal file
View file

@ -0,0 +1,84 @@
package util
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"golang.org/x/crypto/bcrypt"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi/storage"
)
func TestCollect(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
b, _, _ := testrig.Base(nil)
connStr, closeDB := test.PrepareDBConnectionString(t, dbType)
defer closeDB()
db, err := storage.NewUserAPIDatabase(b, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, "localhost", bcrypt.MinCost, 1000, 1000, "")
if err != nil {
t.Error(err)
}
receivedRequest := make(chan struct{}, 1)
// create a test server which responds to our call
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var data map[string]interface{}
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
t.Error(err)
}
defer r.Body.Close()
if _, err := w.Write([]byte("{}")); err != nil {
t.Error(err)
}
// verify the received data matches our expectations
dbEngine, ok := data["database_engine"]
if !ok {
t.Errorf("missing database_engine in JSON request: %+v", data)
}
version, ok := data["version"]
if !ok {
t.Errorf("missing version in JSON request: %+v", data)
}
if version != internal.VersionString() {
t.Errorf("unexpected version: %q, expected %q", version, internal.VersionString())
}
switch {
case dbType == test.DBTypeSQLite && dbEngine != "SQLite":
t.Errorf("unexpected database_engine: %s", dbEngine)
case dbType == test.DBTypePostgres && dbEngine != "Postgres":
t.Errorf("unexpected database_engine: %s", dbEngine)
}
close(receivedRequest)
}))
defer srv.Close()
b.Cfg.Global.ReportStats.Endpoint = srv.URL
stats := phoneHomeStats{
prevData: timestampToRUUsage{},
serverName: "localhost",
startTime: time.Now(),
cfg: b.Cfg,
db: db,
isMonolith: false,
client: &http.Client{Timeout: time.Second},
}
stats.collect()
select {
case <-time.After(time.Second * 5):
t.Error("timed out waiting for response")
case <-receivedRequest:
}
})
}