sigb.us/dendrite
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'main' into neilalexander/purgeroom

Browse source
This commit is contained in:
Neil Alexander 2022-08-24 14:47:11 +01:00 committed by GitHub
parent e31469682d 522bd2999f
commit 965f532bb0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 75 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/dendrite.yml vendored
View file

@ -376,6 +376,8 @@ jobs:
# Build initial Dendrite image
- run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
working-directory: dendrite
env:
DOCKER_BUILDKIT: 1
# Run Complement
- run: |

20
build/scripts/Complement.Dockerfile
View file

@ -1,3 +1,5 @@
#syntax=docker/dockerfile:1.2
FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y sqlite3
WORKDIR /build
@ -8,14 +10,12 @@ RUN mkdir /dendrite
# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
COPY go.mod .
COPY go.sum .
RUN go mod download
COPY . .
RUN go build -o /dendrite ./cmd/dendrite-monolith-server
RUN go build -o /dendrite ./cmd/generate-keys
RUN go build -o /dendrite ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server
WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem
@ -26,7 +26,7 @@ EXPOSE 8008 8448
# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
CMD ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

23
build/scripts/ComplementLocal.Dockerfile
View file

@ -1,3 +1,5 @@
#syntax=docker/dockerfile:1.2
# A local development Complement dockerfile, to be used with host mounts
# /cache -> Contains the entire dendrite code at Dockerfile build time. Builds binaries but only keeps the generate-* ones. Pre-compilation saves time.
# /dendrite -> Host-mounted sources
@ -9,11 +11,10 @@
FROM golang:1.18-stretch
RUN apt-get update && apt-get install -y sqlite3
WORKDIR /runtime
ENV SERVER_NAME=localhost
EXPOSE 8008 8448
WORKDIR /runtime
# This script compiles Dendrite for us.
RUN echo '\
#!/bin/bash -eux \n\
@ -29,25 +30,23 @@ RUN echo '\
RUN echo '\
#!/bin/bash -eu \n\
./generate-keys --private-key matrix_key.pem \n\
./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
' > run.sh && chmod +x run.sh
WORKDIR /cache
# Pre-download deps; we don't need to do this if the GOPATH is mounted.
COPY go.mod .
COPY go.sum .
RUN go mod download
# Build the monolith in /cache - we won't actually use this but will rely on build artifacts to speed
# up the real compilation. Build the generate-* binaries in the true /runtime locations.
# If the generate-* source is changed, this dockerfile needs re-running.
COPY . .
RUN go build ./cmd/dendrite-monolith-server && go build -o /runtime ./cmd/generate-keys && go build -o /runtime ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /runtime ./cmd/generate-config && \
go build -o /runtime ./cmd/generate-keys
WORKDIR /runtime
CMD /runtime/compile.sh && /runtime/run.sh
CMD /runtime/compile.sh && exec /runtime/run.sh

20
build/scripts/ComplementPostgres.Dockerfile
View file

@ -1,3 +1,5 @@
#syntax=docker/dockerfile:1.2
FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
@ -26,14 +28,12 @@ RUN mkdir /dendrite
# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
COPY go.mod .
COPY go.sum .
RUN go mod download
COPY . .
RUN go build -o /dendrite ./cmd/dendrite-monolith-server
RUN go build -o /dendrite ./cmd/generate-keys
RUN go build -o /dendrite ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server
WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem
@ -45,10 +45,10 @@ EXPOSE 8008 8448
# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
# Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

9
cmd/create-account/main.go
View file

@ -66,10 +66,11 @@ var (
resetPassword = flag.Bool("reset-password", false, "Deprecated")
serverURL = flag.String("url", "https://localhost:8448", "The URL to connect to.")
validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
timeout = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
)
var cl = http.Client{
Timeout: time.Second * 10,
Timeout: time.Second * 30,
Transport: http.DefaultTransport,
}
@ -108,6 +109,8 @@ func main() {
logrus.Fatalln(err)
}
cl.Timeout = *timeout
accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
if err != nil {
logrus.Fatalln("Failed to create the account:", err.Error())
@ -124,8 +127,8 @@ type sharedSecretRegistrationRequest struct {
Admin bool `json:"admin"`
}
func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, admin bool) (accesToken string, err error) {
registerURL := fmt.Sprintf("%s/_synapse/admin/v1/register", serverURL)
func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, admin bool) (accessToken string, err error) {
registerURL := fmt.Sprintf("%s/_synapse/admin/v1/register", strings.Trim(serverURL, "/"))
nonceReq, err := http.NewRequest(http.MethodGet, registerURL, nil)
if err != nil {
return "", fmt.Errorf("unable to create http request: %w", err)

5
cmd/generate-keys/main.go
View file

@ -38,6 +38,7 @@ var (
authorityCertFile = flag.String("tls-authority-cert", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
authorityKeyFile = flag.String("tls-authority-key", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
serverName = flag.String("server", "", "Optional: Create TLS certificate/keys with this domain name set. Useful for integration testing.")
keySize = flag.Int("keysize", 4096, "Optional: Create TLS RSA private key with the given key size")
)
func main() {
@ -58,12 +59,12 @@ func main() {
log.Fatal("Zero or both of --tls-key and --tls-cert must be supplied")
}
if *authorityCertFile == "" && *authorityKeyFile == "" {
if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile); err != nil {
if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile, *keySize); err != nil {
panic(err)
}
} else {
// generate the TLS cert/key based on the authority given.
if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile); err != nil {
if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile, *keySize); err != nil {
panic(err)
}
}

7
federationapi/statistics/statistics.go
View file

@ -5,10 +5,11 @@ import (
"sync"
"time"
"github.com/matrix-org/dendrite/federationapi/storage"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"go.uber.org/atomic"
"github.com/matrix-org/dendrite/federationapi/storage"
)
// Statistics contains information about all of the remote federated
@ -126,13 +127,13 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
go func() {
until, ok := s.backoffUntil.Load().(time.Time)
if ok {
if ok && !until.IsZero() {
select {
case <-time.After(time.Until(until)):
case <-s.interrupt:
}
s.backoffStarted.Store(false)
}
s.backoffStarted.Store(false)
}()
}

8
roomserver/api/wrapper.go
View file

@ -19,6 +19,7 @@ import (
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
)
// SendEvents to the roomserver The events are written with KindNew.
@ -69,6 +70,13 @@ func SendEventWithState(
stateEventIDs[i] = stateEvents[i].EventID()
}
logrus.WithContext(ctx).WithFields(logrus.Fields{
"room_id": event.RoomID(),
"event_id": event.EventID(),
"outliers": len(ires),
"state_ids": len(stateEventIDs),
}).Infof("Submitting %q event to roomserver with state snapshot", event.Type())
ires = append(ires, InputRoomEvent{
Kind: kind,
Event: event,

11
roomserver/internal/helpers/helpers.go
View file

@ -254,8 +254,15 @@ func CheckServerAllowedToSeeEvent(
return false, err
}
default:
// Something else went wrong
return false, err
switch err.(type) {
case types.MissingStateError:
// If there's no state then we assume it's open visibility, as Synapse does:
// https://github.com/matrix-org/synapse/blob/aec87a0f9369a3015b2a53469f88d1de274e8b71/synapse/visibility.py#L654-L655
return true, nil
default:
// Something else went wrong
return false, err
}
}
return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil
}

4
roomserver/storage/postgres/events_table.go
View file

@ -74,7 +74,7 @@ const insertEventSQL = "" +
"INSERT INTO roomserver_events AS e (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids, depth, is_rejected)" +
" VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" +
" ON CONFLICT ON CONSTRAINT roomserver_event_id_unique DO UPDATE" +
" SET is_rejected = $8 WHERE e.event_id = $4 AND e.is_rejected = FALSE" +
" SET is_rejected = $8 WHERE e.event_id = $4 AND e.is_rejected = TRUE" +
" RETURNING event_nid, state_snapshot_nid"
const selectEventSQL = "" +
@ -346,7 +346,7 @@ func (s *eventStatements) BulkSelectStateAtEventByID(
// Genuine create events are the only case where it's OK to have no previous state.
isCreate := result.EventTypeNID == types.MRoomCreateNID && result.EventStateKeyNID == 1
if result.BeforeStateSnapshotNID == 0 && !isCreate {
return nil, types.MissingEventError(
return nil, types.MissingStateError(
fmt.Sprintf("storage: missing state for event NID %d", result.EventNID),
)
}

4
roomserver/storage/sqlite3/events_table.go
View file

@ -50,7 +50,7 @@ const insertEventSQL = `
INSERT INTO roomserver_events (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids, depth, is_rejected)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT DO UPDATE
SET is_rejected = $8 WHERE is_rejected = 0
SET is_rejected = $8 WHERE is_rejected = 1
RETURNING event_nid, state_snapshot_nid;
`
@ -362,7 +362,7 @@ func (s *eventStatements) BulkSelectStateAtEventByID(
// Genuine create events are the only case where it's OK to have no previous state.
isCreate := result.EventTypeNID == types.MRoomCreateNID && result.EventStateKeyNID == 1
if result.BeforeStateSnapshotNID == 0 && !isCreate {
return nil, types.MissingEventError(
return nil, types.MissingStateError(
fmt.Sprintf("storage: missing state for event NID %d", result.EventNID),
)
}

2
syncapi/internal/keychange.go
View file

@ -127,7 +127,7 @@ func DeviceListCatchup(
"from": offset,
"to": toOffset,
"response_offset": queryRes.Offset,
}).Debugf("QueryKeyChanges request result: %+v", res.DeviceLists)
}).Tracef("QueryKeyChanges request result: %+v", res.DeviceLists)
return types.StreamPosition(queryRes.Offset), hasNew, nil
}

2
test/http.go
View file

@ -68,7 +68,7 @@ func ListenAndServe(t *testing.T, router http.Handler, withTLS bool) (apiURL str
if withTLS {
certFile := filepath.Join(t.TempDir(), "dendrite.cert")
keyFile := filepath.Join(t.TempDir(), "dendrite.key")
err = NewTLSKey(keyFile, certFile)
err = NewTLSKey(keyFile, certFile, 1024)
if err != nil {
t.Errorf("failed to make TLS key: %s", err)
return

12
test/keys.go
View file

@ -69,8 +69,8 @@ func NewMatrixKey(matrixKeyPath string) (err error) {
const certificateDuration = time.Hour * 24 * 365 * 10
func generateTLSTemplate(dnsNames []string) (*rsa.PrivateKey, *x509.Certificate, error) {
priv, err := rsa.GenerateKey(rand.Reader, 4096)
func generateTLSTemplate(dnsNames []string, bitSize int) (*rsa.PrivateKey, *x509.Certificate, error) {
priv, err := rsa.GenerateKey(rand.Reader, bitSize)
if err != nil {
return nil, nil, err
}
@ -118,8 +118,8 @@ func writePrivateKey(tlsKeyPath string, priv *rsa.PrivateKey) error {
}
// NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
priv, template, err := generateTLSTemplate(nil)
func NewTLSKey(tlsKeyPath, tlsCertPath string, keySize int) error {
priv, template, err := generateTLSTemplate(nil, keySize)
if err != nil {
return err
}
@ -136,8 +136,8 @@ func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
return writePrivateKey(tlsKeyPath, priv)
}
func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string) error {
priv, template, err := generateTLSTemplate([]string{serverName})
func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string, keySize int) error {
priv, template, err := generateTLSTemplate([]string{serverName}, keySize)
if err != nil {
return err
}