mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-29 01:33:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add rate limiting to media api

Browse source
This commit is contained in:
Till Faelligen 2021-10-10 12:11:14 +02:00
parent 31d842f262
commit 07036a3a67
6 changed files with 64 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
clientapi/routing/routing.go
View file

@ -61,7 +61,7 @@ func Setup(
extRoomsProvider api.ExtraPublicRoomsProvider,
mscCfg *config.MSCs,
) {
rateLimits := newRateLimits(&cfg.RateLimiting)
rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg)
unstableFeatures := map[string]bool{
@ -127,7 +127,7 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/join/{roomIDOrAlias}",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -143,7 +143,7 @@ func Setup(
if mscCfg.Enabled("msc2753") {
r0mux.Handle("/peek/{roomIDOrAlias}",
httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -163,7 +163,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/join",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -177,7 +177,7 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/leave",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -211,7 +211,7 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/invite",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -329,14 +329,14 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return Register(req, userAPI, accountDB, cfg)
})).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return RegisterAvailable(req, cfg, accountDB)
@ -410,7 +410,7 @@ func Setup(
r0mux.Handle("/rooms/{roomID}/typing/{userID}",
httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -466,7 +466,7 @@ func Setup(
r0mux.Handle("/account/whoami",
httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return Whoami(req, device)
@ -475,7 +475,7 @@ func Setup(
r0mux.Handle("/account/password",
httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return Password(req, userAPI, accountDB, device, cfg)
@ -484,7 +484,7 @@ func Setup(
r0mux.Handle("/account/deactivate",
httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return Deactivate(req, userInteractiveAuth, userAPI, device)
@ -495,7 +495,7 @@ func Setup(
r0mux.Handle("/login",
httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return Login(req, accountDB, userAPI, cfg)
@ -552,7 +552,7 @@ func Setup(
r0mux.Handle("/profile/{userID}/avatar_url",
httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -577,7 +577,7 @@ func Setup(
r0mux.Handle("/profile/{userID}/displayname",
httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -617,7 +617,7 @@ func Setup(
// Element logs get flooded unless this is handled
r0mux.Handle("/presence/{userID}/status",
httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
// TODO: Set presence (probably the responsibility of a presence server not clientapi)
@ -630,7 +630,7 @@ func Setup(
r0mux.Handle("/voip/turnServer",
httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return RequestTurnServer(req, device, cfg)
@ -709,7 +709,7 @@ func Setup(
r0mux.Handle("/user/{userID}/openid/request_token",
httputil.MakeAuthAPI("openid_request_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -722,7 +722,7 @@ func Setup(
r0mux.Handle("/user_directory/search",
httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
postContent := struct {
@ -767,7 +767,7 @@ func Setup(
r0mux.Handle("/rooms/{roomID}/read_markers",
httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -780,7 +780,7 @@ func Setup(
r0mux.Handle("/rooms/{roomID}/forget",
httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -884,7 +884,7 @@ func Setup(
r0mux.Handle("/capabilities",
httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return GetCapabilities(req, rsAPI)
@ -1100,7 +1100,7 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
if r := rateLimits.Limit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))

2
cmd/dendrite-polylith-multi/personalities/mediaapi.go
View file

@ -24,7 +24,7 @@ func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
client := base.CreateClient()
mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client)
mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, &base.Cfg.ClientAPI.RateLimiting, userAPI, client)
base.SetupAndServeHTTP(
base.Cfg.MediaAPI.InternalAPI.Listen,

22
clientapi/routing/rate_limiting.go → internal/httputil/rate_limiting.go
View file

@ -1,4 +1,4 @@
package routing
package httputil
import (
"net/http"
@ -10,7 +10,7 @@ import (
"github.com/matrix-org/util"
)
type rateLimits struct {
type RateLimits struct {
limits map[string]chan struct{}
limitsMutex sync.RWMutex
cleanMutex sync.RWMutex
@ -19,8 +19,8 @@ type rateLimits struct {
cooloffDuration time.Duration
}
func newRateLimits(cfg *config.RateLimiting) *rateLimits {
l := &rateLimits{
func NewRateLimits(cfg *config.RateLimiting) *RateLimits {
l := &RateLimits{
limits: make(map[string]chan struct{}),
enabled: cfg.Enabled,
requestThreshold: cfg.Threshold,
@ -32,7 +32,7 @@ func newRateLimits(cfg *config.RateLimiting) *rateLimits {
return l
}
func (l *rateLimits) clean() {
func (l *RateLimits) clean() {
for {
// On a 30 second interval, we'll take an exclusive write
// lock of the entire map and see if any of the channels are
@ -52,7 +52,7 @@ func (l *rateLimits) clean() {
}
}
func (l *rateLimits) rateLimit(req *http.Request) *util.JSONResponse {
func (l *RateLimits) Limit(req *http.Request) *util.JSONResponse {
// If rate limiting is disabled then do nothing.
if !l.enabled {
return nil
@ -74,23 +74,23 @@ func (l *rateLimits) rateLimit(req *http.Request) *util.JSONResponse {
// Look up the caller's channel, if they have one.
l.limitsMutex.RLock()
rateLimit, ok := l.limits[caller]
RateLimit, ok := l.limits[caller]
l.limitsMutex.RUnlock()
// If the caller doesn't have a channel, create one and write it
// back to the map.
if !ok {
rateLimit = make(chan struct{}, l.requestThreshold)
RateLimit = make(chan struct{}, l.requestThreshold)
l.limitsMutex.Lock()
l.limits[caller] = rateLimit
l.limits[caller] = RateLimit
l.limitsMutex.Unlock()
}
// Check if the user has got free resource slots for this request.
// If they don't then we'll return an error.
select {
case rateLimit <- struct{}{}:
case RateLimit <- struct{}{}:
default:
// We hit the rate limit. Tell the client to back off.
return &util.JSONResponse{
@ -103,7 +103,7 @@ func (l *rateLimits) rateLimit(req *http.Request) *util.JSONResponse {
// channel. This will free up space in the channel for new requests.
go func() {
<-time.After(l.cooloffDuration)
<-rateLimit
<-RateLimit
}()
return nil
}

6
mediaapi/mediaapi.go
View file

@ -26,7 +26,9 @@ import (
// AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component.
func AddPublicRoutes(
router *mux.Router, cfg *config.MediaAPI,
router *mux.Router,
cfg *config.MediaAPI,
rateLimit *config.RateLimiting,
userAPI userapi.UserInternalAPI,
client *gomatrixserverlib.Client,
) {
@ -36,6 +38,6 @@ func AddPublicRoutes(
}
routing.Setup(
router, cfg, mediaDB, userAPI, client,
router, cfg, rateLimit, mediaDB, userAPI, client,
)
}

27
mediaapi/routing/routing.go
View file

@ -15,6 +15,7 @@
package routing
import (
"encoding/json"
"net/http"
"strings"
@ -45,10 +46,13 @@ type configResponse struct {
func Setup(
publicAPIMux *mux.Router,
cfg *config.MediaAPI,
rateLimit *config.RateLimiting,
db storage.Database,
userAPI userapi.UserInternalAPI,
client *gomatrixserverlib.Client,
) {
rateLimits := httputil.NewRateLimits(rateLimit)
r0mux := publicAPIMux.PathPrefix("/r0").Subrouter()
v1mux := publicAPIMux.PathPrefix("/v1").Subrouter()
@ -59,11 +63,17 @@ func Setup(
uploadHandler := httputil.MakeAuthAPI(
"upload", userAPI,
func(req *http.Request, dev *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return Upload(req, cfg, dev, db, activeThumbnailGeneration)
},
)
configHandler := httputil.MakeAuthAPI("config", userAPI, func(request *http.Request, device *userapi.Device) util.JSONResponse {
configHandler := httputil.MakeAuthAPI("config", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
return *r
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: configResponse{UploadSize: *cfg.MaxFileSizeBytes},
@ -78,20 +88,21 @@ func Setup(
MXCToResult: map[string]*types.RemoteRequestResult{},
}
downloadHandler := makeDownloadAPI("download", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration)
downloadHandler := makeDownloadAPI("download", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration)
r0mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)
v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed
v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed
r0mux.Handle("/thumbnail/{serverName}/{mediaId}",
makeDownloadAPI("thumbnail", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration),
makeDownloadAPI("thumbnail", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration),
).Methods(http.MethodGet, http.MethodOptions)
}
func makeDownloadAPI(
name string,
cfg *config.MediaAPI,
rateLimits *httputil.RateLimits,
db storage.Database,
client *gomatrixserverlib.Client,
activeRemoteRequests *types.ActiveRemoteRequests,
@ -112,6 +123,16 @@ func makeDownloadAPI(
// Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors
w.Header().Set("Content-Type", "application/json")
// Ratelimit requests
if r := rateLimits.Limit(req); r != nil {
rw := json.NewEncoder(w)
w.WriteHeader(http.StatusTooManyRequests)
if err := rw.Encode(r); err != nil {
w.WriteHeader(http.StatusInternalServerError)
}
return
}
vars, _ := httputil.URLDecodeMapValues(mux.Vars(req))
serverName := gomatrixserverlib.ServerName(vars["serverName"])

2
setup/monolith.go
View file

@ -70,7 +70,7 @@ func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ss
m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI,
m.EDUInternalAPI, m.KeyAPI, &m.Config.MSCs, nil,
)
mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, m.UserAPI, m.Client)
mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting, m.UserAPI, m.Client)
syncapi.AddPublicRoutes(
process, csMux, m.UserAPI, m.RoomserverAPI,
m.KeyAPI, m.FedClient, &m.Config.SyncAPI,