started adding proxy_outbound with support for ENV HTTP_PROXY and exclusion of specified destination networks or addresses from proxying

appservice/appservice.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"net/http"
+	"net/url"
 	"sync"
 	"time"
 
@@ -51,7 +52,9 @@ func NewInternalAPI(
 			TLSClientConfig: &tls.Config{
 				InsecureSkipVerify: cfg.AppServiceAPI.DisableTLSValidation,
 			},
-			Proxy: http.ProxyFromEnvironment,
+			Proxy: func(req *http.Request) (*url.URL, error) {
+				return cfg.AppServiceAPI.Proxy.GetApplicableProxy(req, &cfg.Global.Proxy)
+			},
 		},
 	}
 	// Create appserivce query API with an HTTP client that will be used for all

internal/pushgateway/client.go

@@ -7,9 +7,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 type httpClient struct {
@@ -17,15 +19,17 @@ type httpClient struct {
 }
 
 // NewHTTPClient creates a new Push Gateway client.
-func NewHTTPClient(disableTLSValidation bool) Client {
+func NewHTTPClient(cfg *config.Dendrite) Client {
 	hc := &http.Client{
 		Timeout: 30 * time.Second,
 		Transport: &http.Transport{
 			DisableKeepAlives: true,
 			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: disableTLSValidation,
+				InsecureSkipVerify: cfg.UserAPI.PushGatewayDisableTLSValidation,
+			},
+			Proxy: func(req *http.Request) (*url.URL, error) {
+				return cfg.UserAPI.Proxy.GetApplicableProxy(req, &cfg.Global.Proxy)
 			},
-			Proxy: http.ProxyFromEnvironment,
 		},
 	}
 	return &httpClient{hc: hc}

setup/config/config_appservice.go

@@ -34,12 +34,17 @@ type AppServiceAPI struct {
 	DisableTLSValidation bool `yaml:"disable_tls_validation"`
 
 	ConfigFiles []string `yaml:"config_files"`
+
+	// Proxy for outbound requests
+	Proxy Proxy `yaml:"proxy_outbound"`
 }
 
 func (c *AppServiceAPI) Defaults(opts DefaultOpts) {
+	c.Proxy.Defaults()
 }
 
 func (c *AppServiceAPI) Verify(configErrs *ConfigErrors) {
+	c.Proxy.Verify(configErrs)
 }
 
 // ApplicationServiceNamespace is the namespace that a specific application

setup/config/config_global.go

@@ -3,6 +3,10 @@ package config
 import (
 	"fmt"
 	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -78,6 +82,9 @@ type Global struct {
 
 	// Configuration for the caches.
 	Cache Cache `yaml:"cache"`
+
+	// Proxy for outbound requests
+	Proxy Proxy `yaml:"proxy_outbound"`
 }
 
 func (c *Global) Defaults(opts DefaultOpts) {
@@ -102,6 +109,7 @@ func (c *Global) Defaults(opts DefaultOpts) {
 	c.ServerNotices.Defaults(opts)
 	c.ReportStats.Defaults()
 	c.Cache.Defaults()
+	c.Proxy.Defaults()
 }
 
 func (c *Global) Verify(configErrs *ConfigErrors) {
@@ -119,6 +127,7 @@ func (c *Global) Verify(configErrs *ConfigErrors) {
 	c.ServerNotices.Verify(configErrs)
 	c.ReportStats.Verify(configErrs)
 	c.Cache.Verify(configErrs)
+	c.Proxy.Verify(configErrs)
 }
 
 func (c *Global) IsLocalServerName(serverName gomatrixserverlib.ServerName) bool {
@@ -437,3 +446,70 @@ func (d *DataUnit) UnmarshalText(text []byte) error {
 	*d = DataUnit(v * magnitude)
 	return nil
 }
+
+// The config for setting a proxy to use for server->server requests
+type Proxy struct {
+	// Is the proxy enabled?
+	Enabled bool `yaml:"enabled"`
+	// The protocol for the proxy (http / https / socks5)
+	Protocol string `yaml:"protocol"`
+	// The host where the proxy is listening
+	Host string `yaml:"host"`
+	// The port on which the proxy is listening
+	Port uint16 `yaml:"port"`
+	// A list of destination addresses/networks not intended to be proxied
+	ExcludeAddresses []string `yaml:exclude_addresses`
+}
+
+func (c *Proxy) Defaults() {
+	c.Enabled = false
+	c.Protocol = "http"
+	c.Host = "localhost"
+	c.Port = 8080
+}
+
+func (c *Proxy) Verify(configErrs *ConfigErrors) {
+}
+
+func (c *Proxy) RequestAddressIsExcluded(req *http.Request) bool {
+	for _, s := range c.ExcludeAddresses {
+		var exclude *net.IPNet
+
+		if strings.Contains(s, "/") {
+			_, exclude, _ = net.ParseCIDR(s)
+		} else {
+			_, exclude, _ = net.ParseCIDR(fmt.Sprintf("%s/32", s))
+		}
+		_, remote_net, _ := net.ParseCIDR(fmt.Sprintf("%s/32", s))
+
+		if exclude.Contains(remote_net.IP) {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *Proxy) ConfigIsDefault() bool {
+	if c.Enabled == false && c.Protocol == "http" && c.Port == 8080 {
+		return true
+	}
+	return false
+}
+
+func (c *Proxy) GetApplicableProxy(req *http.Request, parent *Proxy) (*url.URL, error) {
+	if parent.RequestAddressIsExcluded(req) || c.RequestAddressIsExcluded(req) {
+		return req.URL, nil
+	}
+	if os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != "" {
+		return http.ProxyFromEnvironment(req)
+	} else if parent.Enabled && c.ConfigIsDefault() {
+		// Proxy defined in Global section
+		return url.Parse(fmt.Sprintf("%s://%s:%s", parent.Protocol, parent.Host, parent.Port))
+	} else if !c.ConfigIsDefault() {
+		// Proxy defined in this section
+		return url.Parse(fmt.Sprintf("%s://%s:%s", c.Protocol, c.Host, c.Port))
+	} else {
+		// No proxy
+		return nil, nil
+	}
+}

setup/config/config_test.go

@@ -79,6 +79,12 @@ global:
     room_name: "Server Alerts"	
   jetstream:
     addresses: ["test"]
+  proxy_outbound:
+    enabled: false
+    protocol: http
+    host: localhost
+    port: 8080
+    exclude_addresses: []
 app_service_api:
   database:
     connection_string: file:appservice.db
@@ -86,6 +92,12 @@ app_service_api:
     max_idle_conns: 2
     conn_max_lifetime: -1
   config_files: []
+  proxy_outbound:
+    enabled: false
+    protocol: http
+    host: localhost
+    port: 8080
+    exclude_addresses: []
 client_api:
   registration_disabled: true
   registration_shared_secret: ""
@@ -103,11 +115,6 @@ client_api:
 federation_api:
   database:
     connection_string: file:federationapi.db
-  proxy_outbound:
-    enabled: false
-    protocol: http
-    host: localhost
-    port: 8080
 key_server:
   database:
     connection_string: file:keyserver.db
@@ -170,6 +177,12 @@ user_api:
     max_open_conns: 100
     max_idle_conns: 2
     conn_max_lifetime: -1
+  proxy_outbound:
+    enabled: false
+    protocol: http
+    host: localhost
+    port: 8080
+    exclude_addresses: []
 relay_api:
   database:
     connection_string: file:relayapi.db

setup/config/config_userapi.go

@@ -21,6 +21,8 @@ type UserAPI struct {
 	// Users who register on this homeserver will automatically
 	// be joined to the rooms listed under this option.
 	AutoJoinRooms []string `yaml:"auto_join_rooms"`
+	// Proxy for outbound requests
+	Proxy Proxy `yaml:"proxy_outbound"`
 }
 
 const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
@@ -33,6 +35,7 @@ func (c *UserAPI) Defaults(opts DefaultOpts) {
 			c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
 		}
 	}
+	c.Proxy.Defaults()
 }
 
 func (c *UserAPI) Verify(configErrs *ConfigErrors) {
@@ -40,4 +43,5 @@ func (c *UserAPI) Verify(configErrs *ConfigErrors) {
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
 	}
+	c.Proxy.Verify(configErrs)
 }

userapi/userapi.go

@@ -47,7 +47,7 @@ func NewInternalAPI(
 	js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream)
 	appServices := dendriteCfg.Derived.ApplicationServices
 
-	pgClient := pushgateway.NewHTTPClient(dendriteCfg.UserAPI.PushGatewayDisableTLSValidation)
+	pgClient := pushgateway.NewHTTPClient(dendriteCfg)
 
 	db, err := storage.NewUserDatabase(
 		processContext.Context(),

userapi/util/phonehomestats.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"math"
 	"net/http"
+	"net/url"
 	"runtime"
 	"syscall"
 	"time"
@@ -57,8 +58,12 @@ func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB
 		db:         statsDB,
 		isMonolith: true,
 		client: &http.Client{
-			Timeout:   time.Second * 30,
+			Timeout: time.Second * 30,
-			Transport: http.DefaultTransport,
+			Transport: &http.Transport{
+				Proxy: func(req *http.Request) (*url.URL, error) {
+					return cfg.UserAPI.Proxy.GetApplicableProxy(req, &cfg.Global.Proxy)
+				},
+			},
 		},
 	}