diff --git a/cmd/furl/main.go b/cmd/furl/main.go
index 0992fd257..a918a4cfe 100644
--- a/cmd/furl/main.go
+++ b/cmd/furl/main.go
@@ -28,6 +28,8 @@ var (
 
 	flagMatrixKey string
 	flagOrigin    string
+
+	flagPort int
 )
 
 func init() {
@@ -45,17 +47,20 @@ func init() {
 
 	flag.StringVar(&flagOrigin, "O", "", "The server name that the request should originate from. The remote server will use this to request server keys. There MUST be a TLS listener at the .well-known address for this server name, i.e it needs to be pointing to a real homeserver. If blank, furl will self-host this on a random high numbered port, but only if the target is localhost. Use $PORT in request URLs/bodies to substitute the port number in.")
 	flag.StringVar(&flagOrigin, "origin", "", "The server name that the request should originate from. The remote server will use this to request server keys. There MUST be a TLS listener at the .well-known address for this server name, i.e it needs to be pointing to a real homeserver. If blank, furl will self-host this on a random high numbered port, but only if the target is localhost. Use $PORT in request URLs/bodies to substitute the port number in.")
+
+	flag.IntVar(&flagPort, "p", 0, "Port to self-host on. If set, always self-hosts. Required because sometimes requests need the same origin.")
 }
 
 type args struct {
-	SkipVerify  bool
-	Method      string
-	Data        []byte
-	MatrixKey   ed25519.PrivateKey
-	MatrixKeyID gomatrixserverlib.KeyID
-	Origin      spec.ServerName
-	SelfHostKey bool
-	TargetURL   *url.URL
+	SkipVerify   bool
+	Method       string
+	Data         []byte
+	MatrixKey    ed25519.PrivateKey
+	MatrixKeyID  gomatrixserverlib.KeyID
+	Origin       spec.ServerName
+	SelfHostKey  bool
+	SelfHostPort int
+	TargetURL    *url.URL
 }
 
 func processArgs() (*args, error) {
@@ -91,8 +96,9 @@ func processArgs() (*args, error) {
 	a.SkipVerify = flagSkipVerify
 	a.Method = strings.ToUpper(flagMethod)
 	a.Origin = spec.ServerName(flagOrigin)
+	a.SelfHostPort = flagPort
 	a.TargetURL = targetURL
-	a.SelfHostKey = a.Origin == "" && a.TargetURL.Hostname() == "localhost"
+	a.SelfHostKey = a.SelfHostPort != 0 || (a.Origin == "" && a.TargetURL.Hostname() == "localhost")
 
 	// load data
 	isFile := strings.HasPrefix(flagData, "@")
@@ -119,7 +125,7 @@ func main() {
 
 	if a.SelfHostKey {
 		fmt.Printf("Self-hosting key...")
-		apiURL, cancel := test.ListenAndServe(tt{}, http.DefaultServeMux, true)
+		apiURL, cancel := test.ListenAndServe(tt{}, http.DefaultServeMux, true, a.SelfHostPort)
 		defer cancel()
 		parsedURL, _ := url.Parse(apiURL)
 		a.Origin = spec.ServerName(parsedURL.Host)
@@ -167,6 +173,11 @@ func main() {
 		if err != nil {
 			panic(err)
 		}
+		if a.Data != nil {
+			data := string(a.Data)
+			data = strings.ReplaceAll(data, "$PORT", port)
+			a.Data = []byte(data)
+		}
 	}
 
 	client := fclient.NewFederationClient(
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index bd49c5301..8f2cd8f59 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -314,7 +314,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
 	federationapi.AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, caching.DisableMetrics)
-	baseURL, cancel := test.ListenAndServe(t, routers.Federation, true)
+	baseURL, cancel := test.ListenAndServe(t, routers.Federation, true, 0)
 	defer cancel()
 	serverName := spec.ServerName(strings.TrimPrefix(baseURL, "https://"))
 
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index c68a6f0cb..79b66b573 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -45,6 +45,7 @@ func MakeJoin(
 ) util.JSONResponse {
 	roomVersion, err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), roomID)
 	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryRoomVersionForRoom failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.InternalServerError(),
diff --git a/test/http.go b/test/http.go
index 74c37ed46..e8d2b1292 100644
--- a/test/http.go
+++ b/test/http.go
@@ -59,8 +59,8 @@ type testInterface interface {
 
 // ListenAndServe will listen on a random high-numbered port and attach the given router.
 // Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
-func ListenAndServe(t testInterface, router http.Handler, withTLS bool) (apiURL string, cancel func()) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
+func ListenAndServe(t testInterface, router http.Handler, withTLS bool, customPort int) (apiURL string, cancel func()) {
+	listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", customPort))
 	if err != nil {
 		t.Fatalf("failed to listen: %s", err)
 	}