p2p: Use JSServer for comms rather than GoJsConn (#888)

* p2p: Use JSServer for comms rather than GoJsConn

This has several benefits:
 - it fixes a bug whereby you could not transmit >4k bytes to/from JS/Go land.
 - it more clearly exposes the interface point between Go and JS: a single
   global function call.
 - it presents a nicer API shape than the previous `net.Conn`.
 - it doesn't needlessly 'stream' data which is already sitting in-memory.

This is currently only active for local CS API traffic, another PR will
add Federation P2P support.

* Typo

cmd/dendritejs/jsServer.go

@@ -0,0 +1,104 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"syscall/js"
+)
+
+// JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it.
+type JSServer struct {
+	// The router which will service requests
+	Mux *http.ServeMux
+}
+
+// OnRequestFromJS is the function that JS will invoke when there is a new request.
+// The JS function signature is:
+//   function(reqString: string): Promise<{result: string, error: string}>
+// Usage is like:
+//   const res = await global._go_js_server.fetch(reqString);
+//   if (res.error) {
+//     // handle error: this is a 'network' error, not a non-2xx error.
+//   }
+//   const rawHttpResponse = res.result;
+func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
+	// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
+	// if this request blocks at all e.g for /sync calls
+	httpStr := args[0].String()
+	promise := js.Global().Get("Promise").New(js.FuncOf(func(pthis js.Value, pargs []js.Value) interface{} {
+		// The initial callback code for new Promise() is also called on the critical path, which is why
+		// we need to put this in an immediately invoked goroutine.
+		go func() {
+			resolve := pargs[0]
+			fmt.Println("Received request:")
+			fmt.Printf("%s\n", httpStr)
+			resStr, err := h.handle(httpStr)
+			errStr := ""
+			if err != nil {
+				errStr = err.Error()
+			}
+			fmt.Println("Sending response:")
+			fmt.Printf("%s\n", resStr)
+			resolve.Invoke(map[string]interface{}{
+				"result": resStr,
+				"error":  errStr,
+			})
+		}()
+		return nil
+	}))
+	return promise
+}
+
+// handle invokes the http.ServeMux for this request and returns the raw HTTP response.
+func (h *JSServer) handle(httpStr string) (resStr string, err error) {
+	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(httpStr)))
+	if err != nil {
+		return
+	}
+	w := httptest.NewRecorder()
+
+	h.Mux.ServeHTTP(w, req)
+
+	res := w.Result()
+	var resBuffer strings.Builder
+	err = res.Write(&resBuffer)
+	return resBuffer.String(), err
+}
+
+// ListenAndServe registers a variable in JS-land with the given namespace. This variable is
+// a function which JS-land can call to 'send' HTTP requests. The function is attached to
+// a global object called "_go_js_server". See OnRequestFromJS for more info.
+func (h *JSServer) ListenAndServe(namespace string) {
+	globalName := "_go_js_server"
+	// register a hook in JS-land for it to invoke stuff
+	server := js.Global().Get(globalName)
+	if !server.Truthy() {
+		server = js.Global().Get("Object").New()
+		js.Global().Set(globalName, server)
+	}
+
+	server.Set(namespace, js.FuncOf(h.OnRequestFromJS))
+
+	fmt.Printf("Listening for requests from JS on function %s.%s\n", globalName, namespace)
+	// Block forever to mimic http.ListenAndServe
+	select {}
+}

cmd/dendritejs/main.go

@@ -156,10 +156,10 @@ func main() {
 	// Expose the matrix APIs via fetch - for local traffic
 	go func() {
 		logrus.Info("Listening for service-worker fetch traffic")
-
+		s := JSServer{
-		listener := go_http_js_libp2p.NewFetchListener()
+			Mux: http.DefaultServeMux,
-		s := &http.Server{}
+		}
-		go s.Serve(listener)
+		s.ListenAndServe("fetch")
 	}()
 
 	// We want to block forever to let the fetch and libp2p handler serve the APIs

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/lib/pq v1.2.0
 	github.com/libp2p/go-libp2p-core v0.5.0
 	github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
-	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437
+	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424

go.sum

@@ -122,6 +122,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306190227-af44d7013315 h1:tE
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306190227-af44d7013315/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 h1:zcGpWvVV6swXw9LBMRsdDHPOugQYSwesH2RByUfBx2I=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY=
+github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c h1:jj/LIZKMO7GK6O0UarpRwse9L3ZyzozpyMtdPA7ddSk=
+github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI=

p2p.md

@@ -16,7 +16,7 @@ $ cp main.wasm ../riot-web/src/vector/dendrite.wasm
 
 This is how peers discover each other and communicate.
 
-By default, Dendrite uses the IPFS-hosted websocket star **Development** relay server at `/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star`.
+By default, Dendrite uses the Matrix-hosted websocket star relay server at TODO `/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star`.
 This is currently hard-coded in `./cmd/dendritejs/main.go` - you can also use a local one if you run your own relay:
 
 ```
@@ -24,13 +24,13 @@ $ npm install --global libp2p-websocket-star-rendezvous
 $ rendezvous --port=9090 --host=127.0.0.1
 ```
 
-Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`. We'll probably run our own relay server at some point.
+Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`.
 
 ### Riot-web
 
 You need to check out these repos:
 
-``
+```
 $ git clone git@github.com:matrix-org/go-http-js-libp2p.git
 $ git clone git@github.com:matrix-org/go-sqlite3-js.git
 ```
@@ -39,6 +39,7 @@ Make sure to `yarn install` in both of these repos. Then:
 
 - `$ cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" ./src/vector/`
 - Comment out the lines in `wasm_exec.js` which contains:
+
 ```
 if (!global.fs && global.require) {
     global.fs = require("fs");
@@ -56,17 +57,13 @@ NB: If you don't run the server with `yarn start` you need to make sure your ser
 
 TODO: Make a Docker image with all of this in it and a volume mount for `dendrite.wasm`.
 
-## Running
+### Running
 
 You need a Chrome and a Firefox running to test locally as service workers don't work in incognito tabs.
 - For Chrome, use `chrome://serviceworker-internals/` to unregister/see logs.
 - For Firefox, use `about:debugging#/runtime/this-firefox` to unregister. Use the console window to see logs.
 
-Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and wait a bit. Then refresh the page (this is required
+Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and register with `http://localhost:8080` as your HS URL.
-because the fetch interceptor races with setting up dendrite. If you don't refresh, you won't be able to contact your HS). After
-the refresh, click Register and use `http://localhost:8080` as your HS URL.
-
-TODO: Fix the race so we don't need multiple refreshes.
 
 You can join rooms by room alias e.g `/join #foo:bar`.
 
@@ -74,7 +71,7 @@ You can join rooms by room alias e.g `/join #foo:bar`.
 
 - When registering you may be unable to find the server, it'll seem flakey. This happens because the SW, particularly in Firefox,
   gets killed after 30s of inactivity. When you are not registered, you aren't doing `/sync` calls to keep the SW alive, so if you
-  don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page *twice*.
+  don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page.
 
 - The libp2p layer has rate limits, so frequent Federation traffic may cause the connection to drop and messages to not be transferred.
   I guess in other words, don't send too much traffic?