From 890bd9cf194109908190c0359d400e10832d6760 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Oct 2017 12:03:11 +0100 Subject: [PATCH] Untangle precommit and travis test scripts --- .travis.yml | 4 +-- hooks/pre-commit | 31 +----------------- scripts/README.md | 21 ++++++++++++ scripts/build-test-lint.sh | 27 ++++++++++++++++ scripts/find-lint.sh | 32 +++++++++++++++++++ .../install-local-kafka.sh | 0 travis-test.sh => scripts/travis-test.sh | 5 ++- 7 files changed, 87 insertions(+), 33 deletions(-) create mode 100644 scripts/README.md create mode 100755 scripts/build-test-lint.sh create mode 100755 scripts/find-lint.sh rename travis-install-kafka.sh => scripts/install-local-kafka.sh (100%) rename travis-test.sh => scripts/travis-test.sh (85%) diff --git a/.travis.yml b/.travis.yml index ab2492d2e..dd7359af9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,8 +24,8 @@ before_script: - openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt -days 365 -nodes -subj /CN=localhost script: - - ./travis-install-kafka.sh - - ./travis-test.sh + - ./scripts/install-local-kafka.sh + - ./scripts/travis-test.sh notifications: webhooks: diff --git a/hooks/pre-commit b/hooks/pre-commit index 904d38dc3..ec7e3e1e3 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -2,33 +2,4 @@ set -eu -# Tune the GC to use more memory to reduce the number of garbage collections -export GOGC=400 -export GOPATH="$(pwd):$(pwd)/vendor" -export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" - -echo "Checking that it builds" -gb build - -# Check that all the packages can build. -# When `go build` is given multiple packages it won't output anything, and just -# checks that everything builds. This seems to do a better job of handling -# missing imports than `gb build` does. -echo "Double checking it builds..." -go build github.com/matrix-org/dendrite/cmd/... - -echo "Installing lint search engine..." -go install github.com/alecthomas/gometalinter/ -gometalinter --config=linter.json ./... --install - -echo "Looking for lint..." -gometalinter --config=linter.json ./... --enable-gc - -echo "Double checking spelling..." -misspell -error src *.md - -echo "Testing..." -gb test - - -echo "Done!" +./scripts/build-test-lint.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..5e48c855c --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,21 @@ +# Dev Scripts + +These are a collection of scripts that should be helpful for those developing +on dendrite. + +- `./scripts/find-lint.sh` runs the linters against dendrite, + `./scripts/find-lint.sh fast` runs a subset of faster lints +- `./scripts/build-test-lint.sh` builds, tests and lints dendrite, and + should be run before pushing commits +- `./scripts/install-local-kafka.sh` downloads, installs and runs a + kafka instance +- `./scripts/travis-test.sh` is what travis runs + + +The linters can take a lot of resources and are slow, so they can be +configured using two enviroment variables: + +- `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run, + gometalinter defaults this to 8 +- `DENDRITE_LINT_DISABLE_GC` - if set then the the go gc will be disabled + when running the linters, speeding them up but using much more memory. diff --git a/scripts/build-test-lint.sh b/scripts/build-test-lint.sh new file mode 100755 index 000000000..e9541ea10 --- /dev/null +++ b/scripts/build-test-lint.sh @@ -0,0 +1,27 @@ +#! /bin/bash + +set -eu + +export GOPATH="$(pwd):$(pwd)/vendor" +export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" + +echo "Checking that it builds" +gb build + +# Check that all the packages can build. +# When `go build` is given multiple packages it won't output anything, and just +# checks that everything builds. This seems to do a better job of handling +# missing imports than `gb build` does. +echo "Double checking it builds..." +go build github.com/matrix-org/dendrite/cmd/... + +./scripts/find-lint.sh + +echo "Double checking spelling..." +misspell -error src *.md + +echo "Testing..." +gb test + + +echo "Done!" diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh new file mode 100755 index 000000000..589032b6e --- /dev/null +++ b/scripts/find-lint.sh @@ -0,0 +1,32 @@ +#! /bin/bash + +set -eu + +export GOPATH="$(pwd):$(pwd)/vendor" +export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" + +args="" +if [ ${1:-""} = "fast" ] +then args="--config=linter-fast.json" +else args="--config=linter.json" +fi + +if [ -n "${DENDRITE_LINT_CONCURRENCY:-}" ] +then args="$args --concurrency=$DENDRITE_LINT_CONCURRENCY" +fi + +if [ -z "${DENDRITE_LINT_DISABLE_GC:-}" ] +then args="$args --enable-gc" +fi + +echo "Installing lint search engine..." +go install github.com/alecthomas/gometalinter/ +gometalinter --config=linter.json ./... --install + +echo "Looking for lint..." +gometalinter ./... $args + +echo "Double checking spelling..." +misspell -error src *.md + +echo "Done!" diff --git a/travis-install-kafka.sh b/scripts/install-local-kafka.sh similarity index 100% rename from travis-install-kafka.sh rename to scripts/install-local-kafka.sh diff --git a/travis-test.sh b/scripts/travis-test.sh similarity index 85% rename from travis-test.sh rename to scripts/travis-test.sh index 20d5f6faa..1ea7c47cf 100755 --- a/travis-test.sh +++ b/scripts/travis-test.sh @@ -2,6 +2,9 @@ set -eu +# Tune the GC to use more memory to reduce the number of garbage collections +export GOGC=400 + # Check that the servers build (this is done explicitly because `gb build` can silently fail (exit 0) and then we'd test a stale binary) gb build github.com/matrix-org/dendrite/cmd/dendrite-room-server gb build github.com/matrix-org/dendrite/cmd/roomserver-integration-tests @@ -13,7 +16,7 @@ gb build github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests gb build github.com/matrix-org/dendrite/cmd/client-api-proxy # Run the pre commit hooks -./hooks/pre-commit +DENDRITE_LINT_DISABLE_GC=1 ./scripts/build-test-lint.sh # Run the integration tests bin/roomserver-integration-tests