Travis tweaks (#831)

* Speed up Travis build

  * Git clone options: limited depth to 10, disabled submodules
  * Cut down number of test combinations

* Tweaks: consistent quoting, clone depth, removed -u from go get

* Run go tests in parallel

* Include osx explicitly on the test matrix

* Run coverage for a single set of tests

* Review comments

  * Env controlled parallelism
  * Removed git settings
  * Named profile files

* Review

Author: codingllama

.travis.yml

@@ -3,59 +3,55 @@ language: go
 
 os:
   - linux
-  - osx
 
 go:
   - 1.8
 
 env:
-  - GOFLAGS=''
+  - WITH_COVERAGE=true
   - GOFLAGS='-race'
-  - GOFLAGS='--tags batched_queue'
   - GOFLAGS='-race --tags batched_queue'
-  - GOFLAGS=''                           WITH_ETCD=true
-  - GOFLAGS='-race'                      WITH_ETCD=true
-  - GOFLAGS='--tags pkcs11'              WITH_PKCS11=true
-  - GOFLAGS='-race --tags pkcs11'        WITH_PKCS11=true
+  - GOFLAGS='-race' WITH_ETCD=true
+  - GOFLAGS='-race --tags pkcs11' WITH_PKCS11=true
 
 matrix:
   fast_finish: true
   allow_failures:
     - os: osx
-  exclude:
-    - os: osx
-      env: GOFLAGS=      WITH_PKCS11=true
-    - os: osx
-      env: GOFLAGS=-race WITH_PKCS11=true
+  include:
+    - os: osx  # Run only "vanilla" tests on osx
+      env:
+
+services: mysql
 
 install:
-  - if [[ "${WITH_PKCS11}" == "true" ]]; then sudo apt-get install softhsm ; fi
+  - if [[ "${WITH_PKCS11}" == "true" ]]; then sudo apt-get install -y softhsm ; fi
   - |
-    if [ ! -d $HOME/gopath/src/github.com/google ]; then
-      mkdir -p $HOME/gopath/src/github.com/google
-      ln -s $TRAVIS_BUILD_DIR $HOME/gopath/src/github.com/google/trillian
+    if [ ! -d "$HOME/gopath/src/github.com/google" ]; then
+      mkdir -p "$HOME/gopath/src/github.com/google"
+      ln -s "$TRAVIS_BUILD_DIR" "$HOME/gopath/src/github.com/google/trillian"
     fi
   - mkdir ../protoc
   - |
     (
       cd ../protoc
-      wget https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-${TRAVIS_OS_NAME}-x86_64.zip
-      unzip protoc-3.2.0-${TRAVIS_OS_NAME}-x86_64.zip
+      wget "https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-${TRAVIS_OS_NAME}-x86_64.zip"
+      unzip "protoc-3.2.0-${TRAVIS_OS_NAME}-x86_64.zip"
     )
-  - export PATH=$(pwd)/../protoc/bin:$PATH
+  - export PATH="$(pwd)/../protoc/bin:$PATH"
   # googleapis is not Go code, but it's required for .pb.go regeneration because of API dependencies.
-  - git clone https://github.com/googleapis/googleapis.git $GOPATH/src/github.com/googleapis/googleapis
+  - git clone --depth=1 https://github.com/googleapis/googleapis.git "$GOPATH/src/github.com/googleapis/googleapis"
   - go get ${GOFLAGS} -d -t ./...
-  - if [[ $TRAVIS_OS_NAME == "osx" ]]; then brew update > /dev/null && brew install mariadb && mysql.server start; fi
-  - go get -u github.com/client9/misspell/cmd/misspell
-  - go get -u github.com/fzipp/gocyclo
-  - go get -u github.com/gordonklaus/ineffassign
-  - go get -u github.com/golang/lint/golint
-  - go get -u github.com/golang/protobuf/proto
-  - go get -u github.com/golang/protobuf/protoc-gen-go
-  - go get -u github.com/kisielk/errcheck
-  - go get -u golang.org/x/tools/cmd/stringer
-  - go get -u github.com/google/certificate-transparency-go
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update > /dev/null && brew install mariadb && mysql.server start; fi
+  - go get github.com/client9/misspell/cmd/misspell
+  - go get github.com/fzipp/gocyclo
+  - go get github.com/gordonklaus/ineffassign
+  - go get github.com/golang/lint/golint
+  - go get github.com/golang/protobuf/proto
+  - go get github.com/golang/protobuf/protoc-gen-go
+  - go get github.com/kisielk/errcheck
+  - go get golang.org/x/tools/cmd/stringer
+  - go get github.com/google/certificate-transparency-go
   - go get -d -t github.com/google/certificate-transparency-go/...
   - go install github.com/golang/{mock/mockgen,protobuf/protoc-gen-go}
   # install vendored protoc-gen-grpc-gateway binary
@@ -64,10 +60,13 @@ install:
   - go install ./vendor/github.com/coreos/etcd/cmd/etcd
   - go install ./vendor/github.com/coreos/etcd/cmd/etcdctl
 
+before_script:
+  - yes | ./scripts/resetdb.sh
+
 script:
   - set -e
-  - cd $HOME/gopath/src/github.com/google/trillian
-  - ./scripts/presubmit.sh
+  - cd "$HOME/gopath/src/github.com/google/trillian"
+  - ./scripts/presubmit.sh ${WITH_COVERAGE:+--coverage}
   - |
       # Check re-generation didn't change anything
       # Skip protoc-generated files (.pb.go) because protoc is not deterministic
@@ -84,17 +83,15 @@ script:
         export ETCD_DIR="${GOPATH}/bin"
       fi
   - ./integration/integration_test.sh
-  - cd $HOME/gopath/src/github.com/google/certificate-transparency-go
+  - cd "$HOME/gopath/src/github.com/google/certificate-transparency-go"
   - ./trillian/integration/integration_test.sh
   - cd $HOME/gopath/src/github.com/google/trillian
   - HAMMER_OPTS="--operations=50" ./integration/maphammer.sh 3
   - set +e
 
-services: mysql
-
-before_script:
-  - yes | ./scripts/resetdb.sh
-
 after_success:
-  - cp /tmp/coverage.txt .
-  - bash <(curl -s https://codecov.io/bash)
+  - |
+      if [[ -f /tmp/coverage.txt ]]; then
+        cp /tmp/coverage.txt .
+        bash <(curl -s https://codecov.io/bash)
+      fi

scripts/presubmit.sh

@@ -1,9 +1,14 @@
 #!/bin/bash
 #
 # Presubmit checks for Trillian.
+#
 # Checks for lint errors, spelling, licensing, correct builds / tests and so on.
 # Flags may be specified to allow suppressing of checks or automatic fixes, try
 # `scripts/presubmit.sh --help` for details.
+#
+# Globals:
+#   GO_TEST_PARALLELISM: max processes to use for Go tests. Optional (defaults
+#       to 10).
 set -eu
 
 check_deps() {
@@ -25,18 +30,22 @@ check_cmd() {
 }
 
 usage() {
-  echo "$0 [--fix] [--no-build] [--no-linters] [--no-generate]"
+  echo "$0 [--coverage] [--fix] [--no-build] [--no-linters] [--no-generate]"
 }
 
 main() {
   check_deps
 
+  local coverage=0
   local fix=0
   local run_build=1
   local run_linters=1
   local run_generate=1
   while [[ $# -gt 0 ]]; do
     case "$1" in
+      --coverage)
+        coverage=1
+        ;;
       --fix)
         fix=1
         ;;
@@ -93,16 +102,35 @@ main() {
     go build ${goflags} ${go_dirs}
 
     echo 'running go test'
-    echo "" > coverage.txt
+
+    # Individual package profiles are written to "$profile.out" files under
+    # /tmp/trillian_profile.
+    # An aggregate profile is created at /tmp/coverage.txt.
+    mkdir -p /tmp/trillian_profile
+    rm -f /tmp/trillian_profile/*
 
     for d in ${go_dirs}; do
-      go test -timeout=5m -short -coverprofile=profile.out -covermode=atomic ${goflags} $d
-      if [ -f profile.out ]; then
-        cat profile.out >> coverage.txt
-        rm profile.out
+      # Create a different -coverprofile for each test (if enabled)
+      local coverflags=
+      if [[ ${coverage} -eq 1 ]]; then
+        # Transform $d to a smaller, valid file name.
+        # For example:
+        # * github.com/google/trillian becomes trillian.out
+        # * github.com/google/trillian/cmd/createtree/keys becomes
+        #   trillian-cmd-createtree-keys.out
+        local profile="${d}.out"
+        profile="${profile#github.com/*/}"
+        profile="${profile//\//-}"
+        coverflags="-covermode=atomic -coverprofile='/tmp/trillian_profile/${profile}'"
       fi
-    done
-    cp coverage.txt /tmp
+
+      # Do not run go test in the loop, instead echo it so we can use xargs to
+      # add some parallelism.
+      echo go test -short -timeout=5m ${coverflags} ${goflags} "$d"
+    done | xargs -I '{}' -P ${GO_TEST_PARALLELISM:=10} bash -c '{}'
+
+    [[ ${coverage} -eq 1 ]] && \
+      cat /tmp/trillian_profile/*.out > /tmp/coverage.txt
   fi
 
   if [[ "${run_linters}" -eq 1 ]]; then