Add CI (#34)

* Add Buildkite CI (#25)

 * Add Buildkite CI

 * Disable shellcheck, clippy, rustfmt, and nits so CI passes

* Switch tests to Travis CI (#31)

* Make CI actually use rust stable (#32)

.buildkite/hooks/post-checkout

@@ -0,0 +1,42 @@
+CI_BUILD_START=$(date +%s)
+export CI_BUILD_START
+
+source ci/env.sh
+
+#
+# Kill any running docker containers, which are potentially left over from the
+# previous CI job
+#
+(
+  containers=$(docker ps -q)
+  if [[ $(hostname) != metrics-solana-com && -n $containers ]]; then
+    echo "+++ Killing stale docker containers"
+    docker ps
+
+    # shellcheck disable=SC2086 # Don't want to double quote $containers
+    docker kill $containers
+  fi
+)
+
+# Processes from previously aborted CI jobs seem to loiter, unclear why as one
+# would expect the buildkite-agent to clean up all child processes of the
+# aborted CI job.
+# But as a workaround for now manually kill some known loiterers.  These
+# processes will all have the `init` process as their PPID:
+(
+  victims=
+  for name in bash cargo docker solana; do
+    victims="$victims $(pgrep -u "$(id -u)" -P 1 -d \  $name)"
+  done
+  for victim in $victims; do
+    echo "Killing pid $victim"
+    kill -9 "$victim" || true
+  done
+)
+
+# HACK: These are in our docker images, need to be removed from CARGO_HOME
+#  because we try to cache downloads across builds with CARGO_HOME
+# cargo lacks a facility for "system" tooling, always tries CARGO_HOME first
+cargo uninstall cargo-audit || true
+cargo uninstall svgbob_cli || true
+cargo uninstall mdbook || true

.buildkite/hooks/post-checkout.sh

@@ -0,0 +1 @@
+post-checkout

.buildkite/hooks/post-command

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+#
+# Save target/ for the next CI build on this machine
+#
+(
+  set -x
+  d=$HOME/cargo-target-cache/"$BUILDKITE_LABEL"
+  mkdir -p "$d"
+  set -x
+  rsync -a --delete --link-dest="$PWD" target "$d"
+  du -hs "$d"
+  read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$d")
+  echo "--- ${cacheSizeInGB}GB: $d"
+)

.buildkite/hooks/post-command.sh

@@ -0,0 +1 @@
+post-command

.buildkite/hooks/pre-command

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -e
+
+eval "$(ejson2env .buildkite/env/secrets.ejson)"
+
+# Ensure the pattern "+++ ..." never occurs when |set -x| is set, as buildkite
+# interprets this as the start of a log group.
+# Ref: https://buildkite.com/docs/pipelines/managing-log-output
+export PS4="++"
+
+#
+# Restore target/ from the previous CI build on this machine
+#
+(
+  set -x
+  d=$HOME/cargo-target-cache/"$BUILDKITE_LABEL"
+  MAX_CACHE_SIZE=18 # gigabytes
+
+  if [[ -d $d ]]; then
+    du -hs "$d"
+    read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$d")
+    echo "--- ${cacheSizeInGB}GB: $d"
+    if [[ $cacheSizeInGB -gt $MAX_CACHE_SIZE ]]; then
+      echo "--- $d is too large, removing it"
+      rm -rf "$d"
+    fi
+  else
+    echo "--- $d not present"
+  fi
+
+  mkdir -p "$d"/target
+  rsync -a --delete --link-dest="$d" "$d"/target .
+)

.buildkite/hooks/pre-command.sh

@@ -0,0 +1 @@
+pre-command

.buildkite/pipeline-upload.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# This script is used to upload the full buildkite pipeline. The steps defined
+# in the buildkite UI should simply be:
+#
+#   steps:
+#    - command: ".buildkite/pipeline-upload.sh"
+#
+
+set -e
+cd "$(dirname "$0")"/..
+
+buildkite-agent pipeline upload ci/buildkite.yml
+
+if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
+  # Add helpful link back to the corresponding Github Pull Request
+  buildkite-agent annotate --style info --context pr-backlink \
+    "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
+fi
+

.travis.yml

@@ -0,0 +1,24 @@
+language: rust
+cache:
+  directories:
+    - $TRAVIS_HOME/.cargo/
+    - $TRAVIS_HOME/.rustup/
+jobs:
+  include:
+    - stage: check
+      rust: nightly-2019-07-19
+      install:
+        - rustup component add rustfmt
+        - ci/cargo-try-install.sh cargo-audit
+      script: ci/checks.sh
+    - stage: test
+      rust: nightly-2019-07-19
+      script: ci/build-all.sh
+    - stage: test
+      language: minimal
+      before_install:
+        - curl -sSf https://build.travis-ci.org/files/rustup-init.sh | sh -s -- --default-toolchain=1.36.0 -y
+        - export PATH=${TRAVIS_HOME}/.cargo/bin:$PATH
+        - rustup override set 1.36.0
+        - rustc --version
+      script: ci/e2e-tests.sh

ci/_

@@ -0,0 +1,11 @@
+# Buildkite log management helper
+#
+# See https://buildkite.com/docs/pipelines/managing-log-output
+#
+# |source| me
+#
+
+_() {
+  echo "--- $*"
+  "$@"
+}

ci/build-all.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -e
+cd "$(dirname "$0")/.."
+
+export RUST_BACKTRACE=1
+export RUSTFLAGS="-D warnings"
+
+cargo build --all ${V:+--verbose}

ci/buildkite.yml

@@ -0,0 +1,5 @@
+steps:
+  - command: "ci/publish-crate.sh"
+    timeout_in_minutes: 90
+    name: "publish crate"
+    branches: "!master"

ci/cargo-try-install.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+cargo install $1
+
+RET=$?
+
+if [[ RET -eq 0 ]] || [[ RET -eq 101 ]]; then
+    exit 0
+fi
+
+exit ${RET}

ci/checks.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -xe
+
+cd "$(dirname "$0")/.."
+
+export RUST_BACKTRACE=1
+export RUSTFLAGS="-D warnings"
+
+cargo fmt --all -- --check
+#cargo clippy --version
+#cargo clippy --all -- --deny=warnings
+cargo update
+cargo audit --version
+cargo audit --ignore RUSTSEC-2019-0011 # https://github.com/solana-labs/solana/issues/5207
+#ci/nits.sh
+ci/order-crates-for-publishing.py

ci/docker-run.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -e
+
+usage() {
+  echo "Usage: $0 [--nopull] [docker image name] [command]"
+  echo
+  echo Runs command in the specified docker image with
+  echo a CI-appropriate environment.
+  echo
+  echo "--nopull   Skip the dockerhub image update"
+  echo "--shell    Skip command and enter an interactive shell"
+  echo
+}
+
+cd "$(dirname "$0")/.."
+
+INTERACTIVE=false
+if [[ $1 = --shell ]]; then
+  INTERACTIVE=true
+  shift
+fi
+
+NOPULL=false
+if [[ $1 = --nopull ]]; then
+  NOPULL=true
+  shift
+fi
+
+IMAGE="$1"
+if [[ -z "$IMAGE" ]]; then
+  echo Error: image not defined
+  exit 1
+fi
+
+$NOPULL || docker pull "$IMAGE"
+shift
+
+ARGS=(
+  --workdir /solana
+  --volume "$PWD:/solana"
+  --rm
+)
+
+if [[ -n $CI ]]; then
+  # Share the real ~/.cargo between docker containers in CI for speed
+  ARGS+=(--volume "$HOME:/home")
+else
+  # Avoid sharing ~/.cargo when building locally to avoid a mixed macOS/Linux
+  # ~/.cargo
+  ARGS+=(--volume "$PWD:/home")
+fi
+ARGS+=(--env "CARGO_HOME=/home/.cargo")
+
+# kcov tries to set the personality of the binary which docker
+# doesn't allow by default.
+ARGS+=(--security-opt "seccomp=unconfined")
+
+# Ensure files are created with the current host uid/gid
+if [[ -z "$SOLANA_DOCKER_RUN_NOSETUID" ]]; then
+  ARGS+=(--user "$(id -u):$(id -g)")
+fi
+
+# Environment variables to propagate into the container
+ARGS+=(
+  --env BUILDKITE
+  --env BUILDKITE_AGENT_ACCESS_TOKEN
+  --env BUILDKITE_JOB_ID
+  --env CI
+  --env CI_BRANCH
+  --env CI_BUILD_ID
+  --env CI_COMMIT
+  --env CI_JOB_ID
+  --env CI_PULL_REQUEST
+  --env CI_REPO_SLUG
+  --env CODECOV_TOKEN
+  --env CRATES_IO_TOKEN
+)
+
+if $INTERACTIVE; then
+  if [[ -n $1 ]]; then
+    echo
+    echo "Note: '$*' ignored due to --shell argument"
+    echo
+  fi
+  set -x
+  exec docker run --interactive --tty "${ARGS[@]}" "$IMAGE" bash
+fi
+
+set -x
+exec docker run "${ARGS[@]}" "$IMAGE" "$@"

ci/e2e-tests.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+cd "$(dirname "$0")/.."
+
+
+export RUST_BACKTRACE=1
+export RUSTFLAGS="-D warnings"
+
+cargo build --manifest-path language/e2e_tests/Cargo.toml ${V:+--verbose}
+cargo test --manifest-path language/e2e_tests/Cargo.toml ${V:+--verbose} -- --nocapture
+

ci/nits.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# Project nits enforced here
+#
+set -e
+
+cd "$(dirname "$0")/.."
+source ci/_
+
+# Logging hygiene: Please don't print from --lib, use the `log` crate instead
+declare prints=(
+  'print!'
+  'println!'
+  'eprint!'
+  'eprintln!'
+  'dbg!'
+)
+
+# Parts of the tree that are expected to be print free
+declare print_free_tree=(
+  'core/src'
+  'drone/src'
+  'metrics/src'
+  'netutil/src'
+  'runtime/src'
+  'sdk/bpf/rust/rust-utils'
+  'sdk/src'
+  'programs/bpf/rust'
+  'programs/stake_api/src'
+  'programs/stake_program/src'
+  'programs/vote_api/src'
+  'programs/vote_program/src'
+)
+
+if _ git --no-pager grep -n --max-depth=0 "${prints[@]/#/-e }" -- "${print_free_tree[@]}"; then
+    exit 1
+fi
+
+
+# Code readability: please be explicit about the type instead of using
+# Default::default()
+#
+# Ref: https://github.com/solana-labs/solana/issues/2630
+if _ git --no-pager grep -n 'Default::default()' -- '*.rs'; then
+    exit 1
+fi
+
+# Let's keep a .gitignore for every crate, ensure it's got
+#  /target/ and /farf/ in it
+declare gitignores_ok=true
+for i in $(git --no-pager ls-files \*/Cargo.toml ); do
+  dir=$(dirname "$i")
+  if [[ ! -f $dir/.gitignore ]]; then
+      echo 'error: nits.sh .gitnore missing for crate '"$dir" >&2
+      gitignores_ok=false
+  else
+    if ! grep -q -e '^/target/$' "$dir"/.gitignore; then
+      echo 'error: nits.sh "/target/" apparently missing from '"$dir"'/.gitignore' >&2
+      gitignores_ok=false
+    fi
+    if ! grep -q -e '^/farf/$' "$dir"/.gitignore ; then
+      echo 'error: nits.sh "/farf/" apparently missing from '"$dir"'/.gitignore' >&2
+      gitignores_ok=false
+    fi
+  fi
+done
+"$gitignores_ok"