Add ci scripts (#1)

* Add ci scripts

* Move config files to ci/kokoro/docker

* Remove dump-logs etc

* Simplified grpc installation

* Add Ubuntu 16.04 kokoro config

* Address code review

CMakeLists.txt

@@ -45,7 +45,7 @@ set(GOOGLE_CLOUD_CPP_GOOGLEAPIS_SHA256
     "6b8a9b2bcb4476e9a5a9872869996f0d639c8d5df76dd8a893e79201f211b1cf")
 
 include(ExternalProject)
-externalproject_add(googleapis_download
+ExternalProject_Add(googleapis_download
                     EXCLUDE_FROM_ALL ON
                     PREFIX "${CMAKE_BINARY_DIR}/external/googleapis"
                     URL ${GOOGLE_CLOUD_CPP_GOOGLEAPIS_URL}
@@ -54,7 +54,7 @@ externalproject_add(googleapis_download
                     BUILD_COMMAND ""
                     INSTALL_COMMAND ""
                     LOG_DOWNLOAD OFF)
-externalproject_get_property(googleapis_download SOURCE_DIR)
+ExternalProject_Get_Property(googleapis_download SOURCE_DIR)
 set(GOOGLEAPIS_CPP_SOURCE "${SOURCE_DIR}")
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
@@ -267,7 +267,8 @@ install(TARGETS ${googleapis_cpp_installed_libraries_list}
 
 foreach (target ${googleapis_cpp_installed_libraries_list})
     google_cloud_cpp_install_proto_library_headers("${target}")
-    google_cloud_cpp_install_proto_library_protos("${target}" "${GOOGLEAPIS_CPP_SOURCE}")
+    google_cloud_cpp_install_proto_library_protos("${target}"
+                                                  "${GOOGLEAPIS_CPP_SOURCE}")
 endforeach ()
 
 # Export the CMake targets to make it easy to create configuration files.
@@ -275,15 +276,12 @@ install(EXPORT googleapis-targets
         DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/googleapis")
 
 # Setup global variables used in the following *.in files.
-set(
+set(GOOGLE_CLOUD_CPP_CONFIG_VERSION_MAJOR
-    GOOGLE_CLOUD_CPP_CONFIG_VERSION_MAJOR ${GOOGLEAPIS_CPP_PROTOS_VERSION_MAJOR}
+    ${GOOGLEAPIS_CPP_PROTOS_VERSION_MAJOR})
-    )
+set(GOOGLE_CLOUD_CPP_CONFIG_VERSION_MINOR
-set(
+    ${GOOGLEAPIS_CPP_PROTOS_VERSION_MINOR})
-    GOOGLE_CLOUD_CPP_CONFIG_VERSION_MINOR ${GOOGLEAPIS_CPP_PROTOS_VERSION_MINOR}
+set(GOOGLE_CLOUD_CPP_CONFIG_VERSION_PATCH
-    )
+    ${GOOGLEAPIS_CPP_PROTOS_VERSION_PATCH})
-set(
-    GOOGLE_CLOUD_CPP_CONFIG_VERSION_PATCH ${GOOGLEAPIS_CPP_PROTOS_VERSION_PATCH}
-    )
 
 # Use a function to create a scope for the variables.
 function (googleapis_cpp_install_pc target)
@@ -322,8 +320,8 @@ function (googleapis_cpp_install_pc target)
     string(CONCAT GOOGLE_CLOUD_CPP_PC_REQUIRES ${_target_pc_requires})
     set(GOOGLE_CLOUD_CPP_PC_LIBS "-l${target}")
     configure_file("cmake/config.pc.in" "${target}.pc" @ONLY)
-    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${target}.pc" DESTINATION
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${target}.pc"
-                  "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+            DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 endfunction ()
 
 # Create and install the pkg-config files.
@@ -355,8 +353,8 @@ string(CONCAT GOOGLE_CLOUD_CPP_PC_REQUIRES
               " libcares")
 set(GOOGLE_CLOUD_CPP_PC_LIBS "")
 configure_file("cmake/config.pc.in" "googleapis.pc" @ONLY)
-install(FILES "${CMAKE_CURRENT_BINARY_DIR}/googleapis.pc" DESTINATION
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/googleapis.pc"
-              "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 
 # Create and install the CMake configuration files.
 configure_file("cmake/config.cmake.in" "googleapis-config.cmake" @ONLY)
@@ -367,5 +365,4 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/googleapis-config.cmake"
               "${PROJECT_SOURCE_DIR}/cmake/FindgRPC.cmake"
               "${PROJECT_SOURCE_DIR}/cmake/FindProtobufTargets.cmake"
               "${PROJECT_SOURCE_DIR}/cmake/CompileProtos.cmake"
-              DESTINATION
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/googleapis")
-              "${CMAKE_INSTALL_LIBDIR}/cmake/googleapis")

ci/check-style.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+if [[ "${CHECK_STYLE}" != "yes" ]]; then
+  echo "Skipping code style check as it is disabled for this build."
+  exit 0
+fi
+
+# This script assumes it is running the top-level google-cloud-cpp directory.
+
+readonly BINDIR="$(dirname "$0")"
+
+# Build paths to ignore in find(1) commands by reading .gitignore.
+declare -a ignore=( -path ./.git )
+if [[ -f .gitignore ]]; then
+  while read -r line; do
+    case "${line}" in
+    [^#]*/*) ignore+=( -o -path "./$(expr "${line}" : '\(.*\)/')" ) ;;
+    [^#]*)   ignore+=( -o -name "${line}" ) ;;
+    esac
+  done < .gitignore
+fi
+
+replace_original_if_changed() {
+  if [[ $# != 2 ]]; then
+    return 1
+  fi
+
+  local original="$1"
+  local reformatted="$2"
+
+  if cmp -s "${original}" "${reformatted}"; then
+    rm -f "${reformatted}"
+  else
+    chmod --reference="${original}" "${reformatted}"
+    mv -f "${reformatted}" "${original}"
+  fi
+}
+
+# Apply cmake_format to all the CMake list files.
+#     https://github.com/cheshirekow/cmake_format
+find . \( "${ignore[@]}" \) -prune -o \
+       \( -name 'CMakeLists.txt' -o -name '*.cmake' \) \
+       -print0 |
+  while IFS= read -r -d $'\0' file; do
+    cmake-format "${file}" >"${file}.tmp"
+    replace_original_if_changed "${file}" "${file}.tmp"
+  done
+
+# Apply buildifier to fix the BUILD and .bzl formatting rules.
+#    https://github.com/bazelbuild/buildtools/tree/master/buildifier
+find . \( "${ignore[@]}" \) -prune -o \
+       \( -name BUILD -o -name '*.bzl' \) \
+       -print0 |
+  xargs -0 buildifier -mode=fix
+
+# Apply shellcheck(1) to emit warnings for common scripting mistakes.
+find . \( "${ignore[@]}" \) -prune -o \
+       -iname '*.sh' -exec shellcheck \
+         --exclude=SC1090 \
+         --exclude=SC2034 \
+         --exclude=SC2153 \
+         --exclude=SC2181 \
+       '{}' \;
+
+# Report any differences created by running the formatting tools.
+git diff --ignore-submodules=all --color --exit-code .

ci/colors.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -z "${COLOR_RESET+x}" ]; then
+  if type tput >/dev/null 2>&1; then
+    readonly COLOR_RED="$(tput setaf 1)"
+    readonly COLOR_GREEN="$(tput setaf 2)"
+    readonly COLOR_YELLOW="$(tput setaf 3)"
+    readonly COLOR_RESET="$(tput sgr0)"
+  else
+    readonly COLOR_RED=""
+    readonly COLOR_GREEN=""
+    readonly COLOR_YELLOW=""
+    readonly COLOR_RESET=""
+  fi
+fi

ci/install-grpc.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+mkdir -p /var/tmp/Downloads
+cd /var/tmp/Downloads
+
+# Install protobuf
+wget -q https://github.com/google/protobuf/archive/v3.8.0.tar.gz
+tar -xf v3.8.0.tar.gz
+(cd protobuf-3.8.0/cmake;
+ cmake \
+   -DCMAKE_BUILD_TYPE=Release \
+   -DBUILD_SHARED_LIBS=yes \
+   -Dprotobuf_BUILD_TESTS=OFF \
+   -H. -Bcmake-out
+ cmake --build cmake-out --target install -- -j "$(nproc)"
+ ldconfig
+)
+
+# Install grpc
+wget -q https://github.com/grpc/grpc/archive/v1.22.0.tar.gz
+tar -xf v1.22.0.tar.gz
+(cd grpc-1.22.0;
+ make -j "$(nproc)"
+ make install
+ ldconfig
+)

ci/kokoro/Dockerfile.centos

@@ -0,0 +1,51 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG DISTRO_VERSION=7
+FROM centos:${DISTRO_VERSION}
+
+# Add /usr/local/lib
+
+# Search paths tweak for the build
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig
+ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64
+ENV PATH=/usr/local/bin:${PATH}
+
+# First install the development tools and OpenSSL. The development tools
+# distributed with CentOS (notably CMake) are too old to build
+# `google-cloud-cpp`. In these instructions, we use `cmake3` obtained from
+# [Software Collections](https://www.softwarecollections.org/).
+
+RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+RUN yum install -y centos-release-scl
+RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
+RUN yum makecache && \
+    yum install -y automake cmake3 curl-devel gcc gcc-c++ git libtool \
+        make openssl-devel pkgconfig tar wget which zlib-devel
+RUN ln -sf /usr/bin/cmake3 /usr/bin/cmake && ln -sf /usr/bin/ctest3 /usr/bin/ctest
+
+# Install c-ares
+RUN mkdir -p /var/tmp/Downloads; \
+    cd /var/tmp/Downloads; \
+    wget -q https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz; \
+    tar -xf cares-1_15_0.tar.gz; \
+    cd /var/tmp/Downloads/c-ares-cares-1_15_0; \
+    ./buildconf && ./configure && make -j $(nproc); \
+    make install; \
+    ldconfig
+
+# Install grpc from source
+WORKDIR /var/tmp/ci
+COPY install-grpc.sh /var/tmp/ci
+RUN /var/tmp/ci/install-grpc.sh

ci/kokoro/Dockerfile.fedora-install

@@ -0,0 +1,38 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG DISTRO_VERSION=30
+FROM fedora:${DISTRO_VERSION}
+
+# Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough
+# for `google-cloud-cpp`. Install these packages and additional development
+# tools to compile the dependencies:
+RUN dnf makecache && \
+    dnf install -y clang clang-tools-extra cmake doxygen findutils gcc-c++ git \
+        grpc-devel grpc-plugins libcxx-devel libcxxabi-devel libcurl-devel \
+        make openssl-devel pkgconfig protobuf-compiler python-pip ShellCheck \
+        tar wget zlib-devel
+
+# Install the the buildifier tool, which does not compile with the default
+# golang compiler for Ubuntu 16.04 and Ubuntu 18.04.
+RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier
+RUN chmod 755 /usr/bin/buildifier
+
+# Install cmake_format to automatically format the CMake list files.
+#     https://github.com/cheshirekow/cmake_format
+# Pin this to an specific version because the formatting changes when the
+# "latest" version is updated, and we do not want the builds to break just
+# because some third party changed something.
+RUN pip install --upgrade pip
+RUN pip install numpy cmake_format==0.5.2

ci/kokoro/Dockerfile.ubuntu

@@ -0,0 +1,81 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG DISTRO_VERSION=18.04
+FROM ubuntu:${DISTRO_VERSION}
+
+RUN apt update && \
+    apt install -y \
+        build-essential \
+        clang \
+        cmake \
+        curl \
+        doxygen \
+        gawk \
+        git \
+        gcc \
+        golang \
+        g++ \
+        libc-ares-dev \
+        libc-ares2 \
+        libssl-dev \
+        make \
+        ninja-build \
+        pkg-config \
+        python-pip \
+        shellcheck \
+        tar \
+        unzip \
+        wget \
+        zlib1g-dev
+
+# Install newer c-ares on Ubuntu 16.04.
+RUN if grep -q 16.04 /etc/lsb-release; then \
+       apt remove libc-ares-dev libc-ares2; \
+       apt install -y automake libtool; \
+       mkdir -p /var/tmp/Downloads; \
+       cd /var/tmp/Downloads; \
+       wget -q https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz; \
+       tar -xf cares-1_15_0.tar.gz; \
+       cd /var/tmp/Downloads/c-ares-cares-1_15_0; \
+       ./buildconf && ./configure && make -j $(nproc); \
+       make install; \
+       ldconfig; \
+    fi
+
+# By default, Ubuntu 18.04 does not install the alternatives for clang-format
+# and clang-tidy, so we need to manually install those.
+RUN if grep -q 18.04 /etc/lsb-release; then \
+      apt update && apt install -y clang-tidy clang-format-7; \
+      update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-6.0 100; \
+      update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-7 100; \
+    fi
+
+# Install the the buildifier tool, which does not compile with the default
+# golang compiler for Ubuntu 16.04 and Ubuntu 18.04.
+RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier
+RUN chmod 755 /usr/bin/buildifier
+
+# Install cmake_format to automatically format the CMake list files.
+#     https://github.com/cheshirekow/cmake_format
+# Pin this to an specific version because the formatting changes when the
+# "latest" version is updated, and we do not want the builds to break just
+# because some third party changed something.
+RUN pip install --upgrade pip
+RUN pip install numpy cmake_format==0.5.2
+
+# Install grpc from source
+WORKDIR /var/tmp/ci
+COPY install-grpc.sh /var/tmp/ci
+RUN /var/tmp/ci/install-grpc.sh

ci/kokoro/create-docker-image.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+# Create a Docker image with all the dependencies necessary to build the
+# project.
+if [[ -z "${PROJECT_ROOT+x}" ]]; then
+  readonly PROJECT_ROOT="$(cd "$(dirname "$0")/../.."; pwd)"
+fi
+source "${PROJECT_ROOT}/ci/kokoro/define-docker-variables.sh"
+
+cd "${PROJECT_ROOT}"
+sudo docker build -t "${IMAGE}:tip" \
+     --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \
+     -f "ci/kokoro/Dockerfile.${DISTRO}" ci

ci/kokoro/define-docker-variables.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+if [[ -n "${IMAGE+x}" ]]; then
+  echo "IMAGE is already defined."
+else
+  readonly IMAGE="apisci-${DISTRO}-${DISTRO_VERSION}"
+  readonly BUILD_OUTPUT="cmake-out/${IMAGE}-${BUILD_NAME}"
+  readonly BUILD_HOME="cmake-out/home/${IMAGE}-${BUILD_NAME}"
+fi

ci/kokoro/docker/build-in-docker-cmake.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+if [[ $# != 2 ]]; then
+  echo "Usage: $(basename "$0") <source-directory> <binary-directory>"
+  exit 1
+fi
+
+readonly SOURCE_DIR="$1"
+readonly BINARY_DIR="$2"
+
+# This script is supposed to run inside a Docker container, see
+# ci/kokoro/cmake/installed-dependencies/build.sh for the expected setup.  The
+# /v directory is a volume pointing to a (clean-ish) checkout of the project:
+if [[ -z "${PROJECT_ROOT+x}" ]]; then
+  readonly PROJECT_ROOT="/v"
+fi
+source "${PROJECT_ROOT}/ci/colors.sh"
+
+echo
+echo "${COLOR_YELLOW}Starting docker build $(date) with $(nproc) cores${COLOR_RESET}"
+echo
+
+echo "================================================================"
+echo "Verify formatting $(date)"
+(cd "${PROJECT_ROOT}" ; ./ci/check-style.sh)
+echo "================================================================"
+
+echo "================================================================"
+echo "Compiling on $(date)"
+echo "================================================================"
+cd "${PROJECT_ROOT}"
+cmake_flags=()
+if [[ "${CLANG_TIDY:-}" = "yes" ]]; then
+  cmake_flags+=("-DGOOGLE_CLOUD_CPP_CLANG_TIDY=yes")
+fi
+if [[ "${GOOGLE_CLOUD_CPP_CXX_STANDARD:-}" != "" ]]; then
+  cmake_flags+=(
+    "-DGOOGLE_CLOUD_CPP_CXX_STANDARD=${GOOGLE_CLOUD_CPP_CXX_STANDARD}")
+fi
+
+if [[ "${CODE_COVERAGE:-}" == "yes" ]]; then
+  cmake_flags+=(
+    "-DCMAKE_BUILD_TYPE=Coverage")
+fi
+
+# Avoid unbound variable error with older bash
+if [[ "${#cmake_flags[@]}" == 0 ]]; then
+  cmake "-H${SOURCE_DIR}" "-B${BINARY_DIR}"
+else
+  cmake "-H${SOURCE_DIR}" "-B${BINARY_DIR}" "${cmake_flags[@]}"
+fi
+cmake --build "${BINARY_DIR}" -- -j "$(nproc)"
+
+# When user a super-build the tests are hidden in a subdirectory. We can tell
+# that ${BINARY_DIR} does not have the tests by checking for this file:
+if [[ -r "${BINARY_DIR}/CTestTestfile.cmake" ]]; then
+  echo "================================================================"
+  # It is Okay to skip the tests in this case because the super build
+  # automatically runs them.
+  echo "Running the unit tests $(date)"
+  env -C "${BINARY_DIR}" ctest \
+      -LE integration-tests \
+      --output-on-failure -j "$(nproc)"
+  echo "================================================================"
+fi
+
+if [[ "${GENERATE_DOCS:-}" = "yes" ]]; then
+  echo "================================================================"
+  echo "Validate Doxygen documentation $(date)"
+  cmake --build "${BINARY_DIR}" --target doxygen-docs
+  echo "================================================================"
+fi
+
+if [[ ${RUN_INTEGRATION_TESTS} == "yes" ]]; then
+  echo "================================================================"
+  echo "Running the integration tests $(date)"
+  echo "================================================================"
+  # shellcheck disable=SC1091
+  source /c/spanner-integration-tests-config.sh
+  export GOOGLE_APPLICATION_CREDENTIALS=/c/spanner-credentials.json
+
+  # Run the integration tests too.
+  env -C "${BINARY_DIR}" ctest \
+      -L integration-tests \
+      --output-on-failure
+  echo "================================================================"
+fi
+
+echo "================================================================"
+echo "Build finished at $(date)"
+echo "================================================================"
+
+exit 0

ci/kokoro/docker/build.sh

@@ -0,0 +1,218 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+export CC=gcc
+export CXX=g++
+export DISTRO=ubuntu
+export DISTRO_VERSION=18.04
+export CMAKE_SOURCE_DIR="."
+
+in_docker_script="ci/kokoro/docker/build-in-docker-cmake.sh"
+
+if [[ $# -eq 1 ]]; then
+  export BUILD_NAME="${1}"
+elif [[ -n "${KOKORO_JOB_NAME:-}" ]]; then
+  # Kokoro injects the KOKORO_JOB_NAME environment variable, the value of this
+  # variable is cloud-cpp/spanner/<config-file-name-without-cfg> (or more
+  # generally <path/to/config-file-without-cfg>). By convention we name these
+  # files `$foo.cfg` for continuous builds and `$foo-presubmit.cfg` for
+  # presubmit builds. Here we extract the value of "foo" and use it as the build
+  # name.
+  BUILD_NAME="$(basename "${KOKORO_JOB_NAME}" "-presubmit")"
+  export BUILD_NAME
+else
+  echo "Aborting build as the build name is not defined."
+  echo "If you are invoking this script via the command line use:"
+  echo "    $0 <build-name>"
+  echo
+  echo "If this script is invoked by Kokoro, the CI system is expected to set"
+  echo "the KOKORO_JOB_NAME environment variable."
+  exit 1
+fi
+
+if [[ "${BUILD_NAME}" = "clang-tidy" ]]; then
+  # Compile with clang-tidy(1) turned on. The build treats clang-tidy warnings
+  # as errors.
+  export DISTRO=fedora-install
+  export DISTRO_VERSION=30
+  export CC=clang
+  export CXX=clang++
+  export CHECK_STYLE=yes
+  export CLANG_TIDY=yes
+elif [[ "${BUILD_NAME}" = "ubuntu-18.04" ]]; then
+  export CC=gcc
+  export CXX=g++
+elif [[ "${BUILD_NAME}" = "ubuntu-16.04" ]]; then
+  export DISTRO_VERSION=16.04
+  export CC=gcc
+  export CXX=g++
+elif [[ "${BUILD_NAME}" = "gcc-4.8" ]]; then
+  # The oldest version of GCC we support is 4.8, this build checks the code
+  # against that version. The use of CentOS 7 for that build is not a
+  # coincidence: the reason we support GCC 4.8 is to support this distribution
+  # (and its commercial cousin: RHEL 7).
+  export CC=gcc
+  export CXX=g++
+  export DISTRO=centos
+  export DISTRO_VERSION=7
+elif [[ "${BUILD_NAME}" = "clang-3.8" ]]; then
+  # The oldest version of Clang we actively test is 3.8. There is nothing
+  # particularly interesting about that version. It is simply the version
+  # included with Ubuntu:16.04, and the oldest version tested by
+  # google-cloud-cpp.
+  export DISTRO=ubuntu
+  export DISTRO_VERSION=16.04
+  export CC=clang
+  export CXX=clang++
+else
+  echo "Unknown BUILD_NAME (${BUILD_NAME}). Fix the Kokoro .cfg file."
+  exit 1
+fi
+
+if [[ -z "${PROJECT_ROOT+x}" ]]; then
+  readonly PROJECT_ROOT="$(cd "$(dirname "$0")/../../.."; pwd)"
+fi
+source "${PROJECT_ROOT}/ci/kokoro/define-docker-variables.sh"
+
+echo "================================================================"
+cd "${PROJECT_ROOT}"
+echo "Building with $(nproc) cores $(date) on ${PWD}."
+
+echo "================================================================"
+echo "Capture Docker version to troubleshoot $(date)."
+sudo docker version
+echo "================================================================"
+
+echo "================================================================"
+echo "Creating Docker image with all the development tools $(date)."
+# We do not want to print the log unless there is an error, so disable the -e
+# flag. Later, we will want to print out the emulator(s) logs *only* if there
+# is an error, so disabling from this point on is the right choice.
+set +e
+mkdir -p "${BUILD_OUTPUT}"
+readonly CREATE_DOCKER_IMAGE_LOG="${BUILD_OUTPUT}/create-build-docker-image.log"
+echo "Logging to ${CREATE_DOCKER_IMAGE_LOG}"
+if ! "${PROJECT_ROOT}/ci/retry-command.sh" \
+       "${PROJECT_ROOT}/ci/kokoro/create-docker-image.sh" \
+         >"${CREATE_DOCKER_IMAGE_LOG}" 2>&1 </dev/null; then
+  cat "${CREATE_DOCKER_IMAGE_LOG}"
+  exit 1
+fi
+echo "Docker image created $(date)."
+sudo docker image ls
+echo "================================================================"
+
+echo "================================================================"
+echo "Running the full build $(date)."
+# The default user for a Docker container has uid 0 (root). To avoid creating
+# root-owned files in the build directory we tell docker to use the current
+# user ID, if known.
+docker_uid="${UID:-0}"
+docker_user="${USER:-root}"
+docker_home_prefix="${PWD}/cmake-out/home"
+if [[ "${docker_uid}" == "0" ]]; then
+  docker_home_prefix="${PWD}/cmake-out/root"
+fi
+
+# Make sure the user has a $HOME directory inside the Docker container.
+mkdir -p "${BUILD_HOME}"
+
+# We use an array for the flags so they are easier to document.
+docker_flags=(
+    # Grant the PTRACE capability to the Docker container running the build,
+    # this is needed by tools like AddressSanitizer.
+    "--cap-add" "SYS_PTRACE"
+
+    # The name and version of the distribution, this is used to call
+    # define-docker-variables.sh and determine the Docker image built, and the
+    # output directory for any artifacts.
+    "--env" "DISTRO=${DISTRO}"
+    "--env" "DISTRO_VERSION=${DISTRO_VERSION}"
+
+    # The C++ and C compiler, both Bazel and CMake use this environment variable
+    # to select the compiler binary.
+    "--env" "CXX=${CXX}"
+    "--env" "CC=${CC}"
+
+    # If set to 'yes', the build script will run the style checks, including
+    # clang-format, cmake-format, and buildifier.
+    "--env" "CHECK_STYLE=${CHECK_STYLE:-}"
+
+    # If set to 'yes', the build script will configure clang-tidy. Currently
+    # only the CMake builds use this flag.
+    "--env" "CLANG_TIDY=${CLANG_TIDY:-}"
+
+    # If set to 'yes', run the integration tests. Currently only the Bazel
+    # builds use this flag.
+    "--env" "RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:-}"
+
+    # If set to 'yes', run compile with code coverage flags. Currently only the
+    # CMake builds use this flag.
+    "--env" "CODE_COVERAGE=${CODE_COVERAGE:-}"
+
+    # If set, pass -DGOOGLE_CLOUD_CPP_CXX_STANDARD=<value> to CMake.
+    "--env" "GOOGLE_CLOUD_CPP_CXX_STANDARD=${GOOGLE_CLOUD_CPP_CXX_STANDARD:-}"
+
+    # When running the integration tests this directory contains the
+    # configuration files needed to run said tests. Make it available inside
+    # the Docker container.
+    "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/c"
+
+    # Let the Docker image script know what kind of terminal we are using, that
+    # produces properly colorized error messages.
+    "--env" "TERM=${TERM:-dumb}"
+
+    # Run the docker script and this user id. Because the docker image gets to
+    # write in ${PWD} you typically want this to be your user id.
+    "--user" "${docker_uid}"
+
+    # Bazel needs this environment variable to work correctly.
+    "--env" "USER=${docker_user}"
+
+    # We give Bazel and CMake a fake $HOME inside the docker image. Bazel caches
+    # build byproducts in this directory. CMake (when ccache is enabled) uses
+    # it to store $HOME/.ccache
+    "--env" "HOME=/h"
+    "--volume" "${PWD}/${BUILD_HOME}:/h"
+
+    # Mount the current directory (which is the top-level directory for the
+    # project) as `/v` inside the docker image, and move to that directory.
+    "--volume" "${PWD}:/v"
+    "--workdir" "/v"
+
+    # Mask any other builds that may exist at the same time. That is, these
+    # directories appear as empty inside the Docker container, this prevents the
+    # container from writing into other builds, or to get confused by the output
+    # of other builds. In the CI system this does not matter, as each build runs
+    # on a completely separate VM. This is useful when running multiple builds
+    # in your workstation.
+    "--volume" "/v/cmake-out/home"
+    "--volume" "/v/cmake-out"
+    "--volume" "${PWD}/${BUILD_OUTPUT}:/v/${BUILD_OUTPUT}"
+)
+
+# When running the builds from the command-line they get a tty, and the scripts
+# running inside the Docker container can produce nicer output. On Kokoro the
+# script does not get a tty, and Docker terminates the program if we pass the
+# `-it` flag.
+if [[ -t 0 ]]; then
+  docker_flags+=("-it")
+fi
+
+sudo docker run "${docker_flags[@]}" "${IMAGE}:tip" \
+    "/v/${in_docker_script}" "${CMAKE_SOURCE_DIR}" \
+    "${BUILD_OUTPUT}"

ci/kokoro/docker/common.cfg

@@ -0,0 +1,17 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build_file: "cpp-cmakefiles/ci/kokoro/docker/build.sh"
+timeout_mins: 120

ci/retry-command.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+# Make three attempts to install the dependencies. It is rare, but from time to
+# time the downloading the packages, building the Docker images, or an installer
+# Bazel, or the Google Cloud SDK fails. To make the CI build more robust, try
+# again when that happens.
+
+if (( $# < 1 )); then
+  echo "Usage: $(basename "$0") program [arguments]"
+  exit 1
+fi
+readonly PROGRAM=${1}
+shift
+
+# Initially, wait at least 2 minutes (the times are in seconds), because it
+# makes no sense to try faster. This used to be 180 seconds, but that ends with
+# sleeps close to 10 minutes, and Travis aborts builds that do not output in
+# 10m.
+min_wait=120
+# Do not exit on failures for this loop.
+set +e
+for i in 1 2 3; do
+  if "${PROGRAM}" "$@"; then
+    exit 0
+  fi
+  # Sleep for a few minutes before trying again.
+  period=$(( (RANDOM % 60) + min_wait ))
+  echo "${PROGRAM} failed (attempt ${i}); trying again in ${period} seconds."
+  sleep ${period}s
+  min_wait=$(( min_wait * 2 ))
+done
+
+exit 1

cmake/CompileProtos.cmake

@@ -54,7 +54,11 @@ endfunction ()
 # `foo/bar/baz.proto` then the directory containing `foo` must be in the search
 # path.
 function (google_cloud_cpp_generate_proto SRCS)
-    cmake_parse_arguments(_opt "" "" "PROTO_PATH_DIRECTORIES" ${ARGN})
+    cmake_parse_arguments(_opt
+                          ""
+                          ""
+                          "PROTO_PATH_DIRECTORIES"
+                          ${ARGN})
     if (NOT _opt_UNPARSED_ARGUMENTS)
         message(SEND_ERROR "Error: google_cloud_cpp_generate_proto() called"
                            " without any proto files")
@@ -100,14 +104,19 @@ function (google_cloud_cpp_generate_proto SRCS)
             OUTPUT "${pb_cc}" "${pb_h}"
             COMMAND $<TARGET_FILE:protobuf::protoc>
                     ARGS
-                    --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+                    --cpp_out
-                              ${protobuf_include_path} "${filename}"
+                    "${CMAKE_CURRENT_BINARY_DIR}"
+                    ${protobuf_include_path}
+                    "${filename}"
             DEPENDS "${filename}" protobuf::protoc
             COMMENT "Running C++ protocol buffer compiler on ${filename}"
             VERBATIM)
     endforeach ()
 
-    set_source_files_properties(${${SRCS}} PROPERTIES GENERATED TRUE)
+    set_source_files_properties(${${SRCS}}
+                                PROPERTIES
+                                GENERATED
+                                TRUE)
     set(${SRCS} ${${SRCS}} PARENT_SCOPE)
 endfunction ()
 
@@ -138,7 +147,11 @@ endfunction ()
 # `foo/bar/baz.proto` then the directory containing `foo` must be in the search
 # path.
 function (google_cloud_cpp_generate_grpcpp SRCS)
-    cmake_parse_arguments(_opt "" "" "PROTO_PATH_DIRECTORIES" ${ARGN})
+    cmake_parse_arguments(_opt
+                          ""
+                          ""
+                          "PROTO_PATH_DIRECTORIES"
+                          ${ARGN})
     if (NOT _opt_UNPARSED_ARGUMENTS)
         message(
             SEND_ERROR "Error: google_cloud_cpp_generate_grpc() called without"
@@ -188,15 +201,19 @@ function (google_cloud_cpp_generate_grpcpp SRCS)
                 $<TARGET_FILE:protobuf::protoc>
                 ARGS
                 --plugin=protoc-gen-grpc=$<TARGET_FILE:gRPC::grpc_cpp_plugin>
-                    "--grpc_out=${CMAKE_CURRENT_BINARY_DIR}"
+                "--grpc_out=${CMAKE_CURRENT_BINARY_DIR}"
-                    "--cpp_out=${CMAKE_CURRENT_BINARY_DIR}"
+                "--cpp_out=${CMAKE_CURRENT_BINARY_DIR}"
-                    ${protobuf_include_path} "${filename}"
+                ${protobuf_include_path}
+                "${filename}"
             DEPENDS "${filename}" protobuf::protoc gRPC::grpc_cpp_plugin
             COMMENT "Running gRPC C++ protocol buffer compiler on ${filename}"
             VERBATIM)
     endforeach ()
 
-    set_source_files_properties(${${SRCS}} PROPERTIES GENERATED TRUE)
+    set_source_files_properties(${${SRCS}}
+                                PROPERTIES
+                                GENERATED
+                                TRUE)
     set(${SRCS} ${${SRCS}} PARENT_SCOPE)
 endfunction ()
 
@@ -215,8 +232,8 @@ function (google_cloud_cpp_install_proto_library_headers target)
                        relative
                        "${header}")
         get_filename_component(dir "${relative}" DIRECTORY)
-        install(FILES "${header}" DESTINATION
+        install(FILES "${header}"
-                      "${CMAKE_INSTALL_INCLUDEDIR}/${dir}")
+                DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${dir}")
     endforeach ()
 endfunction ()
 
@@ -228,21 +245,25 @@ function (google_cloud_cpp_install_proto_library_protos target strip_prefix)
         if (NOT "${proto}" MATCHES "\\.proto$")
             continue()
         endif ()
-	string(REPLACE "${strip_prefix}/"
+        string(REPLACE "${strip_prefix}/"
-		       ""
+                       ""
-		       relative
+                       relative
-		       "${proto}")
+                       "${proto}")
         get_filename_component(dir "${relative}" DIRECTORY)
         # This is modeled after the Protobuf library, it installs the basic
         # protos (think google/protobuf/any.proto) in the include directory for
         # C/C++ code. :shrug:
-        install(FILES "${proto}" DESTINATION
+        install(FILES "${proto}"
-                      "${CMAKE_INSTALL_INCLUDEDIR}/${dir}")
+                DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${dir}")
     endforeach ()
 endfunction ()
 
 function (google_cloud_cpp_proto_library libname)
-    cmake_parse_arguments(_opt "" "" "PROTO_PATH_DIRECTORIES" ${ARGN})
+    cmake_parse_arguments(_opt
+                          ""
+                          ""
+                          "PROTO_PATH_DIRECTORIES"
+                          ${ARGN})
     if (NOT _opt_UNPARSED_ARGUMENTS)
         message(SEND_ERROR "Error: google_cloud_cpp_proto_library() called"
                            " without any proto files")
@@ -267,7 +288,11 @@ function (google_cloud_cpp_proto_library libname)
 endfunction ()
 
 function (google_cloud_cpp_grpcpp_library libname)
-    cmake_parse_arguments(_opt "" "" "PROTO_PATH_DIRECTORIES" ${ARGN})
+    cmake_parse_arguments(_opt
+                          ""
+                          ""
+                          "PROTO_PATH_DIRECTORIES"
+                          ${ARGN})
     if (NOT _opt_UNPARSED_ARGUMENTS)
         message(SEND_ERROR "Error: google_cloud_cpp_proto_library() called"
                            " without any proto files")

cmake/FindProtobufTargets.cmake

@@ -99,14 +99,14 @@ if (protobuf_FOUND)
                    "ProtobufTargets_FOUND = ${ProtobufTargets_FOUND}"
                    " ProtobufTargets_VERSION = ${ProtobufTargets_VERSION}")
     endif ()
-else()
+else ()
     find_package(Protobuf QUIET)
     if (Protobuf_FOUND)
         set(ProtobufTargets_FOUND 1)
         set(ProtobufTargets_VERSION ${Protobuf_VERSION})
 
         if (NOT TARGET protobuf::libprotobuf)
-            add_library(protobuf::libprotobuf INTERFACE IMPORTED)
+            add_library(protobuf::libprotobuf IMPORTED INTERFACE)
             set_property(TARGET protobuf::libprotobuf
                          PROPERTY INTERFACE_INCLUDE_DIRECTORIES
                                   ${Protobuf_INCLUDE_DIR})
@@ -117,7 +117,7 @@ else()
         endif ()
 
         if (NOT TARGET protobuf::libprotobuf-lite)
-            add_library(protobuf::libprotobuf-lite INTERFACE IMPORTED)
+            add_library(protobuf::libprotobuf-lite IMPORTED INTERFACE)
             set_property(TARGET protobuf::libprotobuf-lite
                          PROPERTY INTERFACE_INCLUDE_DIRECTORIES
                                   ${Protobuf_INCLUDE_DIR})
@@ -128,7 +128,7 @@ else()
         endif ()
 
         if (NOT TARGET protobuf::libprotoc)
-            add_library(protobuf::libprotoc INTERFACE IMPORTED)
+            add_library(protobuf::libprotoc IMPORTED INTERFACE)
             set_property(TARGET protobuf::libprotoc
                          PROPERTY INTERFACE_INCLUDE_DIRECTORIES
                                   ${Protobuf_INCLUDE_DIR})

cmake/FindgRPC.cmake

@@ -95,7 +95,7 @@ function (_grpc_fix_grpc_cpp_plugin_target)
             set_property(TARGET gRPC::grpc_cpp_plugin
                          PROPERTY IMPORTED_LOCATION
                                   ${_gRPC_CPP_PLUGIN_EXECUTABLE})
-        else()
+        else ()
             set(gRPC_FOUND "grpc_cpp_plugin-NOTFOUND")
         endif ()
     endif ()
@@ -140,7 +140,7 @@ function (_gRPC_find_library name filename)
     if (${name}_LIBRARY)
         # Use result recorded by a previous call.
         return()
-    else()
+    else ()
         find_library(${name}_LIBRARY_RELEASE NAMES ${filename})
         mark_as_advanced(${name}_LIBRARY_RELEASE)
 
@@ -171,7 +171,7 @@ if (_gRPC_USE_STATIC_LIBS)
     set(_gRPC_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
     if (WIN32)
         set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
-    else()
+    else ()
         set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
     endif ()
 endif ()
@@ -195,7 +195,7 @@ endif ()
 
 if (_gRPC_grpc_LIBRARY)
     if (NOT TARGET gRPC::grpc)
-        add_library(gRPC::grpc UNKNOWN IMPORTED)
+        add_library(gRPC::grpc IMPORTED UNKNOWN)
         set_target_properties(gRPC::grpc
                               PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
                                          "${_gRPC_INCLUDE_DIR}")
@@ -229,7 +229,7 @@ endif ()
 
 if (_gRPC_grpc++_LIBRARY)
     if (NOT TARGET gRPC::grpc++)
-        add_library(gRPC::grpc++ UNKNOWN IMPORTED)
+        add_library(gRPC::grpc++ IMPORTED UNKNOWN)
         set_target_properties(gRPC::grpc++
                               PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
                                          "${_gRPC++_INCLUDE_DIR}")
@@ -266,17 +266,17 @@ if (_gRPC_grpc++_LIBRARY)
             set_property(TARGET gRPC::grpc++
                          APPEND
                          PROPERTY INTERFACE_COMPILE_FEATURES cxx_std_11)
-        elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+        elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
             # CMake 3.5 is still alive and kicking in some older distros, use
             # the compiler-specific versions in these cases.
             set_property(TARGET gRPC::grpc++
                          APPEND
                          PROPERTY INTERFACE_COMPILE_OPTIONS "-std=c++11")
-        elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+        elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
             set_property(TARGET gRPC::grpc++
                          APPEND
                          PROPERTY INTERFACE_COMPILE_OPTIONS "-std=c++11")
-        else()
+        else ()
             message(
                 WARNING
                     "gRPC::grpc++ requires C++11, but this module"