feat(gerrit): Create Gerrit derivation.

This uses the actual Bazel build, using a variety of tricks and hacks to
make it actually work.

Bazel really wants to download linux binaries from the internet and run
them. In lieu of trying to fix the build system to not do this, we
instead put bazel inside an FHS environment, which allows the binaries
to find their dependencies.

We also have to patch a few things:

* We use build --nobuild instead of fetch, so we only fetch the
  dependencies we actually need for the build and not, say, Windows
  binaries.
* We don't remove rules_cc, because we need it as an external
  dependency, not bundled.
* We do some manual fixes on the cache before packing, because we need
  to remove some in-tree sources (so they don't cause the hash to break,
  since the hashes differ each time they're generated), and also remove
  some extraneous files.
* We explicitly turn off the repository and disk caches, because the
  .bazelrc at the root of the Gerrit tree turns them on, with paths
  pointing into the user's home directory.
* detzip is used instead of the zip binary for packing bower_components
  into an archive. detzip doesn't create entries for directories, and
  also doesn't store most metadata (timestamps, etc.), and uses store
  (i.e. uncompressed) compression only. It also sorts the file tree
  before writing them into the file.

Change-Id: I572c43f7175067ecb1b85cdf40dda13a52de1439
Reviewed-on: https://cl.tvl.fyi/c/depot/+/252
Reviewed-by: tazjin <mail@tazj.in>
This commit is contained in:
Luke Granger-Brown 2020-06-14 12:23:47 +01:00 committed by lukegb
parent 02adb10a96
commit a73ca3f43d
3 changed files with 167 additions and 1 deletions

View file

@ -33,6 +33,8 @@ let
autoreconfHook autoreconfHook
bashInteractive bashInteractive
bat bat
buildBazelPackage
buildFHSUserEnv
buildGoModule buildGoModule
buildGoPackage buildGoPackage
buildPackages buildPackages
@ -51,6 +53,7 @@ let
darwin darwin
dockerTools dockerTools
fetchFromGitHub fetchFromGitHub
fetchgit
fetchurl fetchurl
fetchzip fetchzip
fira fira
@ -58,7 +61,6 @@ let
fira-mono fira-mono
fontconfig fontconfig
freetype freetype
gerrit
gettext gettext
glibc glibc
gmock gmock
@ -70,6 +72,7 @@ let
haskell haskell
iana-etc iana-etc
imagemagickBig imagemagickBig
jdk
jetbrains-mono jetbrains-mono
jq jq
kontemplate kontemplate

150
third_party/gerrit/default.nix vendored Normal file
View file

@ -0,0 +1,150 @@
{ depot, pkgs, ... }:
let
detzip = depot.nix.buildGo.program {
name = "detzip";
srcs = [ ./detzip.go ];
};
bazelRunScript = pkgs.writeShellScriptBin "bazel-run" ''
yarn config set cache-folder "$bazelOut/external/yarn_cache"
export HOME="$bazelOut/external/home"
mkdir -p "$bazelOut/external/home"
exec /bin/bazel "$@"
'';
bazelTop = pkgs.buildFHSUserEnv {
name = "bazel";
targetPkgs = pkgs: [
(pkgs.bazel.override { enableNixHacks = true; })
detzip
pkgs.jdk
pkgs.zlib
pkgs.python
pkgs.curl
pkgs.nodejs
pkgs.yarn
pkgs.git
bazelRunScript
];
runScript = "/bin/bazel-run";
};
bazel = bazelTop // { override = x: bazelTop; };
version = "3.2.1";
in
pkgs.buildBazelPackage {
name = "gerrit";
src = pkgs.fetchgit {
url = "https://gerrit.googlesource.com/gerrit";
rev = "v${version}";
sha256 = "1xrckiqc2l07dib22rn6iicgvph8iyxjrzr15bs5x6jdg9cic1s8";
fetchSubmodules = true;
};
patches = [ ./use_detzip.patch ];
bazelTarget = "release";
inherit bazel;
bazelFlags = [
"--repository_cache="
"--disk_cache="
];
removeRulesCC = false;
fetchAttrs = {
sha256 = "18cglkmfi1nwcs8n0j62bkh8l9bcgp1g52ackbzs57v70cg93nq7";
preBuild = ''
rm .bazelversion
'';
buildPhase = ''
runHook preBuild
BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
USER=homeless-shelter \
bazel \
--output_base="$bazelOut" \
--output_user_root="$bazelUserRoot" \
build --nobuild \
--loading_phase_threads=1 \
$bazelFlags \
$bazelFetchFlags \
$bazelTarget
runHook postBuild
'';
installPhase = ''
runHook preInstall
# Remove all built in external workspaces, Bazel will recreate them when building
rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker}
rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker}
rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker}
rm -rf $bazelOut/external/{local_*,\@local_*.marker}
# Clear markers
find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \;
# Remove all vcs files
rm -rf $(find $bazelOut/external -type d -name .git)
rm -rf $(find $bazelOut/external -type d -name .svn)
rm -rf $(find $bazelOut/external -type d -name .hg)
# Removing top-level symlinks along with their markers.
# This is needed because they sometimes point to temporary paths (?).
# For example, in Tensorflow-gpu build:
# platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms
find $bazelOut/external -maxdepth 1 -type l | while read symlink; do
name="$(basename "$symlink")"
rm -rf "$symlink" "$bazelOut/external/@$name.marker"
done
# Patching symlinks to remove build directory reference
find $bazelOut/external -type l | while read symlink; do
new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")"
rm "$symlink"
ln -sf "$new_target" "$symlink"
done
echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version
# Gerrit fixups:
# Remove polymer-bridges and ba-linkify, they're in-repo
rm -rf $bazelOut/external/yarn_cache/v6/npm-polymer-bridges-*
rm -rf $bazelOut/external/yarn_cache/v6/npm-ba-linkify-*
# Normalize permissions on .yarn-{tarball,metadata} files
find $bazelOut/external/yarn_cache \( -name .yarn-tarball.tgz -or -name .yarn-metadata.json \) -exec chmod 644 {} +
(cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/)
runHook postInstall
'';
};
buildAttrs = {
preConfigure = ''
rm .bazelversion
'';
installPhase = ''
mkdir -p "$out"/webapps/
cp bazel-bin/release.war "$out"/webapps/gerrit-${version}.war
'';
};
passthru = {
# A list of plugins that are part of the gerrit.war file.
# Use `java -jar gerrit.war ls | grep -Po '(?<=plugins/)[^.]+' | sed -e 's,^,",' -e 's,$,",' | sort` to generate that list.
plugins = [
"codemirror-editor"
"commit-message-length-validator"
"delete-project"
"download-commands"
"gitiles"
"hooks"
"plugin-manager"
"replication"
"reviewnotes"
"singleusergroup"
"webhooks"
];
};
}

13
third_party/gerrit/use_detzip.patch vendored Normal file
View file

@ -0,0 +1,13 @@
diff --git a/tools/js/download_bower.py b/tools/js/download_bower.py
index 1df4b826bc..65bda74082 100755
--- a/tools/js/download_bower.py
+++ b/tools/js/download_bower.py
@@ -106,7 +106,7 @@ def main():
args.b, '--quiet', 'install', '%s#%s' % (args.p, args.v)))
bc = os.path.join(cwd, 'bower_components')
subprocess.check_call(
- ['zip', '-q', '--exclude', '.bower.json', '-r', cached, args.n],
+ ['detzip', '--exclude', '.bower.json', cached, args.n],
cwd=bc)
if args.s: