refactor(3p/nix): Replace tokenizeStrings with absl::StrSplit

This function was a custom (and inefficient in the case of
single-character delimiters) string splitter which was used all over
the codebase. Abseil provides an appropriate replacement function.

third_party/nix/src/libstore/binary-cache-store.cc

@@ -6,6 +6,7 @@
 
 #include <absl/strings/ascii.h>
 #include <absl/strings/numbers.h>
+#include <absl/strings/str_split.h>
 
 #include "archive.hh"
 #include "compression.hh"
@@ -41,7 +42,7 @@ void BinaryCacheStore::init() {
     upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n",
                "text/x-nix-cache-info");
   } else {
-    for (auto& line : tokenizeString<Strings>(*cacheInfo, "\n")) {
+    for (auto& line : absl::StrSplit(*cacheInfo, absl::ByChar('\n'))) {
       size_t colon = line.find(':');
       if (colon == std::string::npos) {
         continue;

third_party/nix/src/libstore/build.cc

@@ -10,10 +10,12 @@
 #include <queue>
 #include <regex>
 #include <sstream>
+#include <string>
 #include <thread>
 
 #include <absl/strings/ascii.h>
 #include <absl/strings/numbers.h>
+#include <absl/strings/str_split.h>
 #include <fcntl.h>
 #include <grp.h>
 #include <netdb.h>
@@ -1968,7 +1970,7 @@ void DerivationGoal::startBuilder() {
        by `nix-store --register-validity'.  However, the deriver
        fields are left empty. */
     std::string s = get(drv->env, "exportReferencesGraph");
-    auto ss = tokenizeString<Strings>(s);
+    std::vector<std::string> ss = absl::StrSplit(s, absl::ByAnyChar(" \t\n\r"));
     if (ss.size() % 2 != 0) {
       throw BuildError(
           format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s);
@@ -2481,7 +2483,8 @@ void DerivationGoal::initTmpDir() {
      needed (attributes are not passed through the environment, so
      there is no size constraint). */
   if (!parsedDrv->getStructuredAttrs()) {
-    auto passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
+    std::set<std::string> passAsFile =
+        absl::StrSplit(get(drv->env, "passAsFile"), absl::ByAnyChar(" \t\n\r"));
     int fileNr = 0;
     for (auto& i : drv->env) {
       if (passAsFile.find(i.first) == passAsFile.end()) {

third_party/nix/src/libstore/builtins/buildenv.cc

@@ -1,6 +1,7 @@
 #include <algorithm>
 
 #include <absl/strings/match.h>
+#include <absl/strings/str_split.h>
 #include <fcntl.h>
 #include <glog/logging.h>
 #include <sys/stat.h>
@@ -134,12 +135,14 @@ static void addPkg(const Path& pkgDir, int priority) {
   createLinks(pkgDir, out, priority);
 
   try {
-    for (const auto& p : tokenizeString<std::vector<std::string>>(
+    for (auto p : absl::StrSplit(
              readFile(pkgDir + "/nix-support/propagated-user-env-packages"),
-             " \n"))
-      if (!done.count(p)) {
-        postponed.insert(p);
+             absl::ByAnyChar(" \n"))) {
+      auto pkg = std::string(p);
+      if (!done.count(pkg)) {
+        postponed.insert(pkg);
       }
+    }
   } catch (SysError& e) {
     if (e.errNo != ENOENT && e.errNo != ENOTDIR) {
       throw;
@@ -172,7 +175,8 @@ void builtinBuildenv(const BasicDerivation& drv) {
   /* Convert the stuff we get from the environment back into a
    * coherent data type. */
   Packages pkgs;
-  auto derivations = tokenizeString<Strings>(getAttr("derivations"));
+  Strings derivations =
+      absl::StrSplit(getAttr("derivations"), absl::ByAnyChar(" \t\n\r"));
   while (!derivations.empty()) {
     /* !!! We're trusting the caller to structure derivations env var correctly
      */

third_party/nix/src/libstore/derivations.cc

@@ -1,6 +1,8 @@
 #include "derivations.hh"
 
 #include <absl/strings/match.h>
+#include <absl/strings/str_split.h>
+#include <absl/strings/string_view.h>
 
 #include "fs-accessor.hh"
 #include "globals.hh"
@@ -357,13 +359,15 @@ Hash hashDerivationModulo(Store& store, Derivation drv) {
   return hashString(htSHA256, drv.unparse());
 }
 
-DrvPathWithOutputs parseDrvPathWithOutputs(const std::string& s) {
-  size_t n = s.find('!');
-  return n == std::string::npos
-             ? DrvPathWithOutputs(s, std::set<std::string>())
-             : DrvPathWithOutputs(std::string(s, 0, n),
-                                  tokenizeString<std::set<std::string> >(
-                                      std::string(s, n + 1), ","));
+// TODO(tazjin): doc comment?
+DrvPathWithOutputs parseDrvPathWithOutputs(absl::string_view path) {
+  auto pos = path.find('!');
+  if (pos == absl::string_view::npos) {
+    return DrvPathWithOutputs(path, std::set<std::string>());
+  }
+
+  return DrvPathWithOutputs(path.substr(pos + 1),
+                            absl::StrSplit(path, absl::ByChar(',')));
 }
 
 Path makeDrvPathWithOutputs(const Path& drvPath,

third_party/nix/src/libstore/derivations.hh

@@ -88,7 +88,7 @@ extern DrvHashes drvHashes;  // FIXME: global, not thread-safe
    (/nix/store/hash-foo!out1,out2,...) into the derivation path and
    the outputs. */
 typedef std::pair<std::string, std::set<std::string> > DrvPathWithOutputs;
-DrvPathWithOutputs parseDrvPathWithOutputs(const std::string& s);
+DrvPathWithOutputs parseDrvPathWithOutputs(absl::string_view path);
 
 Path makeDrvPathWithOutputs(const Path& drvPath,
                             const std::set<std::string>& outputs);

third_party/nix/src/libstore/download.cc

@@ -3,6 +3,7 @@
 #include <absl/strings/ascii.h>
 #include <absl/strings/match.h>
 #include <absl/strings/numbers.h>
+#include <absl/strings/str_split.h>
 
 #include "archive.hh"
 #include "compression.hh"
@@ -180,7 +181,7 @@ struct CurlDownloader : public Downloader {
                  << "': " << absl::StripAsciiWhitespace(line);
       if (line.compare(0, 5, "HTTP/") == 0) {  // new response starts
         result.etag = "";
-        auto ss = tokenizeString<std::vector<std::string>>(line, " ");
+        std::vector<std::string> ss = absl::StrSplit(line, absl::ByChar(' '));
         status = ss.size() >= 2 ? ss[1] : "";
         result.data = std::make_shared<std::string>();
         result.bodySize = 0;
@@ -894,8 +895,8 @@ CachedDownloadResult Downloader::downloadCached(
     storePath = readLink(fileLink);
     store->addTempRoot(storePath);
     if (store->isValidPath(storePath)) {
-      auto ss =
-          tokenizeString<std::vector<std::string>>(readFile(dataFile), "\n");
+      std::vector<std::string> ss =
+          absl::StrSplit(readFile(dataFile), absl::ByChar('\n'));
       if (ss.size() >= 3 && ss[0] == url) {
         time_t lastChecked;
         if (absl::SimpleAtoi(ss[2], &lastChecked) &&

third_party/nix/src/libstore/gc.cc

@@ -7,6 +7,7 @@
 #include <regex>
 
 #include <absl/strings/match.h>
+#include <absl/strings/str_split.h>
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/statvfs.h>
@@ -422,8 +423,8 @@ void LocalStore::findRuntimeRoots(Roots& roots, bool censor) {
 
         try {
           auto mapFile = fmt("/proc/%s/maps", ent->d_name);
-          auto mapLines = tokenizeString<std::vector<std::string>>(
-              readFile(mapFile, true), "\n");
+          std::vector<std::string> mapLines =
+              absl::StrSplit(readFile(mapFile, true), absl::ByChar('\n'));
           for (const auto& line : mapLines) {
             auto match = std::smatch{};
             if (std::regex_match(line, match, mapRegex)) {
@@ -452,31 +453,9 @@ void LocalStore::findRuntimeRoots(Roots& roots, bool censor) {
     }
   }
 
-#if !defined(__linux__)
-  // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test
-  // to fail. See: https://github.com/NixOS/nix/issues/3011 Because of this we
-  // disable lsof when running the tests.
-  if (getEnv("_NIX_TEST_NO_LSOF") == "") {
-    try {
-      std::regex lsofRegex(R"(^n(/.*)$)");
-      auto lsofLines = tokenizeString<std::vector<std::string>>(
-          runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n");
-      for (const auto& line : lsofLines) {
-        std::smatch match;
-        if (std::regex_match(line, match, lsofRegex))
-          unchecked[match[1]].emplace("{lsof}");
-      }
-    } catch (ExecError& e) {
-      /* lsof not installed, lsof failed */
-    }
-  }
-#endif
-
-#if defined(__linux__)
   readFileRoots("/proc/sys/kernel/modprobe", unchecked);
   readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
   readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
-#endif
 
   for (auto& [target, links] : unchecked) {
     if (isInStore(target)) {

third_party/nix/src/libstore/globals.cc

@@ -5,6 +5,8 @@
 #include <thread>
 
 #include <absl/strings/numbers.h>
+#include <absl/strings/str_cat.h>
+#include <absl/strings/str_split.h>
 #include <dlfcn.h>
 
 #include "archive.hh"
@@ -20,9 +22,6 @@ namespace nix {
    must be deleted and recreated on startup.) */
 #define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
 
-// TODO(tazjin): this was __APPLE__ specific, still needed?
-#define DEFAULT_ALLOWED_IMPURE_PREFIXES ""
-
 Settings settings;
 
 static GlobalConfig::Register r1(&settings);
@@ -55,21 +54,18 @@ Settings::Settings()
   }
 
   /* Backwards compatibility. */
+  // TODO(tazjin): still?
   auto s = getEnv("NIX_REMOTE_SYSTEMS");
   if (!s.empty()) {
     Strings ss;
-    for (auto& p : tokenizeString<Strings>(s, ":")) {
-      ss.push_back("@" + p);
+    for (auto p : absl::StrSplit(s, absl::ByChar(':'))) {
+      ss.push_back(absl::StrCat("@", p));
     }
     builders = concatStringsSep(" ", ss);
   }
 
-#if defined(__linux__) && defined(SANDBOX_SHELL)
-  sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
-#endif
-
-  allowedImpureHostPrefixes =
-      tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
+  sandboxPaths =
+      absl::StrSplit("/bin/sh=" SANDBOX_SHELL, absl::ByAnyChar(" \t\n\r"));
 }
 
 void loadConfFile() {

third_party/nix/src/libstore/local-store.cc

@@ -8,6 +8,7 @@
 #include <iostream>
 
 #include <absl/strings/numbers.h>
+#include <absl/strings/str_split.h>
 #include <fcntl.h>
 #include <glog/logging.h>
 #include <grp.h>
@@ -477,14 +478,15 @@ static void canonicalisePathMetaData_(const Path& path, uid_t fromUid,
       throw SysError("querying extended attributes of '%s'", path);
     }
 
-    for (auto& eaName : tokenizeString<Strings>(
-             std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
+    for (auto& eaName :
+         absl::StrSplit(std::string(eaBuf.data(), eaSize),
+                        absl::ByString(std::string("\000", 1)))) {
       /* Ignore SELinux security labels since these cannot be
          removed even by root. */
       if (eaName == "security.selinux") {
         continue;
       }
-      if (lremovexattr(path.c_str(), eaName.c_str()) == -1) {
+      if (lremovexattr(path.c_str(), std::string(eaName).c_str()) == -1) {
         throw SysError("removing extended attribute '%s' from '%s'", eaName,
                        path);
       }
@@ -702,7 +704,7 @@ void LocalStore::queryPathInfoUncached(
 
       s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 6);
       if (s != nullptr) {
-        info->sigs = tokenizeString<StringSet>(s, " ");
+        info->sigs = absl::StrSplit(s, absl::ByChar(' '));
       }
 
       s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 7);

third_party/nix/src/libstore/local-store.hh

@@ -5,6 +5,8 @@
 #include <string>
 #include <unordered_set>
 
+#include <absl/strings/str_split.h>
+
 #include "pathlocks.hh"
 #include "sqlite.hh"
 #include "store-api.hh"
@@ -94,7 +96,9 @@ class LocalStore : public LocalFSStore {
 
  public:
   // Hack for build-remote.cc.
-  PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
+  // TODO(tazjin): remove this when we've got gRPC
+  PathSet locksHeld =
+      absl::StrSplit(getEnv("NIX_HELD_LOCKS"), absl::ByAnyChar(" \t\n\r"));
 
   /* Initialise the local store, upgrading the schema if
      necessary. */

third_party/nix/src/libstore/machines.cc

@@ -4,6 +4,7 @@
 
 #include <absl/strings/ascii.h>
 #include <absl/strings/match.h>
+#include <absl/strings/str_split.h>
 #include <absl/strings/string_view.h>
 #include <glog/logging.h>
 
@@ -51,14 +52,15 @@ bool Machine::mandatoryMet(const std::set<std::string>& features) const {
 }
 
 void parseMachines(const std::string& s, Machines& machines) {
-  for (auto line : tokenizeString<std::vector<std::string>>(s, "\n;")) {
-    line.erase(std::find(line.begin(), line.end(), '#'), line.end());
-    if (line.empty()) {
+  for (auto line : absl::StrSplit(s, absl::ByAnyChar("\n;"))) {
+    // Skip empty lines & comments
+    line = absl::StripAsciiWhitespace(line);
+    if (line.empty() || line[line.find_first_not_of(" \t")] == '#') {
       continue;
     }
 
     if (line[0] == '@') {
-      auto file = absl::StripAsciiWhitespace(std::string(line, 1));
+      auto file = absl::StripAsciiWhitespace(line.substr(1));
       try {
         parseMachines(readFile(file), machines);
       } catch (const SysError& e) {
@@ -70,7 +72,8 @@ void parseMachines(const std::string& s, Machines& machines) {
       continue;
     }
 
-    auto tokens = tokenizeString<std::vector<std::string>>(line);
+    std::vector<std::string> tokens =
+        absl::StrSplit(line, absl::ByAnyChar(" \t\n\r"));
     auto sz = tokens.size();
     if (sz < 1) {
       throw FormatError("bad machine specification '%s'", line);
@@ -80,15 +83,16 @@ void parseMachines(const std::string& s, Machines& machines) {
       return tokens.size() > n && !tokens[n].empty() && tokens[n] != "-";
     };
 
+    // TODO(tazjin): what???
     machines.emplace_back(
         tokens[0],
-        isSet(1) ? tokenizeString<std::vector<std::string>>(tokens[1], ",")
+        isSet(1) ? absl::StrSplit(tokens[1], absl::ByChar(','))
                  : std::vector<std::string>{settings.thisSystem},
         isSet(2) ? tokens[2] : "", isSet(3) ? std::stoull(tokens[3]) : 1LL,
         isSet(4) ? std::stoull(tokens[4]) : 1LL,
-        isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",")
+        isSet(5) ? absl::StrSplit(tokens[5], absl::ByChar(','))
                  : std::set<std::string>{},
-        isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",")
+        isSet(6) ? absl::StrSplit(tokens[6], absl::ByChar(','))
                  : std::set<std::string>{},
         isSet(7) ? tokens[7] : "");
   }

third_party/nix/src/libstore/nar-info-disk-cache.cc

@@ -1,5 +1,7 @@
 #include "nar-info-disk-cache.hh"
 
+#include <absl/strings/str_cat.h>
+#include <absl/strings/str_split.h>
 #include <glog/logging.h>
 #include <sqlite3.h>
 
@@ -227,14 +229,15 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache {
           narInfo->fileSize = queryNAR.getInt(5);
           narInfo->narHash = Hash(queryNAR.getStr(6));
           narInfo->narSize = queryNAR.getInt(7);
-          for (auto& r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) {
-            narInfo->references.insert(cache.storeDir + "/" + r);
+          for (auto r : absl::StrSplit(queryNAR.getStr(8), absl::ByChar(' '))) {
+            narInfo->references.insert(absl::StrCat(cache.storeDir, "/", r));
           }
           if (!queryNAR.isNull(9)) {
             narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9);
           }
-          for (auto& sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) {
-            narInfo->sigs.insert(sig);
+          for (auto& sig :
+               absl::StrSplit(queryNAR.getStr(10), absl::ByChar(' '))) {
+            narInfo->sigs.insert(std::string(sig));
           }
           narInfo->ca = queryNAR.getStr(11);
 

third_party/nix/src/libstore/nar-info.cc

@@ -1,6 +1,7 @@
 #include "nar-info.hh"
 
 #include <absl/strings/numbers.h>
+#include <absl/strings/str_split.h>
 
 #include "globals.hh"
 
@@ -59,7 +60,7 @@ NarInfo::NarInfo(const Store& store, const std::string& s,
         corrupt();
       }
     } else if (name == "References") {
-      auto refs = tokenizeString<Strings>(value, " ");
+      std::vector<std::string> refs = absl::StrSplit(value, absl::ByChar(' '));
       if (!references.empty()) {
         corrupt();
       }

third_party/nix/src/libstore/parsed-derivations.cc

@@ -1,5 +1,7 @@
 #include "parsed-derivations.hh"
 
+#include <absl/strings/str_split.h>
+
 namespace nix {
 
 ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv)
@@ -86,7 +88,7 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(
     if (i == drv.env.end()) {
       return {};
     }
-    return tokenizeString<Strings>(i->second);
+    return absl::StrSplit(i->second, absl::ByAnyChar(" \t\n\r"));
   }
 }
 

third_party/nix/src/libstore/ssh.cc

@@ -3,6 +3,7 @@
 #include <utility>
 
 #include <absl/strings/match.h>
+#include <absl/strings/str_split.h>
 
 namespace nix {
 
@@ -20,8 +21,9 @@ SSHMaster::SSHMaster(const std::string& host, std::string keyFile,
 }
 
 void SSHMaster::addCommonSSHOpts(Strings& args) {
-  for (auto& i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS"))) {
-    args.push_back(i);
+  for (auto& i :
+       absl::StrSplit(getEnv("NIX_SSHOPTS"), absl::ByAnyChar(" \t\n\r"))) {
+    args.push_back(std::string(i));
   }
   if (!keyFile.empty()) {
     args.insert(args.end(), {"-i", keyFile});

third_party/nix/src/libstore/store-api.cc

@@ -5,6 +5,7 @@
 
 #include <absl/strings/match.h>
 #include <absl/strings/numbers.h>
+#include <absl/strings/str_split.h>
 #include <glog/logging.h>
 
 #include "crypto.hh"
@@ -858,7 +859,8 @@ std::pair<std::string, Store::Params> splitUriAndParams(
   Store::Params params;
   auto q = uri.find('?');
   if (q != std::string::npos) {
-    for (const auto& s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
+    Strings parts = absl::StrSplit(uri.substr(q + 1), absl::ByChar('&'));
+    for (const auto& s : parts) {
       auto e = s.find('=');
       if (e != std::string::npos) {
         auto value = s.substr(e + 1);