refactor(tvix): Factor-out proto utilities

Factor out the shared utilities for interacting with protobufs/grpc from
libstore to a proto.hh header in libproto.

Change-Id: I1cb8d94867d5d4b63a9994be0b53f8f612eb8e3a
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1691
Tested-by: BuildkiteCI
Reviewed-by: kanepyork <rikingcoding@gmail.com>
This commit is contained in:
Griffin Smith 2020-08-08 15:13:22 -04:00 committed by glittershark
parent ef7894273b
commit 747dc65154
3 changed files with 162 additions and 136 deletions

View file

@ -7,6 +7,8 @@
#include <absl/status/status.h>
#include <absl/strings/str_cat.h>
#include <absl/strings/str_format.h>
#include <absl/strings/string_view.h>
#include <glog/logging.h>
#include <google/protobuf/empty.pb.h>
#include <google/protobuf/util/time_util.h>
#include <grpcpp/create_channel.h>
@ -23,6 +25,7 @@
#include "libstore/store-api.hh"
#include "libutil/archive.hh"
#include "libutil/hash.hh"
#include "libutil/proto.hh"
#include "libutil/types.hh"
namespace nix {
@ -35,27 +38,6 @@ using nix::proto::WorkerService;
static google::protobuf::Empty kEmpty;
proto::StorePath StorePath(const Path& path) {
proto::StorePath store_path;
store_path.set_path(path);
return store_path;
}
proto::StorePaths StorePaths(const PathSet& paths) {
proto::StorePaths result;
for (const auto& path : paths) {
result.add_paths(path);
}
return result;
}
template <typename T, typename U>
T FillFrom(const U& src) {
T result;
result.insert(src.begin(), src.end());
return result;
}
template <typename Request>
class RPCSink : public BufferedSink {
public:
@ -85,88 +67,6 @@ class RPCSink : public BufferedSink {
bool good_;
};
constexpr absl::StatusCode GRPCStatusCodeToAbsl(grpc::StatusCode code) {
switch (code) {
case grpc::StatusCode::OK:
return absl::StatusCode::kOk;
case grpc::StatusCode::CANCELLED:
return absl::StatusCode::kCancelled;
case grpc::StatusCode::UNKNOWN:
return absl::StatusCode::kUnknown;
case grpc::StatusCode::INVALID_ARGUMENT:
return absl::StatusCode::kInvalidArgument;
case grpc::StatusCode::DEADLINE_EXCEEDED:
return absl::StatusCode::kDeadlineExceeded;
case grpc::StatusCode::NOT_FOUND:
return absl::StatusCode::kNotFound;
case grpc::StatusCode::ALREADY_EXISTS:
return absl::StatusCode::kAlreadyExists;
case grpc::StatusCode::PERMISSION_DENIED:
return absl::StatusCode::kPermissionDenied;
case grpc::StatusCode::UNAUTHENTICATED:
return absl::StatusCode::kUnauthenticated;
case grpc::StatusCode::RESOURCE_EXHAUSTED:
return absl::StatusCode::kResourceExhausted;
case grpc::StatusCode::FAILED_PRECONDITION:
return absl::StatusCode::kFailedPrecondition;
case grpc::StatusCode::ABORTED:
return absl::StatusCode::kAborted;
case grpc::StatusCode::OUT_OF_RANGE:
return absl::StatusCode::kOutOfRange;
case grpc::StatusCode::UNIMPLEMENTED:
return absl::StatusCode::kUnimplemented;
case grpc::StatusCode::INTERNAL:
return absl::StatusCode::kInternal;
case grpc::StatusCode::UNAVAILABLE:
return absl::StatusCode::kUnavailable;
case grpc::StatusCode::DATA_LOSS:
return absl::StatusCode::kDataLoss;
default:
return absl::StatusCode::kInternal;
}
}
constexpr absl::string_view GRPCStatusCodeDescription(grpc::StatusCode code) {
switch (code) {
case grpc::StatusCode::OK:
return "OK";
case grpc::StatusCode::CANCELLED:
return "CANCELLED";
case grpc::StatusCode::UNKNOWN:
return "UNKNOWN";
case grpc::StatusCode::INVALID_ARGUMENT:
return "INVALID_ARGUMENT";
case grpc::StatusCode::DEADLINE_EXCEEDED:
return "DEADLINE_EXCEEDED";
case grpc::StatusCode::NOT_FOUND:
return "NOT_FOUND";
case grpc::StatusCode::ALREADY_EXISTS:
return "ALREADY_EXISTS";
case grpc::StatusCode::PERMISSION_DENIED:
return "PERMISSION_DENIED";
case grpc::StatusCode::UNAUTHENTICATED:
return "UNAUTHENTICATED";
case grpc::StatusCode::RESOURCE_EXHAUSTED:
return "RESOURCE_EXHAUSTED";
case grpc::StatusCode::FAILED_PRECONDITION:
return "FAILED_PRECONDITION";
case grpc::StatusCode::ABORTED:
return "ABORTED";
case grpc::StatusCode::OUT_OF_RANGE:
return "OUT_OF_RANGE";
case grpc::StatusCode::UNIMPLEMENTED:
return "UNIMPLEMENTED";
case grpc::StatusCode::INTERNAL:
return "INTERNAL";
case grpc::StatusCode::UNAVAILABLE:
return "UNAVAILABLE";
case grpc::StatusCode::DATA_LOSS:
return "DATA_LOSS";
default:
return "<BAD ERROR CODE>";
};
}
// TODO(grfn): Obviously this should go away and be replaced by StatusOr... but
// that would require refactoring the entire store api, which we don't feel like
// doing right now. We should at some point though
@ -177,13 +77,13 @@ void const RpcStore::SuccessOrThrow(const grpc::Status& status,
switch (status.error_code()) {
case grpc::StatusCode::UNIMPLEMENTED:
throw Unsupported(
absl::StrFormat("operation is not supported by store at %s: %s",
uri, status.error_message()));
absl::StrFormat("operation %s is not supported by store at %s: %s",
call, uri, status.error_message()));
default:
throw Error(
absl::StrFormat("Rpc call to %s failed (%s): %s ", uri,
GRPCStatusCodeDescription(status.error_code()),
status.error_message()));
throw Error(absl::StrFormat(
"Rpc call %s to %s failed (%s): %s ", call, uri,
util::proto::GRPCStatusCodeDescription(status.error_code()),
status.error_message()));
}
}
}
@ -191,7 +91,7 @@ void const RpcStore::SuccessOrThrow(const grpc::Status& status,
bool RpcStore::isValidPathUncached(const Path& path) {
ClientContext ctx;
proto::IsValidPathResponse resp;
SuccessOrThrow(stub_->IsValidPath(&ctx, StorePath(path), &resp),
SuccessOrThrow(stub_->IsValidPath(&ctx, util::proto::StorePath(path), &resp),
__FUNCTION__);
return resp.is_valid();
}
@ -200,7 +100,7 @@ PathSet RpcStore::queryAllValidPaths() {
ClientContext ctx;
proto::StorePaths paths;
SuccessOrThrow(stub_->QueryAllValidPaths(&ctx, kEmpty, &paths), __FUNCTION__);
return FillFrom<PathSet>(paths.paths());
return util::proto::FillFrom<PathSet>(paths.paths());
}
PathSet RpcStore::queryValidPaths(const PathSet& paths,
@ -213,7 +113,7 @@ PathSet RpcStore::queryValidPaths(const PathSet& paths,
proto::StorePaths result_paths;
SuccessOrThrow(stub_->QueryValidPaths(&ctx, store_paths, &result_paths),
__FUNCTION__);
return FillFrom<PathSet>(result_paths.paths());
return util::proto::FillFrom<PathSet>(result_paths.paths());
}
void RpcStore::queryPathInfoUncached(
@ -260,33 +160,36 @@ void RpcStore::queryPathInfoUncached(
void RpcStore::queryReferrers(const Path& path, PathSet& referrers) {
ClientContext ctx;
proto::StorePaths paths;
SuccessOrThrow(stub_->QueryReferrers(&ctx, StorePath(path), &paths),
__FUNCTION__);
SuccessOrThrow(
stub_->QueryReferrers(&ctx, util::proto::StorePath(path), &paths),
__FUNCTION__);
referrers.insert(paths.paths().begin(), paths.paths().end());
}
PathSet RpcStore::queryValidDerivers(const Path& path) {
ClientContext ctx;
proto::StorePaths paths;
SuccessOrThrow(stub_->QueryValidDerivers(&ctx, StorePath(path), &paths),
__FUNCTION__);
return FillFrom<PathSet>(paths.paths());
SuccessOrThrow(
stub_->QueryValidDerivers(&ctx, util::proto::StorePath(path), &paths),
__FUNCTION__);
return util::proto::FillFrom<PathSet>(paths.paths());
}
PathSet RpcStore::queryDerivationOutputs(const Path& path) {
ClientContext ctx;
proto::StorePaths paths;
SuccessOrThrow(stub_->QueryDerivationOutputs(&ctx, StorePath(path), &paths),
__FUNCTION__);
return FillFrom<PathSet>(paths.paths());
SuccessOrThrow(
stub_->QueryDerivationOutputs(&ctx, util::proto::StorePath(path), &paths),
__FUNCTION__);
return util::proto::FillFrom<PathSet>(paths.paths());
}
StringSet RpcStore::queryDerivationOutputNames(const Path& path) {
ClientContext ctx;
proto::DerivationOutputNames output_names;
SuccessOrThrow(
stub_->QueryDerivationOutputNames(&ctx, StorePath(path), &output_names));
return FillFrom<StringSet>(output_names.names());
SuccessOrThrow(stub_->QueryDerivationOutputNames(
&ctx, util::proto::StorePath(path), &output_names));
return util::proto::FillFrom<StringSet>(output_names.names());
}
Path RpcStore::queryPathFromHashPart(const std::string& hashPart) {
@ -302,17 +205,17 @@ Path RpcStore::queryPathFromHashPart(const std::string& hashPart) {
PathSet RpcStore::querySubstitutablePaths(const PathSet& paths) {
ClientContext ctx;
proto::StorePaths result;
SuccessOrThrow(
stub_->QuerySubstitutablePaths(&ctx, StorePaths(paths), &result));
return FillFrom<PathSet>(result.paths());
SuccessOrThrow(stub_->QuerySubstitutablePaths(
&ctx, util::proto::StorePaths(paths), &result));
return util::proto::FillFrom<PathSet>(result.paths());
}
void RpcStore::querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos& infos) {
ClientContext ctx;
proto::SubstitutablePathInfos result;
SuccessOrThrow(
stub_->QuerySubstitutablePathInfos(&ctx, StorePaths(paths), &result));
SuccessOrThrow(stub_->QuerySubstitutablePathInfos(
&ctx, util::proto::StorePaths(paths), &result));
for (const auto& path_info : result.path_infos()) {
auto path = path_info.path().path();
@ -321,7 +224,7 @@ void RpcStore::querySubstitutablePathInfos(const PathSet& paths,
if (!info.deriver.empty()) {
assertStorePath(info.deriver);
}
info.references = FillFrom<PathSet>(path_info.references());
info.references = util::proto::FillFrom<PathSet>(path_info.references());
info.downloadSize = path_info.download_size();
info.narSize = path_info.nar_size();
}
@ -353,7 +256,9 @@ void RpcStore::addToStore(const ValidPathInfo& info, Source& narSource,
path_info_req.mutable_path_info()->set_repair(repair);
path_info_req.mutable_path_info()->set_check_sigs(checkSigs);
writer->Write(path_info_req);
if (!writer->Write(path_info_req)) {
throw Error("Could not write to nix daemon");
}
RPCSink sink(std::move(writer));
copyNAR(narSource, sink);
@ -378,7 +283,10 @@ Path RpcStore::addToStore(const std::string& name, const Path& srcPath,
metadata_req.mutable_meta()->set_fixed(!(hashAlgo == htSHA256 && recursive));
metadata_req.mutable_meta()->set_recursive(recursive);
metadata_req.mutable_meta()->set_hash_type(HashTypeToProto(hashAlgo));
writer->Write(metadata_req);
if (!writer->Write(metadata_req)) {
throw Error("Could not write to nix daemon");
}
RPCSink sink(std::move(writer));
dumpPath(std::filesystem::absolute(srcPath), sink);
@ -435,15 +343,17 @@ void RpcStore::ensurePath(const Path& path) {
void RpcStore::addTempRoot(const Path& path) {
ClientContext ctx;
google::protobuf::Empty response;
SuccessOrThrow(stub_->AddTempRoot(&ctx, StorePath(path), &response),
__FUNCTION__);
SuccessOrThrow(
stub_->AddTempRoot(&ctx, util::proto::StorePath(path), &response),
__FUNCTION__);
}
void RpcStore::addIndirectRoot(const Path& path) {
ClientContext ctx;
google::protobuf::Empty response;
SuccessOrThrow(stub_->AddIndirectRoot(&ctx, StorePath(path), &response),
__FUNCTION__);
SuccessOrThrow(
stub_->AddIndirectRoot(&ctx, util::proto::StorePath(path), &response),
__FUNCTION__);
}
void RpcStore::syncWithGC() {
@ -459,7 +369,8 @@ Roots RpcStore::findRoots(bool censor) {
Roots result;
for (const auto& [target, links] : response.roots()) {
auto link_paths = FillFrom<std::unordered_set<std::string>>(links.paths());
auto link_paths =
util::proto::FillFrom<std::unordered_set<std::string>>(links.paths());
result.insert({target, link_paths});
}

View file

@ -18,6 +18,7 @@ set(HEADER_FILES
lru-cache.hh
monitor-fd.hh
pool.hh
proto.hh
ref.hh
serialise.hh
sync.hh

114
third_party/nix/src/libutil/proto.hh vendored Normal file
View file

@ -0,0 +1,114 @@
#pragma once
#include <absl/status/status.h>
#include <grpcpp/impl/codegen/status.h>
#include "libproto/worker.pb.h"
#include "libutil/types.hh"
namespace nix::util::proto {
::nix::proto::StorePath StorePath(const Path& path) {
::nix::proto::StorePath store_path;
store_path.set_path(path);
return store_path;
}
::nix::proto::StorePaths StorePaths(const PathSet& paths) {
::nix::proto::StorePaths result;
for (const auto& path : paths) {
result.add_paths(path);
}
return result;
}
template <typename T, typename U>
T FillFrom(const U& src) {
T result;
result.insert(src.begin(), src.end());
return result;
}
constexpr absl::StatusCode GRPCStatusCodeToAbsl(grpc::StatusCode code) {
switch (code) {
case grpc::StatusCode::OK:
return absl::StatusCode::kOk;
case grpc::StatusCode::CANCELLED:
return absl::StatusCode::kCancelled;
case grpc::StatusCode::UNKNOWN:
return absl::StatusCode::kUnknown;
case grpc::StatusCode::INVALID_ARGUMENT:
return absl::StatusCode::kInvalidArgument;
case grpc::StatusCode::DEADLINE_EXCEEDED:
return absl::StatusCode::kDeadlineExceeded;
case grpc::StatusCode::NOT_FOUND:
return absl::StatusCode::kNotFound;
case grpc::StatusCode::ALREADY_EXISTS:
return absl::StatusCode::kAlreadyExists;
case grpc::StatusCode::PERMISSION_DENIED:
return absl::StatusCode::kPermissionDenied;
case grpc::StatusCode::UNAUTHENTICATED:
return absl::StatusCode::kUnauthenticated;
case grpc::StatusCode::RESOURCE_EXHAUSTED:
return absl::StatusCode::kResourceExhausted;
case grpc::StatusCode::FAILED_PRECONDITION:
return absl::StatusCode::kFailedPrecondition;
case grpc::StatusCode::ABORTED:
return absl::StatusCode::kAborted;
case grpc::StatusCode::OUT_OF_RANGE:
return absl::StatusCode::kOutOfRange;
case grpc::StatusCode::UNIMPLEMENTED:
return absl::StatusCode::kUnimplemented;
case grpc::StatusCode::INTERNAL:
return absl::StatusCode::kInternal;
case grpc::StatusCode::UNAVAILABLE:
return absl::StatusCode::kUnavailable;
case grpc::StatusCode::DATA_LOSS:
return absl::StatusCode::kDataLoss;
default:
return absl::StatusCode::kInternal;
}
}
constexpr absl::string_view GRPCStatusCodeDescription(grpc::StatusCode code) {
switch (code) {
case grpc::StatusCode::OK:
return "OK";
case grpc::StatusCode::CANCELLED:
return "CANCELLED";
case grpc::StatusCode::UNKNOWN:
return "UNKNOWN";
case grpc::StatusCode::INVALID_ARGUMENT:
return "INVALID_ARGUMENT";
case grpc::StatusCode::DEADLINE_EXCEEDED:
return "DEADLINE_EXCEEDED";
case grpc::StatusCode::NOT_FOUND:
return "NOT_FOUND";
case grpc::StatusCode::ALREADY_EXISTS:
return "ALREADY_EXISTS";
case grpc::StatusCode::PERMISSION_DENIED:
return "PERMISSION_DENIED";
case grpc::StatusCode::UNAUTHENTICATED:
return "UNAUTHENTICATED";
case grpc::StatusCode::RESOURCE_EXHAUSTED:
return "RESOURCE_EXHAUSTED";
case grpc::StatusCode::FAILED_PRECONDITION:
return "FAILED_PRECONDITION";
case grpc::StatusCode::ABORTED:
return "ABORTED";
case grpc::StatusCode::OUT_OF_RANGE:
return "OUT_OF_RANGE";
case grpc::StatusCode::UNIMPLEMENTED:
return "UNIMPLEMENTED";
case grpc::StatusCode::INTERNAL:
return "INTERNAL";
case grpc::StatusCode::UNAVAILABLE:
return "UNAVAILABLE";
case grpc::StatusCode::DATA_LOSS:
return "DATA_LOSS";
default:
return "<BAD ERROR CODE>";
};
}
} // namespace nix::util::proto