c891833414
This makes the actual imported database of the ~whole Russian language (all lemmas, grammemes, forms etc.) a Nix build target which is built in CI. This still needs schema normalisation (it's fairly directly mapped to the raw data), but it's already starting to be a useful data set. This also happens to be a pretty cool demonstration of the power of Nix. You can do `nix-build -A corp.russian.data-import.database` and out comes a perfectly valid SQLite database with a valid external data import! Change-Id: I5d6d15e67d0e4a7ff590fad06252be34f5d561fd Reviewed-on: https://cl.tvl.fyi/c/depot/+/7866 Reviewed-by: tazjin <tazjin@tvl.su> Tested-by: BuildkiteCI
43 lines
1.1 KiB
Nix
43 lines
1.1 KiB
Nix
{ depot, lib, pkgs, ... }:
|
|
|
|
let
|
|
buildInputs = with pkgs; [
|
|
sqlite
|
|
pkg-config
|
|
];
|
|
|
|
# mirrored input data from OpenCorpora, as of 2023-01-17.
|
|
#
|
|
# This data is licensed under CC-BY-SA.
|
|
inputDataArchive = pkgs.fetchurl {
|
|
name = "dict.opcorpora.xml.bz";
|
|
url = "https://tazj.in/blobs/dict.opcorpora.xml.bz2";
|
|
sha256 = "04n5g43fkfc93z6xlwf2qfdrfdfl562pc2ivdb3cbbbsy56gkqg6";
|
|
};
|
|
|
|
inputData = pkgs.runCommand "dict.opcorpora.xml" { } ''
|
|
${pkgs.bzip2}/bin/bunzip2 -k -c ${inputDataArchive} > $out
|
|
'';
|
|
|
|
# development shell with native deps
|
|
shell = pkgs.mkShell {
|
|
inherit buildInputs;
|
|
|
|
# make OPENCORPORA_DATA available in the environment
|
|
OPENCORPORA_DATA = inputData;
|
|
};
|
|
|
|
in
|
|
lib.fix (self: depot.third_party.naersk.buildPackage {
|
|
src = depot.third_party.gitignoreSource ./.;
|
|
inherit buildInputs;
|
|
|
|
passthru = depot.nix.readTree.drvTargets {
|
|
inherit shell inputData;
|
|
|
|
# target that actually builds an entire database
|
|
database = pkgs.runCommand "tvl-russian-db.sqlite" { } ''
|
|
${self}/bin/data-import ${inputData} $out
|
|
'';
|
|
};
|
|
})
|