mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore(corp/data-import): make OR data archive available in env

Browse source 
Change-Id: Idacf42743051eae0cf7010f952a4f91af17ad708
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7892
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
This commit is contained in:
Vincent Ambo 2023-01-20 11:54:38 +03:00 committed by tazjin
parent 1a84e3e6d8
commit ee0c0ee951
1 changed files with 15 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
corp/russian/data-import/default.nix
View file

@ -9,22 +9,29 @@ let
# mirrored input data from OpenCorpora, as of 2023-01-17. # mirrored input data from OpenCorpora, as of 2023-01-17.
# #
# This data is licensed under CC-BY-SA. # This data is licensed under CC-BY-SA.
inputDataArchive = pkgs.fetchurl { openCorporaArchive = pkgs.fetchurl {
name = "dict.opcorpora.xml.bz"; name = "dict.opcorpora.xml.bz";
url = "https://tazj.in/blobs/dict.opcorpora.xml.bz2"; url = "https://tazj.in/blobs/opencorpora-20230117.xml.bz2";
sha256 = "04n5g43fkfc93z6xlwf2qfdrfdfl562pc2ivdb3cbbbsy56gkqg6"; sha256 = "04n5g43fkfc93z6xlwf2qfdrfdfl562pc2ivdb3cbbbsy56gkqg6";
}; };
inputData = pkgs.runCommand "dict.opcorpora.xml" { } '' openCorpora = pkgs.runCommand "dict.opcorpora.xml" { } ''
${pkgs.bzip2}/bin/bunzip2 -k -c ${inputDataArchive} > $out ${pkgs.bzip2}/bin/bunzip2 -k -c ${openCorporaArchive} > $out
''; '';
openRussianArchive = pkgs.fetchzip {
name = "openrussian-20230117";
url = "https://tazj.in/blobs/openrussian-20230117.tar.xz";
sha256 = "06jl7i23cx58a0n2626hb82xlzimixvnxp7lxdw0g664kv9bmw25";
};
# development shell with native deps # development shell with native deps
shell = pkgs.mkShell { shell = pkgs.mkShell {
inherit buildInputs; inherit buildInputs;
# make OPENCORPORA_DATA available in the environment # make datasets available in the environment
OPENCORPORA_DATA = inputData; OPENCORPORA_DATA = openCorpora;
OPENRUSSIAN_DATA = openRussianArchive;
}; };
in in
@ -33,11 +40,11 @@ lib.fix (self: depot.third_party.naersk.buildPackage {
inherit buildInputs; inherit buildInputs;
passthru = depot.nix.readTree.drvTargets { passthru = depot.nix.readTree.drvTargets {
inherit shell inputData; inherit shell openCorpora;
# target that actually builds an entire database # target that actually builds an entire database
database = pkgs.runCommand "tvl-russian-db.sqlite" { } '' database = pkgs.runCommand "tvl-russian-db.sqlite" { } ''
${self}/bin/data-import ${inputData} $out ${self}/bin/data-import ${openCorpora} $out
''; '';
}; };
}) })