mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(corp/data-import): add import of OpenRussian 'words' table

Browse source 
This is actually the lemmata table of this corpus, not the forms of
all words (they're in a separate table).

Change-Id: I89a2c2817ccce840f47406fa2a636f4ed3f49154
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7893
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
This commit is contained in:
Vincent Ambo 2023-01-20 13:31:12 +03:00 • committed by tazjin
parent ee0c0ee951
commit 429c0d00c4
6 changed files with 349 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

11
corp/russian/data-import/default.nix
View file

@ -19,6 +19,9 @@ let
${pkgs.bzip2}/bin/bunzip2 -k -c ${openCorporaArchive} > $out
'';
# mirrored input data from OpenRussian, as of 2023-01-17.
#
# This data is licensed under CC-BY-SA.
openRussianArchive = pkgs.fetchzip {
name = "openrussian-20230117";
url = "https://tazj.in/blobs/openrussian-20230117.tar.xz";
@ -43,8 +46,10 @@ lib.fix (self: depot.third_party.naersk.buildPackage {
inherit shell openCorpora;
# target that actually builds an entire database
database = pkgs.runCommand "tvl-russian-db.sqlite" { } ''
${self}/bin/data-import ${openCorpora} $out
'';
database = pkgs.runCommand "tvl-russian-db.sqlite"
{
OPENCORPORA_DATA = openCorpora;
OPENRUSSIAN_DATA = openRussianArchive;
} "${self}/bin/data-import --output $out";
};
})