Remove unused locale scripts
We no longer need these scripts, given our current translatewiki workflows
This commit is contained in:
Andy Allan
parent
37a7996d94
commit
76f141fbd5
4 changed files with 0 additions and 937 deletions
|
|
@ -1,382 +0,0 @@
|
|||
#!/usr/bin/env perl
|
||||
use feature ':5.10';
|
||||
use strict;
|
||||
use warnings;
|
||||
use YAML::Syck qw(Dump LoadFile);
|
||||
use Test::Differences;
|
||||
use Pod::Usage ();
|
||||
use Getopt::Long ();
|
||||
|
||||
=head1 NAME
|
||||
|
||||
locale-diff - Compare two YAML files and print how their datastructures differ
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
# --keys is the default
|
||||
diff en.yml is.yml
|
||||
diff --keys en.yml is.yml
|
||||
|
||||
# --untranslated-values compares prints keys whose values don't differ
|
||||
diff --untranslated-values en.yml is.yml
|
||||
|
||||
# --untranslated-values-all compares prints keys whose values
|
||||
# don't differ. Ignoring the blacklist which prunes things
|
||||
# unlikley to be translated
|
||||
diff --untranslated-values-all en.yml is.yml
|
||||
|
||||
# Check that interpolated variables ({{var}} and [[var]]) are the same
|
||||
diff --validate-variables en.yml is.yml
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
This utility prints the differences between two YAML files using
|
||||
L<Test::Differences>. The purpose of it is to diff the files is
|
||||
F<config/locales> to find out what keys need to be added to the
|
||||
translated files when F<en.yml> changes.
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
=over
|
||||
|
||||
=item -h, --help
|
||||
|
||||
Print this help message.
|
||||
|
||||
=item --keys
|
||||
|
||||
Show the hash keys that differ between the two files, useful merging
|
||||
new entries from F<en.yml> to a local file.
|
||||
|
||||
=item --untranslated-values
|
||||
|
||||
Show keys that B<exist in both the compared files> and whose values
|
||||
are exactly the same. Use C<--keys> to a list of values that hasn't
|
||||
been merged.
|
||||
|
||||
The values are pruned according to global and language specific
|
||||
blacklists found in the C<__DATA__> section of this script.
|
||||
|
||||
This helps to find untranslated values.
|
||||
|
||||
=item --untranslated-values-all
|
||||
|
||||
Like C<--untranslated-values> but ignores blacklists.
|
||||
|
||||
=item --validate-variables
|
||||
|
||||
Check that interpolated Ruby i18n variables (C<{{foo}}> and
|
||||
C<[[foo]]>) are equivalent in the two provided files.
|
||||
|
||||
=item --dump-flat
|
||||
|
||||
Dump a flat version of the translation hash in YAML format,
|
||||
i.e. "foo.bar" instead of "{foo}->{bar}".
|
||||
|
||||
=back
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
E<AElig>var ArnfjE<ouml>rE<eth> Bjarmason <avarab@gmail.com>
|
||||
|
||||
=cut
|
||||
|
||||
# Get the command-line options
|
||||
Getopt::Long::Parser->new(
|
||||
config => [ qw< bundling no_ignore_case no_require_order pass_through > ],
|
||||
)->getoptions(
|
||||
'h|help' => \my $help,
|
||||
'keys' => \my $keys,
|
||||
'dump-flat' => \my $dump_flat,
|
||||
'untranslated-values' => \my $untranslated_values,
|
||||
'untranslated-values-all' => \my $untranslated_values_all,
|
||||
'validate-variables' => \my $validate_variables,
|
||||
'reconstruct' => \my $reconstruct,
|
||||
) or help();
|
||||
|
||||
# --keys is the default
|
||||
$keys = 1 if not $untranslated_values_all and not $untranslated_values and not $validate_variables and not $dump_flat;
|
||||
|
||||
# On --help
|
||||
help() if $help;
|
||||
|
||||
# If we're not given two .yml files
|
||||
help() if (@ARGV != 2 or (!-f $ARGV[0] or !-f $ARGV[1])) and not $dump_flat || $reconstruct;
|
||||
|
||||
my ($from, $to) = @ARGV;
|
||||
|
||||
my $from_data = LoadFile($from);
|
||||
my $from_parsed = { iterate($from_data->{basename($from)}) };
|
||||
|
||||
if ($dump_flat)
|
||||
{
|
||||
mark_utf8($from_parsed);
|
||||
|
||||
print Dump $from_parsed;
|
||||
|
||||
exit 0;
|
||||
}
|
||||
|
||||
if ($reconstruct) {
|
||||
mark_utf8($from_parsed);
|
||||
|
||||
my %out;
|
||||
while (my ($k, $v) = each %$from_parsed) {
|
||||
insert_string_deep(\%out, $k, $v);
|
||||
}
|
||||
|
||||
print Dump { basename($from) => \%out };
|
||||
|
||||
exit 0;
|
||||
}
|
||||
|
||||
my $to_data = LoadFile($to);
|
||||
my $to_parsed = { iterate($to_data->{basename($to)}) };
|
||||
|
||||
if ($keys)
|
||||
{
|
||||
print_key_differences($from_parsed, $to_parsed);
|
||||
}
|
||||
elsif ($untranslated_values or $untranslated_values_all)
|
||||
{
|
||||
my @untranslated = untranslated_keys($from_parsed, $to_parsed);
|
||||
|
||||
# Prune according to blacklist
|
||||
if ($untranslated_values) {
|
||||
@untranslated = prune_untranslated_with_blacklist(basename($to), @untranslated);
|
||||
}
|
||||
|
||||
say for @untranslated;
|
||||
} elsif ($validate_variables)
|
||||
{
|
||||
print_validate_variables($from_parsed, $to_parsed);
|
||||
}
|
||||
|
||||
exit 0;
|
||||
|
||||
sub print_key_differences
|
||||
{
|
||||
my ($f, $t) = @_;
|
||||
|
||||
# Hack around Test::Differences wanting a Test::* module loaded
|
||||
$INC{"Test.pm"} = 1;
|
||||
sub Test::ok { print shift }
|
||||
|
||||
# Diff the tree
|
||||
eq_or_diff([ sort keys %$f ], [ sort keys %$t ]);
|
||||
}
|
||||
|
||||
sub untranslated_keys
|
||||
{
|
||||
my ($from_parsed, $to_parsed) = @_;
|
||||
sort grep { exists $to_parsed->{$_} and $from_parsed->{$_} eq $to_parsed->{$_} } keys %$from_parsed;
|
||||
}
|
||||
|
||||
sub prune_untranslated_with_blacklist
|
||||
{
|
||||
my ($language, @keys) = @_;
|
||||
my %keys;
|
||||
@keys{@keys} = ();
|
||||
|
||||
my $end_yaml = LoadFile(*DATA);
|
||||
my $untranslated_values = $end_yaml->{untranslated_values};
|
||||
my $default = $untranslated_values->{default};
|
||||
my $this_language = $untranslated_values->{$language} || {};
|
||||
|
||||
my %bw_list = (%$default, %$this_language);
|
||||
|
||||
while (my ($key, $blacklisted) = each %bw_list)
|
||||
{
|
||||
# FIXME: Does syck actually support true/false booleans in yaml?
|
||||
delete $keys{$key} if $blacklisted eq 'true'
|
||||
}
|
||||
|
||||
sort keys %keys;
|
||||
}
|
||||
|
||||
sub print_validate_variables
|
||||
{
|
||||
my ($f, $t) = @_;
|
||||
|
||||
while (my ($key, $val) = each %$f)
|
||||
{
|
||||
next if exists $f->{$key} and not exists $t->{$key};
|
||||
|
||||
my @from_var = parse_variables_from_string($f->{$key});
|
||||
my @to_var = parse_variables_from_string($t->{$key});
|
||||
|
||||
unless (@from_var ~~ @to_var) {
|
||||
say "$key in $from has (@from_var) and $to has (@to_var)";
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
sub parse_variables_from_string
|
||||
{
|
||||
my ($string) = @_;
|
||||
|
||||
# This probably matches most of the variables
|
||||
my $var = qr/ [a-z0-9_]+? /xs;
|
||||
|
||||
if (my @var = $string =~ m/ \{\{ ($var) \}\} | \[\[ ($var) \]\] /gsx) {
|
||||
return sort grep { defined } @var;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
sub iterate
|
||||
{
|
||||
my ($hash, @path) = @_;
|
||||
my @ret;
|
||||
|
||||
while (my ($k, $v) = each %$hash)
|
||||
{
|
||||
if (ref $v eq 'HASH')
|
||||
{
|
||||
push @ret => iterate($v, @path, $k);
|
||||
}
|
||||
else
|
||||
{
|
||||
push @ret => join(".",@path, $k), $v;
|
||||
}
|
||||
}
|
||||
|
||||
return @ret;
|
||||
}
|
||||
|
||||
# $s = 'foo.bar.baz.spam.eggs.ham'; $h = \%h; $h = $h->{$_} = {} for split /\./, $s; \%h
|
||||
# ==> {foo => {bar => {baz => {spam => {eggs => {ham => {}}}}}}}
|
||||
sub insert_string_deep {
|
||||
my ($h, $ks, $v) = @_;
|
||||
my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
|
||||
$$p = $v;
|
||||
}
|
||||
|
||||
# sub insert_string_deep
|
||||
# {
|
||||
# my ($hash, $key, $value) = @_;
|
||||
#
|
||||
# my @key = split /\./, $key;
|
||||
# my $h = $hash;
|
||||
#
|
||||
# my $i = 0;
|
||||
# for my $k (@key) {
|
||||
# $i ++;
|
||||
# if ($i == @key) {
|
||||
# $h->{$k} = $value;
|
||||
# } else {
|
||||
# if (ref $h->{$k}) {
|
||||
# $h = $h->{$k};
|
||||
# } else {
|
||||
# $h = $h->{$k} = {};
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
|
||||
sub basename
|
||||
{
|
||||
my $name = shift;
|
||||
$name =~ s[\..*?$][];
|
||||
$name =~ s[.*/][];
|
||||
$name;
|
||||
}
|
||||
|
||||
sub mark_utf8
|
||||
{
|
||||
my ($hash) = @_;
|
||||
|
||||
# Mark as UTF-8
|
||||
map { if (ref $_ eq 'ARRAY') { map { utf8::decode($_) } @$_ } else { utf8::decode($_) } } values %$hash;
|
||||
}
|
||||
|
||||
sub help
|
||||
{
|
||||
my %arg = @_;
|
||||
|
||||
Pod::Usage::pod2usage(
|
||||
-verbose => $arg{ verbose },
|
||||
-exitval => $arg{ exitval } || 0,
|
||||
);
|
||||
}
|
||||
|
||||
__DATA__
|
||||
untranslated_values:
|
||||
|
||||
# Default/Per language blacklist/whitelist for the
|
||||
# --untranslated-values switch. "true" as a value indicates that the
|
||||
# key is to be blacklisted, and "false" that it's to be
|
||||
# whitelisted. "false" is only required to whitelist a key
|
||||
# blacklisted by default on a per-language basis.
|
||||
|
||||
default:
|
||||
html.dir: true
|
||||
layouts.intro_3_bytemark: true
|
||||
layouts.intro_3_ucl: true
|
||||
layouts.project_name.h1: true
|
||||
layouts.project_name.title: true
|
||||
site.index.license.project_url: true
|
||||
browse.relation_member.entry: true
|
||||
|
||||
# #{{id}}
|
||||
changeset.changeset.id: true
|
||||
|
||||
de:
|
||||
activerecord.attributes.message.sender: true
|
||||
activerecord.attributes.trace.name: true
|
||||
activerecord.models.changeset: true
|
||||
activerecord.models.relation: true
|
||||
browse.changeset.changeset: true
|
||||
browse.changeset.changesetxml: true
|
||||
browse.changeset.osmchangexml: true
|
||||
browse.changeset.title: true
|
||||
browse.common_details.version: true
|
||||
browse.containing_relation.relation: true
|
||||
browse.relation.relation: true
|
||||
browse.relation.relation_title: true
|
||||
browse.start_rjs.details: true
|
||||
browse.start_rjs.object_list.details: true
|
||||
browse.tag_details.tags: true
|
||||
changeset.changesets.id: true
|
||||
export.start.export_button: true
|
||||
export.start.format: true
|
||||
export.start.output: true
|
||||
export.start.zoom: true
|
||||
export.start_rjs.export: true
|
||||
layouts.export: true
|
||||
layouts.shop: true
|
||||
site.edit.anon_edits: true
|
||||
site.index.license.license_name: true
|
||||
site.index.permalink: true
|
||||
site.key.table.entry.park: true
|
||||
site.search.submit_text: true
|
||||
trace.edit.tags: true
|
||||
trace.trace.in: true
|
||||
trace.trace_form.tags: true
|
||||
trace.trace_optionals.tags: true
|
||||
trace.view.tags: true
|
||||
user.account.public editing.enabled link: true
|
||||
|
||||
is:
|
||||
# ({{link}})
|
||||
site.edit.anon_edits: true
|
||||
|
||||
# Creative Commons Attribution-Share Alike 2.0
|
||||
site.index.license.license_name: true
|
||||
|
||||
# http://creativecommons.org/licenses/by-sa/2.0/
|
||||
site.index.license.license_url: true
|
||||
|
||||
# {{id}}
|
||||
printable_name.with_id: true
|
||||
|
||||
# {{name}} ({{id}})
|
||||
printable_name.with_name: true
|
||||
|
||||
# {{type}}
|
||||
geocoder.search_osm_namefinder.prefix: true
|
||||
|
||||
# {{suffix}}, {{parentname}}
|
||||
geocoder.search_osm_namefinder.suffix_suburb: true
|
||||
|
|
@ -1,442 +0,0 @@
|
|||
#!/usr/bin/env perl
|
||||
use feature ':5.10';
|
||||
use strict;
|
||||
use warnings;
|
||||
use File::Slurp qw(slurp);
|
||||
use YAML::Syck qw(Dump Load LoadFile DumpFile);
|
||||
BEGIN {
|
||||
$YAML::Syck::Headless = 1;
|
||||
$YAML::Syck::SortKeys = 1;
|
||||
}
|
||||
use WWW::Mechanize;
|
||||
use HTML::TableParser::Grid;
|
||||
use Pod::Usage ();
|
||||
use Getopt::Long ();
|
||||
use Data::Dump 'dump';
|
||||
use File::Spec::Functions qw(catfile);
|
||||
use Storable;
|
||||
use autodie;
|
||||
|
||||
=head1 NAME
|
||||
|
||||
merge-from-translatewiki - Get new translations from L<http://translatewiki.net> and selectively merge them with ours
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
# Run this normally, hopefully...
|
||||
merge-from-translatewiki --locales-dir=config/locales
|
||||
|
||||
# Diff the existing files:
|
||||
config/locales$ for i in $(ls *yml | grep -v en.yml); do perl ../../script/locale/diff --dump-flat $i > $i.0 ;done
|
||||
|
||||
# Merge and find out what changed:
|
||||
rails_port$ perl script/locale/merge-from-translatewiki --locales-dir config/locales
|
||||
|
||||
# Or, more complexy:
|
||||
rails_port$ for i in $(svn st config/locales/ | egrep '^M|\\?' | awk '{print $2}' | grep 'yml$'); do rm -v $i; done && svn up config/locales && perl script/locale/merge-from-translatewiki --locales-dir config/locales && svn st config/locales
|
||||
|
||||
# Diff:
|
||||
config/locales$ for i in $(ls *yml | grep -v en.yml); do perl ../../script/locale/diff --dump-flat $i > $i.1 ;done && for i in $(ls *yml | grep -v en.yml); do diff -ru $i.*; done
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
Translatewiki's export process L<is
|
||||
broken|http://trac.openstreetmap.org/ticket/2305>. This script imports
|
||||
new messages from it while tiptoeing around known bugs.
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
=over
|
||||
|
||||
=item -h, --help
|
||||
|
||||
Print this help message.
|
||||
|
||||
=item --locales-dir
|
||||
|
||||
The locales dir we'll merge stuff into. F<config/locales> by default.
|
||||
|
||||
=item --only-new
|
||||
|
||||
Only import translations that don't exists for us yet.
|
||||
|
||||
=item --cache
|
||||
|
||||
Write a L<Storable> cache for things downloaded from Translatewiki and
|
||||
use it if it exists.
|
||||
|
||||
=back
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
E<AElig>var ArnfjE<ouml>rE<eth> Bjarmason <avarab@gmail.com>
|
||||
|
||||
=cut
|
||||
|
||||
# Get the command-line options
|
||||
Getopt::Long::Parser->new(
|
||||
config => [ qw< bundling no_ignore_case no_require_order pass_through > ],
|
||||
)->getoptions(
|
||||
'h|help' => \my $help,
|
||||
'locales-dir=s' => \(my $locales_dir = 'config/locales'),
|
||||
'only-new' => \my $only_new,
|
||||
'cache' => \my $cache,
|
||||
) or help();
|
||||
|
||||
# On --help
|
||||
help() if $help;
|
||||
|
||||
help() unless $locales_dir and -d $locales_dir;
|
||||
|
||||
###
|
||||
### Main
|
||||
###
|
||||
|
||||
### Get Translatewiki data
|
||||
my %translatewiki_languages = translatewiki_languages();
|
||||
|
||||
# Don't process English from Translatewiki
|
||||
delete $translatewiki_languages{en};
|
||||
|
||||
#say Dump \%translatewiki_languages;
|
||||
|
||||
my @translatewiki_languages_codes = keys %translatewiki_languages;
|
||||
my %translatewiki_translations = get_translatewiki_translations(@translatewiki_languages_codes);
|
||||
|
||||
#say Dump \%translatewiki_translations;
|
||||
|
||||
### Get our existing data
|
||||
my %my_translations;
|
||||
my @my_yaml_files = glob catfile($locales_dir, '*.yml');
|
||||
for my $my_yaml_file (@my_yaml_files) {
|
||||
my $basename = basename($my_yaml_file);
|
||||
my $tw_lang = lc $basename;
|
||||
|
||||
say STDERR "Loading my translation $tw_lang ($my_yaml_file)";
|
||||
$my_translations{$tw_lang} = load_and_flatten_yaml(scalar slurp($my_yaml_file));
|
||||
}
|
||||
|
||||
say "loaded my translations";
|
||||
|
||||
## Write out merged data
|
||||
for my $translatewiki_lang (sort @translatewiki_languages_codes) {
|
||||
my $rails_lang = $translatewiki_lang; $rails_lang =~ s/(?<=-)(\w+)/\U$1\E/;
|
||||
my $out_file = catfile($locales_dir, $rails_lang . '.yml');
|
||||
|
||||
unless (-f $out_file) {
|
||||
# No translation like this exists
|
||||
say STDERR "$rails_lang has no existing translation. Importing as-is from Translatewiki to $out_file";
|
||||
my $expanded = expand_hash($translatewiki_translations{$translatewiki_lang});
|
||||
my $out = +{ $rails_lang => $expanded };
|
||||
spit_out($out_file, $out);
|
||||
} elsif (ref $my_translations{$translatewiki_lang} eq 'HASH' and not $only_new) {
|
||||
say STDERR "$rails_lang has existing translations. Merging the old translation with the new Translatewiki one";
|
||||
|
||||
# Get the data
|
||||
my %tw = %{ $translatewiki_translations{$translatewiki_lang} };
|
||||
my %me = %{ $my_translations{$translatewiki_lang} };
|
||||
my %en = %{ $my_translations{en} };
|
||||
# Use %tw to start with
|
||||
my %new = %tw;
|
||||
|
||||
### Merge stuff
|
||||
|
||||
## These keys shouldn't be removed but are due to
|
||||
## Translatewiki fail (they were missing in the original
|
||||
## import)
|
||||
my @url_keys = qw(
|
||||
browse.relation_member.entry
|
||||
changeset.changeset.id
|
||||
geocoder.search_osm_namefinder.suffix_suburb
|
||||
html.dir
|
||||
layouts.intro_3_bytemark
|
||||
layouts.intro_3_ucl
|
||||
layouts.project_name.h1
|
||||
layouts.project_name.title
|
||||
printable_name.with_version
|
||||
site.edit.anon_edits
|
||||
layouts.help_wiki_url
|
||||
layouts.shop_url
|
||||
notifier.gpx_notification.failure.import_failures_url
|
||||
notifier.signup_confirm_plain.the_wiki_url
|
||||
notifier.signup_confirm_plain.wiki_signup_url
|
||||
trace.edit.visibility_help_url
|
||||
trace.trace_form.help_url
|
||||
trace.trace_form.visibility_help_url
|
||||
);
|
||||
|
||||
for my $key (@url_keys) {
|
||||
if ( exists $me{$key} and not exists $new{$key} ) {
|
||||
$new{$key} = $me{$key} if $me{$key} ne $en{$key};
|
||||
}
|
||||
}
|
||||
|
||||
## When foo exists in this file but only foo.one, foo,other
|
||||
## etc in English or the original file we don't want to throw away what we have
|
||||
my @plural_keys = qw( zero one many few other two );
|
||||
|
||||
while (my ($me_k, $me_v) = each %me) {
|
||||
if (not exists $tw{ $me_k } and
|
||||
not exists $en{ $me_k } and
|
||||
(
|
||||
exists $en{ $me_k . '.zero' } or
|
||||
exists $en{ $me_k . '.one' } or
|
||||
exists $en{ $me_k . '.many' } or
|
||||
exists $en{ $me_k . '.few' } or
|
||||
exists $en{ $me_k . '.other' } or
|
||||
exists $en{ $me_k . '.two' })) {
|
||||
#say STDERR "Bringing back nuked plural form '$me_k' Setting it to '$me{ $me_k }'";
|
||||
$new{ $me_k } = $me{ $me_k };
|
||||
}
|
||||
}
|
||||
|
||||
# Both arrays and strings are supported in the site key. Avoid removing e.g.:
|
||||
# -site.key.table.entry.school: 學校；大學
|
||||
# Just because en.yml has site.key.table.entry.school.0 and site.key.table.entry.school.1
|
||||
while (my ($me_k, $me_v) = each %me) {
|
||||
next unless $me_k =~ /^site\.key\.table\.entry/;
|
||||
next if $me_k =~ /\.\d+$/;
|
||||
|
||||
if (ref $en{ $me_k } eq 'ARRAY' and not ref $me{ $me_k }) {
|
||||
$new{ $me_k } = $me{ $me_k };
|
||||
}
|
||||
}
|
||||
|
||||
# There are a bunch of keys on Translatewiki that are
|
||||
# equivalent to English for some reason. Probably because they
|
||||
# were there at import time. Nuke them.
|
||||
while (my ($new_k, $new_v) = each %new) {
|
||||
if (exists $en{ $new_k } and $en{ $new_k } eq $new_v) {
|
||||
#say "Purging dupe in $rails_lang: $new_k=$new_v";
|
||||
delete $new{ $new_k };
|
||||
}
|
||||
}
|
||||
|
||||
my $expanded = expand_hash( \%new );
|
||||
my $out = +{ $rails_lang => $expanded };
|
||||
spit_out($out_file, $out);
|
||||
} elsif (not $only_new) {
|
||||
die "Internal error on $translatewiki_lang";
|
||||
}
|
||||
}
|
||||
|
||||
sub spit_out
|
||||
{
|
||||
my ($file, $data) = @_;
|
||||
my $yaml_out = Dump $data;
|
||||
|
||||
open my $fh, ">", $file;
|
||||
print $fh $yaml_out;
|
||||
close $fh;
|
||||
}
|
||||
|
||||
#
|
||||
# YAML stuff
|
||||
#
|
||||
|
||||
sub mark_utf8
|
||||
{
|
||||
my ($hash) = @_;
|
||||
|
||||
# Mark as UTF-8
|
||||
map { if (ref $_ eq 'ARRAY') { map { utf8::decode($_) } @$_ } else { utf8::decode($_) } } values %$hash;
|
||||
}
|
||||
|
||||
sub iterate
|
||||
{
|
||||
my ($hash, @path) = @_;
|
||||
my @ret;
|
||||
|
||||
while (my ($k, $v) = each %$hash)
|
||||
{
|
||||
if (ref $v eq 'HASH')
|
||||
{
|
||||
push @ret => iterate($v, @path, $k);
|
||||
}
|
||||
else
|
||||
{
|
||||
push @ret => join(".",@path, $k), $v;
|
||||
}
|
||||
}
|
||||
|
||||
return @ret;
|
||||
}
|
||||
|
||||
sub expand_hash
|
||||
{
|
||||
my ($flat_hash) = @_;
|
||||
my %new_hash;
|
||||
while (my ($k, $v) = each %$flat_hash) {
|
||||
#say "Inserting $k=$v";
|
||||
insert_string_deep(\%new_hash, $k, $v);
|
||||
}
|
||||
|
||||
\%new_hash;
|
||||
}
|
||||
|
||||
# Fails under strict in certain cases:
|
||||
## Inserting browse.start_rjs.object_list.history.type.way=Vía [[id]]
|
||||
## Inserting activerecord.models.relation_tag=Etiqueta de la relación
|
||||
## Inserting browse.changeset_details.has_nodes.one=Tiene el siguiente {{count}} nodo:
|
||||
## Can't use string ("Tiene {{count}} nodos:") as a HASH ref while "strict refs" in use at script/locale/merge-from-translatewiki line 234.
|
||||
# Line 234 = my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
|
||||
|
||||
# sub insert_string_deep_X {
|
||||
# my ($h, $ks, $v) = @_;
|
||||
# my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
|
||||
# $$p = $v;
|
||||
# }
|
||||
|
||||
sub insert_string_deep
|
||||
{
|
||||
my ($hash, $key, $value) = @_;
|
||||
|
||||
my @key = split /\./, $key;
|
||||
my $h = $hash;
|
||||
|
||||
my $i = 0;
|
||||
for my $k (@key) {
|
||||
$i ++;
|
||||
if ($i == @key) {
|
||||
$h->{$k} = $value;
|
||||
} else {
|
||||
if (ref $h->{$k}) {
|
||||
$h = $h->{$k};
|
||||
} else {
|
||||
$h = $h->{$k} = {};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#
|
||||
# Get language from Translatewiki
|
||||
#
|
||||
|
||||
sub get_translatewiki_translations
|
||||
{
|
||||
my @languages = @_;
|
||||
|
||||
my $cache_file = "/tmp/merge-from-translatewiki.storable";
|
||||
if ($cache) {
|
||||
if (-f $cache_file) {
|
||||
my $c = retrieve($cache_file);
|
||||
return %$c;
|
||||
}
|
||||
}
|
||||
|
||||
my %translatewiki_languages;
|
||||
my $all_count = scalar @languages;
|
||||
say "Translatewiki has $all_count languages I'm about to get";
|
||||
my $count = 0;
|
||||
for my $lang (@languages) {
|
||||
$count ++;
|
||||
say STDERR "Getting language $count/$all_count ($lang) from Translatewiki";
|
||||
my $yaml = get_language_from_translatewiki($lang);
|
||||
|
||||
my $flat_data = load_and_flatten_yaml($yaml);
|
||||
|
||||
$translatewiki_languages{$lang} = $flat_data;
|
||||
}
|
||||
|
||||
if ($cache) {
|
||||
store \%translatewiki_languages, $cache_file;
|
||||
}
|
||||
|
||||
return %translatewiki_languages;
|
||||
}
|
||||
|
||||
sub get_language_from_translatewiki
|
||||
{
|
||||
my ($lang) = @_;
|
||||
my $mech = WWW::Mechanize->new;
|
||||
|
||||
$mech->get("http://translatewiki.net/w/i.php?title=Special%3ATranslate&task=export-to-file&group=out-osm-site&language=$lang");
|
||||
|
||||
die "Couldn't get lang $lang lang from Translatewiki" unless $mech->success;
|
||||
|
||||
return $mech->content;
|
||||
}
|
||||
|
||||
#
|
||||
# from language list
|
||||
#
|
||||
|
||||
sub translatewiki_languages
|
||||
{
|
||||
my $mech = WWW::Mechanize->new;
|
||||
|
||||
$mech->get('http://translatewiki.net/wiki/Translating:OpenStreetMap/stats/trunk/site');
|
||||
|
||||
die "Couldn't get translatewiki table" unless $mech->success;
|
||||
|
||||
my $content = $mech->content;
|
||||
my ($sortable) = $content =~ m[(<table class="sortable.*</table>)]s;
|
||||
|
||||
my @table = parse_language_table($sortable);
|
||||
|
||||
# Just get the codes
|
||||
map { $_->{code} => $_->{language} } @table;
|
||||
}
|
||||
|
||||
sub parse_language_table
|
||||
{
|
||||
my ($table) = @_;
|
||||
|
||||
my $parser = HTML::TableParser::Grid->new($table);
|
||||
my @rows;
|
||||
for my $n (0 .. $parser->num_rows - 1) {
|
||||
my %row;
|
||||
@row{qw(code language done fuzzy)} = $parser->row($n);
|
||||
mark_utf8(\%row);
|
||||
|
||||
push @rows => \%row;
|
||||
}
|
||||
|
||||
@rows;
|
||||
}
|
||||
|
||||
#
|
||||
# Misc
|
||||
#
|
||||
|
||||
sub basename
|
||||
{
|
||||
my $name = shift;
|
||||
$name =~ s[\..*?$][];
|
||||
$name =~ s[.*/][];
|
||||
$name;
|
||||
}
|
||||
|
||||
sub load_and_flatten_yaml
|
||||
{
|
||||
my ($yaml) = @_;
|
||||
|
||||
my $data = Load($yaml);
|
||||
|
||||
# Remove the root $lang => key
|
||||
my @keys = keys %$data;
|
||||
die "YAML data had more than 1 root key" if @keys != 1;
|
||||
$data = $data->{$keys[0]};
|
||||
|
||||
# Flatten it
|
||||
my $flat_data = { iterate($data) };
|
||||
|
||||
mark_utf8($flat_data);
|
||||
|
||||
$flat_data;
|
||||
}
|
||||
|
||||
#
|
||||
# Help
|
||||
#
|
||||
|
||||
sub help
|
||||
{
|
||||
my %arg = @_;
|
||||
|
||||
Pod::Usage::pod2usage(
|
||||
-verbose => $arg{ verbose },
|
||||
-exitval => $arg{ exitval } || 0,
|
||||
);
|
||||
}
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
#!/usr/bin/env ruby
|
||||
# po2yaml, for converting gettext .po to the RoR translation YAML
|
||||
# Use:
|
||||
# - To create a language's yaml from a given po file
|
||||
# po2yaml de.po > de.yml
|
||||
|
||||
require "yaml"
|
||||
|
||||
def add_translation(hash, keys, value)
|
||||
key = keys.shift
|
||||
if keys.empty?
|
||||
hash[key] = value
|
||||
else
|
||||
hash[key] ||= {}
|
||||
|
||||
add_translation(hash[key], keys, value)
|
||||
end
|
||||
hash
|
||||
end
|
||||
|
||||
def po2hash(f)
|
||||
trs = {}
|
||||
path = []
|
||||
msgstr = ""
|
||||
f.each_line do |line|
|
||||
line.strip!
|
||||
if line[0..8] == 'msgctxt "'
|
||||
path = line[9..-2].split(":")
|
||||
elsif line[0..7] == 'msgstr "'
|
||||
msgstr = line[8..-2]
|
||||
end
|
||||
|
||||
next if path.empty? || msgstr.empty?
|
||||
|
||||
add_translation(trs, path, msgstr)
|
||||
path = []
|
||||
msgstr = ""
|
||||
end
|
||||
trs
|
||||
end
|
||||
|
||||
filename = ARGV[0]
|
||||
pofile = File.open(filename, "r")
|
||||
|
||||
langcode = File.basename(filename, ".po")
|
||||
|
||||
tr = { langcode => po2hash(pofile) }
|
||||
|
||||
print tr.to_yaml
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
#!/usr/bin/env ruby
|
||||
# yaml2po, for converting RoR translation YAML to the standard gettext
|
||||
# for eventual use with a translation site such as Transifex
|
||||
# Use:
|
||||
# - To create a 'master' .pot
|
||||
# yaml2po > translations.pot
|
||||
# - To create a language's .po (includes from scratch)
|
||||
# yaml2po de > de.po
|
||||
# - To create all languages' .pos and a .pot (under /config/locales/po)
|
||||
# yaml2po --all
|
||||
|
||||
require "yaml"
|
||||
require "optparse"
|
||||
|
||||
LOCALE_DIR = File.dirname(__FILE__) + "/../../config/locales/"
|
||||
EN = YAML.load_file("#{LOCALE_DIR}en.yml")
|
||||
|
||||
def iterate(hash, fhash = {}, path = "", outfile = $stdout)
|
||||
hash.each do |key, val|
|
||||
fhash[key] = {} unless fhash.key? key
|
||||
if val.is_a? Hash
|
||||
fhash[key] = {} unless fhash[key].is_a? Hash
|
||||
iterate(val, fhash[key], "#{path}#{key}:#{outfile}")
|
||||
else
|
||||
outfile.puts "msgctxt \"#{path}#{key}\""
|
||||
outfile.puts "msgid \"#{val}\""
|
||||
outfile.puts "msgstr \"#{fhash[key]}\""
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def lang2po(lang, outfile = $stdout)
|
||||
puts lang
|
||||
infile = "#{LOCALE_DIR}#{lang}.yml"
|
||||
if File.exist? infile
|
||||
oth = YAML.load_file(infile)
|
||||
oth = oth[lang]
|
||||
iterate(EN["en"], oth, "", outfile)
|
||||
else
|
||||
iterate(EN["en"], {}, "", outfile)
|
||||
end
|
||||
end
|
||||
|
||||
opt = ARGV[0]
|
||||
if opt == "--all"
|
||||
# Produce .po files for all langs, and a .pot template
|
||||
po_dir = "#{LOCALE_DIR}po/"
|
||||
Dir.mkdir(po_dir) unless File.directory?(po_dir)
|
||||
Dir.glob("#{LOCALE_DIR}/*.yml") do |filename|
|
||||
lang = File.basename(filename, ".yml")
|
||||
unless lang == "en"
|
||||
outfile = File.new("#{po_dir}#{lang}.po", "w")
|
||||
lang2po(lang, outfile)
|
||||
outfile.close
|
||||
end
|
||||
end
|
||||
outfile = File.new("#{po_dir}rails_port.pot", "w")
|
||||
iterate(EN["en"], {}, "", outfile)
|
||||
outfile.close
|
||||
elsif opt
|
||||
lang2po(opt)
|
||||
else
|
||||
iterate(EN["en"])
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue