line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
# ABSTRACT: push translation files to Smartcat |
2
|
2
|
|
|
2
|
|
1312
|
use strict; |
|
2
|
|
|
|
|
4
|
|
|
2
|
|
|
|
|
60
|
|
3
|
2
|
|
|
2
|
|
11
|
use warnings; |
|
2
|
|
|
|
|
4
|
|
|
2
|
|
|
|
|
52
|
|
4
|
|
|
|
|
|
|
|
5
|
2
|
|
|
2
|
|
9
|
use utf8; |
|
2
|
|
|
|
|
5
|
|
|
2
|
|
|
|
|
9
|
|
6
|
2
|
|
|
2
|
|
95
|
no utf8; |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
9
|
|
7
|
|
|
|
|
|
|
|
8
|
|
|
|
|
|
|
package Smartcat::App::Command::push; |
9
|
2
|
|
|
2
|
|
97
|
use Smartcat::App -command; |
|
2
|
|
|
|
|
4
|
|
|
2
|
|
|
|
|
10
|
|
10
|
|
|
|
|
|
|
|
11
|
2
|
|
|
2
|
|
603
|
use File::Basename; |
|
2
|
|
|
|
|
6
|
|
|
2
|
|
|
|
|
137
|
|
12
|
2
|
|
|
2
|
|
14
|
use File::Spec::Functions qw(catfile catdir); |
|
2
|
|
|
|
|
4
|
|
|
2
|
|
|
|
|
129
|
|
13
|
2
|
|
|
2
|
|
15
|
use File::Find qw(find); |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
133
|
|
14
|
2
|
|
|
2
|
|
14
|
use List::Util qw(first); |
|
2
|
|
|
|
|
2
|
|
|
2
|
|
|
|
|
141
|
|
15
|
|
|
|
|
|
|
|
16
|
2
|
|
|
|
|
114
|
use Smartcat::App::Constants qw( |
17
|
|
|
|
|
|
|
MAX_ITERATION_WAIT_TIMEOUT |
18
|
|
|
|
|
|
|
ITERATION_WAIT_TIMEOUT |
19
|
|
|
|
|
|
|
DOCUMENT_DISASSEMBLING_SUCCESS_STATUS |
20
|
2
|
|
|
2
|
|
14
|
); |
|
2
|
|
|
|
|
11
|
|
21
|
2
|
|
|
2
|
|
24
|
use Smartcat::App::Utils; |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
153
|
|
22
|
|
|
|
|
|
|
|
23
|
2
|
|
|
2
|
|
13
|
use Carp; |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
128
|
|
24
|
2
|
|
|
2
|
|
23
|
use Log::Any qw($log); |
|
2
|
|
|
|
|
5
|
|
|
2
|
|
|
|
|
12
|
|
25
|
|
|
|
|
|
|
|
26
|
|
|
|
|
|
|
# How many documents to delete at a time |
27
|
|
|
|
|
|
|
# (there's a limitation on the number of the document due to |
28
|
|
|
|
|
|
|
# the fact that all document IDs are specified in a URL, |
29
|
|
|
|
|
|
|
# and URLs itself have length limitations). |
30
|
|
|
|
|
|
|
my $DELETE_BATCH_SIZE = 20; |
31
|
|
|
|
|
|
|
|
32
|
|
|
|
|
|
|
sub opt_spec { |
33
|
0
|
|
|
0
|
1
|
|
my ($self) = @_; |
34
|
|
|
|
|
|
|
|
35
|
0
|
|
|
|
|
|
my @opts = $self->SUPER::opt_spec(); |
36
|
|
|
|
|
|
|
|
37
|
0
|
|
|
|
|
|
push @opts, |
38
|
|
|
|
|
|
|
[ 'disassemble-algorithm-name:s' => |
39
|
|
|
|
|
|
|
'Optional disassemble file algorithm' ], |
40
|
|
|
|
|
|
|
[ 'preset-disassemble-algorithm:s' => |
41
|
|
|
|
|
|
|
'Optional disassemble algorithm preset' ], |
42
|
|
|
|
|
|
|
[ 'delete-not-existing' => 'Delete not existing documents' ], |
43
|
|
|
|
|
|
|
$self->project_id_opt_spec, |
44
|
|
|
|
|
|
|
$self->project_workdir_opt_spec, |
45
|
|
|
|
|
|
|
$self->file_params_opt_spec, |
46
|
|
|
|
|
|
|
; |
47
|
|
|
|
|
|
|
|
48
|
0
|
|
|
|
|
|
return @opts; |
49
|
|
|
|
|
|
|
} |
50
|
|
|
|
|
|
|
|
51
|
|
|
|
|
|
|
sub validate_args { |
52
|
0
|
|
|
0
|
1
|
|
my ( $self, $opt, $args ) = @_; |
53
|
|
|
|
|
|
|
|
54
|
0
|
|
|
|
|
|
$self->SUPER::validate_args( $opt, $args ); |
55
|
0
|
|
|
|
|
|
$self->validate_project_id( $opt, $args ); |
56
|
0
|
|
|
|
|
|
$self->validate_project_workdir( $opt, $args ); |
57
|
0
|
|
|
|
|
|
$self->validate_file_params( $opt, $args ); |
58
|
|
|
|
|
|
|
|
59
|
|
|
|
|
|
|
$self->app->{rundata}->{disassemble_algorithm_name} = |
60
|
|
|
|
|
|
|
$opt->{disassemble_algorithm_name} |
61
|
0
|
0
|
|
|
|
|
if defined $opt->{disassemble_algorithm_name}; |
62
|
|
|
|
|
|
|
$self->app->{rundata}->{preset_disassemble_algorithm} = |
63
|
|
|
|
|
|
|
$opt->{preset_disassemble_algorithm} |
64
|
0
|
0
|
|
|
|
|
if defined $opt->{preset_disassemble_algorithm}; |
65
|
|
|
|
|
|
|
$self->app->{rundata}->{delete_not_existing} = |
66
|
0
|
0
|
|
|
|
|
defined $opt->{delete_not_existing} ? $opt->{delete_not_existing} : 0; |
67
|
|
|
|
|
|
|
} |
68
|
|
|
|
|
|
|
|
69
|
|
|
|
|
|
|
sub execute { |
70
|
0
|
|
|
0
|
1
|
|
my ( $self, $opt, $args ) = @_; |
71
|
|
|
|
|
|
|
|
72
|
0
|
|
|
|
|
|
my $app = $self->app; |
73
|
0
|
|
|
|
|
|
my $rundata = $app->{rundata}; |
74
|
|
|
|
|
|
|
$log->info( |
75
|
|
|
|
|
|
|
sprintf( |
76
|
|
|
|
|
|
|
"Running 'push' command for project '%s' and translation files from '%s'...", |
77
|
|
|
|
|
|
|
$rundata->{project_id}, |
78
|
|
|
|
|
|
|
$rundata->{project_workdir} |
79
|
|
|
|
|
|
|
) |
80
|
0
|
|
|
|
|
|
); |
81
|
|
|
|
|
|
|
|
82
|
0
|
|
|
|
|
|
my $project = $app->project_api->get_project; |
83
|
0
|
0
|
|
|
|
|
$app->project_api->update_project_external_tag( $project, "source:Serge" ) if ($#{ $project->documents } >= 0); |
|
0
|
|
|
|
|
|
|
84
|
0
|
|
|
|
|
|
my %documents; |
85
|
0
|
|
|
|
|
|
for ( @{ $project->documents } ) { |
|
0
|
|
|
|
|
|
|
86
|
0
|
|
|
|
|
|
my $key = &get_document_key( $_->full_path, $_->target_language ); |
87
|
0
|
0
|
|
|
|
|
$documents{$key} = [] unless defined $documents{$key}; |
88
|
0
|
|
|
|
|
|
push @{ $documents{$key} }, $_; |
|
0
|
|
|
|
|
|
|
89
|
|
|
|
|
|
|
} |
90
|
|
|
|
|
|
|
|
91
|
0
|
|
|
|
|
|
my %ts_files; |
92
|
|
|
|
|
|
|
find( |
93
|
|
|
|
|
|
|
sub { |
94
|
0
|
|
|
0
|
|
|
my $name = $File::Find::name; |
95
|
0
|
0
|
|
|
|
|
if ($^O !~ /MSWin32/) { # assume we are on Unix if not on Windows |
96
|
0
|
|
|
|
|
|
utf8::decode($name); # assume UTF8 filenames |
97
|
0
|
|
|
|
|
|
utf8::decode($_); |
98
|
|
|
|
|
|
|
} |
99
|
0
|
0
|
0
|
|
|
|
if ( -f $name |
|
|
|
0
|
|
|
|
|
100
|
|
|
|
|
|
|
&& !m/^\.$/ |
101
|
|
|
|
|
|
|
&& m/$rundata->{filetype}$/ ) |
102
|
|
|
|
|
|
|
{ |
103
|
0
|
|
|
|
|
|
s/$rundata->{filetype}$//; |
104
|
0
|
|
|
|
|
|
my $path = catfile( dirname($name), $_ ); |
105
|
0
|
|
|
|
|
|
my $key = &get_ts_file_key($rundata->{project_workdir}, $path); |
106
|
0
|
|
|
|
|
|
utf8::decode($key); |
107
|
0
|
0
|
|
|
|
|
$ts_files{$key} = [] unless defined $ts_files{$key}; |
108
|
0
|
|
|
|
|
|
push @{ $ts_files{$key} }, $name; |
|
0
|
|
|
|
|
|
|
109
|
|
|
|
|
|
|
} |
110
|
|
|
|
|
|
|
}, |
111
|
|
|
|
|
|
|
$rundata->{project_workdir} |
112
|
0
|
|
|
|
|
|
); |
113
|
|
|
|
|
|
|
|
114
|
0
|
|
|
|
|
|
my %stats; |
115
|
0
|
|
|
|
|
|
$stats{$_}++ for ( keys %documents, keys %ts_files ); |
116
|
|
|
|
|
|
|
|
117
|
0
|
|
|
|
|
|
my ( @upload, @obsolete, @update, @skip ); |
118
|
|
|
|
|
|
|
push @{ |
119
|
|
|
|
|
|
|
exists $ts_files{$_} && !$self->_check_if_files_are_empty( $ts_files{$_} ) |
120
|
|
|
|
|
|
|
? defined $documents{$_} ? \@update : \@upload |
121
|
0
|
0
|
0
|
|
|
|
: defined $documents{$_} ? \@obsolete : \@skip |
|
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
122
|
|
|
|
|
|
|
}, |
123
|
|
|
|
|
|
|
$_ |
124
|
0
|
|
|
|
|
|
for ( keys %stats ); |
125
|
|
|
|
|
|
|
|
126
|
|
|
|
|
|
|
$log->info( |
127
|
|
|
|
|
|
|
sprintf( |
128
|
|
|
|
|
|
|
"State:\n Upload [%d]\n %s\n Update [%d]\n %s\n Obsolete [%d]\n %s\n Skip [%d]\n %s\n", |
129
|
|
|
|
|
|
|
scalar @upload, |
130
|
0
|
|
|
|
|
|
join( ', ', map { "'$_'" } @upload ), |
131
|
|
|
|
|
|
|
scalar @update, |
132
|
0
|
|
|
|
|
|
join( ', ', map { "'$_'" } @update ), |
133
|
|
|
|
|
|
|
scalar @obsolete, |
134
|
0
|
|
|
|
|
|
join( ', ', map { "'$_'" } @obsolete ), |
135
|
|
|
|
|
|
|
scalar @skip, |
136
|
0
|
|
|
|
|
|
join( ', ', map { "'$_'" } @skip ) |
|
0
|
|
|
|
|
|
|
137
|
|
|
|
|
|
|
) |
138
|
|
|
|
|
|
|
); |
139
|
|
|
|
|
|
|
|
140
|
0
|
|
|
|
|
|
$self->upload( $project, $ts_files{$_} ) for @upload; |
141
|
0
|
|
|
|
|
|
$self->update( $project, $documents{$_}, $ts_files{$_} ) for @update; |
142
|
|
|
|
|
|
|
|
143
|
0
|
0
|
|
|
|
|
if ($rundata->{delete_not_existing}) { |
144
|
0
|
|
|
|
|
|
my @document_ids; |
145
|
0
|
|
|
|
|
|
push( @document_ids, map { $_->id } @{ $documents{$_} } ) for @obsolete; |
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
146
|
|
|
|
|
|
|
|
147
|
|
|
|
|
|
|
# work in batches |
148
|
0
|
|
|
|
|
|
while (scalar(@document_ids) > 0) { |
149
|
0
|
|
|
|
|
|
my @batch = splice(@document_ids, 0, $DELETE_BATCH_SIZE); |
150
|
0
|
|
|
|
|
|
$self->delete( \@batch ); |
151
|
|
|
|
|
|
|
} |
152
|
|
|
|
|
|
|
} |
153
|
|
|
|
|
|
|
|
154
|
|
|
|
|
|
|
$log->info( |
155
|
|
|
|
|
|
|
sprintf( |
156
|
|
|
|
|
|
|
"Finished 'push' command for project '%s' and translation files from '%s'.", |
157
|
|
|
|
|
|
|
$rundata->{project_id}, |
158
|
|
|
|
|
|
|
$rundata->{project_workdir} |
159
|
|
|
|
|
|
|
) |
160
|
0
|
|
|
|
|
|
); |
161
|
|
|
|
|
|
|
} |
162
|
|
|
|
|
|
|
|
163
|
|
|
|
|
|
|
sub delete { |
164
|
0
|
|
|
0
|
0
|
|
my ( $self, $document_ids ) = @_; |
165
|
|
|
|
|
|
|
|
166
|
0
|
|
|
|
|
|
$self->app->document_api->delete_documents($document_ids); |
167
|
|
|
|
|
|
|
} |
168
|
|
|
|
|
|
|
|
169
|
|
|
|
|
|
|
sub update { |
170
|
0
|
|
|
0
|
0
|
|
my ( $self, $project, $documents, $ts_files ) = @_; |
171
|
|
|
|
|
|
|
|
172
|
0
|
|
|
|
|
|
my $app = $self->app; |
173
|
0
|
|
|
|
|
|
my $api = $app->document_api; |
174
|
0
|
|
|
|
|
|
my $rundata = $app->{rundata}; |
175
|
|
|
|
|
|
|
|
176
|
|
|
|
|
|
|
my @target_languages = |
177
|
0
|
|
|
|
|
|
map { &get_language_from_ts_filepath($rundata->{project_workdir}, $_) } @$ts_files; |
|
0
|
|
|
|
|
|
|
178
|
0
|
|
|
|
|
|
my @project_target_languages = @{ $project->target_languages }; |
|
0
|
|
|
|
|
|
|
179
|
0
|
|
|
|
|
|
my %lang_pairs; |
180
|
|
|
|
|
|
|
my @files_without_documents; |
181
|
|
|
|
|
|
|
|
182
|
|
|
|
|
|
|
#print Dumper $ts_files; |
183
|
0
|
|
|
|
|
|
for (@$ts_files) { |
184
|
|
|
|
|
|
|
|
185
|
|
|
|
|
|
|
#print $_."\n"; |
186
|
0
|
|
|
|
|
|
my $lang = get_language_from_ts_filepath($rundata->{project_workdir}, $_); |
187
|
0
|
|
|
0
|
|
|
my $doc = first { $_->target_language eq $lang } @$documents; |
|
0
|
|
|
|
|
|
|
188
|
|
|
|
|
|
|
|
189
|
|
|
|
|
|
|
#p $doc; |
190
|
0
|
0
|
|
|
|
|
if ( defined $doc ) { |
191
|
0
|
|
|
|
|
|
$lang_pairs{$lang} = [ $_, $doc->id ]; |
192
|
|
|
|
|
|
|
} |
193
|
|
|
|
|
|
|
else { |
194
|
0
|
|
|
|
|
|
push @files_without_documents, $_; |
195
|
|
|
|
|
|
|
} |
196
|
|
|
|
|
|
|
} |
197
|
|
|
|
|
|
|
my @documents_without_files = |
198
|
0
|
|
|
|
|
|
grep { !exists $lang_pairs{ $_->target_language } } @$documents; |
|
0
|
|
|
|
|
|
|
199
|
|
|
|
|
|
|
|
200
|
|
|
|
|
|
|
$log->warn( |
201
|
|
|
|
|
|
|
"No files for documents:" |
202
|
|
|
|
|
|
|
. join( ', ', |
203
|
0
|
0
|
|
|
|
|
map { $_->name . '(' . $_->target_language . ') [' . $_->id . ']' } |
|
0
|
|
|
|
|
|
|
204
|
|
|
|
|
|
|
@documents_without_files ) |
205
|
|
|
|
|
|
|
) if @documents_without_files; |
206
|
|
|
|
|
|
|
|
207
|
0
|
0
|
|
|
|
|
$log->warn( |
208
|
|
|
|
|
|
|
"No documents for files:" . join( ', ', @files_without_documents ) ) |
209
|
|
|
|
|
|
|
if @files_without_documents; |
210
|
|
|
|
|
|
|
|
211
|
0
|
|
|
|
|
|
$api->update_document( @{ $lang_pairs{$_} } ) for ( keys %lang_pairs ); |
|
0
|
|
|
|
|
|
|
212
|
|
|
|
|
|
|
} |
213
|
|
|
|
|
|
|
|
214
|
|
|
|
|
|
|
sub _check_if_files_are_empty { |
215
|
0
|
|
|
0
|
|
|
my ($self, $filepaths) = @_; |
216
|
|
|
|
|
|
|
|
217
|
0
|
|
|
|
|
|
my $rundata = $self->app->{rundata}; |
218
|
|
|
|
|
|
|
|
219
|
0
|
0
|
|
|
|
|
if ($rundata->{filetype} eq ".po") { |
220
|
0
|
|
|
|
|
|
return are_po_files_empty($filepaths); |
221
|
|
|
|
|
|
|
} |
222
|
|
|
|
|
|
|
|
223
|
0
|
|
|
|
|
|
return 0; |
224
|
|
|
|
|
|
|
} |
225
|
|
|
|
|
|
|
|
226
|
|
|
|
|
|
|
sub upload { |
227
|
0
|
|
|
0
|
0
|
|
my ( $self, $project, $ts_files ) = @_; |
228
|
|
|
|
|
|
|
|
229
|
0
|
|
|
|
|
|
my $rundata = $self->app->{rundata}; |
230
|
|
|
|
|
|
|
my @target_languages = |
231
|
0
|
|
|
|
|
|
map { &get_language_from_ts_filepath($rundata->{project_workdir}, $_) } @$ts_files; |
|
0
|
|
|
|
|
|
|
232
|
0
|
|
|
|
|
|
my @project_target_languages = @{ $project->target_languages }; |
|
0
|
|
|
|
|
|
|
233
|
|
|
|
|
|
|
|
234
|
0
|
0
|
0
|
|
|
|
croak("Conflict: one target language to one file expected.") |
235
|
|
|
|
|
|
|
unless @$ts_files == 1 && @target_languages == 1; |
236
|
0
|
|
|
|
|
|
my $path = shift @$ts_files; |
237
|
|
|
|
|
|
|
my $filename = prepare_document_name( $rundata->{project_workdir}, $path, $rundata->{filetype}, |
238
|
0
|
|
|
|
|
|
$target_languages[0] ); |
239
|
0
|
|
|
|
|
|
my $documents = $self->app->project_api->upload_file( $path, $filename, |
240
|
|
|
|
|
|
|
\@target_languages ); |
241
|
|
|
|
|
|
|
$log->info( "Created documents ids:\n " |
242
|
0
|
|
|
|
|
|
. join( ', ', map { $_->id } @$documents ) ); |
|
0
|
|
|
|
|
|
|
243
|
|
|
|
|
|
|
} |
244
|
|
|
|
|
|
|
|
245
|
|
|
|
|
|
|
1; |