File Coverage

lib/Smartcat/App/Command/push.pm

Criterion	Covered	Total	%
statement	42	156	26.9
branch	0	44	0.0
condition	0	12	0.0
subroutine	14	23	60.8
pod	3	6	50.0
total	59	241	24.4

# ABSTRACT: push translation files to Smartcat
use strict;
use warnings;

use utf8;
no utf8;

package Smartcat::App::Command::push;
use Smartcat::App -command;

use File::Basename;
use File::Spec::Functions qw(catfile catdir);
use File::Find qw(find);
use List::Util qw(first);

use Smartcat::App::Constants qw(
  MAX_ITERATION_WAIT_TIMEOUT
  ITERATION_WAIT_TIMEOUT
  DOCUMENT_DISASSEMBLING_SUCCESS_STATUS
);
use Smartcat::App::Utils;

use Carp;
use Log::Any qw($log);
use JSON qw/encode_json/;

# How many documents to delete at a time
# (there's a limitation on the number of the document due to
# the fact that all document IDs are specified in a URL,
# and URLs itself have length limitations).
my $DELETE_BATCH_SIZE = 20;

sub opt_spec {
    my ($self) = @_;

    my @opts = $self->SUPER::opt_spec();

    push @opts,
      [ 'disassemble-algorithm-name:s' =>
          'Optional disassemble file algorithm' ],
      [ 'preset-disassemble-algorithm:s' =>
          'Optional disassemble algorithm preset' ],
      [ 'delete-not-existing' => 'Delete not existing documents' ],
      $self->project_id_opt_spec,
      $self->project_workdir_opt_spec,
      $self->file_params_opt_spec,
      $self->extract_id_from_name_opt_spec,
      $self->external_tag_opt_spec,
      ;

    return @opts;
}

sub validate_args {
    my ( $self, $opt, $args ) = @_;

    $self->SUPER::validate_args( $opt, $args );
    $self->validate_project_id( $opt, $args );
    $self->validate_project_workdir( $opt, $args );
    $self->validate_file_params( $opt, $args );

    $self->app->{rundata}->{disassemble_algorithm_name} =
      $opt->{disassemble_algorithm_name}
      if defined $opt->{disassemble_algorithm_name};
    $self->app->{rundata}->{preset_disassemble_algorithm} =
      $opt->{preset_disassemble_algorithm}
      if defined $opt->{preset_disassemble_algorithm};
    $self->app->{rundata}->{delete_not_existing} =
      defined $opt->{delete_not_existing} ? $opt->{delete_not_existing} : 0;
    $self->app->{rundata}->{extract_id_from_name} =
      defined $opt->{extract_id_from_name} ? $opt->{extract_id_from_name} : 0;
    $self->app->{rundata}->{external_tag} =
      defined $opt->{external_tag} ? $opt->{external_tag} : "source:Serge";
}

sub execute {
    my ( $self, $opt, $args ) = @_;

    my $app     = $self->app;
    my $rundata = $app->{rundata};
    $log->info(
        sprintf(
"Running 'push' command for project '%s' and translation files from '%s'...",
            $rundata->{project_id},
            $rundata->{project_workdir}
        )
    );

    my $project = $app->project_api->get_project;
    $app->project_api->update_project_external_tag( $project, $rundata->{external_tag} ) if ($#{ $project->documents } >= 0);
    my %documents;

    for ( @{ $project->documents } ) {
            my $key = &get_document_key(
                $_->full_path,
                $_->target_language,
                $rundata->{extract_id_from_name} );
            $documents{$key} = [] unless defined $documents{$key};
            push @{ $documents{$key} }, $_;
    }

    my %ts_files;
    find(
        sub {
            my $name = $File::Find::name;
            if ($^O !~ /MSWin32/) { # assume we are on Unix if not on Windows
                utf8::decode($name); # assume UTF8 filenames
                utf8::decode($_);
            }

            if (   -f $name
                && !m/^\.$/
                && m/$rundata->{filetype}$/ )
            {
                s/$rundata->{filetype}$//;
                my $path = catfile( dirname($name), $_ );

                my $key = &get_ts_file_key(
                    $rundata->{project_workdir},
                    $path,
                    $rundata->{extract_id_from_name} );

                utf8::decode($key);
                $ts_files{$key} = [] unless defined $ts_files{$key};
                push @{ $ts_files{$key} }, $name;
            }
        },
        $rundata->{project_workdir}
    );

    my %stats;
    $stats{$_}++ for ( keys %documents, keys %ts_files );

    my ( @upload, @obsolete, @update, @skip );
    push @{
        exists $ts_files{$_} && !$self->_check_if_files_are_empty( $ts_files{$_} )
        ? defined $documents{$_} ? \@update : \@upload
        : defined $documents{$_} ? \@obsolete : \@skip
      },
      $_
      for ( keys %stats );

    $log->info(
        sprintf(
"State:\n  Upload [%d]\n    %s\n  Update [%d]\n    %s\n  Obsolete [%d]\n    %s\n  Skip [%d]\n    %s\n",
            scalar @upload,
            join( ', ', map { "'$_'" } @upload ),
            scalar @update,
            join( ', ', map { "'$_'" } @update ),
            scalar @obsolete,
            join( ', ', map { "'$_'" } @obsolete ),
            scalar @skip,
            join( ', ', map { "'$_'" } @skip )
        )
    );

    $self->upload( $project, $ts_files{$_} ) for @upload;
    $self->update( $project, $documents{$_}, $ts_files{$_} ) for @update;

    if ($rundata->{delete_not_existing}) {
        my @document_ids;
        push( @document_ids, map { $_->id } @{ $documents{$_} } ) for @obsolete;

        # work in batches
        while (scalar(@document_ids) > 0) {
            my @batch = splice(@document_ids, 0, $DELETE_BATCH_SIZE);
            $self->delete( \@batch );
        }
    }

    $log->info(
        sprintf(
"Finished 'push' command for project '%s' and translation files from '%s'.",
            $rundata->{project_id},
            $rundata->{project_workdir}
        )
    );
}

sub delete {
    my ( $self, $document_ids ) = @_;

    $self->app->document_api->delete_documents($document_ids);
}

sub update {
    my ( $self, $project, $documents, $ts_files ) = @_;

    my $app     = $self->app;
    my $api     = $app->document_api;
    my $rundata = $app->{rundata};

    my @target_languages =
      map { &get_language_from_ts_filepath($rundata->{project_workdir}, $_) } @$ts_files;
    my %doc_and_path_by_lang;
    my @files_without_documents;

    #print Dumper $ts_files;
    for (@$ts_files) {

        my $lang = get_language_from_ts_filepath($rundata->{project_workdir}, $_);
        my $doc = first { $_->target_language eq $lang } @$documents;

        # p $doc;
        if ( defined $doc ) {
            $doc_and_path_by_lang{$lang} = { path => $_, doc => $doc };
        }
        else {
            push @files_without_documents, $_;
        }
    }
    my @documents_without_files =
      grep { !exists $doc_and_path_by_lang{ $_->target_language } } @$documents;

    $log->warn(
        "No files for documents:"
          . join( ', ',
            map { $_->name . '(' . $_->target_language . ') [' . $_->id . ']' }
              @documents_without_files )
    ) if @documents_without_files;

    $log->warn(
        "No documents for files:" . join( ', ', @files_without_documents ) )
      if @files_without_documents;

    for ( keys %doc_and_path_by_lang ) {
        my $doc_and_path = $doc_and_path_by_lang{$_};

        $api->update_document( $doc_and_path->{path}, $doc_and_path->{doc}->id );

        if ( $rundata->{extract_id_from_name} ) {
            my $file_name = get_file_name(
                $doc_and_path->{path},
                $rundata->{filetype},
                $target_languages[0]);
            my $document_name = $doc_and_path->{doc}->name;

            if ($file_name ne $document_name) {
                $log->info(
                    sprintf(
                        "Renaming document '%s' from '%s' to '%s'.",
                        $doc_and_path->{doc}->id,
                        $document_name,
                        $file_name
                    )
                );
                $api->rename_document( $doc_and_path->{doc}->id, $file_name );
            }
        }
    }
}

sub _check_if_files_are_empty {
    my ($self, $filepaths) = @_;

    my $rundata = $self->app->{rundata};

    if ($rundata->{filetype} eq ".po") {
        return are_po_files_empty($filepaths);
    }

    return 0;
}

sub upload {
    my ( $self, $project, $ts_files ) = @_;

    my $rundata = $self->app->{rundata};
    my @target_languages =
      map { &get_language_from_ts_filepath($rundata->{project_workdir}, $_) } @$ts_files;
    my @project_target_languages = @{ $project->target_languages };

    croak("Conflict: one target language to one file expected.")
      unless @$ts_files == 1 && @target_languages == 1;
    my $path     = shift @$ts_files;

    my $filename = prepare_document_name( $rundata->{project_workdir}, $path, $rundata->{filetype},
        $target_languages[0] );

    my $meta_info;
    if ( $rundata->{extract_id_from_name}){
        my $file_id = &get_file_id( $path );
        $meta_info = encode_json( { file_id => $file_id } );
    }

    my $documents = $self->app->project_api->upload_file( $path, $filename, undef, $meta_info,
        \@target_languages );
    $log->info( "Created documents ids:\n  "
          . join( ', ', map { $_->id } @$documents ) );
}

1;