File Coverage

blib/lib/HPC/Runner/Command/submit_jobs/Utils/Scheduler/ResolveDeps/BuildTaskDeps.pm
Criterion Covered Total %
statement 15 45 33.3
branch 0 16 0.0
condition n/a
subroutine 5 8 62.5
pod 3 3 100.0
total 23 72 31.9


line stmt bran cond sub pod time code
1             package HPC::Runner::Command::submit_jobs::Utils::Scheduler::ResolveDeps::BuildTaskDeps;
2              
3 1     1   881 use Moose::Role;
  1         5  
  1         13  
4              
5 1     1   8809 use Memoize;
  1         4  
  1         80  
6 1     1   10 use List::MoreUtils qw(first_index);
  1         3  
  1         13  
7 1     1   1328 use Array::Compare;
  1         4  
  1         37  
8 1     1   9 use List::Util qw(first);
  1         4  
  1         1159  
9              
10             =head1 HPC::Runner::Command::submit_jobs::Utils::Scheduler::ResolveDeps::BuildTaskDeps
11              
12             Iterate over the batches and get the the matching TASK_TAGS
13              
14             =cut
15              
16             =head3 process_all_batch_deps
17              
18             =cut
19              
20             sub process_all_batch_deps {
21 0     0 1   my $self = shift;
22              
23 0 0         return if $self->jobs->{ $self->current_job }->submission_failure;
24 0 0         return unless $self->jobs->{ $self->current_job }->submit_by_tags;
25 0 0         return unless $self->jobs->{ $self->current_job }->has_deps;
26              
27 0           my $batch_tags = $self->batch_tags->{ $self->current_job };
28 0           my $scheduler_index = {};
29              
30 0           foreach my $dep ( @{ $self->jobs->{ $self->current_job }->deps } ) {
  0            
31 0 0         next unless $self->jobs->{$dep}->submit_by_tags;
32 0           my $dep_tags = $self->batch_tags->{$dep};
33              
34             ##If they are the same AND UNIQ - and they probably are -
35             ## each element of the array depends upon the same index in the dep array
36             # if ( check_equal_batch_tags( $batch_tags, $dep_tags ) ) {
37             # my $sched_array = build_equal_batch_tags( scalar @{$batch_tags} );
38             # $scheduler_index->{$dep} = $sched_array;
39             # }
40             # else {
41             # my $sched_array =
42             # build_unequal_batch_tags( $batch_tags, $dep_tags );
43             # $scheduler_index->{$dep} = $sched_array;
44             # }
45 0           my $sched_array =
46             build_unequal_batch_tags( $batch_tags, $dep_tags );
47 0           $scheduler_index->{$dep} = $sched_array;
48             }
49              
50 0           return $scheduler_index;
51             }
52              
53             ##TODO Task Tags should go in its own module
54              
55             =head3 build_unequal_batch_tags
56              
57             When they are not the same we have to search
58              
59             =cut
60              
61             memoize('build_unequal_batch_tags');
62              
63             sub build_unequal_batch_tags {
64             my $batch_tags = shift;
65             my $dep_tags = shift;
66              
67             my @sched_array = ();
68              
69             for ( my $x = 0 ; $x < scalar @{$batch_tags} ; $x++ ) {
70             my $batch_tag = $batch_tags->[$x];
71             my @tarray = ();
72             for ( my $y = 0 ; $y < scalar @{$dep_tags} ; $y++ ) {
73             my $dep_tag = $dep_tags->[$y];
74             push( @tarray, $y ) if search_tags( $batch_tag, $dep_tag );
75             }
76             push( @sched_array, \@tarray );
77             }
78              
79             return \@sched_array;
80             }
81              
82             =head3 build_equal_batch_tags
83              
84             If the arrays are equal, each element depends upon the same element previously
85              
86             #TODO Update this - it does not work when we have groups!
87              
88             =cut
89              
90             memoize('build_equal_batch_tags');
91              
92             sub build_equal_batch_tags {
93             my $len = shift;
94              
95             $len = $len - 1;
96             my @array = map { [$_] } ( 0 .. $len );
97             return \@array;
98             }
99              
100             =head3 check_equal_batch_tags
101              
102             If they are the same AND unique each element depends upon the same element from
103             the other array
104              
105             TODO - add back in this method to check for unique
106              
107             =cut
108              
109             memoize('check_equal_batch_tags');
110              
111             sub check_equal_batch_tags {
112             my $batch_tags = shift;
113             my $dep_tags = shift;
114              
115             if ( scalar @{$batch_tags} != scalar @{$dep_tags} ) {
116             return 0;
117             }
118              
119             my $comp = Array::Compare->new;
120             for ( my $x = 0 ; $x < scalar @{$batch_tags} ; $x++ ) {
121             my $tbatch_tags = $batch_tags->[$x];
122             my $tdep_tags = $dep_tags->[$x];
123             if ( !$comp->compare( $tbatch_tags, $tdep_tags ) ) {
124             return 0;
125             }
126             }
127             return 1;
128             }
129              
130             #TODO separate out all batches vs arrays/tasks
131              
132             =head3 process_batch_deps
133              
134             If a job has one or more job tags it is possible to fine tune dependencies
135              
136             #HPC jobname=job01
137             #HPC commands_per_node=1
138             #TASK tags=Sample1
139             gzip Sample1
140             #TASK tags=Sample2
141             gzip Sample2
142              
143             #HPC jobname=job02
144             #HPC jobdeps=job01
145             #HPC commands_per_node=1
146             #TASK tags=Sample1
147             fastqc Sample1
148             #TASK tags=Sample2
149             fastqc Sample2
150              
151             job01 - Sample1 would be submitted as schedulerid 1234
152             job01 - Sample2 would be submitted as schedulerid 1235
153              
154             job02 - Sample1 would be submitted as schedulerid 1236 - with dep on 1234 (with no job tags this would be 1234, 1235)
155             job02 - Sample2 would be submitted as schedulerid 1237 - with dep on 1235 (with no job tags this would be 1234, 1235)
156              
157             ##DEPRACATED - process_all_batch_deps
158              
159             =cut
160              
161             sub process_batch_deps {
162 0     0 1   my $self = shift;
163 0           my $batch = shift;
164              
165 0 0         return if $self->jobs->{ $self->current_job }->submission_failure;
166 0 0         return unless $self->jobs->{ $self->current_job }->submit_by_tags;
167 0 0         return unless $self->jobs->{ $self->current_job }->has_deps;
168              
169 0           my $tags = $batch->batch_tags;
170              
171 0           return $self->search_batches( $self->jobs->{ $self->current_job }->deps,
172             $tags );
173             }
174              
175             =head3 search_batches
176              
177             search the batches for a particular scheduler id
178              
179             #TODO update this to search across all batches - they are usually the same
180             instead of per batch, create an array of array for all batches
181              
182              
183             [['Sample01'], ['Sample03']]
184              
185             =cut
186              
187             sub search_batches {
188 0     0 1   my $self = shift;
189 0           my $job_deps = shift;
190 0           my $tags = shift;
191              
192 0           my $scheduler_ref = {};
193              
194 0           foreach my $dep ( @{$job_deps} ) {
  0            
195 0 0         next unless $self->jobs->{$dep}->submit_by_tags;
196 0           my $dep_batches = $self->jobs->{$dep}->batches;
197 0           $scheduler_ref->{$dep} = search_dep_batch_tags( $dep_batches, $tags );
198             }
199              
200 0           return $scheduler_ref;
201             }
202              
203             =head3 search_dep_batch_tags
204              
205             =cut
206              
207             memoize('search_dep_batch_tags');
208              
209             sub search_dep_batch_tags {
210             my $dep_batches = shift;
211             my $tags = shift;
212              
213             my @scheduler_index = ();
214             my $x = 0;
215             foreach my $dep_batch ( @{$dep_batches} ) {
216             push( @scheduler_index, $x )
217             if search_tags( $dep_batch->batch_tags, $tags );
218             $x++;
219             }
220              
221             return \@scheduler_index;
222             }
223              
224             =head3 search_tags
225              
226             #TODO Update this - we shouldn't check for searches and search separately
227             Check for matching tags. We match against any
228              
229             job02 depends on job01
230              
231             job01 batch01 has tags Sample1,Sample2
232             job01 batch02 has tags Sample3
233              
234             job02 batch01 has tags Sample1
235              
236             job02 batch01 depends upon job01 batch01 - because it has an overlap
237             But not job01 batch02
238              
239             =cut
240              
241             memoize('search_tags');
242              
243             sub search_tags {
244             my $batch_tags = shift;
245             my $search_tags = shift;
246              
247             foreach my $batch_tag ( @{$batch_tags} ) {
248             my $s = first { "$_" eq $batch_tag } @{$search_tags};
249             return $s if $s;
250             }
251              
252             return 0;
253             }
254              
255             1;