| line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
|
1
|
|
|
|
|
|
|
package HPC::Runner::Command::submit_jobs::Plugin::Slurm; |
|
2
|
|
|
|
|
|
|
|
|
3
|
1
|
|
|
1
|
|
738
|
use Moose::Role; |
|
|
1
|
|
|
|
|
2
|
|
|
|
1
|
|
|
|
|
5
|
|
|
4
|
1
|
|
|
1
|
|
4081
|
use namespace::autoclean; |
|
|
1
|
|
|
|
|
2
|
|
|
|
1
|
|
|
|
|
7
|
|
|
5
|
|
|
|
|
|
|
|
|
6
|
1
|
|
|
1
|
|
71
|
use File::Temp qw/ tempfile /; |
|
|
1
|
|
|
|
|
3
|
|
|
|
1
|
|
|
|
|
65
|
|
|
7
|
1
|
|
|
1
|
|
8
|
use File::Slurp; |
|
|
1
|
|
|
|
|
2
|
|
|
|
1
|
|
|
|
|
58
|
|
|
8
|
1
|
|
|
1
|
|
6
|
use File::Spec; |
|
|
1
|
|
|
|
|
3
|
|
|
|
1
|
|
|
|
|
373
|
|
|
9
|
|
|
|
|
|
|
|
|
10
|
|
|
|
|
|
|
with 'HPC::Runner::Command::submit_jobs::Plugin::Role::Log'; |
|
11
|
|
|
|
|
|
|
|
|
12
|
|
|
|
|
|
|
=head1 HPC::Runner::Command::Plugin::Scheduler::Slurm; |
|
13
|
|
|
|
|
|
|
|
|
14
|
|
|
|
|
|
|
Use the SLURM scheduler |
|
15
|
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
=cut |
|
17
|
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
has 'submit_command' => ( |
|
19
|
|
|
|
|
|
|
is => 'rw', |
|
20
|
|
|
|
|
|
|
isa => 'Str', |
|
21
|
|
|
|
|
|
|
default => 'sbatch', |
|
22
|
|
|
|
|
|
|
); |
|
23
|
|
|
|
|
|
|
|
|
24
|
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
=head2 Subroutines |
|
26
|
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
=cut |
|
28
|
|
|
|
|
|
|
|
|
29
|
|
|
|
|
|
|
=head3 submit_jobs |
|
30
|
|
|
|
|
|
|
|
|
31
|
|
|
|
|
|
|
Submit jobs to slurm queue using sbatch. |
|
32
|
|
|
|
|
|
|
|
|
33
|
|
|
|
|
|
|
Format is |
|
34
|
|
|
|
|
|
|
|
|
35
|
|
|
|
|
|
|
Submitted batch job <job_id> |
|
36
|
|
|
|
|
|
|
|
|
37
|
|
|
|
|
|
|
Where <job_id> is just only numeric |
|
38
|
|
|
|
|
|
|
=cut |
|
39
|
|
|
|
|
|
|
|
|
40
|
|
|
|
|
|
|
sub submit_jobs { |
|
41
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
|
42
|
|
|
|
|
|
|
|
|
43
|
0
|
|
|
|
|
|
my ( $exitcode, $stdout, $stderr ) = |
|
44
|
|
|
|
|
|
|
$self->submit_to_scheduler( |
|
45
|
|
|
|
|
|
|
$self->submit_command . " " . $self->slurmfile ); |
|
46
|
|
|
|
|
|
|
|
|
47
|
0
|
|
|
|
|
|
sleep(3); |
|
48
|
|
|
|
|
|
|
|
|
49
|
0
|
0
|
0
|
|
|
|
if ( ! defined $exitcode || $exitcode != 0 ) { |
|
50
|
0
|
|
|
|
|
|
$self->log->warn("Job was not submitted successfully"); |
|
51
|
0
|
0
|
|
|
|
|
$self->log->warn( "STDERR: " . $stderr ) if $stderr; |
|
52
|
0
|
0
|
|
|
|
|
$self->log->warn( "STDOUT: " . $stdout ) if $stdout; |
|
53
|
|
|
|
|
|
|
} |
|
54
|
|
|
|
|
|
|
|
|
55
|
0
|
0
|
|
|
|
|
my ($jobid) = $stdout =~ m/(\d.*)$/ if $stdout; |
|
56
|
|
|
|
|
|
|
|
|
57
|
0
|
0
|
|
|
|
|
if ( !$jobid ) { |
|
58
|
0
|
|
|
|
|
|
$self->job_failure; |
|
59
|
|
|
|
|
|
|
} |
|
60
|
|
|
|
|
|
|
else { |
|
61
|
0
|
|
|
|
|
|
$self->app_log->info( "Submitted job " |
|
62
|
|
|
|
|
|
|
. $self->slurmfile |
|
63
|
|
|
|
|
|
|
. "\n\tWith Slurm jobid $jobid" ); |
|
64
|
|
|
|
|
|
|
} |
|
65
|
|
|
|
|
|
|
|
|
66
|
0
|
|
|
|
|
|
return $jobid; |
|
67
|
|
|
|
|
|
|
} |
|
68
|
|
|
|
|
|
|
|
|
69
|
|
|
|
|
|
|
=head3 update_job_deps |
|
70
|
|
|
|
|
|
|
|
|
71
|
|
|
|
|
|
|
Update the job dependencies if using job_array (not batches) |
|
72
|
|
|
|
|
|
|
|
|
73
|
|
|
|
|
|
|
=cut |
|
74
|
|
|
|
|
|
|
|
|
75
|
|
|
|
|
|
|
sub update_job_deps { |
|
76
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
|
77
|
|
|
|
|
|
|
|
|
78
|
0
|
0
|
|
|
|
|
return unless $self->has_array_deps; |
|
79
|
|
|
|
|
|
|
|
|
80
|
0
|
|
|
|
|
|
my $array_deps_file = File::Spec->catdir( $self->logdir, 'array_deps.tsv' ); |
|
81
|
0
|
|
|
|
|
|
my $array_log_file = File::Spec->catdir( $self->logdir, 'array_deps.log' ); |
|
82
|
|
|
|
|
|
|
|
|
83
|
0
|
|
|
|
|
|
while ( my ( $current_task, $v ) = each %{ $self->array_deps } ) { |
|
|
0
|
|
|
|
|
|
|
|
84
|
0
|
|
|
|
|
|
my $dep_tasks = join( ':', @{$v} ); |
|
|
0
|
|
|
|
|
|
|
|
85
|
0
|
|
|
|
|
|
my $cmd = |
|
86
|
|
|
|
|
|
|
"scontrol update job=$current_task depend=afterok:$dep_tasks"; |
|
87
|
|
|
|
|
|
|
|
|
88
|
0
|
|
|
|
|
|
my ( $exitcode, $stdout, $stderr ) = $self->submit_to_scheduler($cmd); |
|
89
|
0
|
|
|
|
|
|
write_file( |
|
90
|
|
|
|
|
|
|
$array_deps_file, |
|
91
|
|
|
|
|
|
|
{ append => 1 }, |
|
92
|
|
|
|
|
|
|
$current_task . "\t" . $dep_tasks . "\n" |
|
93
|
|
|
|
|
|
|
); |
|
94
|
|
|
|
|
|
|
|
|
95
|
0
|
|
|
|
|
|
my $info = |
|
96
|
|
|
|
|
|
|
"Task Deps:\t" |
|
97
|
|
|
|
|
|
|
. $current_task . "\t" |
|
98
|
|
|
|
|
|
|
. $dep_tasks . "\n" |
|
99
|
|
|
|
|
|
|
. "ExitCode: $exitcode\n"; |
|
100
|
0
|
0
|
|
|
|
|
$info .= "Stderr: $stderr\n" if $stderr; |
|
101
|
0
|
0
|
|
|
|
|
$info .= "Stdout: $stdout\n" if $stdout; |
|
102
|
|
|
|
|
|
|
|
|
103
|
0
|
|
|
|
|
|
write_file( $array_log_file, {append => 1}, $info ); |
|
104
|
|
|
|
|
|
|
} |
|
105
|
|
|
|
|
|
|
} |
|
106
|
|
|
|
|
|
|
|
|
107
|
|
|
|
|
|
|
1; |