| line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
|
1
|
|
|
|
|
|
|
package PBS::Client; |
|
2
|
8
|
|
|
8
|
|
30003
|
use strict; |
|
|
8
|
|
|
|
|
16
|
|
|
|
8
|
|
|
|
|
278
|
|
|
3
|
8
|
|
|
8
|
|
35
|
use vars qw($VERSION); |
|
|
8
|
|
|
|
|
11
|
|
|
|
8
|
|
|
|
|
428
|
|
|
4
|
8
|
|
|
8
|
|
41
|
use Carp; |
|
|
8
|
|
|
|
|
15
|
|
|
|
8
|
|
|
|
|
708
|
|
|
5
|
8
|
|
|
8
|
|
10398
|
use File::Temp qw(tempfile); |
|
|
8
|
|
|
|
|
212556
|
|
|
|
8
|
|
|
|
|
5144
|
|
|
6
|
|
|
|
|
|
|
$VERSION = '0.11'; |
|
7
|
|
|
|
|
|
|
|
|
8
|
|
|
|
|
|
|
#------------------------------------------------ |
|
9
|
|
|
|
|
|
|
# Submit jobs to PBS |
|
10
|
|
|
|
|
|
|
# |
|
11
|
|
|
|
|
|
|
# Included methods: |
|
12
|
|
|
|
|
|
|
# - Construct PBS client object |
|
13
|
|
|
|
|
|
|
# $client = PBS::Client->new(); |
|
14
|
|
|
|
|
|
|
# |
|
15
|
|
|
|
|
|
|
# - Submit jobs |
|
16
|
|
|
|
|
|
|
# $client->qsub($job); |
|
17
|
|
|
|
|
|
|
# -- $client -- client object |
|
18
|
|
|
|
|
|
|
# -- $job ----- job object |
|
19
|
|
|
|
|
|
|
# |
|
20
|
|
|
|
|
|
|
# - Generate job script without job submission |
|
21
|
|
|
|
|
|
|
# $client->genScript($job); |
|
22
|
|
|
|
|
|
|
# -- $client -- client object |
|
23
|
|
|
|
|
|
|
# -- $job ----- job object |
|
24
|
|
|
|
|
|
|
#------------------------------------------------ |
|
25
|
|
|
|
|
|
|
|
|
26
|
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
#------------------------ |
|
28
|
|
|
|
|
|
|
# Constructor method |
|
29
|
|
|
|
|
|
|
# |
|
30
|
|
|
|
|
|
|
# |
|
31
|
|
|
|
|
|
|
# $class -- client object |
|
32
|
|
|
|
|
|
|
# %hash --- argument hash |
|
33
|
|
|
|
|
|
|
# |
|
34
|
|
|
|
|
|
|
# |
|
35
|
|
|
|
|
|
|
# $self -- client object |
|
36
|
|
|
|
|
|
|
sub new |
|
37
|
|
|
|
|
|
|
{ |
|
38
|
13
|
|
|
13
|
1
|
8554
|
my ($class, %hash) = @_; |
|
39
|
13
|
|
|
|
|
30
|
my $self = \%hash; |
|
40
|
13
|
|
|
|
|
61
|
return bless($self, $class); |
|
41
|
|
|
|
|
|
|
} |
|
42
|
|
|
|
|
|
|
#------------------------ |
|
43
|
|
|
|
|
|
|
|
|
44
|
|
|
|
|
|
|
|
|
45
|
|
|
|
|
|
|
#-------------------------- |
|
46
|
|
|
|
|
|
|
# Generic attribute methods |
|
47
|
|
|
|
|
|
|
sub AUTOLOAD |
|
48
|
|
|
|
|
|
|
{ |
|
49
|
0
|
|
|
0
|
|
0
|
my ($self, $key) = @_; |
|
50
|
0
|
|
|
|
|
0
|
my $attr = our $AUTOLOAD; |
|
51
|
0
|
|
|
|
|
0
|
$attr =~ s/.*:://; |
|
52
|
0
|
0
|
|
|
|
0
|
return if ($attr eq 'DESTROY'); # ignore destructor |
|
53
|
0
|
0
|
|
|
|
0
|
$self->{$attr} = $key if (defined $key); |
|
54
|
0
|
|
|
|
|
0
|
return($self->{$attr}); |
|
55
|
|
|
|
|
|
|
} |
|
56
|
|
|
|
|
|
|
#-------------------------- |
|
57
|
|
|
|
|
|
|
|
|
58
|
|
|
|
|
|
|
|
|
59
|
|
|
|
|
|
|
#------------------------------------------------------------------- |
|
60
|
|
|
|
|
|
|
# Submit PBS jobs by qsub command |
|
61
|
|
|
|
|
|
|
# - called subroutines: getScript(), _numPrevJob() and _qsubDepend() |
|
62
|
|
|
|
|
|
|
# |
|
63
|
|
|
|
|
|
|
# |
|
64
|
|
|
|
|
|
|
# $self -- client object |
|
65
|
|
|
|
|
|
|
# $job --- job object |
|
66
|
|
|
|
|
|
|
# |
|
67
|
|
|
|
|
|
|
# |
|
68
|
|
|
|
|
|
|
# \@pbsid -- array reference of PBS job ID |
|
69
|
|
|
|
|
|
|
sub qsub |
|
70
|
|
|
|
|
|
|
{ |
|
71
|
0
|
|
|
0
|
1
|
0
|
my ($self, $job) = @_; |
|
72
|
|
|
|
|
|
|
|
|
73
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
74
|
|
|
|
|
|
|
# Codes for backward compatatible with old private software |
|
75
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
76
|
0
|
0
|
0
|
|
|
0
|
if (!ref($job) || ref($job) eq 'ARRAY') |
|
77
|
|
|
|
|
|
|
{ |
|
78
|
0
|
|
|
|
|
0
|
$self->cmd($job); |
|
79
|
0
|
|
|
|
|
0
|
&qsub($self, $self); |
|
80
|
|
|
|
|
|
|
} |
|
81
|
|
|
|
|
|
|
|
|
82
|
|
|
|
|
|
|
#----------------------------------------------- |
|
83
|
|
|
|
|
|
|
# Dependency: count number of previous jobs |
|
84
|
|
|
|
|
|
|
#----------------------------------------------- |
|
85
|
0
|
|
|
|
|
0
|
my $on = &_numPrevJob($job); |
|
86
|
0
|
0
|
|
|
|
0
|
$job->{depend}{on} = [$on] if ($on); |
|
87
|
0
|
|
|
|
|
0
|
my $file = $job->{script}; |
|
88
|
0
|
|
|
|
|
0
|
my @pbsid = (); |
|
89
|
|
|
|
|
|
|
|
|
90
|
|
|
|
|
|
|
#----------------------------------------------- |
|
91
|
|
|
|
|
|
|
# Single job |
|
92
|
|
|
|
|
|
|
# Thanks to Demian Ricchardi for a bug fix |
|
93
|
|
|
|
|
|
|
#----------------------------------------------- |
|
94
|
0
|
0
|
|
|
|
0
|
if (!ref($job->{cmd})) |
|
95
|
|
|
|
|
|
|
{ |
|
96
|
0
|
|
|
|
|
0
|
my $tempFile = &genScript($self, $job); # generate script |
|
97
|
0
|
|
|
|
|
0
|
my $out = &call_qsub('qsub', $tempFile); # submit script |
|
98
|
0
|
|
|
|
|
0
|
my $pbsid = ($out =~ /^(\d+)/)[0]; # get pid |
|
99
|
0
|
|
|
|
|
0
|
rename($tempFile, "$file.$pbsid"); # rename script |
|
100
|
0
|
|
|
|
|
0
|
push(@pbsid, $pbsid); |
|
101
|
0
|
|
|
|
|
0
|
$job->pbsid($pbsid); |
|
102
|
|
|
|
|
|
|
} |
|
103
|
|
|
|
|
|
|
#----------------------------------------------- |
|
104
|
|
|
|
|
|
|
# Multiple (matrix of) jobs |
|
105
|
|
|
|
|
|
|
# Thanks to Demian Ricchardi for a bug fix |
|
106
|
|
|
|
|
|
|
#----------------------------------------------- |
|
107
|
|
|
|
|
|
|
else |
|
108
|
|
|
|
|
|
|
{ |
|
109
|
0
|
|
|
|
|
0
|
my $subjob = $job->copy; |
|
110
|
0
|
|
|
|
|
0
|
for (my $i = 0; $i < @{$job->{cmd}}; $i++) |
|
|
0
|
|
|
|
|
0
|
|
|
111
|
|
|
|
|
|
|
{ |
|
112
|
|
|
|
|
|
|
# Get command |
|
113
|
0
|
|
|
|
|
0
|
my $list = ${$job->{cmd}}[$i]; |
|
|
0
|
|
|
|
|
0
|
|
|
114
|
0
|
0
|
|
|
|
0
|
my $cmd = (ref $list)? (join("\n", @$list)): $list; |
|
115
|
0
|
|
|
|
|
0
|
$subjob->{cmd} = $cmd; |
|
116
|
|
|
|
|
|
|
|
|
117
|
|
|
|
|
|
|
# Generate and submit job script |
|
118
|
0
|
|
|
|
|
0
|
my $tempFile = &genScript($self, $subjob); # generate script |
|
119
|
0
|
|
|
|
|
0
|
my $out = &call_qsub('qsub', $tempFile); # submit script |
|
120
|
0
|
|
|
|
|
0
|
my $pbsid = ($out =~ /^(\d+)/)[0]; # get pid |
|
121
|
0
|
|
|
|
|
0
|
rename($tempFile, "$file.$pbsid"); # rename script |
|
122
|
0
|
|
|
|
|
0
|
push(@pbsid, $pbsid); |
|
123
|
|
|
|
|
|
|
} |
|
124
|
0
|
|
|
|
|
0
|
$job->pbsid(\@pbsid); |
|
125
|
|
|
|
|
|
|
} |
|
126
|
|
|
|
|
|
|
|
|
127
|
|
|
|
|
|
|
#----------------------------------------------- |
|
128
|
|
|
|
|
|
|
# Dependency: submit previous and following jobs |
|
129
|
|
|
|
|
|
|
#----------------------------------------------- |
|
130
|
0
|
|
|
|
|
0
|
&_qsubDepend($self, $job, \@pbsid); |
|
131
|
|
|
|
|
|
|
|
|
132
|
0
|
|
|
|
|
0
|
return(\@pbsid); |
|
133
|
|
|
|
|
|
|
} |
|
134
|
|
|
|
|
|
|
#------------------------------------------------------------------- |
|
135
|
|
|
|
|
|
|
|
|
136
|
|
|
|
|
|
|
|
|
137
|
|
|
|
|
|
|
#------------------------------------------------------------------- |
|
138
|
|
|
|
|
|
|
# Thanks to Sander Hulst |
|
139
|
|
|
|
|
|
|
sub call_qsub |
|
140
|
|
|
|
|
|
|
{ |
|
141
|
0
|
|
|
0
|
0
|
0
|
my @args = @_; |
|
142
|
|
|
|
|
|
|
|
|
143
|
|
|
|
|
|
|
# If the qsub command fails, for instance, pbs_server is not running, |
|
144
|
|
|
|
|
|
|
# PBS::Client's qsub should not silently ignore. Disable any reaper |
|
145
|
|
|
|
|
|
|
# functions so the exit code can be captured |
|
146
|
8
|
|
|
8
|
|
87
|
use Symbol qw(gensym); |
|
|
8
|
|
|
|
|
16
|
|
|
|
8
|
|
|
|
|
448
|
|
|
147
|
8
|
|
|
8
|
|
6978
|
use IPC::Open3; |
|
|
8
|
|
|
|
|
23674
|
|
|
|
8
|
|
|
|
|
37112
|
|
|
148
|
0
|
|
|
|
|
0
|
my $stdout = gensym(); |
|
149
|
0
|
|
|
|
|
0
|
my $stderr = gensym(); |
|
150
|
|
|
|
|
|
|
{ |
|
151
|
0
|
|
|
0
|
|
0
|
local $SIG{CHLD} = sub{}; |
|
|
0
|
|
|
|
|
0
|
|
|
|
0
|
|
|
|
|
0
|
|
|
152
|
0
|
|
|
|
|
0
|
my $pid = open3(gensym, $stdout, $stderr, @args); |
|
153
|
0
|
|
|
|
|
0
|
waitpid($pid,0); |
|
154
|
|
|
|
|
|
|
} |
|
155
|
0
|
0
|
|
|
|
0
|
confess <$stderr> if ($?); |
|
156
|
0
|
|
|
|
|
0
|
return <$stdout>; |
|
157
|
|
|
|
|
|
|
} |
|
158
|
|
|
|
|
|
|
#------------------------------------------------------------------- |
|
159
|
|
|
|
|
|
|
|
|
160
|
|
|
|
|
|
|
|
|
161
|
|
|
|
|
|
|
#------------------------------------------------------------------- |
|
162
|
|
|
|
|
|
|
# Dispatch PBS jobs by dispatch and qsub command |
|
163
|
|
|
|
|
|
|
# - Codes for backward compatible with old private software |
|
164
|
|
|
|
|
|
|
# - called subroutines: getScript() |
|
165
|
|
|
|
|
|
|
# |
|
166
|
|
|
|
|
|
|
# |
|
167
|
|
|
|
|
|
|
# $self -- server object |
|
168
|
|
|
|
|
|
|
# $job --- job object |
|
169
|
|
|
|
|
|
|
# |
|
170
|
|
|
|
|
|
|
# |
|
171
|
|
|
|
|
|
|
# \@pbsid -- array reference of PBS job ID |
|
172
|
|
|
|
|
|
|
sub dispatch |
|
173
|
|
|
|
|
|
|
{ |
|
174
|
0
|
|
|
0
|
0
|
0
|
my ($self, $job) = @_; |
|
175
|
0
|
|
|
|
|
0
|
my $file = $job->{script}; |
|
176
|
|
|
|
|
|
|
|
|
177
|
|
|
|
|
|
|
#----------------------------------------------- |
|
178
|
|
|
|
|
|
|
# Dependency: count number of previous jobs |
|
179
|
|
|
|
|
|
|
#----------------------------------------------- |
|
180
|
0
|
|
|
|
|
0
|
my $on = &_numPrevJob($job); |
|
181
|
0
|
0
|
|
|
|
0
|
$job->{depend}{on} = [$on] if ($on); |
|
182
|
0
|
|
|
|
|
0
|
my @pbsid = (); |
|
183
|
|
|
|
|
|
|
|
|
184
|
|
|
|
|
|
|
#--------- |
|
185
|
|
|
|
|
|
|
# Dispatch |
|
186
|
|
|
|
|
|
|
#--------- |
|
187
|
0
|
|
|
|
|
0
|
my $subjob = $job->copy; |
|
188
|
0
|
|
|
|
|
0
|
for (my $i = 0; $i < @{$job->{cmd}}; $i++) |
|
|
0
|
|
|
|
|
0
|
|
|
189
|
|
|
|
|
|
|
{ |
|
190
|
|
|
|
|
|
|
#-------------- |
|
191
|
|
|
|
|
|
|
# Dispatch jobs |
|
192
|
|
|
|
|
|
|
#-------------- |
|
193
|
0
|
|
|
|
|
0
|
my $list = ${$job->{cmd}}[$i]; |
|
|
0
|
|
|
|
|
0
|
|
|
194
|
0
|
0
|
|
|
|
0
|
my $djob = (ref $list)? (join("\n", @$list)): $list; |
|
195
|
0
|
|
|
|
|
0
|
$djob =~ s/\\/\\\\/g; |
|
196
|
0
|
|
|
|
|
0
|
$djob =~ s/"/\\\"/g; |
|
197
|
|
|
|
|
|
|
|
|
198
|
|
|
|
|
|
|
#----------------- |
|
199
|
|
|
|
|
|
|
# Dispatch command |
|
200
|
|
|
|
|
|
|
#----------------- |
|
201
|
0
|
|
|
|
|
0
|
my $cmd = 'echo "'."\n$djob\n".'" | dispatch'; |
|
202
|
0
|
|
|
|
|
0
|
$subjob->{cmd} = $cmd; |
|
203
|
|
|
|
|
|
|
|
|
204
|
|
|
|
|
|
|
#------------------------------- |
|
205
|
|
|
|
|
|
|
# Generate and submit job script |
|
206
|
|
|
|
|
|
|
#------------------------------- |
|
207
|
0
|
0
|
|
|
|
0
|
my $numJob = (ref $list)? (scalar(@$list)): 1; |
|
208
|
0
|
0
|
|
|
|
0
|
$subjob->nodes($numJob) if (!defined $subjob->{nodes}); |
|
209
|
0
|
|
|
|
|
0
|
my $tempFile = &genScript($self, $subjob); # generate script |
|
210
|
0
|
|
|
|
|
0
|
my $out = `qsub $tempFile`; # submit script |
|
211
|
0
|
|
|
|
|
0
|
my $pbsid = ($out =~ /^(\d+)/)[0]; # grab pid |
|
212
|
0
|
|
|
|
|
0
|
rename($tempFile, "$file.$pbsid"); # rename script |
|
213
|
0
|
|
|
|
|
0
|
push(@pbsid, $pbsid); |
|
214
|
|
|
|
|
|
|
} |
|
215
|
0
|
|
|
|
|
0
|
$job->pbsid(\@pbsid); |
|
216
|
|
|
|
|
|
|
|
|
217
|
|
|
|
|
|
|
#------------------------------------------------- |
|
218
|
|
|
|
|
|
|
# Dependency: dispatch previous and following jobs |
|
219
|
|
|
|
|
|
|
#------------------------------------------------- |
|
220
|
0
|
|
|
|
|
0
|
&_dphDepend($self, $job, \@pbsid); |
|
221
|
|
|
|
|
|
|
|
|
222
|
0
|
|
|
|
|
0
|
return(\@pbsid); |
|
223
|
|
|
|
|
|
|
} |
|
224
|
|
|
|
|
|
|
#------------------------------------------------------------------- |
|
225
|
|
|
|
|
|
|
|
|
226
|
|
|
|
|
|
|
|
|
227
|
|
|
|
|
|
|
#----------------------------------------------------------------- |
|
228
|
|
|
|
|
|
|
# Generate shell script from command string array |
|
229
|
|
|
|
|
|
|
# - called subroutines: _trace(), _nodes(), _stage() and _depend() |
|
230
|
|
|
|
|
|
|
# - called by qsub() |
|
231
|
|
|
|
|
|
|
# |
|
232
|
|
|
|
|
|
|
# |
|
233
|
|
|
|
|
|
|
# $self -- client object |
|
234
|
|
|
|
|
|
|
# $job --- job object |
|
235
|
|
|
|
|
|
|
# |
|
236
|
|
|
|
|
|
|
# |
|
237
|
|
|
|
|
|
|
# $file -- filename of job script |
|
238
|
|
|
|
|
|
|
sub genScript |
|
239
|
|
|
|
|
|
|
{ |
|
240
|
11
|
|
|
11
|
0
|
58
|
my ($self, $job) = @_; |
|
241
|
|
|
|
|
|
|
|
|
242
|
|
|
|
|
|
|
#------------------------- |
|
243
|
|
|
|
|
|
|
# Process the queue string |
|
244
|
|
|
|
|
|
|
#------------------------- |
|
245
|
11
|
|
|
|
|
21
|
my $queue = ''; |
|
246
|
11
|
100
|
|
|
|
89
|
$queue .= $job->{queue} if (defined $job->{queue}); |
|
247
|
11
|
100
|
|
|
|
70
|
$queue .= '@'.$self->{server} if (defined $self->{server}); |
|
248
|
|
|
|
|
|
|
|
|
249
|
|
|
|
|
|
|
#------------------------------- |
|
250
|
|
|
|
|
|
|
# Process the mail option string |
|
251
|
|
|
|
|
|
|
#------------------------------- |
|
252
|
11
|
|
|
|
|
28
|
my $mailOpt; |
|
253
|
11
|
100
|
|
|
|
39
|
if (defined $job->{mailopt}) |
|
254
|
|
|
|
|
|
|
{ |
|
255
|
1
|
|
|
|
|
2
|
$mailOpt = $job->{mailopt}; |
|
256
|
1
|
|
|
|
|
13
|
$mailOpt =~ s/\s*,\s*//; |
|
257
|
|
|
|
|
|
|
} |
|
258
|
|
|
|
|
|
|
|
|
259
|
|
|
|
|
|
|
#--------------------- |
|
260
|
|
|
|
|
|
|
# Generate node string |
|
261
|
|
|
|
|
|
|
#--------------------- |
|
262
|
11
|
|
|
|
|
38
|
my $nodes = &_nodes($job); |
|
263
|
|
|
|
|
|
|
|
|
264
|
|
|
|
|
|
|
#------------------------------------ |
|
265
|
|
|
|
|
|
|
# Set internal variable: |
|
266
|
|
|
|
|
|
|
# - temporary file for the job script |
|
267
|
|
|
|
|
|
|
#------------------------------------ |
|
268
|
11
|
|
|
|
|
57
|
my (undef, $file) = tempfile(DIR => $job->{wd}); |
|
269
|
11
|
|
|
|
|
287967
|
$job->_tempScript($file); |
|
270
|
|
|
|
|
|
|
|
|
271
|
|
|
|
|
|
|
#------------------------ |
|
272
|
|
|
|
|
|
|
# PBS request option list |
|
273
|
|
|
|
|
|
|
#------------------------ |
|
274
|
11
|
50
|
|
|
|
646
|
open(SH, ">$file") || confess "Can't write $file"; |
|
275
|
11
|
|
|
|
|
271
|
print SH "#!$job->{shell}\n\n"; |
|
276
|
11
|
|
|
|
|
41
|
print SH "#PBS -N $job->{name}\n"; |
|
277
|
11
|
|
|
|
|
42
|
print SH "#PBS -d $job->{wd}\n"; |
|
278
|
11
|
100
|
|
|
|
43
|
print SH "#PBS -e $job->{efile}\n" if (defined $job->{efile}); |
|
279
|
11
|
100
|
|
|
|
57
|
print SH "#PBS -o $job->{ofile}\n" if (defined $job->{ofile}); |
|
280
|
11
|
100
|
|
|
|
41
|
print SH "#PBS -q $queue\n" if ($queue); |
|
281
|
11
|
100
|
|
|
|
43
|
print SH "#PBS -W x=PARTITION:$job->{partition}\n" if |
|
282
|
|
|
|
|
|
|
(defined $job->{partition}); |
|
283
|
11
|
100
|
|
|
|
43
|
print SH "#PBS -W stagein=".&_stage('in', $job->{stagein})."\n" if |
|
284
|
|
|
|
|
|
|
(defined $job->{stagein}); |
|
285
|
11
|
100
|
|
|
|
47
|
print SH "#PBS -W stageout=".&_stage('out', $job->{stageout})."\n" if |
|
286
|
|
|
|
|
|
|
(defined $job->{stageout}); |
|
287
|
11
|
100
|
|
|
|
39
|
print SH "#PBS -M ".&_mailList($job->{maillist})."\n" if |
|
288
|
|
|
|
|
|
|
(defined $job->{maillist}); |
|
289
|
11
|
100
|
|
|
|
34
|
print SH "#PBS -m ".$mailOpt."\n" if (defined $mailOpt); |
|
290
|
11
|
100
|
|
|
|
54
|
print SH "#PBS -v ".&_varList($job->{vars})."\n" if (defined $job->{vars}); |
|
291
|
11
|
100
|
|
|
|
34
|
print SH "#PBS -A $job->{account}\n" if (defined $job->{account}); |
|
292
|
11
|
100
|
|
|
|
42
|
print SH "#PBS -p $job->{pri}\n" if (defined $job->{pri}); |
|
293
|
11
|
|
|
|
|
47
|
print SH "#PBS -l nodes=$nodes\n"; |
|
294
|
11
|
100
|
|
|
|
38
|
print SH "#PBS -l host=$job->{host}\n" if (defined $job->{host}); |
|
295
|
11
|
100
|
|
|
|
35
|
print SH "#PBS -l mem=$job->{mem}\n" if (defined $job->{mem}); |
|
296
|
11
|
100
|
|
|
|
41
|
print SH "#PBS -l pmem=$job->{pmem}\n" if (defined $job->{pmem}); |
|
297
|
11
|
100
|
|
|
|
44
|
print SH "#PBS -l vmem=$job->{vmem}\n" if (defined $job->{vmem}); |
|
298
|
11
|
100
|
|
|
|
37
|
print SH "#PBS -l pvmem=$job->{pvmem}\n" if (defined $job->{pvmem}); |
|
299
|
11
|
100
|
|
|
|
40
|
print SH "#PBS -l cput=$job->{cput}\n" if (defined $job->{cput}); |
|
300
|
11
|
100
|
|
|
|
38
|
print SH "#PBS -l pcput=$job->{pcput}\n" if (defined $job->{pcput}); |
|
301
|
11
|
100
|
|
|
|
47
|
print SH "#PBS -l walltime=$job->{wallt}\n" if (defined $job->{wallt}); |
|
302
|
11
|
100
|
|
|
|
40
|
print SH "#PBS -l nice=$job->{nice}\n" if (defined $job->{nice}); |
|
303
|
11
|
50
|
|
|
|
38
|
print SH "#PBS -l prologue=$job->{prologue}\n" if |
|
304
|
|
|
|
|
|
|
(defined $job->{prologue}); |
|
305
|
11
|
50
|
|
|
|
31
|
print SH "#PBS -l epilogue=$job->{epilogue}\n" if |
|
306
|
|
|
|
|
|
|
(defined $job->{epilogue}); |
|
307
|
|
|
|
|
|
|
|
|
308
|
|
|
|
|
|
|
#--------------- |
|
309
|
|
|
|
|
|
|
# Beginning time |
|
310
|
|
|
|
|
|
|
#--------------- |
|
311
|
11
|
100
|
|
|
|
35
|
if (defined $job->{begint}) |
|
312
|
|
|
|
|
|
|
{ |
|
313
|
2
|
|
|
|
|
4
|
my $begint = $job->{begint}; |
|
314
|
2
|
|
|
|
|
8
|
$begint =~ s/[\-\s]//g; |
|
315
|
2
|
|
|
|
|
17
|
my @a = ($begint =~ /([\d\.]+)/g); |
|
316
|
|
|
|
|
|
|
|
|
317
|
2
|
100
|
|
|
|
9
|
if (scalar(@a) == 3) |
|
318
|
|
|
|
|
|
|
{ |
|
319
|
1
|
|
|
|
|
6
|
print SH "#PBS -a $a[0]$a[1].$a[2]\n"; |
|
320
|
|
|
|
|
|
|
} |
|
321
|
|
|
|
|
|
|
else |
|
322
|
|
|
|
|
|
|
{ |
|
323
|
1
|
|
|
|
|
5
|
$begint = join('', @a); |
|
324
|
1
|
|
|
|
|
7
|
print SH "#PBS -a $begint\n"; |
|
325
|
|
|
|
|
|
|
} |
|
326
|
|
|
|
|
|
|
} |
|
327
|
|
|
|
|
|
|
|
|
328
|
|
|
|
|
|
|
#---------------------- |
|
329
|
|
|
|
|
|
|
# Job dependency option |
|
330
|
|
|
|
|
|
|
#---------------------- |
|
331
|
11
|
50
|
|
|
|
38
|
if (defined $job->{depend}) |
|
332
|
|
|
|
|
|
|
{ |
|
333
|
0
|
|
|
|
|
0
|
my $depend = &_depend($job->{depend}); |
|
334
|
0
|
|
|
|
|
0
|
print SH "#PBS -W depend=$depend\n"; |
|
335
|
|
|
|
|
|
|
} |
|
336
|
11
|
|
|
|
|
20
|
print SH "\n"; |
|
337
|
|
|
|
|
|
|
|
|
338
|
|
|
|
|
|
|
#-------------- |
|
339
|
|
|
|
|
|
|
# Trace the job |
|
340
|
|
|
|
|
|
|
#-------------- |
|
341
|
11
|
|
|
|
|
33
|
my $cmd = $job->{cmd}; |
|
342
|
11
|
|
|
|
|
88
|
my $tracer = $job->{tracer}; |
|
343
|
11
|
50
|
33
|
|
|
74
|
if ($tracer && $tracer ne 'off') |
|
344
|
|
|
|
|
|
|
{ |
|
345
|
0
|
|
|
|
|
0
|
my $server; |
|
346
|
0
|
0
|
|
|
|
0
|
if (defined $self->{server}) |
|
347
|
|
|
|
|
|
|
{ |
|
348
|
0
|
|
|
|
|
0
|
$server = $self->{server}; |
|
349
|
|
|
|
|
|
|
} |
|
350
|
|
|
|
|
|
|
else |
|
351
|
|
|
|
|
|
|
{ |
|
352
|
0
|
|
|
|
|
0
|
$server = `qstat -Bf|head -1`; |
|
353
|
0
|
|
|
|
|
0
|
$server = substr($server, 8); |
|
354
|
|
|
|
|
|
|
} |
|
355
|
|
|
|
|
|
|
|
|
356
|
0
|
0
|
|
|
|
0
|
my ($tfile) = (defined $job->{tfile})? ($job->{tfile}): |
|
357
|
|
|
|
|
|
|
($job->{script}.'.t$PBS_JOBID'); |
|
358
|
0
|
|
|
|
|
0
|
&_trace($server, $tfile, $cmd); |
|
359
|
|
|
|
|
|
|
} |
|
360
|
|
|
|
|
|
|
else |
|
361
|
|
|
|
|
|
|
{ |
|
362
|
|
|
|
|
|
|
#---------------- |
|
363
|
|
|
|
|
|
|
# Execute command |
|
364
|
|
|
|
|
|
|
#---------------- |
|
365
|
11
|
|
|
|
|
35
|
print SH "$cmd\n"; |
|
366
|
|
|
|
|
|
|
} |
|
367
|
11
|
|
|
|
|
625
|
close(SH); |
|
368
|
|
|
|
|
|
|
|
|
369
|
11
|
|
|
|
|
94
|
return($file); |
|
370
|
|
|
|
|
|
|
} |
|
371
|
|
|
|
|
|
|
#----------------------------------------------------------------- |
|
372
|
|
|
|
|
|
|
|
|
373
|
|
|
|
|
|
|
|
|
374
|
|
|
|
|
|
|
#------------------------------ |
|
375
|
|
|
|
|
|
|
# Count number of previous jobs |
|
376
|
|
|
|
|
|
|
sub _numPrevJob |
|
377
|
|
|
|
|
|
|
{ |
|
378
|
0
|
|
|
0
|
|
0
|
my ($job) = @_; |
|
379
|
0
|
|
|
|
|
0
|
my $on = 0; |
|
380
|
0
|
0
|
|
|
|
0
|
if (defined $job->{prev}) |
|
381
|
|
|
|
|
|
|
{ |
|
382
|
0
|
|
|
|
|
0
|
foreach my $type (keys %{$job->{prev}}) |
|
|
0
|
|
|
|
|
0
|
|
|
383
|
|
|
|
|
|
|
{ |
|
384
|
0
|
0
|
|
|
|
0
|
if (ref($job->{prev}{$type}) eq 'ARRAY') |
|
385
|
|
|
|
|
|
|
{ |
|
386
|
0
|
|
|
|
|
0
|
foreach my $jobTmp (@{$job->{prev}{$type}}) |
|
|
0
|
|
|
|
|
0
|
|
|
387
|
|
|
|
|
|
|
{ |
|
388
|
0
|
|
|
|
|
0
|
my $prevcmd = $jobTmp->{cmd}; |
|
389
|
0
|
0
|
|
|
|
0
|
if (ref($prevcmd)) |
|
390
|
|
|
|
|
|
|
{ |
|
391
|
0
|
|
|
|
|
0
|
my $numCmd = scalar(@$prevcmd); |
|
392
|
0
|
|
|
|
|
0
|
$on += $numCmd; |
|
393
|
|
|
|
|
|
|
} |
|
394
|
|
|
|
|
|
|
else |
|
395
|
|
|
|
|
|
|
{ |
|
396
|
0
|
|
|
|
|
0
|
$on++; |
|
397
|
|
|
|
|
|
|
} |
|
398
|
|
|
|
|
|
|
} |
|
399
|
|
|
|
|
|
|
} |
|
400
|
|
|
|
|
|
|
else |
|
401
|
|
|
|
|
|
|
{ |
|
402
|
0
|
|
|
|
|
0
|
my $prevcmd = $job->{prev}{$type}{cmd}; |
|
403
|
0
|
0
|
|
|
|
0
|
if (ref($prevcmd)) |
|
404
|
|
|
|
|
|
|
{ |
|
405
|
0
|
|
|
|
|
0
|
my $numCmd = scalar(@$prevcmd); |
|
406
|
0
|
|
|
|
|
0
|
$on += $numCmd; |
|
407
|
|
|
|
|
|
|
} |
|
408
|
|
|
|
|
|
|
else |
|
409
|
|
|
|
|
|
|
{ |
|
410
|
0
|
|
|
|
|
0
|
$on++; |
|
411
|
|
|
|
|
|
|
} |
|
412
|
|
|
|
|
|
|
} |
|
413
|
|
|
|
|
|
|
} |
|
414
|
|
|
|
|
|
|
} |
|
415
|
0
|
|
|
|
|
0
|
return($on); |
|
416
|
|
|
|
|
|
|
} |
|
417
|
|
|
|
|
|
|
#------------------------------ |
|
418
|
|
|
|
|
|
|
|
|
419
|
|
|
|
|
|
|
|
|
420
|
|
|
|
|
|
|
#---------------------- |
|
421
|
|
|
|
|
|
|
# Submit dependent jobs |
|
422
|
|
|
|
|
|
|
# - called by qsub() |
|
423
|
|
|
|
|
|
|
sub _qsubDepend |
|
424
|
|
|
|
|
|
|
{ |
|
425
|
0
|
|
|
0
|
|
0
|
my ($self, $job, $pbsid) = @_; |
|
426
|
|
|
|
|
|
|
|
|
427
|
0
|
|
|
|
|
0
|
my %type = ( |
|
428
|
|
|
|
|
|
|
'prevstart' => 'before', |
|
429
|
|
|
|
|
|
|
'prevend' => 'beforeany', |
|
430
|
|
|
|
|
|
|
'prevok' => 'beforeok', |
|
431
|
|
|
|
|
|
|
'prevfail' => 'beforenotok', |
|
432
|
|
|
|
|
|
|
'nextstart' => 'after', |
|
433
|
|
|
|
|
|
|
'nextend' => 'afterany', |
|
434
|
|
|
|
|
|
|
'nextok' => 'afterok', |
|
435
|
|
|
|
|
|
|
'nextfail' => 'afternotok', |
|
436
|
|
|
|
|
|
|
); |
|
437
|
|
|
|
|
|
|
|
|
438
|
0
|
|
|
|
|
0
|
foreach my $order (qw(prev next)) |
|
439
|
|
|
|
|
|
|
{ |
|
440
|
0
|
|
|
|
|
0
|
foreach my $cond (qw(start end ok fail)) |
|
441
|
|
|
|
|
|
|
{ |
|
442
|
0
|
0
|
|
|
|
0
|
if (defined $job->{$order}{$cond}) |
|
443
|
|
|
|
|
|
|
{ |
|
444
|
0
|
|
|
|
|
0
|
my $type = $type{$order.$cond}; |
|
445
|
0
|
0
|
|
|
|
0
|
if (ref($job->{$order}{$cond}) eq 'ARRAY') # array of job obj |
|
446
|
|
|
|
|
|
|
{ |
|
447
|
0
|
|
|
|
|
0
|
foreach my $jobTmp (@{$job->{$order}{$cond}}) |
|
|
0
|
|
|
|
|
0
|
|
|
448
|
|
|
|
|
|
|
{ |
|
449
|
0
|
|
|
|
|
0
|
$$jobTmp{depend}{$type} = $pbsid; |
|
450
|
0
|
|
|
|
|
0
|
&qsub($self, $jobTmp); |
|
451
|
|
|
|
|
|
|
} |
|
452
|
|
|
|
|
|
|
} |
|
453
|
|
|
|
|
|
|
else |
|
454
|
|
|
|
|
|
|
{ |
|
455
|
0
|
|
|
|
|
0
|
my $jobTmp = $job->{$order}{$cond}; |
|
456
|
0
|
|
|
|
|
0
|
$$jobTmp{depend}{$type} = $pbsid; |
|
457
|
0
|
|
|
|
|
0
|
&qsub($self, $jobTmp); |
|
458
|
|
|
|
|
|
|
} |
|
459
|
|
|
|
|
|
|
} |
|
460
|
|
|
|
|
|
|
} |
|
461
|
|
|
|
|
|
|
} |
|
462
|
|
|
|
|
|
|
} |
|
463
|
|
|
|
|
|
|
#---------------------- |
|
464
|
|
|
|
|
|
|
|
|
465
|
|
|
|
|
|
|
|
|
466
|
|
|
|
|
|
|
#------------------------ |
|
467
|
|
|
|
|
|
|
# Dispatch dependent jobs |
|
468
|
|
|
|
|
|
|
# - called by dispatch() |
|
469
|
|
|
|
|
|
|
sub _dphDepend |
|
470
|
|
|
|
|
|
|
{ |
|
471
|
0
|
|
|
0
|
|
0
|
my ($self, $job, $pbsid) = @_; |
|
472
|
|
|
|
|
|
|
|
|
473
|
0
|
|
|
|
|
0
|
my %type = ( |
|
474
|
|
|
|
|
|
|
'prevstart' => 'before', |
|
475
|
|
|
|
|
|
|
'prevend' => 'beforeany', |
|
476
|
|
|
|
|
|
|
'prevok' => 'beforeok', |
|
477
|
|
|
|
|
|
|
'prevfail' => 'beforenotok', |
|
478
|
|
|
|
|
|
|
'nextstart' => 'after', |
|
479
|
|
|
|
|
|
|
'nextend' => 'afterany', |
|
480
|
|
|
|
|
|
|
'nextok' => 'afterok', |
|
481
|
|
|
|
|
|
|
'nextfail' => 'afternotok', |
|
482
|
|
|
|
|
|
|
); |
|
483
|
|
|
|
|
|
|
|
|
484
|
0
|
|
|
|
|
0
|
foreach my $order (qw(prev next)) |
|
485
|
|
|
|
|
|
|
{ |
|
486
|
0
|
|
|
|
|
0
|
foreach my $cond (qw(start end ok fail)) |
|
487
|
|
|
|
|
|
|
{ |
|
488
|
0
|
0
|
|
|
|
0
|
if (defined $job->{$order}{$cond}) |
|
489
|
|
|
|
|
|
|
{ |
|
490
|
0
|
|
|
|
|
0
|
my $type = $type{$order.$cond}; |
|
491
|
0
|
0
|
|
|
|
0
|
if (ref($job->{$order}{$cond}) eq 'ARRAY') # array of job obj |
|
492
|
|
|
|
|
|
|
{ |
|
493
|
0
|
|
|
|
|
0
|
foreach my $jobTmp (@{$job->{$order}{$cond}}) |
|
|
0
|
|
|
|
|
0
|
|
|
494
|
|
|
|
|
|
|
{ |
|
495
|
0
|
|
|
|
|
0
|
$$jobTmp{depend}{$type} = $pbsid; |
|
496
|
0
|
|
|
|
|
0
|
&dispatch($self, $jobTmp); |
|
497
|
|
|
|
|
|
|
} |
|
498
|
|
|
|
|
|
|
} |
|
499
|
|
|
|
|
|
|
else |
|
500
|
|
|
|
|
|
|
{ |
|
501
|
0
|
|
|
|
|
0
|
my $jobTmp = $job->{$order}{$cond}; |
|
502
|
0
|
|
|
|
|
0
|
$$jobTmp{depend}{$type} = $pbsid; |
|
503
|
0
|
|
|
|
|
0
|
&dispatch($self, $jobTmp); |
|
504
|
|
|
|
|
|
|
} |
|
505
|
|
|
|
|
|
|
} |
|
506
|
|
|
|
|
|
|
} |
|
507
|
|
|
|
|
|
|
} |
|
508
|
|
|
|
|
|
|
} |
|
509
|
|
|
|
|
|
|
#------------------------ |
|
510
|
|
|
|
|
|
|
|
|
511
|
|
|
|
|
|
|
|
|
512
|
|
|
|
|
|
|
#----------------------------------------------------- |
|
513
|
|
|
|
|
|
|
# Trace the job by recording the location of execution |
|
514
|
|
|
|
|
|
|
# - called by genScript() |
|
515
|
|
|
|
|
|
|
# |
|
516
|
|
|
|
|
|
|
# |
|
517
|
|
|
|
|
|
|
# $cmd -- command string |
|
518
|
|
|
|
|
|
|
sub _trace |
|
519
|
|
|
|
|
|
|
{ |
|
520
|
0
|
|
|
0
|
|
0
|
my ($server, $tfile, $cmd) = @_; |
|
521
|
|
|
|
|
|
|
|
|
522
|
0
|
|
|
|
|
0
|
print SH "server=$server\n"; |
|
523
|
0
|
|
|
|
|
0
|
print SH "tfile=$tfile\n"; |
|
524
|
0
|
|
|
|
|
0
|
print SH 'tfile=${tfile/%.$server/}'."\n"; |
|
525
|
|
|
|
|
|
|
|
|
526
|
|
|
|
|
|
|
# Get machine, start and finish time |
|
527
|
0
|
|
|
|
|
0
|
print SH 'echo MACHINES : > $tfile'."\n"; |
|
528
|
0
|
|
|
|
|
0
|
print SH 'cat $PBS_NODEFILE >> $tfile'."\n"; |
|
529
|
0
|
|
|
|
|
0
|
print SH 'echo "" >> $tfile'."\n"; |
|
530
|
0
|
|
|
|
|
0
|
print SH 'start=`date +\'%F %T\'`'."\n"; |
|
531
|
0
|
|
|
|
|
0
|
print SH 'echo "START : $start" >> $tfile'."\n"; |
|
532
|
0
|
|
|
|
|
0
|
print SH "\n$cmd\n\n"; |
|
533
|
0
|
|
|
|
|
0
|
print SH 'finish=`date +\'%F %T\'`'."\n"; |
|
534
|
0
|
|
|
|
|
0
|
print SH 'echo "FINISH : $finish" >> $tfile'."\n"; |
|
535
|
|
|
|
|
|
|
|
|
536
|
|
|
|
|
|
|
# Calculate the duration of the command |
|
537
|
0
|
|
|
|
|
0
|
print SH 'begin=`date +%s -d "$start"`'."\n"; |
|
538
|
0
|
|
|
|
|
0
|
print SH 'end=`date +%s -d "$finish"`'."\n"; |
|
539
|
0
|
|
|
|
|
0
|
print SH 'sec=`expr $end - $begin`'."\n"; |
|
540
|
0
|
|
|
|
|
0
|
print SH 'if [ $sec -ge 60 ]'."\n"; |
|
541
|
0
|
|
|
|
|
0
|
print SH 'then'."\n"; |
|
542
|
0
|
|
|
|
|
0
|
print SH ' min=`expr $sec / 60`'."\n"; |
|
543
|
0
|
|
|
|
|
0
|
print SH ' sec=`expr $sec % 60`'."\n\n"; |
|
544
|
0
|
|
|
|
|
0
|
print SH ' if [ $min -ge 60 ]'."\n"; |
|
545
|
0
|
|
|
|
|
0
|
print SH ' then'."\n"; |
|
546
|
0
|
|
|
|
|
0
|
print SH ' hr=`expr $min / 60`'."\n"; |
|
547
|
0
|
|
|
|
|
0
|
print SH ' min=`expr $min % 60`'."\n"; |
|
548
|
0
|
|
|
|
|
0
|
print SH ' echo "RUNTIME : $hr hr $min min $sec sec" >> $tfile'."\n"; |
|
549
|
0
|
|
|
|
|
0
|
print SH ' else'."\n"; |
|
550
|
0
|
|
|
|
|
0
|
print SH ' echo "RUNTIME : $min min $sec sec" >> $tfile'."\n"; |
|
551
|
0
|
|
|
|
|
0
|
print SH ' fi'."\n"; |
|
552
|
0
|
|
|
|
|
0
|
print SH 'else'."\n"; |
|
553
|
0
|
|
|
|
|
0
|
print SH ' echo "RUNTIME : $sec sec" >> $tfile'."\n"; |
|
554
|
0
|
|
|
|
|
0
|
print SH 'fi'."\n"; |
|
555
|
|
|
|
|
|
|
} |
|
556
|
|
|
|
|
|
|
#----------------------------------------------------- |
|
557
|
|
|
|
|
|
|
|
|
558
|
|
|
|
|
|
|
|
|
559
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
560
|
|
|
|
|
|
|
# Construct node request string |
|
561
|
|
|
|
|
|
|
# - called by genScript() |
|
562
|
|
|
|
|
|
|
# |
|
563
|
|
|
|
|
|
|
# |
|
564
|
|
|
|
|
|
|
# $job -- job object |
|
565
|
|
|
|
|
|
|
# |
|
566
|
|
|
|
|
|
|
# |
|
567
|
|
|
|
|
|
|
# $str -- node request string |
|
568
|
|
|
|
|
|
|
sub _nodes |
|
569
|
|
|
|
|
|
|
{ |
|
570
|
11
|
|
|
11
|
|
18
|
my ($job) = @_; |
|
571
|
11
|
100
|
|
|
|
86
|
$job->nodes('1') if (!defined $job->{nodes}); |
|
572
|
11
|
|
|
|
|
30
|
my $type = ref($job->{nodes}); |
|
573
|
|
|
|
|
|
|
|
|
574
|
|
|
|
|
|
|
#------------------------------------------- |
|
575
|
|
|
|
|
|
|
# String |
|
576
|
|
|
|
|
|
|
# Example: |
|
577
|
|
|
|
|
|
|
# (1) nodes => 2, ppn => 2 |
|
578
|
|
|
|
|
|
|
# (2) nodes => "delta01+delta02", ppn => 2 |
|
579
|
|
|
|
|
|
|
# (3) nodes => "delta01:ppn=2+delta02:ppn=1" |
|
580
|
|
|
|
|
|
|
#------------------------------------------- |
|
581
|
11
|
100
|
|
|
|
75
|
if ($type eq '') |
|
|
|
100
|
|
|
|
|
|
|
|
|
50
|
|
|
|
|
|
|
582
|
|
|
|
|
|
|
{ |
|
583
|
9
|
100
|
|
|
|
52
|
if ($job->{nodes} =~ /^\d+$/) |
|
584
|
|
|
|
|
|
|
{ |
|
585
|
8
|
|
|
|
|
22
|
my $str = "$job->{nodes}"; |
|
586
|
8
|
100
|
|
|
|
26
|
$str .= ":ppn=$job->{ppn}" if (defined $job->{ppn}); |
|
587
|
8
|
|
|
|
|
27
|
return($str); |
|
588
|
|
|
|
|
|
|
} |
|
589
|
|
|
|
|
|
|
else |
|
590
|
|
|
|
|
|
|
{ |
|
591
|
1
|
50
|
|
|
|
4
|
if (defined $job->{ppn}) |
|
592
|
|
|
|
|
|
|
{ |
|
593
|
1
|
|
|
|
|
10
|
my @node = split(/\s*\+\s*/, $job->{nodes}); |
|
594
|
1
|
|
|
|
|
6
|
my $str = join(":ppn=$job->{ppn}+", @node); |
|
595
|
1
|
|
|
|
|
3
|
$str .= ":ppn=$job->{ppn}"; |
|
596
|
1
|
|
|
|
|
23
|
return($str); |
|
597
|
|
|
|
|
|
|
} |
|
598
|
0
|
|
|
|
|
0
|
return($job->{nodes}); |
|
599
|
|
|
|
|
|
|
} |
|
600
|
|
|
|
|
|
|
} |
|
601
|
|
|
|
|
|
|
#----------------------------------------------- |
|
602
|
|
|
|
|
|
|
# Array |
|
603
|
|
|
|
|
|
|
# Example: |
|
604
|
|
|
|
|
|
|
# (1) nodes => [qw(delta01 delta02)], ppn => 2 |
|
605
|
|
|
|
|
|
|
# (2) nodes => [qw(delta01:ppn=2 delta02:ppn=1)] |
|
606
|
|
|
|
|
|
|
#----------------------------------------------- |
|
607
|
|
|
|
|
|
|
elsif ($type eq 'ARRAY') |
|
608
|
|
|
|
|
|
|
{ |
|
609
|
1
|
50
|
|
|
|
5
|
if (defined $job->{ppn}) |
|
610
|
|
|
|
|
|
|
{ |
|
611
|
1
|
|
|
|
|
6
|
my $str = join( ":ppn=$job->{ppn}+", @{$job->{nodes}} ); |
|
|
1
|
|
|
|
|
3
|
|
|
612
|
1
|
|
|
|
|
3
|
$str .= ":ppn=$job->{ppn}"; |
|
613
|
1
|
|
|
|
|
3
|
return($str); |
|
614
|
|
|
|
|
|
|
} |
|
615
|
0
|
|
|
|
|
0
|
return( join('+', @{$job->{nodes}}) ); |
|
|
0
|
|
|
|
|
0
|
|
|
616
|
|
|
|
|
|
|
} |
|
617
|
|
|
|
|
|
|
#------------------------------------------ |
|
618
|
|
|
|
|
|
|
# Hash |
|
619
|
|
|
|
|
|
|
# Example: |
|
620
|
|
|
|
|
|
|
# (1) nodes => {delta01 => 2, delta02 => 1} |
|
621
|
|
|
|
|
|
|
#------------------------------------------ |
|
622
|
|
|
|
|
|
|
elsif ($type eq 'HASH') |
|
623
|
|
|
|
|
|
|
{ |
|
624
|
1
|
|
|
|
|
2
|
my $str = ''; |
|
625
|
1
|
|
|
|
|
2
|
foreach my $node (keys %{$job->{nodes}}) |
|
|
1
|
|
|
|
|
4
|
|
|
626
|
|
|
|
|
|
|
{ |
|
627
|
1
|
|
|
|
|
4
|
$str .= "$node:ppn=${$job->{nodes}}{$node}+"; |
|
|
1
|
|
|
|
|
5
|
|
|
628
|
|
|
|
|
|
|
} |
|
629
|
1
|
|
|
|
|
3
|
chop($str); |
|
630
|
1
|
|
|
|
|
3
|
return($str); |
|
631
|
|
|
|
|
|
|
} |
|
632
|
|
|
|
|
|
|
} |
|
633
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
634
|
|
|
|
|
|
|
|
|
635
|
|
|
|
|
|
|
|
|
636
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
637
|
|
|
|
|
|
|
# Construct string for file staging (in and out) |
|
638
|
|
|
|
|
|
|
# - called by genScript() |
|
639
|
|
|
|
|
|
|
sub _stage |
|
640
|
|
|
|
|
|
|
{ |
|
641
|
2
|
|
|
2
|
|
5
|
my ($act, $file) = @_; |
|
642
|
2
|
|
|
|
|
3
|
my $type = ref($file); |
|
643
|
|
|
|
|
|
|
|
|
644
|
|
|
|
|
|
|
#------------------------------------------- |
|
645
|
|
|
|
|
|
|
# String |
|
646
|
|
|
|
|
|
|
# Example: |
|
647
|
|
|
|
|
|
|
# stagein => "to01.file@fromMachine:from01.file,". |
|
648
|
|
|
|
|
|
|
# "to02.file@fromMachine:from02.file" |
|
649
|
|
|
|
|
|
|
# stageout => "from01.file@toMachine:to01.file,". |
|
650
|
|
|
|
|
|
|
# "from02.file@toMachine:to02.file" |
|
651
|
|
|
|
|
|
|
#------------------------------------------- |
|
652
|
2
|
50
|
|
|
|
13
|
return($file) if ($type eq ''); |
|
653
|
|
|
|
|
|
|
|
|
654
|
|
|
|
|
|
|
#------------------------------------------- |
|
655
|
|
|
|
|
|
|
# Array |
|
656
|
|
|
|
|
|
|
# Example: |
|
657
|
|
|
|
|
|
|
# stagein => ['to01.file@fromMachine:from01.file', |
|
658
|
|
|
|
|
|
|
# 'to02.file@fromMachine:from02.file'] |
|
659
|
|
|
|
|
|
|
# stageout => ['from01.file@toMachine:to01.file', |
|
660
|
|
|
|
|
|
|
# 'from02.file@toMachine:to02.file'] |
|
661
|
|
|
|
|
|
|
#------------------------------------------- |
|
662
|
0
|
0
|
|
|
|
0
|
return(join(',', @$file)) if ($type eq 'ARRAY'); |
|
663
|
|
|
|
|
|
|
|
|
664
|
|
|
|
|
|
|
#------------------------------------------- |
|
665
|
|
|
|
|
|
|
# Hash |
|
666
|
|
|
|
|
|
|
# Example: |
|
667
|
|
|
|
|
|
|
# stagein => {'fromMachine:from01.file' => 'to01.file', |
|
668
|
|
|
|
|
|
|
# 'fromMachine:from02.file' => 'to02.file'} |
|
669
|
|
|
|
|
|
|
# stageout => {'from01.file' => 'toMachine:to01.file', |
|
670
|
|
|
|
|
|
|
# 'from02.file' => 'toMachine:to02.file'} |
|
671
|
|
|
|
|
|
|
#------------------------------------------- |
|
672
|
0
|
0
|
|
|
|
0
|
if ($type eq 'HASH') |
|
673
|
|
|
|
|
|
|
{ |
|
674
|
0
|
0
|
|
|
|
0
|
if ($act eq 'in') |
|
|
|
0
|
|
|
|
|
|
|
675
|
|
|
|
|
|
|
{ |
|
676
|
0
|
|
|
|
|
0
|
my @stages; |
|
677
|
0
|
|
|
|
|
0
|
foreach my $f (keys %$file) |
|
678
|
|
|
|
|
|
|
{ |
|
679
|
0
|
|
|
|
|
0
|
push(@stages, "$$file{$f}".'@'."$f"); |
|
680
|
|
|
|
|
|
|
} |
|
681
|
0
|
|
|
|
|
0
|
return(join(',', @stages)); |
|
682
|
|
|
|
|
|
|
} |
|
683
|
|
|
|
|
|
|
elsif ($act eq 'out') |
|
684
|
|
|
|
|
|
|
{ |
|
685
|
0
|
|
|
|
|
0
|
my @stages; |
|
686
|
0
|
|
|
|
|
0
|
foreach my $f (keys %$file) |
|
687
|
|
|
|
|
|
|
{ |
|
688
|
0
|
|
|
|
|
0
|
push(@stages, "$f".'@'."$$file{$f}"); |
|
689
|
|
|
|
|
|
|
} |
|
690
|
0
|
|
|
|
|
0
|
return(join(',', @stages)); |
|
691
|
|
|
|
|
|
|
} |
|
692
|
|
|
|
|
|
|
} |
|
693
|
|
|
|
|
|
|
} |
|
694
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
695
|
|
|
|
|
|
|
|
|
696
|
|
|
|
|
|
|
|
|
697
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
698
|
|
|
|
|
|
|
# Construct the job dependency string |
|
699
|
|
|
|
|
|
|
# - called by genScript() |
|
700
|
|
|
|
|
|
|
# |
|
701
|
|
|
|
|
|
|
# |
|
702
|
|
|
|
|
|
|
# $arg -- hash reference of job dependency |
|
703
|
|
|
|
|
|
|
# |
|
704
|
|
|
|
|
|
|
# |
|
705
|
|
|
|
|
|
|
# $str -- job dependency string |
|
706
|
|
|
|
|
|
|
sub _depend |
|
707
|
|
|
|
|
|
|
{ |
|
708
|
0
|
|
|
0
|
|
0
|
my ($arg) = @_; |
|
709
|
0
|
|
|
|
|
0
|
my $str = ''; |
|
710
|
|
|
|
|
|
|
|
|
711
|
0
|
|
|
|
|
0
|
foreach my $type (keys %$arg) |
|
712
|
|
|
|
|
|
|
{ |
|
713
|
0
|
0
|
|
|
|
0
|
$str .= ',' unless ($str eq ''); |
|
714
|
0
|
|
|
|
|
0
|
my $joblist = join(':', @{$$arg{$type}}); |
|
|
0
|
|
|
|
|
0
|
|
|
715
|
0
|
|
|
|
|
0
|
$str .= "$type:$joblist"; |
|
716
|
|
|
|
|
|
|
} |
|
717
|
0
|
|
|
|
|
0
|
return($str); |
|
718
|
|
|
|
|
|
|
} |
|
719
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
720
|
|
|
|
|
|
|
|
|
721
|
|
|
|
|
|
|
|
|
722
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
723
|
|
|
|
|
|
|
# Construct the mail address list string |
|
724
|
|
|
|
|
|
|
# - called by genScript() |
|
725
|
|
|
|
|
|
|
# |
|
726
|
|
|
|
|
|
|
# |
|
727
|
|
|
|
|
|
|
# "abc@ABC.com, def@DEF.com" or |
|
728
|
|
|
|
|
|
|
# [qw(abc@ABC.com def@DEF.com)] |
|
729
|
|
|
|
|
|
|
# |
|
730
|
|
|
|
|
|
|
# |
|
731
|
|
|
|
|
|
|
# abc@ABC.com,def@DEF.com |
|
732
|
|
|
|
|
|
|
sub _mailList |
|
733
|
|
|
|
|
|
|
{ |
|
734
|
1
|
|
|
1
|
|
7
|
my ($arg) = @_; |
|
735
|
1
|
50
|
|
|
|
4
|
if (ref($arg) eq 'ARRAY') |
|
736
|
|
|
|
|
|
|
{ |
|
737
|
0
|
|
|
|
|
0
|
return(join(',', @$arg)); |
|
738
|
|
|
|
|
|
|
} |
|
739
|
|
|
|
|
|
|
else |
|
740
|
|
|
|
|
|
|
{ |
|
741
|
1
|
|
|
|
|
7
|
$arg =~ s/,\s+/,/g; |
|
742
|
1
|
|
|
|
|
5
|
return($arg); |
|
743
|
|
|
|
|
|
|
} |
|
744
|
|
|
|
|
|
|
} |
|
745
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
746
|
|
|
|
|
|
|
|
|
747
|
|
|
|
|
|
|
|
|
748
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
749
|
|
|
|
|
|
|
# Construct the environment variable list string |
|
750
|
|
|
|
|
|
|
# - called by genScript() |
|
751
|
|
|
|
|
|
|
# |
|
752
|
|
|
|
|
|
|
# |
|
753
|
|
|
|
|
|
|
# ['A', 'B =b', {C => '', D => 'd'}], |
|
754
|
|
|
|
|
|
|
# |
|
755
|
|
|
|
|
|
|
# |
|
756
|
|
|
|
|
|
|
# A,B=b,c,D=d |
|
757
|
|
|
|
|
|
|
sub _varList |
|
758
|
|
|
|
|
|
|
{ |
|
759
|
2
|
|
|
2
|
|
3
|
my ($arg) = @_; |
|
760
|
|
|
|
|
|
|
|
|
761
|
2
|
100
|
|
|
|
8
|
if (ref($arg) eq 'ARRAY') |
|
|
|
50
|
|
|
|
|
|
|
762
|
|
|
|
|
|
|
{ |
|
763
|
1
|
|
|
|
|
2
|
my $str; |
|
764
|
1
|
|
|
|
|
3
|
foreach my $ele (@$arg) |
|
765
|
|
|
|
|
|
|
{ |
|
766
|
4
|
100
|
|
|
|
11
|
$str .= ',' if (defined $str); |
|
767
|
4
|
50
|
|
|
|
8
|
if (ref($ele) eq 'HASH') |
|
768
|
|
|
|
|
|
|
{ |
|
769
|
0
|
|
|
|
|
0
|
$str .= &_hashVar($ele); |
|
770
|
|
|
|
|
|
|
} |
|
771
|
|
|
|
|
|
|
else |
|
772
|
|
|
|
|
|
|
{ |
|
773
|
4
|
|
|
|
|
11
|
$ele =~ s/\s*=\s*/=/; # remove possible spaces around "=" |
|
774
|
4
|
|
|
|
|
12
|
$str .= $ele; |
|
775
|
|
|
|
|
|
|
} |
|
776
|
|
|
|
|
|
|
} |
|
777
|
1
|
|
|
|
|
4
|
return($str); |
|
778
|
|
|
|
|
|
|
} |
|
779
|
|
|
|
|
|
|
elsif (ref($arg) eq 'HASH') |
|
780
|
|
|
|
|
|
|
{ |
|
781
|
0
|
|
|
|
|
0
|
return(&_hashVar($arg)); |
|
782
|
|
|
|
|
|
|
} |
|
783
|
|
|
|
|
|
|
else |
|
784
|
|
|
|
|
|
|
{ |
|
785
|
1
|
|
|
|
|
2
|
my $str = $arg; |
|
786
|
1
|
|
|
|
|
8
|
$str =~ s/\s*=\s*/=/g; |
|
787
|
1
|
|
|
|
|
6
|
$str =~ s/\s*,\s+/,/g; |
|
788
|
1
|
|
|
|
|
4
|
return($str); |
|
789
|
|
|
|
|
|
|
} |
|
790
|
|
|
|
|
|
|
|
|
791
|
|
|
|
|
|
|
# Construct environment variable list string from hash |
|
792
|
|
|
|
|
|
|
sub _hashVar |
|
793
|
|
|
|
|
|
|
{ |
|
794
|
0
|
|
|
0
|
|
0
|
my ($h) = @_; |
|
795
|
0
|
|
|
|
|
0
|
my $str; |
|
796
|
0
|
|
|
|
|
0
|
foreach my $key (keys %$h) |
|
797
|
|
|
|
|
|
|
{ |
|
798
|
0
|
0
|
|
|
|
0
|
$str .= ',' if (defined $str); |
|
799
|
0
|
|
|
|
|
0
|
$str .= "$key"; |
|
800
|
0
|
0
|
|
|
|
0
|
$str .= "=$$h{$key}" if ($$h{$key} ne ''); |
|
801
|
|
|
|
|
|
|
} |
|
802
|
0
|
|
|
|
|
0
|
return($str); |
|
803
|
|
|
|
|
|
|
} |
|
804
|
|
|
|
|
|
|
} |
|
805
|
|
|
|
|
|
|
#---------------------------------------------------------- |
|
806
|
|
|
|
|
|
|
|
|
807
|
|
|
|
|
|
|
|
|
808
|
|
|
|
|
|
|
#################### PBS::Client::Job #################### |
|
809
|
|
|
|
|
|
|
|
|
810
|
|
|
|
|
|
|
package PBS::Client::Job; |
|
811
|
8
|
|
|
8
|
|
143
|
use strict; |
|
|
8
|
|
|
|
|
13
|
|
|
|
8
|
|
|
|
|
326
|
|
|
812
|
|
|
|
|
|
|
|
|
813
|
|
|
|
|
|
|
#------------------------------------------------ |
|
814
|
|
|
|
|
|
|
# Job class |
|
815
|
|
|
|
|
|
|
#------------------------------------------------ |
|
816
|
|
|
|
|
|
|
|
|
817
|
8
|
|
|
8
|
|
44
|
use Cwd; |
|
|
8
|
|
|
|
|
17
|
|
|
|
8
|
|
|
|
|
652
|
|
|
818
|
8
|
|
|
8
|
|
42
|
use Carp; |
|
|
8
|
|
|
|
|
14
|
|
|
|
8
|
|
|
|
|
4651
|
|
|
819
|
|
|
|
|
|
|
|
|
820
|
|
|
|
|
|
|
#------------------------- |
|
821
|
|
|
|
|
|
|
# Constructor method |
|
822
|
|
|
|
|
|
|
# |
|
823
|
|
|
|
|
|
|
# |
|
824
|
|
|
|
|
|
|
# $class -- job object |
|
825
|
|
|
|
|
|
|
# %hash --- argument hash |
|
826
|
|
|
|
|
|
|
# |
|
827
|
|
|
|
|
|
|
# |
|
828
|
|
|
|
|
|
|
# $self -- job object |
|
829
|
|
|
|
|
|
|
sub new |
|
830
|
|
|
|
|
|
|
{ |
|
831
|
14
|
|
|
14
|
|
214
|
my ($class, %hash) = @_; |
|
832
|
|
|
|
|
|
|
|
|
833
|
|
|
|
|
|
|
#------------- |
|
834
|
|
|
|
|
|
|
# set defaults |
|
835
|
|
|
|
|
|
|
#------------- |
|
836
|
14
|
100
|
|
|
|
17633
|
$hash{wd} = cwd if (!defined $hash{wd}); |
|
837
|
14
|
100
|
|
|
|
166
|
$hash{script} = 'pbsjob.sh' if (!defined $hash{script}); |
|
838
|
14
|
50
|
|
|
|
141
|
$hash{tracer} = 'off' if (!defined $hash{tracer}); |
|
839
|
14
|
100
|
|
|
|
103
|
$hash{shell} = '/bin/sh' if (!defined $hash{shell}); |
|
840
|
14
|
100
|
|
|
|
91
|
$hash{name} = $hash{script} if (!defined $hash{name}); |
|
841
|
|
|
|
|
|
|
|
|
842
|
14
|
|
|
|
|
56
|
my $self = \%hash; |
|
843
|
14
|
|
|
|
|
186
|
return bless($self, $class); |
|
844
|
|
|
|
|
|
|
} |
|
845
|
|
|
|
|
|
|
#------------------------- |
|
846
|
|
|
|
|
|
|
|
|
847
|
|
|
|
|
|
|
|
|
848
|
|
|
|
|
|
|
#-------------------------- |
|
849
|
|
|
|
|
|
|
# Generic attribute methods |
|
850
|
|
|
|
|
|
|
sub AUTOLOAD |
|
851
|
|
|
|
|
|
|
{ |
|
852
|
25
|
|
|
25
|
|
173
|
my ($self, $key) = @_; |
|
853
|
25
|
|
|
|
|
54
|
my $attr = our $AUTOLOAD; |
|
854
|
25
|
|
|
|
|
242
|
$attr =~ s/.*:://; |
|
855
|
25
|
50
|
|
|
|
96
|
return if ($attr eq 'DESTROY'); # ignore destructor |
|
856
|
25
|
50
|
|
|
|
122
|
$self->{$attr} = $key if (defined $key); |
|
857
|
25
|
|
|
|
|
97
|
return($self->{$attr}); |
|
858
|
|
|
|
|
|
|
} |
|
859
|
|
|
|
|
|
|
#-------------------------- |
|
860
|
|
|
|
|
|
|
|
|
861
|
|
|
|
|
|
|
|
|
862
|
|
|
|
|
|
|
#--------------------------------------- |
|
863
|
|
|
|
|
|
|
# Pack commands |
|
864
|
|
|
|
|
|
|
# |
|
865
|
|
|
|
|
|
|
# |
|
866
|
|
|
|
|
|
|
# $self -- job object |
|
867
|
|
|
|
|
|
|
# %args -- argument hash |
|
868
|
|
|
|
|
|
|
# -- numQ -- number of queues |
|
869
|
|
|
|
|
|
|
# -- cpq --- commands per queue |
|
870
|
|
|
|
|
|
|
sub pack |
|
871
|
|
|
|
|
|
|
{ |
|
872
|
2
|
|
|
2
|
|
58
|
my ($self, %args) = @_; |
|
873
|
2
|
|
|
|
|
26
|
my $cmdlist = $self->{cmd}; |
|
874
|
2
|
50
|
|
|
|
19
|
return if (ref($cmdlist) ne 'ARRAY'); |
|
875
|
|
|
|
|
|
|
|
|
876
|
2
|
|
|
|
|
6
|
my @pack = (); |
|
877
|
2
|
|
|
|
|
7
|
my $jc = 0; # job counter |
|
878
|
2
|
100
|
|
|
|
30
|
if (defined $args{numQ}) |
|
|
|
50
|
|
|
|
|
|
|
879
|
|
|
|
|
|
|
{ |
|
880
|
1
|
|
|
|
|
10
|
for (my $i = 0; $i < @$cmdlist; $i++) |
|
881
|
|
|
|
|
|
|
{ |
|
882
|
10
|
100
|
|
|
|
17
|
if (ref($$cmdlist[$i])) |
|
883
|
|
|
|
|
|
|
{ |
|
884
|
9
|
|
|
|
|
9
|
foreach my $cell (@{$$cmdlist[$i]}) |
|
|
9
|
|
|
|
|
23
|
|
|
885
|
|
|
|
|
|
|
{ |
|
886
|
15
|
|
|
|
|
18
|
my $row = $jc % $args{numQ}; |
|
887
|
15
|
|
|
|
|
15
|
push(@{$pack[$row]}, $cell); |
|
|
15
|
|
|
|
|
38
|
|
|
888
|
15
|
|
|
|
|
36
|
$jc++; |
|
889
|
|
|
|
|
|
|
} |
|
890
|
|
|
|
|
|
|
} |
|
891
|
|
|
|
|
|
|
else |
|
892
|
|
|
|
|
|
|
{ |
|
893
|
1
|
|
|
|
|
5
|
my $row = $jc % $args{numQ}; |
|
894
|
1
|
|
|
|
|
1
|
push(@{$pack[$row]}, $$cmdlist[$i]); |
|
|
1
|
|
|
|
|
45
|
|
|
895
|
1
|
|
|
|
|
3
|
$jc++; |
|
896
|
|
|
|
|
|
|
} |
|
897
|
|
|
|
|
|
|
} |
|
898
|
|
|
|
|
|
|
} |
|
899
|
|
|
|
|
|
|
elsif (defined $args{cpq}) |
|
900
|
|
|
|
|
|
|
{ |
|
901
|
1
|
|
|
|
|
25
|
for (my $i = 0; $i < @$cmdlist; $i++) |
|
902
|
|
|
|
|
|
|
{ |
|
903
|
10
|
100
|
|
|
|
46
|
if (ref($$cmdlist[$i])) |
|
904
|
|
|
|
|
|
|
{ |
|
905
|
9
|
|
|
|
|
20
|
foreach my $cell (@{$$cmdlist[$i]}) |
|
|
9
|
|
|
|
|
49
|
|
|
906
|
|
|
|
|
|
|
{ |
|
907
|
15
|
|
|
|
|
82
|
my $row = int($jc / $args{cpq}); |
|
908
|
15
|
|
|
|
|
29
|
push(@{$pack[$row]}, $cell); |
|
|
15
|
|
|
|
|
63
|
|
|
909
|
15
|
|
|
|
|
79
|
$jc++; |
|
910
|
|
|
|
|
|
|
} |
|
911
|
|
|
|
|
|
|
} |
|
912
|
|
|
|
|
|
|
else |
|
913
|
|
|
|
|
|
|
{ |
|
914
|
1
|
|
|
|
|
10
|
my $row = int($jc / $args{cpq}); |
|
915
|
1
|
|
|
|
|
3
|
push(@{$pack[$row]}, $$cmdlist[$i]); |
|
|
1
|
|
|
|
|
9
|
|
|
916
|
1
|
|
|
|
|
6
|
$jc++; |
|
917
|
|
|
|
|
|
|
} |
|
918
|
|
|
|
|
|
|
} |
|
919
|
|
|
|
|
|
|
} |
|
920
|
2
|
|
|
|
|
47
|
$self->cmd([@pack]); |
|
921
|
|
|
|
|
|
|
} |
|
922
|
|
|
|
|
|
|
#--------------------------------------- |
|
923
|
|
|
|
|
|
|
|
|
924
|
|
|
|
|
|
|
|
|
925
|
|
|
|
|
|
|
#------------------------------- |
|
926
|
|
|
|
|
|
|
# Copy object using Data::Dumper |
|
927
|
|
|
|
|
|
|
sub copy |
|
928
|
|
|
|
|
|
|
{ |
|
929
|
2
|
|
|
2
|
|
51
|
my ($self, $num) = @_; |
|
930
|
8
|
|
|
8
|
|
9657
|
use Data::Dumper; |
|
|
8
|
|
|
|
|
63488
|
|
|
|
8
|
|
|
|
|
1392
|
|
|
931
|
2
|
100
|
66
|
|
|
32
|
if (!defined $num || $num == 1) |
|
932
|
|
|
|
|
|
|
{ |
|
933
|
1
|
|
|
|
|
9
|
my $clone; |
|
934
|
1
|
|
|
|
|
45
|
eval(Data::Dumper->Dump([$self], ['clone'])); |
|
935
|
1
|
|
|
|
|
41
|
return($clone); |
|
936
|
|
|
|
|
|
|
} |
|
937
|
|
|
|
|
|
|
|
|
938
|
1
|
|
|
|
|
10
|
my @clones = (); |
|
939
|
1
|
|
|
|
|
9
|
for (my $i = 0; $i < $num; $i++) |
|
940
|
|
|
|
|
|
|
{ |
|
941
|
2
|
|
|
|
|
5
|
my $clone; |
|
942
|
2
|
|
|
|
|
16
|
eval(Data::Dumper->Dump([$self], ['clone'])); |
|
943
|
2
|
|
|
|
|
37
|
push(@clones, $clone) |
|
944
|
|
|
|
|
|
|
} |
|
945
|
1
|
|
|
|
|
52
|
return(@clones); |
|
946
|
|
|
|
|
|
|
} |
|
947
|
|
|
|
|
|
|
#------------------------------- |
|
948
|
|
|
|
|
|
|
|
|
949
|
|
|
|
|
|
|
|
|
950
|
|
|
|
|
|
|
__END__ |