line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Stepford::Runner; |
2
|
|
|
|
|
|
|
|
3
|
39
|
|
|
39
|
|
3935130
|
use strict; |
|
39
|
|
|
|
|
352
|
|
|
39
|
|
|
|
|
1289
|
|
4
|
39
|
|
|
39
|
|
195
|
use warnings; |
|
39
|
|
|
|
|
79
|
|
|
39
|
|
|
|
|
988
|
|
5
|
39
|
|
|
39
|
|
12912
|
use namespace::autoclean; |
|
39
|
|
|
|
|
522286
|
|
|
39
|
|
|
|
|
175
|
|
6
|
|
|
|
|
|
|
|
7
|
|
|
|
|
|
|
our $VERSION = '0.006000'; |
8
|
|
|
|
|
|
|
|
9
|
39
|
|
|
39
|
|
22828
|
use List::AllUtils qw( first ); |
|
39
|
|
|
|
|
387824
|
|
|
39
|
|
|
|
|
3243
|
|
10
|
39
|
|
|
39
|
|
17828
|
use Module::Pluggable::Object; |
|
39
|
|
|
|
|
276758
|
|
|
39
|
|
|
|
|
1626
|
|
11
|
39
|
|
|
39
|
|
18009
|
use MooseX::Params::Validate qw( validated_list ); |
|
39
|
|
|
|
|
19510973
|
|
|
39
|
|
|
|
|
282
|
|
12
|
39
|
|
|
39
|
|
28870
|
use Parallel::ForkManager; |
|
39
|
|
|
|
|
2336901
|
|
|
39
|
|
|
|
|
1658
|
|
13
|
39
|
|
|
39
|
|
17737
|
use Stepford::Error; |
|
39
|
|
|
|
|
154
|
|
|
39
|
|
|
|
|
1385
|
|
14
|
39
|
|
|
39
|
|
15956
|
use Stepford::GraphBuilder; |
|
39
|
|
|
|
|
166
|
|
|
39
|
|
|
|
|
2475
|
|
15
|
39
|
|
|
|
|
365
|
use Stepford::Types qw( |
16
|
|
|
|
|
|
|
ArrayOfClassPrefixes ArrayOfSteps Bool ClassName |
17
|
|
|
|
|
|
|
HashRef Logger Maybe PositiveInt Step |
18
|
39
|
|
|
39
|
|
345
|
); |
|
39
|
|
|
|
|
90
|
|
19
|
39
|
|
|
39
|
|
389461
|
use Try::Tiny; |
|
39
|
|
|
|
|
118
|
|
|
39
|
|
|
|
|
2557
|
|
20
|
|
|
|
|
|
|
|
21
|
39
|
|
|
39
|
|
231
|
use Moose; |
|
39
|
|
|
|
|
82
|
|
|
39
|
|
|
|
|
342
|
|
22
|
39
|
|
|
39
|
|
250858
|
use MooseX::StrictConstructor; |
|
39
|
|
|
|
|
85
|
|
|
39
|
|
|
|
|
337
|
|
23
|
|
|
|
|
|
|
|
24
|
|
|
|
|
|
|
has _step_namespaces => ( |
25
|
|
|
|
|
|
|
traits => ['Array'], |
26
|
|
|
|
|
|
|
is => 'ro', |
27
|
|
|
|
|
|
|
isa => ArrayOfClassPrefixes, |
28
|
|
|
|
|
|
|
coerce => 1, |
29
|
|
|
|
|
|
|
required => 1, |
30
|
|
|
|
|
|
|
init_arg => 'step_namespaces', |
31
|
|
|
|
|
|
|
handles => { |
32
|
|
|
|
|
|
|
step_namespaces => 'elements', |
33
|
|
|
|
|
|
|
}, |
34
|
|
|
|
|
|
|
); |
35
|
|
|
|
|
|
|
|
36
|
|
|
|
|
|
|
has logger => ( |
37
|
|
|
|
|
|
|
is => 'ro', |
38
|
|
|
|
|
|
|
isa => Logger, |
39
|
|
|
|
|
|
|
lazy => 1, |
40
|
|
|
|
|
|
|
builder => '_build_logger', |
41
|
|
|
|
|
|
|
); |
42
|
|
|
|
|
|
|
|
43
|
|
|
|
|
|
|
has jobs => ( |
44
|
|
|
|
|
|
|
is => 'ro', |
45
|
|
|
|
|
|
|
isa => PositiveInt, |
46
|
|
|
|
|
|
|
default => 1, |
47
|
|
|
|
|
|
|
); |
48
|
|
|
|
|
|
|
|
49
|
|
|
|
|
|
|
has _memory_stats => ( |
50
|
|
|
|
|
|
|
is => 'ro', |
51
|
|
|
|
|
|
|
isa => Maybe ['Memory::Stats'], |
52
|
|
|
|
|
|
|
default => sub { |
53
|
|
|
|
|
|
|
return try { |
54
|
|
|
|
|
|
|
require Memory::Stats; |
55
|
|
|
|
|
|
|
my $s = Memory::Stats->new; |
56
|
|
|
|
|
|
|
$s->start; |
57
|
|
|
|
|
|
|
$s; |
58
|
|
|
|
|
|
|
} |
59
|
|
|
|
|
|
|
catch { |
60
|
|
|
|
|
|
|
undef; |
61
|
|
|
|
|
|
|
}; |
62
|
|
|
|
|
|
|
}, |
63
|
|
|
|
|
|
|
); |
64
|
|
|
|
|
|
|
|
65
|
|
|
|
|
|
|
has _step_classes => ( |
66
|
|
|
|
|
|
|
is => 'ro', |
67
|
|
|
|
|
|
|
isa => ArrayOfSteps, |
68
|
|
|
|
|
|
|
init_arg => undef, |
69
|
|
|
|
|
|
|
lazy => 1, |
70
|
|
|
|
|
|
|
builder => '_build_step_classes', |
71
|
|
|
|
|
|
|
); |
72
|
|
|
|
|
|
|
|
73
|
|
|
|
|
|
|
# We want to preload all the step classes so that the final_steps passed to |
74
|
|
|
|
|
|
|
# run are recognized as valid classes. |
75
|
|
|
|
|
|
|
sub BUILD { |
76
|
145
|
|
|
145
|
0
|
530
|
my $self = shift; |
77
|
|
|
|
|
|
|
|
78
|
145
|
|
|
|
|
4697
|
$self->_step_classes; |
79
|
|
|
|
|
|
|
|
80
|
144
|
|
|
|
|
3261
|
return; |
81
|
|
|
|
|
|
|
} |
82
|
|
|
|
|
|
|
|
83
|
|
|
|
|
|
|
sub run { |
84
|
146
|
|
|
146
|
1
|
1052945
|
my $self = shift; |
85
|
146
|
|
|
|
|
922
|
my ( $final_steps, $config, $force_step_execution, $dry_run ) |
86
|
|
|
|
|
|
|
= validated_list( |
87
|
|
|
|
|
|
|
\@_, |
88
|
|
|
|
|
|
|
final_steps => { |
89
|
|
|
|
|
|
|
isa => ArrayOfSteps, |
90
|
|
|
|
|
|
|
coerce => 1, |
91
|
|
|
|
|
|
|
}, |
92
|
|
|
|
|
|
|
config => { |
93
|
|
|
|
|
|
|
isa => HashRef, |
94
|
|
|
|
|
|
|
default => {}, |
95
|
|
|
|
|
|
|
}, |
96
|
|
|
|
|
|
|
force_step_execution => { |
97
|
|
|
|
|
|
|
isa => Bool, |
98
|
|
|
|
|
|
|
default => 0, |
99
|
|
|
|
|
|
|
}, |
100
|
|
|
|
|
|
|
dry_run => { |
101
|
|
|
|
|
|
|
isa => Bool, |
102
|
|
|
|
|
|
|
default => 'none', |
103
|
|
|
|
|
|
|
}, |
104
|
|
|
|
|
|
|
); |
105
|
|
|
|
|
|
|
|
106
|
146
|
|
|
|
|
191045
|
my $root_graph |
107
|
|
|
|
|
|
|
= $self->_make_root_graph_builder( $final_steps, $config )->graph; |
108
|
|
|
|
|
|
|
|
109
|
143
|
100
|
|
|
|
4458
|
if ( $dry_run ne 'none' ) { |
|
|
100
|
|
|
|
|
|
110
|
|
|
|
|
|
|
## no critic (InputOutput::RequireCheckedSyscalls) |
111
|
1
|
|
|
|
|
36
|
print $root_graph->as_string($dry_run); |
112
|
|
|
|
|
|
|
} |
113
|
|
|
|
|
|
|
elsif ( $self->jobs > 1 ) { |
114
|
77
|
|
|
|
|
410
|
$self->_run_parallel( $root_graph, $force_step_execution ); |
115
|
|
|
|
|
|
|
} |
116
|
|
|
|
|
|
|
else { |
117
|
65
|
|
|
|
|
254
|
$self->_run_sequential( $root_graph, $force_step_execution ); |
118
|
|
|
|
|
|
|
} |
119
|
|
|
|
|
|
|
|
120
|
108
|
|
|
|
|
6872
|
return; |
121
|
|
|
|
|
|
|
} |
122
|
|
|
|
|
|
|
|
123
|
|
|
|
|
|
|
sub _run_sequential { |
124
|
65
|
|
|
65
|
|
128
|
my $self = shift; |
125
|
65
|
|
|
|
|
97
|
my $root_graph = shift; |
126
|
65
|
|
|
|
|
95
|
my $force_step_execution = shift; |
127
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
$root_graph->traverse( |
129
|
|
|
|
|
|
|
sub { |
130
|
314
|
|
|
314
|
|
431
|
my $graph = shift; |
131
|
|
|
|
|
|
|
|
132
|
|
|
|
|
|
|
return |
133
|
314
|
100
|
|
|
|
811
|
unless $self->_should_run_step( |
134
|
|
|
|
|
|
|
$graph, |
135
|
|
|
|
|
|
|
$force_step_execution |
136
|
|
|
|
|
|
|
); |
137
|
|
|
|
|
|
|
|
138
|
181
|
|
|
|
|
571
|
$self->_run_step_in_process($graph); |
139
|
181
|
|
|
|
|
598
|
return; |
140
|
|
|
|
|
|
|
} |
141
|
65
|
|
|
|
|
449
|
); |
142
|
|
|
|
|
|
|
} |
143
|
|
|
|
|
|
|
|
144
|
|
|
|
|
|
|
sub _run_parallel { |
145
|
77
|
|
|
77
|
|
187
|
my $self = shift; |
146
|
77
|
|
|
|
|
169
|
my $root_graph = shift; |
147
|
77
|
|
|
|
|
135
|
my $force_step_execution = shift; |
148
|
|
|
|
|
|
|
|
149
|
77
|
|
|
|
|
1814
|
my $pm = Parallel::ForkManager->new( $self->jobs ); |
150
|
|
|
|
|
|
|
|
151
|
77
|
|
|
|
|
130053
|
my %graphs; |
152
|
77
|
|
|
|
|
165
|
my $steps_finished_since_last_iteration = 0; |
153
|
|
|
|
|
|
|
|
154
|
|
|
|
|
|
|
$pm->run_on_finish( |
155
|
|
|
|
|
|
|
sub { |
156
|
167
|
|
|
167
|
|
279474068
|
my ( $pid, $exit_code, $step_name, $signal, $message ) |
157
|
|
|
|
|
|
|
= @_[ 0 .. 3, 5 ]; |
158
|
|
|
|
|
|
|
|
159
|
167
|
100
|
|
|
|
2408
|
if ($exit_code) { |
|
|
100
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
160
|
1
|
|
|
|
|
37
|
$pm->wait_all_children; |
161
|
1
|
|
|
|
|
121
|
die "Child process $pid failed while running step $step_name" |
162
|
|
|
|
|
|
|
. " (exited with code $exit_code)"; |
163
|
|
|
|
|
|
|
} |
164
|
|
|
|
|
|
|
elsif ( !$message ) { |
165
|
3
|
|
|
|
|
93
|
$pm->wait_all_children; |
166
|
3
|
|
|
|
|
69
|
my $error = "Child process $pid did not send back any data"; |
167
|
3
|
|
|
|
|
42
|
$error .= " while running step $step_name"; |
168
|
3
|
50
|
|
|
|
81
|
$error .= " (exited because of signal $signal)" if $signal; |
169
|
3
|
|
|
|
|
333
|
die $error; |
170
|
|
|
|
|
|
|
} |
171
|
|
|
|
|
|
|
elsif ( $message->{error} ) { |
172
|
4
|
|
|
|
|
132
|
$pm->wait_all_children; |
173
|
4
|
|
|
|
|
500
|
die "Child process $pid died" |
174
|
|
|
|
|
|
|
. " while running step $step_name" |
175
|
|
|
|
|
|
|
. " with error:\n$message->{error}"; |
176
|
|
|
|
|
|
|
} |
177
|
|
|
|
|
|
|
else { |
178
|
159
|
|
|
|
|
912
|
my $graph = $graphs{$pid}; |
179
|
159
|
50
|
|
|
|
3008
|
die "Could not find step graph for $pid" |
180
|
|
|
|
|
|
|
unless defined $graph; |
181
|
|
|
|
|
|
|
|
182
|
159
|
|
|
|
|
14456
|
$graph->set_last_run_time( $message->{last_run_time} ); |
183
|
|
|
|
|
|
|
$graph->set_step_productions_as_hashref( |
184
|
159
|
|
|
|
|
8371
|
$message->{productions} ); |
185
|
159
|
|
|
|
|
6943
|
$graph->set_is_being_processed(0); |
186
|
159
|
|
|
|
|
7918
|
$graph->set_has_been_processed(1); |
187
|
159
|
|
|
|
|
1490
|
$steps_finished_since_last_iteration++; |
188
|
|
|
|
|
|
|
} |
189
|
|
|
|
|
|
|
} |
190
|
77
|
|
|
|
|
1068
|
); |
191
|
|
|
|
|
|
|
|
192
|
77
|
|
|
|
|
3445
|
while ( !$root_graph->has_been_processed ) { |
193
|
|
|
|
|
|
|
$root_graph->traverse( |
194
|
|
|
|
|
|
|
sub { |
195
|
901
|
|
|
901
|
|
2328
|
my $graph = shift; |
196
|
|
|
|
|
|
|
|
197
|
901
|
|
|
|
|
27571
|
my $class = $graph->step_class; |
198
|
|
|
|
|
|
|
|
199
|
|
|
|
|
|
|
return |
200
|
901
|
100
|
|
|
|
5832
|
unless $self->_should_run_step( |
201
|
|
|
|
|
|
|
$graph, |
202
|
|
|
|
|
|
|
$force_step_execution |
203
|
|
|
|
|
|
|
); |
204
|
|
|
|
|
|
|
|
205
|
201
|
100
|
|
|
|
976
|
unless ( $graph->is_serializable ) { |
206
|
4
|
|
|
|
|
1294
|
$self->_run_step_in_process($graph); |
207
|
4
|
|
|
|
|
12
|
$steps_finished_since_last_iteration++; |
208
|
4
|
|
|
|
|
14
|
return; |
209
|
|
|
|
|
|
|
} |
210
|
|
|
|
|
|
|
|
211
|
197
|
100
|
|
|
|
21099
|
if ( my $pid = $pm->start($class) ) { |
212
|
170
|
|
|
|
|
404519
|
$graph->set_is_being_processed(1); |
213
|
|
|
|
|
|
|
|
214
|
170
|
|
|
|
|
3477
|
$graphs{$pid} = $graph; |
215
|
|
|
|
|
|
|
|
216
|
170
|
|
|
|
|
10909
|
$self->logger->debug( |
217
|
|
|
|
|
|
|
"Forked child to run $class - pid $pid"); |
218
|
170
|
|
|
|
|
28231
|
return; |
219
|
|
|
|
|
|
|
} |
220
|
|
|
|
|
|
|
|
221
|
|
|
|
|
|
|
# child |
222
|
27
|
|
|
|
|
122174
|
my $error; |
223
|
|
|
|
|
|
|
try { |
224
|
27
|
|
|
|
|
7349
|
$self->_run_step_in_process($graph); |
225
|
|
|
|
|
|
|
} |
226
|
|
|
|
|
|
|
catch { |
227
|
1
|
|
|
|
|
58
|
$error = $_; |
228
|
27
|
|
|
|
|
2692
|
}; |
229
|
|
|
|
|
|
|
|
230
|
26
|
100
|
|
|
|
3213
|
my %message |
231
|
|
|
|
|
|
|
= defined $error |
232
|
|
|
|
|
|
|
? ( error => $error . q{} ) |
233
|
|
|
|
|
|
|
: ( |
234
|
|
|
|
|
|
|
last_run_time => scalar $graph->last_run_time, |
235
|
|
|
|
|
|
|
productions => $graph->step_productions_as_hashref, |
236
|
|
|
|
|
|
|
); |
237
|
|
|
|
|
|
|
|
238
|
26
|
|
|
|
|
1517
|
$pm->finish( 0, \%message ); |
239
|
|
|
|
|
|
|
} |
240
|
195
|
|
|
|
|
4018
|
); |
241
|
|
|
|
|
|
|
|
242
|
168
|
|
|
|
|
8980
|
$pm->reap_finished_children; |
243
|
|
|
|
|
|
|
|
244
|
|
|
|
|
|
|
# If none of the child processes have finished, there is no point in |
245
|
|
|
|
|
|
|
# wasting a bunch of CPU checking for steps that can be worked on. We |
246
|
|
|
|
|
|
|
# wait until at least one step has finished. |
247
|
168
|
100
|
|
|
|
15125
|
$pm->wait_one_child unless $steps_finished_since_last_iteration; |
248
|
|
|
|
|
|
|
|
249
|
160
|
|
|
|
|
7686
|
$steps_finished_since_last_iteration = 0; |
250
|
|
|
|
|
|
|
} |
251
|
|
|
|
|
|
|
|
252
|
42
|
|
|
|
|
1847
|
$self->logger->debug('Waiting for children'); |
253
|
42
|
|
|
|
|
3046
|
$pm->wait_all_children; |
254
|
|
|
|
|
|
|
} |
255
|
|
|
|
|
|
|
|
256
|
|
|
|
|
|
|
sub _should_run_step { |
257
|
1215
|
|
|
1215
|
|
2650
|
my $self = shift; |
258
|
1215
|
|
|
|
|
2199
|
my $graph = shift; |
259
|
1215
|
|
|
|
|
1723
|
my $force_step_execution = shift; |
260
|
|
|
|
|
|
|
|
261
|
1215
|
100
|
|
|
|
5151
|
return 0 unless $graph->can_run_step; |
262
|
|
|
|
|
|
|
|
263
|
619
|
100
|
|
|
|
3148
|
if ( $graph->step_is_up_to_date($force_step_execution) ) { |
264
|
237
|
|
|
|
|
6007
|
$self->logger->info( 'Skipping ' . $graph->step_class ); |
265
|
237
|
|
|
|
|
12856
|
$graph->set_has_been_processed(1); |
266
|
237
|
|
|
|
|
878
|
return 0; |
267
|
|
|
|
|
|
|
} |
268
|
|
|
|
|
|
|
|
269
|
382
|
|
|
|
|
1273
|
return 1; |
270
|
|
|
|
|
|
|
} |
271
|
|
|
|
|
|
|
|
272
|
|
|
|
|
|
|
sub _run_step_in_process { |
273
|
212
|
|
|
212
|
|
3057
|
my $self = shift; |
274
|
212
|
|
|
|
|
493
|
my $graph = shift; |
275
|
|
|
|
|
|
|
|
276
|
212
|
|
|
|
|
9491
|
$self->_log_memory_usage( 'Before running ' . $graph->step_class ); |
277
|
212
|
|
|
|
|
560448
|
$graph->run_step; |
278
|
210
|
|
|
|
|
5125
|
$self->_log_memory_usage( 'After running ' . $graph->step_class ); |
279
|
|
|
|
|
|
|
|
280
|
210
|
|
|
|
|
504640
|
return; |
281
|
|
|
|
|
|
|
} |
282
|
|
|
|
|
|
|
|
283
|
|
|
|
|
|
|
sub _make_root_graph_builder { |
284
|
151
|
|
|
151
|
|
2072
|
my $self = shift; |
285
|
151
|
|
|
|
|
320
|
my $final_steps = shift; |
286
|
151
|
|
|
|
|
230
|
my $config = shift; |
287
|
|
|
|
|
|
|
|
288
|
151
|
|
|
|
|
4633
|
return Stepford::GraphBuilder->new( |
289
|
|
|
|
|
|
|
config => $config, |
290
|
|
|
|
|
|
|
step_classes => $self->_step_classes, |
291
|
|
|
|
|
|
|
final_steps => $final_steps, |
292
|
|
|
|
|
|
|
logger => $self->logger, |
293
|
|
|
|
|
|
|
); |
294
|
|
|
|
|
|
|
} |
295
|
|
|
|
|
|
|
|
296
|
|
|
|
|
|
|
sub _build_step_classes { |
297
|
145
|
|
|
145
|
|
332
|
my $self = shift; |
298
|
|
|
|
|
|
|
|
299
|
|
|
|
|
|
|
# Module::Pluggable does not document whether it returns class names in |
300
|
|
|
|
|
|
|
# any specific order. |
301
|
145
|
|
|
|
|
606
|
my $sorter = $self->_step_class_sorter; |
302
|
|
|
|
|
|
|
|
303
|
145
|
|
|
|
|
334
|
my @classes; |
304
|
|
|
|
|
|
|
|
305
|
145
|
|
|
|
|
5181
|
for my $class ( |
306
|
653
|
|
|
|
|
1092998
|
sort { $sorter->() } Module::Pluggable::Object->new( |
307
|
|
|
|
|
|
|
search_path => [ $self->step_namespaces ], |
308
|
|
|
|
|
|
|
require => 1, |
309
|
|
|
|
|
|
|
)->plugins |
310
|
|
|
|
|
|
|
) { |
311
|
|
|
|
|
|
|
|
312
|
|
|
|
|
|
|
# We need to skip roles |
313
|
602
|
100
|
|
|
|
7364
|
next unless $class->isa('Moose::Object'); |
314
|
|
|
|
|
|
|
|
315
|
601
|
100
|
|
|
|
3637
|
unless ( $class->does('Stepford::Role::Step') ) { |
316
|
1
|
|
|
|
|
268
|
Stepford::Error->throw( message => |
317
|
|
|
|
|
|
|
qq{Found a class which doesn't do the Stepford::Role::Step role: $class} |
318
|
|
|
|
|
|
|
); |
319
|
|
|
|
|
|
|
} |
320
|
|
|
|
|
|
|
|
321
|
600
|
|
|
|
|
107421
|
$self->logger->debug("Found step class $class"); |
322
|
600
|
|
|
|
|
20796
|
push @classes, $class; |
323
|
|
|
|
|
|
|
} |
324
|
|
|
|
|
|
|
|
325
|
144
|
|
|
|
|
6108
|
return \@classes; |
326
|
|
|
|
|
|
|
} |
327
|
|
|
|
|
|
|
|
328
|
|
|
|
|
|
|
sub _step_class_sorter { |
329
|
145
|
|
|
145
|
|
319
|
my $self = shift; |
330
|
|
|
|
|
|
|
|
331
|
145
|
|
|
|
|
325
|
my $x = 0; |
332
|
145
|
|
|
|
|
7096
|
my @namespaces = $self->step_namespaces; |
333
|
145
|
|
|
|
|
333
|
my %order = map { $_ => $x++ } @namespaces; |
|
146
|
|
|
|
|
983
|
|
334
|
|
|
|
|
|
|
|
335
|
|
|
|
|
|
|
return sub { |
336
|
653
|
|
|
653
|
|
3281
|
my $a_prefix = first { $a =~ /^\Q$_/ } @namespaces; |
|
664
|
|
|
|
|
3808
|
|
337
|
653
|
|
|
|
|
2834
|
my $b_prefix = first { $b =~ /^\Q$_/ } @namespaces; |
|
662
|
|
|
|
|
2798
|
|
338
|
|
|
|
|
|
|
|
339
|
653
|
|
66
|
|
|
4360
|
return ( $order{$a_prefix} <=> $order{$b_prefix} or $a cmp $b ); |
340
|
145
|
|
|
|
|
1597
|
}; |
341
|
|
|
|
|
|
|
} |
342
|
|
|
|
|
|
|
|
343
|
|
|
|
|
|
|
sub _build_logger { |
344
|
130
|
|
|
130
|
|
292
|
my $self = shift; |
345
|
|
|
|
|
|
|
|
346
|
130
|
|
|
|
|
17098
|
require Log::Dispatch; |
347
|
130
|
|
|
|
|
5697879
|
require Log::Dispatch::Null; |
348
|
130
|
|
|
|
|
1019021
|
return Log::Dispatch->new( |
349
|
|
|
|
|
|
|
outputs => [ [ Null => min_level => 'emerg' ] ] ); |
350
|
|
|
|
|
|
|
} |
351
|
|
|
|
|
|
|
|
352
|
|
|
|
|
|
|
sub _log_memory_usage { |
353
|
422
|
|
|
422
|
|
1792
|
my $self = shift; |
354
|
422
|
|
|
|
|
777
|
my $checkpoint = shift; |
355
|
|
|
|
|
|
|
|
356
|
422
|
50
|
|
|
|
11888
|
return unless $self->_memory_stats; |
357
|
|
|
|
|
|
|
|
358
|
422
|
|
|
|
|
9713
|
$self->_memory_stats->checkpoint($checkpoint); |
359
|
|
|
|
|
|
|
|
360
|
|
|
|
|
|
|
# There's no way to get the total use so far without calling stop(), which |
361
|
|
|
|
|
|
|
# is quite annoying. See |
362
|
|
|
|
|
|
|
# https://github.com/celogeek/perl-memory-stats/issues/3. |
363
|
|
|
|
|
|
|
|
364
|
|
|
|
|
|
|
## no critic (Subroutines::ProtectPrivateSubs) |
365
|
422
|
|
|
|
|
1193161
|
$self->logger->info( |
366
|
|
|
|
|
|
|
sprintf( |
367
|
|
|
|
|
|
|
'Total memory use since Stepford started is %d', |
368
|
|
|
|
|
|
|
$self->_memory_stats->_memory_usage->[-1][1] |
369
|
|
|
|
|
|
|
- $self->_memory_stats->_memory_usage->[0][1] |
370
|
|
|
|
|
|
|
) |
371
|
|
|
|
|
|
|
); |
372
|
422
|
|
|
|
|
25941
|
$self->logger->info( |
373
|
|
|
|
|
|
|
sprintf( |
374
|
|
|
|
|
|
|
'Memory since last checked is %d', |
375
|
|
|
|
|
|
|
$self->_memory_stats->delta_usage |
376
|
|
|
|
|
|
|
) |
377
|
|
|
|
|
|
|
); |
378
|
|
|
|
|
|
|
} |
379
|
|
|
|
|
|
|
|
380
|
|
|
|
|
|
|
__PACKAGE__->meta->make_immutable; |
381
|
|
|
|
|
|
|
|
382
|
|
|
|
|
|
|
1; |
383
|
|
|
|
|
|
|
|
384
|
|
|
|
|
|
|
# ABSTRACT: Takes a set of steps and figures out what order to run them in |
385
|
|
|
|
|
|
|
|
386
|
|
|
|
|
|
|
__END__ |
387
|
|
|
|
|
|
|
|
388
|
|
|
|
|
|
|
=pod |
389
|
|
|
|
|
|
|
|
390
|
|
|
|
|
|
|
=encoding UTF-8 |
391
|
|
|
|
|
|
|
|
392
|
|
|
|
|
|
|
=head1 NAME |
393
|
|
|
|
|
|
|
|
394
|
|
|
|
|
|
|
Stepford::Runner - Takes a set of steps and figures out what order to run them in |
395
|
|
|
|
|
|
|
|
396
|
|
|
|
|
|
|
=head1 VERSION |
397
|
|
|
|
|
|
|
|
398
|
|
|
|
|
|
|
version 0.006000 |
399
|
|
|
|
|
|
|
|
400
|
|
|
|
|
|
|
=head1 SYNOPSIS |
401
|
|
|
|
|
|
|
|
402
|
|
|
|
|
|
|
use Stepford::Runner; |
403
|
|
|
|
|
|
|
|
404
|
|
|
|
|
|
|
my $runner = Stepford::Runner->new( |
405
|
|
|
|
|
|
|
step_namespaces => 'My::Step', |
406
|
|
|
|
|
|
|
); |
407
|
|
|
|
|
|
|
|
408
|
|
|
|
|
|
|
$runner->run( |
409
|
|
|
|
|
|
|
final_steps => [ |
410
|
|
|
|
|
|
|
'My::Step::DeployCatDatabase', |
411
|
|
|
|
|
|
|
'My::Step::DeployDogDatabase', |
412
|
|
|
|
|
|
|
], |
413
|
|
|
|
|
|
|
config => {...}, |
414
|
|
|
|
|
|
|
); |
415
|
|
|
|
|
|
|
|
416
|
|
|
|
|
|
|
=head1 DESCRIPTION |
417
|
|
|
|
|
|
|
|
418
|
|
|
|
|
|
|
This class takes a set of objects which do the L<Stepford::Role::Step> role |
419
|
|
|
|
|
|
|
and determines what order they should be run so as to get to one or more final |
420
|
|
|
|
|
|
|
steps. |
421
|
|
|
|
|
|
|
|
422
|
|
|
|
|
|
|
Steps which are up to date are skipped during the run, so no unnecessary work |
423
|
|
|
|
|
|
|
is done. |
424
|
|
|
|
|
|
|
|
425
|
|
|
|
|
|
|
=for Pod::Coverage BUILD add_step |
426
|
|
|
|
|
|
|
|
427
|
|
|
|
|
|
|
=head1 METHODS |
428
|
|
|
|
|
|
|
|
429
|
|
|
|
|
|
|
This class provides the following methods: |
430
|
|
|
|
|
|
|
|
431
|
|
|
|
|
|
|
=head2 Stepford::Runner->new(...) |
432
|
|
|
|
|
|
|
|
433
|
|
|
|
|
|
|
This method returns a new runner object. It accepts the following arguments: |
434
|
|
|
|
|
|
|
|
435
|
|
|
|
|
|
|
=over 4 |
436
|
|
|
|
|
|
|
|
437
|
|
|
|
|
|
|
=item * step_namespaces |
438
|
|
|
|
|
|
|
|
439
|
|
|
|
|
|
|
This argument is required. |
440
|
|
|
|
|
|
|
|
441
|
|
|
|
|
|
|
This can either be a string or an array reference of strings. Each string |
442
|
|
|
|
|
|
|
should contain a namespace which contains step classes. |
443
|
|
|
|
|
|
|
|
444
|
|
|
|
|
|
|
For example, if your steps are named C<My::Step::CreateFoo>, |
445
|
|
|
|
|
|
|
C<My::Step::MergeFooAndBar>, and C<My::Step::DeployMergedFooAndBar>, the |
446
|
|
|
|
|
|
|
namespace you'd provide is C<'My::Step'>. |
447
|
|
|
|
|
|
|
|
448
|
|
|
|
|
|
|
The order of the step namespaces I<is> relevant. If more than one step has a |
449
|
|
|
|
|
|
|
production of the same name, then the first step "wins". Stepford sorts step |
450
|
|
|
|
|
|
|
class names based on the order of the namespaces provided to the constructor, |
451
|
|
|
|
|
|
|
and then the full name of the class. You can take advantage of this feature to |
452
|
|
|
|
|
|
|
provide different steps in different environments (for example, for testing). |
453
|
|
|
|
|
|
|
|
454
|
|
|
|
|
|
|
The runner object assumes that every B<class> it finds in a step namespace is |
455
|
|
|
|
|
|
|
a step. Specifically, if it finds a package that is a subclass of |
456
|
|
|
|
|
|
|
L<Moose::Object>, then it throws an error if that package does not also |
457
|
|
|
|
|
|
|
consume the L<Stepford::Role::Step> role. |
458
|
|
|
|
|
|
|
|
459
|
|
|
|
|
|
|
This means you can have utility packages and roles in a step namespace, but |
460
|
|
|
|
|
|
|
not Moose objects which aren't steps. |
461
|
|
|
|
|
|
|
|
462
|
|
|
|
|
|
|
=item * jobs |
463
|
|
|
|
|
|
|
|
464
|
|
|
|
|
|
|
This argument defaults to 1. |
465
|
|
|
|
|
|
|
|
466
|
|
|
|
|
|
|
The number of jobs to run at a time. By default, all steps are run |
467
|
|
|
|
|
|
|
sequentially. However, if you set this to a value greater than 1 then the |
468
|
|
|
|
|
|
|
runner will run steps in parallel, up to the value you set. |
469
|
|
|
|
|
|
|
|
470
|
|
|
|
|
|
|
=item * logger |
471
|
|
|
|
|
|
|
|
472
|
|
|
|
|
|
|
This argument is optional. |
473
|
|
|
|
|
|
|
|
474
|
|
|
|
|
|
|
This should be an object that provides C<debug>, C<info>, C<notice>, |
475
|
|
|
|
|
|
|
C<warning>, and C<error> methods. |
476
|
|
|
|
|
|
|
|
477
|
|
|
|
|
|
|
This object will receive log messages from the runner and (possibly your |
478
|
|
|
|
|
|
|
steps). |
479
|
|
|
|
|
|
|
|
480
|
|
|
|
|
|
|
If this is not provided, Stepford will create a L<Log::Dispatch> object with a |
481
|
|
|
|
|
|
|
single L<Log::Dispatch::Null> output (which silently eats all the logging |
482
|
|
|
|
|
|
|
messages). |
483
|
|
|
|
|
|
|
|
484
|
|
|
|
|
|
|
Note that if you I<do> provide a logger object of your own, Stepford will not |
485
|
|
|
|
|
|
|
load L<Log::Dispatch> into memory. |
486
|
|
|
|
|
|
|
|
487
|
|
|
|
|
|
|
=back |
488
|
|
|
|
|
|
|
|
489
|
|
|
|
|
|
|
=head2 $runner->run |
490
|
|
|
|
|
|
|
|
491
|
|
|
|
|
|
|
When this method is called, the runner comes up with a plan of the steps |
492
|
|
|
|
|
|
|
needed to get to the requested final steps. |
493
|
|
|
|
|
|
|
|
494
|
|
|
|
|
|
|
When this method is called, we check for circular dependencies among the steps |
495
|
|
|
|
|
|
|
and will throw a L<Stepford::Error> exception if it finds one. We also check |
496
|
|
|
|
|
|
|
for unsatisfied dependencies for steps in the plan. Finally, we check to make |
497
|
|
|
|
|
|
|
sure that no step provides its own dependencies as productions. |
498
|
|
|
|
|
|
|
|
499
|
|
|
|
|
|
|
For each step, the runner checks if it is up to date compared to its |
500
|
|
|
|
|
|
|
dependencies (as determined by the C<< $step->last_run_time >> method. If the |
501
|
|
|
|
|
|
|
step is up to date, it is skipped, otherwise the runner calls C<< $step->run |
502
|
|
|
|
|
|
|
>> on the step. You can avoid this check and force all steps to be executed |
503
|
|
|
|
|
|
|
with the C< force_step_execution > parameter (documented below.) |
504
|
|
|
|
|
|
|
|
505
|
|
|
|
|
|
|
Note that the step objects are always I<constructed>, so you should avoid |
506
|
|
|
|
|
|
|
doing a lot of work in your constructor. Save that for the C<run> method. |
507
|
|
|
|
|
|
|
|
508
|
|
|
|
|
|
|
This method accepts the following parameters: |
509
|
|
|
|
|
|
|
|
510
|
|
|
|
|
|
|
=over 4 |
511
|
|
|
|
|
|
|
|
512
|
|
|
|
|
|
|
=item * final_steps |
513
|
|
|
|
|
|
|
|
514
|
|
|
|
|
|
|
This argument is required. |
515
|
|
|
|
|
|
|
|
516
|
|
|
|
|
|
|
This can either be a string or an array reference of strings. Each string |
517
|
|
|
|
|
|
|
should be a step's class name. These classes must do the |
518
|
|
|
|
|
|
|
L<Stepford::Role::Step> role. |
519
|
|
|
|
|
|
|
|
520
|
|
|
|
|
|
|
These are the final steps run when the C<< $runner->run >> method is |
521
|
|
|
|
|
|
|
called. |
522
|
|
|
|
|
|
|
|
523
|
|
|
|
|
|
|
=item * config |
524
|
|
|
|
|
|
|
|
525
|
|
|
|
|
|
|
This is an optional hash reference. For each step constructed, the runner |
526
|
|
|
|
|
|
|
looks at the attributes that the step accepts. If they match any of the keys |
527
|
|
|
|
|
|
|
in this hash reference, the key/value pair from this hash reference will be |
528
|
|
|
|
|
|
|
passed to the step constructor. This matching is done based on attribute |
529
|
|
|
|
|
|
|
C<init_arg> values. |
530
|
|
|
|
|
|
|
|
531
|
|
|
|
|
|
|
Note that values generated as productions from previous steps will override |
532
|
|
|
|
|
|
|
the corresponding key in the config hash reference. |
533
|
|
|
|
|
|
|
|
534
|
|
|
|
|
|
|
=item * force_step_execution |
535
|
|
|
|
|
|
|
|
536
|
|
|
|
|
|
|
This argument defaults to false. |
537
|
|
|
|
|
|
|
|
538
|
|
|
|
|
|
|
This controls if we should force all steps to be executed rather than checking |
539
|
|
|
|
|
|
|
which steps are up to date and do not need re-executing. Even with this set |
540
|
|
|
|
|
|
|
each step will only be executed once per run regardless of how many other |
541
|
|
|
|
|
|
|
steps depend on it during execution. |
542
|
|
|
|
|
|
|
|
543
|
|
|
|
|
|
|
=item * dry_run |
544
|
|
|
|
|
|
|
|
545
|
|
|
|
|
|
|
The argument is a boolean that defaults to false. When set to a true value, |
546
|
|
|
|
|
|
|
Stepford prints out the steps that need to be executed and exits without |
547
|
|
|
|
|
|
|
executing them. |
548
|
|
|
|
|
|
|
|
549
|
|
|
|
|
|
|
=back |
550
|
|
|
|
|
|
|
|
551
|
|
|
|
|
|
|
=head2 $runner->step_namespaces |
552
|
|
|
|
|
|
|
|
553
|
|
|
|
|
|
|
This method returns the step namespaces passed to the constructor as a list |
554
|
|
|
|
|
|
|
(not an arrayref). |
555
|
|
|
|
|
|
|
|
556
|
|
|
|
|
|
|
=head2 $runner->logger |
557
|
|
|
|
|
|
|
|
558
|
|
|
|
|
|
|
This method returns the C<logger> used by the runner, either what you passed |
559
|
|
|
|
|
|
|
to the constructor or a default. |
560
|
|
|
|
|
|
|
|
561
|
|
|
|
|
|
|
=head1 PARALLEL RUNS AND SERIALIZATION |
562
|
|
|
|
|
|
|
|
563
|
|
|
|
|
|
|
When running steps in parallel, the results of a step (its productions) are |
564
|
|
|
|
|
|
|
sent from a child process to the parent by serializing them. This means that |
565
|
|
|
|
|
|
|
productions which can't be serialized (like a filehandle or L<DBI> handle) |
566
|
|
|
|
|
|
|
will cause the runner to die when it tries to serialize their productions. |
567
|
|
|
|
|
|
|
|
568
|
|
|
|
|
|
|
You can force a step class to be run in the same process as the runner itself |
569
|
|
|
|
|
|
|
by having that step consume the L<Stepford::Role::Step::Unserializable> |
570
|
|
|
|
|
|
|
role. Note that the runner may still fork I<after> a production has been |
571
|
|
|
|
|
|
|
generated, so the values returned for a production must be able to survive a |
572
|
|
|
|
|
|
|
fork, even if they cannot be serialized. |
573
|
|
|
|
|
|
|
|
574
|
|
|
|
|
|
|
You can also work around this by changing the production entirely. For |
575
|
|
|
|
|
|
|
example, instead of passing a DBI handle you could pass a data structure with |
576
|
|
|
|
|
|
|
a DSN, username, password, and connection options. |
577
|
|
|
|
|
|
|
|
578
|
|
|
|
|
|
|
=head1 SUPPORT |
579
|
|
|
|
|
|
|
|
580
|
|
|
|
|
|
|
Bugs may be submitted through L<https://github.com/maxmind/Stepford/issues>. |
581
|
|
|
|
|
|
|
|
582
|
|
|
|
|
|
|
=head1 AUTHOR |
583
|
|
|
|
|
|
|
|
584
|
|
|
|
|
|
|
Dave Rolsky <drolsky@maxmind.com> |
585
|
|
|
|
|
|
|
|
586
|
|
|
|
|
|
|
=head1 COPYRIGHT AND LICENSE |
587
|
|
|
|
|
|
|
|
588
|
|
|
|
|
|
|
This software is copyright (c) 2014 - 2019 by MaxMind, Inc. |
589
|
|
|
|
|
|
|
|
590
|
|
|
|
|
|
|
This is free software; you can redistribute it and/or modify it under |
591
|
|
|
|
|
|
|
the same terms as the Perl 5 programming language system itself. |
592
|
|
|
|
|
|
|
|
593
|
|
|
|
|
|
|
=cut |