File Coverage

blib/lib/Devel/Git/MultiBisect/AllCommits.pm
Criterion Covered Total %
statement 23 74 31.0
branch 0 10 0.0
condition n/a
subroutine 8 12 66.6
pod 4 4 100.0
total 35 100 35.0


line stmt bran cond sub pod time code
1             package Devel::Git::MultiBisect::AllCommits;
2 4     4   2028 use v5.14.0;
  4         33  
3 4     4   21 use warnings;
  4         18  
  4         133  
4 4     4   1583 use parent( qw| Devel::Git::MultiBisect | );
  4         1118  
  4         21  
5 4     4   1791 use Devel::Git::MultiBisect::Opts qw( process_options );
  4         10  
  4         269  
6 4     4   27 use Carp;
  4         8  
  4         174  
7 4     4   20 use Cwd;
  4         7  
  4         164  
8 4     4   20 use File::Temp;
  4         6  
  4         227  
9 4     4   20 use List::Util qw(first sum);
  4         7  
  4         2410  
10              
11             our $VERSION = '0.20';
12             $VERSION = eval $VERSION;
13              
14             =head1 NAME
15              
16             Devel::Git::MultiBisect::AllCommits - Gather test output over an entire range of F commits
17              
18             =head1 SYNOPSIS
19              
20             use Devel::Git::MultiBisect::AllCommits;
21              
22             $self = Devel::Git::MultiBisect::AllCommits->new(\%parameters);
23              
24             $commit_range = $self->get_commits_range();
25              
26             $full_targets = $self->set_targets(\@target_args);
27              
28             $outputs = $self->run_test_files_on_one_commit($commit_range->[0]);
29              
30             $all_outputs = $self->run_test_files_on_all_commits();
31              
32             $rv = $self->get_digests_by_file_and_commit();
33              
34             $transitions = $self->examine_transitions();
35              
36             =head1 DESCRIPTION
37              
38             Given a Perl library or application kept in F for version control, it is
39             often useful to be able to compare the output collected from running one or
40             several test files over a range of F commits. If that range is sufficiently
41             large, a test may fail in B over that range.
42              
43             If that is the case, then simply asking, I<"When did this file start to
44             fail?"> is insufficient. We may want to capture the test output for each
45             commit, or, more usefully, may want to capture the test output only at those
46             commits where the output changed. F
47             provides methods for the first of those objectives. That is:
48              
49             =over 4
50              
51             =item *
52              
53             When you want to capture the test output for each commit in a specified range,
54             you can use this package, F.
55              
56             =item *
57              
58             When the number of commits in the specified range is large and you only need
59             the test output at those commits where the output materially changed, you can
60             use another package found in this library, F.
61              
62             =back
63              
64             =head1 METHODS
65              
66             This package inherits methods from F. Only methods unique to
67             F are documented here. See the documentation for
68             F for all other methods, including:
69              
70             get_commits_range()
71             set_targets()
72             run_test_files_on_one_commit()
73              
74             =head2 C
75              
76             =over 4
77              
78             =item * Purpose
79              
80             Constructor.
81              
82             =item * Arguments
83              
84             $self = Devel::Git::MultiBisect::AllCommits->new(\%params);
85              
86             Reference to a hash, typically the return value of
87             C.
88              
89             =item * Return Value
90              
91             Object of Devel::Git::MultiBisect child class.
92              
93             =back
94              
95             =cut
96              
97             sub new {
98 0     0 1   my ($class, $params) = @_;
99 0           my $data = $class->SUPER::new($params);
100              
101 0           delete $data->{probe};
102              
103 0           return bless $data, $class;
104             }
105              
106             =head2 C
107              
108             =over 4
109              
110             =item * Purpose
111              
112             Capture the output from a run of the selected test files at each specific git
113             checkout in the selected commit range.
114              
115             =item * Arguments
116              
117             $all_outputs = $self->run_test_files_on_all_commits();
118              
119             None; all data needed is already present in the object.
120              
121             =item * Return Value
122              
123             Array reference, each of whose elements is an array reference, each of whose
124             elements is a hash reference with the same four keys as in the return value
125             from C:
126              
127             commit
128             commit_short
129             file
130             file_stub
131             md5_hex
132              
133             Example:
134              
135             [
136             # Array where each element corresponds to a single git checkout
137              
138             [
139             # Array where each element corresponds to one of the selected test
140             # files (here, 2 test files were targetd)
141              
142             {
143             # Hash where each element correponds to the result of running a
144             # single test file at a single commit point
145              
146             commit => "2a2e54af709f17cc6186b42840549c46478b6467",
147             commit_short => "2a2e54a",
148             file => "/tmp/AHC_YkwUYg/2a2e54a.t_44_func_hashes_mult_unsorted_t.output.txt",
149             file_stub => "t_44_func_hashes_mult_unsorted_t",
150             md5_hex => "31b7c93474e15a16d702da31989ab565",
151             },
152             {
153             commit => "2a2e54af709f17cc6186b42840549c46478b6467",
154             commit_short => "2a2e54a",
155             file => "/tmp/AHC_YkwUYg/2a2e54a.t_45_func_hashes_alt_dual_sorted_t.output.txt",
156             file_stub => "t_45_func_hashes_alt_dual_sorted_t",
157             md5_hex => "6ee767b9d2838e4bbe83be0749b841c1",
158             },
159             ],
160             [
161             {
162             commit => "a624024294a56964eca53ec4617a58a138e91568",
163             commit_short => "a624024",
164             file => "/tmp/AHC_YkwUYg/a624024.t_44_func_hashes_mult_unsorted_t.output.txt",
165             file_stub => "t_44_func_hashes_mult_unsorted_t",
166             md5_hex => "31b7c93474e15a16d702da31989ab565",
167             },
168             {
169             commit => "a624024294a56964eca53ec4617a58a138e91568",
170             commit_short => "a624024",
171             file => "/tmp/AHC_YkwUYg/a624024.t_45_func_hashes_alt_dual_sorted_t.output.txt",
172             file_stub => "t_45_func_hashes_alt_dual_sorted_t",
173             md5_hex => "6ee767b9d2838e4bbe83be0749b841c1",
174             },
175             ],
176             # ...
177             [
178             {
179             commit => "d304a207329e6bd7e62354df4f561d9a7ce1c8c2",
180             commit_short => "d304a20",
181             file => "/tmp/AHC_YkwUYg/d304a20.t_44_func_hashes_mult_unsorted_t.output.txt",
182             file_stub => "t_44_func_hashes_mult_unsorted_t",
183             md5_hex => "31b7c93474e15a16d702da31989ab565",
184             },
185             {
186             commit => "d304a207329e6bd7e62354df4f561d9a7ce1c8c2",
187             commit_short => "d304a20",
188             file => "/tmp/AHC_YkwUYg/d304a20.t_45_func_hashes_alt_dual_sorted_t.output.txt",
189             file_stub => "t_45_func_hashes_alt_dual_sorted_t",
190             md5_hex => "6ee767b9d2838e4bbe83be0749b841c1",
191             },
192             ],
193             ]
194              
195             =item * Comment
196              
197             Note: If the number of commits in the commits range is large, this method
198             will take a long time to run. That time will be even longer if the
199             configuration and build times for each commit are large. For example, to run
200             one test over 160 commits from the Perl 5 core distribution might take 15
201             hours. YMMV. If either of these conditions holds, you are probably better
202             off using F.
203              
204             The implementation of this method is very much subject to change.
205              
206             =back
207              
208             =cut
209              
210             sub run_test_files_on_all_commits {
211 0     0 1   my $self = shift;
212 0           my $all_commits = $self->get_commits_range();
213 0           my @all_outputs;
214 0           my $start_time = time();
215 0           for my $commit (@{$all_commits}) {
  0            
216 0           my $outputs = $self->run_test_files_on_one_commit($commit);
217 0           push @all_outputs, $outputs;
218             }
219 0           my $end_time = time();
220             my %timings = (
221             elapsed => $end_time - $start_time,
222 0           runs => scalar(@{$all_commits}),
  0            
223             );
224 0 0         if ($timings{runs}) {
225 0           $timings{mean} = sprintf("%.02f" => $timings{elapsed} / $timings{runs}); }
226 0 0         if ($self->{verbose}) {
227 0           my $msg = "Ran $timings{runs} runs; elapsed: $timings{elapsed} sec";
228 0 0         if ($timings{runs}) {
229 0           $msg .= "; mean: $timings{mean} sec";
230             }
231 0           say $msg;
232             }
233 0           $self->{all_outputs} = [ @all_outputs ];
234 0           $self->{timings} = \%timings;
235 0           return \@all_outputs;
236             }
237              
238              
239             =head2 C
240              
241             =over 4
242              
243             =item * Purpose
244              
245             Present the same outcomes as C, but formatted
246             by target file, then commit.
247              
248             =item * Arguments
249              
250             $rv = $self->get_digests_by_file_and_commit();
251              
252             None; all data needed is already present in the object.
253              
254             =item * Return Value
255              
256             Reference to a hash keyed on the basename of the target file, modified to
257             substitute underscores for forward slashes and dots. The value of each
258             element in the hash is a reference to an array which, in turn, holds a list of
259             hash references, one per F commit. Each such hash has the following keys:
260              
261             commit
262             file
263             md5_hex
264              
265             Example:
266              
267             {
268             t_44_func_hashes_mult_unsorted_t => [
269             {
270             commit => "2a2e54af709f17cc6186b42840549c46478b6467",
271             file => "/tmp/Xhilc8ZSgS/2a2e54a.t_44_func_hashes_mult_unsorted_t.output.txt",
272             md5_hex => "31b7c93474e15a16d702da31989ab565",
273             },
274             {
275             commit => "a624024294a56964eca53ec4617a58a138e91568",
276             file => "/tmp/Xhilc8ZSgS/a624024.t_44_func_hashes_mult_unsorted_t.output.txt",
277             md5_hex => "31b7c93474e15a16d702da31989ab565",
278             },
279             # ...
280             {
281             commit => "d304a207329e6bd7e62354df4f561d9a7ce1c8c2",
282             file => "/tmp/Xhilc8ZSgS/d304a20.t_44_func_hashes_mult_unsorted_t.output.txt",
283             md5_hex => "31b7c93474e15a16d702da31989ab565",
284             },
285             ],
286             t_45_func_hashes_alt_dual_sorted_t => [
287             {
288             commit => "2a2e54af709f17cc6186b42840549c46478b6467",
289             file => "/tmp/Xhilc8ZSgS/2a2e54a.t_45_func_hashes_alt_dual_sorted_t.output.txt",
290             md5_hex => "6ee767b9d2838e4bbe83be0749b841c1",
291             },
292             {
293             commit => "a624024294a56964eca53ec4617a58a138e91568",
294             file => "/tmp/Xhilc8ZSgS/a624024.t_45_func_hashes_alt_dual_sorted_t.output.txt",
295             md5_hex => "6ee767b9d2838e4bbe83be0749b841c1",
296             },
297             # ...
298             {
299             commit => "d304a207329e6bd7e62354df4f561d9a7ce1c8c2",
300             file => "/tmp/Xhilc8ZSgS/d304a20.t_45_func_hashes_alt_dual_sorted_t.output.txt",
301             md5_hex => "6ee767b9d2838e4bbe83be0749b841c1",
302             },
303             ],
304             }
305              
306              
307             =item * Comment
308              
309             This method currently presumes that you have called
310             C. It will die otherwise.
311              
312             =back
313              
314             =cut
315              
316             sub get_digests_by_file_and_commit {
317 0     0 1   my $self = shift;
318 0 0         unless (exists $self->{all_outputs}) {
319 0           croak "You must call run_test_files_on_all_commits() before calling get_digests_by_file_and_commit()";
320             }
321 0           my $rv = {};
322 0           for my $commit (@{$self->{all_outputs}}) {
  0            
323 0           for my $target (@{$commit}) {
  0            
324 0           push @{$rv->{$target->{file_stub}}},
325             {
326             commit => $target->{commit},
327             file => $target->{file},
328             md5_hex => $target->{md5_hex},
329 0           };
330             }
331             }
332 0           return $rv;
333             }
334              
335             =head2 C
336              
337             =over 4
338              
339             =item * Purpose
340              
341             Determine whether a run of the same targeted test file run at two consecutive
342             commits produced the same or different output (as measured by string equality
343             or inequality of each commit's md5_hex value.
344              
345             =item * Arguments
346              
347             $hashref = $self->get_digests_by_file_and_commit();
348              
349             $transitions = $self->examine_transitions($hashref);
350              
351             Hash reference returned by C;
352              
353             =item * Return Value
354              
355             Reference to a hash keyed on the basename of the target file, modified to
356             substitute underscores for forward slashes and dots. The value of each
357             element in the hash is a reference to an array which, in turn, holds a list of
358             hash references, one per each pair of consecutive F commits. Each such hash
359             has the following keys:
360              
361             older
362             newer
363             compare
364              
365             The value for each of the C and C elements is a reference to a
366             hash with two elements:
367              
368             md5_hex
369             idx
370              
371             ... where C is the digest of the test output file and C is the
372             position (count starting at C<0>) of that element in the list of commits in
373             the commit range.
374              
375             Example:
376              
377             {
378             t_44_func_hashes_mult_unsorted_t => [
379             {
380             compare => "same",
381             newer => { md5_hex => "31b7c93474e15a16d702da31989ab565", idx => 1 },
382             older => { md5_hex => "31b7c93474e15a16d702da31989ab565", idx => 0 },
383             },
384             {
385             compare => "same",
386             newer => { md5_hex => "31b7c93474e15a16d702da31989ab565", idx => 2 },
387             older => { md5_hex => "31b7c93474e15a16d702da31989ab565", idx => 1 },
388             },
389             # ...
390             {
391             compare => "same",
392             newer => { md5_hex => "31b7c93474e15a16d702da31989ab565", idx => 9 },
393             older => { md5_hex => "31b7c93474e15a16d702da31989ab565", idx => 8 },
394             },
395             ],
396             t_45_func_hashes_alt_dual_sorted_t => [
397             {
398             compare => "same",
399             newer => { md5_hex => "6ee767b9d2838e4bbe83be0749b841c1", idx => 1 },
400             older => { md5_hex => "6ee767b9d2838e4bbe83be0749b841c1", idx => 0 },
401             },
402             {
403             compare => "same",
404             newer => { md5_hex => "6ee767b9d2838e4bbe83be0749b841c1", idx => 2 },
405             older => { md5_hex => "6ee767b9d2838e4bbe83be0749b841c1", idx => 1 },
406             },
407             {
408             compare => "same",
409             newer => { md5_hex => "6ee767b9d2838e4bbe83be0749b841c1", idx => 3 },
410             older => { md5_hex => "6ee767b9d2838e4bbe83be0749b841c1", idx => 2 },
411             },
412             # ...
413             {
414             compare => "same",
415             newer => { md5_hex => "6ee767b9d2838e4bbe83be0749b841c1", idx => 9 },
416             older => { md5_hex => "6ee767b9d2838e4bbe83be0749b841c1", idx => 8 },
417             },
418             ],
419             }
420              
421             =item * Comment
422              
423             This method currently may be called only after calling
424             C and will die otherwise.
425              
426             Since in this method we are concerned with the B in the test
427             output between a pair of commits, the second-level arrays returned by this
428             method will have one fewer element than the second-level arrays returned by
429             C.
430              
431             =back
432              
433             =cut
434              
435             sub examine_transitions {
436 0     0 1   my ($self, $rv) = @_;
437 0           my %transitions;
438 0           for my $k (sort keys %{$rv}) {
  0            
439 0           my @arr = @{$rv->{$k}};
  0            
440 0           for (my $i = 1; $i <= $#arr; $i++) {
441 0           my $older = $arr[$i-1]->{md5_hex};
442 0           my $newer = $arr[$i]->{md5_hex};
443 0 0         if ($older eq $newer) {
444 0           push @{$transitions{$k}}, {
  0            
445             older => { idx => $i-1, md5_hex => $older },
446             newer => { idx => $i, md5_hex => $newer },
447             compare => 'same',
448             }
449             }
450             else {
451 0           push @{$transitions{$k}}, {
  0            
452             older => { idx => $i-1, md5_hex => $older },
453             newer => { idx => $i, md5_hex => $newer },
454             compare => 'different',
455             }
456             }
457             }
458             }
459 0           return \%transitions;
460             }
461              
462             1;
463