NewsExtractor/TextUtil.pm

filter: Statementstmt 76.7% 43 / 56 Branchbran 60.0% 6 / 10 Conditioncond 50.0% 7 / 14 MC/DCmcdc 0.0% 0 / 7 Subroutinesub 72.7% 8 / 11 Podpod 0.0% 0 / 7 totaltotal 60.9% 64 / 105 CC 13 SCAR 14.1
CC
13
Cov
70%
reformat_dateline
6.2
is_empty
6.0
parse_dateline_ymdhms
5.5
CRAP 4.1
« NewsExtractor/SiteSpecificExtractor/news_ebc_net_tw.pm ↑ Summary NewsExtractor/CSSExtractor.pm »
1 package NewsExtractor::TextUtil;
2 6 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 6 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 6 ▶use Encode qw(is_utf8 decode_utf8);
Subroutine BEGIN: called CC=1 SCAR=0.0
5 6 ▶use Mojo::DOM;
Subroutine BEGIN: called CC=1 SCAR=0.0
6  
7 6 our @EXPORT = (
8 'u',
9 'normalize_whitespace',
10 'html2text',
11 'is_empty',
12 'parse_dateline_ymdhms',
13 'reformat_dateline',
14 'remove_control_characters',
15 );
16  
17 43 ▶sub u($) {
Subroutine u: called CC=3 SCAR=11.1
Pod undocumented
18 43 ▶ defined($_[0]) or return undef;
Branch: 1 branch
BranchTrueFalseTotal
defined $_[0] or return undef 43 0 43
19  
20 43 my $v = "".$_[0];
21 43 ▶ return is_utf8($v) ? $v : decode_utf8($v);
Branch: 1 branch
BranchTrueFalseTotal
&is_utf8($v) ? : 17 26 43
22 }
23  
24 0 ▶sub is_empty {
Subroutine is_empty: not called CC=2 SCAR=17.9
Pod undocumented
25 0 ▶ (! defined($_[0])) || $_[0] eq '';
Branch: 1 branch
BranchTrueFalseTotal
not defined $_[0] or $_[0] eq "" 0 0 0
26 }
27  
28 2 ▶sub normalize_whitespace {
Subroutine normalize_whitespace: called CC=1 SCAR=0.0
Pod undocumented
29 2 local $_ = $_[0];
30 2 s/\h+/ /g;
31 2 s/\r\n/\n/g;
32 2 s/\A\s+//;
33 2 s/\s+\z//;
34 2 return $_;
35 }
36  
37 0 ▶sub remove_control_characters {
Subroutine remove_control_characters: not called CC=1 SCAR=6.9
Pod undocumented
38 0 local $_ = $_[0];
39 0 s/\p{PosixCntrl}//g;
40 0 return $_;
41 }
42  
43 0 ▶sub html2text {
Subroutine html2text: not called CC=1 SCAR=6.9
Pod undocumented
44 0 my $html = $_[0];
45  
46 0 my $content_dom = Mojo::DOM->new('<body>' . $html . '</body>');
47 0 $content_dom->find('br')->map(replace => "\n");
48 0 $content_dom->find('div,p')->map(append => "\n\n");
49  
50 0 my @paragraphs = grep { $_ ne '' } map { remove_control_characters($_) } map { normalize_whitespace($_) } split /\n\n+/, $content_dom->all_text;
51  
52 0 return join "\n\n", @paragraphs;
53 }
54  
55 7 ▶sub reformat_dateline {
Subroutine reformat_dateline: called CC=6 SCAR=18.2
Pod undocumented
56 7 my ($text, $offset) = @_;
57 7 ▶ $offset //= '';
Condition50%1/2
$offset //= ""
l!l
70
MC/DC0%0/1
$offset //= ""
$offset
Truth table50%1/2
A
A= $offset
Aresult
11
00
58  
59 7 my @t = $text =~ m/([0-9]+)/g;
60 7 ▶ return undef unless 3 <= @t;
Branch: 1 branch
BranchTrueFalseTotal
unless 3 <= @t 0 7 7
61  
62 7 my $format_date = '%04d-%02d-%02d';
63 7 my $format_time = '%02d:%02d:%02d';
64  
65 7 ▶ if (@t == 3) {
Branch: 1 branch
BranchTrueFalseTotal
if (@t == 3) 2 5 7
66 2 my $format = $format_date . '%s';
67 2 return u(sprintf($format, $t[0], $t[1], $t[2], $offset));
68 }
69  
70 5 ▶ $t[5] //= 0;
Condition50%1/2
$t[5] //= 0
l!l
05
MC/DC0%0/1
$t[5] //= 0
$t[5]
Truth table50%1/2
A
A= $t[5]
Aresult
11
00
71 5 ▶ $t[6] //= 0;
Condition50%1/2
$t[6] //= 0
l!l
05
MC/DC0%0/1
$t[6] //= 0
$t[6]
Truth table50%1/2
A
A= $t[6]
Aresult
11
00
72  
73 5 my $format = $format_date . 'T' . $format_time . '%s';
74 5 return u(sprintf($format, $t[0], $t[1], $t[2], $t[3], $t[4], $t[5], $offset));
75 }
76  
77 4 ▶sub parse_dateline_ymdhms {
Subroutine parse_dateline_ymdhms: called CC=5 SCAR=17.0
Pod undocumented
78 4 my ($text, $offset) = @_;
79  
80 4 ▶ $offset //= '';
Condition50%1/2
$offset //= ""
l!l
40
MC/DC0%0/1
$offset //= ""
$offset
Truth table50%1/2
A
A= $offset
Aresult
11
00
81  
82 4 my @t = $text =~ m/([0-9]+)/g;
83 4 ▶ $t[3] //= 23;
Condition50%1/2
$t[3] //= 23
l!l
40
MC/DC0%0/1
$t[3] //= 23
$t[3]
Truth table50%1/2
A
A= $t[3]
Aresult
11
00
84 4 ▶ $t[4] //= 59;
Condition50%1/2
$t[4] //= 59
l!l
40
MC/DC0%0/1
$t[4] //= 59
$t[4]
Truth table50%1/2
A
A= $t[4]
Aresult
11
00
85 4 ▶ $t[5] //= 59;
Condition50%1/2
$t[5] //= 59
l!l
04
MC/DC0%0/1
$t[5] //= 59
$t[5]
Truth table50%1/2
A
A= $t[5]
Aresult
11
00
86  
87 4 return u(
88 sprintf(
89 '%04d-%02d-%02dT%02d:%02d:%02d%s',
90 $t[0], $t[1], $t[2], $t[3], $t[4], $t[5], $offset
91 )
92 );
93 }
94  
95 6 1;
« NewsExtractor/SiteSpecificExtractor/news_ebc_net_tw.pm ↑ Summary NewsExtractor/CSSExtractor.pm »