NewsExtractor/Role/ContentTextExtractor.pm

filter: Statementstmt 52.1% 24 / 46 Branchbran 0.0% 0 / 10 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 77.7% 7 / 9 Podpod n/a totaltotal 47.6% 31 / 65 CC 6 SCAR 24.9
CC
6
Cov
43%
_build_content_text
30.0
_build_site_name
6.0
BEGIN
1.0
CRAP 12.1
« NewsExtractor/SiteSpecificExtractor/news_cts_com_tw.pm ↑ Summary NewsExtractor/SiteSpecificExtractor/www_setn_com.pm »
1 package NewsExtractor::Role::ContentTextExtractor;
2 1 ▶use utf8;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 1 ▶use Moo::Role;
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 1 ▶use Types::Standard qw(Str Maybe);
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 ▶use List::Util qw(max);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 1 ▶use HTML::ExtractContent;
Subroutine BEGIN: called CC=1 SCAR=0.0
8  
9 1 ▶use Importer 'NewsExtractor::TextUtil' => qw( html2text );
Subroutine BEGIN: called CC=1 SCAR=0.0
10 1 ▶use Importer 'NewsExtractor::Constants' => qw( %RE );
Subroutine BEGIN: called CC=1 SCAR=0.0
11  
12 1 has site_name => (
13 is => "lazy",
14 isa => Maybe[Str],
15 );
16  
17 1 has content_text => (
18 is => "lazy",
19 isa => Maybe[Str],
20 );
21  
22 0 ▶sub _build_site_name {
Subroutine _build_site_name: not called CC=2 SCAR=17.9
23 0 my ($self) = @_;
24  
25 0 my $el = $self->dom->at("meta[property='og:site_name']");
26 0 ▶ if ($el) {
Branch: 1 branch
BranchTrueFalseTotal
if ($el) 0 0 0
27 0 return $el->attr('content');
28 }
29  
30 0 return undef;
31 }
32  
33 0 ▶sub _build_content_text {
Subroutine _build_content_text: not called CC=5 SCAR=34.0
34 0 my ($self) = @_;
35 0 my ($el, $html);
36  
37 # Cleanup some noisy elements that are known to interfere.
38 0 $self->dom->find('script, style, p.appE1121, div.sexmask, div.cat-list, div#marquee, #setting_weather')->map('remove');
39  
40 0 my $extractor = HTML::ExtractContent->new;
41 0 ▶ if ($el = $self->dom->at('article')) {
Branch: 1 branch
BranchTrueFalseTotal
if ($el = $self->dom->at('article')) { } 0 0 0
42 0 $html = $extractor->extract("$el")->as_html;
43 } else {
44 0 $html = $extractor->extract( $self->dom->to_string )->as_html;
45 }
46  
47 0 my $text = html2text( $html );
48  
49 0 ▶ my @paragraphs = split(/\n\n/, $text) or return undef;
Branch: 1 branch
BranchTrueFalseTotal
my @paragraphs = split(/\n\n/, $text, 0) or return undef 0 0 0
50  
51 0 ▶ if (my $site_name = $self->site_name) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $site_name = $self->site_name) 0 0 0
52 0 $paragraphs[-1] =~ s/\A \s* \p{Punct}? \s* ${site_name} \s* \p{Punct}? \s* \z//x;
53 0 $paragraphs[-1] =~ s/${site_name}//x;
54 }
55  
56 0 $paragraphs[-1] =~ s/\A \s* \p{Punct}? \s* $RE{newspaper_names} \s* \p{Punct}? \s* \z//x;
57  
58 0 ▶ if (max( map { length($_) } @paragraphs ) < 30) {
Branch: 1 branch
BranchTrueFalseTotal
if (&max(map({length $_;} @paragraphs)) < 30) 0 0 0
59 # err "[$$] Not enough contents";
60 0 return undef;
61 }
62  
63 0 return join "\n\n", @paragraphs;
64 }
65  
66 1 1;
« NewsExtractor/SiteSpecificExtractor/news_cts_com_tw.pm ↑ Summary NewsExtractor/SiteSpecificExtractor/www_setn_com.pm »