File Coverage

blib/lib/Text/TermExtract.pm

Criterion	Covered	Total	%
statement	42	45	93.3
branch	15	20	75.0
condition	1	3	33.3
subroutine	8	8	100.0
pod	3	3	100.0
total	69	79	87.3

###########################################
package Text::TermExtract;
###########################################

use strict;
use warnings;
use Lingua::StopWords;
use Text::Language::Guess;
use Log::Log4perl qw(:easy);

our $VERSION = "0.02";

###########################################
sub new {
###########################################
    my($class, %options) = @_;

    my $self = {
        languages => ['en'],
        %options,
    };

    bless $self, $class;
}

###########################################
sub exclude {
###########################################
    my($self, $aref) = @_;

    for (@$aref) {
        $self->{exclude}->{$_}++;
    }
}

###########################################
sub terms_extract {
###########################################
    my($self, $text, $opts) = @_;

    $opts = {} unless defined $opts;

    my $guesser = Text::Language::Guess->
            new(languages => $self->{languages});

    my $lang =
        $guesser->language_guess_string($text);

    $lang = $self->{languages}->[0] unless $lang;
    DEBUG "Guessed language: $lang\n";

    my $stopwords =
      Lingua::StopWords::getStopWords($lang);

    my %words;

    while($text =~ /\b(\w+)\b/g) {
        my $word = lc($1);
        next if $stopwords->{$word};
        next if $word =~ /^\d+$/;
        next if length($word) <= 2;
        next if exists $self->{exclude}->{$word};
        $words{$word}++;
        $words{$word} += 3 if length $word > 6;
    }

    my @weighted_words = sort {
      $words{$b} <=> $words{$a} or
      $a cmp $b  # sort alphabetically on equal score
    } keys %words;

    if(get_logger()->is_debug()) {
        for my $word (@weighted_words) {
            DEBUG "$word scores $words{$word}";
        }
    }

    if(exists $opts->{max} and $opts->{max} < @weighted_words) {
        return @weighted_words[0..($opts->{max}-1)];
    } else {
        return @weighted_words;
    }
}

1;

__END__