File Coverage

blib/lib/Search/Tools/Tokenizer.pm

Criterion	Covered	Total	%
statement	44	46	95.6
branch	16	18	88.8
condition	3	3	100.0
subroutine	8	8	100.0
pod	2	2	100.0
total	73	77	94.8

package Search::Tools::Tokenizer;
use Moo;
extends 'Search::Tools::Object';
use Search::Tools;    # XS package required
use Search::Tools::Token;
use Search::Tools::TokenList;
use Search::Tools::UTF8;
use Carp;

our $VERSION = '1.007';

has 're' => ( is => 'rw', default => sub {qr/\w+(?:[\'\-\.]\w+)*/} );

sub BUILD {
    my $self = shift;
    if ( $self->debug ) {
        $self->set_debug( $self->debug - 1 );    # XS debug a level higher
    }
    return $self;
}

sub tokenize_pp {
    require Search::Tools::TokenPP;
    require Search::Tools::TokenListPP;

    my $self = shift;
    if ( !defined $_[0] ) {
        croak "str required";
    }

    # XS modifies the original arg, so we do too.
    # this is same slight optimization XS does. ~5%
    if ( !is_ascii( $_[0] ) ) {
        $_[0] = to_utf8( $_[0] );
    }
    my $heat_seeker = $_[1];

    # match_num ($_[2]) not supported in PP

    my @heat   = ();
    my @tokens = ();
    my $i      = 0;
    my $re     = $self->{re};
    my $heat_seeker_is_coderef
        = ( defined $heat_seeker and ref($heat_seeker) eq 'CODE' ) ? 1 : 0;

    # TODO is_sentence_* logic
    for ( split( m/($re)/, $_[0] ) ) {
        next unless length($_);
        my $tok = bless(
            {   'pos'    => $i++,
                str      => $_,
                is_hot   => 0,
                is_match => 0,
                len      => byte_length($_),
                u8len    => length($_),
            },
            'Search::Tools::TokenPP'
        );
        if ( $_ =~ m/^$re$/ ) {
            $tok->{is_match} = 1;
            if ($heat_seeker_is_coderef) {
                $heat_seeker->($tok);
            }
            elsif ( defined $heat_seeker ) {
                $tok->{is_hot} = $_ =~ m/$heat_seeker/;
            }
        }
        push( @heat, $tok->{pos} ) if $tok->{is_hot};
        push @tokens, $tok;
    }
    return bless(
        {   tokens => \@tokens,
            num    => $i,
            'pos'  => 0,
            heat   => \@heat,
        },
        'Search::Tools::TokenListPP'
    );
}

1;

__END__