File Coverage

blib/lib/KinoSearch1/Analysis/Tokenizer.pm

Criterion	Covered	Total	%
statement	37	37	100.0
branch	4	4	100.0
condition	5	9	55.5
subroutine	9	9	100.0
pod	2	2	100.0
total	57	61	93.4

package KinoSearch1::Analysis::Tokenizer;
use strict;
use warnings;
use KinoSearch1::Util::ToolSet;
use base qw( KinoSearch1::Analysis::Analyzer );
use locale;

BEGIN {
    __PACKAGE__->init_instance_vars(

        # constructor params / members
        token_re => undef,    # regex for a single token

        # members
        separator_re => undef,    # regex for separations between tokens
    );
}

use KinoSearch1::Analysis::TokenBatch;

sub init_instance {
    my $self = shift;

    # supply defaults if token_re wasn't specified
    if ( !defined $self->{token_re} ) {
        $self->{token_re}     = qr/\b\w+(?:'\w+)?\b/;
        $self->{separator_re} = qr/\W*/;
    }

    # if user-defined token_re...
    if ( !defined $self->{separator_re} ) {

        # define separator using lookahead
        $self->{separator_re} = qr/
            .*?                    # match up to...
            (?=                    # but not including...
                $self->{token_re}  # a token,
                |\z                # or the end of the string
            )/xsm;
    }
}

sub analyze {
    my ( $self, $batch ) = @_;

    my $new_batch    = KinoSearch1::Analysis::TokenBatch->new;
    my $token_re     = $self->{token_re};
    my $separator_re = $self->{separator_re};

    # alias input to $_
    while ( $batch->next ) {
        local $_ = $batch->get_text;

        # ensure that pos is set to 0 for this scalar
        pos = 0;

        # accumulate token start_offsets and end_offsets
        my ( @starts, @ends );
        1 while ( m/$separator_re/g and push @starts,
            pos and m/$token_re/g and push @ends, pos );

        # correct for overshoot
        $#starts = $#ends;

        # add the new tokens to the batch
        $new_batch->add_many_tokens( $_, \@starts, \@ends );
    }

    return $new_batch;
}

1;

__END__