File Coverage

blib/lib/Plucene/Analysis/UTF8Tokenizer.pm

Criterion	Covered	Total	%
statement	9	30	30.0
branch	0	10	0.0
condition	0	3	0.0
subroutine	3	6	50.0
pod	1	3	33.3
total	13	52	25.0

package Plucene::Analysis::UTF8Tokenizer;

our $VERSION = '0.02';

use base qw/Plucene::Analysis::Tokenizer/;
use strict;
use warnings;

sub token_re { //o }

sub next
{
    my $self = shift;
    my $fh = $self->{reader};

    if (!defined $self->{buffer} or length $self->{buffer} == 0) {
	$self->{buffer} .= <$fh>;
    }

    if ($self->{buffer}) {
	$self->{word} = [] unless ref $self->{word};
	$self->scantext($self->{buffer});
	$self->{buffer} = undef;
    }
}

sub scantext
{
    my $self = shift;
    my $text = shift or return;

    $text = lc $text;

    my %tok;

    foreach ($self->{word}) {
	$tok{$_} = 1;
    }

    my $c = undef;
    while ($text =~ /([a-z\d]+|\S)/go) {
	$tok{$1} = 1;
	$tok{$c . $1} = 1 if defined $c;
	$c = $1;
    }

    $self->{word} = [keys %tok];
}

1;
__END__