File Coverage

blib/lib/KSx/Analysis/StripAccents.pm

Criterion	Covered	Total	%
statement	23	23	100.0
branch			n/a
condition			n/a
subroutine	7	7	100.0
pod	0	1	0.0
total	30	31	96.7

use strict;
use warnings;

package KSx::Analysis::StripAccents;
use base qw( KinoSearch::Analysis::Analyzer );

our $VERSION = '0.05';

use Encode qw 'encode decode';
use Text::Unaccent 'unac_string_utf16';

sub analyze_batch {
    my ( $self, $batch ) = @_;

    # lc and unaccent all of the terms, one by one
    while ( my $token = $batch->next ) {
        # I have to use UTF-16BE, since, although it’s not documented,
        # Text::Unaccent only supports big-endian. And I have to encode it,
        # since it doesn’t support Perl’s Unicode strings. (And it’ll con-
        # vert it to UTF-16 behind the scenes anyway, if I don’t.)
        $token->set_text(
            lc uc decode 'utf-16be', unac_string_utf16
                encode 'UTF-16BE', $token->get_text );
        # We have an ‘lc uc’ there, since some letters won’t be normalised
        # properly without it; e.g., ‘Σσς’ should be normalised to three
        # instances of the same character (‘σσσ’ as opposed to ‘σσς’).
    }

    $batch->reset;
    return $batch;
}

*transform = *analyze_batch;

1;

__END__