File Coverage

blib/lib/Text/StemTagPOS.pm

Criterion	Covered	Total	%
statement	343	444	77.2
branch	62	136	45.5
condition	12	27	44.4
subroutine	40	44	90.9
pod	7	7	100.0
total	464	658	70.5

package Text::StemTagPOS;

require 5.006002;
use strict;
use warnings;
use Carp;
use Encode;
use Lingua::Stem::Snowball;
use Lingua::EN::Tagger;
use Data::Dump qw(dump);

BEGIN {
    use Exporter ();
    use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
    $VERSION     = '0.61';
    @ISA         = qw(Exporter);
    @EXPORT      = qw();
    @EXPORT_OK   = qw();
    %EXPORT_TAGS = ();
}

use constant WORD_STEMMED => 0;
use constant WORD_ORIGINAL => 1;
use constant WORD_POSTAG => 2;
use constant WORD_INDEX => 3;
use constant WORD_CHAR_POSITION => 4;
use constant WORD_CHAR_LENGTH => 5;
use constant WORD_SENTENCE_ID => 6;
use constant WORD_USER_DEFINED => 7;

use constant POSTAGS_PERIOD => 'PP';
use constant POSTAGS_PUNCTUATION => qw(PP PGP PPC PPD PPL PPR PPS LRB RRB SYM);
use constant POSTAGS_NOUN => qw(NN NNP NNPS NNS);
use constant POSTAGS_ADJECTIVE => qw(CD JJ JJR JJS);
use constant POSTAGS_VERB => qw(VB VBD VBG VBN VBP VBZ);
use constant POSTAGS_ADVERB => qw(RB RBR RBS RP WRB);
use constant POSTAGS_CONTENT_ADVERB => qw(RBR RBS RP);
use constant POSTAGS_ALL => qw(CC CD DET EX FW IN JJ JJR JJS LS MD NN NNP NNPS NNS PDT POS PRP PRPS RB RBR RBS RP SYM TO UH VB VBD VBG VBN VBP VBZ WDT WP WPS WRB PP PGP PPC PPD PPL PPR PPS LRB RRB);
use constant POSTAGS_CONTENT => (POSTAGS_CONTENT_ADVERB, POSTAGS_VERB, POSTAGS_ADJECTIVE, POSTAGS_NOUN);
use constant POSTAGS_TEXTRANK => (POSTAGS_NOUN, POSTAGS_ADJECTIVE);

=head1 NAME

C - Computes stemmed/POS tagged lists of text.

=head1 SYNOPSIS

  use Text::StemTagPOS;
  use Data::Dump qw(dump);
  my $stemTagger = Text::StemTagPOS->new;
  my $text = 'The first sentence. Sentence number two.';
  my $listOfStemmedTaggedSentences = $stemTagger->getStemmedAndTaggedText ($text);
  dump $listOfStemmedTaggedSentences;

=head1 DESCRIPTION

C uses the modules L and L
to do part-of-speech tagging and stemming of English text. It was developed
to pre-process text for other modules. Encoding of all text
should be in Perl's internal format; see L for converting text from
various encodes to a Perl string.

=head1 CONSTRUCTOR

=head2 C

The method C creates an instance of the C class with the following
parameters:

=over

=item C

 isoLangCode => 'en'

C is the ISO language code of the language that will be tagged and
stemmed by the object. It must be 'en', which is the default; other languages
may be added when POS taggers for them are added to CPAN.

=item C

 endingSentenceTag => 'PP'

C is the part-of-speech tag from L
that will be used to indicate
the end of a sentence. The default is 'PP'. The value of C must be
a tag generated by the module L; see method
C for all the possible tags; which are based on the
Penn Treebank tagset.


=item C and/or C

 listOfPOSTypesToKeep => [...], listOfPOSTagsToKeep => [...]

The method C uses C and C
to build the default list of the
parts-of-speech to be retained when filtering previously tagged text.
The default list is C<[qw(TEXTRANK_WORDS)]>,
which is all the nouns and adjectives in the text, as used in the textrank algorithm. Permitted
types for C are 'ALL', 'ADJECTIVES', 'ADVERBS', 'CONTENT_WORDS', 'NOUNS', 'PUNCTUATION',
'TEXTRANK_WORDS', and 'VERBS'. C provides finer control over the
parts-of-speech to be retained. For a list
of all the possible tags see method C.

=back

=cut

sub new
{
  # create the class object.
  my ($Class, %Parameters) = @_;
  my $Self = bless {}, ref($Class) || $Class;

  # get the stemmer to normalize the words, default is english.
  my $isoLangCode = 'en';
  $isoLangCode = lc $Parameters{isoLangCode} if (exists ($Parameters{isoLangCode}));
  $Self->{stemmer} = Lingua::Stem::Snowball->new (lang => $isoLangCode, encode => 'UTF-8');

  # get the POS tagger.
  if ($isoLangCode eq 'en')
  {
    $Self->{tagger} = new Lingua::EN::Tagger;
  }
  else
  {
    croak "parameter isoLangCode must be 'en'.\n";
  }

  # read in the part of speech tags.
  $Self->_getPartOfSpeechTags ();

  # get the ending sentence part of speech tag which will be
  # used to break text into array of sentences.
  unless (exists ($Parameters{endingSentenceTag}))
  {
    $Parameters{endingSentenceTag} = 'PP';
  }
  $Self->{endingSentenceTag} = uc $Parameters{endingSentenceTag};
  $Self->{endingSentenceTag} =~ tr/A-Z//cd;
  $Self->{endingSentenceTag} = '/' . $Self->{endingSentenceTag} if (substr ($Self->{endingSentenceTag}, 0, 1) ne '/');

  # get the list of parts of speech to keep when filtering.
  # the default is to keep only nouns and adjetives.
  $Self->{hashOfPOSTagsToKeep} = $Self->_getHashOfPOSTagsToKeep (%Parameters, instantiation => 1);
  return $Self;
}

=head1 METHODS

=head2 C

 getStemmedAndTaggedText (@Text, $Text, \@Text)

The method C returns a hierarchy of array references containing the stemmed words,
the original words, their part-of-speech tag, and their word position index within the original text.
The hierarchy is of the form

  [
    [ # sentence level: first sentence.
      [ # word level: first word.
        stemmed word, original word, part-of-speech tag, word index, word position, word length
      ]
      [ # word level: second word.
        stemmed word, original word, part-of-speech tag, word index, word position, word length
      ]
      ...
    ]
    [ # sentence level: second sentence.
      [ # word level: first word.
        stemmed word, original word, part-of-speech tag, word index, word position, word length
      ]
      [ # word level: second word.
        stemmed word, original word, part-of-speech tag, word index, word position, word length
      ]
      ...
    ]
  ]

Its only parameters are any combination of strings of text as scalars, references to
scalars, arrays of strings of text, or references to arrays of strings of text, etc...
The following examples below show the various ways to call the method; note that the constants
Text::StemTagPOS::WORD_STEMMED,
Text::StemTagPOS::WORD_ORIGINAL,
Text::StemTagPOS::WORD_POSTAG,
Text::StemTagPOS::WORD_INDEX,
Text::StemTagPOS::WORD_CHAR_POSITION,
Text::StemTagPOS::WORD_CHAR_LENGTH,
Text::StemTagPOS::WORD_SENTENCE_ID, and
Text::StemTagPOS::WORD_USER_DEFINED,
are used to access the information about each word.

  use Text::StemTagPOS;
  use Data::Dump qw(dump);
  my $stemTagger = Text::StemTagPOS->new;
  my $text = 'The first sentence. Sentence number two.';
  my $listOfStemmedTaggedSentences = $stemTagger->getStemmedAndTaggedText ($text);
  dump $listOfStemmedTaggedSentences;

  #  dumps:
  #  [
  #    [
  #      ["the", "The", "/DET", 0, 0, 3, 0],
  #      [" ", " ", "/PGP", 1, 3, 1, 0],
  #      ["first", "first", "/JJ", 2, 4, 5, 0],
  #      [" ", " ", "/PGP", 3, 9, 1, 0],
  #      ["sentenc", "sentence", "/NN", 4, 10, 8, 0],
  #      [".", ".", "/PP", 5, 18, 1, 0],
  #      [" ", " ", "/PGP", 6, 19, 1, 0],
  #    ],
  #    [
  #      ["sentenc", "Sentence", "/NN", 7, 20, 8, 1],
  #      [" ", " ", "/PGP", 8, 28, 1, 1],
  #      ["number", "number", "/NN", 9, 29, 6, 1],
  #      [" ", " ", "/PGP", 10, 35, 1, 1],
  #      ["two", "two", "/CD", 11, 36, 3, 1],
  #      [".", ".", "/PP", 12, 39, 1, 1],
  #    ],
  #  ]

  my $word = $listOfStemmedTaggedSentences->[0][0];
  print
    'WORD_STEMMED: ' .
    "'" . $word->[Text::StemTagPOS::WORD_STEMMED] . "'\n" .
    'WORD_ORIGINAL: ' .
    "'" . $word->[Text::StemTagPOS::WORD_ORIGINAL] . "'\n" .
    'WORD_POSTAG: ' .
    "'" . $word->[Text::StemTagPOS::WORD_POSTAG] . "'\n" .
    'WORD_INDEX: ' .
    $word->[Text::StemTagPOS::WORD_INDEX] . "\n" .
    'WORD_CHAR_POSITION: ' .
    $word->[Text::StemTagPOS::WORD_CHAR_POSITION] . "\n" .
    'WORD_CHAR_LENGTH: ' .
    $word->[Text::StemTagPOS::WORD_CHAR_LENGTH] . "\n";

  #  prints:
  #  WORD_STEMMED: 'the'
  #  WORD_ORIGINAL: 'The'
  #  WORD_POSTAG: '/DET'
  #  WORD_INDEX: 0
  #  WORD_CHAR_POSITION: 0
  #  WORD_CHAR_LENGTH: 3

The following example shows the various ways the text can be passed to the method:

  use Text::StemTagPOS;
  use Data::Dump qw(dump);
  my $stemTagger = Text::StemTagPOS->new;
  my $text = 'This is a sentence with seven words.';
  dump $stemTagger->getStemmedAndTaggedText ($text,
    [$text, \$text], ($text, \$text));

=cut

sub getStemmedAndTaggedText
{
  my ($Self, @Text) = @_;

  # convert the data to process to a list of strings.
  my $ListOfStrings = $Self->_flattenList (@Text);

  my @sentences;
  my $initialIndex = 0;
  foreach my $text (@$ListOfStrings)
  {
    # skip undefined text.
    next unless defined $text;

    # replace all backslashes with a space, since the tagger adds these.
    my $cleanedText = $text;
    $cleanedText =~ tr/\/\-/ /;
    if ($cleanedText !~ m/^\s*$/)
    {
      # tag the text.
      my $taggedText = $Self->{tagger}->get_readable ($cleanedText);

      # convert to a list of sentence word,tag pairs.
      my $listOfSentences = $Self->_convertTextToListOfSentenceWordTags ($taggedText);

      # add the position in the original text and the length of the words.
      $listOfSentences = $Self->_addPositionLengthWordInfo ($initialIndex, $text, $listOfSentences);

      # save the list of sentences.
      push @sentences, @$listOfSentences;
    }

    # update the initial index for the next string.
    $initialIndex += length $text;
  }

  my $wordIndex = 0;
  foreach my $sentenceList (@sentences)
  {
    # pull off the words in the sentence and put them in a list.
    # words must be encoded to utf8 for stemmer.
    my @wordList = map {encode("utf8", $_->[WORD_ORIGINAL])} @$sentenceList;

    # stem the list of words.
    $Self->{stemmer}->stem_in_place (\@wordList);

    # insert the stemmed words to the list as:
    # [stemmed word, original word, part of speech tag, word index, word position, word length]
    for (my $i = 0; $i < @wordList; $i++)
    {
      my $wordTags = $sentenceList->[$i];
      # insert the stemmed word and convert it from utf8 to Perl's internal format.
      $wordTags->[WORD_STEMMED] = lc decode ('utf8', $wordList[$i]);
      $wordTags->[WORD_INDEX] = $wordIndex++;
    }
  }

  return \@sentences;
}

# add the position of the words in the original text to the info about each
# of the words. really should rewrite Lingua::EN::Tagger to provide this
# information; sloppy to recompute it and method used is fragile.
sub _addPositionLengthWordInfo
{
  my ($Self, $PositionOffset, $Text, $ListOfSentences) = @_;

  my $startingIndex = 0;
  my @listOfWords;
  foreach my $sentence (@$ListOfSentences)
  {
    foreach my $word (@$sentence)
    {
      # store all the words for subsequent analysis for missing words.
      push @listOfWords, $word;

      # get and store the length of the word.
      my $length = length ($word->[WORD_ORIGINAL]);
      $word->[WORD_CHAR_LENGTH] = $length;

      # default position is -1, like index function.
      $word->[WORD_CHAR_POSITION] = -1;

      # find the starting position of the word.
      my $position = index ($Text, $word->[WORD_ORIGINAL], $startingIndex);

      if ($position >= $startingIndex)
      {
        # add position and update starting position for next search.
        $word->[WORD_CHAR_POSITION] = $PositionOffset + $position;
        $startingIndex = $position + $length;
      }
    }
  }

  # estimate the positions of the missing words.
  my $totalWords = @listOfWords;
  my $i = -1;
  while (++$i < $totalWords)
  {
    if ($listOfWords[$i]->[WORD_CHAR_POSITION] == -1)
    {
      # get the starting and ending index of the sequence of missing words.
      my $noPositionStartIndex = $i;
      while (++$i < $totalWords)
      {
        last if ($listOfWords[$i]->[WORD_CHAR_POSITION] != -1);
      }
      my $noPositionEndIndex = $i;
      my $totalMissingWords = $noPositionEndIndex - $noPositionStartIndex;
      $totalMissingWords = 0 if ($totalMissingWords < 0);

      # get the substring corresponding to the missing words.

      # compute the starting index of the substring.
      my $startIndexPosition = $PositionOffset;
      if ($noPositionStartIndex > 0)
      {
        $startIndexPosition = $listOfWords[$noPositionStartIndex - 1]->[WORD_CHAR_POSITION] + $listOfWords[$noPositionStartIndex - 1]->[WORD_CHAR_LENGTH];
      }

      # get the ending string index.
      my $endIndexPosition = length $Text;
      if ($noPositionEndIndex < $totalWords)
      {
        $endIndexPosition = $listOfWords[$noPositionEndIndex]->[WORD_CHAR_POSITION];
      }

      # get the substring.
      my $substring = substr ($Text, $startIndexPosition - $PositionOffset, $endIndexPosition - $startIndexPosition);

      # extract the non-white space items.
      my @listOfTokens;
      while ($substring =~ m/([^\s]+)/g)
      {
        push @listOfTokens, [$-[0], $+[0] - $-[1], $1];
      }

      my $substringsToMatch = @listOfTokens;
      $substringsToMatch = $totalMissingWords if ($totalMissingWords < $substringsToMatch);
      for (my $i = 0; $ i < $substringsToMatch; $i++)
      {
        $listOfWords[$noPositionStartIndex + $i]->[WORD_CHAR_POSITION] = $startIndexPosition + $listOfTokens[$i]->[0];
        $listOfWords[$noPositionStartIndex + $i]->[WORD_CHAR_LENGTH] = $listOfTokens[$i]->[1];
      }
    }
  }

  # add the gaps.
  # still need some work here to get the words into the list of sentences that they belong in.
  my $listOfWordsAndGaps = _addGapsToListOfWords (listOfWords => \@listOfWords, stringLength => length ($Text), positionOffset => $PositionOffset);

  # add the sentence id to each gap.
  my $currentSentenceId = 0;
  my @newListOfSentences;
  foreach my $word (@$listOfWordsAndGaps)
  {
    $newListOfSentences[$currentSentenceId] = [] unless defined $newListOfSentences[$currentSentenceId];

    if ($word->[WORD_POSTAG] eq '/PGP')
    {
      $word->[WORD_SENTENCE_ID] = $currentSentenceId;
      $word->[WORD_ORIGINAL] = substr ($Text, $word->[WORD_CHAR_POSITION] - $PositionOffset, $word->[WORD_CHAR_LENGTH]);
    }
    else
    {
      $currentSentenceId = $word->[WORD_SENTENCE_ID];
    }

    push @{$newListOfSentences[$currentSentenceId]}, $word;
  }

  return \@newListOfSentences;
}


sub _addGapsToListOfWords
{
  # get the parameters.
  my %Parameters = @_;

  # get the length of the original string.
  my $stringLength = $Parameters{stringLength};

  # get the list of words.
  my $listOfWords = $Parameters{listOfWords};

  # get the position offset if it exists and is defined.
  my $positionOffset = 0;
  $positionOffset = $Parameters{positionOffset} if (exists ($Parameters{positionOffset}) && defined ($Parameters{positionOffset}));

  # build the list of complete tokens.
  my @listOfWordsAndGaps = @$listOfWords;

  # get the list of missing substring positions.
  my $listOfMissingSubstrings = _getListOfMissingSubstringPositions(listOfSubstringPositions => $listOfWords, stringLength => $stringLength, positionOffset => $positionOffset);

  # add the gaps to the list.
  foreach my $gapInfo (@$listOfMissingSubstrings)
  {
    $gapInfo->[WORD_STEMMED] = ' ';
    $gapInfo->[WORD_ORIGINAL] = undef;
    $gapInfo->[WORD_POSTAG] = '/PGP';
    $gapInfo->[WORD_INDEX] = undef;
  }

  # sort the substrings by position.
  @listOfWordsAndGaps = sort { $a->[WORD_CHAR_POSITION] <=> $b->[WORD_CHAR_POSITION] } (@listOfWordsAndGaps, @$listOfMissingSubstrings);

  # if test is true, make sure things were computed correctly.
  if (exists($Parameters{test}) && $Parameters{test})
  {
    my $totalSubstrings = @listOfWordsAndGaps;

    for (my $i = 1 ; $i < $totalSubstrings ; $i++)
    {

      # make sure the strings are sorted.
      if ($listOfWordsAndGaps[ $i - 1 ]->[WORD_CHAR_POSITION] > $listOfWordsAndGaps[$i]->[WORD_CHAR_POSITION])
      {
        croak 'error: substrings in $listOfSubstringInfo are not sorted and should be.';
      }

      # make sure the strings have at least one character.
      if ($listOfWordsAndGaps[ $i - 1 ]->[WORD_CHAR_LENGTH] < 1)
      {
        croak 'error: substrings in $listOfSubstringInfo has length less than one.';
      }
    }
  }

  # returns the complete list of tokens sorted by their starting index in ascending order.
  return \@listOfWordsAndGaps;
}



# routine returns an array reference of the gaps or missing substrings given
# a list of substrings. for example, if listOfSubstringPositions is
# [[2,4], [9,2], [11,5], [20,1]] and stringLength is 25, then the list returned
# is [[0, 2], [6, 3], [16, 4], [21, 4]].
sub _getListOfMissingSubstringPositions    # (listOfSubstringPositions => \@, stringLength => n, positionOffset => n)
{
  # get the parameters.
  my %Parameters = @_;

  # get the position offset if it exists and is defined.
  my $positionOffset = 0;
  $positionOffset = $Parameters{positionOffset} if (exists ($Parameters{positionOffset}) && defined ($Parameters{positionOffset}));

  # if listOfSubstringPositions is not defined, we have one special case.
  if (!defined($Parameters{listOfSubstringPositions}))
  {
    if (!defined($Parameters{stringLength}))
    {

      # no parameters defined, so return the empty list.
      return [];
    }
    else
    {
      if (int($Parameters{stringLength}) > 0)
      {

        # positive string length, but no substrings, so gap is entire string.
        return [ 0, int($Parameters{stringLength}) - 1 ];
      }
      else
      {

        # non-positive string length, so return empty list.
        return [];
      }
    }
  }

  # get the list of [WORD_CHAR_POSITION, WORD_CHAR_LENGTH].
  my $listOfSubstringPositions = $Parameters{listOfSubstringPositions};

  # get the number of subtrings.
  my $totalSubstrings = $#$listOfSubstringPositions + 1;

  # skip substrings having length less than one or a negative position.
  my @filteredListOfSubstringPositions;
  for (my $i = 0 ; $i < $totalSubstrings ; $i++)
  {
    next if ($listOfSubstringPositions->[$i][WORD_CHAR_LENGTH] < 1);
    next if ($listOfSubstringPositions->[$i][WORD_CHAR_POSITION] < 0);
    push @filteredListOfSubstringPositions, $listOfSubstringPositions->[$i];
  }
  $listOfSubstringPositions = \@filteredListOfSubstringPositions;
  $totalSubstrings          = $#$listOfSubstringPositions + 1;

  # get the entire strings length if defined.
  my $stringLength;
  $stringLength = int abs $Parameters{stringLength} if exists $Parameters{stringLength};

  # if $stringLength is undefined use the last substring to compute the length
  # of the entire string; the string will not end with a gap in this case.
  if (!defined($stringLength) && $totalSubstrings)
  {
    $stringLength = 0;
    foreach my $currentSubstringInfo (@$listOfSubstringPositions)
    {
      my $last = $currentSubstringInfo->[WORD_CHAR_POSITION] + $currentSubstringInfo->[WORD_CHAR_LENGTH];
      $stringLength = $last if $last > $stringLength;
    }
    $stringLength -= $positionOffset;
  }

  # if $stringLength is not defined at this point there are no gaps.
  return [] unless ((defined $stringLength) && ($stringLength > 0));

  # if $totalSubstrings is zero, then the entire string is a gap.
  unless ($totalSubstrings)
  {
    my @substringGapInfo;
    $substringGapInfo[WORD_CHAR_POSITION] = 0;
    $substringGapInfo[WORD_CHAR_LENGTH]   = $stringLength;
    return [ \@substringGapInfo ];
  }

  # sort the pairs by their position.
  my @listOfSubstringPositions = sort { $a->[WORD_CHAR_POSITION] <=> $b->[WORD_CHAR_POSITION] } @$listOfSubstringPositions;

  # @listOfMissingSubstringPositions holds all the gaps.
  my @listOfMissingSubstringPositions;

  # get the first substring position.
  my $currentSubstringInfo = $listOfSubstringPositions[0];

  # if the first substring does not start with position 0, add the beginning gap.
  if ($currentSubstringInfo->[WORD_CHAR_POSITION] > $positionOffset)
  {
    my @substringGapInfo;
    $substringGapInfo[WORD_CHAR_POSITION] = $positionOffset;
    $substringGapInfo[WORD_CHAR_LENGTH]   = $currentSubstringInfo->[WORD_CHAR_POSITION] - $positionOffset;
    push @listOfMissingSubstringPositions, \@substringGapInfo;
  }

  # compute the gaps.
  for (my $i = 1 ; $i < $totalSubstrings ; $i++)
  {

    # get the information about the previous and current substrings.
    my $previousSubstringInfo = $listOfSubstringPositions[ $i - 1 ];
    my $currentSubstringInfo  = $listOfSubstringPositions[$i];

    # compute the starting index and length of the gap.
    my $gapStartPosition = $previousSubstringInfo->[WORD_CHAR_POSITION] + $previousSubstringInfo->[WORD_CHAR_LENGTH];
    my $gapEndPosition   = $currentSubstringInfo->[WORD_CHAR_POSITION] - 1;
    my $gapLength        = $gapEndPosition - $gapStartPosition + 1;

    # if the gap is not a positive size, skip it.
    # maybe a warning should be logged since it really should not happen.
    if ($gapLength > 0)
    {

      # store the information about the gap.
      my @substringGapInfo;
      $substringGapInfo[WORD_CHAR_POSITION] = $gapStartPosition;
      $substringGapInfo[WORD_CHAR_LENGTH]   = $gapLength;
      push @listOfMissingSubstringPositions, \@substringGapInfo;
    }
  }

  # add any trailing gap to the list.
  $currentSubstringInfo = $listOfSubstringPositions[-1];
  if ($currentSubstringInfo->[WORD_CHAR_POSITION] + $currentSubstringInfo->[WORD_CHAR_LENGTH] < $stringLength)
  {
    my @substringGapInfo;
    $substringGapInfo[WORD_CHAR_POSITION] = $currentSubstringInfo->[WORD_CHAR_POSITION] + $currentSubstringInfo->[WORD_CHAR_LENGTH];
    $substringGapInfo[WORD_CHAR_LENGTH] = $stringLength - ($currentSubstringInfo->[WORD_CHAR_POSITION] + $currentSubstringInfo->[WORD_CHAR_LENGTH]);
    push @listOfMissingSubstringPositions, \@substringGapInfo;
  }

  # if test is true, check if gaps were computed correctly.
  if (exists($Parameters{test}) && $Parameters{test})
  {
    my @allSubstrings =
      sort { $a->[WORD_CHAR_POSITION] <=> $b->[WORD_CHAR_POSITION] } (@listOfSubstringPositions, @listOfMissingSubstringPositions);
    my $totalSubstrings = @allSubstrings;

    for (my $i = 1 ; $i < $totalSubstrings ; $i++)
    {

      # make sure the strings are sorted.
      if ($allSubstrings[ $i - 1 ]->[WORD_CHAR_POSITION] + $allSubstrings[ $i - 1 ]->[WORD_CHAR_LENGTH] < $allSubstrings[$i]->[WORD_CHAR_POSITION])
      {
        my $logger = Log::Log4perl->get_logger();
        $logger->logdie("error: missed computing a gap.");
      }
    }
  }

  # returns the list of missing substrings.
  return \@listOfMissingSubstringPositions;
}


=head2 C

 getTaggedTextToKeep (listOfStemmedTaggedSentences => [...],
  listOfPOSTypesToKeep => [...], listOfPOSTagsToKeep => [...]);

The method C returns all the array references of the words
that have a part-of-speech tag that is of a type specified by
C or C. The word lists
returned have the same hierarchical sentence structure used by C.
Note C and C
are optional parameters, if neither is defined, then the values used when the
object was instantiated are used. If one of them is defined, its values override the default
values.

=over

=item C

 listOfStemmedTaggedSentences => [...]

C is the array reference returned by
C or
a previous call to C.

=item C and/or C

 listOfPOSTypesToKeep => [...], listOfPOSTagsToKeep => [...]

C and C define the list of
parts-of-speech types to be retained when filtering previously tagged text.
Permitted values for C are
are 'ALL', 'ADJECTIVES', 'ADVERBS', 'CONTENT_WORDS', 'NOUNS', 'PUNCTUATION',
'TEXTRANK_WORDS', and 'VERBS'. For the possible value of C
see the method C.
Note C and C
are optional parameters, if neither is defined, then the values used when the
object was instantiated are used. If one of them is defined, its values override the default
values.

=back

  use Text::StemTagPOS;
  use Data::Dump qw(dump);
  my $stemTagger = Text::StemTagPOS->new;
  my $text = 'This is the first sentence. This is the last sentence.';
  my $listOfStemmedTaggedSentences = $stemTagger->getStemmedAndTaggedText ($text);
  dump $stemTagger->getTaggedTextToKeep (
    listOfStemmedTaggedSentences => $listOfStemmedTaggedSentences);

  #  dumps:
  #  [
  #    [
  #      ["first", "first", "/JJ", 6, 12, 5, 0],
  #      ["sentenc", "sentence", "/NN", 8, 18, 8, 0],
  #    ],
  #    [
  #      ["last", "last", "/JJ", 17, 40, 4, 1],
  #      ["sentenc", "sentence", "/NN", 19, 45, 8, 1],
  #    ],
  #  ]

=cut

sub getTaggedTextToKeep
{
  my ($Self, %Parameters) = @_;

  # get the list of sentences.
  my $listOfStemmedTaggedSentences = $Parameters{listOfStemmedTaggedSentences};

  # get the hash of tags to keep.
  my $hashOfPOSToKeep = $Self->_getHashOfPOSTagsToKeep (%Parameters);

  # copy off the tokens to keep in each sentence.
  my @listOfFilteredSentences;
  foreach my $sentence (@$listOfStemmedTaggedSentences)
  {
    my @newSentence;
    foreach my $token (@$sentence)
    {
      if (exists ($hashOfPOSToKeep->{$token->[WORD_POSTAG]}))
      {
        push @newSentence, $token;
      }
    }
    #push @listOfFilteredSentences, \@newSentence if (@newSentence > 0);
    push @listOfFilteredSentences, \@newSentence;
  }

  # return the list of filtered sentences.
  return \@listOfFilteredSentences;
}

=head2 C

 getWordsPhrasesInTaggedText (listOfStemmedTaggedSentences => ...,
    listOfPhrasesToFind => [...],  listOfPOSTypesToKeep => [...],
    listOfPOSTagsToKeep => [...]);

The method C returns a reference to an array where
each entry in the array corresponds
to the word or phrase in C. The value of each entry is a list
of word indices
where the words or phrases were found. Each list contains integer
pairs of the form [first-word-index, last-word-index] where first-word-index is the index to the first
word of the phrase and last-word-index the index of the last word. The values of the index are those
assigned to the stemmed and tagged word in C.

  [
    [ # first phrase locations
      [first word index, last word index],
      [first word index, last word index], ...]
    ]
    [ # second phrase locations
      [first word index, last word index],
      [first word index, last word index], ...]
    ]
    ...
  ]

=over

=item C

 listOfStemmedTaggedSentences => [...]

C is the array reference returned by C or
C.

=item C

 listOfPhrasesToFind => [...]

C is an array reference containing a list of strings of
text that are either single words or phrases that are to be located in the text
provided by C. Before the words or phrases are located they are filtered
using C or C.


=item C and/or C

 listOfPOSTypesToKeep => [...], listOfPOSTagsToKeep => [...]

C and C defines the list of
parts-of-speech types to be retained when filtering previously tagged text.
Permitted values for C are
are 'ALL', 'ADJECTIVES', 'ADVERBS', 'CONTENT_WORDS', 'NOUNS', 'PUNCTUATION',
'TEXTRANK_WORDS', and 'VERBS'. For the possible value of C
see the method C.
Note C and C
are optional parameters, if neither is defined, then the values used when the
object was instantiated are used. If one of them is defined, its values override the default
values.

=back

The code below illustrates the output format:

  use Text::StemTagPOS;
  use Data::Dump qw(dump);
  my $stemTagger = Text::StemTagPOS->new;
  my $text = 'This is the first sentence. This is the last sentence.';
  my $listOfStemmedTaggedSentences = $stemTagger->getStemmedAndTaggedText ($text);
  dump $listOfStemmedTaggedSentences;
  my $listOfWordsOrPhrasesToFind = ['first sentence','this is',
    'third sentence', 'sentence'];
  my $phraseLocations = $stemTagger->getWordsPhrasesInTaggedText (
    listOfPOSTypesToKeep => [qw(ALL)],
    listOfStemmedTaggedSentences => $listOfStemmedTaggedSentences,
    listOfWordsOrPhrasesToFind => $listOfWordsOrPhrasesToFind);
  dump $phraseLocations;
  # [
  #   [[6, 8]],           # 'first sentence'
  #   [[0, 2], [11, 13]], # 'this is': note period in text has index 5.
  #   [],                 # 'third sentence'
  #   [[8, 8], [19, 19]]  # 'sentence'
  # ]

=cut

sub getWordsPhrasesInTaggedText
{
  my ($Self, %Parameters) = @_;

  # stem and tag all of the word and phrase text.
  # put all the listOfStemmedTaggedSentences into an array,
  # create the suffix array of the listOfStemmedTaggedSentences array.

  # get the hash of tags to keep.
  my $hashOfPOSToKeep = $Self->_getHashOfPOSTagsToKeep (%Parameters);

  # get the tagged text.
  my $listOfStemmedTaggedSentences = $Parameters{listOfStemmedTaggedSentences};

  # create a list of the filtered stemmed words of the text.
  my @wordsOfText;
  foreach my $sentence (@$listOfStemmedTaggedSentences)
  {
    foreach my $word (@$sentence)
    {
      # skip over the words types to ignore.
      next unless (exists ($hashOfPOSToKeep->{$word->[WORD_POSTAG]}));
      push @wordsOfText, $word;
    }
  }

  # create the suffix array of the tagged words, sorting by the stemmed word.
  my $totalWords = @wordsOfText;

  # hash all the indices of the stemmed words in the suffix array of the text.
  my %wordIndices;
  for(my $index = 0; $index < $totalWords; $index++)
  {
    my $word = $wordsOfText[$index];
    $wordIndices{$word->[WORD_STEMMED]} = [] unless exists $wordIndices{$word->[WORD_STEMMED]};
    push @{$wordIndices{$word->[WORD_STEMMED]}}, $index;
  }

  # stem and part-of-speech tag the word or phrases to find.
  my @listOfStemmedWordsInPhrasesToFind;
  my $listOfWordsOrPhrasesToFind = $Parameters{listOfWordsOrPhrasesToFind};
  foreach my $wordOrPhrase (@$listOfWordsOrPhrasesToFind)
  {
    my $stemmed = $Self->getStemmedAndTaggedText ($wordOrPhrase);

    # flatten the words of a phrase into on list.
    my @allWords;
    foreach my $sentence (@$stemmed)
    {
      foreach my $word (@$sentence)
      {
        # remove the words to ignore.
        next unless (exists ($hashOfPOSToKeep->{$word->[WORD_POSTAG]}));
        push @allWords, $word;
      }
    }

    push @listOfStemmedWordsInPhrasesToFind, \@allWords ;
  }

  # given the starting index into the suffix array and a word list, return
  # the number of words matching the word list at that index.
  my $getMatchingWords = sub
  {
    my ($StartingIndex, $StemmedWordList) = @_;
    my $stemmedIndex = 0;
    my $textIndex = $StartingIndex;
    while (($textIndex < $totalWords) && ($stemmedIndex < @$StemmedWordList))
    {
      last if ($wordsOfText[$textIndex]->[WORD_STEMMED] cmp $StemmedWordList->[$stemmedIndex][WORD_STEMMED]);
      $textIndex++;
      $stemmedIndex++;
    }
    return $textIndex - $StartingIndex;
  };

  # find all the phrases in the filtered text.
  my @phrasesFound;
  my $phraseIndex = 0;
  foreach my $phraseStemmedWordList (@listOfStemmedWordsInPhrasesToFind)
  {
    my @fullPhrasesFound;
    my @stemmedPhraseWords = @$phraseStemmedWordList;
    my $wordsInPhrase = $#stemmedPhraseWords + 1;

    # get the first word in the phrase.
    my $firstWord = $stemmedPhraseWords[0];
    if (exists ($wordIndices{$firstWord->[WORD_STEMMED]}))
    {
      # get all the indices in the suffix array that the first word occurs at.
      my $listOfStartingIndices = $wordIndices{$firstWord->[WORD_STEMMED]};

      # compute the number of matching phrase words at each position.
      foreach my $startingIndex (@$listOfStartingIndices)
      {
        my $matchingLength = &$getMatchingWords ($startingIndex, \@stemmedPhraseWords);
        my $firstWordIndex = $wordsOfText[$startingIndex]->[WORD_INDEX];
        my $lastWordIndex = $wordsOfText[$startingIndex + $matchingLength - 1]->[WORD_INDEX];

        if ($matchingLength == $wordsInPhrase)
        {
          # full match of the phrase.
          push @fullPhrasesFound, [$firstWordIndex, $lastWordIndex];
        }
      }
    }
    $phrasesFound[$phraseIndex++] = \@fullPhrasesFound;
  }
  return \@phrasesFound;
}

=head2 C

The method C takes no parameters. It returns an array
reference where each item in the list is of the form C<[part of speech tag, description, examples]>.
It is meant for getting the part-of-speech tags that can be used to populate C.

  use Text::StemTagPOS;
  use Data::Dump qw(dump);
  my $stemTagger = Text::StemTagPOS->new;
  dump $stemTagger->getListOfPartOfSpeechTags;

=cut

# this method takes no parameters. it reads in the information about the
# part of speech tags stored in the DATA section of this file. the information
# is stored in the object at posTags as an array reference containing the
# array [POS tag, description, examples] for each possible tag.
sub getListOfPartOfSpeechTags
{
  my $Self = shift;

  # return a copy of the posTags array.
  my @partOfSpeechTags = map {[@$_]} @{$Self->{posTags}};
  return \@partOfSpeechTags;
}


# _convertTextToListOfSentenceWordTags->('string of tagged words returned from xxxx');
sub _convertTextToListOfSentenceWordTags
{
  # get the method class object.
  my $Self = $_[0];

  # split the string of tagged words into a list of the words and tags. note,
  # all POS tags are a slash followed by 2, 3  or 4 uppercase letters.
  my @list = split (/(\/[A-Z]{2,4})/, $_[1]);

  # trim off any leading or trailing spaces from each word or tag.
  foreach my $item (@list)
  {
    $item =~ s/^\s*//;
    $item =~ s/\s+$//;
  }

  # restructure the list so each item is of the form [word, tag].
  my $wordOrPhrase;
  my @taggerWordList;
  foreach my $item (@list)
  {
    if (substr ($item, 0, 1) eq '/')
    {
      # at this point the item in the list is a tag, so form the new [word,tag] pair
      # and save it.
      my $wordInfo = [];
      $wordInfo->[WORD_ORIGINAL] = $wordOrPhrase;
      $wordInfo->[WORD_POSTAG] = $item;
      push @taggerWordList, $wordInfo;
      $wordOrPhrase = undef;
    }
    else
    {
      # at this point we have a new word, so append it to $wordOrPhrase.
      if (!defined ($wordOrPhrase))
      {
        $wordOrPhrase = $item;
      }
      else
      {
        $wordOrPhrase .= ' ' . $item;
      }
    }
  }

  # now partition the list into sentences.
  my @sentences;
  my $currentSentence = [];
  foreach my $wordTag (@taggerWordList)
  {
    if ($wordTag->[WORD_POSTAG] eq $Self->{endingSentenceTag})
    {
      # if the sentence has no words, skip it.
      next unless ($#$currentSentence > -1);

      # add the POS sentence ender tag to the end of the sentence list.
      push @$currentSentence, $wordTag;

      # add the sentence to the list of sentences.
      push @sentences, $currentSentence;
      $currentSentence = [];
    }
    else
    {
       push @$currentSentence, $wordTag;
    }
  }

  # if there is a sentence left, save it.
  if ($#$currentSentence > -1)
  {
    push @sentences, $currentSentence;
  }

  # add the sentence id to each word.
  for (my $i = 0; $i < @sentences; $i++)
  {
    foreach my $word (@{$sentences[$i]})
    {
      $word->[WORD_SENTENCE_ID] = $i;
    }
  }

  # return the list of sentences of word,tag pairs.
  return \@sentences;
}


# converts a list of strings, string refs, arrays of strings, etc... to a
# simpler list of strings; essentially flattening a list.
sub _flattenList
{
  my ($Self, @Data) = @_;

  my @ListOfStrings;
  foreach my $item (@_)
  {
    my $type = ref ($item);
    if ($type eq '')
    {
      push @ListOfStrings, $item;
    }
    elsif ($type eq 'SCALAR')
    {
      push @ListOfStrings, $$item;
    }
    elsif ($type eq 'ARRAY')
    {
       push @ListOfStrings, @{$Self->_flattenList (@$item)};
    }
    elsif ($type eq 'REF')
    {
       push @ListOfStrings, @{$Self->_flattenList ($$item)};
    }
  }
  return \@ListOfStrings;
}


# this method takes no parameters. it reads in the information about the
# part of speech tags stored in the DATA section of this file. the information
# is stored in the object at posTags as an array reference containing the
# array [POS tag, description, examples] for each possible tag.
sub _getPartOfSpeechTags
{
  my $Self = shift;

  my $posTags =
  [
    ['CC', 'Conjunction, coordinating', 'and, or'],
    ['CD', 'Adjective, cardinal number', '3, fifteen'],
    ['DET', 'Determiner', 'this, each, some'],
    ['EX', 'Pronoun, existential there', 'there'],
    ['FW', 'Foreign words', ''],
    ['IN', 'Preposition / Conjunction', 'for, of, although, that'],
    ['JJ', 'Adjective', 'happy, bad'],
    ['JJR', 'Adjective, comparative', 'happier, worse'],
    ['JJS', 'Adjective, superlative', 'happiest, worst'],
    ['LS', 'Symbol, list item', 'A, A.'],
    ['MD', 'Verb, modal', "can, could, 'll"],
    ['NN', 'Noun', 'aircraft, data'],
    ['NNP', 'Noun, proper', 'London, Michael'],
    ['NNPS', 'Noun, proper, plural', 'Australians, Methodists'],
    ['NNS', 'Noun, plural', 'women, books'],
    ['PDT', 'Determiner, prequalifier', 'quite, all, half'],
    ['POS', 'Possessive', "'s"],
    ['PRP', 'Determiner, possessive second', 'mine, yours'],
    ['PRPS', 'Determiner, possessive', 'their, your'],
    ['RB', 'Adverb', 'often, not, very, here'],
    ['RBR', 'Adverb, comparative', 'faster'],
    ['RBS', 'Adverb, superlative', 'fastest'],
    ['RP', 'Adverb, particle', 'up, off, out'],
    ['SYM', 'Symbol', '*'],
    ['TO', 'Preposition', 'to'],
    ['UH', 'Interjection', 'oh, yes, mmm'],
    ['VB', 'Verb, infinitive', 'take, live'],
    ['VBD', 'Verb, past tense', 'took, lived'],
    ['VBG', 'Verb, gerund', 'taking, living'],
    ['VBN', 'Verb, past/passive participle', 'taken, lived'],
    ['VBP', 'Verb, base present form', 'take, live'],
    ['VBZ', 'Verb, present 3SG -s form', 'takes, lives'],
    ['WDT', 'Determiner, question', 'which, whatever'],
    ['WP', 'Pronoun, question', 'who, whoever'],
    ['WPS', 'Determiner, possessive & question', 'whose'],
    ['WRB', 'Adverb, question', 'when, how, however'],
    ['PP', 'Punctuation, sentence ender', '., !, ?'],
    ['PPC', 'Punctuation, comma', ','],
    ['PGP', 'Punctuation, whitespace, gap', ' '],
    ['PPD', 'Punctuation, dollar sign', '$'],
    ['PPL', 'Punctuation, quotation mark left', '``'],
    ['PPR', 'Punctuation, quotation mark right', "''"],
    ['PPS', 'Punctuation, colon, semicolon, elipsis', ':, ..., -'],
    ['LRB', 'Punctuation, left bracket', '(, {, ['],
    ['RRB', 'Punctuation, right bracket', '), }, ]']
  ];

  # store the pos array.
  $Self->{posTags} = $posTags;
  return;
}


sub _getListOfPOSTagsFromPOSTypesList
{
  my ($Self, %Parameters) = @_;

  my $listOfPOSTypesToKeep = [];
  $listOfPOSTypesToKeep = $Parameters{listOfPOSTypesToKeep} if exists $Parameters{listOfPOSTypesToKeep};

  # ALL, NOUNS, VERBS, PUNCTUATION, ADJECTIVES, ADVERBS, CONTENT_WORDS, TEXTRANK_WORDS, VERBS
  my @listOfPOSTagsToKeep;
  foreach my $type (@$listOfPOSTypesToKeep)
  {
    my $ucType = uc $type;
    $ucType =~ tr/A-Z\_//cd;
    if (($ucType cmp 'NOUNS') == 0) { push @listOfPOSTagsToKeep, POSTAGS_NOUN; }
    elsif (($ucType cmp 'VERBS')  == 0) { push @listOfPOSTagsToKeep, POSTAGS_VERB; }
    elsif (($ucType cmp 'ADJECTIVES')  == 0) { push @listOfPOSTagsToKeep, POSTAGS_ADJECTIVE; }
    elsif (($ucType cmp 'ADVERBS')  == 0) { push @listOfPOSTagsToKeep, POSTAGS_ADVERB; }
    elsif (($ucType cmp 'CONTENT_ADVERBS')  == 0) { push @listOfPOSTagsToKeep, POSTAGS_CONTENT_ADVERB; }
    elsif (($ucType cmp 'PUNCTUATION')  == 0) { push @listOfPOSTagsToKeep, POSTAGS_PUNCTUATION; }
    elsif (($ucType cmp 'CONTENT_WORDS')  == 0) { push @listOfPOSTagsToKeep, POSTAGS_CONTENT; }
    elsif (($ucType cmp 'TEXTRANK_WORDS')  == 0) { push @listOfPOSTagsToKeep, POSTAGS_TEXTRANK; }
    elsif (($ucType cmp 'ALL')  == 0) { push @listOfPOSTagsToKeep, POSTAGS_ALL; }
  }

  return \@listOfPOSTagsToKeep;
}


# returns a hash reference of the part of speech tags that are
# provided in listOfPOSTagsToKeep.
sub _getHashOfPOSTagsToKeep
{
  my ($Self, %Parameters) = @_;

  # get the list of POS tags to keep.
  my $listOfPOSTagsToKeep;
  if (!exists ($Parameters{listOfPOSTagsToKeep}) && !exists ($Parameters{listOfPOSTypesToKeep}))
  {
    return $Self->{hashOfPOSTagsToKeep} unless exists $Parameters{instantiation};
    $listOfPOSTagsToKeep = [POSTAGS_TEXTRANK];
  }
  else
  {
    $listOfPOSTagsToKeep = $Self->_getListOfPOSTagsFromPOSTypesList (%Parameters);
    push @$listOfPOSTagsToKeep, @{$Parameters{listOfPOSTagsToKeep}} if (exists ($Parameters{listOfPOSTagsToKeep}));
  }

  my %hashOfPOSTagsToKeep;
  foreach my $pos (@$listOfPOSTagsToKeep)
  {
    my $posClean = uc $pos;
    $posClean =~ tr/A-Z0-9//cd;

    # prefix with a slash.
    $hashOfPOSTagsToKeep{'/' . $posClean} = 1;
  }

  return \%hashOfPOSTagsToKeep;
}

=head2 C

The method C returns an array reference of the sorted stemmed
words in the text given by C.

=over

=item C

 listOfStemmedTaggedSentences => [...]

C is the array reference returned by C or
C of the text.

=back

  use Text::StemTagPOS;
  use Data::Dump qw(dump);
  my $stemTagger = Text::StemTagPOS->new;
  my $text = 'The first sentence. Sentence number two.';
  my $listOfStemmedTaggedSentences = $stemTagger->getStemmedAndTaggedText ($text);
  dump $listOfStemmedTaggedSentences;

=cut

sub getListOfStemmedWordsInText
{
  my ($Self, %Parameters) = @_;

  # get the hash of tags to keep.
  my $hashOfPOSToKeep = $Self->_getHashOfPOSTagsToKeep (%Parameters);

  # get the tagged text.
  my $listOfStemmedTaggedSentences = $Parameters{listOfStemmedTaggedSentences};

  # create a list of the filtered stemmed words of the text.
  my %wordsOfText;
  foreach my $sentence (@$listOfStemmedTaggedSentences)
  {
    foreach my $word (@$sentence)
    {
      # skip over the words types to ignore.
      next unless (exists ($hashOfPOSToKeep->{$word->[WORD_POSTAG]}));
      $wordsOfText{$word->[WORD_STEMMED]} = 1;
    }
  }
  my @words = sort keys %wordsOfText;
  return \@words;
}

=head2 C

The method C returns an array reference of the sorted stemmed
words of the intersection of all the words in the documents given by C;

=over

=item C

 listOfStemmedTaggedDocuments => [...]

C is a list of document references returned by C or
C.

=back

=cut

sub getListOfStemmedWordsInAllDocuments
{
  my ($Self, %Parameters) = @_;

  # get the list of documents to process.
  my $listOfStemmedTaggedDocuments = $Parameters{listOfStemmedTaggedDocuments};
  my $totalDocuments = @$listOfStemmedTaggedDocuments;
  my %wordOccurence;

  # put the words of the first document into a hash.
  my $sentences = $listOfStemmedTaggedDocuments->[0];
  foreach my $sentence (@$sentences)
  {
    foreach my $word (@$sentence)
    {
      $wordOccurence{$word->[WORD_STEMMED]} = 1;
    }
  }

  # add the words from the remaining documents, only if the word
  # occurred in all previous documents.
  for (my $i = 1; $i < $totalDocuments; $i++)
  {
    my $sentences = $listOfStemmedTaggedDocuments->[$i];
    foreach my $sentence (@$sentences)
    {
      foreach my $word (@$sentence)
      {
        # if the word has not occurred in a previous document, skip it.
        next unless exists $wordOccurence{$word->[WORD_STEMMED]};

        # if the word has not occurred in all previous documents, delete it.
        if ($wordOccurence{$word->[WORD_STEMMED]} < $i)
        {
          delete $wordOccurence{$word->[WORD_STEMMED]};
          next;
        }

        # set the words occurence.
        $wordOccurence{$word->[WORD_STEMMED]} = $i + 1;
      }
    }
  }

  # return the words that occured in all the documents.
  my @wordsInAll;
  while (my ($word, $occurence) = each %wordOccurence)
  {
    push @wordsInAll, $word if ($occurence == $totalDocuments);
  }
  @wordsInAll = sort @wordsInAll;
  return \@wordsInAll;
}

=head1 INSTALLATION

To install the module run the following commands:

  perl Makefile.PL
  make
  make test
  make install

If you are on a windows box you should use 'nmake' rather than 'make'.

=head1 AUTHOR

 Jeff Kubina

=head1 BUGS

Please email bugs reports or feature requests to C, or through
the web interface at L.  The author
will be notified and you can be automatically notified of progress on the bug fix or feature request.

=head1 COPYRIGHT

Copyright (c) 2010 Jeff Kubina. All rights reserved.
This program is free software; you can redistribute
it and/or modify it under the same terms as Perl itself.

The full text of the license can be found in the
LICENSE file included with this module.

=head1 KEYWORDS

natural language processing, NLP, part of speech tagging, POS, stemming

=head1 SEE ALSO

L, L, L, L,
L, L

=begin html

See the Lingua::EN::Tagger README
file for a list of the part-of-speech tags.

=end html

=cut

1;
# The preceding line will help the module return a true value