File Coverage

blib/lib/PPIx/Regexp/Lexer.pm

Criterion	Covered	Total	%
statement	250	264	94.7
branch	64	80	80.0
condition	26	35	74.2
subroutine	44	45	97.7
pod	5	5	100.0
total	389	429	90.6

=head1 NAME

PPIx::Regexp::Lexer - Assemble tokenizer output.

=head1 SYNOPSIS

 use PPIx::Regexp::Lexer;
 use PPIx::Regexp::Dumper;
 my $lex = PPIx::Regexp::Lexer->new('qr{foo}smx');
 my $dmp = PPIx::Regexp::Dumper->new( $lex );
 $dmp->print();

=head1 INHERITANCE

C is a
L.

C has no descendants.

=head1 DESCRIPTION

This class takes the token stream generated by
L and generates the
parse tree.

=head1 METHODS

This class provides the following public methods. Methods not documented
here are private, and unsupported in the sense that the author reserves
the right to change or remove them without notice.

=cut

package PPIx::Regexp::Lexer;

use strict;
use warnings;

use base qw{ PPIx::Regexp::Support };

use Carp qw{ confess };
use PPIx::Regexp::Constant qw{
    ARRAY_REF
    TOKEN_LITERAL
    TOKEN_UNKNOWN
    @CARP_NOT
};
use PPIx::Regexp::Node::Range				();
use PPIx::Regexp::Node::Unknown				();
use PPIx::Regexp::Structure				();
use PPIx::Regexp::Structure::Assertion			();
use PPIx::Regexp::Structure::Atomic_Script_Run		();
use PPIx::Regexp::Structure::BranchReset		();
use PPIx::Regexp::Structure::Code			();
use PPIx::Regexp::Structure::Capture			();
use PPIx::Regexp::Structure::CharClass			();
use PPIx::Regexp::Structure::Subexpression		();
use PPIx::Regexp::Structure::Main			();
use PPIx::Regexp::Structure::Modifier			();
use PPIx::Regexp::Structure::NamedCapture		();
use PPIx::Regexp::Structure::Quantifier			();
use PPIx::Regexp::Structure::Regexp			();
use PPIx::Regexp::Structure::RegexSet			();
use PPIx::Regexp::Structure::Replacement		();
use PPIx::Regexp::Structure::Script_Run			();
use PPIx::Regexp::Structure::Switch			();
use PPIx::Regexp::Structure::Unknown			();
use PPIx::Regexp::Token::Unmatched			();
use PPIx::Regexp::Tokenizer				();
use PPIx::Regexp::Util qw{ __choose_tokenizer_class __instance };

our $VERSION = '0.087_01';

=head2 new

This method instantiates the lexer. It takes as its argument either a
L or the text to be
parsed. In the latter case the tokenizer is instantiated from the text.

Any optional name/value pairs after the first argument are passed to the
tokenizer, which interprets them or not as the case may be.

=cut

{

    my $errstr;

    sub new {
	my ( $class, $tokenizer, %args ) = @_;
	ref $class and $class = ref $class;

	unless ( __instance( $tokenizer, 'PPIx::Regexp::Tokenizer' ) ) {
	    my $tokenizer_class = __choose_tokenizer_class(
		$tokenizer, \%args )
		or do {
		    $errstr = 'Data not supported';
		    return;
		};
	    $tokenizer = $tokenizer_class->new( $tokenizer, %args )
		or do {
		    $errstr = $tokenizer_class->errstr();
		    return;
		};
	}

	my $self = {
	    deferred	=> [],	# Deferred tokens
	    failures	=> 0,
	    strict	=> $args{strict},
	    tokenizer	=> $tokenizer,
	};

	bless $self, $class;
	return $self;
    }

    sub errstr {
	return $errstr;
    }

}

=head2 errstr

This method returns the error string from the last attempt to
instantiate a C. If the last attempt succeeded, the
error will be C.

=cut

# Defined above

=head2 failures

 print $lexer->failures(), " parse failures\n";

This method returns the number of parse failures encountered. A
parse failure is either a tokenization failure (see
L<< PPIx::Regexp::Tokenizer->failures()|PPIx::Regexp::Tokenizer/failures >>)
or a structural error.

=cut

sub failures {
    my ( $self ) = @_;
    return $self->{failures};
}

=head2 lex

This method lexes the tokens in the text, and returns the lexed list of
elements.

=cut

sub lex {
    my ( $self ) = @_;

    my @content;
    $self->{failures} = 0;

    # Accept everything up to the first delimiter.
    my $kind;	# Initial PPIx::Regexp::Token::Structure
    {
	my $token = $self->_get_token()
	    or return $self->_finalize( @content );
	$token->isa( 'PPIx::Regexp::Token::Delimiter' ) or do {
	    not $kind
		and $token->isa( 'PPIx::Regexp::Token::Structure' )
		and $kind = $token;
	    push @content, $token;
	    redo;
	};
	$self->_unget_token( $token );
    }

    my ( $part_0_class, $part_1_class ) =
	$self->{tokenizer}->__part_classes();

    # Accept the first delimited structure.
    push @content, ( my $part_0 = $self->_get_delimited(
	    $part_0_class ) );

    # If we are a substitution ...
    if ( defined $part_1_class ) {

	# Accept any insignificant stuff.
	while ( my $token = $self->_get_token() ) {
	    if ( $token->significant() ) {
		$self->_unget_token( $token );
		last;
	    } else {
		push @content, $token;
	    }
	}

	# Figure out if we should expect an opening bracket.
	my $expect_open_bracket = $self->close_bracket(
	    $part_0->start( 0 ) ) || 0;

	# Accept the next delimited structure.
	push @content, $self->_get_delimited(
		$part_1_class,
		$expect_open_bracket,
	);
    }

    # Accept the modifiers (we hope!) plus any trailing white space.
    while ( my $token = $self->_get_token() ) {
	push @content, $token;
    }

    # Let all the elements finalize themselves, recording any additional
    # errors as they do so.
    $self->_finalize( @content );

    # If we found a regular expression (and we should have done so) ...
    if ( $part_0 && $part_0->can( 'max_capture_number' ) ) {
	# TODO the above line is really ugly. I'm wondering about
	# string implementations like:
	# * return a $part_0_class of undef (but that complicates the
	#   lexing of the structure itself);
	# * hang this logic on the tokenizer somehow (where it seems out
	#   of place)
	# * hang this logic on PPIx::Regexp::Structure::Regexp and
	#   ::Replacement.
	# I also need to figure out how to make \n backreferences come
	# out as literals. Maybe that is a job best done by the
	# tokenizer.

	# Retrieve the maximum capture group.
	my $max_capture = $part_0->max_capture_number();

	# Hashify the known capture names
	my $capture_name = {
	    map { $_ => 1 } $part_0->capture_names(),
	};

	# For all the backreferences found
	foreach my $elem ( @{ $part_0->find(
	    'PPIx::Regexp::Token::Backreference' ) || [] } ) {
	    # Rebless them as needed, recording any errors found.
	    $self->{failures} +=
		$elem->__PPIX_LEXER__rebless(
		    capture_name	=> $capture_name,
		    max_capture		=> $max_capture,
		);
	}
    }

    return @content;

}

=head2 strict

This method returns true or false based on the value of the C<'strict'>
argument to C.

=cut

sub strict {
    my ( $self ) = @_;
    return $self->{strict};
}

# Finalize the content array, updating the parse failures count as we
# go.
sub _finalize {
    my ( $self, @content ) = @_;
    foreach my $elem ( @content ) {
	$self->{failures} += $elem->__PPIX_LEXER__finalize( $self );
    }
    defined wantarray and return @content;
    return;
}

{

    my %bracket = (
	'{' => '}',
	'(' => ')',
	'[' => ']',
	'(?['	=> '])',
    ##  '<' => '>',
    );

    my %unclosed = (
	'{' => '_recover_curly',
    );

    sub _get_delimited {
	my ( $self, $class, $expect_open_bracket ) = @_;
	defined $expect_open_bracket or $expect_open_bracket = 1;

	my @rslt;
	$self->{_rslt} = \@rslt;

	if ( $expect_open_bracket ) {
	    if ( my $token = $self->_get_token() ) {
		push @rslt, [];
		if ( $token->isa( 'PPIx::Regexp::Token::Delimiter' ) ) {
		    push @{ $rslt[-1] }, '', $token;
		} else {
		    push @{ $rslt[-1] }, '', undef;
		    $self->_unget_token( $token );
		}
	    } else {
		return;
	    }
	} else {
	    push @rslt, [ '', undef ];
	}

	while ( my $token = $self->_get_token() ) {
	    if ( $token->isa( 'PPIx::Regexp::Token::Delimiter' ) ) {
		$self->_unget_token( $token );
		last;
	    }
	    if ( $token->isa( 'PPIx::Regexp::Token::Structure' ) ) {
		my $content = $token->content();

		if ( my $finish = $bracket{$content} ) {
		    # Open bracket
		    push @rslt, [ $finish, $token ];

		} elsif ( $content eq $rslt[-1][0] ) {

		    # Matched close bracket
		    $self->_make_node( $token );

		} elsif ( $content ne ')' ) {

		    # If the close bracket is not a parenthesis, it becomes
		    # a literal.
		    TOKEN_LITERAL->__PPIX_ELEM__rebless( $token );
		    push @{ $rslt[-1] }, $token;

		} elsif ( $content eq ')'
			and @rslt > 1	# Ignore enclosing delimiter
			and my $recover = $unclosed{$rslt[-1][1]->content()} ) {
		    # If the close bracket is a parenthesis and there is a
		    # recovery procedure, we use it.
		    $self->$recover( $token );

		} else {

		    # Unmatched close with no recovery.
		    $self->{failures}++;
		    PPIx::Regexp::Token::Unmatched->
			__PPIX_ELEM__rebless( $token );
		    push @{ $rslt[-1] }, $token;
		}

	    } else {
		push @{ $rslt[-1] }, $token;
	    }

	    # We have to hand-roll the Range object.
	    if ( __instance( $rslt[-1][-2], 'PPIx::Regexp::Token::Operator' )
		&& $rslt[-1][-2]->content() eq '-'
		&& $rslt[-1][0] eq ']'	# It's a character class
	    ) {
		my @tokens = splice @{ $rslt[-1] }, -3;
		push @{ $rslt[-1] },
		    PPIx::Regexp::Node::Range->__new( @tokens );
	    }
	}

	while ( @rslt > 1 ) {
	    if ( my $recover = $unclosed{$rslt[-1][1]->content()} ) {
		$self->$recover();
	    } else {
		$self->{failures}++;
		$self->_make_node( undef );
	    }
	}

	if ( @rslt == 1 ) {
	    my @last = @{ pop @rslt };
	    shift @last;
	    push @last, $self->_get_token();
	    return $class->__new( @last );
	} else {
	    confess "Missing data";
	}

    }

}

#	$token = $self->_get_token();
#
#	This method returns the next token from the tokenizer.

sub _get_token {
    my ( $self ) = @_;

    if ( @{ $self->{deferred} } ) {
	return shift @{ $self->{deferred} };
    }

    my $token = $self->{tokenizer}->next_token() or return;

    return $token;
}

{

    my %handler = (
	'(' => '_round',
	'[' => '_square',
	'{' => '_curly',
	'(?['	=> '_regex_set',
    );

    sub _make_node {
	my ( $self, $token ) = @_;
	my @args = @{ pop @{ $self->{_rslt} } };
	shift @args;
	push @args, $token;
	my @node;
	if ( my $method = $handler{ $args[0]->content() } ) {
	    @node = $self->$method( \@args );
	}
	@node or @node = PPIx::Regexp::Structure->__new( @args );
	push @{ $self->{_rslt}[-1] }, @node;
	return;
    }

}

# Called as $self->$method( ... ) in _make_node(), above
sub _curly {	## no critic (ProhibitUnusedPrivateSubroutines)
    my ( $self, $args ) = @_;

    if ( $args->[-1] && $args->[-1]->is_quantifier() ) {

	# If the tokenizer has marked the right curly as a quantifier,
	# make the whole thing a quantifier structure.
	return PPIx::Regexp::Structure::Quantifier->__new( @{ $args } );

    } elsif ( $args->[-1] ) {

	# If there is a right curly but it is not a quantifier,
	# make both curlys into literals.
	foreach my $inx ( 0, -1 ) {
	    TOKEN_LITERAL->__PPIX_ELEM__rebless( $args->[$inx] );
	}

	# Try to recover possible quantifiers not recognized because we
	# thought this was a structure.
	$self->_recover_curly_quantifiers( $args );

	return @{ $args };

    } else {

	# If there is no right curly, just make a generic structure
	# TODO maybe this should be something else?
	return PPIx::Regexp::Structure->__new( @{ $args } );
    }
}

# Recover from an unclosed left curly.
# Called as $self->$revover( ... ) in _get_delimited, above
sub _recover_curly {	## no critic (ProhibitUnusedPrivateSubroutines)
    my ( $self, $token ) = @_;

    # Get all the stuff we have accumulated for this curly.
    my @content = @{ pop @{ $self->{_rslt} } };

    # Lose the right bracket, which we have already failed to match.
    shift @content;

    # Rebless the left curly appropriately
    if ( $self->{_rslt}[0][-1]->isa( 'PPIx::Regexp::Token::Assertion' )
	&& q<\b> eq $self->{_rslt}[0][-1]->content() ) {
	# If following \b, it becomes an unknown.
	TOKEN_UNKNOWN->__PPIX_ELEM__rebless( $content[0],
	    error	=> 'Unterminated bound type',
	);
    } else {
	# Rebless the left curly to a literal.
	TOKEN_LITERAL->__PPIX_ELEM__rebless( $content[0] );
    }

    # Try to recover possible quantifiers not recognized because we
    # thought this was a structure.
    $self->_recover_curly_quantifiers( \@content );

    # Shove the curly and its putative contents into whatever structure
    # we have going.
    # The checks are to try to trap things like RT 56864, though on
    # further reflection it turned out that you could get here with an
    # empty $self->{_rslt} on things like 'm{)}'. This one did not get
    # made into an RT ticket, but was fixed by not calling the recovery
    # code if $self->{_rslt} contained only the enclosing delimiters.
    ARRAY_REF eq ref $self->{_rslt}
	or confess 'Programming error - $self->{_rslt} not array ref, ',
	    "parsing '", $self->{tokenizer}->content(), "' at ",
	    $token->content();
    @{ $self->{_rslt} }
	or confess 'Programming error - $self->{_rslt} empty, ',
	    "parsing '", $self->{tokenizer}->content(), "' at ",
	    $token->content();
    push @{ $self->{_rslt}[-1] }, @content;

    # Shove the mismatched delimiter back into the input so we can have
    # another crack at it.
    $token and $self->_unget_token( $token );

    # We gone.
    return;
}

sub _recover_curly_quantifiers {
    my ( undef, $args ) = @_;	# Invocant unused

    if ( __instance( $args->[0], TOKEN_LITERAL )
	&& __instance( $args->[1], TOKEN_UNKNOWN )
	&& PPIx::Regexp::Token::Quantifier->could_be_quantifier(
	$args->[1]->content() )
    ) {
	PPIx::Regexp::Token::Quantifier->
	    __PPIX_ELEM__rebless( $args->[1] );

	if ( __instance( $args->[2], TOKEN_UNKNOWN )
	    && PPIx::Regexp::Token::Greediness->could_be_greediness(
		$args->[2]->content() )
	) {
	    PPIx::Regexp::Token::Greediness
		->__PPIX_ELEM__rebless( $args->[2] );
	}

    }

    return;
}

sub _in_regex_set {
    my ( $self ) = @_;
    foreach my $stack_entry ( reverse @{ $self->{_rslt} } ) {
	$stack_entry->[0] eq '])'
	    and return 1;
    }
    return 0;
}

# Called as $self->$method( ... ) in _make_node(), above
sub _round {	## no critic (ProhibitUnusedPrivateSubroutines)
    my ( $self, $args ) = @_;

    # If we're inside a regex set, parens do not capture.
    $self->_in_regex_set()
	and return PPIx::Regexp::Structure->__new( @{ $args } );

    # If /n is asserted, parens do not capture.
    $self->{tokenizer}->modifier( 'n' )
	and return PPIx::Regexp::Structure->__new( @{ $args } );

    # The instantiator will rebless based on the first token if need be.
    return PPIx::Regexp::Structure::Capture->__new( @{ $args } );
}

# Called as $self->$method( ... ) in _make_node(), above
sub _square {	## no critic (ProhibitUnusedPrivateSubroutines)
    my ( undef, $args ) = @_;	# Invocant unused
    return PPIx::Regexp::Structure::CharClass->__new( @{ $args } );
}

# Called as $self->$method( ... ) in _make_node(), above
sub _regex_set {	## no critic (ProhibitUnusedPrivateSubroutines)
    my ( undef, $args ) = @_;	# Invocant unused
    return PPIx::Regexp::Structure::RegexSet->__new( @{ $args } );
}

#	$self->_unget_token( $token );
#
#	This method caches its argument so that it will be returned by
#	the next call to C<_get_token()>. If more than one argument is
#	passed, they will be returned in the order given; that is,
#	_unget_token/_get_token work like unshift/shift.

sub _unget_token {
    my ( $self, @args ) = @_;
    unshift @{ $self->{deferred} }, @args;
    return $self;
}

1;

__END__