system: Linux mars.sprixweb.com 3.10.0-1160.119.1.el7.x86_64 #1 SMP Tue Jun 4 14:43:51 UTC 2024 x86_64
cmd: 

Direktori : /usr/share/doc/perl-Parse-RecDescent-1.967009/demo/
Upload File :
Current File : //usr/share/doc/perl-Parse-RecDescent-1.967009/demo/demo_lexer.pl

#!/usr/bin/perl -sw

# IT'S A PARSER *AND* A LEXER...

use Parse::RecDescent;
use Data::Dumper;

$lexer = new Parse::RecDescent q
{
	lex: token(s)

	token: 'I\b'		<token:POSS>
	     | 'see\b'		<token:VERB>
	     | 'on\b'		<token:PREP>
	     | 'by\b'		<token:PREP>
	     | /the\b|a\b/i	<token:ARTICLE>
	     | /\w+/		<token:WORD,PUNCT,OTHER>
};

$data = join '', <DATA>;

print_tokens($lexer->lex(\$data));

print "left: [$data]\n";

sub print_tokens
{
	foreach $token ( @{$_[0]} )
	{
		print Dumper($token), "\n";
	}
}

__DATA__

I see a cat on the windowsill by the door!!!!!