view release on metacpan or  search on metacpan

local/lib/perl5/PPI/Tokenizer.pm  view on Meta::CPAN

		endprotoent
		endpwent
		endservent
		fork
		getgrent
		gethostent
		getlogin
		getnetent
		getppid
		getprotoent
		getpwent
		getservent
		setgrent
		setpwent
		time
		times
		wait
		wantarray
		__SUB__
);



#####################################################################
# Creation and Initialization

=pod

=head2 new $file | \@lines | \$source

The main C<new> constructor creates a new Tokenizer object. These
objects have no configuration parameters, and can only be used once,
to tokenize a single perl source file.

It takes as argument either a normal scalar containing source code,
a reference to a scalar containing source code, or a reference to an
ARRAY containing newline-terminated lines of source code.

Returns a new C<PPI::Tokenizer> object on success, or throws a
L<PPI::Exception> exception on error.

=cut

sub new {
	my $class = ref($_[0]) || $_[0];

	# Create the empty tokenizer struct
	my $self = bless {
		# Source code
		source       => undef,
		source_bytes => undef,

		# Line buffer
		line         => undef,
		line_length  => undef,
		line_cursor  => undef,
		line_count   => 0,

		# Parse state
		token        => undef,
		class        => 'PPI::Token::BOM',
		zone         => 'PPI::Token::Whitespace',

		# Output token buffer
		tokens       => [],
		token_cursor => 0,
		token_eof    => 0,

		# Perl 6 blocks
		perl6        => [],
	}, $class;

	if ( ! defined $_[1] ) {
		# We weren't given anything
		PPI::Exception->throw("No source provided to Tokenizer");

	} elsif ( ! ref $_[1] ) {
		my $source = PPI::Util::_slurp($_[1]);
		if ( ref $source ) {
			# Content returned by reference
			$self->{source} = $$source;
		} else {
			# Errors returned as a string
			return( $source );
		}

	} elsif ( _SCALAR0($_[1]) ) {
		$self->{source} = ${$_[1]};

	} elsif ( _ARRAY0($_[1]) ) {
		$self->{source} = join '', map { "\n" } @{$_[1]};

	} else {
		# We don't support whatever this is
		PPI::Exception->throw(ref($_[1]) . " is not supported as a source provider");
	}

	# We can't handle a null string
	$self->{source_bytes} = length $self->{source};
	if ( $self->{source_bytes} ) {
		# Split on local newlines
		$self->{source} =~ s/(?:\015{1,2}\012|\015|\012)/\n/g;
		$self->{source} = [ split /(?<=\n)/, $self->{source} ];

	} else {
		$self->{source} = [ ];
	}

	### EVIL
	# I'm explaining this earlier than I should so you can understand
	# why I'm about to do something that looks very strange. There's
	# a problem with the Tokenizer, in that tokens tend to change
	# classes as each letter is added, but they don't get allocated
	# their definite final class until the "end" of the token, the
	# detection of which occurs in about a hundred different places,
	# all through various crufty code (that triples the speed).
	#
	# However, in general, this does not apply to tokens in which a
	# whitespace character is valid, such as comments, whitespace and
	# big strings.
	#