flex++ lexer for moot PoS tagger native text input (guts for moot::TokenReaderNative) More...

Inheritance diagram for mootTokenLexer:

Collaboration diagram for mootTokenLexer:

[legend]

Public Types
typedef moot::mootTokenType	TokenType

typedef moot::mootTokenType	TokenType

Public Member Functions
int	yylex (void)

	mootTokenLexer ()

virtual	~mootTokenLexer (void)

virtual void	reset (void)

void	on_EOA (void)

void	loc_set (moot::OffsetT off, int len)

void	loc_add (int len)

virtual void *	mgl_yy_current_buffer_p (void)

virtual void *	mgl_yy_create_buffer (int size, FILE *unused=stdin)

virtual void	mgl_yy_init_buffer (void buf, FILE unused=stdin)

virtual void	mgl_yy_delete_buffer (void *buf)

virtual void	mgl_yy_switch_to_buffer (void *buf)

virtual void	mgl_begin (int stateno)

int	yylex (void)

	mootTokenLexer ()

virtual	~mootTokenLexer (void)

virtual void	reset (void)

void	on_EOA (void)

void	loc_set (moot::OffsetT off, int len)

void	loc_add (int len)

virtual void *	mgl_yy_current_buffer_p (void)

virtual void *	mgl_yy_create_buffer (int size, FILE *unused=stdin)

virtual void	mgl_yy_init_buffer (void buf, FILE unused=stdin)

virtual void	mgl_yy_delete_buffer (void *buf)

virtual void	mgl_yy_switch_to_buffer (void *buf)

virtual void	mgl_begin (int stateno)

Public Member Functions inherited from moot::GenericLexer
	GenericLexer (const std::string &myname="moot::GenericLexer", size_t line=0, size_t column=0, mootio::ByteOffset byte=0)

virtual	~GenericLexer (void)

virtual void	clear (bool clear_input=true, bool clear_output=true)

virtual void	from_mstream (mootio::mistream *in=__null)

virtual void	from_filename (const std::string &filename)

virtual void	from_file (FILE *in=stdin)

virtual void	from_buffer (const char *buf, size_t len)

void	from_string (const char *s)

void	from_string (const std::string &s)

void	select_streams (FILE in, FILE out=stdout, const char *myname=__null)

void	select_string (const char in, FILE out=stderr, const char *myname=__null)

virtual void	to_mstream (mootio::mostream *out=__null)

virtual void	to_filename (const std::string &filename)

virtual void	to_file (FILE *out=stdout)

int	yyinput (char *buffer, int &result, int max_size)

void	tokbuf_append (const char *text, size_t len)

void	add_columns (ByteOffset len)

void	add_lines (ByteOffset len=1)

virtual void	yycarp (const char *fmt,...)

Public Attributes
unsigned char *	yytext

int	yyleng

FILE *	yyin

FILE *	yyout

int	lasttyp

int	nextstate

moot::mootToken	mtoken_default

moot::mootToken *	mtoken

moot::mootToken::Analysis *	manalysis

bool	ignore_comments

bool	first_analysis_is_best

bool	current_analysis_is_best

bool	ignore_first_analysis

bool	ignore_current_analysis

bool	parse_location

bool	parse_analysis_cost

bool	analysis_cost_details

Public Attributes inherited from moot::GenericLexer
mootio::mistream *	mglin

mootio::mostream *	mglout

bool	mgl_in_created

bool	mgl_out_created

size_t	theLine

size_t	theColumn

mootio::ByteOffset	theByte

std::string	lexname

std::string	tokbuf

bool	tokbuf_clear

Static Public Attributes
static const int	LexTypeText = moot::NTokTypes+1

static const int	LexTypeTag = moot::NTokTypes+2

static const int	LexTypeDetails = moot::NTokTypes+3

static const int	LexTypeEOA = moot::NTokTypes+4

static const int	LexTypeEOT = moot::NTokTypes+5

static const int	LexTypeIgnore = moot::NTokTypes+6

Static Public Attributes inherited from moot::GenericLexer
static const int	MGL_DEFAULT_BUFFER_SIZE = 16384

Protected Member Functions
void	yyrestart (FILE *input_file)

void	yy_switch_to_buffer (YY_BUFFER_STATE new_buffer)

void	yy_load_buffer_state (void)

YY_BUFFER_STATE	yy_create_buffer (FILE *file, int size)

void	yy_delete_buffer (YY_BUFFER_STATE b)

void	yy_init_buffer (YY_BUFFER_STATE b, FILE *file)

virtual void	yy_echo ()

virtual int	yy_input (char *buf, int &result, int max_size)

virtual void	yy_fatal_error (const char *msg)

virtual int	yy_wrap ()

void	yyrestart (FILE *input_file)

void	yy_switch_to_buffer (YY_BUFFER_STATE new_buffer)

void	yy_load_buffer_state (void)

YY_BUFFER_STATE	yy_create_buffer (FILE *file, int size)

void	yy_delete_buffer (YY_BUFFER_STATE b)

void	yy_init_buffer (YY_BUFFER_STATE b, FILE *file)

virtual void	yy_echo ()

virtual int	yy_input (char *buf, int &result, int max_size)

virtual void	yy_fatal_error (const char *msg)

virtual int	yy_wrap ()

Protected Attributes
YY_BUFFER_STATE	yy_current_buffer

Private Member Functions
void	yy_initialize ()

int	input ()

int	yyinput ()

int	yy_get_next_buffer ()

void	yyunput (unsigned char c, unsigned char *buf_ptr)

long	yy_get_previous_state_ (void)

long	yy_try_NUL_trans_ (long current_state_)

void	yy_initialize ()

int	input ()

int	yyinput ()

int	yy_get_next_buffer ()

void	yyunput (unsigned char c, unsigned char *buf_ptr)

long	yy_get_previous_state_ (void)

long	yy_try_NUL_trans_ (long current_state_)

Private Attributes
unsigned char *	yy_c_buf_p

unsigned char	yy_hold_char

int	yy_n_chars

int	yy_init

int	yy_start

int	yy_did_buffer_switch_on_eof

Detailed Description

Assumes pre-tokenized input:

One token per line
Lines are parsed into TAB-separated fields
Blank lines are interpreted as end-of-sentence ( EOS)
Input contains raw text only (no markup!)
Comments are introduced with '%%', and continue to the end of the line
Word- and sentence-break hints are special comments "\%\%\$WB\$", "\%\%\$SB\$"
Supports multiple tags per token

Format:

 FILE       ::= SENTENCE*
 SENTENCE   ::= ((TOKEN|COMMENT)*) "\n"
 TOKEN      ::= TOKEN_TEXT ("\t" LOCATION)? ("\t" ANALYSIS)* ("\r"*) "\n"
 TOKEN_TEXT ::= [^\t\r\n]*
 LOCATION   ::= OFFSET_INT " " LENGTH_INT
 ANALYSIS   ::= (DETAIL*) (COST?) (DETAIL*) "[" TAG ( "]" | " " ) (DETAIL*) (COST?) (DETAIL*)
 DETAIL     ::= [^\t\r\n]
 COST       ::= "<" FLOAT ">"
 COMMENT    ::= "^%%" (.*) "\n"

Member Typedef Documentation

◆ TokenType [1/2]

typedef moot::mootTokenType mootTokenLexer::TokenType

◆ TokenType [2/2]

typedef moot::mootTokenType mootTokenLexer::TokenType

Constructor & Destructor Documentation

◆ mootTokenLexer() [1/2]

mootTokenLexer::mootTokenLexer ( )

Referenced by yyinput().

◆ ~mootTokenLexer() [1/2]

virtual mootTokenLexer::~mootTokenLexer ( void )

inlinevirtual

References moot::mootToken::loc_length(), and moot::mootToken::location().

◆ mootTokenLexer() [2/2]

mootTokenLexer::mootTokenLexer ( )

◆ ~mootTokenLexer() [2/2]

virtual mootTokenLexer::~mootTokenLexer ( void )

inlinevirtual

References loc_add(), moot::mootToken::loc_length(), loc_set(), moot::mootToken::location(), mgl_yy_create_buffer(), mgl_yy_current_buffer_p(), on_EOA(), reset(), yy_current_buffer, and yyleng.

Member Function Documentation

◆ yy_initialize() [1/2]

void mootTokenLexer::yy_initialize ( )

private

◆ input() [1/2]

int mootTokenLexer::input ( )

private

Referenced by yyinput().

◆ yyinput() [1/2]

int mootTokenLexer::yyinput ( )

inlineprivate

◆ yy_get_next_buffer() [1/2]

int mootTokenLexer::yy_get_next_buffer ( )

private

Referenced by yyinput().

◆ yyunput() [1/2]

void mootTokenLexer::yyunput	(	unsigned char	c,
		unsigned char *	buf_ptr
	)

private

Referenced by yyinput().

◆ yy_get_previous_state_() [1/2]

long mootTokenLexer::yy_get_previous_state_ ( void )

private

Referenced by yyinput().

◆ yy_try_NUL_trans_() [1/2]

long mootTokenLexer::yy_try_NUL_trans_ ( long current_state_ )

private

Referenced by yyinput().

◆ yyrestart() [1/2]

void mootTokenLexer::yyrestart ( FILE * input_file )

protected

Referenced by yyinput().

◆ yy_switch_to_buffer() [1/2]

void mootTokenLexer::yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer )

protected

Referenced by mgl_yy_switch_to_buffer(), and yyinput().

◆ yy_load_buffer_state() [1/2]

void mootTokenLexer::yy_load_buffer_state ( void )

protected

Referenced by yyinput().

◆ yy_create_buffer() [1/2]

YY_BUFFER_STATE mootTokenLexer::yy_create_buffer	(	FILE *	file,
		int	size
	)

protected

Referenced by mgl_yy_init_buffer(), and yyinput().

◆ yy_delete_buffer() [1/2]

void mootTokenLexer::yy_delete_buffer ( YY_BUFFER_STATE b )

protected

Referenced by mgl_yy_switch_to_buffer(), and yyinput().

◆ yy_init_buffer() [1/2]

void mootTokenLexer::yy_init_buffer	(	YY_BUFFER_STATE	b,
		FILE *	file
	)

protected

Referenced by mgl_yy_switch_to_buffer(), and yyinput().

◆ yy_echo() [1/2]

virtual void mootTokenLexer::yy_echo ( )

protectedvirtual

Referenced by yyinput().

◆ yy_input() [1/2]

virtual int mootTokenLexer::yy_input	(	char *	buf,
		int &	result,
		int	max_size
	)

protectedvirtual

Referenced by yyinput().

◆ yy_fatal_error() [1/2]

virtual void mootTokenLexer::yy_fatal_error ( const char * msg )

protectedvirtual

Referenced by yyinput().

◆ yy_wrap() [1/2]

virtual int mootTokenLexer::yy_wrap ( )

protectedvirtual

Referenced by yyinput().

◆ yylex() [1/2]

int mootTokenLexer::yylex ( void )

Referenced by yyinput().

◆ reset() [1/2]

virtual void mootTokenLexer::reset ( void )

virtual

Reset lexer state

Reimplemented from moot::GenericLexer.

Referenced by moot::wasteLexer::length_attr(), and ~mootTokenLexer().

◆ on_EOA() [1/2]

void mootTokenLexer::on_EOA ( void )

Referenced by ~mootTokenLexer().

◆ loc_set() [1/2]

void mootTokenLexer::loc_set	(	moot::OffsetT	off,
		int	len
	)

inline

Referenced by ~mootTokenLexer().

◆ loc_add() [1/2]

void mootTokenLexer::loc_add ( int len )

inline

Referenced by ~mootTokenLexer().

◆ mgl_yy_current_buffer_p() [1/2]

virtual void* mootTokenLexer::mgl_yy_current_buffer_p ( void )

inlinevirtual

Reimplemented from moot::GenericLexer.

Referenced by ~mootTokenLexer().

◆ mgl_yy_create_buffer() [1/2]

virtual void* mootTokenLexer::mgl_yy_create_buffer	(	int	size,
		FILE *	unused = `stdin`
	)

inlinevirtual

Flex++ should define this (backwards), but it doesn't get through to us

Implements moot::GenericLexer.

Referenced by ~mootTokenLexer().

◆ mgl_yy_init_buffer() [1/2]

virtual void mootTokenLexer::mgl_yy_init_buffer	(	void *	buf,
		FILE *	unused = `stdin`
	)

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Implements moot::GenericLexer.

References mgl_yy_init_buffer().

Referenced by mgl_yy_init_buffer().

◆ mgl_yy_delete_buffer() [1/2]

virtual void mootTokenLexer::mgl_yy_delete_buffer ( void * buf )

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Reimplemented from moot::GenericLexer.

Referenced by mgl_yy_switch_to_buffer().

◆ mgl_yy_switch_to_buffer() [1/2]

virtual void mootTokenLexer::mgl_yy_switch_to_buffer ( void * buf )

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Implements moot::GenericLexer.

References mgl_yy_switch_to_buffer().

Referenced by mgl_yy_switch_to_buffer().

◆ mgl_begin() [1/2]

virtual void mootTokenLexer::mgl_begin ( int stateno )

virtual

Set current start-state. Descendants must override this method (section 3 only!)

Reimplemented from moot::GenericLexer.

Referenced by mgl_yy_switch_to_buffer().

◆ yy_initialize() [2/2]

void mootTokenLexer::yy_initialize ( )

private

◆ input() [2/2]

int mootTokenLexer::input ( )

private

◆ yyinput() [2/2]

int mootTokenLexer::yyinput ( )

inlineprivate

◆ yy_get_next_buffer() [2/2]

int mootTokenLexer::yy_get_next_buffer ( )

private

◆ yyunput() [2/2]

void mootTokenLexer::yyunput	(	unsigned char	c,
		unsigned char *	buf_ptr
	)

private

◆ yy_get_previous_state_() [2/2]

long mootTokenLexer::yy_get_previous_state_ ( void )

private

◆ yy_try_NUL_trans_() [2/2]

long mootTokenLexer::yy_try_NUL_trans_ ( long current_state_ )

private

◆ yyrestart() [2/2]

void mootTokenLexer::yyrestart ( FILE * input_file )

protected

◆ yy_switch_to_buffer() [2/2]

void mootTokenLexer::yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer )

protected

◆ yy_load_buffer_state() [2/2]

void mootTokenLexer::yy_load_buffer_state ( void )

protected

◆ yy_create_buffer() [2/2]

YY_BUFFER_STATE mootTokenLexer::yy_create_buffer	(	FILE *	file,
		int	size
	)

protected

◆ yy_delete_buffer() [2/2]

void mootTokenLexer::yy_delete_buffer ( YY_BUFFER_STATE b )

protected

◆ yy_init_buffer() [2/2]

void mootTokenLexer::yy_init_buffer	(	YY_BUFFER_STATE	b,
		FILE *	file
	)

protected

◆ yy_echo() [2/2]

virtual void mootTokenLexer::yy_echo ( )

protectedvirtual

◆ yy_input() [2/2]

virtual int mootTokenLexer::yy_input	(	char *	buf,
		int &	result,
		int	max_size
	)

protectedvirtual

◆ yy_fatal_error() [2/2]

virtual void mootTokenLexer::yy_fatal_error ( const char * msg )

protectedvirtual

◆ yy_wrap() [2/2]

virtual int mootTokenLexer::yy_wrap ( )

protectedvirtual

◆ yylex() [2/2]

int mootTokenLexer::yylex ( void )

◆ reset() [2/2]

virtual void mootTokenLexer::reset ( void )

virtual

Reset lexer state

Reimplemented from moot::GenericLexer.

◆ on_EOA() [2/2]

void mootTokenLexer::on_EOA ( void )

◆ loc_set() [2/2]

void mootTokenLexer::loc_set	(	moot::OffsetT	off,
		int	len
	)

inline

◆ loc_add() [2/2]

void mootTokenLexer::loc_add ( int len )

inline

◆ mgl_yy_current_buffer_p() [2/2]

virtual void* mootTokenLexer::mgl_yy_current_buffer_p ( void )

inlinevirtual

Reimplemented from moot::GenericLexer.

◆ mgl_yy_create_buffer() [2/2]

virtual void* mootTokenLexer::mgl_yy_create_buffer	(	int	size,
		FILE *	unused = `stdin`
	)

inlinevirtual

Flex++ should define this (backwards), but it doesn't get through to us

Implements moot::GenericLexer.

◆ mgl_yy_init_buffer() [2/2]

virtual void mootTokenLexer::mgl_yy_init_buffer	(	void *	buf,
		FILE *	unused = `stdin`
	)

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Implements moot::GenericLexer.

References mgl_yy_init_buffer(), and yy_create_buffer().

Referenced by mgl_yy_init_buffer().

◆ mgl_yy_delete_buffer() [2/2]

virtual void mootTokenLexer::mgl_yy_delete_buffer ( void * buf )

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Reimplemented from moot::GenericLexer.

◆ mgl_yy_switch_to_buffer() [2/2]

virtual void mootTokenLexer::mgl_yy_switch_to_buffer ( void * buf )

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Implements moot::GenericLexer.

References mgl_begin(), mgl_yy_delete_buffer(), mgl_yy_switch_to_buffer(), yy_delete_buffer(), yy_init_buffer(), and yy_switch_to_buffer().

Referenced by mgl_yy_switch_to_buffer().

◆ mgl_begin() [2/2]

virtual void mootTokenLexer::mgl_begin ( int stateno )

virtual

Set current start-state. Descendants must override this method (section 3 only!)

Reimplemented from moot::GenericLexer.

Member Data Documentation

◆ yy_c_buf_p

unsigned char * mootTokenLexer::yy_c_buf_p

private

◆ yy_hold_char

unsigned char mootTokenLexer::yy_hold_char

private

◆ yy_n_chars

int mootTokenLexer::yy_n_chars

private

◆ yy_init

int mootTokenLexer::yy_init

private

◆ yy_start

int mootTokenLexer::yy_start

private

◆ yy_did_buffer_switch_on_eof

int mootTokenLexer::yy_did_buffer_switch_on_eof

private

◆ yy_current_buffer

YY_BUFFER_STATE mootTokenLexer::yy_current_buffer

protected

Referenced by yyinput(), and ~mootTokenLexer().

◆ yytext

unsigned char * mootTokenLexer::yytext

Referenced by yyinput().

◆ yyleng

int mootTokenLexer::yyleng

Referenced by yyinput(), and ~mootTokenLexer().

◆ yyin

FILE * mootTokenLexer::yyin

Referenced by yyinput().

◆ yyout

FILE * mootTokenLexer::yyout

Referenced by yyinput().

◆ LexTypeText

static const int mootTokenLexer::LexTypeText = moot::NTokTypes+1

static

◆ LexTypeTag

static const int mootTokenLexer::LexTypeTag = moot::NTokTypes+2

static

◆ LexTypeDetails

static const int mootTokenLexer::LexTypeDetails = moot::NTokTypes+3

static

◆ LexTypeEOA

static const int mootTokenLexer::LexTypeEOA = moot::NTokTypes+4

static

◆ LexTypeEOT

static const int mootTokenLexer::LexTypeEOT = moot::NTokTypes+5

static

◆ LexTypeIgnore

static const int mootTokenLexer::LexTypeIgnore = moot::NTokTypes+6

static

◆ lasttyp

int mootTokenLexer::lasttyp

◆ nextstate

int mootTokenLexer::nextstate

◆ mtoken_default

moot::mootToken mootTokenLexer::mtoken_default

Referenced by moot::TokenReaderNative::TokenReaderNative().

◆ mtoken

moot::mootToken * mootTokenLexer::mtoken

◆ manalysis

moot::mootToken::Analysis * mootTokenLexer::manalysis

◆ ignore_comments

bool mootTokenLexer::ignore_comments

◆ first_analysis_is_best

bool mootTokenLexer::first_analysis_is_best

Referenced by moot::TokenReaderNative::input_is_tagged().

◆ current_analysis_is_best

bool mootTokenLexer::current_analysis_is_best

◆ ignore_first_analysis

bool mootTokenLexer::ignore_first_analysis

Referenced by moot::TokenReaderNative::input_is_tagged().

◆ ignore_current_analysis

bool mootTokenLexer::ignore_current_analysis

◆ parse_location

bool mootTokenLexer::parse_location

Referenced by moot::TokenReaderNative::input_has_locations().

◆ parse_analysis_cost

bool mootTokenLexer::parse_analysis_cost

Referenced by moot::TokenReaderNative::input_has_cost().

◆ analysis_cost_details

bool mootTokenLexer::analysis_cost_details

Referenced by moot::TokenReaderNative::input_has_cost().

The documentation for this class was generated from the following files:

Public Types

Public Member Functions

Public Attributes

Static Public Attributes

Protected Member Functions

Protected Attributes

Private Member Functions

Private Attributes

Detailed Description

Member Typedef Documentation

◆ TokenType [1/2]

◆ TokenType [2/2]

Constructor & Destructor Documentation

◆ mootTokenLexer() [1/2]

◆ ~mootTokenLexer() [1/2]

◆ mootTokenLexer() [2/2]

◆ ~mootTokenLexer() [2/2]

Member Function Documentation

◆ yy_initialize() [1/2]

◆ input() [1/2]

◆ yyinput() [1/2]

◆ yy_get_next_buffer() [1/2]

◆ yyunput() [1/2]

◆ yy_get_previous_state_() [1/2]

◆ yy_try_NUL_trans_() [1/2]

◆ yyrestart() [1/2]

◆ yy_switch_to_buffer() [1/2]

◆ yy_load_buffer_state() [1/2]

◆ yy_create_buffer() [1/2]

◆ yy_delete_buffer() [1/2]

◆ yy_init_buffer() [1/2]

◆ yy_echo() [1/2]

◆ yy_input() [1/2]

◆ yy_fatal_error() [1/2]

◆ yy_wrap() [1/2]

◆ yylex() [1/2]

◆ reset() [1/2]

◆ on_EOA() [1/2]

◆ loc_set() [1/2]

◆ loc_add() [1/2]

◆ mgl_yy_current_buffer_p() [1/2]

◆ mgl_yy_create_buffer() [1/2]

◆ mgl_yy_init_buffer() [1/2]

◆ mgl_yy_delete_buffer() [1/2]

◆ mgl_yy_switch_to_buffer() [1/2]

◆ mgl_begin() [1/2]

◆ yy_initialize() [2/2]

◆ input() [2/2]

◆ yyinput() [2/2]

◆ yy_get_next_buffer() [2/2]

◆ yyunput() [2/2]

◆ yy_get_previous_state_() [2/2]

◆ yy_try_NUL_trans_() [2/2]

◆ yyrestart() [2/2]

◆ yy_switch_to_buffer() [2/2]

◆ yy_load_buffer_state() [2/2]

◆ yy_create_buffer() [2/2]

◆ yy_delete_buffer() [2/2]

◆ yy_init_buffer() [2/2]

◆ yy_echo() [2/2]

◆ yy_input() [2/2]

◆ yy_fatal_error() [2/2]

◆ yy_wrap() [2/2]

◆ yylex() [2/2]

◆ reset() [2/2]

◆ on_EOA() [2/2]

◆ loc_set() [2/2]

◆ loc_add() [2/2]

◆ mgl_yy_current_buffer_p() [2/2]

◆ mgl_yy_create_buffer() [2/2]

◆ mgl_yy_init_buffer() [2/2]

◆ mgl_yy_delete_buffer() [2/2]

◆ mgl_yy_switch_to_buffer() [2/2]

◆ mgl_begin() [2/2]

Member Data Documentation

◆ yy_c_buf_p

◆ yy_hold_char

◆ yy_n_chars

◆ yy_init

◆ yy_start