mootPPLexer Class Reference

flex++ lexer for 'mootpp' rudimentary raw-text preprocessor More...

Inheritance diagram for mootPPLexer:

Collaboration diagram for mootPPLexer:

Public Types
enum	TokenType {   PPEOF, UNKNOWN, EOS, XML_START_TAG,   XML_END_TAG, XML_ENTITY, WORD, INTEGER,   FLOAT, PUNCT }

Public Member Functions
int	yylex (void)
	mootPPLexer ()
virtual	~mootPPLexer (void)
virtual void *	mgl_yy_current_buffer_p (void)
virtual void *	mgl_yy_create_buffer (int size, FILE *unused=stdin)
virtual void	mgl_yy_init_buffer (void *buf, FILE *unused=stdin)
virtual void	mgl_yy_delete_buffer (void *buf)
virtual void	mgl_yy_switch_to_buffer (void *buf)
virtual void	mgl_begin (int stateno)
Public Member Functions inherited from moot::GenericLexer
	GenericLexer (const std::string &myname="moot::GenericLexer", size_t line=0, size_t column=0, mootio::ByteOffset byte=0)
virtual	~GenericLexer (void)
virtual void	reset (void)
virtual void	clear (bool clear_input=true, bool clear_output=true)
virtual void	from_mstream (mootio::mistream *in=__null)
virtual void	from_filename (const std::string &filename)
virtual void	from_file (FILE *in=stdin)
virtual void	from_buffer (const char *buf, size_t len)
void	from_string (const char *s)
void	from_string (const std::string &s)
void	select_streams (FILE *in, FILE *out=stdout, const char *myname=__null)
void	select_string (const char *in, FILE *out=stderr, const char *myname=__null)
virtual void	to_mstream (mootio::mostream *out=__null)
virtual void	to_filename (const std::string &filename)
virtual void	to_file (FILE *out=stdout)
int	yyinput (char *buffer, int &result, int max_size)
void	tokbuf_append (const char *text, size_t len)
void	add_columns (ByteOffset len)
void	add_lines (ByteOffset len=1)
virtual void	yycarp (const char *fmt,...)

Public Attributes
unsigned char *	yytext
int	yyleng
FILE *	yyin
FILE *	yyout
bool	verbose
unsigned int	ntokens
std::string	output_sentence_separator
std::string	output_token_separator
Public Attributes inherited from moot::GenericLexer
mootio::mistream *	mglin
mootio::mostream *	mglout
bool	mgl_in_created
bool	mgl_out_created
size_t	theLine
size_t	theColumn
mootio::ByteOffset	theByte
std::string	lexname
std::string	tokbuf
bool	tokbuf_clear

Protected Member Functions
void	yyrestart (FILE *input_file)
void	yy_switch_to_buffer (YY_BUFFER_STATE new_buffer)
void	yy_load_buffer_state (void)
YY_BUFFER_STATE	yy_create_buffer (FILE *file, int size)
void	yy_delete_buffer (YY_BUFFER_STATE b)
void	yy_init_buffer (YY_BUFFER_STATE b, FILE *file)
virtual void	yy_echo ()
virtual int	yy_input (char *buf, int &result, int max_size)
virtual void	yy_fatal_error (const char *msg)
virtual int	yy_wrap ()

Protected Attributes
YY_BUFFER_STATE	yy_current_buffer

Private Member Functions
void	yy_initialize ()
int	input ()
int	yyinput ()
int	yy_get_next_buffer ()
void	yyunput (unsigned char c, unsigned char *buf_ptr)
long	yy_get_previous_state_ (void)
long	yy_try_NUL_trans_ (long current_state_)

Private Attributes
unsigned char *	yy_c_buf_p
unsigned char	yy_hold_char
int	yy_n_chars
int	yy_init
int	yy_start
int	yy_did_buffer_switch_on_eof

Additional Inherited Members
Static Public Attributes inherited from moot::GenericLexer
static const int	MGL_DEFAULT_BUFFER_SIZE = 16384

Detailed Description

Does rudimentary end-of-sentence and abbreviation recognition, and filters out (most) XML markup.

Member Enumeration Documentation

TokenType

enum mootPPLexer::TokenType

Enumerator
PPEOF
UNKNOWN
EOS
XML_START_TAG
XML_END_TAG
XML_ENTITY
WORD
INTEGER
FLOAT
PUNCT

Constructor & Destructor Documentation

mootPPLexer()

mootPPLexer::mootPPLexer

(

)

~mootPPLexer()

virtual mootPPLexer::~mootPPLexer ( void  )

inlinevirtual

References ~mootPPLexer().

Referenced by ~mootPPLexer().

Member Function Documentation

yy_initialize()

void mootPPLexer::yy_initialize ( )

private

input()

int mootPPLexer::input ( )

private

yyinput()

int mootPPLexer::yyinput ( )

inlineprivate

yy_get_next_buffer()

int mootPPLexer::yy_get_next_buffer ( )

private

yyunput()

void mootPPLexer::yyunput ( unsigned char  c, unsigned char *  buf_ptr  )

private

yy_get_previous_state_()

long mootPPLexer::yy_get_previous_state_ ( void  )

private

yy_try_NUL_trans_()

long mootPPLexer::yy_try_NUL_trans_ ( long  current_state_ )

private

yyrestart()

void mootPPLexer::yyrestart ( FILE *  input_file )

protected

yy_switch_to_buffer()

void mootPPLexer::yy_switch_to_buffer ( YY_BUFFER_STATE  new_buffer )

protected

yy_load_buffer_state()

void mootPPLexer::yy_load_buffer_state ( void  )

protected

yy_create_buffer()

YY_BUFFER_STATE mootPPLexer::yy_create_buffer ( FILE *  file, int  size  )

protected

yy_delete_buffer()

void mootPPLexer::yy_delete_buffer ( YY_BUFFER_STATE  b )

protected

yy_init_buffer()

void mootPPLexer::yy_init_buffer ( YY_BUFFER_STATE  b, FILE *  file  )

protected

yy_echo()

virtual void mootPPLexer::yy_echo ( )

protectedvirtual

yy_input()

virtual int mootPPLexer::yy_input ( char *  buf, int &  result, int  max_size  )

protectedvirtual

yy_fatal_error()

virtual void mootPPLexer::yy_fatal_error ( const char *  msg )

protectedvirtual

yy_wrap()

virtual int mootPPLexer::yy_wrap ( )

protectedvirtual

yylex()

int mootPPLexer::yylex

(

void

)

mgl_yy_current_buffer_p()

virtual void* mootPPLexer::mgl_yy_current_buffer_p ( void  )

inlinevirtual

Reimplemented from moot::GenericLexer.

mgl_yy_create_buffer()

virtual void* mootPPLexer::mgl_yy_create_buffer ( int  size, FILE *  unused = stdin  )

inlinevirtual

Flex++ should define this (backwards), but it doesn't get through to us

Implements moot::GenericLexer.

mgl_yy_init_buffer()

virtual void mootPPLexer::mgl_yy_init_buffer ( void *  buf, FILE *  unused = stdin  )

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Implements moot::GenericLexer.

References mgl_yy_init_buffer().

Referenced by mgl_yy_init_buffer().

mgl_yy_delete_buffer()

virtual void mootPPLexer::mgl_yy_delete_buffer ( void *  buf )

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Reimplemented from moot::GenericLexer.

mgl_yy_switch_to_buffer()

virtual void mootPPLexer::mgl_yy_switch_to_buffer ( void *  buf )

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Implements moot::GenericLexer.

References mgl_yy_switch_to_buffer().

Referenced by mgl_yy_switch_to_buffer().

mgl_begin()

virtual void mootPPLexer::mgl_begin ( int  stateno )

virtual

Set current start-state. Descendants must override this method (section 3 only!)

Reimplemented from moot::GenericLexer.

Member Data Documentation

yy_c_buf_p

unsigned char* mootPPLexer::yy_c_buf_p

private

yy_hold_char

unsigned char mootPPLexer::yy_hold_char

private

yy_n_chars

int mootPPLexer::yy_n_chars

private

yy_init

int mootPPLexer::yy_init

private

yy_start

int mootPPLexer::yy_start

private

yy_did_buffer_switch_on_eof

int mootPPLexer::yy_did_buffer_switch_on_eof

private

yy_current_buffer

YY_BUFFER_STATE mootPPLexer::yy_current_buffer

protected

yytext

unsigned char* mootPPLexer::yytext

yyleng

int mootPPLexer::yyleng

yyin

FILE* mootPPLexer::yyin

yyout

FILE * mootPPLexer::yyout

verbose

bool mootPPLexer::verbose

ntokens

unsigned int mootPPLexer::ntokens

output_sentence_separator

std::string mootPPLexer::output_sentence_separator

output_token_separator

std::string mootPPLexer::output_token_separator

---

The documentation for this class was generated from the following file:

• mootPPLexer.h