Abstract base class for Flex++ lexers. More...

Inheritance diagram for moot::GenericLexer:

[legend]

Collaboration diagram for moot::GenericLexer:

Public Member Functions
Constructors etc.
	GenericLexer (const std::string &myname="moot::GenericLexer", size_t line=0, size_t column=0, mootio::ByteOffset byte=0)

virtual	~GenericLexer (void)

virtual void	reset (void)

virtual void	clear (bool clear_input=true, bool clear_output=true)

Required Wrapper Methods
virtual void *	mgl_yy_current_buffer_p (void)

virtual void	mgl_begin (int stateno)

virtual void	mgl_yy_delete_buffer (void *buf)

virtual void *	mgl_yy_create_buffer (int size, FILE *unused=stdin)=0

virtual void	mgl_yy_switch_to_buffer (void *buf)=0

virtual void	mgl_yy_init_buffer (void buf, FILE unused=stdin)=0

Input selection
virtual void	from_mstream (mootio::mistream *in=__null)

virtual void	from_filename (const std::string &filename)

virtual void	from_file (FILE *in=stdin)

virtual void	from_buffer (const char *buf, size_t len)

void	from_string (const char *s)

void	from_string (const std::string &s)

void	select_streams (FILE in, FILE out=stdout, const char *myname=__null)

void	select_string (const char in, FILE out=stderr, const char *myname=__null)

Output selection
virtual void	to_mstream (mootio::mostream *out=__null)

virtual void	to_filename (const std::string &filename)

virtual void	to_file (FILE *out=stdout)

Input Reverse-Override Hacks
int	yyinput (char *buffer, int &result, int max_size)

Token Buffering
void	tokbuf_append (const char *text, size_t len)

Location-tracking
void	add_columns (ByteOffset len)

void	add_lines (ByteOffset len=1)

Diagnostics
virtual void	yycarp (const char *fmt,...)

Public Attributes
Stream Parameters
mootio::mistream *	mglin

mootio::mostream *	mglout

bool	mgl_in_created

bool	mgl_out_created

Positional Parameters
size_t	theLine

size_t	theColumn

mootio::ByteOffset	theByte

Diagnostic Data
std::string	lexname

Low-level data
std::string	tokbuf

bool	tokbuf_clear

Static Public Attributes
static const int	MGL_DEFAULT_BUFFER_SIZE = 16384

Detailed Description

Descendant lexer specifications should include the following:

%header{
#include <mootGenericLexer.h>
using namespace moot;
%}
%define CLASS myLexer

%define INHERIT   : public GenericLexer
r

%define INPUT_CODE   return moot::GenericLexer::yyinput(buffer,result,max_size);
;

%define MEMBERS   / * ... local members ... * /   / *----- moot::GenericLexer helpers ----- * /   virtual void  *mgl_yy_current_buffer_p(void)                  {return (&yy_current_buffer);};  virtual void  *mgl_yy_create_buffer(int size, FILE *unused=stdin)                  {return (void*)(yy_create_buffer(unused,size));};  virtual void   mgl_yy_init_buffer(void *buf, FILE *unused=stdin)                  {yy_init_buffer((YY_BUFFER_STATE)buf,unused);};  virtual void   mgl_yy_delete_buffer(void *buf)                  {yy_delete_buffer((YY_BUFFER_STATE)buf);};  virtual void   mgl_yy_switch_to_buffer(void *buf)                  {yy_switch_to_buffer((YY_BUFFER_STATE)buf);};  virtual void   mgl_begin(int stateno);   //...
gin(int stateno); \
  //...

%define CONSTRUCTOR_INIT :   moot::GenericLexer("myLexer")   // ...
...

%%
%{
//-- rules go here ...
%}
%%
void myLexer::mgl_begin(int stateno) {BEGIN(stateno);}

Constructor & Destructor Documentation

◆ GenericLexer()

moot::GenericLexer::GenericLexer	(	const std::string &	myname = `"moot::GenericLexer"`,
		size_t	line = `0`,
		size_t	column = `0`,
		mootio::ByteOffset	byte = `0`
	)

inline

Default constructor

◆ ~GenericLexer()

virtual moot::GenericLexer::~GenericLexer ( void )

virtual

Destructor

Member Function Documentation

◆ reset()

virtual void moot::GenericLexer::reset ( void )

virtual

Reset lexer state

Reimplemented in moot::wasteScanner, mootTokenLexer, and mootTokenLexer.

◆ clear()

virtual void moot::GenericLexer::clear	(	bool	clear_input = `true`,
		bool	clear_output = `true`
	)

virtual

Clear any selected streams

◆ mgl_yy_current_buffer_p()

virtual void* moot::GenericLexer::mgl_yy_current_buffer_p ( void )

virtual

Reimplemented in mootTokenLexer, mootTokenLexer, mootClassfreqsLexer, mootLexfreqsLexer, mootNgramsLexer, and mootPPLexer.

◆ mgl_begin()

virtual void moot::GenericLexer::mgl_begin ( int stateno )

virtual

Set current start-state. Descendants must override this method (section 3 only!)

Reimplemented in mootTokenLexer, mootTokenLexer, mootClassfreqsLexer, mootLexfreqsLexer, mootNgramsLexer, and mootPPLexer.

◆ mgl_yy_delete_buffer()

virtual void moot::GenericLexer::mgl_yy_delete_buffer ( void * buf )

inlinevirtual

Flex++ should define this, but it doesn't get through to us

Reimplemented in mootTokenLexer, mootTokenLexer, mootClassfreqsLexer, mootLexfreqsLexer, mootNgramsLexer, and mootPPLexer.

◆ mgl_yy_create_buffer()

virtual void* moot::GenericLexer::mgl_yy_create_buffer	(	int	size,
		FILE *	unused = `stdin`
	)

pure virtual

Flex++ should define this (backwards), but it doesn't get through to us

Implemented in mootTokenLexer, mootTokenLexer, mootClassfreqsLexer, mootLexfreqsLexer, mootNgramsLexer, mootPPLexer, and moot::wasteScanner.

◆ mgl_yy_switch_to_buffer()

virtual void moot::GenericLexer::mgl_yy_switch_to_buffer ( void * buf )

pure virtual

Flex++ should define this, but it doesn't get through to us

Implemented in mootTokenLexer, mootTokenLexer, mootClassfreqsLexer, mootLexfreqsLexer, mootNgramsLexer, mootPPLexer, and moot::wasteScanner.

◆ mgl_yy_init_buffer()

virtual void moot::GenericLexer::mgl_yy_init_buffer	(	void *	buf,
		FILE *	unused = `stdin`
	)

pure virtual

Flex++ should define this, but it doesn't get through to us

Implemented in mootTokenLexer, mootTokenLexer, mootClassfreqsLexer, mootLexfreqsLexer, mootNgramsLexer, mootPPLexer, and moot::wasteScanner.

◆ from_mstream()

virtual void moot::GenericLexer::from_mstream ( mootio::mistream * in = __null )

virtual

Select mootio::mstream input

Reimplemented in moot::wasteScanner.

◆ from_filename()

virtual void moot::GenericLexer::from_filename ( const std::string & filename )

virtual

Select named file input

◆ from_file()

virtual void moot::GenericLexer::from_file ( FILE * in = stdin )

virtual

Select C-stream input

◆ from_buffer()

virtual void moot::GenericLexer::from_buffer	(	const char *	buf,
		size_t	len
	)

virtual

Select C-buffer input

◆ from_string() [1/2]

void moot::GenericLexer::from_string ( const char * s )

inline

Select C-string input

◆ from_string() [2/2]

void moot::GenericLexer::from_string ( const std::string & s )

inline

Select C++ string input

◆ select_streams()

void moot::GenericLexer::select_streams	(	FILE *	in,
		FILE *	out = `stdout`,
		const char *	myname = `__null`
	)

inline

Backwards-compatible alias

Referenced by moot::mootClassfreqsCompiler::select_streams(), moot::mootNgramsCompiler::select_streams(), and moot::mootLexfreqsCompiler::select_streams().

◆ select_string()

void moot::GenericLexer::select_string	(	const char *	in,
		FILE *	out = `stderr`,
		const char *	myname = `__null`
	)

inline

Backwards-compatible alias

Referenced by moot::mootClassfreqsCompiler::select_string(), moot::mootNgramsCompiler::select_string(), and moot::mootLexfreqsCompiler::select_string().

◆ to_mstream()

virtual void moot::GenericLexer::to_mstream ( mootio::mostream * out = __null )

virtual

Select mootio::mstream output

◆ to_filename()

virtual void moot::GenericLexer::to_filename ( const std::string & filename )

virtual

Select named file output

◆ to_file()

virtual void moot::GenericLexer::to_file ( FILE * out = stdout )

virtual

Select C-stream output

Referenced by moot::TokenReaderNative::TokenReaderNative().

◆ yyinput()

int moot::GenericLexer::yyinput	(	char *	buffer,
		int &	result,
		int	max_size
	)

inline

yyinput() hack, for use with flex++ INPUT_CODE

References mootio::mistream::read().

◆ tokbuf_append()

void moot::GenericLexer::tokbuf_append	(	const char *	text,
		size_t	len
	)

inline

Add some characters to 'tokbuf' buffer

◆ add_columns()

void moot::GenericLexer::add_columns ( ByteOffset len )

inline

Increment both 'theColumn' and 'theByte' by len

◆ add_lines()

void moot::GenericLexer::add_lines ( ByteOffset len = 1 )

inline

Increment both 'theLine' and 'theByte' by len, set theColumn=0

◆ yycarp()

virtual void moot::GenericLexer::yycarp	(	const char *	fmt,
			...
	)

virtual

Complain

Referenced by moot::wasteScanner::mgl_yy_create_buffer(), moot::wasteScanner::mgl_yy_init_buffer(), and moot::wasteScanner::mgl_yy_switch_to_buffer().

Member Data Documentation

◆ MGL_DEFAULT_BUFFER_SIZE

const int moot::GenericLexer::MGL_DEFAULT_BUFFER_SIZE = 16384

static

◆ mglin

mootio::mistream* moot::GenericLexer::mglin

input stream wrapper

◆ mglout

mootio::mostream* moot::GenericLexer::mglout

output stream wrapper

◆ mgl_in_created

bool moot::GenericLexer::mgl_in_created

whether we created mglin

◆ mgl_out_created

bool moot::GenericLexer::mgl_out_created

whether we created mglout

◆ theLine

size_t moot::GenericLexer::theLine

Current line number (counting from BOF==0)

Referenced by moot::wasteTokenScanner::line_number(), and moot::TokenReaderNative::line_number().

◆ theColumn

size_t moot::GenericLexer::theColumn

Current column number

Referenced by moot::wasteTokenScanner::column_number(), and moot::TokenReaderNative::column_number().

◆ theByte

mootio::ByteOffset moot::GenericLexer::theByte

Current byte number

Referenced by moot::wasteTokenScanner::byte_number(), and moot::TokenReaderNative::byte_number().

◆ lexname

std::string moot::GenericLexer::lexname

symbolic name of lexer (or program)

◆ tokbuf

std::string moot::GenericLexer::tokbuf

used for token-buffering

◆ tokbuf_clear

bool moot::GenericLexer::tokbuf_clear

whether to clear tokbuf on next tokbuf_append()

The documentation for this class was generated from the following file:

mootGenericLexer.h

Public Member Functions

Public Attributes

Static Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ GenericLexer()

◆ ~GenericLexer()

Member Function Documentation

◆ reset()

◆ clear()

◆ mgl_yy_current_buffer_p()

◆ mgl_begin()

◆ mgl_yy_delete_buffer()

◆ mgl_yy_create_buffer()

◆ mgl_yy_switch_to_buffer()

◆ mgl_yy_init_buffer()

◆ from_mstream()

◆ from_filename()

◆ from_file()

◆ from_buffer()

◆ from_string() [1/2]

◆ from_string() [2/2]

◆ select_streams()

◆ select_string()

◆ to_mstream()

◆ to_filename()

◆ to_file()

◆ yyinput()

◆ tokbuf_append()

◆ add_columns()

◆ add_lines()

◆ yycarp()

Member Data Documentation

◆ MGL_DEFAULT_BUFFER_SIZE

◆ mglin

◆ mglout

◆ mgl_in_created

◆ mgl_out_created

◆ theLine

◆ theColumn

◆ theByte

◆ lexname

◆ tokbuf

◆ tokbuf_clear