moot::TokenReaderNative Class Reference

Class for native "cooked" text-format token input.

Inheritance diagram for moot::TokenReaderNative:

Collaboration diagram for moot::TokenReaderNative:

Public Member Functions
Constructors etc.
	TokenReaderNative (int fmt=tiofWellDone, const std::string &name="TokenReaderNative")
virtual	~TokenReaderNative (void)
Input Selection
virtual void	from_mstream (mootio::mistream *mis)
Overrides
virtual mootTokenType	get_token (void)
virtual mootTokenType	get_sentence (void)
Diagnostics
virtual size_t	line_number (void)
virtual size_t	line_number (size_t n)
virtual size_t	column_number (void)
virtual size_t	column_number (size_t n)
virtual mootio::ByteOffset	byte_number (void)
virtual mootio::ByteOffset	byte_number (mootio::ByteOffset n)
New Methods
bool	input_is_tagged (void)
bool	input_is_tagged (bool is_tagged)
bool	input_has_locations (void)
bool	input_has_locations (bool has_locs)
bool	input_has_cost (void)
bool	input_has_cost (bool has_cost)
Public Member Functions inherited from moot::TokenReader
	TokenReader (int fmt=tiofUnknown, const std::string &name="TokenReader")
virtual	~TokenReader (void)
void	tr_clear (void)
virtual void	from_mstream (mootio::mistream &mis)
virtual void	from_filename (const char *filename)
virtual void	from_file (FILE *file)
virtual void	from_fd (int fd)
virtual void	from_buffer (const void *buf, size_t len)
virtual void	from_string (const char *s)
virtual void	from_cxxstream (std::istream &is)
virtual void	close (void)
virtual bool	opened (void)
mootToken *	token (void)
mootSentence *	sentence (void)
virtual void	reader_name (const std::string &myname)
virtual mootio::ByteOffset	byte_number (size_t n)
virtual void	carp (const char *fmt,...)

Public Attributes
mootTokenLexer	lexer
mootSentence	trn_sentence
Public Attributes inherited from moot::TokenReader
int	tr_format
std::string	tr_name
mootio::mistream *	tr_istream
bool	tr_istream_created
mootToken *	tr_token
mootSentence *	tr_sentence
void *	tr_data

Additional Inherited Members
Static Public Member Functions inherited from moot::TokenIO
static int	parse_format_string (const std::string &fmtString)
static int	guess_filename_format (const char *filename)
static bool	is_empty_format (int fmt)
static int	sanitize_format (int fmt, int fmt_implied=tiofNone, int fmt_default=tiofNone)
static int	parse_format_request (const char *request, const char *filename=__null, int fmt_implied=tiofNone, int fmt_default=tiofNone)
static std::string	format_canonical_string (int fmt)
static class TokenReader *	new_reader (int fmt)
static class TokenWriter *	new_writer (int fmt)
static class TokenReader *	file_reader (const char *filename, const char *fmt_request=__null, int fmt_implied=tiofNone, int fmt_default=tiofNone)
static class TokenWriter *	file_writer (const char *filename, const char *fmt_request=__null, int fmt_implied=tiofNone, int fmt_default=tiofNone)
static size_t	pipe_tokens (class TokenReader *reader, class TokenWriter *writer)
static size_t	pipe_sentences (class TokenReader *reader, class TokenWriter *writer)
Static Public Attributes inherited from moot::TokenReader
static const size_t	TR_DEFAULT_BUFSIZE = 256

Constructor & Destructor Documentation

TokenReaderNative()

moot::TokenReaderNative::TokenReaderNative ( int  fmt = tiofWellDone, const std::string &  name = "TokenReaderNative"  )

inline

Default constructor

Parameters

fmt	bitmask of moot::TokenIOFormat flags.
name	name of input source, for diagnostics.

References mootTokenLexer::mtoken_default, moot::tiofCost, moot::tiofLocation, moot::tiofNative, moot::tiofTagged, and moot::GenericLexer::to_file().

~TokenReaderNative()

virtual moot::TokenReaderNative::~TokenReaderNative ( void  )

inlinevirtual

Default destructor

Member Function Documentation

from_mstream()

virtual void moot::TokenReaderNative::from_mstream ( mootio::mistream *  mis )

virtual

Select input from a mootio::mstream object.

Reimplemented from moot::TokenReader.

get_token()

virtual mootTokenType moot::TokenReaderNative::get_token ( void  )

virtual

Get the next token from the buffer. On completion, current token (if any) is in *tr_token. Descendants must override this method.

Reimplemented from moot::TokenReader.

get_sentence()

virtual mootTokenType moot::TokenReaderNative::get_sentence ( void  )

virtual

Read in next sentence. On completion, current sentence (if any) is in *tr_sentence. Descendants may override this method for sentence-wise input.

Reimplemented from moot::TokenReader.

line_number() [1/2]

virtual size_t moot::TokenReaderNative::line_number ( void  )

inlinevirtual

Get current line number.

Reimplemented from moot::TokenReader.

References moot::GenericLexer::theLine.

line_number() [2/2]

virtual size_t moot::TokenReaderNative::line_number ( size_t  n )

inlinevirtual

Set current line number.

Reimplemented from moot::TokenReader.

References moot::GenericLexer::theLine.

column_number() [1/2]

virtual size_t moot::TokenReaderNative::column_number ( void  )

inlinevirtual

Get current column number.

Reimplemented from moot::TokenReader.

References moot::GenericLexer::theColumn.

column_number() [2/2]

virtual size_t moot::TokenReaderNative::column_number ( size_t  n )

inlinevirtual

Set current column number.

Reimplemented from moot::TokenReader.

References moot::GenericLexer::theColumn.

byte_number() [1/2]

virtual mootio::ByteOffset moot::TokenReaderNative::byte_number ( void  )

inlinevirtual

Get current byte offset.

Reimplemented from moot::TokenReader.

References moot::GenericLexer::theByte.

byte_number() [2/2]

virtual mootio::ByteOffset moot::TokenReaderNative::byte_number ( mootio::ByteOffset  n )

inlinevirtual

Set current byte offset.

References moot::GenericLexer::theByte.

input_is_tagged() [1/2]

bool moot::TokenReaderNative::input_is_tagged ( void  )

inline

Get value of the 'tagged' flag : whether we think the input has been tagged with initial best-tags.

References mootTokenLexer::first_analysis_is_best.

input_is_tagged() [2/2]

bool moot::TokenReaderNative::input_is_tagged ( bool  is_tagged )

inline

Set value of the 'tagged' flag: whether we think the input has been tagged with initial best-tags.

References mootTokenLexer::first_analysis_is_best, mootTokenLexer::ignore_first_analysis, and moot::tiofTagged.

input_has_locations() [1/2]

bool moot::TokenReaderNative::input_has_locations ( void  )

inline

Get value of the 'locations' flag : whether we think the input has been marked with 'offset length' pairs.

References mootTokenLexer::parse_location.

input_has_locations() [2/2]

bool moot::TokenReaderNative::input_has_locations ( bool  has_locs )

inline

Set value of the 'locations' flag: whether we think the input has been marked with 'offset length' pairs.

References mootTokenLexer::parse_location, and moot::tiofLocation.

input_has_cost() [1/2]

bool moot::TokenReaderNative::input_has_cost ( void  )

inline

Get value of the 'cost' flag : whether we want to parse analysis costs (prob field) from input stream

References mootTokenLexer::parse_analysis_cost.

input_has_cost() [2/2]

bool moot::TokenReaderNative::input_has_cost ( bool  has_cost )

inline

Set value of the 'cost' flag: whether we want to parse analysis costs (prob field) from input stream

References mootTokenLexer::analysis_cost_details, mootTokenLexer::parse_analysis_cost, and moot::tiofCost.

Member Data Documentation

lexer

mootTokenLexer moot::TokenReaderNative::lexer

The underlying lexer (does the dirty work).

trn_sentence

mootSentence moot::TokenReaderNative::trn_sentence

Default construction buffer for get_sentence()

---

The documentation for this class was generated from the following file:

• mootTokenIO.h