High-level token information object.

Collaboration diagram for moot::mootToken:

Classes
class	Analysis
	Type for a single morphological analysis. More...

class	Location

Public Types
typedef float	Cost

typedef list< Analysis >	Analyses

Public Member Functions
	mootToken (mootTokenType type=TokTypeVanilla)

	mootToken (const mootTokString &text, mootTokenType type=TokTypeVanilla)

	mootToken (const mootTokString &text, const Analyses &analyses)

	mootToken (const mootTokString &text, const Analyses &analyses, const mootTagString &besttag)

	mootToken (const mootToken &x)

	~mootToken (void)

mootToken &	operator= (const mootToken &x)

void	clear (void)

const mootTokString &	text (void) const

mootTokString &	text (const mootTokString &text)

mootTokString &	text (const char *s, size_t len)

mootTokString &	textAppend (const mootTokString &text)

mootTokString &	textAppend (const char *s, size_t len)

const mootTagString &	besttag (void) const

mootTagString &	besttag (const mootTagString &besttag)

mootTagString &	besttagAppend (const mootTagString &text)

mootTagString &	besttagAppend (const char *s, size_t len)

mootTokenType	toktype (void) const

mootTokenType	toktype (const mootTokenType type)

const Analyses &	analyses (void) const

const Analyses &	analyses (const Analyses &analyses)

void	insert (const Analysis &analysis)

void	insert (const mootTagString &tag, const mootTagString &details, ProbT p=0)

void	insert (const char tag, const char details, ProbT p=0)

bool	has_analysis_for_tag (const mootTagString &tag) const

void	erase (const Analysis &analysis)

void	prune (void)

const Location &	location (void) const

const Location &	location (const Location &loc)

const Location &	location (const OffsetT offset, const OffsetT length=0)

OffsetT	loc_offset (void) const

OffsetT	loc_offset (const OffsetT off)

OffsetT	loc_length (void) const

OffsetT	loc_length (const OffsetT len)

void	tokImport (const mootTokString src_toktext=__null, const mootTagSet src_tagset=__null)

void	tokExport (mootTokString dst_toktext=__null, mootTagSet dst_tagset=__null, bool want_besttag_in_tagset=true) const

void	dump (const char label=__null, FILE f=__null) const

Public Attributes
mootTokenType	tok_type

mootTokString	tok_text

mootTagString	tok_besttag

Analyses	tok_analyses

Location	tok_location

void *	tok_data

Friends
bool	operator== (const mootToken &x, const mootToken &y)

bool	operator< (const mootToken &x, const mootToken &y)

Member Typedef Documentation

◆ Cost

typedef float moot::mootToken::Cost

Type for analysis weights

◆ Analyses

typedef list<Analysis> moot::mootToken::Analyses

Type for multiple concurrent analyses : tag => set<Analysis>

Constructor & Destructor Documentation

◆ mootToken() [1/5]

moot::mootToken::mootToken ( mootTokenType type = TokTypeVanilla )

inline

Default constructor: empty text, no analyses

◆ mootToken() [2/5]

moot::mootToken::mootToken	(	const mootTokString &	text,
		mootTokenType	type = `TokTypeVanilla`
	)

inline

Constructor given only token text: no analyses

◆ mootToken() [3/5]

moot::mootToken::mootToken	(	const mootTokString &	text,
		const Analyses &	analyses
	)

inline

Constructor given text & analyses

◆ mootToken() [4/5]

moot::mootToken::mootToken	(	const mootTokString &	text,
		const Analyses &	analyses,
		const mootTagString &	besttag
	)

inline

Constructor given text & analyses & best tag

◆ mootToken() [5/5]

moot::mootToken::mootToken ( const mootToken & x )

inline

copy constructor

◆ ~mootToken()

moot::mootToken::~mootToken ( void )

inline

Constructor for text-only tokens from C buffer of known length

Member Function Documentation

◆ operator=()

mootToken& moot::mootToken::operator= ( const mootToken & x )

inline

assignment operator

References tok_analyses, tok_besttag, tok_data, tok_location, tok_text, and tok_type.

◆ clear()

void moot::mootToken::clear ( void )

inline

Clear this object (except for tok_data)

References moot::mootToken::Location::clear(), and moot::TokTypeVanilla.

Referenced by moot::TokenReader::tr_clear().

◆ text() [1/3]

const mootTokString& moot::mootToken::text ( void ) const

inline

Get token text

Referenced by moot::mootHMM::viterbi_step().

◆ text() [2/3]

mootTokString& moot::mootToken::text ( const mootTokString & text )

inline

Set token text

◆ text() [3/3]

mootTokString& moot::mootToken::text	(	const char *	s,
		size_t	len
	)

inline

Set token text

◆ textAppend() [1/2]

mootTokString& moot::mootToken::textAppend ( const mootTokString & text )

inline

Append to token text, returns new token-text

◆ textAppend() [2/2]

mootTokString& moot::mootToken::textAppend	(	const char *	s,
		size_t	len
	)

inline

Append to token text, returns new token-text

◆ besttag() [1/2]

const mootTagString& moot::mootToken::besttag ( void ) const

inline

Get best tag

◆ besttag() [2/2]

mootTagString& moot::mootToken::besttag ( const mootTagString & besttag )

inline

Set best tag

◆ besttagAppend() [1/2]

mootTagString& moot::mootToken::besttagAppend ( const mootTagString & text )

inline

Append to 'besttag' text, returns new text

◆ besttagAppend() [2/2]

mootTagString& moot::mootToken::besttagAppend	(	const char *	s,
		size_t	len
	)

inline

Append to 'besttag' text, returns new text

◆ toktype() [1/2]

mootTokenType moot::mootToken::toktype ( void ) const

inline

Get token type

Referenced by moot::mootHMM::viterbi_step().

◆ toktype() [2/2]

mootTokenType moot::mootToken::toktype ( const mootTokenType type )

inline

Set token type

◆ analyses() [1/2]

const Analyses& moot::mootToken::analyses ( void ) const

inline

Get token analyses

◆ analyses() [2/2]

const Analyses& moot::mootToken::analyses ( const Analyses & analyses )

inline

Set token analyses

◆ insert() [1/3]

void moot::mootToken::insert ( const Analysis & analysis )

inline

Insert an analysis

◆ insert() [2/3]

void moot::mootToken::insert	(	const mootTagString &	tag,
		const mootTagString &	details,
		ProbT	p = `0`
	)

inline

Insert a new analysis

◆ insert() [3/3]

void moot::mootToken::insert	(	const char *	tag,
		const char *	details,
		ProbT	p = `0`
	)

inline

Insert a new analysis, C-style

◆ has_analysis_for_tag()

bool moot::mootToken::has_analysis_for_tag ( const mootTagString & tag ) const

inline

Test presence of an analysis for 'tag'

◆ erase()

void moot::mootToken::erase ( const Analysis & analysis )

inline

Remove an analysis

◆ prune()

void moot::mootToken::prune ( void )

inline

Prune analyses, retaining only those for 'besttag'

◆ location() [1/3]

const Location& moot::mootToken::location ( void ) const

inline

Get token location

Referenced by mootTokenLexer::~mootTokenLexer().

◆ location() [2/3]

const Location& moot::mootToken::location ( const Location & loc )

inline

Set token location

◆ location() [3/3]

const Location& moot::mootToken::location	(	const OffsetT	offset,
		const OffsetT	length = `0`
	)

inline

Set token location directly from member variables

◆ loc_offset() [1/2]

OffsetT moot::mootToken::loc_offset ( void ) const

inline

Get token location offset

References moot::mootToken::Location::offset.

◆ loc_offset() [2/2]

OffsetT moot::mootToken::loc_offset ( const OffsetT off )

inline

Set token location offset

References moot::mootToken::Location::offset.

◆ loc_length() [1/2]

OffsetT moot::mootToken::loc_length ( void ) const

inline

Get token location length

References moot::mootToken::Location::length.

Referenced by mootTokenLexer::~mootTokenLexer().

◆ loc_length() [2/2]

OffsetT moot::mootToken::loc_length ( const OffsetT len )

inline

Set token location length

References moot::mootToken::Location::length.

◆ tokImport()

void moot::mootToken::tokImport	(	const mootTokString *	src_toktext = `__null`,
		const mootTagSet *	src_tagset = `__null`
	)

inline

For backwards-compatibility: convert old-style 'text,tags' pair to a mootToken.

Warning: current analysis-set is NOT cleared by this method.

◆ tokExport()

void moot::mootToken::tokExport	(	mootTokString *	dst_toktext = `__null`,
		mootTagSet *	dst_tagset = `__null`,
		bool	want_besttag_in_tagset = `true`
	)		const

inline

For lazy backwards-compatibility: convert mootToken back to old-style 'text,tagset' pair.

Parameters

dst_toktext	pointer to destination token-text. NULL (default) means don't export text.
dst_tagset	pointer to destination tagset. NULL (default) means don't export tagset.
want_besttag_in_tagset	whether to include besttag in exported tagset. Default=true.

Warning: argument tagset is NOT cleared by this method.

◆ dump()

void moot::mootToken::dump	(	const char *	label = `__null`,
		FILE *	f = `__null`
	)		const

dump token data in human-readable format to file f (default=stderr)

Friends And Related Function Documentation

◆ operator==

bool operator==	(	const mootToken &	x,
		const mootToken &	y
	)

friend

Equality operator

◆ operator<

bool operator<	(	const mootToken &	x,
		const mootToken &	y
	)

friend

Comparison operaor

Member Data Documentation

◆ tok_type

mootTokenType moot::mootToken::tok_type

Token content type.

Referenced by operator=().

◆ tok_text

mootTokString moot::mootToken::tok_text

Literal token text.

Referenced by operator=().

◆ tok_besttag

mootTagString moot::mootToken::tok_besttag

Best tag for this token.

Referenced by operator=().

◆ tok_analyses

Analyses moot::mootToken::tok_analyses

Set of possible analyses as a mootToken::AnalysisSet

Referenced by operator=().

◆ tok_location

Location moot::mootToken::tok_location

Token location in input stream, for backtraces (optional)

Referenced by operator=().

◆ tok_data

void* moot::mootToken::tok_data

Additional user data for this token, defaults to NULL. User is responsible for memory management.

Referenced by operator=().

The documentation for this class was generated from the following file:

mootToken.h

Classes

Public Types

Public Member Functions

Public Attributes

Friends

Member Typedef Documentation

◆ Cost

◆ Analyses

Constructor & Destructor Documentation

◆ mootToken() [1/5]

◆ mootToken() [2/5]

◆ mootToken() [3/5]

◆ mootToken() [4/5]

◆ mootToken() [5/5]

◆ ~mootToken()

Member Function Documentation

◆ operator=()

◆ clear()

◆ text() [1/3]

◆ text() [2/3]

◆ text() [3/3]

◆ textAppend() [1/2]

◆ textAppend() [2/2]

◆ besttag() [1/2]

◆ besttag() [2/2]

◆ besttagAppend() [1/2]

◆ besttagAppend() [2/2]

◆ toktype() [1/2]

◆ toktype() [2/2]

◆ analyses() [1/2]

◆ analyses() [2/2]

◆ insert() [1/3]

◆ insert() [2/3]

◆ insert() [3/3]

◆ has_analysis_for_tag()

◆ erase()

◆ prune()

◆ location() [1/3]

◆ location() [2/3]

◆ location() [3/3]

◆ loc_offset() [1/2]

◆ loc_offset() [2/2]

◆ loc_length() [1/2]

◆ loc_length() [2/2]

◆ tokImport()

◆ tokExport()

◆ dump()

Friends And Related Function Documentation

◆ operator==

◆ operator<

Member Data Documentation

◆ tok_type

◆ tok_text

◆ tok_besttag

◆ tok_analyses

◆ tok_location

◆ tok_data