Main Page   Namespace List   Class Hierarchy   Alphabetical List   Compound List   File List   Namespace Members   Compound Members   File Members  

mootTokenLexer Class Reference

Flex++ lexer for moot PoS tagger native text input. More...

#include <mootTokenLexer.h>

Inheritance diagram for mootTokenLexer:

Inheritance graph
[legend]
Collaboration diagram for mootTokenLexer:

Collaboration graph
[legend]
List of all members.

Public Types

Public Methods

Public Attributes

Static Public Attributes

Protected Methods

Protected Attributes

Private Methods

Private Attributes


Detailed Description

Assumes pre-tokenized input: one token per line, blank lines = EOS, raw text only (no markup!). Format: 1 tok/line, comments introduced with '%%'. Supports multiple tags/tok.

token-line format (TAB-separated):

TOKEN_TEXT ANALYSIS_1 ... ANALYSIS_N

Analysis format(s):

...(COST?)... "[" TAG ["]"|" "] ...(COST?)...

COST format(s):

"<" COST_FLOAT ">"


Member Typedef Documentation

typedef moot::mootTokenType mootTokenLexer::TokenType
 


Constructor & Destructor Documentation

mootTokenLexer::mootTokenLexer  
 

virtual mootTokenLexer::~mootTokenLexer void    [inline, virtual]
 


Member Function Documentation

int mootTokenLexer::input   [private]
 

virtual void mootTokenLexer::mgl_begin int    stateno [virtual]
 

Set current start-state. Descendants must override this method (section 3 only!)

Reimplemented from moot::GenericLexer.

virtual void* mootTokenLexer::mgl_yy_create_buffer int    size,
FILE *    unused = stdin
[inline, virtual]
 

Flex++ should define this (backwards), but it doesn't get through to us

Implements moot::GenericLexer.

virtual void** mootTokenLexer::mgl_yy_current_buffer_p void    [inline, virtual]
 

Reimplemented from moot::GenericLexer.

virtual void mootTokenLexer::mgl_yy_delete_buffer void *    buf [inline, virtual]
 

Flex++ should define this, but it doesn't get through to us

Reimplemented from moot::GenericLexer.

virtual void mootTokenLexer::mgl_yy_init_buffer void *    buf,
FILE *    unused = stdin
[inline, virtual]
 

Flex++ should define this, but it doesn't get through to us

Implements moot::GenericLexer.

virtual void mootTokenLexer::mgl_yy_switch_to_buffer void *    buf [inline, virtual]
 

Flex++ should define this, but it doesn't get through to us

Implements moot::GenericLexer.

void mootTokenLexer::on_EOA void    [inline]
 

virtual void mootTokenLexer::reset void    [virtual]
 

Reset lexer state

Reimplemented from moot::GenericLexer.

YY_BUFFER_STATE mootTokenLexer::yy_create_buffer FILE *    file,
int    size
[protected]
 

void mootTokenLexer::yy_delete_buffer YY_BUFFER_STATE    b [protected]
 

virtual void mootTokenLexer::yy_echo   [protected, virtual]
 

virtual void mootTokenLexer::yy_fatal_error char *    msg [protected, virtual]
 

int mootTokenLexer::yy_get_next_buffer   [private]
 

long mootTokenLexer::yy_get_previous_state_ void    [private]
 

void mootTokenLexer::yy_init_buffer YY_BUFFER_STATE    b,
FILE *    file
[protected]
 

void mootTokenLexer::yy_initialize   [private]
 

virtual int mootTokenLexer::yy_input char *    buf,
int &    result,
int    max_size
[protected, virtual]
 

void mootTokenLexer::yy_load_buffer_state void    [protected]
 

void mootTokenLexer::yy_switch_to_buffer YY_BUFFER_STATE    new_buffer [protected]
 

long mootTokenLexer::yy_try_NUL_trans_ long    current_state_ [private]
 

virtual int mootTokenLexer::yy_wrap   [protected, virtual]
 

int mootTokenLexer::yyinput   [inline, private]
 

int mootTokenLexer::yylex void   
 

void mootTokenLexer::yyrestart FILE *    input_file [protected]
 

void mootTokenLexer::yyunput unsigned char    c,
unsigned char *    buf_ptr
[private]
 


Member Data Documentation

bool mootTokenLexer::current_analysis_is_best
 

bool mootTokenLexer::first_analysis_is_best
 

bool mootTokenLexer::ignore_comments
 

bool mootTokenLexer::ignore_current_analysis
 

bool mootTokenLexer::ignore_first_analysis
 

int mootTokenLexer::lasttyp
 

const int mootTokenLexer::LexTypeDetails = moot::NTokTypes+3 [static]
 

const int mootTokenLexer::LexTypeEOA = moot::NTokTypes+4 [static]
 

const int mootTokenLexer::LexTypeEOT = moot::NTokTypes+5 [static]
 

const int mootTokenLexer::LexTypeIgnore = moot::NTokTypes+6 [static]
 

const int mootTokenLexer::LexTypeTag = moot::NTokTypes+2 [static]
 

const int mootTokenLexer::LexTypeText = moot::NTokTypes+1 [static]
 

moot::mootToken::Analysis* mootTokenLexer::manalysis
 

moot::mootToken* mootTokenLexer::mtoken
 

moot::mootToken mootTokenLexer::mtoken_default
 

unsigned char* mootTokenLexer::yy_c_buf_p [private]
 

YY_BUFFER_STATE mootTokenLexer::yy_current_buffer [protected]
 

int mootTokenLexer::yy_did_buffer_switch_on_eof [private]
 

unsigned char mootTokenLexer::yy_hold_char [private]
 

int mootTokenLexer::yy_init [private]
 

int mootTokenLexer::yy_n_chars [private]
 

int mootTokenLexer::yy_start [private]
 

FILE* mootTokenLexer::yyin
 

int mootTokenLexer::yyleng
 

FILE * mootTokenLexer::yyout
 

unsigned char* mootTokenLexer::yytext
 


The documentation for this class was generated from the following file:
Generated on Mon Sep 11 16:10:34 2006 for libmoot by doxygen1.2.18