Main Page Namespace List Class Hierarchy Alphabetical List Compound List File List Namespace Members Compound Members File Members
mootTokenLexer Class Reference
Flex++ lexer for moot PoS tagger native text input.
More...
#include <mootTokenLexer.h>
Inheritance diagram for mootTokenLexer:
[legend]Collaboration diagram for mootTokenLexer:
[legend]List of all members.Public Types
Public Methods
Public Attributes
Static Public Attributes
Protected Methods
Protected Attributes
Private Methods
Private Attributes
Detailed Description
Assumes pre-tokenized input: one token per line, blank lines = EOS, raw text only (no markup!). Format: 1 tok/line, comments introduced with '%%'. Supports multiple tags/tok.
token-line format (TAB-separated):
TOKEN_TEXT ANALYSIS_1 ... ANALYSIS_N
Analysis format(s):
...(COST?)... "[" TAG ["]"|" "] ...(COST?)...
COST format(s):
"<" COST_FLOAT ">"
Member Typedef Documentation
Constructor & Destructor Documentation
mootTokenLexer::mootTokenLexer |
( |
|
) |
|
|
virtual mootTokenLexer::~mootTokenLexer |
( |
void |
|
) |
[inline, virtual] |
|
Member Function Documentation
int mootTokenLexer::input |
( |
|
) |
[private] |
|
virtual void mootTokenLexer::mgl_begin |
( |
int |
stateno |
) |
[virtual] |
|
|
Set current start-state. Descendants must override this method (section 3 only!)
Reimplemented from moot::GenericLexer. |
virtual void* mootTokenLexer::mgl_yy_create_buffer |
( |
int |
size, |
|
|
FILE * |
unused = stdin |
|
) |
[inline, virtual] |
|
|
Flex++ should define this (backwards), but it doesn't get through to us
Implements moot::GenericLexer. |
virtual void** mootTokenLexer::mgl_yy_current_buffer_p |
( |
void |
|
) |
[inline, virtual] |
|
virtual void mootTokenLexer::mgl_yy_delete_buffer |
( |
void * |
buf |
) |
[inline, virtual] |
|
|
Flex++ should define this, but it doesn't get through to us
Reimplemented from moot::GenericLexer. |
virtual void mootTokenLexer::mgl_yy_init_buffer |
( |
void * |
buf, |
|
|
FILE * |
unused = stdin |
|
) |
[inline, virtual] |
|
|
Flex++ should define this, but it doesn't get through to us
Implements moot::GenericLexer. |
virtual void mootTokenLexer::mgl_yy_switch_to_buffer |
( |
void * |
buf |
) |
[inline, virtual] |
|
|
Flex++ should define this, but it doesn't get through to us
Implements moot::GenericLexer. |
void mootTokenLexer::on_EOA |
( |
void |
|
) |
[inline] |
|
virtual void mootTokenLexer::reset |
( |
void |
|
) |
[virtual] |
|
YY_BUFFER_STATE mootTokenLexer::yy_create_buffer |
( |
FILE * |
file, |
|
|
int |
size |
|
) |
[protected] |
|
virtual void mootTokenLexer::yy_echo |
( |
|
) |
[protected, virtual] |
|
virtual void mootTokenLexer::yy_fatal_error |
( |
char * |
msg |
) |
[protected, virtual] |
|
int mootTokenLexer::yy_get_next_buffer |
( |
|
) |
[private] |
|
long mootTokenLexer::yy_get_previous_state_ |
( |
void |
|
) |
[private] |
|
void mootTokenLexer::yy_init_buffer |
( |
YY_BUFFER_STATE |
b, |
|
|
FILE * |
file |
|
) |
[protected] |
|
void mootTokenLexer::yy_initialize |
( |
|
) |
[private] |
|
virtual int mootTokenLexer::yy_input |
( |
char * |
buf, |
|
|
int & |
result, |
|
|
int |
max_size |
|
) |
[protected, virtual] |
|
void mootTokenLexer::yy_load_buffer_state |
( |
void |
|
) |
[protected] |
|
void mootTokenLexer::yy_switch_to_buffer |
( |
YY_BUFFER_STATE |
new_buffer |
) |
[protected] |
|
long mootTokenLexer::yy_try_NUL_trans_ |
( |
long |
current_state_ |
) |
[private] |
|
virtual int mootTokenLexer::yy_wrap |
( |
|
) |
[protected, virtual] |
|
int mootTokenLexer::yyinput |
( |
|
) |
[inline, private] |
|
int mootTokenLexer::yylex |
( |
void |
|
) |
|
|
void mootTokenLexer::yyrestart |
( |
FILE * |
input_file |
) |
[protected] |
|
void mootTokenLexer::yyunput |
( |
unsigned char |
c, |
|
|
unsigned char * |
buf_ptr |
|
) |
[private] |
|
Member Data Documentation
bool mootTokenLexer::current_analysis_is_best
|
|
bool mootTokenLexer::first_analysis_is_best
|
|
bool mootTokenLexer::ignore_comments
|
|
bool mootTokenLexer::ignore_current_analysis
|
|
bool mootTokenLexer::ignore_first_analysis
|
|
int mootTokenLexer::lasttyp
|
|
const int mootTokenLexer::LexTypeDetails = moot::NTokTypes+3 [static]
|
|
const int mootTokenLexer::LexTypeEOA = moot::NTokTypes+4 [static]
|
|
const int mootTokenLexer::LexTypeEOT = moot::NTokTypes+5 [static]
|
|
const int mootTokenLexer::LexTypeIgnore = moot::NTokTypes+6 [static]
|
|
const int mootTokenLexer::LexTypeTag = moot::NTokTypes+2 [static]
|
|
const int mootTokenLexer::LexTypeText = moot::NTokTypes+1 [static]
|
|
unsigned char* mootTokenLexer::yy_c_buf_p [private]
|
|
int mootTokenLexer::yy_did_buffer_switch_on_eof [private]
|
|
unsigned char mootTokenLexer::yy_hold_char [private]
|
|
int mootTokenLexer::yy_init [private]
|
|
int mootTokenLexer::yy_n_chars [private]
|
|
int mootTokenLexer::yy_start [private]
|
|
FILE* mootTokenLexer::yyin
|
|
int mootTokenLexer::yyleng
|
|
FILE * mootTokenLexer::yyout
|
|
unsigned char* mootTokenLexer::yytext
|
|
The documentation for this class was generated from the following file:
Generated on Mon Sep 11 16:10:34 2006 for libmoot by
1.2.18