flex++
lexer for moot PoS tagger native text input (guts for moot::TokenReaderNative)
More...
Public Types | |
typedef moot::mootTokenType | TokenType |
typedef moot::mootTokenType | TokenType |
Public Member Functions | |
int | yylex (void) |
mootTokenLexer () | |
virtual | ~mootTokenLexer (void) |
virtual void | reset (void) |
void | on_EOA (void) |
void | loc_set (moot::OffsetT off, int len) |
void | loc_add (int len) |
virtual void * | mgl_yy_current_buffer_p (void) |
virtual void * | mgl_yy_create_buffer (int size, FILE *unused=stdin) |
virtual void | mgl_yy_init_buffer (void *buf, FILE *unused=stdin) |
virtual void | mgl_yy_delete_buffer (void *buf) |
virtual void | mgl_yy_switch_to_buffer (void *buf) |
virtual void | mgl_begin (int stateno) |
int | yylex (void) |
mootTokenLexer () | |
virtual | ~mootTokenLexer (void) |
virtual void | reset (void) |
void | on_EOA (void) |
void | loc_set (moot::OffsetT off, int len) |
void | loc_add (int len) |
virtual void * | mgl_yy_current_buffer_p (void) |
virtual void * | mgl_yy_create_buffer (int size, FILE *unused=stdin) |
virtual void | mgl_yy_init_buffer (void *buf, FILE *unused=stdin) |
virtual void | mgl_yy_delete_buffer (void *buf) |
virtual void | mgl_yy_switch_to_buffer (void *buf) |
virtual void | mgl_begin (int stateno) |
Public Member Functions inherited from moot::GenericLexer | |
GenericLexer (const std::string &myname="moot::GenericLexer", size_t line=0, size_t column=0, mootio::ByteOffset byte=0) | |
virtual | ~GenericLexer (void) |
virtual void | clear (bool clear_input=true, bool clear_output=true) |
virtual void | from_mstream (mootio::mistream *in=__null) |
virtual void | from_filename (const std::string &filename) |
virtual void | from_file (FILE *in=stdin) |
virtual void | from_buffer (const char *buf, size_t len) |
void | from_string (const char *s) |
void | from_string (const std::string &s) |
void | select_streams (FILE *in, FILE *out=stdout, const char *myname=__null) |
void | select_string (const char *in, FILE *out=stderr, const char *myname=__null) |
virtual void | to_mstream (mootio::mostream *out=__null) |
virtual void | to_filename (const std::string &filename) |
virtual void | to_file (FILE *out=stdout) |
int | yyinput (char *buffer, int &result, int max_size) |
void | tokbuf_append (const char *text, size_t len) |
void | add_columns (ByteOffset len) |
void | add_lines (ByteOffset len=1) |
virtual void | yycarp (const char *fmt,...) |
Public Attributes | |
unsigned char * | yytext |
int | yyleng |
FILE * | yyin |
FILE * | yyout |
int | lasttyp |
int | nextstate |
moot::mootToken | mtoken_default |
moot::mootToken * | mtoken |
moot::mootToken::Analysis * | manalysis |
bool | ignore_comments |
bool | first_analysis_is_best |
bool | current_analysis_is_best |
bool | ignore_first_analysis |
bool | ignore_current_analysis |
bool | parse_location |
bool | parse_analysis_cost |
bool | analysis_cost_details |
Public Attributes inherited from moot::GenericLexer | |
mootio::mistream * | mglin |
mootio::mostream * | mglout |
bool | mgl_in_created |
bool | mgl_out_created |
size_t | theLine |
size_t | theColumn |
mootio::ByteOffset | theByte |
std::string | lexname |
std::string | tokbuf |
bool | tokbuf_clear |
Static Public Attributes | |
static const int | LexTypeText = moot::NTokTypes+1 |
static const int | LexTypeTag = moot::NTokTypes+2 |
static const int | LexTypeDetails = moot::NTokTypes+3 |
static const int | LexTypeEOA = moot::NTokTypes+4 |
static const int | LexTypeEOT = moot::NTokTypes+5 |
static const int | LexTypeIgnore = moot::NTokTypes+6 |
Static Public Attributes inherited from moot::GenericLexer | |
static const int | MGL_DEFAULT_BUFFER_SIZE = 16384 |
Protected Member Functions | |
void | yyrestart (FILE *input_file) |
void | yy_switch_to_buffer (YY_BUFFER_STATE new_buffer) |
void | yy_load_buffer_state (void) |
YY_BUFFER_STATE | yy_create_buffer (FILE *file, int size) |
void | yy_delete_buffer (YY_BUFFER_STATE b) |
void | yy_init_buffer (YY_BUFFER_STATE b, FILE *file) |
virtual void | yy_echo () |
virtual int | yy_input (char *buf, int &result, int max_size) |
virtual void | yy_fatal_error (const char *msg) |
virtual int | yy_wrap () |
void | yyrestart (FILE *input_file) |
void | yy_switch_to_buffer (YY_BUFFER_STATE new_buffer) |
void | yy_load_buffer_state (void) |
YY_BUFFER_STATE | yy_create_buffer (FILE *file, int size) |
void | yy_delete_buffer (YY_BUFFER_STATE b) |
void | yy_init_buffer (YY_BUFFER_STATE b, FILE *file) |
virtual void | yy_echo () |
virtual int | yy_input (char *buf, int &result, int max_size) |
virtual void | yy_fatal_error (const char *msg) |
virtual int | yy_wrap () |
Protected Attributes | |
YY_BUFFER_STATE | yy_current_buffer |
Private Member Functions | |
void | yy_initialize () |
int | input () |
int | yyinput () |
int | yy_get_next_buffer () |
void | yyunput (unsigned char c, unsigned char *buf_ptr) |
long | yy_get_previous_state_ (void) |
long | yy_try_NUL_trans_ (long current_state_) |
void | yy_initialize () |
int | input () |
int | yyinput () |
int | yy_get_next_buffer () |
void | yyunput (unsigned char c, unsigned char *buf_ptr) |
long | yy_get_previous_state_ (void) |
long | yy_try_NUL_trans_ (long current_state_) |
Private Attributes | |
unsigned char * | yy_c_buf_p |
unsigned char | yy_hold_char |
int | yy_n_chars |
int | yy_init |
int | yy_start |
int | yy_did_buffer_switch_on_eof |
Assumes pre-tokenized input:
Format:
FILE ::= SENTENCE* SENTENCE ::= ((TOKEN|COMMENT)*) "\n" TOKEN ::= TOKEN_TEXT ("\t" LOCATION)? ("\t" ANALYSIS)* ("\r"*) "\n" TOKEN_TEXT ::= [^\t\r\n]* LOCATION ::= OFFSET_INT " " LENGTH_INT ANALYSIS ::= (DETAIL*) (COST?) (DETAIL*) "[" TAG ( "]" | " " ) (DETAIL*) (COST?) (DETAIL*) DETAIL ::= [^\t\r\n] COST ::= "<" FLOAT ">" COMMENT ::= "^%%" (.*) "\n"
mootTokenLexer::mootTokenLexer | ( | ) |
Referenced by yyinput().
|
inlinevirtual |
References moot::mootToken::loc_length(), and moot::mootToken::location().
mootTokenLexer::mootTokenLexer | ( | ) |
|
inlinevirtual |
|
private |
|
private |
Referenced by yyinput().
|
inlineprivate |
|
private |
Referenced by yyinput().
|
private |
Referenced by yyinput().
|
private |
Referenced by yyinput().
|
private |
Referenced by yyinput().
|
protected |
Referenced by yyinput().
|
protected |
Referenced by mgl_yy_switch_to_buffer(), and yyinput().
|
protected |
Referenced by yyinput().
|
protected |
Referenced by mgl_yy_init_buffer(), and yyinput().
|
protected |
Referenced by mgl_yy_switch_to_buffer(), and yyinput().
|
protected |
Referenced by mgl_yy_switch_to_buffer(), and yyinput().
|
protectedvirtual |
Referenced by yyinput().
|
protectedvirtual |
Referenced by yyinput().
|
protectedvirtual |
Referenced by yyinput().
|
protectedvirtual |
Referenced by yyinput().
int mootTokenLexer::yylex | ( | void | ) |
Referenced by yyinput().
|
virtual |
Reset lexer state
Reimplemented from moot::GenericLexer.
Referenced by moot::wasteLexer::length_attr(), and ~mootTokenLexer().
void mootTokenLexer::on_EOA | ( | void | ) |
Referenced by ~mootTokenLexer().
|
inline |
Referenced by ~mootTokenLexer().
|
inline |
Referenced by ~mootTokenLexer().
|
inlinevirtual |
Reimplemented from moot::GenericLexer.
Referenced by ~mootTokenLexer().
|
inlinevirtual |
Flex++ should define this (backwards), but it doesn't get through to us
Implements moot::GenericLexer.
Referenced by ~mootTokenLexer().
|
inlinevirtual |
Flex++ should define this, but it doesn't get through to us
Implements moot::GenericLexer.
References mgl_yy_init_buffer().
Referenced by mgl_yy_init_buffer().
|
inlinevirtual |
Flex++ should define this, but it doesn't get through to us
Reimplemented from moot::GenericLexer.
Referenced by mgl_yy_switch_to_buffer().
|
inlinevirtual |
Flex++ should define this, but it doesn't get through to us
Implements moot::GenericLexer.
References mgl_yy_switch_to_buffer().
Referenced by mgl_yy_switch_to_buffer().
|
virtual |
Set current start-state. Descendants must override this method (section 3 only!)
Reimplemented from moot::GenericLexer.
Referenced by mgl_yy_switch_to_buffer().
|
private |
|
private |
|
inlineprivate |
References input(), mootTokenLexer(), yy_create_buffer(), yy_current_buffer, yy_delete_buffer(), yy_echo(), yy_fatal_error(), yy_get_next_buffer(), yy_get_previous_state_(), yy_init_buffer(), yy_input(), yy_load_buffer_state(), yy_switch_to_buffer(), yy_try_NUL_trans_(), yy_wrap(), yyin, yyleng, yylex(), yyout, yyrestart(), yytext, and yyunput().
|
private |
|
private |
|
private |
|
private |
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
|
protectedvirtual |
|
protectedvirtual |
|
protectedvirtual |
|
protectedvirtual |
int mootTokenLexer::yylex | ( | void | ) |
|
virtual |
Reset lexer state
Reimplemented from moot::GenericLexer.
void mootTokenLexer::on_EOA | ( | void | ) |
|
inline |
|
inline |
|
inlinevirtual |
Reimplemented from moot::GenericLexer.
|
inlinevirtual |
Flex++ should define this (backwards), but it doesn't get through to us
Implements moot::GenericLexer.
|
inlinevirtual |
Flex++ should define this, but it doesn't get through to us
Implements moot::GenericLexer.
References mgl_yy_init_buffer(), and yy_create_buffer().
Referenced by mgl_yy_init_buffer().
|
inlinevirtual |
Flex++ should define this, but it doesn't get through to us
Reimplemented from moot::GenericLexer.
|
inlinevirtual |
Flex++ should define this, but it doesn't get through to us
Implements moot::GenericLexer.
References mgl_begin(), mgl_yy_delete_buffer(), mgl_yy_switch_to_buffer(), yy_delete_buffer(), yy_init_buffer(), and yy_switch_to_buffer().
Referenced by mgl_yy_switch_to_buffer().
|
virtual |
Set current start-state. Descendants must override this method (section 3 only!)
Reimplemented from moot::GenericLexer.
|
private |
|
private |
|
private |
|
private |
|
private |
|
private |
|
protected |
Referenced by yyinput(), and ~mootTokenLexer().
unsigned char * mootTokenLexer::yytext |
Referenced by yyinput().
int mootTokenLexer::yyleng |
Referenced by yyinput(), and ~mootTokenLexer().
FILE * mootTokenLexer::yyin |
Referenced by yyinput().
FILE * mootTokenLexer::yyout |
Referenced by yyinput().
|
static |
|
static |
|
static |
|
static |
|
static |
|
static |
int mootTokenLexer::lasttyp |
int mootTokenLexer::nextstate |
moot::mootToken mootTokenLexer::mtoken_default |
Referenced by moot::TokenReaderNative::TokenReaderNative().
moot::mootToken * mootTokenLexer::mtoken |
moot::mootToken::Analysis * mootTokenLexer::manalysis |
bool mootTokenLexer::ignore_comments |
bool mootTokenLexer::first_analysis_is_best |
Referenced by moot::TokenReaderNative::input_is_tagged().
bool mootTokenLexer::current_analysis_is_best |
bool mootTokenLexer::ignore_first_analysis |
Referenced by moot::TokenReaderNative::input_is_tagged().
bool mootTokenLexer::ignore_current_analysis |
bool mootTokenLexer::parse_location |
Referenced by moot::TokenReaderNative::input_has_locations().
bool mootTokenLexer::parse_analysis_cost |
Referenced by moot::TokenReaderNative::input_has_cost().
bool mootTokenLexer::analysis_cost_details |
Referenced by moot::TokenReaderNative::input_has_cost().