Main Page   Namespace List   Class Hierarchy   Alphabetical List   Compound List   File List   Namespace Members   Compound Members   File Members  

mootGenericLexer.h

Go to the documentation of this file.
00001 /* -*- Mode: C++ -*- */
00002 
00003 /*
00004    libmoot : moocow's part-of-speech tagging library
00005    Copyright (C) 2003-2005 by Bryan Jurish <moocow@ling.uni-potsdam.de>
00006 
00007    This library is free software; you can redistribute it and/or
00008    modify it under the terms of the GNU Lesser General Public
00009    License as published by the Free Software Foundation; either
00010    version 2.1 of the License, or (at your option) any later version.
00011    
00012    This library is distributed in the hope that it will be useful,
00013    but WITHOUT ANY WARRANTY; without even the implied warranty of
00014    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00015    Lesser General Public License for more details.
00016    
00017    You should have received a copy of the GNU Lesser General Public
00018    License along with this library; if not, write to the Free Software
00019    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
00020 */
00021 
00022 /*--------------------------------------------------------------------------
00023  * File: mootGenericLexer.h
00024  * Author: Bryan Jurish <moocow@ling.uni-potsdam.de>
00025  * Description:
00026  *   + moocow's PoS tagger : generic lexer routines
00027  *--------------------------------------------------------------------------*/
00028 
00029 #ifndef _MOOT_GENERIC_LEXER_H
00030 #define _MOOT_GENERIC_LEXER_H
00031 
00032 #include <stdio.h>
00033 #include <string.h>
00034 #include <stdarg.h>
00035 #include <string>
00036 
00037 #include <mootIO.h>
00038 #include <mootCIO.h>
00039 #include <mootBufferIO.h>
00040 
00041 namespace moot {
00042   using namespace std;
00043 
00089 class GenericLexer {
00090 public:
00091   /*--------------------------------------------------------------------------
00092    * mootGenericLexer: Statics
00093    */
00094   static const int MGL_DEFAULT_BUFFER_SIZE = 8192;
00095 
00096 public:
00097   /*--------------------------------------------------------------------
00098    * mootGenericLexer: Data
00099    */
00100   /*------------------------------------------------------------*/
00103   mootio::mistream        *mglin; 
00104   mootio::mostream       *mglout; 
00105 
00106   bool mgl_in_created;   
00107   bool mgl_out_created;  
00108 
00109 
00110   /*------------------------------------------------------------*/
00113   size_t theLine;    
00114   size_t theColumn;  
00115   size_t theByte;    
00116 
00117 
00118   /*------------------------------------------------------------*/
00121   std::string  lexname; 
00122 
00123 
00124 
00125   /*------------------------------------------------------------*/
00128   std::string tokbuf;               
00129   bool        tokbuf_clear;         
00130 
00131 
00132 public:
00133   /*--------------------------------------------------------------------
00134    * mootGenericLexer: Methods
00135    */
00136   /*------------------------------------------------------------*/
00140   GenericLexer(const std::string &myname="moot::GenericLexer",
00141                size_t line=0, size_t column=0, size_t byte=0)
00142     : mglin(NULL),
00143       mglout(NULL),
00144       mgl_in_created(false),
00145       mgl_out_created(false),
00146       theLine(line),
00147       theColumn(column),
00148       theByte(byte),
00149       lexname(myname),
00150       tokbuf_clear(false)
00151   {};
00152 
00154   virtual ~GenericLexer(void);
00155 
00157   virtual void reset(void);
00158 
00160   virtual void clear(bool clear_input=true, bool clear_output=true);
00162 
00163   /*------------------------------------------------------------*/
00166   virtual void **mgl_yy_current_buffer_p(void);
00167   //{ return &((void *)yy_current_buffer); }
00168 
00173   virtual void mgl_begin(int stateno);
00174   //{ BEGIN(stateno); };
00175 
00177   virtual void mgl_yy_delete_buffer(void *buf)
00178   {
00179     yycarp("abstract method mgl_yy_delete_buffer() called!");
00180     abort();
00181   };
00182   
00184   virtual void *mgl_yy_create_buffer(int size, FILE *unused=stdin) =0;
00185 
00187   virtual void mgl_yy_switch_to_buffer(void *buf) =0;
00188 
00190   virtual void mgl_yy_init_buffer(void *buf, FILE *unused=stdin) =0;
00192 
00193   /*------------------------------------------------------------*/
00197   virtual void from_mstream(mootio::mistream *in=NULL);
00198 
00200   virtual void from_filename(const std::string &filename);
00201 
00203   virtual void from_file(FILE *in=stdin);
00204   
00206   virtual void from_buffer(const char *buf, size_t len);
00207 
00209   inline void from_string(const char *s) {
00210     from_buffer(s, strlen(s));
00211   };
00213   inline void from_string(const std::string &s) {
00214     from_buffer(s.data(), s.size());
00215   };
00216 
00218   inline void select_streams(FILE *in, FILE *out=stdout, const char *myname=NULL) {
00219     from_file(in);
00220     to_file(out);
00221     if (myname) lexname = myname;
00222   };
00223 
00225   inline void select_string(const char *in, FILE *out=stderr, const char *myname=NULL) {
00226     from_string(in);
00227     to_file(out);
00228     if (myname) lexname = myname;
00229   };
00231 
00232   /*------------------------------------------------------------*/
00236   virtual void to_mstream(mootio::mostream *out=NULL);
00237 
00239   virtual void to_filename(const std::string &filename);
00240 
00242   virtual void to_file(FILE *out=stdout);
00244 
00245 
00246   /*------------------------------------------------------------*/
00250   inline int yyinput(char *buffer, int &result, int max_size)
00251   {
00252     return (result = (mglin ? mglin->read(buffer, max_size) : 0));
00253   };
00255 
00256   /*------------------------------------------------------------*/
00260   inline void tokbuf_append(const char *text, size_t len)
00261   {
00262     if (tokbuf_clear) {
00263       tokbuf.assign(text,len);
00264       tokbuf_clear = false;
00265     } else {
00266       tokbuf.append(text,len);
00267     }
00268   };
00270 
00271   /*------------------------------------------------------------*/
00275   virtual void yycarp(const char *fmt, ...);
00277 }; //-- /class mootGenericLexer
00278 
00279 }; //-- /namespace moot
00280 
00281 #endif // _MOOT_GENERIC_LEXER_H

Generated on Mon Sep 11 16:10:33 2006 for libmoot by doxygen1.2.18