mootGenericLexer.h

Go to the documentation of this file.

/* -*- Mode: C++ -*- */

/*
   libmoot : moocow's part-of-speech tagging library
   Copyright (C) 2003-2009 by Bryan Jurish <moocow@cpan.org>

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
   
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.
   
   You should have received a copy of the GNU Lesser General Public
   License along with this library; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

/*--------------------------------------------------------------------------
 * File: mootGenericLexer.h
 * Author: Bryan Jurish <moocow@cpan.org>
 * Description:
 *   + moocow's PoS tagger : generic lexer routines
 *--------------------------------------------------------------------------*/

#ifndef _MOOT_GENERIC_LEXER_H
#define _MOOT_GENERIC_LEXER_H

#include <mootBufferIO.h> //-- (indirectly) includes mootCIO.h

namespace moot {
  using namespace std;

class GenericLexer {
public:
  /*--------------------------------------------------------------------------
   * mootGenericLexer: Statics
   */
  /*static const int MGL_DEFAULT_BUFFER_SIZE = 8192;*/ //-- old: flex++ now uses 16384
  static const int MGL_DEFAULT_BUFFER_SIZE = 16384;

public:
  /*--------------------------------------------------------------------
   * mootGenericLexer: Data
   */
  /*------------------------------------------------------------*/
  mootio::mistream        *mglin; 
  mootio::mostream       *mglout; 
  bool mgl_in_created;   
  bool mgl_out_created;  

  /*------------------------------------------------------------*/
  size_t theLine;               
  size_t theColumn;             
  mootio::ByteOffset theByte;   

  /*------------------------------------------------------------*/
  std::string  lexname; 


  /*------------------------------------------------------------*/
  std::string tokbuf;               
  bool        tokbuf_clear;         

public:
  /*--------------------------------------------------------------------
   * mootGenericLexer: Methods
   */

  /*------------------------------------------------------------*/
  GenericLexer(const std::string &myname="moot::GenericLexer",
               size_t line=0, size_t column=0, mootio::ByteOffset byte=0)
    : mglin(NULL),
      mglout(NULL),
      mgl_in_created(false),
      mgl_out_created(false),
      theLine(line),
      theColumn(column),
      theByte(byte),
      lexname(myname),
      tokbuf_clear(false)
  {};

  virtual ~GenericLexer(void);

  virtual void reset(void);

  virtual void clear(bool clear_input=true, bool clear_output=true);

  /*------------------------------------------------------------*/
  virtual void *mgl_yy_current_buffer_p(void);
  //{ return (&yy_current_buffer); }

  virtual void mgl_begin(int stateno);
  //{ BEGIN(stateno); };

  virtual void mgl_yy_delete_buffer(void *buf)
  {
    yycarp("abstract method mgl_yy_delete_buffer() called!");
    abort();
  };
  
  virtual void *mgl_yy_create_buffer(int size, FILE *unused=stdin) =0;

  virtual void mgl_yy_switch_to_buffer(void *buf) =0;

  virtual void mgl_yy_init_buffer(void *buf, FILE *unused=stdin) =0;

  /*------------------------------------------------------------*/
  virtual void from_mstream(mootio::mistream *in=NULL);

  virtual void from_filename(const std::string &filename);

  virtual void from_file(FILE *in=stdin);
  
  virtual void from_buffer(const char *buf, size_t len);

  inline void from_string(const char *s) {
    from_buffer(s, strlen(s));
  };
  inline void from_string(const std::string &s) {
    from_buffer(s.data(), s.size());
  };

  inline void select_streams(FILE *in, FILE *out=stdout, const char *myname=NULL) {
    from_file(in);
    to_file(out);
    if (myname) lexname = myname;
  };

  inline void select_string(const char *in, FILE *out=stderr, const char *myname=NULL) {
    from_string(in);
    to_file(out);
    if (myname) lexname = myname;
  };

  /*------------------------------------------------------------*/
  virtual void to_mstream(mootio::mostream *out=NULL);

  virtual void to_filename(const std::string &filename);

  virtual void to_file(FILE *out=stdout);


  /*------------------------------------------------------------*/
  inline int yyinput(char *buffer, int &result, int max_size)
  {
    return (result = (mglin ? mglin->read(buffer, max_size) : 0));
  };

  /*------------------------------------------------------------*/
  inline void tokbuf_append(const char *text, size_t len)
  {
    if (tokbuf_clear) {
      tokbuf.assign(text,len);
      tokbuf_clear = false;
    } else {
      tokbuf.append(text,len);
    }
  };

  /*------------------------------------------------------------*/
  inline void add_columns(ByteOffset len)
  {
    theColumn += len;
    theByte   += len;
  };

  inline void add_lines(ByteOffset len=1)
  {
    theLine += len;
    theByte += len;
    theColumn = 0;
  };


  /*------------------------------------------------------------*/
  virtual void yycarp(const char *fmt, ...);
}; //-- /class mootGenericLexer

}; //-- /namespace moot

#endif // _MOOT_GENERIC_LEXER_H