wasteLexer.h

Go to the documentation of this file.

/* -*- Mode: C++; coding: utf-8; c-basic-offset: 4; -*- */
/*
   libmoot : moot part-of-speech tagging library
   Copyright (C) 2013-2016 by Bryan Jurish <moocow@cpan.org> and Kay-Michael Würzner

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
   
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.
   
   You should have received a copy of the GNU Lesser General Public
   License along with this library; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

#ifndef _WASTE_LEXER_H
#define _WASTE_LEXER_H

#include <mootTokenIO.h>
#include <wasteTypes.h>
#include <wasteLexicon.h>

#include <vector>
#include <list>

namespace moot
{

  enum wasteLexer_state
  {
    ls_flush   = 0x0001,
    ls_hyph    = 0x0002,
    ls_head    = 0x0004,
    ls_tail    = 0x0008,
    ls_nl      = 0x0010,
    ls_sb_fw   = 0x0020,
    ls_wb_fw   = 0x0040,
    ls_blanked = 0x0080,
  };
  static const int ls_init         = (ls_wb_fw | ls_sb_fw | ls_blanked); 
  static const int ls_head_hyph    = ( ls_head | ls_hyph ); 
  static const int ls_head_hyph_nl = ( ls_head_hyph | ls_nl ); 
  /*============================================================================
   * wasteLexerToken
   */
  class wasteLexerToken
  {
    public:
      /*------------------------------------------------------------*/
      mootToken       wlt_token;   
      wasteLexerType  wlt_type;    
      bool            wlt_blanked; 
      bool            s;
      bool            S;
      bool            w;

      /*--------------------------------------------------------------------
       * wasteLexer: Methods
       */
      /*------------------------------------------------------------*/
      wasteLexerToken(wasteLexerType type=wLexerTypeOther, bool blanked=true, bool bos=true, bool eos=false, bool bow=true)
        : wlt_type(type),
          wlt_blanked(blanked),
          s(bos),
          S(eos),
          w(bow)
    {}


      inline void set_wlt_data(wasteLexerType lextype, bool blanked, bool s, bool S, bool w)
      {
        this->wlt_type    = lextype;
        this->wlt_blanked = blanked;
        this->s           = s;
        this->S           = S;
        this->w           = w;
      }
  };

  /*============================================================================
   * wasteLexer
   */
  class wasteLexer
  {
    public:
      //--------------------------------------------------------------------


      typedef std::vector<std::vector<std::vector<std::vector<std::vector<std::vector<std::string> > > > > > wasteTagset;

      typedef std::list<wasteLexerToken> wasteLexerBuffer;

      //--------------------------------------------------------------------


      enum cls
      {
        stop     = 0,
        rom      = 1,
        alpha    = 2,
        num      = 3,
        dot      = 4,
        comma    = 5,
        colon    = 6,
        scolon   = 7,
        eos      = 8,
        lbr      = 9,
        rbr      = 10,
        hyphen   = 11,
        plus     = 12,
        slash    = 13,
        quote    = 14,
        apos     = 15,
        sc       = 16,
        other    = 17,
        n_cls    = 18
      };

      enum cas
      {
        non      = 0,
        lo       = 1,
        up       = 2,
        cap      = 3,
        n_cas    = 4
      };

      enum binary
      {
        uk       = 0,
        kn       = 1,
        n_binary = 2
      };

      enum len
      {
        le_null  = 0,
        le_one   = 1,
        le_three = 2,
        le_five  = 3,
        longer   = 4,
        n_len    = 5
      };

      static const int n_hidden = 7;


      /*--------------------------------------------------------------------
       * wasteLexer: data
       */
      /*------------------------------------------------------------*/
      wasteTagset       wl_tagset;        
      int               wl_state;         
      wasteLexerBuffer  wl_lexbuf;        
      wasteLexerToken  *wl_current_tok;   
      wasteLexerToken  *wl_head_tok;      
      bool              wl_dehyph_mode;   

      /*------------------------------------------------------------*/
      wasteLexicon      wl_stopwords;     
      wasteLexicon      wl_abbrevs;       
      wasteLexicon      wl_conjunctions;  

      /*--------------------------------------------------------------------
       * wasteLexer: Methods
       */
      /*------------------------------------------------------------*/
      wasteLexer();

      virtual ~wasteLexer();

      /*------------------------------------------------------------*/

      inline len length_attr(size_t length) const
      {
        switch (length)
        {
          case 0:
            return le_null;
          case 1:
            return le_one;
          case 2:
          case 3:
            return le_three;
          case 4:
          case 5:
            return le_five;
          default:
            return longer;
        }
      }

      void set_token(mootToken &token, const wasteLexerToken &lex_token);

      void buffer_token(const mootToken& stok);

      void reset(void);

      /*------------------------------------------------------------*/
      inline void lexbuf_pop_front(void)
      {
          wasteLexerToken *front = &(wl_lexbuf.front());
          if (wl_current_tok==front) wl_current_tok=NULL; 
          if (wl_head_tok==front)    wl_head_tok=NULL;
          wl_lexbuf.pop_front();
      };
  };

  /*============================================================================
   * wasteLexerReader
   */
  class wasteLexerReader : public TokenReader {
  public:
    /*----------------------------------------
     * wasteLexerReader: Data
     */
    wasteLexer lexer;

    TokenReader *scanner;

    mootToken wlr_token;

    mootSentence wlr_sentence;

  public:
    //------------------------------------------------------------
    wasteLexerReader(int                fmt  =tiofText,
                     const std::string &name ="wasteLexerReader");

    virtual ~wasteLexerReader(void);

    /*------------------------------------------------------------
     * TokenReader: Input Selection
     */
    void from_reader(TokenReader *reader);

    virtual void from_mstream(mootio::mistream *mistreamp);

    virtual void close(void);

    inline void dehyph_mode(bool on)
    {
      lexer.wl_dehyph_mode = on;
    };

    /*------------------------------------------------------------
     * TokenReader: Token-Level Access
     */
    virtual mootTokenType get_token(void);
    
    virtual mootTokenType get_sentence(void);

    /*------------------------------------------------------------
     * TokenReader: Diagnostics
     */

    virtual size_t line_number(void) { return scanner ? scanner->line_number() : 0; };

    virtual size_t line_number(size_t n) { return scanner ? scanner->line_number(n) : 0; };

    virtual size_t column_number(void) { return scanner ? scanner->column_number() : 0; };

    virtual size_t column_number(size_t n) { return scanner ? scanner->column_number(n) : 0; };

    virtual mootio::ByteOffset byte_number(void) { return scanner ? scanner->byte_number() : 0; };

    virtual mootio::ByteOffset byte_number(size_t n) { return scanner ? scanner->byte_number(n) : 0; };
  };


} // namespace moot


#endif /* _WASTE_LEXER_H */