mootTokenIO.h

Go to the documentation of this file.

/* -*- Mode: C++; c-basic-offset: 2; -*- */

/*
   libmoot : moocow's part-of-speech tagging library
   Copyright (C) 2003-2014 by Bryan Jurish <moocow@cpan.org>

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
   
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.
   
   You should have received a copy of the GNU Lesser General Public
   License along with this library; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

/*--------------------------------------------------------------------------
 * File: mootTokenIO.h
 * Author: Bryan Jurish <moocow@cpan.org>
 * Description:
 *   + moocow's PoS tagger : token I/O
 *--------------------------------------------------------------------------*/

#ifndef _moot_TOKEN_IO_H
#define _moot_TOKEN_IO_H

#include <mootTokenLexer.h> //-- includes GenericLexer -> BufferIO -> Utils -> CIO -> IO
#include <mootCxxIO.h>

#include <stdexcept>

/*moot_BEGIN_NAMESPACE*/
namespace moot {

//==========================================================================
// Globals

enum TokenIOFormatE {
  tiofNone      = 0x00000000,  
  tiofUnknown   = 0x00000001,  
  tiofNull      = 0x00000002,  
  tiofUser      = 0x00000004,  
  tiofNative    = 0x00000008,  
  tiofXML       = 0x00000010,  
  tiofConserve  = 0x00000020,  
  tiofPretty    = 0x00000040,  
  tiofText      = 0x00000080,  
  tiofAnalyzed  = 0x00000100,  
  tiofTagged    = 0x00000200,  
  tiofPruned    = 0x00000400,  
  tiofLocation  = 0x00000800,  
  tiofCost      = 0x00001000,  
  tiofTrace     = 0x00002000,  
  tiofPredict   = 0x00004000,  
  tiofFlush     = 0x00008000   
};
typedef TokenIOFormatE TokenIOFormat;

static const int tiofRare = tiofText;

static const int tiofMediumRare = tiofText|tiofAnalyzed; //|tiofCost

static const int tiofMedium = tiofText|tiofTagged;

static const int tiofWellDone = tiofText|tiofAnalyzed|tiofTagged; //|tiofCost

//==========================================================================
// TokenIO

class TokenIO {
public:
  //--------------------------------------------------------------------


  static int parse_format_string(const std::string &fmtString);

  static int guess_filename_format(const char *filename);

  static bool is_empty_format(int fmt);

  static int sanitize_format(int fmt,
                             int fmt_implied=tiofNone,
                             int fmt_default=tiofNone);

  static int parse_format_request(const char *request,
                                  const char *filename=NULL,
                                  int fmt_implied=tiofNone,
                                  int fmt_default=tiofNone);

  static std::string format_canonical_string(int fmt);

  //--------------------------------------------------------------------


  static class TokenReader *new_reader(int fmt);

  static class TokenWriter *new_writer(int fmt);

  static class TokenReader *file_reader(const char *filename, const char *fmt_request=NULL, int fmt_implied=tiofNone, int fmt_default=tiofNone);

  static class TokenWriter *file_writer(const char *filename, const char *fmt_request=NULL, int fmt_implied=tiofNone, int fmt_default=tiofNone);

  //--------------------------------------------------------------------


  static size_t pipe_tokens(class TokenReader *reader, class TokenWriter *writer);

  static size_t pipe_sentences(class TokenReader *reader, class TokenWriter *writer);
};

//==========================================================================
// TokenReader

class TokenReader : public TokenIO {
public:
  static const size_t TR_DEFAULT_BUFSIZE = 256;

public:
  int tr_format;

  std::string tr_name;

  mootio::mistream *tr_istream;

  bool tr_istream_created;

  mootToken *tr_token;

  mootSentence *tr_sentence;

  void *tr_data;

public:
  /*------------------------------------------------------------
   * TokenReader: Constructors
   */
  TokenReader(int                fmt  =tiofUnknown,
              const std::string &name ="TokenReader")
    : tr_format(fmt),
      tr_name(name),
      tr_istream(NULL),
      tr_istream_created(false),
      tr_token(NULL),
      tr_sentence(NULL),
      tr_data(NULL)
  {};

  virtual ~TokenReader(void)
  {
    TokenReader::close();
  };

  inline void tr_clear(void)
  {
    if (tr_token) tr_token->clear();
    if (tr_sentence) tr_sentence->clear();
  };


  /*------------------------------------------------------------
   * TokenReader: Input Selection
   */

  virtual void from_mstream(mootio::mistream *mistreamp)
  {
    this->close();
    tr_istream = mistreamp;
    byte_number(1);
    line_number(1);
    column_number(0);
    tr_istream_created = false;
  };

  virtual void from_mstream(mootio::mistream &mis)
  {
    this->from_mstream(&mis);
  };

  virtual void from_filename(const char *filename)
  {
    this->from_mstream(new mootio::mifstream(filename,"rb"));
    tr_istream_created = true;
    if (!tr_istream || !tr_istream->valid()) {
      this->carp("open failed for \"%s\": %s", filename, strerror(errno));
      this->close();
    }
  };

  virtual void from_file(FILE *file)
  {
    this->from_mstream(new mootio::micstream(file));
    tr_istream_created = true;
  };

  virtual void from_fd(int fd)
  {
    this->close();
    throw domain_error("from_fd(): not implemented");
  };

  virtual void from_buffer(const void *buf, size_t len)
  {
    this->from_mstream(new mootio::micbuffer(buf,len));
    tr_istream_created = true;
  };

  virtual void from_string(const char *s)
  {
    from_buffer(s,strlen(s));
  };

  virtual void from_cxxstream(std::istream &is)
  {
    this->from_mstream(new mootio::micxxstream(is));
    tr_istream_created = true;
  };

  virtual void close(void) {
    if (tr_istream_created) {
      tr_istream->close();
      if (tr_istream) delete tr_istream;
    }
    tr_istream_created = false;
    tr_istream = NULL;
  };

  virtual bool opened(void)
  {
    return tr_istream!=NULL && tr_istream->valid();
  };

  /*------------------------------------------------------------
   * TokenReader: Token-Level Access
   */

  inline mootToken *token(void) { return tr_token; };

  inline mootSentence *sentence(void) { return tr_sentence; };

  virtual mootTokenType get_token(void) {
    throw domain_error("TokenReader: get_token() not implemented");
  };

  virtual mootTokenType get_sentence(void);

  /*------------------------------------------------------------
   * TokenReader: Diagnostics
   */
  virtual void reader_name(const std::string &myname) { tr_name = myname; };

  virtual size_t line_number(void) { return 0; };

  virtual size_t line_number(size_t n) { return n; };

  virtual size_t column_number(void) { return 0; };

  virtual size_t column_number(size_t n) { return n; };

  virtual mootio::ByteOffset byte_number(void) { return 0; };

  virtual mootio::ByteOffset byte_number(size_t n) { return n; };

  virtual void carp(const char *fmt, ...);
};

//==========================================================================
// TokenReaderNative

class TokenReaderNative : public TokenReader {
public:
  /*----------------------------------------
   * Reader: Native: Data
   */
  mootTokenLexer lexer;

  mootSentence   trn_sentence;

public:
  /*----------------------------------------
   * Reading: Native: Methods: Constructors
   */
  TokenReaderNative(int                fmt  =tiofWellDone,
                    const std::string &name ="TokenReaderNative")
    : TokenReader(fmt,name)
  {
    tr_format |= tiofNative;
    input_is_tagged(tr_format&tiofTagged);
    input_has_locations(tr_format&tiofLocation);
    input_has_cost(tr_format&tiofCost);

    tr_sentence = &trn_sentence;
    tr_token    = &lexer.mtoken_default;

    lexer.to_file(stderr);
  };

  virtual ~TokenReaderNative(void)
  {
    this->close();
  };

  /*----------------------------------------
   * Reader: Native: Methods: Input Selection
   */


  virtual void from_mstream(mootio::mistream *mis);


  /*----------------------------------------
   * Reader: Native: Methods: Input
   */
  virtual mootTokenType get_token(void);
  virtual mootTokenType get_sentence(void);


  /*----------------------------------------
   * Reader: Native: Methods: Diagnostics
   */

  virtual size_t line_number(void) { return lexer.theLine; };

  virtual size_t line_number(size_t n) { return lexer.theLine = n; };

  virtual size_t column_number(void) { return lexer.theColumn; };

  virtual size_t column_number(size_t n) { return lexer.theColumn = n; };

  virtual mootio::ByteOffset byte_number(void) { return lexer.theByte; };

  virtual mootio::ByteOffset byte_number(mootio::ByteOffset n) { return lexer.theByte = n; };


  /*----------------------------------------
   * Reader: Native: Methods: New methods
   */
  inline bool input_is_tagged(void)
  {
    return lexer.first_analysis_is_best;
  };

  inline bool input_is_tagged(bool is_tagged)
  {
    if (is_tagged) {
      tr_format |= tiofTagged;
      lexer.first_analysis_is_best = true;
      lexer.ignore_first_analysis = true;
    } else {
      tr_format &= ~tiofTagged;
      lexer.first_analysis_is_best = false;
      lexer.ignore_first_analysis = false;
    }
    return is_tagged;
  };

  inline bool input_has_locations(void)
  {
    return lexer.parse_location;
  };

  inline bool input_has_locations(bool has_locs)
  {
    if (has_locs) {
      tr_format |= tiofLocation;
      lexer.parse_location = true;
    } else {
      tr_format &= ~tiofLocation;
      lexer.parse_location = false;
    }
    return has_locs;
  };

  inline bool input_has_cost(void)
  {
    return lexer.parse_analysis_cost;
  };

  inline bool input_has_cost(bool has_cost)
  {
    if (has_cost) {
      tr_format |= tiofCost;
      lexer.parse_analysis_cost = true;
      lexer.analysis_cost_details = false;
    } else {
      tr_format &= ~tiofCost;
      lexer.parse_analysis_cost = false;
      lexer.analysis_cost_details = true;
    }
    return has_cost;
  };
};


//==========================================================================
// TokenWriter

class TokenWriter : public TokenIO {
public:
  int tw_format;

  std::string tw_name;

  mootio::mostream *tw_ostream;

  bool tw_ostream_created;

  bool tw_is_comment_block;

  void *tw_data;

public:
  /*----------------------------------------
   * Writer: Methods
   */
  TokenWriter(int fmt=tiofWellDone,
              const std::string &name="TokenWriter")
    : tw_format(fmt),
      tw_name(name),
      tw_ostream(NULL),
      tw_ostream_created(false),
      tw_is_comment_block(false),
      tw_data(NULL)
  {};

  virtual ~TokenWriter(void)
  {
    //close();
  };

  /*------------------------------------------------------------
   * Writer: Methods: Output Selection
   */

  virtual void to_mstream(mootio::mostream *mostreamp)
  {
    this->close();
    tw_ostream = mostreamp;
    if ( !(tw_format&tiofNull) && (!tw_ostream || !tw_ostream->valid())) {
      this->carp("Warning: selecting output to invalid stream");
    }
    tw_ostream_created = false;
  };

  virtual void to_mstream(mootio::mostream &mos)
  {
    this->to_mstream(&mos);
  };

  virtual void to_filename(const char *filename)
  {
    this->to_mstream(new mootio::mofstream(filename,"wb"));
    tw_ostream_created = true;
    if (!tw_ostream || !tw_ostream->valid()) {
      this->carp("open failed for \"%s\": %s", filename, strerror(errno));
      this->close();
    }
  };

  virtual void to_file(FILE *file)
  {
    this->to_mstream(new mootio::mocstream(file));
    tw_ostream_created = true;
  };

  virtual void to_fd(int fd)
  {
    this->close();
    throw domain_error("to_fd(): not implemented.");
  };

  virtual void to_cxxstream(std::ostream &os)
  {
    this->to_mstream(new mootio::mocxxstream(os));
    tw_ostream_created = true;
  };

  virtual void close(void) {
    if (tw_is_comment_block) this->put_comment_block_end();
    if (tw_ostream && tw_ostream_created) {
      tw_ostream->close();
      delete tw_ostream;
    }
    tw_ostream_created = false; 
    tw_ostream = NULL;
  };

  virtual bool opened(void)
  {
    return tw_ostream!=NULL && tw_ostream->valid();
  };

  virtual bool flush(void)
  {
    return this->opened() && tw_ostream->flush();
  };

  inline bool autoflush(mootio::mostream *os)
  {
    return (tw_format&tiofFlush)==0 || os->flush();
  };


  /*----------------------------------------*/
  virtual void put_token(const mootToken &token) {
    throw domain_error("TokenWriter: put_token() not implemented");
  };

  virtual void put_tokens(const mootSentence &tokens)
  {
    for (mootSentence::const_iterator si=tokens.begin(); si!=tokens.end(); si++)
      this->put_token(*si);
  };

  virtual void put_sentence(const mootSentence &sentence)
  {
    this->put_tokens(sentence);
  };

  /*----------------------------------------*/
  virtual void put_comment_block_begin(void) {
    tw_is_comment_block = true;
  };

  virtual void put_comment_block_end(void) {
    tw_is_comment_block = false;
  };

  virtual void put_comment_buffer(const char *buf, size_t len) {
    this->put_comment_block_begin();
    this->put_raw_buffer(buf,len);
    this->put_comment_block_end();
  };

  virtual void put_comment(const char *s) {
    this->put_comment_buffer(s,strlen(s));
  };

  virtual void put_comment_buffer(const std::string &s) {
    this->put_comment_buffer(s.data(),s.size());
  };

  virtual void printf_comment(const char *fmt, ...);

  /*----------------------------------------*/
  virtual void put_raw_buffer(const char *buf, size_t len)
  {};
  virtual void put_raw(const char *s) {
    this->put_raw_buffer(s,strlen(s));
  };
  virtual void put_raw(const std::string &s) {
    this->put_raw_buffer(s.data(),s.size());
  };

  virtual void printf_raw(const char *fmt, ...);

  /*----------------------------------------*/
  virtual void writer_name(const std::string &myname) { tw_name = myname; };

  virtual void carp(const char *fmt, ...);
};

//==========================================================================
// TokenWriterNative

class TokenWriterNative : public TokenWriter {
public:
  /*----------------------------------------
   * Writer: Native: Data
   */
  mootio::mocbuffer twn_tmpbuf;

public:
  /*----------------------------------------
   * Writer: Native: Methods: construction
   */
  TokenWriterNative(int fmt=tiofWellDone,
                    const std::string name="TokenWriterNative")
    : TokenWriter(fmt,name)
  {
    if (! (tw_format&tiofNative) ) tw_format |= tiofNative;
  };

  virtual ~TokenWriterNative(void)
  {
    //TokenWriterNative::close();
  };

  /*----------------------------------------
   * Writer: Native: Methods: Output Selection
   */
  // @ {

  /*
   * Finish output to currently selected sink & perform any required
   * cleanup operations.
   * Used by named-file interface.
   */
  //virtual void close(void);
  // @ }

  /*----------------------------------------
   * Writer: Native: Methods: Output
   */
  virtual void put_token(const mootToken &token) {
    _put_token(token,tw_ostream);
  };
  virtual void put_tokens(const mootSentence &tokens) {
    _put_tokens(tokens,tw_ostream);
  };
  virtual void put_sentence(const mootSentence &sentence) {
    _put_sentence(sentence,tw_ostream);
  };

  virtual void put_raw_buffer(const char *buf, size_t len) {
    _put_raw_buffer(buf,len,tw_ostream);
  };

  /*----------------------------------------
   * Writer: Native: Methods: Utilities
   */
  void _put_token(const mootToken &token, mootio::mostream *os);

  void _put_tokens(const mootSentence &tokens, mootio::mostream *os);

  void _put_sentence(const mootSentence &sentence, mootio::mostream *os);

  void _put_comment(const char *buf, size_t len, mootio::mostream *os);

  void _put_raw_buffer(const char *buf, size_t len, mootio::mostream *os);

  inline std::string token2string(const mootToken &token)
  {
    mostream *tw_ostream_old = tw_ostream;
    twn_tmpbuf.clear();
    tw_ostream = &twn_tmpbuf;
    _put_token(token,tw_ostream);
    std::string t2s(twn_tmpbuf.data(), twn_tmpbuf.size());
    tw_ostream = tw_ostream_old;
    return t2s;
  };

  inline std::string sentence2string(const mootSentence &sentence)
  {
    twn_tmpbuf.clear();
    _put_sentence(sentence,&twn_tmpbuf);
    return std::string(twn_tmpbuf.data(), twn_tmpbuf.size());
  };
};


//==========================================================================
// TokenBuffer

class TokenBuffer : public TokenReader, public TokenWriter {
public:
  //------------------------------------------------------------
  // Buffer: typedefs
  typedef mootSentence Buffer;

public:
  //------------------------------------------------------------
  // Buffer: Data
  Buffer tb_buffer;             
  mootSentence tb_sentence;     
public:
  //------------------------------------------------------------


  TokenBuffer(int fmt=tiofUnknown, const std::string name="TokenBuffer")
    : TokenReader(fmt,name),
      TokenWriter(fmt,name)
  {
    tr_sentence = &tb_sentence;
  };

  virtual ~TokenBuffer(void);

  //------------------------------------------------------------


  virtual void from_mstream(mootio::mistream *mistreamp)
  {
    throw domain_error("from_mstream(): not implemented for class moot::TokenBuffer");
  };

  virtual void to_mstream(mootio::mostream *mostreamp)
  {
    throw domain_error("to_mstream(): not implemented for class moot::TokenBuffer");
  };

  virtual void close()
  {};

  virtual bool opened()
  { return true; };

  virtual void from_reader(TokenReader *reader);

  virtual void to_writer(TokenWriter *writer);

  virtual void clear_buffer();

  //------------------------------------------------------------


  virtual mootTokenType get_token(void);

  virtual mootTokenType get_sentence(void);

  //------------------------------------------------------------


  virtual void put_token(const mootToken &token);

  virtual void put_tokens(const mootSentence &tokens);

  virtual void put_sentence(const mootSentence &tokens);

  //------------------------------------------------------------


  virtual void put_raw_buffer(const char *buf, size_t len);
};

}; /*moot_END_NAMESPACE*/

#endif /* _moot_TOKEN_IO_H */