mootTokenExpatIO.h

Go to the documentation of this file.

/* -*- Mode: C++ -*- */

/*
   libmoot : moocow's part-of-speech tagging library
   Copyright (C) 2003-2013 by Bryan Jurish <moocow@cpan.org>

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
   
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.
   
   You should have received a copy of the GNU Lesser General Public
   License along with this library; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

/*--------------------------------------------------------------------------
 * File: mootTokenExpatIO.h
 * Author: Bryan Jurish <moocow@cpan.org>
 * Description:
 *   + moocow's PoS tagger : token I/O : XML: Expat
 *--------------------------------------------------------------------------*/

#ifndef _MOOT_TOKEN_EXPAT_IO_H
#define _MOOT_TOKEN_EXPAT_IO_H

#include <mootConfig.h>

#ifdef MOOT_EXPAT_ENABLED

#include <mootTokenIO.h>
#include <mootExpatParser.h>
#include <mootRecode.h>

#ifdef __GNUC__
# include <ext/slist>
#else
# include <slist>
#endif

/*#define MOOT_DEBUG_EXPAT*/

moot_BEGIN_NAMESPACE

using namespace std;

/*======================================================================
 * mootTokenExpatIO
 *======================================================================*/

/*--------------------------------------------------------------------------
 * TokenReaderExpat
 */
class TokenReaderExpat : public TokenReader, public mootExpatParser {
public:
  /*----------------------------------------------------
   * TokenReaderExpat: Types
   */
  typedef enum {
    TRX_Default     = 0x00000000,  
    TRX_IsOuter     = 0x00000001,  
    TRX_IsRoot      = 0x00000002,  
    TRX_IsBodyE     = 0x00000004,  
    TRX_IsBodyD     = 0x00000008,  
    TRX_IsTokenE    = 0x00000010,  
    TRX_IsTokenD    = 0x00000020,  
    TRX_IsTokTextE  = 0x00000040,  
    TRX_IsTokTextD  = 0x00000080,  
    TRX_IsAnalysisE = 0x00000100,  
    TRX_IsAnalysisD = 0x00000200,  
    TRX_IsBestTagE  = 0x00000400,  
    TRX_IsBestTagD  = 0x00000800,  
    TRX_All         = 0xffffffff   
  } xmlNodeFlags;

  const static int defaultNodeInheritanceMask
   = TRX_IsBodyD|TRX_IsTokenD|TRX_IsTokTextD|TRX_IsAnalysisD|TRX_IsBestTagD;

  //typedef list<int> NodeInfoStack;
  typedef slist<int> NodeInfoStack;

public:
  /*----------------------------------------------------
   * TokenReaderExpat: Data
   */
  //----------------------------
  /* I/O Behavior */

  //----------------------------
  bool        save_raw_xml;  

  std::string body_elt;      
  std::string eos_elt;       
  std::string token_elt;     
  std::string text_elt;      
  std::string analysis_elt;  
  std::string postag_attr;   
  std::string besttag_elt;   
  std::string location_elt;  
  std::string offset_attr;   
  std::string length_attr;   


  //----------------------------
  //-- position tracking
  NodeInfoStack         stack;      
  int                   done;       

  //-- construction buffers
  mootSentence          cb_nxtsent; 
  mootToken            *cb_nxttok;  

  //-- output buffers
  mootSentence         cb_fullsents; 
  mootSentence         trx_sentbuf;  

 
 public:
  /*----------------------------------------------------
   * TokenReaderExpat: Constructor
   */

  TokenReaderExpat(int                fmt      =tiofXML,
                   size_t             buflen   =MOOT_DEFAULT_EXPAT_BUFLEN,
                   //size_t           buflen   =128, //-- DEBUG
                   const std::string &encoding ="",
                   const std::string &name     ="TokenReaderExpat")
    : TokenReader(fmt,name),
      mootExpatParser(buflen,encoding),
      save_raw_xml(false),
      body_elt(""),
      eos_elt("eos"),
      token_elt("token"),
      text_elt("text"),
      analysis_elt("analysis"),
      postag_attr("pos"),
      besttag_elt("moot.tag"),
      location_elt("moot.loc"),
      offset_attr("offset"),
      length_attr("length"),
      done(1)
  {
    //-- TokenReader pointers
    tr_sentence = &trx_sentbuf;
    tr_token    = NULL;

    save_raw_xml = tr_format & tiofConserve;
  };

  /*----------------------------------------------------
   * TokenReaderExpat: Destructor
   */
  virtual ~TokenReaderExpat(void) {};

  /*----------------------------------------------------
   * TokenReaderExpat: Reset
   */
  virtual void reset(void);

  /*----------------------------------------------------*/

  virtual void reader_name(const std::string &myname)
  {
    TokenReader::reader_name(myname);
    //mootExpatParser::setSrcName(myname);
  };

  virtual void close(void);

  virtual void from_mstream(mootio::mistream *mistreamp) {
    TokenReader::from_mstream(mistreamp);
    mootExpatParser::from_mstream(tr_istream,false);
    done = 0;
  };
  virtual void from_mstream(mootio::mistream &mis) {
    TokenReader::from_mstream(mis);
    mootExpatParser::from_mstream(tr_istream,false);
    done = 0;
  };
  virtual void from_filename(const char *filename) {
    TokenReader::from_filename(filename);
    mootExpatParser::from_mstream(tr_istream,false);
  };
  virtual void from_file(FILE *infile) {
    TokenReader::from_file(infile);
    mootExpatParser::from_mstream(tr_istream,false);
  };
  virtual void from_fd(int fd) {
    TokenReader::from_fd(fd);
    mootExpatParser::from_mstream(tr_istream,false);
  };
  virtual void from_buffer(const void *buf, size_t len) {
    TokenReader::from_buffer(buf,len);
    mootExpatParser::from_mstream(tr_istream,false);
  };
  virtual void from_cxxstream(std::istream &is) {
    TokenReader::from_cxxstream(is);
    mootExpatParser::from_mstream(tr_istream,false);
  };

  /*----------------------------------------------------*/


  virtual mootTokenType get_token(void);

  virtual mootTokenType get_sentence(void);

  /*----------------------------------------------------*/

  /*----------------------------------------------------
   * TokenReaderExpat: XML Utilities
   */
  bool ensure_cb_fullsents(void);

  inline int next_node_info(int emptyStackValue=TRX_IsOuter,
                            int inheritanceMask=defaultNodeInheritanceMask)
  {
    return (stack.empty()
            ? emptyStackValue
            : (stack.front() & inheritanceMask));
  };

  inline int top_node_info(int emptyStackValue=TRX_IsOuter)
  {
    return stack.empty() ? emptyStackValue : stack.front();
  };


#ifdef MOOT_DEBUG_EXPAT

  void save_context(mootTokenType toktype=TokTypeXMLRaw, int info=0);

  void save_context_data(const mootio::micbuffer &buf, mootTokenType toktype=TokTypeXMLRaw, int info=0);

#else

  inline void save_context(mootTokenType toktype=TokTypeXMLRaw, int info=0)
  {
    if (!save_raw_xml && toktype == TokTypeXMLRaw) return;
    if (!info) info = top_node_info();
    ContextBuffer ctb(parser);
    save_context_data(ctb, toktype, info);
  };

  inline void save_context_data(const mootio::micbuffer &buf,
                                mootTokenType toktype=TokTypeXMLRaw,
                                int info=0)
  {
    save_context_data(buf.cb_rdata + buf.cb_offset,
                      buf.cb_used  - buf.cb_offset,
                      toktype, info);
  };
#endif /* MOOT_DEBUG_EXPAT */

  void save_context_data(const char *text, size_t len,
                         mootTokenType toktype=TokTypeXMLRaw,
                         int info=0);
  
  /*----------------------------------------------------
   * TokenReaderExpat: Expat Handlers
   */
  virtual void XmlDeclHandler(const XML_Char  *version,
                              const XML_Char  *encoding,
                              int             standalone);
  virtual void StartElementHandler(const char *el, const char **attr);
  virtual void EndElementHandler(const char *el);
  virtual void CharacterDataHandler(const XML_Char *s, int len);
  virtual void CommentHandler(const XML_Char *s);
  virtual void DefaultHandler(const XML_Char *s, int len);

  /*----------------------------------------------------*/
  virtual size_t line_number(void) {
    return parser ? static_cast<size_t>(XML_GetCurrentLineNumber(parser)) : 0;
  };

  virtual size_t line_number(size_t n) { return line_number(); };

  virtual size_t column_number(void) {
      return parser ? static_cast<size_t>(XML_GetCurrentLineNumber(parser)) : 0;
  };

  virtual size_t column_number(size_t n) { return column_number(); };

  virtual mootio::ByteOffset byte_number(void) {
      return parser ? static_cast<mootio::ByteOffset>(XML_GetCurrentByteIndex(parser)) : 0;
  };

  virtual mootio::ByteOffset byte_number(mootio::ByteOffset n) { return byte_number(); };

  virtual void carp(char *fmt, ...);
};

moot_END_NAMESPACE

/*#endif // moot_EXPAT_ENABLED*/


moot_BEGIN_NAMESPACE

/*======================================================================
 * WRITER
 *======================================================================*/

/*--------------------------------------------------------------------------
 * TokenWriterExpat
 */
class TokenWriterExpat : public TokenWriter {
public:
  /*----------------------------------------------------
   * TokenWriterExpat: Data
   */
  //----------------------------
  /* I/O Behavior */

  //----------------------------

  bool        use_raw_xml;

  std::string root_elt;      
  std::string eos_elt;       
  std::string token_elt;     
  std::string text_elt;      
  std::string analysis_elt;  
  std::string postag_attr;   
  std::string besttag_elt;   
  std::string location_elt;   
  std::string offset_attr;   
  std::string length_attr;   


  //----------------------------
  std::string    twx_encoding;

  mootXMLRecoder twx_recoder;

  int lastc;
 
 public:
  /*----------------------------------------------------
   * TokenReaderExpat: Constructor
   */

  TokenWriterExpat(int                   fmt         =tiofXML
                   , bool                got_raw_xml =false
                   , const std::string  &encoding    =""
                   , const std::string  &name        ="TokenWriterExpat"
                   );

  /*----------------------------------------------------
   * TokenWriterExpat: encoding
   */
  inline void setEncoding(const std::string &encoding="")
  {
    twx_encoding = encoding;
    twx_recoder.scan_request("UTF-8", (twx_encoding.empty()
                                       ? "XML-standalone"
                                       : twx_encoding));
  };


  /*----------------------------------------------------
   * TokenWriterExpat: Destructor
   */
  virtual ~TokenWriterExpat(void)
  {
    close();
  };

  /*----------------------------------------
   * Writer: Expat: Methods: Output Selection
   */
  virtual void to_mstream(mootio::mostream *os);

  virtual void close(void);

  /*----------------------------------------
   * Writer: Expat: Methods: Output
   */
  virtual void put_token(const mootToken &token) {
    _put_token(token,tw_ostream);
  };

  virtual void put_tokens(const mootSentence &tokens) {
    _put_tokens(tokens,tw_ostream);
  };

  virtual void put_sentence(const mootSentence &sentence) {
    _put_sentence(sentence,tw_ostream);
  };

  virtual void put_comment_block_begin(void) {
    _put_comment_block_begin(tw_ostream);
  };

  virtual void put_comment_block_end(void) {
    _put_comment_block_end(tw_ostream);
  };

  virtual void put_raw_buffer(const char *buf, size_t len) {
    _put_raw_buffer(buf,len,tw_ostream);
  };

  /*----------------------------------------
   * Writer: Expat: Methods: Utilities
   */
  void _put_token_raw(const mootToken &token, mootio::mostream *os);

  void _put_token_gen(const mootToken &token, mootio::mostream *os);

  inline void _put_token(const mootToken &token, mootio::mostream *os)
  {
    if (use_raw_xml)  _put_token_raw(token,os);
    else              _put_token_gen(token,os);
  };

  inline void _put_tokens(const mootSentence &tokens, mootio::mostream *os)
  {
    if (!os || (tw_format&tiofNone) || !os->valid()) return;
    if (use_raw_xml) {
      for (mootSentence::const_iterator si=tokens.begin(); si!=tokens.end(); si++) _put_token_raw(*si, os);
    } else {
      for (mootSentence::const_iterator si=tokens.begin(); si!=tokens.end(); si++) _put_token_gen(*si, os);
    }
  };

  inline void _put_sentence(const mootSentence &sentence, mootio::mostream *os)
  {
    if (!os || (tw_format&tiofNone) || !os->valid()) return;
    if (use_raw_xml) {
      for (mootSentence::const_iterator si=sentence.begin(); si!=sentence.end(); si++) _put_token_raw(*si, os);
    } else {
      for (mootSentence::const_iterator si=sentence.begin(); si!=sentence.end(); si++) _put_token_gen(*si, os);
      _put_token_gen(mootToken(TokTypeEOS), os);
    }
  };

  void _put_comment_block_begin(mootio::mostream *os);

  void _put_comment_block_end(mootio::mostream *os);

  void _put_raw_buffer(const char *buf, size_t len, mootio::mostream *os);
};

moot_END_NAMESPACE

#endif // moot_EXPAT_ENABLED

#endif // MOOT_EXPAT_TOKEN_IO_H