mootToken.h

Go to the documentation of this file.

/* -*- Mode: C++ -*- */

/*
   libmoot : moocow's part-of-speech tagging library
   Copyright (C) 2003-2014 by Bryan Jurish <moocow@cpan.org>

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
   
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.
   
   You should have received a copy of the GNU Lesser General Public
   License along with this library; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

/*--------------------------------------------------------------------------
 * File: mootToken.h
 * Author: Bryan Jurish <moocow@cpan.org>
 * Description:
 *   + moocow's PoS tagger : token information
 *--------------------------------------------------------------------------*/

#ifndef _moot_TOKEN_H
#define _moot_TOKEN_H

#include <ctype.h>

#include <list>
#include <vector>
#include <mootTypes.h>

#define MOOT_TNT_COMPAT 1
//#undef MOOT_TNT_COMPAT

#define MOOT_TOKEN_DEBUG 1
#undef MOOT_TOKEN_DEBUG

#ifdef MOOT_TOKEN_DEBUG
# define TOKDEBUG(x) x
#else
# define TOKDEBUG(x)
#endif

namespace moot {
  using namespace std;

/*----------------------------------------------------------------------
 * Basic Types
 *----------------------------------------------------------------------*/

typedef string mootTagString;

typedef string mootTokString;

typedef set<mootTagString> mootTagSet;


/*----------------------------------------------------------------------
 * Basic Token Types
 *----------------------------------------------------------------------*/
enum mootTokenTypeE {
  /* Output token-types */
  TokTypeUnknown,    
  TokTypeVanilla,    
  TokTypeLibXML,     
  TokTypeXMLRaw,     
  TokTypeComment,    
  TokTypeEOS,        
  TokTypeEOF,        
  TokTypeWB,         
  TokTypeSB,         
  TokTypeUser,       
  NTokTypes          
};
typedef mootTokenTypeE mootTokenType;

extern const char* mootTokenTypeNames[NTokTypes];

/*--------------------------------------------------------------------------
 * mootToken
 *--------------------------------------------------------------------------*/
class mootToken {
public:
  /*---------------------------------------------------------------------
   * Embedded Types
   */

  typedef float Cost;

  class Analysis {
  public:
    //mootTokenType type;

    mootTagString tag;

    mootTagString details;

    ProbT prob;

    void *data;

    /*--------------------------------------------------
     * Constructor / Destructor
     */
    Analysis(void)
      : prob(0),
        data(NULL)
    {};

    Analysis(const mootTagString &my_tag
             //, mootTokenType typ=TokTypeVanilla
             )
      : tag(my_tag),
        prob(0),
        data(NULL)
    {};

    Analysis(const mootTagString &my_tag,
             const mootTagString &my_details)
      : tag(my_tag),
        details(my_details),
        prob(0),
        data(NULL)
    {};

    Analysis(const mootTagString &my_tag,
             const mootTagString &my_details,
             const ProbT my_prob)
      : tag(my_tag),
        details(my_details),
        prob(my_prob),
        data(NULL)
    {};

    Analysis(const Analysis &x)
      : tag(x.tag),
        details(x.details),
        prob(x.prob),
        data(x.data)
    {};

    inline void clear(void) {
      tag.clear();
      details.clear();
      prob = 0;
    };

    inline bool empty(void) const {
      return tag.empty() && details.empty();
    }

    friend bool operator<(const Analysis &x, const Analysis &y)
    {
      if (x.prob != y.prob) return x.prob < y.prob;
      int tcomp = x.tag.compare(y.tag);
      return (tcomp < 0
              ? true
              : (tcomp > 0
                 ? false
                 : x.details < y.details));
    };

    friend bool operator==(const Analysis &x, const Analysis &y)
    {
      return x.prob == y.prob && x.tag == y.tag && x.details == y.details;
    }

    void dump(const char *label=NULL, FILE *f=NULL) const;
  }; //-- /mootToken::Analysis


  //typedef set<Analysis> AnalysisSet;
  //typedef vector<Analysis> Analyses;
  typedef list<Analysis> Analyses;

  class Location {
  public:
    OffsetT offset;  
    OffsetT length;  
  public:
    inline Location(const OffsetT my_offset=0, const OffsetT my_length=0)
      : offset(my_offset),
        length(my_length)
    {};

    inline Location(const Location &x)
      : offset(x.offset),
        length(x.length)
    {};
    
    inline void clear(void)
    {
      offset = 0;
      length = 0;
    };
  }; //-- /mootToken::Location

public:
  /*---------------------------------------------------------------------*
   * Data Members
   */

  mootTokenType   tok_type;

  mootTokString   tok_text;
  
  mootTagString   tok_besttag;

  Analyses       tok_analyses;

  Location       tok_location;

  void *tok_data;

public:
  /*------------------------------------------------------------
   * Constructors / Destructors
   */
  mootToken(mootTokenType type=TokTypeVanilla)
    : tok_type(type),
      tok_data(NULL)
  {};

  mootToken(const mootTokString &text, mootTokenType type=TokTypeVanilla)
    : tok_type(type),
      tok_text(text),
      tok_data(NULL)
  {};

  mootToken(const mootTokString &text,
            const Analyses &analyses)
    : tok_type(TokTypeVanilla),
      tok_text(text),
      tok_analyses(analyses),
      tok_data(NULL)
  {};

  mootToken(const mootTokString &text,
            const Analyses &analyses,
            const mootTagString &besttag)
    : tok_type(TokTypeVanilla),
      tok_text(text),
      tok_besttag(besttag),
      tok_analyses(analyses),
      tok_data(NULL)
  {};

  mootToken(const mootToken& x)
    : tok_type(x.tok_type),
      tok_text(x.tok_text),
      tok_besttag(x.tok_besttag),
      tok_analyses(x.tok_analyses),
      tok_location(x.tok_location),
      tok_data(x.tok_data)
  {};

  /*
  mootToken(mootTokenType type=TokTypeVanilla, const char *text, size_t len)
    : tok_type(type),
      tok_text(text,len),
      tok_data(NULL)
  {};
  */

  /* Destructor */
  ~mootToken(void) {
#if 0
    if (tok_location.offset>=680307 && tok_location.offset<=680400) {
      dump();   //-- debug
    }
#endif
  };

  /*------------------------------------------------------------
   * Operators
   */
  friend bool operator==(const mootToken &x, const mootToken &y)
  {
    return
      x.tok_type == y.tok_type
      && x.tok_text == y.tok_text
      && x.tok_besttag == y.tok_besttag
      && x.tok_analyses == y.tok_analyses;
  };

  friend bool operator <(const mootToken &x, const mootToken &y)
  {
    return
      x.tok_text < y.tok_text
      || x.tok_besttag < y.tok_besttag
      || x.tok_analyses < y.tok_analyses;
  };

  inline mootToken& operator =(const mootToken &x)
  {
    tok_type = x.tok_type;
    tok_text = x.tok_text;
    tok_besttag = x.tok_besttag;
    tok_analyses = x.tok_analyses;
    tok_location = x.tok_location;
    tok_data = x.tok_data;
    return *this;
  };

  /*------------------------------------------------------------
   * Manipulators: General
   */
  inline void clear(void) {
    tok_type = TokTypeVanilla;
    tok_text.clear();
    tok_besttag.clear();
    tok_analyses.clear();
    tok_location.clear();
  };

  /*------------------------------------------------------------
   * Manipulators: specific
   */
  inline const mootTokString &text(void) const {
    return tok_text;
  };
  inline mootTokString &text(const mootTokString &text) {
    tok_text = text;
    return tok_text;
  }; 
  inline mootTokString &text(const char *s, size_t len) {
    tok_text.assign(s,len);
    return tok_text;
  }; 
  inline mootTokString &textAppend(const mootTokString &text) {
    tok_text.append(text);
    return tok_text;
  };
  inline mootTokString &textAppend(const char *s, size_t len) {
    tok_text.append(s, len);
    return tok_text;
  };

  inline const mootTagString &besttag(void) const {
    return tok_besttag;
  };
  inline mootTagString &besttag(const mootTagString &besttag) {
    tok_besttag = besttag;
    return tok_besttag;
  };
  inline mootTagString &besttagAppend(const mootTagString &text) {
    tok_besttag.append(text);
    return tok_besttag;
  };
  inline mootTagString &besttagAppend(const char *s, size_t len) {
    tok_besttag.append(s, len);
    return tok_besttag;
  };

  inline mootTokenType toktype(void) const { return tok_type; }
  inline mootTokenType toktype(const mootTokenType type) {
    tok_type = type;
    return tok_type;
  };

  inline const Analyses &analyses(void) const { return tok_analyses; };
  inline const Analyses &analyses(const Analyses &analyses) {
    tok_analyses = analyses;
    return tok_analyses;
  };
  inline void insert(const Analysis &analysis)
  {
    //tok_analyses.insert(analysis);
    tok_analyses.push_back(analysis);
  };
  inline void insert(const mootTagString &tag, const mootTagString &details, ProbT p=0)
  {
    //insert(Analysis(tag,details));
    tok_analyses.push_back(Analysis());
    tok_analyses.back().tag = tag;
    tok_analyses.back().details = details;
    tok_analyses.back().prob = p;
  };
  inline void insert(const char *tag, const char *details, ProbT p=0)
  {
    //insert(Analysis(tag,details));
    tok_analyses.push_back(Analysis());
    tok_analyses.back().tag = tag;
    tok_analyses.back().details = details;
    tok_analyses.back().prob = p;
  };
  inline bool has_analysis_for_tag(const mootTagString &tag) const
  {
    for (Analyses::const_iterator asi = tok_analyses.begin();
         asi != tok_analyses.end();
         asi++)
      {
        if (asi->tag == tag) return true;
      }
    return false;
  };
  inline void erase(const Analysis &analysis)
  {
    for (Analyses::iterator asi = tok_analyses.begin();
         asi != tok_analyses.end();
         )
      {
        if (*asi == analysis) tok_analyses.erase(asi);
        else asi++;
      }
  };
  inline void prune(void)
  {
    for (Analyses::iterator asi = tok_analyses.begin();
         asi != tok_analyses.end();
         )
      {
        if (asi->tag != tok_besttag) tok_analyses.erase(asi);
        else asi++;
      }
  };

  inline const Location &location(void) const
  { return tok_location; }

  inline const Location &location(const Location &loc)
  { tok_location=loc; return location(); }

  inline const Location &location(const OffsetT offset, const OffsetT length=0)
  { return location(Location(offset,length)); };

  inline OffsetT loc_offset(void) const { return tok_location.offset; };

  inline OffsetT loc_offset(const OffsetT off)
  { tok_location.offset=off; return loc_offset(); };

  inline OffsetT loc_length(void) const { return tok_location.length; };

  inline OffsetT loc_length(const OffsetT len)
  { tok_location.length=len; return loc_length(); };


  /*------------------------------------------------------------
   * Compatibility
   */
  inline void tokImport(const mootTokString *src_toktext=NULL,
                        const mootTagSet    *src_tagset=NULL)
  {
    if (src_toktext) tok_text = *src_toktext;
    if (src_tagset) {
      for (mootTagSet::const_iterator tsi = src_tagset->begin();
           tsi != src_tagset->end();
           tsi++)
        {
          insert(Analysis(*tsi));
        }
    }
  };

  inline void tokExport(mootTokString *dst_toktext=NULL,
                        mootTagSet *dst_tagset=NULL,
                        bool want_besttag_in_tagset = true) const
  {
    if (dst_toktext) *dst_toktext = tok_text;
    if (dst_tagset) {
      for (Analyses::const_iterator asi = tok_analyses.begin();
           asi != tok_analyses.end();
           asi++
             //asi = upper_bound(asi->tag)
           )
        {
          dst_tagset->insert(asi->tag);
        }
      if (want_besttag_in_tagset && !tok_besttag.empty())
        dst_tagset->insert(tok_besttag);
    }
  };

  /*------------------------------------------------------------
   * Debugging
   */
  void dump(const char *label=NULL, FILE *f=NULL) const;

}; //-- /mootToken


/*--------------------------------------------------------------------------
 * mootSentence
 *--------------------------------------------------------------------------*/

typedef list<mootToken> mootSentence;

//typedef vector<mootToken> mootSentence;

mootToken &sentence_printf_append(mootSentence &s, mootTokenType typ, const char *fmt, ...);

/*----------------------------------------------------------------------
 * Pattern-based Typification
 *  - obsolete; see mootFlavor.h
 *----------------------------------------------------------------------*/

}; /* namespace moot */

#endif /* _moot_TOKEN_H */