mootDynHMM.h

Go to the documentation of this file.

/* -*- Mode: C++ -*- */
/*
   libmoot : moocow's part-of-speech tagging library
   Copyright (C) 2009 by Bryan Jurish <moocow@cpan.org>

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
   
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.
   
   You should have received a copy of the GNU Lesser General Public
   License along with this library; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

/*--------------------------------------------------------------------------
 * File: mootDynHMM.h
 * Author: Bryan Jurish <moocow@cpan.org>
 * Description:
 *   + moot PoS tagger : HMM: for large "tag"-sets with dynamic lexical probabilities
 *--------------------------------------------------------------------------*/

#ifndef _MOOT_DYNHMM_H
#define _MOOT_DYNHMM_H

#include <mootHMM.h>
#include <assert.h>

moot_BEGIN_NAMESPACE


/*======================================================================
 *Generic  Utilities
 */

typedef enum {
  dheUnknown,   
  dheFreq,      
  dheBoltzmann, 
  dheMIParser,  
  dheN          
} DynHMMClassId;

class mootDynHMMOptions {
public:
  //-- mootDynLexHMM
  bool        invert_lexp;     
  std::string newtag_str;      
  ProbT       Ftw_eps;         
  //
  //-- mootDynLexHMM_Boltzmann
  ProbT       dynlex_base;     
  ProbT       dynlex_beta;     
  //
  //-- mootMIParser
  bool        text_tags;        
public:
  mootDynHMMOptions(void)
    : invert_lexp(true),
      newtag_str("@NEW"),
      Ftw_eps(0.5),
      dynlex_base(2.0),
      dynlex_beta(1.0),
      text_tags(false)
  {};

  ~mootDynHMMOptions(void) {};
};

class mootDynHMM *newDynHMM(DynHMMClassId which=dheFreq, const mootDynHMMOptions &opts=mootDynHMMOptions());

class mootDynHMM *newDynHMM(const std::string &which="Freq", const mootDynHMMOptions &opts=mootDynHMMOptions());


/*======================================================================
 * mootDynHMM : "dynamic" HMM class
 */

class mootDynHMM: public mootHMM {
public:
  /*---------------------------------------------------------------------*/
  /* Types */

public:
  /*---------------------------------------------------------------------*/
  /* Data */

public:
  /*---------------------------------------------------------------------*/
  mootDynHMM(void)
  { hash_ngrams=true; };

  virtual ~mootDynHMM(void) {};

  virtual void set_options(const mootDynHMMOptions &opts)
  {};

  //------------------------------------------------------------
  // Utilities

  inline void lex_clear(void) {
    lexprobs.clear();
    tokids.clear();         //-- leaves '@UNKNOWN' tok entry
    lexprobs.resize(1);     //-- re-insert empty entry for '@UNKNOWN' tok
    n_toks = tokids.size(); //-- sanity check
  };

  inline void lex_resize(TokID tokid_max) {
    if (n_toks <= tokid_max) {
      tokids.resize(tokid_max+1);
      lexprobs.resize(tokid_max+1);
      n_toks = tokids.size();
    }
  };

  inline TokID lex_get_tokid(const mootTokString &tok_text)
  {
    TokID tokid = tokids.get_id(tok_text);
    lex_resize(tokid);
    return tokid;
  };

  inline void tagset_resize(TagID tagid_max) {
    assert(hash_ngrams==true);
    if (n_tags <= tagid_max) {
      tagids.resize(tagid_max+1);
      n_tags = tagids.size();
    }
  };

  inline TagID get_tagid(const mootTagString &tagstr)
  {
    if (tagids.nameExists(tagstr))
      return tagids.name2id(tagstr);
    TagID tagid = tagids.get_id(tagstr);
    tagset_resize(tagid);
    return tagid;
  };

  //------------------------------------------------------------

  virtual void tag_hook_pre(mootSentence &sent)
  {};

  virtual void tag_hook_post(mootSentence &sent)
  {};

  /*---------------------------------------------------------------------*/


  virtual void tw_put_info(moot::TokenWriter *tw)
  {
    tw->printf_raw("  +DynHMM class      : %s\n", "mootDynHMM (?)");
  };

  //------------------------------------------------------------
  // Tagging: Top-level

  virtual void tag_io(TokenReader *reader, TokenWriter *writer);

  virtual void tag_sentence(mootSentence &sentence);
};


/*======================================================================
 * class mootDynLexHMM
 */
class mootDynLexHMM : public mootDynHMM {
public:
  //---------------------------------------------------------------------
  // Types
  typedef mootTokString TokStr;  
  typedef mootTagString TagStr;  
  typedef std::map<TokStr,ProbT>      TokProbMap;     
  typedef std::map<TagStr,ProbT>      TagProbMap;     
  typedef std::map<TagStr,TokProbMap> TagTokProbMap;  
  typedef std::map<TagStr,TagProbMap> TokTagProbMap;  
public:
  //---------------------------------------------------------------------
  // Data
  bool           invert_lexp;

  TagStr         newtag_str;  
  TagID          newtag_id;   
  ProbT          newtag_f;    
  TagTokProbMap  Ftw;         
  TokProbMap     Fw;          
  TokProbMap     Ft;          
  ProbT          Ftw_eps;     
  size_t         tagids_size_orig; 
public:
  //---------------------------------------------------------------------

  mootDynLexHMM(void)
    : invert_lexp(true),
      newtag_str("@NEW"),
      newtag_id(0),
      Ftw_eps(0.5),
      tagids_size_orig(0)
  {};

  virtual ~mootDynLexHMM(void) {};

  virtual void set_options(const mootDynHMMOptions &opts)
  {
    invert_lexp = opts.invert_lexp;
    newtag_str  = opts.newtag_str;
    Ftw_eps     = opts.Ftw_eps;
  };

  //---------------------------------------------------------------------


  virtual bool load_model(const string &modelname,
                          const mootTagString &start_tag_str="__$",
                          const char *myname="mootDynLexHMM::load_model()",
                          bool  do_estimate_nglambdas=true,
                          bool  do_estimate_wlambdas=true,
                          bool  do_estimate_clambdas=true,
                          bool  do_build_suffix_trie=true,
                          bool  do_compute_logprobs=true);

  virtual bool compile(mootLexfreqs &lexfreqs,
                       mootNgrams &ngrams,
                       mootClassfreqs &classfreqs,
                       const mootTagString &start_tag_str="__$");

  //---------------------------------------------------------------------

  virtual void tag_hook_pre(mootSentence &sent);
  virtual void tag_hook_post(mootSentence &sent);

  //---------------------------------------------------------------------


  virtual void tw_put_info(moot::TokenWriter *tw);

  //---------------------------------------------------------------------


  void dynlex_clear(void);

  virtual ProbT dynlex_analysis_freq(const mootToken &tok, const mootToken::Analysis &a)
  {
    return a.prob + Ftw_eps;
  };

  virtual void dynlex_populate_lexprobs(void);
};

/*======================================================================
 * class mootDynLexHMM_Boltzmann
 */
class mootDynLexHMM_Boltzmann : public mootDynLexHMM {
public:
  ProbT dynlex_base;

  ProbT dynlex_beta;

public:
  mootDynLexHMM_Boltzmann(void)
    : dynlex_base(2),
      dynlex_beta(1)
  {};

  virtual void set_options(const mootDynHMMOptions &opts)
  {
    mootDynLexHMM::set_options(opts);
    dynlex_base = opts.dynlex_base;
    dynlex_beta = opts.dynlex_beta;
  };

  virtual ProbT dynlex_analysis_freq(const mootToken &tok, const mootToken::Analysis &a)
  {
    return Ftw_eps + pow(dynlex_base, -dynlex_beta * a.prob);
  };

  virtual void tw_put_info(moot::TokenWriter *tw)
  {
    mootDynLexHMM::tw_put_info(tw);
    tw->printf_raw("  +DynHMM class      : %s\n", "mootDynLexHMM_Boltzmann");
    tw->printf_raw("   dynlex_base       : %g\n", dynlex_base);
    tw->printf_raw("   dynlex_beta       : %g\n", dynlex_beta);
  };
};


moot_END_NAMESPACE

#endif /* _MOOT_DYNHMM_H */