mootHMM.h

Go to the documentation of this file.

/* -*- Mode: C++ -*- */
/*
   libmoot : moocow's part-of-speech tagging library
   Copyright (C) 2003-2014 by Bryan Jurish <moocow@cpan.org>

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
   
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.
   
   You should have received a copy of the GNU Lesser General Public
   License along with this library; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

/*--------------------------------------------------------------------------
 * File: mootHMM.h
 * Author: Bryan Jurish <moocow@cpan.org>
 * Description:
 *   + moot PoS tagger : Hidden Markov Model (Disambiguator): headers
 *--------------------------------------------------------------------------*/

#ifndef _MOOT_HMM_H
#define _MOOT_HMM_H

#include <math.h>

#include <mootFlavor.h>
#include <mootTokenIO.h>
#include <mootZIO.h>
#include <mootBinHeader.h>
#include <mootUtils.h>

#include <mootClassfreqs.h>
//#include <mootLexfreqs.h> //-- included by mootClassfreqs.h

#include <mootSuffixTrie.h>
/*
#ifdef MOOT_ENABLE_SUFFIX_TRIE
# include <mootSuffixTrie.h>
#else
# include <mootEnum.h>
# include <mootAssocVector.h>
# include <mootTrieVector.h>
# include <mootNgrams.h>
#endif //--MOOT_ENABLE_SUFFIX_TRUE
*/

#define MOOT_LEX_UNKNOWN_TOKENS
//#undef MOOT_LEX_UNKNOWN_TOKENS

#define MOOT_LEX_UNKNOWN_CLASSES
//#undef MOOT_LEX_UNKNOWN_CLASSES

#define MOOT_LEX_NONALPHA
//#undef MOOT_LEX_NONALPHA

#undef MOOT_LEX_IS_TIEBREAKER

moot_BEGIN_NAMESPACE

/*--------------------------------------------------------------------------
 * mootHMM : HMM class
 *--------------------------------------------------------------------------*/

class mootHMM {
public:
  /*---------------------------------------------------------------------*/

  typedef mootEnumID TagID;

  typedef mootEnumID TokID;

  typedef mootEnumID FlavorID;

  typedef mootEnumID ClassID;

  typedef moot::VerbosityLevel VerbosityLevel;

  /*------------------------------------------------------------
   * public typedefs : lexical classes
   */


  typedef set<TagID> LexClass;

  struct LexClassHash {
  public:
    inline size_t operator()(const LexClass &x) const {
      size_t hv = 0;
      for (LexClass::const_iterator xi = x.begin(); xi != x.end(); ++xi) {
        hv = 5*hv + *xi;
      }
      return hv;
    };
  };

  struct LexClassEqual {
  public:
    inline size_t operator()(const LexClass &x, const LexClass &y) const {
      return x==y;
    };
  };

  /*---------------------------------------------------------------------*/
  
  typedef mootEnum<mootTagString,
                    moot_hash<mootTagString>,
                    equal_to<mootTagString> >
          TagIDTable;

  typedef mootEnum<mootTokString,
                    moot_hash<mootTokString>,
                    equal_to<mootTokString> >
          TokIDTable;

  typedef mootEnum<LexClass,
                   LexClassHash,
                   LexClassEqual>
          ClassIDTable;

  //typedef map<TagID,ProbT> LexProbSubTable;
  typedef AssocVector<TagID,ProbT> LexProbSubTable;

  typedef LexProbSubTable LexClassProbSubTable;

  typedef vector<LexProbSubTable> LexProbTable;

  typedef LexProbTable LexClassProbTable;

  typedef ProbT* UnigramProbTable;

  class Trigram {
  public:
    TagID tag1;  
    TagID tag2;  
    TagID tag3;  
    //
  public:
    struct HashFcn {
    public:
      inline size_t operator()(const Trigram &x) const {
        return (0xdeece66d * ((0xdeece66d * x.tag1) + x.tag2)) + x.tag3;
      };
    };
    //
    struct EqualFcn {
    public:
      inline size_t operator()(const Trigram &x, const Trigram &y) const {
        return 
          x.tag1==y.tag1 && x.tag2==y.tag2 && x.tag3==y.tag3;
      };
    };
    //
  public:
    Trigram(TagID t1=0, TagID t2=0, TagID t3=0)
      : tag1(t1), tag2(t2), tag3(t3)
    {};
    //
    ~Trigram(void) {};
  };

  typedef
    hash_map<Trigram,ProbT,
             Trigram::HashFcn,
             Trigram::EqualFcn>
    TrigramProbHash;

  typedef ProbT* TrigramProbArray;

  typedef Trigram          NgramProbKey;    
  typedef TrigramProbHash  NgramProbHash;   
  typedef TrigramProbArray NgramProbArray;  


  /*---------------------------------------------------------------------*/

  class ViterbiNode {
  public:
    TagID tagid;                  
    TagID ptagid;                 
    ProbT lprob;                  

    class ViterbiNode *pth_prev;  
    class ViterbiNode *nod_next;  
  };

  class ViterbiRow {
  public:
    TagID  tagid;                 
    ProbT  wprob;                 
    class ViterbiNode *nodes;     
    class ViterbiRow  *row_next;  
  };

  class ViterbiColumn {
  public:
    ViterbiRow    *rows;     
    ViterbiColumn *col_prev; 
    ProbT          bbestpr;  
    ProbT          bpprmin;  
  };

  struct ViterbiPathNode {
  public:
    ViterbiNode      *node;      
    ViterbiPathNode  *path_next; 
  };


public:
  /*---------------------------------------------------------------------*/
  int verbose;

  size_t ndots;

  bool save_ambiguities;

  bool save_flavors;

  bool save_mark_unknown;

  /*---------------------------------------------------------------------*/

  bool      hash_ngrams;

  bool      relax;

  bool      use_lex_classes;

  bool      use_flavors;

  TagID     start_tagid;

  ProbT     unknown_lex_threshhold;

  ProbT     unknown_class_threshhold;

  LexClass   uclass;

  /*---------------------------------------------------------------------*/
  ProbT             nglambda1;    
  ProbT             nglambda2;    
  ProbT             nglambda3;    
  ProbT             wlambda0;     
  ProbT             wlambda1;     
  ProbT             clambda0;     
  ProbT             clambda1;     
  ProbT             beamwd;

  /*---------------------------------------------------------------------*/
  TokIDTable        tokids;     
  TagIDTable        tagids;     
  ClassIDTable      classids;   
  mootTaster        taster;     

  /*---------------------------------------------------------------------*/
  size_t            n_tags;     
  size_t            n_toks;     
  size_t            n_classes;  
  LexProbTable      lexprobs;   
  LexClassProbTable lcprobs;    
  NgramProbHash     ngprobsh;   
  NgramProbArray    ngprobsa;   
#ifdef MOOT_ENABLE_SUFFIX_TRIE
  SuffixTrie        suftrie;    
#endif


  /*---------------------------------------------------------------------*/
  ViterbiColumn     *vtable;    

  /*---------------------------------------------------------------------*/
  size_t             nsents;      
  size_t             ntokens;     
  size_t             nnewtokens;  
  size_t             nunclassed;  
  size_t             nnewclasses; 
  size_t             nunknown;    
  size_t             nfallbacks;  

protected:
  /*---------------------------------------------------------------------*/
  ViterbiNode     *trash_nodes;     
  ViterbiRow      *trash_rows;      
  ViterbiColumn   *trash_columns;   
  ViterbiPathNode *trash_pathnodes; 

  /*---------------------------------------------------------------------*/
  TagID             vtagid;     
  ProbT             vbestpr;    
  ProbT             vtagpr;     
  ProbT             vwordpr;    
  ViterbiNode      *vbestpn;    
  ViterbiPathNode  *vbestpath;  
  //ProbT           bbestpr;   /**< Best current (log-)probability for beam pruning */
  //ProbT           bpprmin;   /**< Minimum previous probability for beam pruning */

public:
  /*---------------------------------------------------------------------*/
  mootHMM(void)
    : verbose(1),
      ndots(0),
      save_ambiguities(false),
      save_flavors(false),
      save_mark_unknown(false),
      hash_ngrams(false),
      relax(true),
      use_lex_classes(true),
      use_flavors(true),
      start_tagid(0),
      unknown_lex_threshhold(1.0),
      unknown_class_threshhold(1.0),
      nglambda1(mootProbEpsilon),
      nglambda2(1.0 - mootProbEpsilon),
      wlambda0(mootProbEpsilon),
      wlambda1(1.0 - mootProbEpsilon),
      clambda0(mootProbEpsilon),
      clambda1(1.0 - mootProbEpsilon),
      beamwd(1000),
      n_tags(0),
      n_toks(0),
      n_classes(0),
      ngprobsa(NULL),
      vtable(NULL),
      nsents(0),
      ntokens(0),
      nnewtokens(0),
      nunclassed(0),
      nnewclasses(0),
      nunknown(0),
      nfallbacks(0),
      trash_nodes(NULL),
      trash_rows(NULL),
      trash_columns(NULL), 
      trash_pathnodes(NULL),
      vbestpn(NULL),
      vbestpath(NULL)
  {
    //-- create special token entries
    unknown_token_name("@UNKNOWN");
    unknown_tag_name("UNKNOWN");
    uclass = LexClass();
  };


  //~mootHMM(void) { clear(true,false); };
  virtual ~mootHMM(void) { clear(false,false); };

  /*------------------------------------------------------------*/
  void clear(bool wipe_everything=true, bool unlogify=false);

  /*------------------------------------------------------------*/
  bool save(const char *filename, int compression_level=MOOT_DEFAULT_COMPRESSION);

  bool save(mootio::mostream *obs, const char *filename=NULL);

  bool _bindump(mootio::mostream *obs, const mootBinIO::HeaderInfo &hdr, const char *filename=NULL);

  bool load(const char *filename=NULL);

  bool load(mootio::mistream *ibs, const char *filename=NULL);

  bool _binload(mootio::mistream *ibs, const mootBinIO::HeaderInfo &hdr, const char *filename=NULL);

  /*------------------------------------------------------------*/
  inline void unknown_token_name(const mootTokString &name)
  {
    tokids.unknown_name(name);
  };

  inline void unknown_tag_name(const mootTokString &name)
  {
    tagids.unknown_name(name);
  };

  inline void unknown_class_name(const mootTagSet &tagset)
  {
    tagset2lexclass(tagset,&uclass,false);
  };


  //------------------------------------------------------------

  virtual bool load_model(const string &modelname,
                          const mootTagString &start_tag_str="__$",
                          const char *myname="mootHMM::load_model()",
                          bool  do_estimate_nglambdas=true,
                          bool  do_estimate_wlambdas=true,
                          bool  do_estimate_clambdas=true,
                          bool  do_build_suffix_trie=true,
                          bool  do_compute_logprobs=true);

  virtual bool compile(const mootLexfreqs &lexfreqs,
                       const mootNgrams &ngrams,
                       const mootClassfreqs &classfreqs,
                       const mootTagString &start_tag_str="__$",
                       const mootTaster &mtaster=builtinTaster);

  void assign_ids_fl(void);

  void assign_ids_lf(const mootLexfreqs &lexfreqs);

  void assign_ids_ng(const mootNgrams   &ngrams);

  void assign_ids_cf(const mootClassfreqs &classfreqs);

  void compile_unknown_lexclass(const mootClassfreqs &classfreqs);

  bool estimate_lambdas(const mootNgrams &ngrams);

  bool estimate_wlambdas(const mootLexfreqs &lf);

  bool estimate_clambdas(const mootClassfreqs &cf);

  bool build_suffix_trie(const mootLexfreqs &lf,
                         const mootNgrams   &ng,
                         bool  verbose=false)
  {
#ifdef MOOT_ENABLE_SUFFIX_TRIE
    return suftrie.build(lf,ng,tagids,start_tagid,verbose);
#else
    return false;
#endif
  };

  bool compute_logprobs(void);

  inline void set_ngram_prob(ProbT p, TagID t1=0, TagID t2=0, TagID t3=0)
  {
    if (hash_ngrams) {   // +hash
      ngprobsh[Trigram(t1,t2,t3)] = p; 
    } else {             // -hash
      ngprobsa[(n_tags*((n_tags*t1)+t2))+t3] = p;
    }
  };

  //------------------------------------------------------------
  // Tagging: Top-level

  void tag_sentence(mootSentence &sentence);

  virtual void tag_io(TokenReader *reader, TokenWriter *writer);

  virtual void tag_stream(TokenReader *reader, TokenWriter *writer);

  /*====================================================================
   * VITERBI: Mid-level
   *====================================================================*/

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  void viterbi_clear(void);

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: single iteration: (mootToken)
  inline void viterbi_step(const mootToken &token) {
    if (token.toktype() != TokTypeVanilla) return; //-- ignore non-vanilla tokens
    ++ntokens;
    LexClass tok_class;
    token2lexclass(token, tok_class);
    viterbi_step(token2id(token.text()), tok_class, token.text());
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: single iteration: (TokID,LexClass=set<ClassID>)
  inline void viterbi_step(TokID tokid,
                           const LexClass &lexclass,
                           const mootTokString &toktext="")
  {
    if (use_lex_classes) {
      if (lexclass.empty()) {
        ++nunclassed;
        viterbi_step(tokid, 0, uclass, toktext);
      } else {
        //-- non-empty class : get ID (assign empty distribution if unknown)
        ClassID classid = class2id(lexclass,0,1);
        viterbi_step(tokid, classid, lexclass, toktext);
      }
    } else {
      //-- !use_lex_classes
      if (lexclass.empty()) {
        ++nunclassed;
        viterbi_step(tokid, toktext);
      } else {
        viterbi_step(tokid, 0, lexclass, toktext);
      }
    }
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: single iteration: (TokID,ClassID,LexClass)
  void viterbi_step(TokID tokid,
                    ClassID classid,
                    const LexClass &lclass,
                    const mootTokString &toktext="");

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: single iteration: (TokID)
  void viterbi_step(TokID tokid, const mootTokString &toktext="");

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: single iteration: (TokString)
  inline void viterbi_step(const mootTokString &token_text) {
    return viterbi_step(token2id(token_text), token_text);
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: single iteration: (TokString,set<TagString>)
  inline void viterbi_step(const mootTokString &token_text, const set<mootTagString> &tags)
  {
    LexClass lclass;
    tagset2lexclass(tags,&lclass);
    viterbi_step(token2id(token_text), lclass, token_text);
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: single iteration: (TokID,TagID,col=NULL)
  void viterbi_step(TokID tokid, TagID tagid, ViterbiColumn *col=NULL);

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: single iteration: (TokString,TagString)
  inline void viterbi_step(const mootTokString &toktext, const mootTagString &tag)
  {
    return viterbi_step(token2id(toktext), tagids.name2id(tag));
  };


  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: finish
  inline void viterbi_finish(const TagID final_tagid)
  {
    viterbi_step(0, final_tagid);
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline void viterbi_finish(void)
  {
    viterbi_step(0, start_tagid);
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  // Viterbi: finish
  void viterbi_flush(TokenWriter *writer, mootSentence &toks, ViterbiNode *nod);

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline void tag_mark_best(mootSentence &sentence)
  {
    tag_mark_best(viterbi_best_path(), sentence);
  };

  //@param skip_first if true (default), the first path node will be skipped (it's usually an implicit BOS marker) 
  void tag_mark_best(ViterbiPathNode *pnod, mootSentence &sentence);

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  void tag_dump_trace(mootSentence &sentence, bool dumpPredict=false);



  //------------------------------------------------------------
  // Viterbi: Low-Level: path utilities


  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline ViterbiPathNode *viterbi_best_path(void)
  {
    return viterbi_node_path(viterbi_best_node());
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline ViterbiPathNode *viterbi_best_path(TagID tagid)
  {
    return viterbi_node_path(viterbi_best_node(tagid));
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline ViterbiPathNode *viterbi_best_path(const mootTagString &tagstr)
  {
    return viterbi_best_path(tagids.name2id(tagstr));
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ViterbiNode *viterbi_best_node(void);

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ViterbiNode *viterbi_best_node(TagID tagid);

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ViterbiNode* viterbi_flushable_node(void);
 
  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ViterbiPathNode *viterbi_node_path(ViterbiNode *node);

  //------------------------------------------------------------
  // Viterbi: low-level: iteration

  //{@

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline bool viterbi_column_ok(const ViterbiColumn *col) const
  {
    return (col && col->rows  && col->rows->nodes);
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ViterbiColumn *viterbi_populate_row(TagID             curtagid,
                                      ProbT             wordpr   =MOOT_PROB_ONE,
                                      ViterbiColumn     *col     =NULL,
                                      ProbT             probmin  =MOOT_PROB_NONE);


  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  void viterbi_clear_bestpath(void);

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  void _viterbi_step_fallback(TokID tokid, ViterbiColumn *col);


  //------------------------------------------------------------

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline ViterbiNode *viterbi_get_node(void)
  {
    ViterbiNode *nod;
    if (trash_nodes != NULL) {
      nod         = trash_nodes;
      trash_nodes = nod->nod_next;
    } else {
      nod = new ViterbiNode();
    }
    return nod;
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline ViterbiRow *viterbi_get_row(void)
  {
    ViterbiRow *row;
    if (trash_rows != NULL) {
      row        = trash_rows;
      trash_rows = row->row_next;
    } else {
      row = new ViterbiRow();
    }
    return row;
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline ViterbiColumn *viterbi_get_column(void)
  {
    ViterbiColumn *col;
    if (trash_columns != NULL) {
      col           = trash_columns;
      trash_columns = col->col_prev;
    } else {
      col = new ViterbiColumn();
    }
    return col;
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline ViterbiPathNode *viterbi_get_pathnode(void)
  {
    ViterbiPathNode *pnod;
    if (trash_pathnodes != NULL) {
      pnod            = trash_pathnodes;
      trash_pathnodes = pnod->path_next;
    } else {
      pnod = new ViterbiPathNode();
    }
    return pnod;
  };


  //------------------------------------------------------------
  // Low-level: ID Lookup

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  inline TokID token2id(const mootTokString &token) const
  {
#ifdef MOOT_LEX_NONALPHA
    TokID tokid = tokids.name2id(token);
    return tokid || !use_flavors ? tokid : taster.flavor_id(token);
#else
    TokID tokid = use_flavors ? taster.flavor_id(token) : 0;
    return tokid ? tokid : tokids.name2id(token);
#endif
  };

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  void token2lexclass(const mootToken &token, LexClass &tok_class) const;

  //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  LexClass *tagset2lexclass(const mootTagSet &tagset, LexClass *lclass=NULL, bool add_tagids=false);


  ClassID class2id(const LexClass &lclass, bool autopopulate=true, bool autocreate=true);


  //------------------------------------------------------------

  /*------------------------------------------------------------
   * Lexical Probability Lookup
   */

  inline const ProbT wordp(const TokID tokid, const TagID tagid) const
  {
    if (tokid >= lexprobs.size()) return MOOT_PROB_ZERO;
    const LexProbSubTable &lps = lexprobs[tokid];
    LexProbSubTable::const_iterator lpsi = lps.find(tagid);
    return lpsi != lps.end() ? lpsi->value() : MOOT_PROB_ZERO;
  };

  inline const ProbT wordp(const mootTokString &tokstr, const mootTagString &tagstr) const
  {
    return wordp(token2id(tokstr), tagids.name2id(tagstr));
  };

  /*------------------------------------------------------------
   * Lexical-Class Probability Lookup
   */
  inline const ProbT classp(const ClassID classid, const TagID tagid) const
  {
    if (classid >= lcprobs.size()) return MOOT_PROB_ZERO;
    const LexClassProbSubTable &lps = lcprobs[classid];
    LexClassProbSubTable::const_iterator lpsi = lps.find(tagid);
    return lpsi != lps.end() ? lpsi->value() : MOOT_PROB_ZERO;
  };

  inline const ProbT classp(const LexClass &lclass, const mootTagString &tagstr) const
  {
    return classp(classids.name2id(lclass), tagids.name2id(tagstr));
  };

  /*------------------------------------------------------------
   * Unigram Probability Lookup
   */
  inline const ProbT tagp(const TagID tagid) const
  {
    return tagp(0,0,tagid);
  };

  inline const ProbT tagp(const mootTagString &tag) const
  {
    return tagp(tagids.name2id(tag));
  };

  /*------------------------------------------------------------
   * Bigram Probability Lookup
   */
  inline const ProbT tagp(const TagID prevtagid, const TagID tagid) const
  {
    return tagp(0,prevtagid,tagid);
  };

  inline const ProbT tagp(const mootTagString &prevtag, const mootTagString &tag) const
  {
    return tagp(tagids.name2id(prevtag), tagids.name2id(tag));
  };

  /*------------------------------------------------------------
   * Trigram probability lookup
   */
  inline const ProbT tagp(const Trigram &trigram, ProbT ProbZero=MOOT_PROB_ZERO) const
  {
    if (!hash_ngrams) return tagp(trigram.tag1, trigram.tag2, trigram.tag3);
    NgramProbHash::const_iterator ngpi = ngprobsh.find(trigram);
    return ngpi != ngprobsh.end() ? ngpi->second : ProbZero;
  };

  inline const ProbT tagp(const TagID prevtagid2, const TagID prevtagid1, const TagID tagid) const
  {
    if (!hash_ngrams) { //-- -hash
      return
        ngprobsa && prevtagid2 < n_tags && prevtagid1 < n_tags && tagid < n_tags
        ? ngprobsa[(n_tags*((n_tags*prevtagid2)+prevtagid1))+tagid]
        : MOOT_PROB_ZERO;
    } else {            //-- +hash
      //-- trigram as stored (pre-smoothed)
      Trigram ng(prevtagid2,prevtagid1,tagid);
      ProbT p = tagp(ng, 1);
      if (p != 1) return p;
      //-- fallback: bigram as stored (pre-smoothed)
      ng.tag1 = 0;
      p = tagp(ng, 1);
      if (p != 1) return p;
      //-- fallback: unigram as stored (pre-smoothed)
      ng.tag2 = 0;
      p = tagp(ng, 1);
      if (p != 1) return p;
      //-- fallback: "unknown" unigram (pre-smoothed)
      ng.tag3 = 0;
      return tagp(ng, MOOT_PROB_ZERO);
    }
  };

  inline const ProbT tagp(const mootTagString &prevtag2,
                          const mootTagString &prevtag1,
                          const mootTagString &tag)
    const
  {
    return tagp(tagids.name2id(prevtag2), tagids.name2id(prevtag1), tagids.name2id(tag));
  };


  //------------------------------------------------------------
  // Error Reporting

  void carp(const char *fmt, ...);

  //------------------------------------------------------------
  // public methods: low-level: debugging

  void txtdump(FILE *file, bool dump_constants=true, bool dump_lexprobs=true, bool dump_classprobs=true, bool dump_suftrie=true, bool dump_ngprobs=true);

  void viterbi_txtdump(TokenWriter *w, int ncols=0);

  void viterbi_txtdump_col(TokenWriter *w, ViterbiColumn *col, int colnum=0);
};

moot_END_NAMESPACE

#endif /* _MOOT_HMM_H */