Lemmatizers.h

Go to the documentation of this file.

//-*- Mode: C++; c-basic-offset: 8; -*-*/
// ==========  This file is under  LGPL, the GNU Lesser General Public Licence
// ==========  Dialing Lemmatizer (www.aot.ru)
// ==========  Copyright by Alexey Sokirko, Andrey Putrin

#ifndef __LEMMATIZERS_H_
#define __LEMMATIZERS_H_

#pragma warning (disable : 4786)

#include "MorphDict.h"
#include "Paradigm.h"
#include "Statistic.h"
#include "Predict.h"
#include "../common/ddcIconv.h"

#pragma warning (disable : 4250)

class CGraphmatFile;

typedef enum {  subjFinance = 1, subjComputer = 2, subjLiterature = 4   } SubjectEnum;


class CLemmatizer  : public CMorphDict
{
protected:
        string                          m_Registry;
        
    // Postfixes-particles, that do not change the meaning 
        set<string>             m_HyphenPostfixes;
    // productive prefixes
    set<string>                 m_HyphenPrefixes;

        CStatistic                      m_Statistic;
        CPredictBase            m_Predict;
        set<string>                     m_PrefixesSet;


        virtual void                    FilterSrc(string& src) const = 0;

        string                                  GetRegistryString()  const      {return m_Registry;     };
        string                                  GetPath()  const;
        void                                    ReadOptions(string FileName);
                
        
        bool                                    LemmatizeWord(string& InputWordStr, const bool cap, const bool predict, vector<CAutomAnnotationInner>& results, bool bGetLemmaInfos) const;
        void                                    AssignWeightIfNeed(vector<CAutomAnnotationInner>& FindResults) const;

        // prediction by suffix
        bool                                    CheckAbbreviation(string InputWordStr,vector<CAutomAnnotationInner>& FindResults, bool is_cap) const;
        CAutomAnnotationInner   ConvertPredictTupleToAnnot(const CPredictTuple& input) const;
        void                                    PredictByDataBase(string InputWordStr,  vector<CAutomAnnotationInner>&  results,bool is_cap) const;
        bool                                    IsPrefix(const string& Prefix) const;


public:
        bool                                    m_bLoaded;
        bool                                    m_bMaximalPrediction;
        bool                                    m_bUseStatistic;
        bool                                    m_bAllowRussianJo;


public:
        //-- BEGIN moo: character set hacks
        std::string                             m_enc_int;     
        std::string                             m_enc_ext;     
        ddcIconv                               *m_ic_ext2int;  
        ddcIconv                               *m_ic_int2ext;  
        //-- END moo
        
        
        CLemmatizer(MorphLanguageEnum Language);
        virtual ~CLemmatizer();

        //  basic methods
        MorphLanguageEnum GetLanguage()  const {return m_pFormAutomat->m_Language; };
        const CStatistic& GetStatistic() const;
        bool    CheckABC(const string& WordForm) const;
        bool    IsHyphenPostfix(const string& Postfix) const;
        bool    IsHyphenPrefix(const string& Prefix) const;


        //-- moo: character set hacks
        bool initIconv(const string &enc_internal="", const string &enc_external="UTF8");
        inline std::string recode_ext2int(const std::string s_ext) const { return m_ic_ext2int==NULL ? s_ext : m_ic_ext2int->convert(s_ext); }
        inline std::string recode_int2ext(const std::string s_int) const { return m_ic_int2ext==NULL ? s_int : m_ic_int2ext->convert(s_int); }

        
        //  loading
        bool    LoadDictionariesRegistry(string& strError) ;
        bool    LoadStatisticRegistry(SubjectEnum subj);

        //  main interfaces
        bool    CreateParadigmCollection(bool bNorm, string& WordStr, bool capital, bool bUsePrediction, vector<CFormInfo>& Result) const;
        void    GetAllAncodesQuick(const BYTE* WordForm, bool capital, BYTE* OutBuffer, bool bUsePrediction) const;
        //string        GetAllAncodesAndLemmasQuick(string& InputWordStr, bool capital) const;
        bool    GetAllAncodesAndLemmasQuick(string& InputWordStr, bool capital, char* OutBuffer, size_t MaxBufferSize, bool bUsePrediction) const;
        bool    CreateParadigmFromID(DWORD id, CFormInfo& Result) const;
        bool    ProcessHyphenWords(CGraphmatFile* piGraphmatFile) const;


};





class CLemmatizerRussian : public CLemmatizer
{
public:
        CLemmatizerRussian();
        virtual ~CLemmatizerRussian() {};
        void FilterSrc(string& src) const;
};


class  CLemmatizerEnglish : public CLemmatizer
{
        void FilterSrc(string& src) const;
public:
        CLemmatizerEnglish();
        virtual ~CLemmatizerEnglish() {};
        
};




class  CLemmatizerGerman: public CLemmatizer
{
        void FilterSrc(string& src) const;
public:
        CLemmatizerGerman();
        virtual ~CLemmatizerGerman() {};
        
};





#endif //__LEMMATIZERS_H_