ConcCommon.h

Go to the documentation of this file.

//-*- Mode: C++ -*-
//
//  This file is part of DDC.
// 
//  DDC is free software: you can redistribute it and/or modify
//  it under the terms of the GNU Lesser General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
// 
//  DDC is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU Lesser General Public License for more details.
// 
//  You should have received a copy of the GNU Lesser General Public License
//  along with DDC.  If not, see <http://www.gnu.org/licenses/>.
//
#ifndef __ConcCommon_H_
#define __ConcCommon_H_

#include "../CommonLib/utilit.h"
#include "../CommonLib/ddcLog.h"
#include "list"
#include "limits.h"


#include "../GraphanLib/GraphmatFile.h"
#include "../LemmatizerLib/Lemmatizers.h"

#include "../AgramtabLib/EngGramTab.h"
#include "../AgramtabLib/RusGramTab.h"
#include "../AgramtabLib/GerGramTab.h"
#include "../CommonLib/DDC_common.h"
#include "../CommonLib/ddcObject.h"
#include "../tinyxml/tinyxml.h"

#include "../ConcordLib/CCurl.h"

inline void ddcInitGlobal(void) {
    ddcInitLocale();
    ddcInitCurl();
}

const char globalFieldDelimeter = '\t';
const string PredefinedTableLineTag = "l";
const string ChunkIndexName = "chunk";
const string PredefinedFileBreakName = "file";
const string PredefinedTextAreaBreakName = "textarea";

typedef DWORD CTokenNo;

typedef DWORD CFileNo;

const size_t MaxShortOccurCacheSize = 1000000;

#define DDC_SORTKEY_MAXLEN 256

const size_t MaxBiblStringLen = 20000;

class CShortOccurCache 
{
    struct CDataReference 
    {
        // the start index of subvector in m_OccurrencesBody
        size_t m_VectorStartOffset;
        // the length of subvector in m_OccurrencesBody
        size_t m_VectorLength;


        CDataReference (size_t VectorStartOffset, size_t VectorLength)
        {
            m_VectorStartOffset = VectorStartOffset;
            m_VectorLength = VectorLength;
        };
    };
    //map<CTokenNo, CDataReference> m_IndexItemNo2Occurrences;
    vector <CDataReference> m_IndexItemNo2Occurrences;

    vector<CTokenNo>                        m_OccurrencesBody;
public:
    void                            Clear();
    size_t                          AddNewIndexItemNoToCache(const CTokenNo* pStart, const CTokenNo* pEnd );
    const CTokenNo*                 GetOccurrencesFromCache(const int CacheId, DWORD& Length) const;
    bool                            CouldContainMore() const;

};

enum HitSortEnum {
    NoSort = 0,
    LessByDate = 1,
    GreaterByDate = 2,
    LessBySize = 3,
    GreaterBySize = 4,
    LessByFreeBiblField = 5,
    GreaterByFreeBiblField = 6,
    LessByRank = 7,
    GreaterByRank = 8,
    LessByMiddleContext = 9,
    GreaterByMiddleContext = 10,
    LessByLeftContext = 11,
    GreaterByLeftContext = 12,
    LessByRightContext = 13,
    GreaterByRightContext = 14,
    RandomSort = 15,
    LessByCountKey = 16,
    GreaterByCountKey = 17,
    LessByCountValue = 18,
    GreaterByCountValue = 19,
    LessByPruneKey = 20,
    GreaterByPruneKey = 21,
    HitSortsCount = 22
};

inline bool IsPruneFilterType(HitSortEnum e)
{ return e==LessByPruneKey || e==GreaterByPruneKey; }

enum HitSortOrderEnum {
    hsoDescending = -1,
    hsoNone = 0,
    hsoAscending = 1,
    hsoAscendingCountKeys = 2,
    hsoDescendingCountKeys = 3,
    hsoAscendingCountValues = 4,
    hsoDescendingCountValues =  5
};

inline bool IsNullSort(HitSortOrderEnum e)
{ return e==hsoNone; }

inline bool IsCountSort(HitSortOrderEnum e)
{ return e >= hsoAscendingCountKeys && e <= hsoDescendingCountValues; }

inline bool IsCountKeySort(HitSortOrderEnum e)
{ return e==hsoAscendingCountKeys || e==hsoDescendingCountKeys; }

inline bool IsCountValueSort(HitSortOrderEnum e)
{ return e==hsoAscendingCountValues || e==hsoDescendingCountValues; }

static const char *HitSortEnumNames[HitSortsCount] =
{
    "NoSort",
    "LessByDate",
    "GreaterByDate",
    "LessBySize",
    "GreaterBySize",
    "LessByFreeBiblField",
    "GreaterByFreeBiblField",
    "LessByRank",
    "GreaterByRank",
    "LessByMiddleContext",
    "GreaterByMiddleContext",
    "LessByLeftContext",
    "GreaterByLeftContext",
    "LessByRightContext",
    "GreaterByRightContext",
    "RandomSort",
    "LessByCountKey",
    "GreaterByCountKey",
    "LessByCountValue",
    "GreaterByCountValue",
    "LessByPruneKey",
    "GreaterByPruneKey"
};

static const char *HitSortEnumStrings[HitSortsCount] = {
    "no_sort",
    "asc_by_date",
    "desc_by_date",
    "asc_by_size",
    "desc_by_size",
    "asc",
    "desc",
    "asc_by_rank",
    "desc_by_rank",
    "asc_middle",
    "desc_middle",
    "asc_left",
    "desc_left",
    "asc_right",
    "desc_right",
    "random",
    "asc_by_key",
    "desc_by_key",
    "asc_by_count",
    "desc_by_count",
    "prune_asc",
    "prune_desc"
};


class CFreeBiblIndexInterface {
public:
    virtual ~CFreeBiblIndexInterface()
    {};
    
    virtual string GetStringValue(DWORD FileNo) const = 0;

    virtual DWORD GetIntegerLowerBound(const string &Value) const = 0;
};

struct CHitSortKey {
    int     i;

    string  s;

    CHitSortKey(int i_=0) : i(i_)
    {};

    CHitSortKey(int i_, const string& s_) : i(i_), s(s_)
    {};

    inline void clear()
    { i=0; s.clear(); };

    inline bool operator< (const CHitSortKey &x) const
    { return (i < x.i) || (i==x.i && s < x.s); };

    inline bool operator> (const CHitSortKey &x) const
    { return (i > x.i) || (i==x.i && s > x.s); };

    inline void assign(int i_)
    { i=i_; s.clear(); };

    inline void assign(int i_, const string &s_)
    { i=i_; s=s_; };

    inline void operator= (const std::pair<int,string>& p)
    { i=p.first; s=p.second; };

    inline void operator= (int i_)
    { i=i_; s.clear(); };

    inline void operator= (const string &s_)
    { i=0; s=s_; };
};

struct CDDCFilterWithBounds 
{
    HitSortEnum             m_FilterType;

    string                  m_AttrName;

    bool                    m_bSet;

    bool                    m_bNegated; 

    int m_ContextMatchId;

    int m_ContextOffset;

    CHitSortKey m_KeyLo;

    CHitSortKey m_KeyHi;

    set<int> m_SatisfiedValues;

    const CFreeBiblIndexInterface* m_BiblIndex;

    class CQFilter * m_Parent;

    inline void clear()
    {
        m_SatisfiedValues.clear();
    };

    inline HitSortOrderEnum SortOrder() const {
        switch (m_FilterType) {
            case GreaterByDate:
            case GreaterByRank:
            case GreaterBySize:
            case GreaterByFreeBiblField:
            case GreaterByMiddleContext:
            case GreaterByLeftContext:
            case GreaterByRightContext:
            case GreaterByCountKey:
            case GreaterByCountValue:
            case GreaterByPruneKey:
                return hsoDescending;
            case NoSort:
                return hsoNone;
            default:
                return hsoAscending;
        };
    };

    inline bool IsFileFilter() const
    {
        switch (m_FilterType) {
            case NoSort:
            case LessByDate:
            case GreaterByDate:
                return true; //-- special handling in CConcXml::FileMatches()
            case LessByFreeBiblField:
            case GreaterByFreeBiblField:
                return true; //-- usual case of metadata-based filtering
            case LessByRank:
            case GreaterByRank:
            case RandomSort:
                return true; //-- non-filter operators are ignored
            case LessByPruneKey:
            case GreaterByPruneKey:
                //return !m_Parent || !((CQFPrune*)m_Parent)->m_keys || ((CQFPrune*)m_Parent)->m_keys->canCountByFile(); //-- not declared here
            case LessBySize:
            case GreaterBySize:
            case LessByMiddleContext:
            case GreaterByMiddleContext:
            case LessByLeftContext:
            case GreaterByLeftContext:
            case LessByRightContext:
            case GreaterByRightContext:
            default:
                return false; //-- not file-based
        }
    };

    inline bool IsTrivialFilter(void) const
    { return m_FilterType == NoSort; };

    inline bool IsPruneFilter(void) const
    { return IsPruneFilterType(m_FilterType); };


    CDDCFilterWithBounds()
        : m_bSet(false),
          m_bNegated(false),
          m_FilterType(NoSort),
          m_ContextMatchId(0),
          m_ContextOffset(0),
          m_KeyLo(INT_MIN),
          m_KeyHi(INT_MAX),
          m_BiblIndex(NULL),
          m_Parent(NULL)
    {};
};

struct CHit 
{
    DWORD   m_BreakNo;
    DWORD   m_HighlightOccurrenceEnd;
    DWORD   m_FileNo;

    union {
        size_t  m_DebugRankNo;

        size_t  m_Count;
    } m_Value;

    CHitSortKey m_SortKey;

    CHit(DWORD BreakNo=0) : m_BreakNo(BreakNo), m_SortKey(0) { m_Value.m_Count=0; };
};

enum DDCFormatTypeEnum {
    DDC_ResultHTML,
    DDC_ResultText,
    DDC_ResultTable,
    DDC_ResultDocIds,
    DDC_ResultJson
};

typedef map<string,CShortOccurCache> CShortOccurCacheMap;

typedef vector<CTokenNo> COccurrBuffer;

const string MorphAnnotationsDelim = "#";
const string MorphAnnotationsDelimRegExp = "[^#]*";

extern bool InitConcordDicts();
extern void FreeConcordDicts();
extern const CLemmatizer* GetLemmatizerByLanguage (MorphLanguageEnum Langua);
extern const CAgramtab* GetGramtabByLanguage (MorphLanguageEnum Langua);

struct CHitCompareByBreak
{
    CHitCompareByBreak() {};

    inline bool operator()(const CHit& h, const DWORD brkno) const
    { return h.m_BreakNo < brkno; };
};

string GetIndexItemSetByVectorString(const vector<string> &TokenProperties, bool bRegexp);

#endif

/*--- emacs style variables ---
 * Local Variables:
 *  mode: C++
 *  c-file-style: "ellemtel"
 *  c-basic-offset: 4
 *  tab-width: 8
 *  indent-tabs-mode: nil
 * End:
 */