IndexSetForQueryingStage.h

Go to the documentation of this file.

//-*- Mode: C++ -*-
//
//  DDC originally by Alexey Sokirko
//  Changes and modifications 2011-2018 by Bryan Jurish
//
//  This file is part of DDC.
// 
//  DDC is free software: you can redistribute it and/or modify
//  it under the terms of the GNU Lesser General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
// 
//  DDC is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU Lesser General Public License for more details.
// 
//  You should have received a copy of the GNU Lesser General Public License
//  along with DDC.  If not, see <http://www.gnu.org/licenses/>.
//
#ifndef IndexSetForQueryingStage_h
#define IndexSetForQueryingStage_h


#include "../CommonLib/bserialize.h"
#include "../CommonLib/ddcThread.h"
#include "../CommonLib/ddcMMap.h"

// PeriodsDivisionMap : type for mappping an index item no to its \ref perdiv_def "period division"

//typedef map<DWORD, vector<DWORD> > PeriodsDivisionMap__OLD
typedef ddcMapFile<DWORD, DWORD> PeriodsDivisionMapT;
typedef PeriodsDivisionMapT::MapT PeriodsDivisionMapR;  
typedef PeriodsDivisionMapT::RecFileT PeriodsDivisionMapV;  



// Flags for CIndexItem::m_IndexItemOffsetAndFlags

const DWORD TheOnlyOccurIsInEndOccurNo = 1 << 31;
/*-- 2018-08-15: EndOfSentenceForBigrams flag is obsolete --*/
//const DWORD EndOfSentenceForBigrams = 1<<30;
//const DWORD AllFlags = TheOnlyOccurIsInEndOccurNo  | EndOfSentenceForBigrams;
const DWORD AllFlags = TheOnlyOccurIsInEndOccurNo;

class CIndexItem {
    // pointer to CStringIndexSet::m_StringBuffer and flags;
    DWORD m_IndexItemOffsetAndFlags;

    // upper bound offset of occurrences in CIndexSetForQueryingStage::m_OccursFp
    DWORD m_EndOccurOffset;

public:
    DWORD GetIndexItemOffsetAndFlags() const {
        return m_IndexItemOffsetAndFlags;
    };

    void SetIndexItemOffsetAndFlags(DWORD Value) {
        m_IndexItemOffsetAndFlags = Value;
    };

    static size_t GetMaximalNumberOfRunningTokens() {
        return ~0 & ~AllFlags;
    };

    DWORD GetIndexItemOffset() const {
        return m_IndexItemOffsetAndFlags & ~AllFlags;
    };

    void SetIndexItemOffset(DWORD Value) {
        m_IndexItemOffsetAndFlags &= AllFlags;
        m_IndexItemOffsetAndFlags |= Value & ~AllFlags;
    };

    DWORD GetItemIndexFlags() const {
        return m_IndexItemOffsetAndFlags & AllFlags;
    };

    void AddItemIndexFlags(DWORD Value) {
        m_IndexItemOffsetAndFlags |= Value & AllFlags;
    };

    DWORD GetEndOccurOffset() const {
        return m_EndOccurOffset;
    };

    void SetEndOccurOffset(DWORD EndOccurOffset) {
        m_EndOccurOffset = EndOccurOffset;
    };

    bool HasOneOccurrence() const {
        return (m_IndexItemOffsetAndFlags & TheOnlyOccurIsInEndOccurNo) != 0;
    }
};

inline size_t get_size_in_bytes(const CIndexItem &t) {
    return 8;
};

inline size_t save_to_bytes(const CIndexItem &i, BYTE *buf) {
    buf += save_to_bytes(i.GetIndexItemOffsetAndFlags(), buf);
    buf += save_to_bytes(i.GetEndOccurOffset(), buf);
    return get_size_in_bytes(i);
}

inline size_t restore_from_bytes(CIndexItem &i, const BYTE *buf) {
    DWORD d;

    buf += restore_from_bytes(d, buf);
    i.SetIndexItemOffsetAndFlags(d);

    buf += restore_from_bytes(d, buf);
    i.SetEndOccurOffset(d);

    return get_size_in_bytes(i);
}

// suffix-index

typedef ddcVecFile<DWORD> CSuffixIndex;


// forward decls

class CStringIndexator;

class CItemIndexForLoading;


class CIndexSetForQueryingStage {
    ddcFileOrMMap m_OccursFp;

    bool LoadPeriodDivision();

    void ReadOccurrences(CTokenNo *OutBuffer, file_off_t FilePosition, size_t Count) const;

protected:

    void AssertHasPath() const;

    void AddOccurs(size_t IndexItemNo, const bool bOneOccurrence, const size_t StartOccurNo, const size_t EndOccurNo,
                   vector<CTokenNo> &Occurs, size_t PeriodNo, COccurrBuffer &OccursBuffer,
                   CShortOccurCache *pCacheByIndexSet, int &CacheId) const;

    string GetOccursFileName() const;

    string GetOccHdrFileName() const;

    string GetSuffixFileName() const;


    string GetPeriodsDivisionFileName() const;

    string GetFileNameForInfos() const;

    file_off_t GetOccurrsFileSize() const;

    size_t GetStartOccurNo(size_t IndexNo) const;

    bool BuildPeriodsDivisionAndCompress(const DWORD TokenId, vector<CTokenNo> &InputTokens);

    bool
    AddOneIndexItem(CItemIndexForLoading &M, FILE *res_fp, size_t &CurrPositionInResFile, const CTokenNo EndTokeNo);

    bool WritePeriodsDivision();

    bool LoadIndexSet(bool bLoadHeaderOfOccurrences = true);


public:
    ddcVecFile<CIndexItem> m_Index;

    CSuffixIndex m_rIndex;

    PeriodsDivisionMapT m_EndPeriodOffsets;

    const CStringIndexator *m_pParent;

    bool m_bCompressOccurrences;


    CIndexSetForQueryingStage(const CStringIndexator *pParent);

    virtual ~CIndexSetForQueryingStage();

    virtual string GetName() const = 0;

    bool DestroyIndexSet();

    void ReadAllOccurrences(size_t IndexItemNo, vector<CTokenNo> &Occurs) const;

};


#endif

/*--- emacs style variables ---
 * Local Variables:
 *  mode: C++
 *  c-file-style: "ellemtel"
 *  c-basic-offset: 4
 *  tab-width: 8
 *  indent-tabs-mode: nil
 * End:
 */