ddcCorpusList.h

Go to the documentation of this file.

//-*- Mode: C++ -*-
//
//  Copyright 2018 by Bryan Jurish
//
//  This file is part of DDC.
//  DDC originally by Alexey Sokirko
// 
//  DDC is free software: you can redistribute it and/or modify
//  it under the terms of the GNU Lesser General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
// 
//  DDC is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU Lesser General Public License for more details.
// 
//  You should have received a copy of the GNU Lesser General Public License
//  along with DDC.  If not, see <http://www.gnu.org/licenses/>.
//
#ifndef DDC_CORPUS_FILES_H
#define DDC_CORPUS_FILES_H

#include "ddcMMap.h"
#include "ddcVersion.h"

//======================================================================
// ddcCorpusList
template <typename OffT_=DWORD>
class ddcCorpusList
{
public:
    //----------------------------------------------------------
    // typedefs
    typedef OffT_  OffT; 
    typedef DWORD  IdT;  

public:
    //----------------------------------------------------------
    // data
    ddcVecFile<char>  m_cdata;    
    ddcVecFile<OffT>  m_offsets;  
    vector<string>    m_strings;  
    string            m_verstr;   

    size_t            m_bufsize;  

public:
    //----------------------------------------------------------
    // constructors etc.

    ddcCorpusList()
        : m_bufsize(20000)
    {};

    ddcCorpusList(const string& conFile, const string& cdataFile="", const string& offsetsFile="", bool useMMap=false)
        : m_bufsize(20000)
    { open(conFile,cdataFile,offsetsFile,useMMap); };

    ~ddcCorpusList()
    { close(); };

    inline void clear()
    { close(); };

public:
    //----------------------------------------------------------
    // accessors & properties

    inline size_t size() const
    { return m_offsets ? m_offsets.size() : m_strings.size(); };

    inline bool empty() const
    { return m_offsets ? m_offsets.empty() : m_strings.empty(); };

    inline bool opened() const
    { return m_offsets.opened() || !m_strings.empty(); };

    const string& VersionString() const
    { return m_verstr; };

    inline OffT itemStartOffset(IdT idx) const
    { return m_offsets[idx]; };

    inline OffT itemEndOffset(IdT idx) const
    { return ((idx+1)>=size() ? m_cdata.size() : m_offsets[idx+1]) - 1; };

    inline void itemGet(IdT idx, string& str) const
    {
        OffT i = itemStartOffset(idx);
        OffT j = itemEndOffset(idx);
        str.assign( &m_cdata[i], j-i );
    };


public:
    //----------------------------------------------------------
    // operators

    inline operator bool() const
    { return opened(); };

    inline string operator[](IdT idx) const
    {
        if (!m_offsets) return m_strings[idx];
        OffT i = itemStartOffset(idx);
        OffT j = itemEndOffset(idx);
        return string( &m_cdata[i], j-i );
    };

    //----------------------------------------------------------
    // read/write access (auto-demote)

    inline void push_back(const string& value)
    {
        ensureVec();
        m_strings.push_back(value);
    };

    void append_vector(vector<string>& v) const
    {
        v.reserve(v.size()+size());
        if (m_offsets) {
            for (IdT idx=0; idx < size(); ++idx)
                v.push_back(operator[](idx));
        } else {
            v.insert(v.end(), m_strings.begin(), m_strings.end());
        }
    };
    
    inline void ensureVec()
    {
        if (m_offsets) {
            ddcLogWarn(Format("WARNING: ddcCorpusList(\"%s\") demoting to std::vector<> implementation", m_cdata.filename().c_str()));
            ClearVector(m_strings);
            append_vector(m_strings);
            m_offsets.close();
            m_cdata.close();
        }
    };

public:
    //----------------------------------------------------------
    // search

    //-- re numeric_limits<>::max(): see NO_ID comments in ddcStringEnum.h
    //static const IdT NO_ID = std::numeric_limits<IdT>::max();
    static const IdT NO_ID = DWORD_MAX;

    inline IdT find(const string& key) const
    {
        if (m_offsets) {
            string tmp;
            for (IdT idx=0; idx < size(); ++idx) {
                itemGet(idx,tmp);
                if (tmp == key)
                    return idx;
            }
        }
        else {
            vector<string>::const_iterator it = std::find(m_strings.begin(), m_strings.end(), key);
            if (it != m_strings.end())
                return (it - m_strings.begin());
        }
        return NO_ID;
    };

    inline IdT rfind(const string& key) const
    {
        if (m_offsets) {
            if (empty()) return NO_ID;
            string tmp;
            IdT idx = size()-1;
            do {
                itemGet(idx,tmp);
                if (tmp == key)
                    return idx;
                --idx;
            } while (idx != 0);
        }
        else {
            vector<string>::const_reverse_iterator it = std::find(m_strings.rbegin(), m_strings.rend(), key);
            if (it != m_strings.rend())
                return (m_strings.size()-1) - (it - m_strings.rbegin());
        }
        return NO_ID;
    };
    
public:
    //----------------------------------------------------------
    // guts

    ddcCorpusList& open(const string& conFile, const string& cdataFile="", const string& offsetsFile="", bool useMMap=false)
    {
        if (opened()) close();

        if (!cdataFile.empty() && !offsetsFile.empty() && FileExists(cdataFile) && FileExists(offsetsFile)) {
            openCompat(conFile,false);
            m_cdata.open(cdataFile, useMMap);
            m_offsets.open(offsetsFile, useMMap);
            ddcLogDebug(Format("ddcCorpusList[%s]::open(): read %zi item(s) from %s + %s", (useMMap ? "mmap" : "slurp"), size(), cdataFile.c_str(), offsetsFile.c_str()));
        }
        else {
            ddcLogWarn("WARNING: ddcCorpusList::open(): loading corpus list '"+conFile+"' in compatibility mode: please upgrade your index!");
            openCompat(conFile,true);
            ddcLogDebug(Format("ddcCorpusList[compat]::open(): read %zi item(s) from %s", size(), conFile.c_str()));
        }
        //ddc_format_version_check(m_verstr.c_str(), cdataFile.c_str());
        return *this;
    };

    void openCompat(const string& filename, bool slurpList=true)
    {
        if (opened()) close();

        FILE * fp = fopen(filename.c_str(), "rb");
        if (!fp)
            throw invalid_argument(Format("ddcCorpusList::openCompat(): open failed for '%s': %s'", filename.c_str(), strerror(errno)));

        char buf[m_bufsize+1];
        if (!fgets(buf, m_bufsize, fp))
            throw invalid_argument("ddcCorpusList::openCompat(): failed to read version information from '"+filename+"'");
        m_verstr = buf;
        Trim(m_verstr);

        if (slurpList) {
            string tmp;
            while (fgets(buf, m_bufsize, fp)) {
                tmp = buf;
                Trim(tmp);
                m_strings.push_back(tmp);
            }
        }

        fclose(fp);
    };

    void close()
    {
        m_cdata.close();
        m_offsets.close();
        ClearVector(m_strings);
        ClearString(m_verstr);
    };

};


#endif /* DDC_STRING_ENUM_H */

/*--- emacs style variables ---
 * Local Variables:
 *  mode: C++
 *  c-file-style: "ellemtel"
 *  c-basic-offset: 4
 *  tab-width: 8
 *  indent-tabs-mode: nil
 * End:
 */