ddc
Public Member Functions | Public Attributes | List of all members
CDDCBranchServer Class Reference

#include <BranchServer.h>

Inheritance diagram for CDDCBranchServer:
Inheritance graph
[legend]
Collaboration diagram for CDDCBranchServer:
Collaboration graph
[legend]

Public Member Functions

 CDDCBranchServer (bool master=true)
 
virtual ~CDDCBranchServer ()
 
virtual void Shutdown ()
 
virtual void SetOptions (const CDDCServerOptions &Options)
 
virtual CDDCServerWorkerClone (void) const
 
virtual void WorkerCloneInit (const CDDCServer *Parent, size_t WorkerId)
 
virtual void WorkerCloneFree (void)
 
virtual bool Close (int &rConnectedSocket, bool linger=false, int timeout=5)
 
virtual bool CloseDtr (int &rConnectedSocket, bool linger=false, int timeout=5)
 
virtual void handle__get_first_hits (string &Result, const char *Query, int TimeOut, size_t ResultLimit, const string &ReqPath, const NavHint &Hint)
 
virtual void handle__get_hit_strings (string &Result, const char *ResultType, size_t StartHitNo, size_t ResultLimit)
 
virtual void handle__run_query (string &Result, const char *CorpusName, const char *Query, const char *ResultType, size_t StartHitNo, size_t ResultLimit, int TimeOut, const string &ReqPath, const NavHint &Hint)
 
virtual void handle__expand_terms (string &Result, const string &pipeline, const string &terms, int timeout, const string &corpus)
 
virtual void handle__status (string &Result, int timeout=10, bool verbose=false)
 
virtual void handle__info (string &Result, int timeout=10, bool verbose=false)
 
virtual void handle__reload (string &Result, int depth=0)
 
virtual void handle__clear_cache (string &Result, int depth=0)
 
virtual void handle__nodes (string &Result, int depth=-1)
 
virtual void handle__close_socket (string &Result, int &rConnectedSocket)
 
void InitHostMap ()
 
void GetSubcorpusJsonStrings (vector< string > &Results, const string &request, int timeout=10)
 
string GetSubcorpusJsonArray (const string &request, int timeout=10)
 
void RunDistributed ()
 
NetworkErrorsEnum RunGetFirstHits ()
 
void CheckFirstHitsResult (NetworkErrorsEnum networkError=neSuccess)
 
NetworkErrorsEnum RunGetHitStrings ()
 
NetworkErrorsEnum RunDistributed (string Query, const string &ResultType, size_t StartHitNo, size_t ResultLimit, int &iInternalError, size_t &EndHitNo, size_t &HitsCount, int TimeOut, string &ResultString, size_t &RelevantDocsCount, string &HitDistrib, string &DocDistrib, string &HitsKey)
 
void UpdateNavCache ()
 
NetworkErrorsEnum GetHitContexts (CFirstHitsQueryResult &FHResult, const string &ResultType, DDCFormatTypeEnum ResultTypeE, CSelectedHosts &SelectedHosts, int &InternalError, int TimeOut, string &ResultString)
 
NetworkErrorsEnum GetHitCounts (CFirstHitsQueryResult &FHResult, DDCFormatTypeEnum ResultTypeE, string &ResultString)
 
void GetFirstHitsFromCorpora (const CSelectedHosts &SelectedHosts, const string Query, const size_t StartHitNo, const size_t ResultLimit, const int TimeOut, CFirstHitsQueryResult &Result)
 
NetworkErrorsEnum SendGetHitStringsRequest (BYTE SelectedHostNo, const vector< vector< size_t > > &SortedHits, const vector< size_t > &FirstOffsets, const string &ResultType)
 
NetworkErrorsEnum GetHitStringsFromOneCorpus (CHost &Host, const string &ResultType, size_t StartHitNo, size_t ResultLimit, int TimeOut, int &CorpusSocket, int &InternalError, string &Result, string &ResultHint)
 
bool ReadSelectedHosts (string &CleanQuery)
 
size_t FindHostID (const string &CorpusName) const
 
CHostFindHost (const string &CorpusName)
 
void AlignCorpusPath (const string &UserPath, string &Dtr, string &Rest)
 
- Public Member Functions inherited from CDDCServer
 CDDCServer ()
 
virtual ~CDDCServer ()
 
virtual void Shutdown (bool quiet=false)
 
virtual void StartClientThreads ()
 
bool UseClientThreads (void) const
 
virtual void RunThread ()
 
virtual void HandleClientConnection (int &rConnectedSocket)
 
virtual string ProcessSocketString (const string &SocketInput, int &rConnectedSocket)
 
virtual void handle__get_first_hits (string &Result, const char *Query, int TimeOut, size_t ResultLimit)
 
virtual void handle__run_query (string &Result, const char *CorpusName, const char *Query, const char *ResultType, size_t StartHitNo, size_t ResultLimit, int TimeOut)
 
virtual void handle__get_paradigm (string &Result, const char *Word, int iLemmatizeOnly, int iLang)
 
virtual void handle__version (string &Result)
 
virtual void handle__vstatus (string &Result, int timeout=10)
 
virtual void handle__vinfo (string &Result, int timeout=10)
 
void method_not_implemented (const char *methodName)
 
virtual size_t residentMemorySize (void) const
 
virtual size_t totalMemorySize (void) const
 
void findPathHint (const string &ReqStr, size_t StartPos, size_t &PathPos, size_t &HintPos)
 
- Public Member Functions inherited from CHost
 CHost ()
 
virtual ~CHost ()
 
bool ReadFromString (string S)
 
virtual void CreateListener ()
 
void CopyAddressParametersFrom (const CHost &X)
 
bool CloseListenSocket (int timeout=0)
 
int CreateAndConnectSocket (string &strError) const
 
string GetAddressStr () const
 
const string & GetLocalPathOfIndex () const
 
const CSocketAddrSocketAddr () const
 
const string & HostAddr () const
 
const int Port () const
 
bool InitEmbedded (const CHost &)
 
void InitShortName ()
 
bool IsListening () const
 
bool GetRequest (int &rConnectedSocket, int TimeOut, string &Request)
 
void SendResponse (int &rConnectedSocket, const string &Result)
 
void HandleSocketError (int &rConnectedSocket, const string &ErrorStr="ERROR", const string &Where="CHost::HandleSocketError()")
 
void HandleClientRequest (int &rConnectedSocket, int TimeOut)
 
virtual void ClientError (int code=-1, const string &msg="")
 
virtual void ClientError (int code, const string &msg, int &clientSocket)
 

Public Attributes

map< string, CQueryStatisticsm_QueryToStatis
 
vector< CHostm_Hosts
 
CHostIdMap m_HostMap
 maps short daughter labels (without "server:" prefix) to m_Hosts[] offsets More...
 
string m_QueryResultString
 
string m_ErrorStr
 
CFirstHitsQueryResult m_FHResult
 
CRunQueryData m_QueryData
 
NavHintCachem_pNavCache
 
size_t m_NavCacheStep
 
bool m_bHitStringsParallel
 send get_hit_strings requests in parallel? More...
 
- Public Attributes inherited from CDDCServer
DDCVersionT m_VersionCompat
 
time_t m_tStarted
 
CDDCServerStats m_ServerStats
 
CDDCServerStats m_SessionStats
 
bool m_bLogRequests
 
set< string > m_noLogCommands
 
string m_FQHostName
 
size_t m_nWorkers
 
size_t m_ThreadQueueSize
 
double m_SlowQueryTimeout
 
size_t m_WorkerId
 
class CDDCServerThreadPoolm_ThreadPool
 
- Public Attributes inherited from CHost
string m_CorpusName
 full node label, including optional "server:" prefix More...
 
string m_ShortName
 short node label, without optional "server:" prefix More...
 

Additional Inherited Members

- Static Public Attributes inherited from CDDCServer
static const char * AdditionalHitDelimiter = "\x01"
 
static const size_t MaxSubhostsSize = 256
 
- Protected Member Functions inherited from CHost
int CreateSocket (bool bBind, string &strError) const
 
bool CheckValidAddr () const
 
void StopListening ()
 
- Protected Attributes inherited from CHost
CSocketAddr m_SockAddr
 socket address (tcp or UNIX); formerly m_HostAddr, m_Port, m_SocketPath More...
 
bool m_bExpectMore
 moo: whether to expect more requests from current client More...
 
bool m_bRespond
 moo: whether current request has a real (possibly empty) response More...
 
bool m_bLogFinish
 moo: whether to log connection completion More...
 
string m_LocalPath
 path to local project-file (ddc_local_corpora.cfg) or label of parent branch node (ddc_server.cfg) More...
 
uint32_t m_maxReceiveBytes
 maximum receivable packet length (default=0: no limit) More...
 
mode_t m_UnixSocketPerms
 permissions mask for unix socket (defualt=0: OS-default = 0777&~umask()) More...
 

Detailed Description

DDC branch server node, formerly CDDCCorpusListenHost

Constructor & Destructor Documentation

◆ CDDCBranchServer()

CDDCBranchServer::CDDCBranchServer ( bool  master = true)

default constructor

Parameters
mastertrue (default) iff this is to be a "master" object (i.e. not a worker clone)

References DDC_DEFAULT_NAV_HINT_CACHE_CAPACITY, DDC_DEFAULT_NAV_HINT_CACHE_STEP, DDC_STATIC_BUFLEN, ddcLLDebug, and ddcLogLevel.

◆ ~CDDCBranchServer()

CDDCBranchServer::~CDDCBranchServer ( )
virtual

default destructor

Member Function Documentation

◆ Shutdown()

void CDDCBranchServer::Shutdown ( )
virtual

close any open socket(s) this server administers, called on termination signal

References ddcLogInfo, and Format().

Referenced by ShutdownDDCServer().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ SetOptions()

void CDDCBranchServer::SetOptions ( const CDDCServerOptions Options)
virtual

set server options (override handles option m_NavCacheSize)

Reimplemented from CDDCServer.

References ddcLogInfo, Format(), CDDCServerOptions::m_bHitStringsParallel, CDDCServerOptions::m_NavCacheSize, CDDCServerOptions::m_NavCacheStep, and CDDCServer::SetOptions().

Referenced by LoadDDCServer().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ WorkerClone()

virtual CDDCServer* CDDCBranchServer::WorkerClone ( void  ) const
inlinevirtual

create a minimal copy of this object (and any subordinate data) for use by a worker thread

Reimplemented from CDDCServer.

References neSuccess, and SOCKET.

◆ WorkerCloneInit()

void CDDCBranchServer::WorkerCloneInit ( const CDDCServer Parent,
size_t  WorkerId 
)
virtual

initialize a newly created worker-clone (and any subordinate data)

Reimplemented from CDDCServer.

References m_bHitStringsParallel, m_HostMap, m_Hosts, m_pNavCache, and CDDCServer::WorkerCloneInit().

Here is the call graph for this function:

◆ WorkerCloneFree()

void CDDCBranchServer::WorkerCloneFree ( void  )
virtual

perform local cleanup of a worker clone prior to deletion

Reimplemented from CDDCServer.

References CDDCServer::WorkerCloneFree().

Here is the call graph for this function:

◆ Close()

bool CDDCBranchServer::Close ( int &  rConnectedSocket,
bool  linger = false,
int  timeout = 5 
)
virtual

close connected socket, implies cleanup/close of m_FHResult and m_QueryData

Reimplemented from CHost.

References CHost::Close().

Here is the call graph for this function:

◆ CloseDtr()

bool CDDCBranchServer::CloseDtr ( int &  rConnectedSocket,
bool  linger = false,
int  timeout = 5 
)
virtual

close a socket connected to a daugher subcorpus, NO implicit cleanup of m_FHResult and m_QueryData

References CHost::Close(), and SendString().

Here is the call graph for this function:

◆ handle__get_first_hits()

void CDDCBranchServer::handle__get_first_hits ( string &  Result,
const char *  Query,
int  TimeOut,
size_t  ResultLimit,
const string &  ReqPath,
const NavHint Hint 
)
virtual

◆ handle__get_hit_strings()

void CDDCBranchServer::handle__get_hit_strings ( string &  Result,
const char *  ResultType,
size_t  StartHitNo,
size_t  ResultLimit 
)
virtual

◆ handle__run_query()

void CDDCBranchServer::handle__run_query ( string &  Result,
const char *  CorpusName,
const char *  Query,
const char *  ResultType,
size_t  StartHitNo,
size_t  ResultLimit,
int  TimeOut,
const string &  ReqPath,
const NavHint Hint 
)
virtual

request handler: run_query: required (branch)

Reimplemented from CDDCServer.

References ddcLogError, errParseError, Format(), CRunQueryData::Init(), CRunQueryData::sQuery, and CRunQueryData::toString().

Here is the call graph for this function:

◆ handle__expand_terms()

void CDDCBranchServer::handle__expand_terms ( string &  Result,
const string &  pipeline,
const string &  terms,
int  timeout,
const string &  corpus 
)
virtual

request handler: expand_terms: required (leaf|branch)

Reimplemented from CDDCServer.

References CHost::CreateAndConnectSocket(), errOther, errParseError, Format(), CHost::m_CorpusName, neSuccess, ReceiveString(), and SendString().

Here is the call graph for this function:

◆ handle__status()

void CDDCBranchServer::handle__status ( string &  Result,
int  timeout = 10,
bool  verbose = false 
)
virtual

◆ handle__info()

void CDDCBranchServer::handle__info ( string &  Result,
int  timeout = 10,
bool  verbose = false 
)
virtual

request handler: info: required (leaf|branch)

Reimplemented from CDDCServer.

References jsonStr(), and PACKAGE_VERSION.

Here is the call graph for this function:

◆ handle__reload()

void CDDCBranchServer::handle__reload ( string &  Result,
int  depth = 0 
)
virtual

request handler: reload: required (leaf|branch)

Reimplemented from CDDCServer.

References Format(), and jsonStr().

Here is the call graph for this function:

◆ handle__clear_cache()

void CDDCBranchServer::handle__clear_cache ( string &  Result,
int  depth = 0 
)
virtual

request handler: clear_cache: required (leaf|branch)

Reimplemented from CDDCServer.

References ddcLRUCache< Key, Val >::clear(), ddcCurlCache, Format(), jsonStr(), ddcLRUCache< Key, Val >::lock(), and ddcLRUCache< Key, Val >::unlock().

Here is the call graph for this function:

◆ handle__nodes()

void CDDCBranchServer::handle__nodes ( string &  Result,
int  depth = -1 
)
virtual

request handler: nodes: required (leaf|branch; since v2.2.8)

Reimplemented from CDDCServer.

References Format(), and jsonStr().

Here is the call graph for this function:

◆ handle__close_socket()

void CDDCBranchServer::handle__close_socket ( string &  Result,
int &  rConnectedSocket 
)
virtual

request handler: ddc_close_socket: required (leaf|branch)

Reimplemented from CDDCServer.

◆ InitHostMap()

void CDDCBranchServer::InitHostMap ( )

initialize m_HostMap[] from m_Hosts[]

Referenced by LoadDDCServer().

Here is the caller graph for this function:

◆ GetSubcorpusJsonStrings()

void CDDCBranchServer::GetSubcorpusJsonStrings ( vector< string > &  Results,
const string &  request,
int  timeout = 10 
)

pass request to each subcorpus host and fetch results into a vector<>

References CHost::CreateAndConnectSocket(), Format(), jsonStr(), CHost::m_CorpusName, neSuccess, ReceiveString(), SendString(), and SOCKET.

Here is the call graph for this function:

◆ GetSubcorpusJsonArray()

string CDDCBranchServer::GetSubcorpusJsonArray ( const string &  request,
int  timeout = 10 
)

pass request to each subcorpus host, fetch results, and collect into a single JSON array

◆ RunDistributed() [1/2]

void CDDCBranchServer::RunDistributed ( )

◆ RunGetFirstHits()

NetworkErrorsEnum CDDCBranchServer::RunGetFirstHits ( )

◆ CheckFirstHitsResult()

void CDDCBranchServer::CheckFirstHitsResult ( NetworkErrorsEnum  networkError = neSuccess)

error-message propagation for get_first_hits requests; uses m_FHResult and m_ErrorStr; assigns to m_ErrorStr if currently empty

References errNone, errRuntime, GetNetworkErrorString(), CFirstHitsQueryResult::m_ErrorStr, CFirstHitsQueryResult::m_InternalError, CFirstHitsQueryResult::m_NetworkError, and neSuccess.

Here is the call graph for this function:

◆ RunGetHitStrings()

NetworkErrorsEnum CDDCBranchServer::RunGetHitStrings ( )

new top-level guts for handle__get_hit_strings() and handle__run_query() phase 2; uses m_FHResult and m_QueryData

◆ RunDistributed() [2/2]

NetworkErrorsEnum CDDCBranchServer::RunDistributed ( string  Query,
const string &  ResultType,
size_t  StartHitNo,
size_t  ResultLimit,
int &  iInternalError,
size_t &  EndHitNo,
size_t &  HitsCount,
int  TimeOut,
string &  ResultString,
size_t &  RelevantDocsCount,
string &  HitDistrib,
string &  DocDistrib,
string &  HitsKey 
)

◆ UpdateNavCache()

void CDDCBranchServer::UpdateNavCache ( )

mid-level: update the navigation cache from runq.navHint, runq.navKey if applicable

References ddcLogDebug, Format(), IsCountSort(), NavHint::m_Offset, CFirstHitsQueryResult::m_SortOrder, NAVCACHE_DEBUG, CRunQueryData::navHint, CRunQueryData::navKey, CRunQueryData::SelectedHosts, CSelectedHosts::size(), NavHintKey::toJson(), and NavHint::toJson().

Here is the call graph for this function:

◆ GetHitContexts()

NetworkErrorsEnum CDDCBranchServer::GetHitContexts ( CFirstHitsQueryResult FHResult,
const string &  ResultType,
DDCFormatTypeEnum  ResultTypeE,
CSelectedHosts SelectedHosts,
int &  InternalError,
int  TimeOut,
string &  ResultString 
)

◆ GetHitCounts()

NetworkErrorsEnum CDDCBranchServer::GetHitCounts ( CFirstHitsQueryResult FHResult,
DDCFormatTypeEnum  ResultTypeE,
string &  ResultString 
)

◆ GetFirstHitsFromCorpora()

void CDDCBranchServer::GetFirstHitsFromCorpora ( const CSelectedHosts SelectedHosts,
const string  Query,
const size_t  StartHitNo,
const size_t  ResultLimit,
const int  TimeOut,
CFirstHitsQueryResult Result 
)

◆ SendGetHitStringsRequest()

NetworkErrorsEnum CDDCBranchServer::SendGetHitStringsRequest ( BYTE  SelectedHostNo,
const vector< vector< size_t > > &  SortedHits,
const vector< size_t > &  FirstOffsets,
const string &  ResultType 
)

send a single get_hit_strings request (used by both parallel and serial modes)

References ErrorMessage(), Format(), neCouldNotSendData, neNetworkError, neSuccess, SendString(), and SOCKET.

Here is the call graph for this function:

◆ GetHitStringsFromOneCorpus()

NetworkErrorsEnum CDDCBranchServer::GetHitStringsFromOneCorpus ( CHost Host,
const string &  ResultType,
size_t  StartHitNo,
size_t  ResultLimit,
int  TimeOut,
int &  CorpusSocket,
int &  InternalError,
string &  Result,
string &  ResultHint 
)

◆ ReadSelectedHosts()

bool CDDCBranchServer::ReadSelectedHosts ( string &  CleanQuery)

heuristically extract user-specified subcorpora from ":CORPORA" clause of current query, called by RunDistributed()

  • uses (parsed) input query from m_QueryData
  • appends parsed daughter paths to m_QueryData.SelectedHosts (which is assumed to be empty when this method is called)
  • returns false only when a network error occurs

References ddcLogWarn, Format(), CSelectedHosts::Insert(), CSelectedHosts::InsertAll(), CQueryOptions::m_Subcorpora, CSelectedHosts::m_vec, CRunQueryData::pQCompiler, CRunQueryData::QueryOptions(), CRunQueryData::SelectedHosts, CSelectedHosts::size(), CRunQueryData::sQuery, Trim(), and unescapeCString().

Here is the call graph for this function:

◆ FindHostID()

size_t CDDCBranchServer::FindHostID ( const string &  CorpusName) const

find Host-ID (index in m_Hosts[]) for a given CorpusName (short form, uses m_HostIdMap[])

Returns
host-ID for CorpusName on success, otherwise m_Hosts.size()

◆ FindHost()

CHost * CDDCBranchServer::FindHost ( const string &  CorpusName)

find pointer to host in m_Hosts[] for a given CorpusName

Returns
pointer into m_Hosts[] for CorpusName on success, otherwise NULL

◆ AlignCorpusPath()

void CDDCBranchServer::AlignCorpusPath ( const string &  UserPath,
string &  Dtr,
string &  Rest 
)

aligns user-request CorpusPath into shared (~runq.ReqPath), initial (immediate daughter), and non-initial (grandchildren etc.) components

References ddcLogDebug, Format(), CRunQueryData::ReqPath, and stringSplit().

Here is the call graph for this function:

Member Data Documentation

◆ m_QueryToStatis

map<string, CQueryStatistics> CDDCBranchServer::m_QueryToStatis

◆ m_Hosts

vector<CHost> CDDCBranchServer::m_Hosts

Referenced by LoadDDCServer(), and WorkerCloneInit().

◆ m_HostMap

CHostIdMap CDDCBranchServer::m_HostMap

maps short daughter labels (without "server:" prefix) to m_Hosts[] offsets

Referenced by WorkerCloneInit().

◆ m_QueryResultString

string CDDCBranchServer::m_QueryResultString

◆ m_ErrorStr

string CDDCBranchServer::m_ErrorStr

◆ m_FHResult

CFirstHitsQueryResult CDDCBranchServer::m_FHResult

◆ m_QueryData

CRunQueryData CDDCBranchServer::m_QueryData

◆ m_pNavCache

NavHintCache* CDDCBranchServer::m_pNavCache

Referenced by WorkerCloneInit().

◆ m_NavCacheStep

size_t CDDCBranchServer::m_NavCacheStep

◆ m_bHitStringsParallel

bool CDDCBranchServer::m_bHitStringsParallel

send get_hit_strings requests in parallel?

Referenced by WorkerCloneInit().


The documentation for this class was generated from the following files: