SearchIndex Class Reference

Writes search index for doxygen provided server based search engine that uses PHP. More...

#include <src/searchindex.h>

Classes
struct	URL
struct	URLInfo
class	IndexWord

Public Member Functions
	SearchIndex ()
void	setCurrentDoc (const Definition *ctx, const DString &anchor, bool isSourceFile)
void	addWord (const DString &word, bool hiPriority)
void	write (const DString &file)

Private Member Functions
void	addWordRec (const DString &word, bool hiPrio, bool recurse)

Private Attributes
std::unordered_map< std::string, int >	m_words
std::vector< std::vector< IndexWord > >	m_index
std::unordered_map< std::string, int >	m_url2IdMap
std::map< int, URL >	m_urls
int	m_urlIndex = -1
int	m_urlMaxIndex = 0

Detailed Description

Writes search index for doxygen provided server based search engine that uses PHP.

Definition at line 63 of file searchindex.h.

Constructor & Destructor Documentation

SearchIndex()

SearchIndex::SearchIndex

(

)

Definition at line 67 of file searchindex.cpp.

{
  m_index.resize(numIndexEntries);
}

References m_index, and numIndexEntries.

Member Function Documentation

addWord()

void SearchIndex::addWord	(	const DString &	word,
		bool	hiPriority )

Definition at line 234 of file searchindex.cpp.

{
  std::lock_guard<std::mutex> lock(g_searchIndexMutex);
  addWordRec(word,hiPriority,false);
}

References addWordRec(), and g_searchIndexMutex.

addWordRec()

void SearchIndex::addWordRec ( const DString & word, bool hiPrio, bool recurse )

private

Definition at line 193 of file searchindex.cpp.

{
  if (word.empty()) return;
  DString wStr = DString(word).lower();
  //printf("SearchIndex::addWord(%s,%d) wStr=%s\n",word,hiPriority,qPrint(wStr));
  int idx=charsToIndex(wStr);
  if (idx<0 || idx>=static_cast<int>(m_index.size())) return;
  auto it = m_words.find(wStr.str());
  if (it==m_words.end())
  {
    //fprintf(stderr,"addWord(%s) at index %d\n",word,idx);
    m_index[idx].emplace_back(wStr);
    it = m_words.emplace( wStr.str(), static_cast<int>(m_index[idx].size())-1 ).first;
  }
  m_index[idx][it->second].addUrlIndex(m_urlIndex,hiPriority);
  bool found=false;
  if (!recurse) // the first time we check if we can strip the prefix
  {
    int i=getPrefixIndex(word);
    if (i>0)
    {
      addWordRec(word.data()+i,hiPriority,true);
      found=true;
    }
  }
  if (!found) // no prefix stripped
  {
    int i=0;
    while (word[i]!=0 &&
           !((word[i]=='_' || word[i]==':' || (word[i]>='a' && word[i]<='z')) &&  // [_a-z:]
             (word[i+1]>='A' && word[i+1]<='Z')))                                 // [A-Z]
    {
      i++;
    }
    if (word[i]!=0 && i>=1)
    {
      addWordRec(word.data()+i+1,hiPriority,true);
    }
  }
}

References addWordRec(), charsToIndex(), DString::data(), DString::empty(), getPrefixIndex(), DString::lower(), m_index, m_urlIndex, m_words, and DString::str().

Referenced by addWord(), and addWordRec().

setCurrentDoc()

void SearchIndex::setCurrentDoc	(	const Definition *	ctx,
		const DString &	anchor,
		bool	isSourceFile )

Definition at line 72 of file searchindex.cpp.

{
  if (ctx==nullptr) return;
  std::lock_guard<std::mutex> lock(g_searchIndexMutex);
  ASSERT(!isSourceFile || ctx->definitionType()==Definition::TypeFile);
  //printf("SearchIndex::setCurrentDoc(%s,%s,%s)\n",name,baseName,anchor);
  DString url=isSourceFile ? (toFileDef(ctx))->getSourceFileBase() : ctx->getOutputFileBase();
  url+=Config_getString(HTML_FILE_EXTENSION);
  DString baseUrl = url;
  if (!anchor.empty()) url+=DString("#")+anchor;
  if (!isSourceFile) baseUrl=url;
  DString name=ctx->qualifiedName();
  if (ctx->definitionType()==Definition::TypeMember)
  {
    const MemberDef *md = toMemberDef(ctx);
    name.prepend((md->getLanguage()==SrcLangExt::Fortran  ?
                 theTranslator->trSubprogram(true,true) :
                 theTranslator->trMember(true,true))+" ");
  }
  else // compound type
  {
    SrcLangExt lang = ctx->getLanguage();
    DString sep = getLanguageSpecificSeparator(lang);
    if (sep!="::")
    {
      name = substitute(name,"::",sep);
    }
    switch (ctx->definitionType())
    {
      case Definition::TypePage:
        {
          const PageDef *pd = toPageDef(ctx);
          if (pd->hasTitle())
          {
            name = theTranslator->trPage(true,true)+" "+pd->title();
          }
          else
          {
            name = theTranslator->trPage(true,true)+" "+pd->name();
          }
        }
        break;
      case Definition::TypeClass:
        {
          const ClassDef *cd = toClassDef(ctx);
          name.prepend(cd->compoundTypeString()+" ");
        }
        break;
      case Definition::TypeNamespace:
        {
          if (lang==SrcLangExt::Java || lang==SrcLangExt::CSharp)
          {
            name = theTranslator->trPackage(name);
          }
          else if (lang==SrcLangExt::Fortran)
          {
            name.prepend(theTranslator->trModule(true,true)+" ");
          }
          else
          {
            name.prepend(theTranslator->trNamespace(true,true)+" ");
          }
        }
        break;
      case Definition::TypeGroup:
        {
          const GroupDef *gd = toGroupDef(ctx);
          if (!gd->groupTitle().empty())
          {
            name = theTranslator->trGroup(true,true)+" "+gd->groupTitle();
          }
          else
          {
            name.prepend(theTranslator->trGroup(true,true)+" ");
          }
        }
        break;
      case Definition::TypeModule:
        {
          name.prepend(theTranslator->trModule(true,true)+" ");
        }
        break;
      default:
        break;
    }
  }
 
  auto it = m_url2IdMap.find(baseUrl.str());
  if (it == m_url2IdMap.end()) // new entry
  {
    m_urlIndex = m_urlMaxIndex++;
    m_url2IdMap.emplace(baseUrl.str(),m_urlIndex);
    m_urls.emplace(m_urlIndex,URL(name,url));
  }
  else // existing entry
  {
    m_urlIndex=it->second;
    m_urls.emplace(it->second,URL(name,url));
  }
}

References ASSERT, ClassDef::compoundTypeString(), Config_getString, Definition::definitionType(), DString::empty(), g_searchIndexMutex, Definition::getLanguage(), getLanguageSpecificSeparator(), Definition::getOutputFileBase(), GroupDef::groupTitle(), PageDef::hasTitle(), m_url2IdMap, m_urlIndex, m_urlMaxIndex, m_urls, Definition::name(), DString::prepend(), Definition::qualifiedName(), DString::str(), substitute(), theTranslator, PageDef::title(), toClassDef(), toFileDef(), toGroupDef(), toMemberDef(), toPageDef(), Translator::trGroup(), Translator::trMember(), Translator::trModule(), Translator::trNamespace(), Translator::trPackage(), Translator::trPage(), Translator::trSubprogram(), Definition::TypeClass, Definition::TypeFile, Definition::TypeGroup, Definition::TypeMember, Definition::TypeModule, Definition::TypeNamespace, and Definition::TypePage.

write()

void SearchIndex::write

(

const DString &

file

)

Definition at line 255 of file searchindex.cpp.

{
  size_t size=4; // for the header
  size+=4*numIndexEntries; // for the index
  size_t wordsOffset = size;
  // first pass: compute the size of the wordlist
  for (size_t i=0;i<numIndexEntries;i++)
  {
    const auto &wlist = m_index[i];
    if (!wlist.empty())
    {
      for (const auto &iw : wlist)
      {
        size_t ws = iw.word().length()+1;
        size+=ws+4; // word + url info list offset
      }
      size+=1; // zero list terminator
    }
  }
 
  // second pass: compute the offsets in the index
  size_t indexOffsets[numIndexEntries];
  size_t offset=wordsOffset;
  for (size_t i=0;i<numIndexEntries;i++)
  {
    const auto &wlist = m_index[i];
    if (!wlist.empty())
    {
      indexOffsets[i]=offset;
      for (const auto &iw : wlist)
      {
        offset+= iw.word().length()+1;
        offset+=4; // word + offset to url info array
      }
      offset+=1; // zero list terminator
    }
    else
    {
      indexOffsets[i]=0;
    }
  }
  size_t padding = size;
  size = (size+3)&~3; // round up to 4 byte boundary
  padding = size - padding;
 
  std::vector<size_t> wordStatOffsets(m_words.size());
 
  int count=0;
 
  // third pass: compute offset to stats info for each word
  for (size_t i=0;i<numIndexEntries;i++)
  {
    const auto &wlist = m_index[i];
    if (!wlist.empty())
    {
      for (const auto &iw : wlist)
      {
        //printf("wordStatOffsets[%d]=%d\n",count,size);
        wordStatOffsets[count++] = size;
        size+=4 + iw.urls().size() * 8; // count + (url_index,freq) per url
      }
    }
  }
  std::vector<size_t> urlOffsets(m_urls.size());
  for (const auto &udi : m_urls)
  {
    urlOffsets[udi.first]=size;
    size+=udi.second.name.length()+1+
          udi.second.url.length()+1;
  }
 
  //printf("Total size %x bytes (word=%x stats=%x urls=%x)\n",size,wordsOffset,statsOffset,urlsOffset);
  std::ofstream f = Portable::openOutputStream(fileName);
  if (f.is_open())
  {
    // write header
    f.put('D'); f.put('O'); f.put('X'); f.put('S');
    // write index
    for (size_t i=0;i<numIndexEntries;i++)
    {
      writeInt(f,indexOffsets[i]);
    }
    // write word lists
    count=0;
    for (size_t i=0;i<numIndexEntries;i++)
    {
      const auto &wlist = m_index[i];
      if (!wlist.empty())
      {
        for (const auto &iw : wlist)
        {
          writeString(f,iw.word());
          writeInt(f,wordStatOffsets[count++]);
        }
        f.put(0);
      }
    }
    // write extra padding bytes
    for (size_t i=0;i<padding;i++) f.put(0);
    // write word statistics
    for (size_t i=0;i<numIndexEntries;i++)
    {
      const auto &wlist = m_index[i];
      if (!wlist.empty())
      {
        for (const auto &iw : wlist)
        {
          size_t numUrls = iw.urls().size();
          writeInt(f,numUrls);
          for (const auto &ui : iw.urls())
          {
            writeInt(f,urlOffsets[ui.second.urlIdx]);
            writeInt(f,ui.second.freq);
          }
        }
      }
    }
    // write urls
    for (const auto &udi : m_urls)
    {
      writeString(f,udi.second.name);
      writeString(f,udi.second.url);
    }
  }
 
}

References m_index, m_urls, m_words, numIndexEntries, Portable::openOutputStream(), writeInt(), and writeString().

Member Data Documentation

m_index

std::vector< std::vector< IndexWord> > SearchIndex::m_index

private

Definition at line 101 of file searchindex.h.

Referenced by addWordRec(), SearchIndex(), and write().

m_url2IdMap

std::unordered_map<std::string,int> SearchIndex::m_url2IdMap

private

Definition at line 102 of file searchindex.h.

Referenced by setCurrentDoc().

m_urlIndex

int SearchIndex::m_urlIndex = -1

private

Definition at line 104 of file searchindex.h.

Referenced by addWordRec(), and setCurrentDoc().

m_urlMaxIndex

int SearchIndex::m_urlMaxIndex = 0

private

Definition at line 105 of file searchindex.h.

Referenced by setCurrentDoc().

m_urls

std::map<int,URL> SearchIndex::m_urls

private

Definition at line 103 of file searchindex.h.

Referenced by setCurrentDoc(), and write().

m_words

std::unordered_map<std::string,int> SearchIndex::m_words

private

Definition at line 100 of file searchindex.h.

Referenced by addWordRec(), and write().

---

The documentation for this class was generated from the following files:

• src/searchindex.h
• src/searchindex.cpp