FilterCache Class Reference

Classes
struct	FilterCacheItem

Public Member Functions
bool	getFileContents (const DString &fileName, size_t startLine, size_t endLine, std::string &str)

Static Public Member Functions
static FilterCache &	instance ()

Private Types
using	LineOffsets = std::vector<size_t>

Private Member Functions
bool	getFileContentsPipe (const DString &fileName, const DString &filter, size_t startLine, size_t endLine, std::string &str)
bool	getFileContentsDisk (const DString &fileName, size_t startLine, size_t endLine, std::string &str)
void	compileLineOffsets (const DString &fileName, const std::string &str)
auto	getFragmentLocation (const LineOffsets &lineOffsets, size_t startLine, size_t endLine) -> std::tuple< size_t, size_t >
void	shrinkBuffer (std::string &str, const DString &fileName, size_t startLine, size_t endLine)
void	readFragmentFromFile (std::string &str, const DString &fileName, size_t startOffset, size_t size=0)
	FilterCache ()

Private Attributes
std::unordered_map< std::string, FilterCacheItem >	m_cache
std::unordered_map< std::string, LineOffsets >	m_lineOffsets
std::mutex	m_mutex
size_t	m_endPos

Detailed Description

Cache for storing the result of filtering a file

Definition at line 545 of file definition.cpp.

Member Typedef Documentation

LineOffsets

using FilterCache::LineOffsets = std::vector<size_t>

private

Definition at line 553 of file definition.cpp.

Constructor & Destructor Documentation

FilterCache()

FilterCache::FilterCache ( )

inlineprivate

Definition at line 737 of file definition.cpp.

: m_endPos(0) { }

References m_endPos.

Referenced by instance().

Member Function Documentation

compileLineOffsets()

void FilterCache::compileLineOffsets ( const DString & fileName, const std::string & str )

inlineprivate

computes the starting offset for each line for file fileName, whose contents should already be stored in buffer str.

Definition at line 681 of file definition.cpp.

    {
      // line 1 (index 0) is at offset 0
      auto it = m_lineOffsets.emplace(fileName.data(),LineOffsets{0}).first;
      const char *p=str.data();
      while (*p)
      {
        char c=0;
        while ((c=*p)!='\n' && c!=0) p++; // search until end of the line
        if (c!=0) p++;
        it->second.push_back(p-str.data());
      }
    }

References DString::data(), and m_lineOffsets.

Referenced by shrinkBuffer().

getFileContents()

bool FilterCache::getFileContents ( const DString & fileName, size_t startLine, size_t endLine, std::string & str )

inline

collects the part of file fileName starting at startLine and ending at endLine into buffer str. Applies filtering if FILTER_SOURCE_FILES is enabled and the file extension matches a filter. Caches file information so that subsequent extraction of blocks from the same file can be performed efficiently

Definition at line 562 of file definition.cpp.

    {
      bool filterSourceFiles = Config_getBool(FILTER_SOURCE_FILES);
      DString filter = getFileFilter(fileName,true);
      bool usePipe = !filter.empty() && filterSourceFiles;
      return usePipe ? getFileContentsPipe(fileName,filter,startLine,endLine,str)
                     : getFileContentsDisk(fileName,startLine,endLine,str);
    }

References Config_getBool, DString::empty(), getFileContentsDisk(), getFileContentsPipe(), and getFileFilter().

Referenced by readCodeFragment().

getFileContentsDisk()

bool FilterCache::getFileContentsDisk ( const DString & fileName, size_t startLine, size_t endLine, std::string & str )

inlineprivate

reads the fragment start at startLine and ending at endLine from file fileName into buffer str

Definition at line 655 of file definition.cpp.

    {
      std::unique_lock<std::mutex> lock(m_mutex);
      // normal file
      //printf("getFileContents(%s): no filter\n",qPrint(fileName));
      auto it = m_lineOffsets.find(fileName.str());
      if (it == m_lineOffsets.end()) // new file
      {
        // read file completely into str buffer
        readFragmentFromFile(str,fileName,0);
        // shrink buffer to [startLine..endLine] part
        shrinkBuffer(str,fileName,startLine,endLine);
      }
      else // file already processed before
      {
        lock.unlock();
        auto [ startLineOffset, fragmentSize] = getFragmentLocation(it->second,startLine,endLine);
        //printf("%s: existing file [%zu-%zu] -> start=%zu size=%zu\n",
        //    qPrint(fileName),startLine,endLine,startLineOffset,fragmentSize);
        readFragmentFromFile(str,fileName,startLineOffset,fragmentSize);
      }
      return true;
    }

References getFragmentLocation(), m_lineOffsets, m_mutex, readFragmentFromFile(), shrinkBuffer(), and DString::str().

Referenced by getFileContents().

getFileContentsPipe()

bool FilterCache::getFileContentsPipe ( const DString & fileName, const DString & filter, size_t startLine, size_t endLine, std::string & str )

inlineprivate

Definition at line 571 of file definition.cpp.

    {
      std::unique_lock<std::mutex> lock(m_mutex);
      auto it = m_cache.find(fileName.str());
      if (it!=m_cache.end()) // cache hit: reuse stored result
      {
        lock.unlock();
        auto item = it->second;
        //printf("getFileContents(%s): cache hit\n",qPrint(fileName));
        // file already processed, get the results after filtering from the tmp file
        Debug::print(Debug::FilterOutput,0,"Reusing filter result for {} from {} at offset={} size={}\n",
               fileName,Doxygen::filterDBFileName,item.filePos,item.fileSize);
 
        auto it_off = m_lineOffsets.find(fileName.str());
        assert(it_off!=m_lineOffsets.end());
        auto [ startLineOffset, fragmentSize] = getFragmentLocation(it_off->second,startLine,endLine);
        //printf("%s: existing file [%zu-%zu]->[%zu-%zu] size=%zu\n",
        //    qPrint(fileName),startLine,endLine,startLineOffset,endLineOffset,fragmentSize);
        readFragmentFromFile(str, Doxygen::filterDBFileName.data(),
                             item.filePos+startLineOffset, fragmentSize);
        return true;
      }
      else // cache miss: filter active but file not previously processed
      {
        //printf("getFileContents(%s): cache miss\n",qPrint(fileName));
        // filter file
        DString cmd=filter+" \""+fileName+"\"";
        Debug::print(Debug::ExtCmd,0,"Executing popen(`{}`)\n",cmd);
        FILE *f = Portable::popen(cmd,"r");
        if (f==nullptr)
        {
          // handle error
          err("Error opening filter pipe command '{}'\n",cmd);
          return false;
        }
        FILE *bf = Portable::fopen(Doxygen::filterDBFileName,"a+b");
        FilterCacheItem item;
        item.filePos = m_endPos;
        if (bf==nullptr)
        {
          // handle error
          err("Error opening filter database file {}\n",Doxygen::filterDBFileName);
          Portable::pclose(f);
          return false;
        }
        // append the filtered output to the database file
        size_t size=0;
        while (!feof(f))
        {
          const int blockSize = 4096;
          char buf[blockSize];
          size_t bytesRead = fread(buf,1,blockSize,f);
          size_t bytesWritten = fwrite(buf,1,bytesRead,bf);
          if (bytesRead!=bytesWritten)
          {
            // handle error
            err("Failed to write to filter database {}. Wrote {} out of {} bytes\n",
                Doxygen::filterDBFileName,bytesWritten,bytesRead);
            Portable::pclose(f);
            fclose(bf);
            return false;
          }
          size+=bytesWritten;
          str+=std::string_view(buf,bytesWritten);
        }
        item.fileSize = size;
        // add location entry to the dictionary
        m_cache.emplace(fileName.str(),item);
        Debug::print(Debug::FilterOutput,0,"Storing new filter result for {} in {} at offset={} size={}\n",
               fileName,Doxygen::filterDBFileName,item.filePos,item.fileSize);
        // update end of file position
        m_endPos += size;
        Portable::pclose(f);
        fclose(bf);
 
        // shrink buffer to [startLine..endLine] part
        shrinkBuffer(str,fileName,startLine,endLine);
      }
      return true;
    }

References err, Debug::ExtCmd, FilterCache::FilterCacheItem::filePos, FilterCache::FilterCacheItem::fileSize, Doxygen::filterDBFileName, Debug::FilterOutput, Portable::fopen(), getFragmentLocation(), m_cache, m_endPos, m_lineOffsets, m_mutex, Portable::pclose(), Portable::popen(), Debug::print(), readFragmentFromFile(), shrinkBuffer(), and DString::str().

Referenced by getFileContents().

getFragmentLocation()

auto FilterCache::getFragmentLocation ( const LineOffsets & lineOffsets, size_t startLine, size_t endLine ) -> std::tuple< size_t, size_t >

inlineprivate

Returns the byte offset and size within a file of a fragment given the array of line offsets and the start and end line of the fragment.

Definition at line 697 of file definition.cpp.

    {
      assert(startLine > 0);
      assert(startLine <= endLine);
      const size_t startLineOffset = lineOffsets[std::min(startLine-1,lineOffsets.size()-1)];
      const size_t endLineOffset   = lineOffsets[std::min(endLine,    lineOffsets.size()-1)];
      assert(startLineOffset <= endLineOffset);
      const size_t fragmentSize = endLineOffset-startLineOffset;
      return std::tie(startLineOffset,fragmentSize);
    }

Referenced by getFileContentsDisk(), getFileContentsPipe(), and shrinkBuffer().

instance()

FilterCache & FilterCache::instance ( )

static

Definition at line 744 of file definition.cpp.

{
  static FilterCache theInstance;
  return theInstance;
}

References FilterCache().

Referenced by readCodeFragment().

readFragmentFromFile()

void FilterCache::readFragmentFromFile ( std::string & str, const DString & fileName, size_t startOffset, size_t size = 0 )

inlineprivate

Reads the fragment start at byte offset startOffset of file fileName into buffer str. Result will be a null terminated. If size==0 the whole file will be read and startOffset is ignored. If size>0, size bytes will be read.

Definition at line 728 of file definition.cpp.

    {
      std::ifstream ifs = Portable::openInputStream(fileName,true,true);
      if (size==0) { startOffset=0; size = static_cast<size_t>(ifs.tellg()); }
      ifs.seekg(startOffset, std::ios::beg);
      str.resize(size);
      ifs.read(str.data(), size);
    }

References Portable::openInputStream().

Referenced by getFileContentsDisk(), and getFileContentsPipe().

shrinkBuffer()

void FilterCache::shrinkBuffer ( std::string & str, const DString & fileName, size_t startLine, size_t endLine )

inlineprivate

Shrinks buffer str which should hold the contents of fileName to the fragment starting a line startLine and ending at line endLine

Definition at line 711 of file definition.cpp.

    {
      // compute offsets from start for each line
      compileLineOffsets(fileName,str);
      auto it = m_lineOffsets.find(fileName.str());
      assert(it!=m_lineOffsets.end());
      const LineOffsets &lineOffsets = it->second;
      auto [ startLineOffset, fragmentSize] = getFragmentLocation(lineOffsets,startLine,endLine);
      //printf("%s: new file [%zu-%zu]->[%zu-%zu] size=%zu\n",
      //    qPrint(fileName),startLine,endLine,startLineOffset,endLineOffset,fragmentSize);
      str.erase(0,startLineOffset);
      str.resize(fragmentSize);
    }

References compileLineOffsets(), getFragmentLocation(), m_lineOffsets, and DString::str().

Referenced by getFileContentsDisk(), and getFileContentsPipe().

Member Data Documentation

m_cache

std::unordered_map<std::string,FilterCacheItem> FilterCache::m_cache

private

Definition at line 738 of file definition.cpp.

Referenced by getFileContentsPipe().

m_endPos

size_t FilterCache::m_endPos

private

Definition at line 741 of file definition.cpp.

Referenced by FilterCache(), and getFileContentsPipe().

m_lineOffsets

std::unordered_map<std::string,LineOffsets> FilterCache::m_lineOffsets

private

Definition at line 739 of file definition.cpp.

Referenced by compileLineOffsets(), getFileContentsDisk(), getFileContentsPipe(), and shrinkBuffer().

m_mutex

std::mutex FilterCache::m_mutex

private

Definition at line 740 of file definition.cpp.

Referenced by getFileContentsDisk(), and getFileContentsPipe().

---

The documentation for this class was generated from the following file:

• src/definition.cpp