docparser.cpp

Go to the documentation of this file.

/******************************************************************************
 *
 * Copyright (C) 1997-2022 by Dimitri van Heesch.
 *
 * Permission to use, copy, modify, and distribute this software and its
 * documentation under the terms of the GNU General Public License is hereby
 * granted. No representations are made about the suitability of this software
 * for any purpose. It is provided "as is" without express or implied warranty.
 * See the GNU General Public License for more details.
 *
 * Documents produced by Doxygen are derivative works derived from the
 * input used in their production; they are not affected by this license.
 *
 */
 
// own header
#include "docparser.h"
 
// standard includes
#include <cstdint>
#include <cstdio>
 
// other includes
#include "classlist.h"
#include "config.h"
#include "debug.h"
#include "dir.h"
#include "docparser_p.h"
#include "doxygen.h"
#include "filedef.h"
#include "fileinfo.h"
#include "filename.h"
#include "groupdef.h"
#include "htmldocvisitor.h"
#include "indexlist.h"
#include "message.h"
#include "namespacedef.h"
#include "outputlist.h"
#include "pagedef.h"
#include "portable.h"
#include "printdocvisitor.h"
#include "stringutil.h"
#include "textdocvisitor.h"
#include "trace.h"
#include "util.h"
 
#if !ENABLE_DOCPARSER_TRACING
#undef  AUTO_TRACE
#undef  AUTO_TRACE_ADD
#undef  AUTO_TRACE_EXIT
#define AUTO_TRACE(...)      (void)0
#define AUTO_TRACE_ADD(...)  (void)0
#define AUTO_TRACE_EXIT(...) (void)0
#endif
 
 
//---------------------------------------------------------------------------
 
IDocParserPtr createDocParser()
{
  return std::make_unique<DocParser>();
}
 
void DocParser::pushContext()
{
  //DString indent;
  //indent.fill(' ',contextStack.size()*2+2);
  //printf("%sdocParserPushContext() count=%zu\n",qPrint(indent),context.nodeStack.size());
 
  tokenizer.pushContext();
  contextStack.emplace();
  auto &ctx = contextStack.top();
  ctx           = context;
  ctx.fileName  = tokenizer.getFileName();
  ctx.lineNo    = tokenizer.getLineNr();
  context.token = tokenizer.token();
}
 
void DocParser::popContext()
{
  auto &ctx = contextStack.top();
  context = ctx;
  tokenizer.setFileName(ctx.fileName);
  tokenizer.setLineNr(ctx.lineNo);
  contextStack.pop();
  tokenizer.popContext();
  context.token = tokenizer.token();
 
  //DString indent;
  //indent.fill(' ',contextStack.size()*2+2);
  //printf("%sdocParserPopContext() count=%zu\n",qPrint(indent),context.nodeStack.size());
}
 
//---------------------------------------------------------------------------
 
/*! search for an image in the imageNameDict and if found
 * copies the image to the output directory (which depends on the \a type
 * parameter).
 */
DString DocParser::findAndCopyImage(const DString &fileName, DocImage::Type type, bool doWarn)
{
  DString result;
  bool ambig = false;
  FileDef *fd = Doxygen::imageNameLinkedMap->findFileDef(fileName,ambig);
  //printf("Search for %s\n",fileName);
  if (fd)
  {
    if (ambig & doWarn)
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),
          "image file name '{}' is ambiguous.\n"
          "Possible candidates:\n{}", fileName,Doxygen::imageNameLinkedMap->showFileDefMatches(fileName));
    }
 
    DString inputFile = fd->absFilePath();
    FileInfo infi(inputFile.str());
    if (infi.exists())
    {
      result = fileName;
      if (size_t i = result.rfind('/') ; i!=DString::npos || (i=result.rfind('\\'))!=DString::npos)
      {
        result = result.mid(i+1);
      }
      //printf("fileName=%s result=%s\n",fileName,qPrint(result));
      DString outputDir;
      switch(type)
      {
        case DocImage::Html:
          if (!Config_getBool(GENERATE_HTML)) return result;
          outputDir = Config_getString(HTML_OUTPUT);
          break;
        case DocImage::Latex:
          if (!Config_getBool(GENERATE_LATEX)) return result;
          outputDir = Config_getString(LATEX_OUTPUT);
          break;
        case DocImage::DocBook:
          if (!Config_getBool(GENERATE_DOCBOOK)) return result;
          outputDir = Config_getString(DOCBOOK_OUTPUT);
          break;
        case DocImage::Rtf:
          if (!Config_getBool(GENERATE_RTF)) return result;
          outputDir = Config_getString(RTF_OUTPUT);
          break;
        case DocImage::Xml:
          if (!Config_getBool(GENERATE_XML)) return result;
          outputDir = Config_getString(XML_OUTPUT);
          break;
      }
      DString outputFile = outputDir+"/"+result;
      FileInfo outfi(outputFile.str());
      if (outfi.isSymLink())
      {
        Dir().remove(outputFile.str());
        warn_doc_error(context.fileName,tokenizer.getLineNr(),
            "destination of image {} is a symlink, replacing with image",
            outputFile);
      }
      if (outputFile!=inputFile) // prevent copying to ourself
      {
        if (copyFile(inputFile,outputFile) && type==DocImage::Html)
        {
          Doxygen::indexList->addImageFile(result);
        }
      }
    }
    else
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),
                     "could not open image {}",fileName);
    }
 
    if (type==DocImage::Latex && Config_getBool(USE_PDFLATEX) &&
        fd->name().endsWith(".eps")
       )
    { // we have an .eps image in pdflatex mode => convert it to a pdf.
      DString outputDir = Config_getString(LATEX_OUTPUT);
      DString baseName  = fd->name().left(fd->name().length()-4);
      DString epstopdfArgs(4096, DString::ExplicitSize);
      epstopdfArgs.sprintf("\"%s/%s.eps\" --outfile=\"%s/%s.pdf\"",
                           qPrint(outputDir), qPrint(baseName),
                           qPrint(outputDir), qPrint(baseName));
      if (Portable::system("epstopdf",epstopdfArgs)!=0)
      {
        err("Problems running epstopdf. Check your TeX installation!\n");
      }
      else
      {
        Dir().remove(outputDir.str()+"/"+baseName.str()+".eps");
      }
      return baseName;
    }
  }
  else
  {
    result=fileName;
    if (!result.startsWith("http:") && !result.startsWith("https:") && doWarn)
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),
           "image file {} is not found in IMAGE_PATH: "
           "assuming external image.",fileName
          );
    }
  }
  return result;
}
 
/*! Collects the parameters found with \@param command
 *  in a list context.paramsFound. If
 *  the parameter is not an actual parameter of the current
 *  member context.memberDef, then a warning is raised (unless warnings
 *  are disabled altogether).
 */
void DocParser::checkArgumentName()
{
  if (!(Config_getBool(WARN_IF_DOC_ERROR) || Config_getBool(WARN_IF_INCOMPLETE_DOC))) return;
  if (context.memberDef==nullptr) return; // not a member
  std::string name = context.token->name.str();
  const ArgumentList &al=context.memberDef->isDocsForDefinition() ?
                         context.memberDef->argumentList() :
                         context.memberDef->declArgumentList();
  SrcLangExt lang      = context.memberDef->getLanguage();
  context.numParameters = static_cast<int>(al.size());
  //printf("isDocsForDefinition()=%d\n",context.memberDef->isDocsForDefinition());
  if (al.empty()) return; // no argument list
 
  static const reg::Ex re(R"(\$?\w+\.*)");
  static const reg::Ex re_digits(R"(\d+)");
  reg::Iterator it(name,re);
  reg::Iterator end;
  for (; it!=end ; ++it)
  {
    const auto &match = *it;
    DString aName=match.str();
    int number = reg::match(aName.view(),re_digits) ? std::atoi(aName.data()) : -1;
    if (lang==SrcLangExt::Fortran) aName=aName.lower();
    //printf("aName='%s'\n",qPrint(aName));
    bool found=false;
    int position=1;
    for (const Argument &a : al)
    {
      DString argName = context.memberDef->isDefine() ? a.type : a.name;
      if (lang==SrcLangExt::Fortran) argName=argName.lower();
      argName=argName.stripWhiteSpace();
      //printf("argName='%s' aName=%s\n",qPrint(argName),qPrint(aName));
      if (argName.endsWith("...")) argName=argName.left(argName.length()-3);
      bool sameName = aName==argName;
      bool samePosition = position==number;
      if (samePosition || sameName) // @param <number> or @param name
      {
        if (!sameName) // replace positional argument with real name or -
        {
          context.token->name = argName.empty() ? "-" : argName;
        }
        context.paramsFound.insert(aName.str());
        found=true;
        break;
      }
      else if (aName==".") // replace . by - in the output
      {
        context.token->name = "-";
      }
      position++;
    }
    if (!found)
    {
      //printf("member type=%d\n",context.memberDef->memberType());
      DString scope=context.memberDef->getScopeString();
      if (!scope.empty()) scope+="::"; else scope="";
      DString inheritedFrom = "";
      DString docFile = context.memberDef->docFile();
      int docLine = context.memberDef->docLine();
      const MemberDef *inheritedMd = context.memberDef->inheritsDocsFrom();
      if (inheritedMd) // documentation was inherited
      {
        inheritedFrom.sprintf(" inherited from member %s at line "
            "%d in file %s",qPrint(inheritedMd->name()),
            inheritedMd->docLine(),qPrint(inheritedMd->docFile()));
        docFile = context.memberDef->getDefFileName();
        docLine = context.memberDef->getDefLine();
      }
      DString alStr = argListToString(al);
      if (number==0)
      {
        warn_doc_error(docFile,docLine,
            "positional argument with value '0' of command @param "
            "is invalid, first parameter has index '1' for {}{}{}{}",
            scope, context.memberDef->name(),
            alStr, inheritedFrom);
        context.token->name = "-";
      }
      else if (number>0)
      {
        warn_doc_error(docFile,docLine,
            "positional argument '{}' of command @param "
            "is larger than the number of parameters ({}) for {}{}{}{}",
            number, al.size(), scope, context.memberDef->name(),
            alStr, inheritedFrom);
        context.token->name = "-";
      }
      else
      {
        warn_doc_error(docFile,docLine,
            "argument '{}' of command @param "
            "is not found in the argument list of {}{}{}{}",
            aName, scope, context.memberDef->name(),
            alStr, inheritedFrom);
      }
    }
  }
}
/*! Collects the return values found with \@retval command
 *  in a global list g_parserContext.retvalsFound.
 */
void DocParser::checkRetvalName()
{
  DString name = context.token->name;
  if (!Config_getBool(WARN_IF_DOC_ERROR)) return;
  if (context.memberDef==nullptr || name.empty()) return; // not a member or no valid name
  if (context.retvalsFound.count(name.str())==1) // only report the first double entry
  {
     warn_doc_error(context.memberDef->getDefFileName(),
                    context.memberDef->getDefLine(),
                    "return value '{}' of {} has multiple documentation sections",
                    name, context.memberDef->qualifiedName());
  }
  context.retvalsFound.insert(name.str());
}
 
/*! Checks if the parameters that have been specified using \@param are
 *  indeed all parameters and that a parameter does not have multiple
 *  \@param blocks.
 *  Must be called after checkArgumentName() has been called for each
 *  argument.
 */
void DocParser::checkUnOrMultipleDocumentedParams()
{
  if (context.memberDef && context.hasParamCommand)
  {
    const ArgumentList &al=context.memberDef->isDocsForDefinition() ?
      context.memberDef->argumentList() :
      context.memberDef->declArgumentList();
    SrcLangExt lang = context.memberDef->getLanguage();
    if (!al.empty())
    {
      ArgumentList undocParams;
      int position = 1;
      for (const Argument &a: al)
      {
        DString argName = context.memberDef->isDefine() ? a.type : a.name;
        if (lang==SrcLangExt::Fortran) argName = argName.lower();
        argName=argName.stripWhiteSpace();
        DString aName = argName;
        if (argName.endsWith("...")) argName=argName.left(argName.length()-3);
        if (lang==SrcLangExt::Python && (argName=="self" || argName=="cls"))
        {
          // allow undocumented self / cls parameter for Python
        }
        else if (lang==SrcLangExt::Cpp && (a.type=="this" || a.type.startsWith("this ")))
        {
          // allow undocumented this (for C++23 deducing this), see issue #11123
        }
        else if (!argName.empty())
        {
          size_t count_named      = context.paramsFound.count(argName.str());
          size_t count_positional = context.paramsFound.count(std::to_string(position));
          if (count_named==0 && count_positional==0 && a.docs.empty())
          {
            undocParams.push_back(a);
          }
          else if (count_named==1 && count_positional==1 && Config_getBool(WARN_IF_DOC_ERROR))
          {
            warn_doc_error(context.memberDef->docFile(),
                           context.memberDef->docLine(),
                           "argument {} from the argument list of {} has both named and positional @param documentation sections",
                           aName, context.memberDef->qualifiedName());
          }
          else if (count_named+count_positional>1 && Config_getBool(WARN_IF_DOC_ERROR))
          {
            warn_doc_error(context.memberDef->docFile(),
                           context.memberDef->docLine(),
                           "argument {} from the argument list of {} has multiple @param documentation sections",
                           aName, context.memberDef->qualifiedName());
          }
        }
        position++;
      }
      if (!undocParams.empty() && Config_getBool(WARN_IF_INCOMPLETE_DOC))
      {
        bool first=true;
        DString errMsg = "The following parameter";
        if (undocParams.size()>1) errMsg+="s";
        errMsg+=DString(" of ")+
            context.memberDef->qualifiedName() +
            argListToString(al) +
            (undocParams.size()>1 ? " are" : " is") + " not documented:\n";
        for (const Argument &a : undocParams)
        {
          DString argName = context.memberDef->isDefine() ? a.type : a.name;
          if (lang==SrcLangExt::Fortran) argName = argName.lower();
          argName=argName.stripWhiteSpace();
          if (!first) errMsg+="\n";
          first=false;
          errMsg+="  parameter '"+argName+"'";
        }
        warn_incomplete_doc(context.memberDef->docFile(), context.memberDef->docLine(), "{}", errMsg);
      }
 
      if (Config_getBool(WARN_IF_DOC_ERROR) && context.paramPosition-1 > context.numParameters)
      {
        warn_doc_error(context.memberDef->docFile(),
                       context.memberDef->docLine(),
                       "too many @param commands for function {}. Found {} while function has {} parameter{}",
                       context.memberDef->qualifiedName(), context.paramPosition-1, context.numParameters, context.numParameters!=1?"s":"");
      }
    }
    else
    {
      if (context.paramsFound.empty() && Config_getBool(WARN_IF_DOC_ERROR))
      {
        warn_doc_error(context.memberDef->docFile(),
                       context.memberDef->docLine(),
                       "{} has @param documentation sections but no arguments",
                       context.memberDef->qualifiedName());
      }
    }
  }
}
 
 
//---------------------------------------------------------------------------
 
//---------------------------------------------------------------------------
 
//---------------------------------------------------------------------------
/*! Looks for a documentation block with name commandName in the current
 *  context (g_parserContext.context). The resulting documentation string is
 *  put in pDoc, the definition in which the documentation was found is
 *  put in pDef.
 *  @retval true if name was found.
 *  @retval false if name was not found.
 */
bool DocParser::findDocsForMemberOrCompound(const DString &commandName,
                                            DString *pDoc,
                                            DString *pBrief,
                                            const Definition **pDef)
{
  AUTO_TRACE("commandName={}",commandName);
  *pDoc="";
  *pBrief="";
  *pDef=nullptr;
  DString cmdArg=commandName;
  if (cmdArg.empty())
  {
    AUTO_TRACE_EXIT("empty");
    return false;
  }
 
  const FileDef      *fd=nullptr;
  const GroupDef     *gd=nullptr;
  const PageDef      *pd=nullptr;
  gd = Doxygen::groupLinkedMap->find(cmdArg);
  if (gd) // group
  {
    *pDoc=gd->documentation();
    *pBrief=gd->briefDescription();
    *pDef=gd;
    AUTO_TRACE_EXIT("group");
    return true;
  }
  pd = Doxygen::pageLinkedMap->find(cmdArg);
  if (pd) // page
  {
    *pDoc=pd->documentation();
    *pBrief=pd->briefDescription();
    *pDef=pd;
    AUTO_TRACE_EXIT("page");
    return true;
  }
  bool ambig = false;
  fd = Doxygen::inputNameLinkedMap->findFileDef(cmdArg,ambig);
  if (fd && !ambig) // file
  {
    *pDoc=fd->documentation();
    *pBrief=fd->briefDescription();
    *pDef=fd;
    AUTO_TRACE_EXIT("file");
    return true;
  }
 
  // for symbols we need to normalize the separator, so A#B, or A\B, or A.B becomes A::B
  cmdArg = substitute(cmdArg,"#","::");
  cmdArg = substitute(cmdArg,"\\","::");
  bool extractAnonNs = Config_getBool(EXTRACT_ANON_NSPACES);
  if (extractAnonNs &&
      cmdArg.startsWith("anonymous_namespace{")
      )
  {
    size_t rightBracePos = cmdArg.find("}", dstrlen("anonymous_namespace{"));
    if (rightBracePos != DString::npos)
    {
      DString leftPart = cmdArg.left(rightBracePos + 1);
      DString rightPart = cmdArg.mid(rightBracePos + 1);
      rightPart = substitute(rightPart, ".", "::");
      cmdArg = leftPart + rightPart;
    }
    else
    {
      cmdArg = substitute(cmdArg,".","::");
    }
  }
  else
  {
    cmdArg = substitute(cmdArg,".","::");
  }
 
  int l=static_cast<int>(cmdArg.length());
 
  size_t funcStart=cmdArg.find('(');
  if (funcStart==DString::npos)
  {
    funcStart=l;
  }
  else
  {
    // Check for the case of operator() and the like.
    // beware of scenarios like operator()((foo)bar)
    size_t secondParen = cmdArg.find('(', funcStart+1);
    size_t leftParen   = cmdArg.find(')', funcStart+1);
    if (leftParen!=DString::npos && secondParen!=DString::npos)
    {
      if (leftParen<secondParen)
      {
        funcStart=secondParen;
      }
    }
  }
 
  DString name=removeRedundantWhiteSpace(cmdArg.left(funcStart));
  DString args=cmdArg.right(l-funcStart);
  // try if the link is to a member
  GetDefInput input(
      context.context.find('.')==DString::npos ? context.context : DString(), // find('.') is a hack to detect files
      name,
      args);
  input.checkCV=true;
  GetDefResult result = getDefs(input);
  //printf("found=%d context=%s name=%s\n",result.found,qPrint(context.context),qPrint(name));
  if (result.found && result.md)
  {
    *pDoc=result.md->documentation();
    *pBrief=result.md->briefDescription();
    *pDef=result.md;
    AUTO_TRACE_EXIT("member");
    return true;
  }
 
  int scopeOffset=static_cast<int>(context.context.length());
  do // for each scope
  {
    DString fullName=cmdArg;
    if (scopeOffset>0)
    {
      fullName.prepend(context.context.left(scopeOffset)+"::");
    }
    //printf("Trying fullName='%s'\n",qPrint(fullName));
 
    // try class, namespace, group, page, file reference
    const ClassDef *cd = Doxygen::classLinkedMap->find(fullName);
    if (cd) // class
    {
      *pDoc=cd->documentation();
      *pBrief=cd->briefDescription();
      *pDef=cd;
      AUTO_TRACE_EXIT("class");
      return true;
    }
    const NamespaceDef *nd = Doxygen::namespaceLinkedMap->find(fullName);
    if (nd) // namespace
    {
      *pDoc=nd->documentation();
      *pBrief=nd->briefDescription();
      *pDef=nd;
      AUTO_TRACE_EXIT("namespace");
      return true;
    }
    if (scopeOffset==0)
    {
      scopeOffset=-1;
    }
    else
    {
      size_t o = context.context.rfind("::",scopeOffset-1);
      scopeOffset = o!=DString::npos ? static_cast<int>(o) : 0;
    }
  } while (scopeOffset>=0);
 
  AUTO_TRACE_EXIT("not found");
  return false;
}
 
//---------------------------------------------------------------------------
void DocParser::errorHandleDefaultToken(DocNodeVariant *parent,Token tok,
                                        DocNodeList &children,const DString &txt)
{
  switch (tok.value())
  {
    case TokenRetval::TK_COMMAND_AT:
      // fall through
    case TokenRetval::TK_COMMAND_BS:
      {
        char cs[2] = { tok.command_to_char(), 0 };
        children.append<DocWord>(this,parent,cs + context.token->name);
        warn_doc_error(context.fileName,tokenizer.getLineNr(),"Illegal command '{:c}{}' found as part of a {}",
         tok.command_to_char(),context.token->name,txt);
      }
      break;
    case TokenRetval::TK_SYMBOL:
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"Unsupported symbol '{}' found as part of a {}",
           qPrint(context.token->name), qPrint(txt));
      break;
    case TokenRetval::TK_HTMLTAG:
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"Unsupported HTML tag <{}{}> found as part of a {}",
           context.token->endTag ? "/" : "",context.token->name, txt);
      break;
    default:
      children.append<DocWord>(this,parent,context.token->name);
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"Unexpected token {} found as part of a {}",
        tok.to_string(), txt);
      break;
  }
}
 
//---------------------------------------------------------------------------
 
Token DocParser::handleStyleArgument(DocNodeVariant *parent,DocNodeList &children,const DString &cmdName)
{
  AUTO_TRACE("cmdName={}",cmdName);
  DString saveCmdName = cmdName;
  Token tok=tokenizer.lex();
  size_t styleStackSizeAtStart = context.styleStack.size();
  if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after \\{} command",
                   saveCmdName);
    return tok;
  }
  tok = tokenizer.lex();
  while (!tok.is_any_of(TokenRetval::TK_NONE, TokenRetval::TK_EOF, TokenRetval::TK_WHITESPACE,
                        TokenRetval::TK_NEWPARA, TokenRetval::TK_LISTITEM, TokenRetval::TK_ENDLIST)
        )
  {
    static const reg::Ex specialChar(R"([.,|()\‍[\‍]:;?])");
    if (tok.is(TokenRetval::TK_WORD) && context.token->name.length()==1 &&
        reg::match(context.token->name.str(),specialChar))
    {
      // special character that ends the markup command
      AUTO_TRACE_ADD("special character ending style argument: '{}' styleStackSize {}->{}",context.token->name,styleStackSizeAtStart,context.styleStack.size());
      if (context.styleStack.size() > styleStackSizeAtStart) // new styles opened inside command, but not closed
      {
        handlePendingStyleCommands(parent,children,context.styleStack.size()-styleStackSizeAtStart);
      }
      return tok;
    }
    if (!defaultHandleToken(parent,tok,children))
    {
      switch (tok.value())
      {
        case TokenRetval::TK_HTMLTAG:
          if (insideLI(parent) && Mappers::htmlTagMapper->map(context.token->name)!=HtmlTagType::UNKNOWN && context.token->endTag)
          {
            // ignore </li> as the end of a style command
          }
          else
          {
            AUTO_TRACE_EXIT("end tok={}",tok.to_string());
            return tok;
          }
          break;
        default:
          errorHandleDefaultToken(parent,tok,children,"\\" + saveCmdName + " command");
          break;
      }
      break;
    }
    tok = tokenizer.lex();
  }
  AUTO_TRACE_EXIT("end tok={}",tok.to_string());
  return (tok.is_any_of(TokenRetval::TK_NEWPARA,TokenRetval::TK_LISTITEM,TokenRetval::TK_ENDLIST)) ? tok : Token::make_RetVal_OK();
}
 
/*! Called when a style change starts. For instance a <b> command is
 *  encountered.
 */
void DocParser::handleStyleEnter(DocNodeVariant *parent,DocNodeList &children,
          DocStyleChange::Style s,const DString &tagName,const HtmlAttribList *attribs)
{
  AUTO_TRACE("tagName={}",tagName);
  children.append<DocStyleChange>(this,parent,context.nodeStack.size(),s,tagName,true,
                                  context.fileName,tokenizer.getLineNr(),attribs);
  context.styleStack.push(&children.back());
  context.inCodeStyle = s==DocStyleChange::Style::Typewriter;
}
 
/*! Called when a style change ends. For instance a </b> command is
 *  encountered.
 */
void DocParser::handleStyleLeave(DocNodeVariant *parent,DocNodeList &children,
         DocStyleChange::Style s,const DString &tagName)
{
  AUTO_TRACE("tagName={}",tagName);
  DString tagNameLower = DString(tagName).lower();
 
  auto topStyleChange = [](const DocStyleChangeStack &stack) -> const DocStyleChange &
  {
    return std::get<DocStyleChange>(*stack.top());
  };
 
  if (context.styleStack.empty() ||                            // no style change
      topStyleChange(context.styleStack).style()!=s ||         // wrong style change
      topStyleChange(context.styleStack).tagName()!=tagNameLower ||  // wrong style change
      topStyleChange(context.styleStack).position()!=context.nodeStack.size() // wrong position
     )
  {
    if (context.styleStack.empty())
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"found </{0}> tag without matching <{0}>",tagName);
    }
    else if (topStyleChange(context.styleStack).tagName()!=tagNameLower ||
             topStyleChange(context.styleStack).style()!=s)
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"found </{}> tag while expecting </{}>",
          tagName,topStyleChange(context.styleStack).tagName());
    }
    else
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"found </{}> at different nesting level ({}) than expected ({})",
          tagName,context.nodeStack.size(),topStyleChange(context.styleStack).position());
    }
  }
  else // end the section
  {
    children.append<DocStyleChange>(
          this,parent,context.nodeStack.size(),s,
          topStyleChange(context.styleStack).tagName(),false);
    context.styleStack.pop();
  }
  if (s==DocStyleChange::Style::Typewriter)
  {
    context.inCodeStyle = false;
  }
}
 
/*! Called at the end of a paragraph to close all open style changes
 *  (e.g. a <b> without a </b>). The closed styles are pushed onto a stack
 *  and entered again at the start of a new paragraph.
 */
void DocParser::handlePendingStyleCommands(DocNodeVariant *parent,DocNodeList &children, size_t numberOfElementsToClose)
{
  AUTO_TRACE("context.styleStack.size()={} numberOfElementsToClose={}",context.styleStack.size(),numberOfElementsToClose);
  if (!context.styleStack.empty())
  {
    if (numberOfElementsToClose==0) numberOfElementsToClose = context.styleStack.size(); // 0 is special value for "close all"
    const DocStyleChange *sc = &std::get<DocStyleChange>(*context.styleStack.top());
    while (sc && sc->position()>=context.nodeStack.size() && numberOfElementsToClose>0)
    { // there are unclosed style modifiers in the paragraph
      AUTO_TRACE_ADD("unclosed style {} at position {}",sc->styleString(),sc->position());
      children.append<DocStyleChange>(this,parent,context.nodeStack.size(),
                                           sc->style(),sc->tagName(),false);
      context.initialStyleStack.push(context.styleStack.top());
      context.styleStack.pop();
      sc = !context.styleStack.empty() ? &std::get<DocStyleChange>(*context.styleStack.top()) : nullptr;
      numberOfElementsToClose--;
    }
  }
}
 
void DocParser::handleInitialStyleCommands(DocNodeVariant *parent,DocNodeList &children)
{
  AUTO_TRACE();
  while (!context.initialStyleStack.empty())
  {
    const DocStyleChange &sc = std::get<DocStyleChange>(*context.initialStyleStack.top());
    handleStyleEnter(parent,children,sc.style(),sc.tagName(),&sc.attribs());
    context.initialStyleStack.pop();
  }
}
 
Token DocParser::handleAHref(DocNodeVariant *parent,DocNodeList &children,
                           const HtmlAttribList &tagHtmlAttribs)
{
  AUTO_TRACE();
  size_t index=0;
  Token retval = Token::make_RetVal_OK();
  for (const auto &opt : tagHtmlAttribs)
  {
    if (opt.name=="name" || opt.name=="id") // <a name=label> or <a id=label> tag
    {
      if (!opt.value.empty())
      {
        children.append<DocAnchor>(this,parent,opt.value,true);
        break; // stop looking for other tag attribs
      }
      else
      {
        warn_doc_error(context.fileName,tokenizer.getLineNr(),"found <a> tag with name option but without value!");
      }
    }
    else if (opt.name=="href") // <a href=url>..</a> tag
    {
      // copy attributes
      HtmlAttribList attrList = tagHtmlAttribs;
      // and remove the href attribute
      attrList.erase(attrList.begin()+index);
      DString relPath;
      if (opt.value.at(0) != '#') relPath = context.relPath;
      children.append<DocHRef>(this, parent, attrList,
                               opt.value, relPath,
                               convertNameToFile(context.fileName, false, true));
      context.insideHtmlLink=true;
      retval = children.get_last<DocHRef>()->parse();
      context.insideHtmlLink=false;
      tokenizer.setStatePara();
      break;
    }
    else // unsupported option for tag a
    {
    }
    ++index;
  }
  return retval;
}
 
void DocParser::handleUnclosedStyleCommands()
{
  AUTO_TRACE("content.initialStyleStack.size()={}",context.initialStyleStack.size());
  if (!context.initialStyleStack.empty())
  {
    DString tagName = std::get<DocStyleChange>(*context.initialStyleStack.top()).tagName();
    DString fileName = std::get<DocStyleChange>(*context.initialStyleStack.top()).fileName();
    int lineNr = std::get<DocStyleChange>(*context.initialStyleStack.top()).lineNr();
    context.initialStyleStack.pop();
    handleUnclosedStyleCommands();
    if (lineNr != -1)
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),
               "end of comment block while expecting "
               "command </{}> (Probable start '{}' at line {})",tagName, fileName, lineNr);
    }
    else
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),
               "end of comment block while expecting command </{}>",tagName);
    }
  }
}
 
void DocParser::handleLinkedWord(DocNodeVariant *parent,DocNodeList &children,bool ignoreAutoLinkFlag,bool typeLinkOnly)
{
  // helper to check if word w starts with any of the words in AUTOLINK_IGNORE_WORDS
  auto ignoreWord = [](const DString &w) -> bool {
    auto list = Config_getList(AUTOLINK_IGNORE_WORDS);
    return std::find_if(list.begin(), list.end(),
                       [&w](const auto &ignore) { return w.startsWith(ignore); }
                      )!=list.end();
  };
  DString name = linkToText(context.lang,context.token->name,true);
  AUTO_TRACE("word={}",name);
  if (!context.autolinkSupport || ignoreAutoLinkFlag || ignoreWord(context.token->name)) // no autolinking -> add as normal word
  {
    children.append<DocWord>(this,parent,name);
    return;
  }
 
  // ------- try to turn the word 'name' into a link
 
  const Definition *compound=nullptr;
  const MemberDef  *member=nullptr;
  size_t len = context.token->name.length();
  ClassDef *cd=nullptr;
  bool ambig = false;
  FileDef *fd = Doxygen::inputNameLinkedMap->findFileDef(context.fileName,ambig);
  auto lang = context.lang;
  bool inSeeBlock = context.inSeeBlock || context.inCodeStyle;
  //printf("handleLinkedWord(%s) context.context=%s\n",qPrint(context.token->name),qPrint(context.context));
  if (!context.insideHtmlLink &&
      (resolveRef(context.context,context.token->name,inSeeBlock,&compound,&member,lang,true,fd,true)
       || (!context.context.empty() &&  // also try with global scope
           resolveRef(DString(),context.token->name,inSeeBlock,&compound,&member,lang,false,nullptr,true))
      )
     )
  {
    //printf("ADD %s = %p (linkable?=%d typeLinkOnly=%d)\n",
    //    qPrint(context.token->name),(void*)member,member ? member->isLinkable() : false, typeLinkOnly);
    if (member && member->isLinkable())
    {
      if (!typeLinkOnly || context.token->name.startsWith("#") ||
          member->isTypedef() || member->isEnumerate() || member->isEnumValue()) // filter on type links
      {
        AUTO_TRACE_ADD("resolved reference as member link");
        if (member->isObjCMethod())
        {
          bool localLink = context.memberDef ? member->getClassDef()==context.memberDef->getClassDef() : false;
          name = member->objCMethodName(localLink,inSeeBlock);
        }
        children.append<DocLinkedWord>(
              this,parent,name,
              member->getReference(),
              member->getOutputFileBase(),
              member->anchor(),
              member->briefDescriptionAsTooltip());
      }
      else // explicit type link, but not a type
      {
        AUTO_TRACE_ADD("no link as request is type but member is not a type or explicit link");
        children.append<DocWord>(this,parent,context.token->name);
      }
    }
    else if (compound->isLinkable()) // compound link
    {
      AUTO_TRACE_ADD("resolved reference as compound link");
      DString anchor = compound->anchor();
      if (compound->definitionType()==Definition::TypeFile)
      {
        name=context.token->name;
      }
      else if (compound->definitionType()==Definition::TypeGroup)
      {
        name=toGroupDef(compound)->groupTitle();
      }
      children.append<DocLinkedWord>(
            this,parent,name,
            compound->getReference(),
            compound->getOutputFileBase(),
            anchor,
            compound->briefDescriptionAsTooltip());
    }
    else if (compound->definitionType()==Definition::TypeFile &&
             (toFileDef(compound))->generateSourceFile()
            ) // undocumented file that has source code we can link to
    {
      AUTO_TRACE_ADD("resolved reference as source link");
      children.append<DocLinkedWord>(
             this,parent,context.token->name,
             compound->getReference(),
             compound->getSourceFileBase(),
             "",
             compound->briefDescriptionAsTooltip());
    }
    else // not linkable
    {
      AUTO_TRACE_ADD("resolved reference as unlinkable compound={} (linkable={}) member={} (linkable={})",
                     compound ? compound->name() : "<none>", compound ? (int)compound->isLinkable() : -1,
                     member   ? member->name()   : "<none>", member   ? (int)member->isLinkable()   : -1);
      children.append<DocWord>(this,parent,name);
    }
  }
  else if (!context.insideHtmlLink && len>1 && context.token->name.at(len-1)==':')
  {
    // special case, where matching Foo: fails to be an Obj-C reference,
    // but Foo itself might be linkable.
    context.token->name=context.token->name.left(len-1);
    handleLinkedWord(parent,children,ignoreAutoLinkFlag);
    children.append<DocWord>(this,parent,":");
  }
  else if (!context.insideHtmlLink && (cd=getClass(context.token->name+"-p")))
  {
    // special case 2, where the token name is not a class, but could
    // be a Obj-C protocol
    children.append<DocLinkedWord>(
          this,parent,name,
          cd->getReference(),
          cd->getOutputFileBase(),
          cd->anchor(),
          cd->briefDescriptionAsTooltip());
  }
  else if (const RequirementIntf *req = RequirementManager::instance().find(name); req!=nullptr) // link to requirement
  {
    if (Config_getBool(GENERATE_REQUIREMENTS))
    {
      children.append<DocLinkedWord>(
           this,parent,name,
           DString(), // link to local requirements overview also for external references
           req->getOutputFileBase(),
           req->id(),
           req->title()
          );
    }
    else // cannot link to a page that does not exist
    {
      children.append<DocWord>(this,parent,name);
    }
  }
  else // normal non-linkable word
  {
    AUTO_TRACE_ADD("non-linkable");
    if (context.token->name.startsWith("#"))
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"explicit link request to '{}' could not be resolved",name);
    }
    children.append<DocWord>(this,parent,context.token->name);
  }
}
 
void DocParser::handleParameterType(DocNodeVariant *parent,DocNodeList &children,const DString &paramTypes)
{
  DString name = context.token->name; // save token name
  AUTO_TRACE("name={}",name);
  DString name1;
  size_t p=0, i=0, ii=0;
  while ((i=paramTypes.find('|',p))!=DString::npos)
  {
    name1 = paramTypes.mid(p,i-p);
    ii=name1.find('[');
    context.token->name=ii!=DString::npos ? name1.mid(0,ii) : name1; // take part without []
    handleLinkedWord(parent,children);
    if (ii!=DString::npos) children.append<DocWord>(this,parent,name1.mid(ii)); // add [] part
    p=i+1;
    children.append<DocSeparator>(this,parent,"|");
  }
  name1 = paramTypes.mid(p);
  ii=name1.find('[');
  context.token->name=ii!=DString::npos ? name1.mid(0,ii) : name1;
  handleLinkedWord(parent,children);
  if (ii!=DString::npos) children.append<DocWord>(this,parent,name1.mid(ii));
  context.token->name = name; // restore original token name
}
 
void DocParser::handleInternalRef(DocNodeVariant *parent,DocNodeList &children)
{
  Token tok=tokenizer.lex();
  DString tokenName = context.token->name;
  AUTO_TRACE("name={}",tokenName);
  if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after \\{} command", tokenName);
    return;
  }
  tokenizer.setStateInternalRef();
  tok=tokenizer.lex(); // get the reference id
  if (!tok.is_any_of(TokenRetval::TK_WORD,TokenRetval::TK_LNKWORD))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected token {} as the argument of {}",
        tok.to_string(),tokenName);
    return;
  }
  children.append<DocInternalRef>(this,parent,context.token->name);
  children.get_last<DocInternalRef>()->parse();
}
 
void DocParser::handleAnchor(DocNodeVariant *parent,DocNodeList &children)
{
  AUTO_TRACE();
  Token tok=tokenizer.lex();
  if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after \\{} command",
        context.token->name);
    return;
  }
 
  { DocTokenizer::AutoSaveState saveState(tokenizer);
    tokenizer.setStateAnchor();
    tok=tokenizer.lex();
    if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected end of comment block while parsing the "
          "argument of command {}",context.token->name);
      return;
    }
    else if (!tok.is_any_of(TokenRetval::TK_WORD,TokenRetval::TK_LNKWORD))
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected token {} as the argument of {}",
          tok.to_string(),context.token->name);
      return;
    }
  }
  children.append<DocAnchor>(this,parent,context.token->name,false);
}
 
void DocParser::handleCite(DocNodeVariant *parent,DocNodeList &children)
{
  AUTO_TRACE();
  // get the argument of the cite command.
  Token tok=tokenizer.lex();
 
  CiteInfoOption option;
  if (tok.is(TokenRetval::TK_WORD) && context.token->name=="{")
  {
    tokenizer.setStateOptions();
    tokenizer.lex();
    StringVector optList=split(context.token->name.str(),",");
    for (auto const &opt : optList)
    {
      if (opt == "number")
      {
        if (!option.isUnknown())
        {
          warn(context.fileName,tokenizer.getLineNr(),"Multiple options specified with \\{}, discarding '{}'", context.token->name, opt);
        }
        else
        {
          option = CiteInfoOption::makeNumber();
        }
      }
      else if (opt == "year")
      {
        if (!option.isUnknown())
        {
          warn(context.fileName,tokenizer.getLineNr(),"Multiple options specified with \\{}, discarding '{}'", context.token->name, opt);
        }
        else
        {
          option = CiteInfoOption::makeYear();
        }
      }
      else if (opt == "shortauthor")
      {
        if (!option.isUnknown())
        {
          warn(context.fileName,tokenizer.getLineNr(),"Multiple options specified with \\{}, discarding '{}'", context.token->name, opt);
        }
        else
        {
          option = CiteInfoOption::makeShortAuthor();
        }
      }
      else if (opt == "nopar")
      {
        option.setNoPar();
      }
      else if (opt == "nocite")
      {
        option.setNoCite();
      }
      else
      {
        warn(context.fileName,tokenizer.getLineNr(),"Unknown option specified with \\{}, discarding '{}'", context.token->name, opt);
      }
    }
 
    if (option.isUnknown()) option.changeToNumber();
 
    tokenizer.setStatePara();
    tok=tokenizer.lex();
    if (!tok.is(TokenRetval::TK_WHITESPACE))
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after \\{} command",
          context.token->name);
      return;
    }
  }
  else if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after '\\{}' command",
      context.token->name);
    return;
  }
  else
  {
    option = CiteInfoOption::makeNumber();
  }
 
  { DocTokenizer::AutoSaveState saveState(tokenizer);
    tokenizer.setStateCite();
    tok=tokenizer.lex();
    if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected end of comment block while parsing the "
          "argument of command '\\{}'",context.token->name);
      return;
    }
    else if (!tok.is_any_of(TokenRetval::TK_WORD,TokenRetval::TK_LNKWORD))
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected token {} as the argument of '\\{}'",
          tok.to_string(),context.token->name);
      return;
    }
    context.token->sectionId = context.token->name;
    children.append<DocCite>(this,parent,context.token->name,context.context,option);
  }
 
}
 
void DocParser::handlePrefix(DocNodeVariant *parent,DocNodeList &children)
{
  AUTO_TRACE();
  Token tok=tokenizer.lex();
  if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after \\{} command", context.token->name);
    return;
  }
  tokenizer.setStatePrefix();
  tok=tokenizer.lex();
  if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected end of comment block while parsing the "
        "argument of command {}",context.token->name);
    return;
  }
  else if (!tok.is(TokenRetval::TK_WORD))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected token {} as the argument of {}",
        tok.to_string(),context.token->name);
    return;
  }
  context.prefix = context.token->name;
  tokenizer.setStatePara();
}
 
/* Helper function that deals with the title, width, and height arguments of various commands.
 * @param[in] cmd        Command id for which to extract caption and size info.
 * @param[in] parent     Parent node, owner of the children list passed as
 *                       the third argument.
 * @param[in] children   The list of child nodes to which the node representing
 *                       the token can be added.
 * @param[out] width     the extracted width specifier
 * @param[out] height    the extracted height specifier
 */
void DocParser::defaultHandleTitleAndSize(const CommandType cmd, DocNodeVariant *parent, DocNodeList &children, DString &width,DString &height)
{
  AUTO_TRACE();
  auto ns = AutoNodeStack(this,parent);
 
  // parse title
  tokenizer.setStateTitle();
  Token tok = tokenizer.lex();
  while (!tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
  {
    if (tok.is(TokenRetval::TK_WORD) && (context.token->name=="width=" || context.token->name=="height="))
    {
      // special case: no title, but we do have a size indicator
      break;
    }
    else if (tok.is(TokenRetval::TK_HTMLTAG))
    {
      tokenizer.unputString(context.token->text);
      break;
    }
    if (!defaultHandleToken(parent,tok,children))
    {
      errorHandleDefaultToken(parent,tok,children,Mappers::cmdMapper->find(cmd));
    }
    tok = tokenizer.lex();
  }
  // parse size attributes
  if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
  {
    tok=tokenizer.lex();
  }
  while (tok.is_any_of(TokenRetval::TK_WHITESPACE,TokenRetval::TK_WORD,TokenRetval::TK_HTMLTAG)) // there are values following the title
  {
    if (tok.is(TokenRetval::TK_WORD))
    {
      if (context.token->name=="width=" || context.token->name=="height=")
      {
        tokenizer.setStateTitleAttrValue();
        context.token->name = context.token->name.left(context.token->name.length()-1);
      }
 
      if (context.token->name=="width")
      {
        width = context.token->chars;
      }
      else if (context.token->name=="height")
      {
        height = context.token->chars;
      }
      else // other text after the title -> treat as normal text
      {
        tokenizer.unputString(context.token->name);
        //warn_doc_error(context.fileName,tokenizer.getLineNr(),"Unknown option '{}' after \\{} command, expected 'width' or 'height'",
        //               context.token->name, Mappers::cmdMapper->find(cmd));
        break;
      }
    }
 
    tok=tokenizer.lex();
    // if we found something we did not expect, push it back to the stream
    // so it can still be processed
    if (tok.is_any_of(TokenRetval::TK_COMMAND_AT,TokenRetval::TK_COMMAND_BS))
    {
      tokenizer.unputString(context.token->name);
      tokenizer.unputString(tok.is(TokenRetval::TK_COMMAND_AT) ? "@" : "\\");
      break;
    }
    else if (tok.is(TokenRetval::TK_SYMBOL))
    {
      tokenizer.unputString(context.token->name);
      break;
    }
    else if (tok.is(TokenRetval::TK_HTMLTAG))
    {
      tokenizer.unputString(context.token->text);
      break;
    }
  }
  tokenizer.setStatePara();
 
  handlePendingStyleCommands(parent,children);
  AUTO_TRACE_EXIT("width={} height={}",width,height);
}
 
void DocParser::handleImage(DocNodeVariant *parent, DocNodeList &children)
{
  AUTO_TRACE();
  bool inlineImage = false;
  DString anchorStr;
 
  Token tok=tokenizer.lex();
  if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    if (tok.is(TokenRetval::TK_WORD))
    {
      if (context.token->name == "{")
      {
        tokenizer.setStateOptions();
        tokenizer.lex();
        tokenizer.setStatePara();
        StringVector optList=split(context.token->name.str(),",");
        for (const auto &opt : optList)
        {
          if (opt.empty()) continue;
          DString locOpt(opt);
          DString locOptLow;
          locOpt = locOpt.stripWhiteSpace();
          locOptLow = locOpt.lower();
          if (locOptLow == "inline")
          {
            inlineImage = true;
          }
          else if (locOptLow.startsWith("anchor:"))
          {
            if (!anchorStr.empty())
            {
               warn_doc_error(context.fileName,tokenizer.getLineNr(),
                  "multiple use of option 'anchor' for 'image' command, ignoring: '{}'",
                  locOpt.mid(7));
            }
            else
            {
               anchorStr = locOpt.mid(7);
            }
          }
          else
          {
            warn_doc_error(context.fileName,tokenizer.getLineNr(),
                  "unknown option '{}' for 'image' command specified",
                  locOpt);
          }
        }
        tok=tokenizer.lex();
        if (!tok.is(TokenRetval::TK_WHITESPACE))
        {
          warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after \\image command");
          return;
        }
      }
    }
    else
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after \\image command");
      return;
    }
  }
  tok=tokenizer.lex();
  if (!tok.is_any_of(TokenRetval::TK_WORD,TokenRetval::TK_LNKWORD))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected token {} as the argument of \\image",
        tok.to_string());
    return;
  }
  tok=tokenizer.lex();
  if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after \\image command");
    return;
  }
  DocImage::Type t = DocImage::Html;
  DString imgType = context.token->name.lower();
  if      (imgType=="html")    t=DocImage::Html;
  else if (imgType=="latex")   t=DocImage::Latex;
  else if (imgType=="docbook") t=DocImage::DocBook;
  else if (imgType=="rtf")     t=DocImage::Rtf;
  else if (imgType=="xml")     t=DocImage::Xml;
  else
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"output format `{}` specified as the first argument of "
        "\\image command is not valid", imgType);
    return;
  }
  tokenizer.setStateFile();
  tok=tokenizer.lex();
  tokenizer.setStatePara();
  if (!tok.is(TokenRetval::TK_WORD))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected token {} as the argument of \\image", tok.to_string());
    return;
  }
  if (!anchorStr.empty())
  {
    children.append<DocAnchor>(this,parent,anchorStr,true);
  }
  HtmlAttribList attrList;
  children.append<DocImage>(this,parent,attrList,
                 findAndCopyImage(context.token->name,t),t,"",inlineImage);
  children.get_last<DocImage>()->parse();
}
 
void DocParser::handleRef(DocNodeVariant *parent, DocNodeList &children, char cmdChar, const DString &cmdName)
{
  AUTO_TRACE("cmdName={}",cmdName);
  DString saveCmdName = cmdName;
  DocTokenizer::AutoSaveState saveState(tokenizer);
  Token tok=tokenizer.lex();
  if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after '{:c}{}' command",
      cmdChar,qPrint(saveCmdName));
    return;
  }
  tokenizer.setStateRef();
  tok=tokenizer.lex(); // get the reference id
  if (!tok.is(TokenRetval::TK_WORD))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected token {} as the argument of '{:c}{}'",
        tok.to_string(),cmdChar,saveCmdName);
    return;
  }
  children.append<DocRef>(this,parent,
                            context.token->name,
                            context.context);
  children.get_last<DocRef>()->parse(cmdChar,saveCmdName);
}
 
void DocParser::handleIFile(char cmdChar,const DString &cmdName)
{
  AUTO_TRACE();
  Token tok=tokenizer.lex();
  if (!tok.is(TokenRetval::TK_WHITESPACE))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"expected whitespace after '{:c}{}' command",
      cmdChar,cmdName);
    return;
  }
  tokenizer.setStateIFile();
  tok=tokenizer.lex();
  tokenizer.setStatePara();
  if (!tok.is(TokenRetval::TK_WORD))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"unexpected token {} as the argument of '{:c}{}'",
      tok.to_string(),cmdChar,cmdName);
    return;
  }
  context.fileName = context.token->name;
}
 
void DocParser::handleILine(char cmdChar,const DString &cmdName)
{
  AUTO_TRACE();
  tokenizer.setStateILine();
  Token tok = tokenizer.lex();
  tokenizer.setStatePara();
  if (!tok.is(TokenRetval::TK_WORD))
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"invalid argument for command '{:c}{}'",
      cmdChar,cmdName);
    return;
  }
}
 
/* Helper function that deals with the most common tokens allowed in
 * title like sections.
 * @param parent     Parent node, owner of the children list passed as
 *                   the third argument.
 * @param tok        The token to process.
 * @param children   The list of child nodes to which the node representing
 *                   the token can be added.
 * @param handleWord Indicates if word token should be processed
 * @retval true      The token was handled.
 * @retval false     The token was not handled.
 */
bool DocParser::defaultHandleToken(DocNodeVariant *parent,Token &tok, DocNodeList &children,bool handleWord)
{
  AUTO_TRACE("token={} handleWord={}",tok.to_string(),handleWord);
  if (tok.is_any_of(TokenRetval::TK_WORD,TokenRetval::TK_LNKWORD,TokenRetval::TK_SYMBOL,TokenRetval::TK_URL,
                    TokenRetval::TK_COMMAND_AT,TokenRetval::TK_COMMAND_BS,TokenRetval::TK_HTMLTAG)
     )
  {
  }
reparsetoken:
  DString tokenName = context.token->name;
  AUTO_TRACE_ADD("tokenName={}",tokenName);
  switch (tok.value())
  {
    case TokenRetval::TK_COMMAND_AT:
        // fall through
    case TokenRetval::TK_COMMAND_BS:
      switch (Mappers::cmdMapper->map(tokenName))
      {
        case CommandType::CMD_BSLASH:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_BSlash);
          break;
        case CommandType::CMD_AT:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_At);
          break;
        case CommandType::CMD_LESS:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Less);
          break;
        case CommandType::CMD_GREATER:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Greater);
          break;
        case CommandType::CMD_AMP:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Amp);
          break;
        case CommandType::CMD_DOLLAR:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Dollar);
          break;
        case CommandType::CMD_HASH:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Hash);
          break;
        case CommandType::CMD_DCOLON:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_DoubleColon);
          break;
        case CommandType::CMD_PERCENT:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Percent);
          break;
        case CommandType::CMD_NDASH:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Minus);
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Minus);
          break;
        case CommandType::CMD_MDASH:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Minus);
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Minus);
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Minus);
          break;
        case CommandType::CMD_QUOTE:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Quot);
          break;
        case CommandType::CMD_PUNT:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Dot);
          break;
        case CommandType::CMD_EXCLAMATION:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Exclam);
          break;
        case CommandType::CMD_QUESTION:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Quest);
          break;
        case CommandType::CMD_PLUS:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Plus);
          break;
        case CommandType::CMD_MINUS:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Minus);
          break;
        case CommandType::CMD_EQUAL:
          children.append<DocSymbol>(this,parent,HtmlEntityMapper::Sym_Equal);
          break;
        case CommandType::CMD_EMPHASIS:
          {
            children.append<DocStyleChange>(this,parent,context.nodeStack.size(),DocStyleChange::Italic,tokenName,true);
            tok=handleStyleArgument(parent,children,tokenName);
            children.append<DocStyleChange>(this,parent,context.nodeStack.size(),DocStyleChange::Italic,tokenName,false);
            if (!tok.is(TokenRetval::TK_WORD)) children.append<DocWhiteSpace>(this,parent," ");
            if (tok.is(TokenRetval::TK_NEWPARA)) goto handlepara;
            else if (tok.is_any_of(TokenRetval::TK_WORD,TokenRetval::TK_HTMLTAG))
            {
              AUTO_TRACE_ADD("CommandType::CMD_EMPHASIS: reparsing");
              goto reparsetoken;
            }
          }
          break;
        case CommandType::CMD_BOLD:
          {
            children.append<DocStyleChange>(this,parent,context.nodeStack.size(),DocStyleChange::Bold,tokenName,true);
            tok=handleStyleArgument(parent,children,tokenName);
            children.append<DocStyleChange>(this,parent,context.nodeStack.size(),DocStyleChange::Bold,tokenName,false);
            if (!tok.is(TokenRetval::TK_WORD)) children.append<DocWhiteSpace>(this,parent," ");
            if (tok.is(TokenRetval::TK_NEWPARA)) goto handlepara;
            else if (tok.is_any_of(TokenRetval::TK_WORD,TokenRetval::TK_HTMLTAG))
            {
              AUTO_TRACE_ADD("CommandType::CMD_BOLD: reparsing");
              goto reparsetoken;
            }
          }
          break;
        case CommandType::CMD_CODE:
          {
            children.append<DocStyleChange>(this,parent,context.nodeStack.size(),DocStyleChange::Code,tokenName,true);
            tok=handleStyleArgument(parent,children,tokenName);
            children.append<DocStyleChange>(this,parent,context.nodeStack.size(),DocStyleChange::Code,tokenName,false);
            if (!tok.is(TokenRetval::TK_WORD)) children.append<DocWhiteSpace>(this,parent," ");
            if (tok.is(TokenRetval::TK_NEWPARA)) goto handlepara;
            else if (tok.is_any_of(TokenRetval::TK_WORD,TokenRetval::TK_HTMLTAG))
            {
              AUTO_TRACE_ADD("CommandType::CMD_CODE: reparsing");
              goto reparsetoken;
            }
          }
          break;
        case CommandType::CMD_HTMLONLY:
          {
            tokenizer.setStateHtmlOnly();
            tok = tokenizer.lex();
            children.append<DocVerbatim>(this,parent,context.context,context.token->verb,DocVerbatim::HtmlOnly,context.isExample,context.exampleName,context.token->name=="block");
            if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
            {
              warn_doc_error(context.fileName,tokenizer.getLineNr(),"htmlonly section ended without end marker");
            }
            tokenizer.setStatePara();
          }
          break;
        case CommandType::CMD_MANONLY:
          {
            tokenizer.setStateManOnly();
            tok = tokenizer.lex();
            children.append<DocVerbatim>(this,parent,context.context,context.token->verb,DocVerbatim::ManOnly,context.isExample,context.exampleName);
            if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
            {
              warn_doc_error(context.fileName,tokenizer.getLineNr(),"manonly section ended without end marker");
            }
            tokenizer.setStatePara();
          }
          break;
        case CommandType::CMD_RTFONLY:
          {
            tokenizer.setStateRtfOnly();
            tok = tokenizer.lex();
            children.append<DocVerbatim>(this,parent,context.context,context.token->verb,DocVerbatim::RtfOnly,context.isExample,context.exampleName);
            if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
            {
              warn_doc_error(context.fileName,tokenizer.getLineNr(),"rtfonly section ended without end marker");
            }
            tokenizer.setStatePara();
          }
          break;
        case CommandType::CMD_LATEXONLY:
          {
            tokenizer.setStateLatexOnly();
            tok = tokenizer.lex();
            children.append<DocVerbatim>(this,parent,context.context,context.token->verb,DocVerbatim::LatexOnly,context.isExample,context.exampleName);
            if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
            {
              warn_doc_error(context.fileName,tokenizer.getLineNr(),"latexonly section ended without end marker");
            }
            tokenizer.setStatePara();
          }
          break;
        case CommandType::CMD_XMLONLY:
          {
            tokenizer.setStateXmlOnly();
            tok = tokenizer.lex();
            children.append<DocVerbatim>(this,parent,context.context,context.token->verb,DocVerbatim::XmlOnly,context.isExample,context.exampleName);
            if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
            {
              warn_doc_error(context.fileName,tokenizer.getLineNr(),"xmlonly section ended without end marker");
            }
            tokenizer.setStatePara();
          }
          break;
        case CommandType::CMD_DBONLY:
          {
            tokenizer.setStateDbOnly();
            tok = tokenizer.lex();
            children.append<DocVerbatim>(this,parent,context.context,context.token->verb,DocVerbatim::DocbookOnly,context.isExample,context.exampleName);
            if (tok.is_any_of(TokenRetval::TK_NONE,TokenRetval::TK_EOF))
            {
              warn_doc_error(context.fileName,tokenizer.getLineNr(),"docbookonly section ended without end marker");
            }
            tokenizer.setStatePara();
          }
          break;
        case CommandType::CMD_FORMULA:
          {
            children.append<DocFormula>(this,parent,context.token->id);
          }
          break;
        case CommandType::CMD_ANCHOR:
        case CommandType::CMD_IANCHOR:
          {
            handleAnchor(parent,children);
          }
          break;
        case CommandType::CMD_CITE:
          {
            handleCite(parent,children);
          }
          break;
        case CommandType::CMD_IPREFIX:
          {
            handlePrefix(parent,children);
          }
          break;
        case CommandType::CMD_INTERNALREF:
          {
            handleInternalRef(parent,children);
            tokenizer.setStatePara();
          }
          break;
        case CommandType::CMD_SETSCOPE:
          {
            tokenizer.setStateSetScope();
            (void)tokenizer.lex();
            context.context = context.token->name;
            //printf("Found scope='%s'\n",qPrint(context.context));
            tokenizer.setStatePara();
          }
          break;
        case CommandType::CMD_IMAGE:
          handleImage(parent,children);
          break;
        case CommandType::CMD_ILINE:
          handleILine(tok.command_to_char(),tokenName);
          break;
        case CommandType::CMD_IFILE:
          handleIFile(tok.command_to_char(),tokenName);
          break;
        default:
          return false;
      }
      break;
    case TokenRetval::TK_HTMLTAG:
      {
        auto handleEnterLeaveStyle = [this,&parent,&children,&tokenName](DocStyleChange::Style style) {
            if (!context.token->endTag)
            {
              handleStyleEnter(parent,children,style,tokenName,&context.token->attribs);
            }
            else
            {
              handleStyleLeave(parent,children,style,tokenName);
            }
        };
        switch (Mappers::htmlTagMapper->map(tokenName))
        {
          case HtmlTagType::HTML_DIV:
            warn_doc_error(context.fileName,tokenizer.getLineNr(),"found <div> tag in heading");
            break;
          case HtmlTagType::HTML_PRE:
            warn_doc_error(context.fileName,tokenizer.getLineNr(),"found <pre> tag in heading");
            break;
          case HtmlTagType::HTML_SPAN:
            handleEnterLeaveStyle(DocStyleChange::Span);
            break;
          case HtmlTagType::HTML_BOLD:
            handleEnterLeaveStyle(DocStyleChange::Bold);
            break;
          case HtmlTagType::HTML_S:
            handleEnterLeaveStyle(DocStyleChange::S);
            break;
          case HtmlTagType::HTML_STRIKE:
            handleEnterLeaveStyle(DocStyleChange::Strike);
            break;
          case HtmlTagType::HTML_DEL:
            handleEnterLeaveStyle(DocStyleChange::Del);
            break;
          case HtmlTagType::HTML_UNDERLINE:
            handleEnterLeaveStyle(DocStyleChange::Underline);
            break;
          case HtmlTagType::HTML_INS:
            handleEnterLeaveStyle(DocStyleChange::Ins);
            break;
          case HtmlTagType::HTML_CODE:
          case HtmlTagType::XML_C:
            handleEnterLeaveStyle(DocStyleChange::Code);
            break;
          case HtmlTagType::HTML_KBD:
            handleEnterLeaveStyle(DocStyleChange::Kbd);
            break;
          case HtmlTagType::HTML_TT:
            handleEnterLeaveStyle(DocStyleChange::Typewriter);
            break;
          case HtmlTagType::HTML_EMPHASIS:
            handleEnterLeaveStyle(DocStyleChange::Italic);
            break;
          case HtmlTagType::HTML_SUB:
            handleEnterLeaveStyle(DocStyleChange::Subscript);
            break;
          case HtmlTagType::HTML_SUP:
            handleEnterLeaveStyle(DocStyleChange::Superscript);
            break;
          case HtmlTagType::HTML_CENTER:
            handleEnterLeaveStyle(DocStyleChange::Center);
            break;
          case HtmlTagType::HTML_SMALL:
            handleEnterLeaveStyle(DocStyleChange::Small);
            break;
          case HtmlTagType::HTML_CITE:
            handleEnterLeaveStyle(DocStyleChange::Cite);
            break;
          case HtmlTagType::HTML_IMG:
            if (!context.token->endTag)
            {
              handleImg(parent,children,context.token->attribs);
            }
            break;
          default:
            return false;
            break;
        }
      }
      break;
    case TokenRetval::TK_SYMBOL:
      {
        HtmlEntityMapper::SymType s = DocSymbol::decodeSymbol(tokenName);
        if (s!=HtmlEntityMapper::Sym_Unknown)
        {
          children.append<DocSymbol>(this,parent,s);
        }
        else
        {
          return false;
        }
      }
      break;
    case TokenRetval::TK_WHITESPACE:
    case TokenRetval::TK_NEWPARA:
handlepara:
      if (insidePRE(parent) || !children.empty())
      {
        children.append<DocWhiteSpace>(this,parent,context.token->chars);
      }
      break;
    case TokenRetval::TK_LNKWORD:
      if (handleWord)
      {
        handleLinkedWord(parent,children);
      }
      else
        return false;
      break;
    case TokenRetval::TK_WORD:
      if (handleWord)
      {
        children.append<DocWord>(this,parent,context.token->name);
      }
      else
        return false;
      break;
    case TokenRetval::TK_URL:
      if (context.insideHtmlLink)
      {
        children.append<DocWord>(this,parent,context.token->name);
      }
      else
      {
        children.append<DocURL>(this,parent,context.token->name,context.token->isEMailAddr);
      }
      break;
    default:
      return false;
  }
  return true;
}
 
//---------------------------------------------------------------------------
 
void DocParser::handleImg(DocNodeVariant *parent, DocNodeList &children,const HtmlAttribList &tagHtmlAttribs)
{
  AUTO_TRACE();
  bool found=false;
  size_t index=0;
  for (const auto &opt : tagHtmlAttribs)
  {
    AUTO_TRACE_ADD("option name={} value='{}'",opt.name,opt.value);
    if (opt.name=="src" && !opt.value.empty())
    {
      // copy attributes
      HtmlAttribList attrList = tagHtmlAttribs;
      // and remove the src attribute
      attrList.erase(attrList.begin()+index);
      DocImage::Type t = DocImage::Html;
      children.append<DocImage>(
            this,parent,attrList,
            findAndCopyImage(opt.value,t,false),
            t,opt.value);
      found = true;
    }
    ++index;
  }
  if (!found)
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"IMG tag does not have a SRC attribute!");
  }
}
 
//---------------------------------------------------------------------------
 
Token DocParser::internalValidatingParseDoc(DocNodeVariant *parent,DocNodeList &children,
                                    const DString &doc)
{
  AUTO_TRACE();
  Token retval = Token::make_RetVal_OK();
 
  if (doc.empty()) return retval;
 
  tokenizer.init(doc.data(),context.fileName,context.markdownSupport,context.insideHtmlLink);
 
  // first parse any number of paragraphs
  bool isFirst=true;
  DocPara *lastPar=!children.empty() ? std::get_if<DocPara>(&children.back()): nullptr;
  if (lastPar)
  { // last child item was a paragraph
    isFirst=false;
  }
  do
  {
    children.append<DocPara>(this,parent);
    DocPara *par  = children.get_last<DocPara>();
    if (isFirst) { par->markFirst(); isFirst=false; }
    retval=par->parse();
    if (!par->empty())
    {
      if (lastPar) lastPar->markLast(false);
      lastPar=par;
    }
    else
    {
      children.pop_back();
    }
  } while (retval.is(TokenRetval::TK_NEWPARA));
  if (lastPar) lastPar->markLast();
 
  AUTO_TRACE_EXIT("isFirst={} isLast={}",lastPar?lastPar->isFirst():-1,lastPar?lastPar->isLast():-1);
  return retval;
}
 
//---------------------------------------------------------------------------
 
void DocParser::readTextFileByName(const DString &file,DString &text)
{
  AUTO_TRACE("file={} text={}",file,text);
  bool ambig = false;
  DString filePath = findExampleFilePath(file,ambig);
  if (!filePath.empty())
  {
    size_t indent = 0;
    text = detab(fileToString(filePath,Config_getBool(FILTER_SOURCE_FILES)),indent);
    if (ambig)
    {
      warn_doc_error(context.fileName,tokenizer.getLineNr(),"included file name '{}' is ambiguous"
          "Possible candidates:\n{}",file, Doxygen::exampleNameLinkedMap->showFileDefMatches(file));
    }
  }
  else
  {
    warn_doc_error(context.fileName,tokenizer.getLineNr(),"included file '{}' is not found. "
           "Check your EXAMPLE_PATH",file);
  }
}
 
//---------------------------------------------------------------------------
 
static DString extractCopyDocId(const char *data, size_t &j, size_t len)
{
  size_t s=j;
  int round=0;
  bool insideDQuote=false;
  bool insideSQuote=false;
  bool found=false;
  while (j<len && !found)
  {
    if (!insideSQuote && !insideDQuote)
    {
      switch (data[j])
      {
        case '(': round++; break;
        case ')': round--; break;
        case '"': insideDQuote=true; break;
        case '\'': insideSQuote=true; break;
        case '\\': // fall through, begin of command
        case '@':  // fall through, begin of command
        case '\t': // fall through
        case '\n':
          found=(round==0);
          break;
        case ' ':  // allow spaces in cast operator (see issue #11169)
          found=(round==0) && (j<8 || !literal_at(data+j-8,"operator"));
          break;
      }
    }
    else if (insideSQuote) // look for single quote end
    {
      if (data[j]=='\'' && (j==0 || data[j]!='\\'))
      {
        insideSQuote=false;
      }
    }
    else if (insideDQuote) // look for double quote end
    {
      if (data[j]=='"' && (j==0 || data[j]!='\\'))
      {
        insideDQuote=false;
      }
    }
    if (!found) j++;
  }
 
  // include const and volatile
  if (literal_at(data+j," const"))
  {
    j+=6;
  }
  else if (literal_at(data+j," volatile"))
  {
    j+=9;
  }
 
  // allow '&' or '&&' or ' &' or ' &&' at the end
  size_t k=j;
  while (k<len && data[k]==' ') k++;
  if      (k<len-1 && data[k]=='&' && data[k+1]=='&') j=k+2;
  else if (k<len   && data[k]=='&'                  ) j=k+1;
 
  // do not include punctuation added by Definition::_setBriefDescription()
  size_t e=j;
  if (j>0 && data[j-1]=='.') { e--; }
  DString id(data+s,e-s);
  //printf("extractCopyDocId='%s' input='%s'\n",qPrint(id),&data[s]);
  return id;
}
 
// macro to check if the input starts with a specific command.
// note that data[i] should point to the start of the command (\ or @ character)
// and the sizeof(str) returns the size of str including the '\0' terminator;
// a fact we abuse to skip over the start of the command character.
#define CHECK_FOR_COMMAND(str,action) \
   do if ((i+sizeof(str)<len) && literal_at(data+i+1,str)) \
   { j=i+sizeof(str); action; } while(0)
 
static size_t isCopyBriefOrDetailsCmd(const char *data, size_t i,size_t len,bool &brief)
{
  size_t j=0;
  if (i==0 || (data[i-1]!='@' && data[i-1]!='\\')) // not an escaped command
  {
    CHECK_FOR_COMMAND("copybrief",brief=true);    // @copybrief or \copybrief
    CHECK_FOR_COMMAND("copydetails",brief=false); // @copydetails or \copydetails
  }
  return j;
}
 
static size_t isVerbatimSection(const char *data,size_t i,size_t len,DString &endMarker)
{
  size_t j=0;
  if (i==0 || (data[i-1]!='@' && data[i-1]!='\\')) // not an escaped command
  {
    CHECK_FOR_COMMAND("dot",endMarker="enddot");
    CHECK_FOR_COMMAND("icode",endMarker="endicode");
    CHECK_FOR_COMMAND("code",endMarker="endcode");
    CHECK_FOR_COMMAND("msc",endMarker="endmsc");
    CHECK_FOR_COMMAND("mermaid",endMarker="endmermaid");
    CHECK_FOR_COMMAND("iverbatim",endMarker="endiverbatim");
    CHECK_FOR_COMMAND("verbatim",endMarker="endverbatim");
    CHECK_FOR_COMMAND("iliteral",endMarker="endiliteral");
    CHECK_FOR_COMMAND("latexonly",endMarker="endlatexonly");
    CHECK_FOR_COMMAND("htmlonly",endMarker="endhtmlonly");
    CHECK_FOR_COMMAND("xmlonly",endMarker="endxmlonly");
    CHECK_FOR_COMMAND("rtfonly",endMarker="endrtfonly");
    CHECK_FOR_COMMAND("manonly",endMarker="endmanonly");
    CHECK_FOR_COMMAND("docbookonly",endMarker="enddocbookonly");
    CHECK_FOR_COMMAND("startuml",endMarker="enduml");
  }
  //printf("isVerbatimSection(%s)=%d)\n",qPrint(DString(&data[i]).left(10)),j);
  return j;
}
 
static size_t skipToEndMarker(const char *data,size_t i,size_t len,const DString &endMarker)
{
  while (i<len)
  {
    if ((data[i]=='@' || data[i]=='\\') &&  // start of command character
        (i==0 || (data[i-1]!='@' && data[i-1]!='\\'))) // that is not escaped
    {
      if (i+endMarker.length()+1<=len && dstrncmp(data+i+1,endMarker.data(),endMarker.length())==0)
      {
        return i+endMarker.length()+1;
      }
    }
    i++;
  }
  // oops no endmarker found...
  return i<len ? i+1 : len;
}
 
 
DString DocParser::processCopyDoc(const char *data,size_t &len)
{
  AUTO_TRACE("data={} len={}",Trace::trunc(data),len);
  DString result;
  result.reserve(len+32);
  size_t i=0;
  int lineNr = tokenizer.getLineNr();
  while (i<len)
  {
    char c = data[i];
    if (c=='@' || c=='\\') // look for a command
    {
      bool isBrief=true;
      size_t j=isCopyBriefOrDetailsCmd(data,i,len,isBrief);
      if (j>0)
      {
        // skip whitespace
        while (j<len && (data[j]==' ' || data[j]=='\t')) j++;
        // extract the argument
        DString id = extractCopyDocId(data,j,len);
        const Definition *def = nullptr;
        DString doc,brief;
        //printf("resolving docs='%s'\n",qPrint(id));
        bool found = findDocsForMemberOrCompound(id,&doc,&brief,&def);
        if (found && def->isReference())
        {
          warn_doc_error(context.fileName,tokenizer.getLineNr(),
               "@copy{} or @copydoc target '{}' found but is from a tag file, skipped",
               isBrief?"brief":"details", id);
        }
        else if (found)
        {
          //printf("found it def=%p brief='%s' doc='%s' isBrief=%d\n",def,qPrint(brief),qPrint(doc),isBrief);
          auto it = std::find(context.copyStack.begin(),context.copyStack.end(),def);
          if (it==context.copyStack.end()) // definition not parsed earlier
          {
            DString orgFileName = context.fileName;
            context.copyStack.push_back(def);
            auto addDocs = [&](const DString &file_,int line_,const DString &doc_)
            {
              result+=" \\ifile \""+file_+"\" ";
              result+="\\iline "+DString().setNum(line_)+" \\ilinebr ";
              size_t len_ = doc_.length();
              result+=processCopyDoc(doc_.data(),len_);
            };
            if (isBrief)
            {
              addDocs(def->briefFile(),def->briefLine(),brief);
            }
            else
            {
              addDocs(def->docFile(),def->docLine(),doc);
              if (def->definitionType()==Definition::TypeMember)
              {
                const MemberDef *md = toMemberDef(def);
                const ArgumentList &docArgList = md->templateMaster() ?
                    md->templateMaster()->argumentList() :
                    md->argumentList();
                result+=inlineArgListToDoc(docArgList);
              }
            }
            context.copyStack.pop_back();
            result+=" \\ilinebr \\ifile \""+context.fileName+"\" ";
            result+="\\iline "+DString().setNum(lineNr)+" ";
          }
          else
          {
            warn_doc_error(context.fileName,tokenizer.getLineNr(),
                 "Found recursive @copy{} or @copydoc relation for argument '{}'.",
                 isBrief?"brief":"details",id);
          }
        }
        else
        {
          warn_doc_error(context.fileName,tokenizer.getLineNr(),
               "@copy{} or @copydoc target '{}' not found", isBrief?"brief":"details",id);
        }
        // skip over command
        i=j;
      }
      else
      {
        DString endMarker;
        size_t k = isVerbatimSection(data,i,len,endMarker);
        if (k>0)
        {
          size_t orgPos = i;
          i=skipToEndMarker(data,k,len,endMarker);
          result+=DString(data+orgPos,i-orgPos);
          // TODO: adjust lineNr
        }
        else
        {
          result+=c;
          i++;
        }
      }
    }
    else // not a command, just copy
    {
      result+=c;
      i++;
      lineNr += (c=='\n') ? 1 : 0;
    }
  }
  len = result.length();
  AUTO_TRACE_EXIT("result={}",Trace::trunc(result));
  return result;
}
 
 
//---------------------------------------------------------------------------
 
IDocNodeASTPtr validatingParseDoc(IDocParser &parserIntf,
                                  const DString &fileName,
                                  int startLine,
                                  const Definition *ctx,
                                  const MemberDef *md,
                                  const DString &input,
                                  const DocOptions &options)
{
  DocParser *parser = dynamic_cast<DocParser*>(&parserIntf);
  ASSERT(parser!=nullptr);
  if (parser==nullptr) return nullptr;
  //printf("validatingParseDoc(%s,%s)=[%s]\n",ctx?qPrint(ctx->name()):"<none>",
  //                                     md?qPrint(md->name()):"<none>",
  //                                     qPrint(input));
  //printf("========== validating %s at line %d\n",qPrint(fileName),startLine);
  //printf("---------------- input --------------------\n%s\n----------- end input -------------------\n",qPrint(input));
 
  // set initial token
  parser->context.token = parser->tokenizer.resetToken();
 
  if (ctx && ctx!=Doxygen::globalScope &&
      (ctx->definitionType()==Definition::TypeClass ||
       ctx->definitionType()==Definition::TypeNamespace
      )
     )
  {
    parser->context.context = substitute(ctx->qualifiedName(),getLanguageSpecificSeparator(ctx->getLanguage(),true),"::");
  }
  else if (ctx && ctx->definitionType()==Definition::TypePage)
  {
    const Definition *scope = (toPageDef(ctx))->getPageScope();
    if (scope && scope!=Doxygen::globalScope)
    {
      parser->context.context = substitute(scope->name(),getLanguageSpecificSeparator(scope->getLanguage(),true),"::");
    }
  }
  else if (ctx && ctx->definitionType()==Definition::TypeGroup)
  {
    const Definition *scope = (toGroupDef(ctx))->getGroupScope();
    if (scope && scope!=Doxygen::globalScope)
    {
      parser->context.context = substitute(scope->name(),getLanguageSpecificSeparator(scope->getLanguage(),true),"::");
    }
  }
  else
  {
    parser->context.context = "";
  }
  parser->context.scope = ctx;
  parser->context.lang = getLanguageFromFileName(fileName);
 
  if (options.indexWords() && Doxygen::searchIndex.enabled())
  {
    if (md)
    {
      parser->context.searchUrl=md->getOutputFileBase();
      Doxygen::searchIndex.setCurrentDoc(md,md->anchor(),false);
    }
    else if (ctx)
    {
      parser->context.searchUrl=ctx->getOutputFileBase();
      Doxygen::searchIndex.setCurrentDoc(ctx,ctx->anchor(),false);
    }
  }
  else
  {
    parser->context.searchUrl="";
  }
 
  parser->context.fileName = fileName;
  parser->context.relPath = (!options.linkFromIndex() && ctx) ?
               relativePathToRoot(ctx->getOutputFileBase()) :
               DString("");
  //printf("ctx->name=%s relPath=%s\n",qPrint(ctx->name()),qPrint(parser->context.relPath));
  parser->context.memberDef = md;
  while (!parser->context.nodeStack.empty()) parser->context.nodeStack.pop();
  while (!parser->context.styleStack.empty()) parser->context.styleStack.pop();
  while (!parser->context.initialStyleStack.empty()) parser->context.initialStyleStack.pop();
  parser->context.inSeeBlock = false;
  parser->context.inCodeStyle = false;
  parser->context.xmlComment = false;
  parser->context.insideHtmlLink = false;
  parser->context.includeFileText = "";
  parser->context.includeFileOffset = 0;
  parser->context.includeFileLength = 0;
  parser->context.isExample = options.isExample();
  parser->context.exampleName = options.exampleName();
  parser->context.hasParamCommand = false;
  parser->context.hasReturnCommand = false;
  parser->context.retvalsFound.clear();
  parser->context.paramsFound.clear();
  parser->context.markdownSupport = options.markdownSupport();
  parser->context.autolinkSupport = options.autolinkSupport();
  if (md)
  {
    const ArgumentList &al=md->isDocsForDefinition() ? md->argumentList() : md->declArgumentList();
    parser->context.numParameters = static_cast<int>(al.size());
  }
  else
  {
    parser->context.numParameters = 0;
  }
  parser->context.paramPosition = 1;
 
  //printf("Starting comment block at %s:%d\n",qPrint(parser->context.fileName),startLine);
  parser->tokenizer.setFileName(fileName);
  parser->tokenizer.setLineNr(startLine);
  size_t ioLen = input.length();
  DString inpStr = parser->processCopyDoc(input.data(),ioLen);
  if (inpStr.empty() || inpStr.at(inpStr.length()-1)!='\n')
  {
    inpStr+='\n';
  }
  //printf("processCopyDoc(in='%s' out='%s')\n",qPrint(input),qPrint(inpStr));
  parser->tokenizer.init(inpStr.data(),parser->context.fileName,
                         parser->context.markdownSupport,parser->context.insideHtmlLink);
 
  // build abstract syntax tree
  auto ast = std::make_unique<DocNodeAST>(DocRoot(parser,md!=nullptr,options.singleLine()));
  std::get<DocRoot>(ast->root).parse();
 
  if (Debug::isFlagSet(Debug::PrintTree))
  {
    // pretty print the result
    std::visit(PrintDocVisitor{},ast->root);
  }
 
  if (md && md->isFunction())
  {
    parser->checkUnOrMultipleDocumentedParams();
  }
  if (parser->context.memberDef) parser->context.memberDef->detectUndocumentedParams(parser->context.hasParamCommand,parser->context.hasReturnCommand);
 
  // reset token
  parser->tokenizer.resetToken();
 
  //printf(">>>>>> end validatingParseDoc(%s,%s)\n",ctx?qPrint(ctx->name()):"<none>",
  //                                     md?qPrint(md->name()):"<none>");
 
  return ast;
}
 
IDocNodeASTPtr validatingParseTitle(IDocParser &parserIntf,const DString &fileName,int lineNr,const DString &input)
{
  DocParser *parser = dynamic_cast<DocParser*>(&parserIntf);
  ASSERT(parser!=nullptr);
  if (parser==nullptr) return nullptr;
 
  // set initial token
  parser->context.token = parser->tokenizer.resetToken();
 
  //printf("------------ input ---------\n%s\n"
  //       "------------ end input -----\n",input);
  parser->context.context = "";
  parser->context.fileName = fileName;
  parser->context.relPath = "";
  parser->context.memberDef = nullptr;
  while (!parser->context.nodeStack.empty()) parser->context.nodeStack.pop();
  while (!parser->context.styleStack.empty()) parser->context.styleStack.pop();
  while (!parser->context.initialStyleStack.empty()) parser->context.initialStyleStack.pop();
  parser->context.inSeeBlock = false;
  parser->context.inCodeStyle = false;
  parser->context.xmlComment = false;
  parser->context.insideHtmlLink = false;
  parser->context.includeFileText = "";
  parser->context.includeFileOffset = 0;
  parser->context.includeFileLength = 0;
  parser->context.isExample = false;
  parser->context.exampleName = "";
  parser->context.hasParamCommand = false;
  parser->context.hasReturnCommand = false;
  parser->context.retvalsFound.clear();
  parser->context.paramsFound.clear();
  parser->context.searchUrl="";
  parser->context.lang = SrcLangExt::Unknown;
  parser->context.markdownSupport = Config_getBool(MARKDOWN_SUPPORT);
  parser->context.autolinkSupport = false;
 
  auto ast = std::make_unique<DocNodeAST>(DocTitle(parser,nullptr));
 
  if (!input.empty())
  {
    // build abstract syntax tree from title string
    std::get<DocTitle>(ast->root).parseFromString(nullptr,input);
 
    if (Debug::isFlagSet(Debug::PrintTree))
    {
      // pretty print the result
      std::visit(PrintDocVisitor{},ast->root);
    }
  }
 
  return ast;
}
 
IDocNodeASTPtr validatingParseText(IDocParser &parserIntf,const DString &input)
{
  DocParser *parser = dynamic_cast<DocParser*>(&parserIntf);
  ASSERT(parser!=nullptr);
  if (parser==nullptr) return nullptr;
 
  // set initial token
  parser->context.token = parser->tokenizer.resetToken();
 
  //printf("------------ input ---------\n%s\n"
  //       "------------ end input -----\n",input);
  //parser->context.token = new TokenInfo;
  parser->context.context = "";
  parser->context.fileName = "<parseText>";
  parser->context.relPath = "";
  parser->context.memberDef = nullptr;
  while (!parser->context.nodeStack.empty()) parser->context.nodeStack.pop();
  while (!parser->context.styleStack.empty()) parser->context.styleStack.pop();
  while (!parser->context.initialStyleStack.empty()) parser->context.initialStyleStack.pop();
  parser->context.inSeeBlock = false;
  parser->context.inCodeStyle = false;
  parser->context.xmlComment = false;
  parser->context.insideHtmlLink = false;
  parser->context.includeFileText = "";
  parser->context.includeFileOffset = 0;
  parser->context.includeFileLength = 0;
  parser->context.isExample = false;
  parser->context.exampleName = "";
  parser->context.hasParamCommand = false;
  parser->context.hasReturnCommand = false;
  parser->context.retvalsFound.clear();
  parser->context.paramsFound.clear();
  parser->context.searchUrl="";
  parser->context.lang = SrcLangExt::Unknown;
  parser->context.markdownSupport = Config_getBool(MARKDOWN_SUPPORT);
  parser->context.autolinkSupport = false;
 
 
  auto ast = std::make_unique<DocNodeAST>(DocText(parser));
 
  if (!input.empty())
  {
    parser->tokenizer.setLineNr(1);
    parser->tokenizer.init(input.data(),parser->context.fileName,
                           parser->context.markdownSupport,parser->context.insideHtmlLink);
 
    // build abstract syntax tree
    std::get<DocText>(ast->root).parse();
 
    if (Debug::isFlagSet(Debug::PrintTree))
    {
      // pretty print the result
      std::visit(PrintDocVisitor{},ast->root);
    }
  }
 
  return ast;
}
 
IDocNodeASTPtr createRef(IDocParser &parserIntf,const DString &target,const DString &context, const DString &srcFile, int srcLine )
{
  DocParser *parser = dynamic_cast<DocParser*>(&parserIntf);
  ASSERT(parser!=nullptr);
  if (parser==nullptr) return nullptr;
  if (!srcFile.empty())
  {
    parser->context.fileName = srcFile;
    parser->tokenizer.setFileName(srcFile);
    parser->tokenizer.setLineNr(srcLine);
  }
  return std::make_unique<DocNodeAST>(DocRef(parser,nullptr,target,context));
}
 
void docFindSections(const DString &input,
                     const Definition *d,
                     const DString &fileName)
{
  DocParser parser;
  parser.tokenizer.findSections(input,d,fileName);
}
 
//--------------------------------------------------------------------------------------
 
static int nextUTF8CharPosition(const DString &utf8Str,uint32_t len,uint32_t startPos)
{
  if (startPos>=len) return len;
  uint8_t c = static_cast<uint8_t>(utf8Str[startPos]);
  int bytes=getUTF8CharNumBytes(c);
  if (c=='&') // skip over character entities
  {
    bytes=1;
    int (*matcher)(int) = nullptr;
    c = static_cast<uint8_t>(utf8Str[startPos+bytes]);
    if (c=='#') // numerical entity?
    {
      bytes++;
      c = static_cast<uint8_t>(utf8Str[startPos+bytes]);
      if (c=='x') // hexadecimal entity?
      {
        bytes++;
        matcher = std::isxdigit;
      }
      else // decimal entity
      {
        matcher = std::isdigit;
      }
    }
    else if (std::isalnum(c)) // named entity?
    {
      bytes++;
      matcher = std::isalnum;
    }
    if (matcher)
    {
      while ((c = static_cast<uint8_t>(utf8Str[startPos+bytes]))!=0 && matcher(c))
      {
        bytes++;
      }
    }
    if (c!=';')
    {
      bytes=1; // not a valid entity, reset bytes counter
    }
  }
  return startPos+bytes;
}
 
 
DString parseCommentAsText(const Definition *scope,const MemberDef *md,
                           const DString &doc,const DString &fileName,int lineNr)
{
  if (doc.empty()) return "";
  static std::mutex                              s_docCacheMutex;
  static std::unordered_map<std::string,DString> s_docCache;
 
  std::lock_guard lock(s_docCacheMutex);
  auto it = s_docCache.find(doc.str());
  if (it != s_docCache.end())
  {
    //printf("Cache: [%s]->[%s]\n",qPrint(doc),qPrint(it->second));
    return it->second;
  }
  //printf("parseCommentAsText(%s)\n",qPrint(doc));
  TextStream t;
  auto parser { createDocParser() };
  auto ast    { validatingParseDoc(*parser.get(),
                                   fileName,
                                   lineNr,
                                   scope,
                                   md,
                                   doc,
                                   DocOptions()
                                   .setAutolinkSupport(false))
              };
  auto astImpl = dynamic_cast<const DocNodeAST*>(ast.get());
  if (astImpl)
  {
    TextDocVisitor visitor(t);
    std::visit(visitor,astImpl->root);
  }
  DString result = HtmlEntityMapper::instance().convertCharEntitiesToUTF8(t.str()).stripWhiteSpace();
  int i=0;
  int charCnt=0;
  int l=static_cast<int>(result.length());
  while ((i=nextUTF8CharPosition(result,l,i))<l)
  {
    charCnt++;
    if (charCnt>=80) break;
  }
  if (charCnt>=80) // try to truncate the string
  {
    while ((i=nextUTF8CharPosition(result,l,i))<l && charCnt<100)
    {
      charCnt++;
      if (result.at(i)==',' ||
          result.at(i)=='.' ||
          result.at(i)=='!' ||
          result.at(i)=='?' ||
          result.at(i)=='}')    // good for UTF-16 characters and } otherwise also a good point to stop the string
      {
        i++; // we want to be "behind" last inspected character
        break;
      }
    }
  }
  if ( i < l) result=result.left(i)+"...";
  s_docCache.insert(std::make_pair(doc.str(),result));
  return result.data();
}
 
//--------------------------------------------------------------------------------------
 
DString parseCommentAsHtml(const Definition *scope,const MemberDef *member,
                           const DString &doc,const DString &fileName,int lineNr)
{
  static std::mutex                              s_docCacheMutex;
  static std::unordered_map<std::string,DString> s_docCache;
 
  std::lock_guard lock(s_docCacheMutex);
  auto it = s_docCache.find(doc.str());
  if (it != s_docCache.end())
  {
    //printf("Cache: [%s]->[%s]\n",qPrint(doc),qPrint(it->second));
    return it->second;
  }
  auto parser { createDocParser() };
  auto ast    { validatingParseTitle(*parser.get(),fileName,lineNr,doc) };
  auto astImpl = dynamic_cast<const DocNodeAST*>(ast.get());
  DString result;
  if (astImpl)
  {
    TextStream t;
    OutputCodeList codeList;
    codeList.add<HtmlCodeGenerator>(&t);
    HtmlDocVisitor visitor(t,codeList,scope,fileName);
    std::visit(visitor,astImpl->root);
    result = t.str();
  }
  else // fallback, should not happen
  {
    result = filterTitle(doc);
  }
  //printf("Conversion: [%s]->[%s]\n",qPrint(doc),qPrint(result));
  s_docCache.insert(std::make_pair(doc.str(),result));
  return result;
}