Doxygen
Loading...
Searching...
No Matches
doctokenizer.l File Reference
#include <stdint.h>
#include "doctokenizer.h"
#include <cctype>
#include <stack>
#include <string>
#include "cmdmapper.h"
#include "config.h"
#include "debug.h"
#include "definition.h"
#include "docnode.h"
#include "message.h"
#include "portable.h"
#include "regex.h"
#include "section.h"
#include "stringutil.h"
#include "doxygen_lex.h"
#include "doctokenizer.l.h"
Include dependency graph for doctokenizer.l:

Go to the source code of this file.

Classes

struct  DocLexerContext
struct  doctokenizerYY_state
struct  DocTokenizer::Private

Macros

#define YY_TYPEDEF_YY_SCANNER_T
#define YY_NO_INPUT   1
#define YY_NO_UNISTD_H   1
#define lineCount(s, len)
#define unput_string(yytext, yyleng)
#define YY_INPUT(buf, result, max_size)
#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)
#define yyterminate()

Typedefs

typedef yyguts_t * yyscan_t

Functions

static const char * stateToString (int state)
static int yyread (yyscan_t yyscanner, char *buf, int max_size)
static void handleHtmlTag (yyscan_t yyscanner, const char *text)
static void processSection (yyscan_t yyscanner)
DString extractPartAfterNewLine (const DString &text)
static int computeIndent (const char *str, size_t length)
static const char * getLexerFILE ()
int yylex (yyscan_t yyscanner)

Macro Definition Documentation

◆ lineCount

#define lineCount ( s,
len )
Value:
do { for(int i=0;i<(int)len;i++) if (s[i]=='\n') yyextra->yyLineNr++; } while(0)

Definition at line 98 of file doctokenizer.l.

Referenced by endBrief(), initMethodProtection(), and CitationManager::insertCrossReferencesForBibFile().

◆ unput_string

#define unput_string ( yytext,
yyleng )
Value:
do { for (int i=(int)yyleng-1;i>=0;i--) unput(yytext[i]); } while(0)

Definition at line 160 of file doctokenizer.l.

◆ YY_DECL

#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)

Definition at line 171 of file doctokenizer.l.

◆ YY_INPUT

#define YY_INPUT ( buf,
result,
max_size )
Value:
result=yyread(yyscanner,buf,max_size);
yyread
static int yyread(yyscan_t yyscanner, char *buf, int max_size)
Definition doctokenizer.l:1648

Definition at line 164 of file doctokenizer.l.

◆ YY_NO_INPUT

#define YY_NO_INPUT   1

Definition at line 53 of file doctokenizer.l.

◆ YY_NO_UNISTD_H

#define YY_NO_UNISTD_H   1

Definition at line 54 of file doctokenizer.l.

◆ YY_TYPEDEF_YY_SCANNER_T

#define YY_TYPEDEF_YY_SCANNER_T

Definition at line 26 of file doctokenizer.l.

◆ yyterminate

#define yyterminate ( )
Value:
return Token::make_TK_EOF()

Definition at line 174 of file doctokenizer.l.

Typedef Documentation

◆ yyscan_t

typedef yyguts_t* yyscan_t

Definition at line 28 of file doctokenizer.l.

Function Documentation

◆ computeIndent()

int computeIndent ( const char * str,
size_t length )
static

Definition at line 127 of file doctokenizer.l.

128{
129 if (str==0 || length==std::string::npos) return 0;
130 size_t i;
131 int indent=0;
132 int tabSize=Config_getInt(TAB_SIZE);
133 for (i=0;i<length;i++)
134 {
135 if (str[i]=='\t')
136 {
137 indent+=tabSize - (indent%tabSize);
138 }
139 else if (str[i]=='\n')
140 {
141 indent=0;
142 }
143 else if (literal_at(&str[i],"\\ilinebr"))
144 {
145 indent=0;
146 i+=7;
147 if (str[i+1]==' ') i++; // also eat space after \\ilinebr if present
148 }
149 else
150 {
151 indent++;
152 }
153 }
154 //printf("input('%s')=%d\n",str,indent);
155 return indent;
Config_getInt
#define Config_getInt(name)
Definition config.h:34
literal_at
bool literal_at(const char *data, const char(&str)[N])
returns true iff data points to a substring that matches string literal str
Definition stringutil.h:101
156}

References Config_getInt, and literal_at().

◆ extractPartAfterNewLine()

DString extractPartAfterNewLine ( const DString & text)

Definition at line 109 of file doctokenizer.l.

110{
111 size_t nl1 = text.find('\n');
112 size_t nl2 = text.find("\\ilinebr");
113 if (nl1!=DString::npos && nl2!=DString::npos && nl1<nl2)
114 {
115 return text.mid(nl1+1);
116 }
117 if (nl2!=DString::npos)
118 {
119 if (text.at(nl2+8)==' ') nl2++; // skip space after \\ilinebr
120 return text.mid(nl2+8);
121 }
122 return text;
DString::mid
DString mid(size_t index, size_t len=npos) const
Definition dstring.h:318
DString::npos
static constexpr size_t npos
value used to indicate 'not found' or 'to the end of the string', matching std::string::npos
Definition dstring.h:178
DString::at
char & at(size_t i)
Returns a reference to the character at index i.
Definition dstring.h:686
DString::find
size_t find(char c, size_t pos=0) const
Definition dstring.h:239
123}

References DString::at(), DString::find(), DString::mid(), and DString::npos.

◆ getLexerFILE()

const char * getLexerFILE ( )
inlinestatic

Definition at line 167 of file doctokenizer.l.

167{return __FILE__;}

◆ handleHtmlTag()

void handleHtmlTag ( yyscan_t yyscanner,
const char * text )
static

Definition at line 1679 of file doctokenizer.l.

1680{
1681 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1682
1683 DString tagText(text);
1684 yyextra->token.text = tagText;
1685 yyextra->token.attribs.clear();
1686 yyextra->token.endTag = false;
1687 yyextra->token.emptyTag = false;
DString
A String class for use with Doxygen wrapping std::string and adding some additional functionality off...
Definition dstring.h:84
1688
1689 // Check for end tag
1690 int startNamePos=1;
1691 if (tagText.at(1)=='/')
1692 {
1693 yyextra->token.endTag = true;
1694 startNamePos++;
1695 }
1696
1697 // Parse the name portion
1698 int i = startNamePos;
1699 for (i=startNamePos; i < (int)yyleng; i++)
1700 {
1701 // Check for valid HTML/XML name chars (including namespaces)
1702 char c = tagText.at(i);
1703 if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
1704 }
1705 yyextra->token.name = tagText.mid(startNamePos,i-startNamePos);
1706
1707 // Parse the attributes. Each attribute is a name, value pair
1708 // The result is stored in yyextra->token.attribs.
1709 int startAttribList = i;
1710 while (i<(int)yyleng)
1711 {
1712 char c=tagText.at(i);
1713 // skip spaces
1714 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1715 // check for end of the tag
1716 if (c == '>') break;
1717 // Check for XML style "empty" tag.
1718 if (c == '/')
1719 {
1720 yyextra->token.emptyTag = true;
1721 break;
1722 }
1723 int startName=i;
1724 // search for end of name
1725 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='=' && c!= '>') { c=tagText.at(++i); }
1726 int endName=i;
1727 DString optName,optValue;
1728 optName = tagText.mid(startName,endName-startName).lower();
1729 // skip spaces
1730 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1731 if (tagText.at(i)=='=') // option has value
1732 {
1733 int startAttrib=0, endAttrib=0;
1734 c=tagText.at(++i);
1735 // skip spaces
1736 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1737 if (tagText.at(i)=='\'') // option '...'
1738 {
1739 c=tagText.at(++i);
1740 startAttrib=i;
DString::lower
DString lower() const
Definition dstring.h:326
1741
1742 // search for matching quote
1743 while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
1744 endAttrib=i;
1745 if (i<(int)yyleng) { c=tagText.at(++i);}
1746 }
1747 else if (tagText.at(i)=='"') // option "..."
1748 {
1749 c=tagText.at(++i);
1750 startAttrib=i;
1751 // search for matching quote
1752 while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
1753 endAttrib=i;
1754 if (i<(int)yyleng) { c=tagText.at(++i);}
1755 }
1756 else // value without any quotes
1757 {
1758 startAttrib=i;
1759 // search for separator or end symbol
1760 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='>') { c=tagText.at(++i); }
1761 endAttrib=i;
1762 if (i<(int)yyleng) { c=tagText.at(++i);}
1763 }
1764 optValue = tagText.mid(startAttrib,endAttrib-startAttrib);
1765 if (optName == "align") optValue = optValue.lower();
1766 else if (optName == "valign")
1767 {
1768 optValue = optValue.lower();
1769 if (optValue == "center") optValue="middle";
1770 }
1771 }
1772 else // start next option
1773 {
1774 }
1775 //printf("=====> Adding option name=<%s> value=<%s>\n",
1776 // qPrint(optName),qPrint(optValue));
1777 yyextra->token.attribs.emplace_back(optName,optValue);
1778 }
1779 yyextra->token.attribsStr = tagText.mid(startAttribList,i-startAttribList);
1780}

References DString::at(), DString::clear(), DString::lower(), and DString::mid().

◆ processSection()

void processSection ( yyscan_t yyscanner)
static

Definition at line 1658 of file doctokenizer.l.

1659{
1660 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1661 //printf("%s: found section/anchor with name '%s'\n",qPrint(yyextra->fileName),qPrint(yyextra->secLabel));
1662 DString file;
1663 if (yyextra->definition)
1664 {
1665 file = yyextra->definition->getOutputFileBase();
1666 }
1667 else
1668 {
1669 warn(yyextra->fileName,yyextra->yyLineNr,"Found section/anchor {} without context",yyextra->secLabel);
1670 }
1671 SectionInfo *si = SectionManager::instance().find(yyextra->secLabel);
1672 if (si)
1673 {
1674 si->setFileName(file);
1675 si->setType(yyextra->secType);
1676 }
LinkedMap::find
const T * find(const std::string &key) const
Definition linkedmap.h:47
SectionInfo
class that provide information about a section.
Definition section.h:58
SectionInfo::setType
void setType(SectionType t)
Definition section.h:81
SectionInfo::setFileName
void setFileName(const DString &fn)
Definition section.h:80
SectionManager::instance
static SectionManager & instance()
returns a reference to the singleton
Definition section.h:179
warn
#define warn(file, line, fmt,...)
Definition message.h:97
1677}

References LinkedMap< T, Hash, KeyEqual, Map >::find(), SectionManager::instance(), SectionInfo::setFileName(), SectionInfo::setType(), and warn.

◆ stateToString()

const char * stateToString ( int state)
static

◆ yylex()

int yylex ( yyscan_t yyscanner)

Definition at line 340 of file doctokenizer.l.

342 {LISTITEM} { /* list item */
343 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
344 lineCount(yytext,yyleng);
345 DString text(yytext);
346 size_t dashPos = text.rfind('-');
347 ASSERT(dashPos!=DString::npos);
348 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
349 yyextra->token.isCheckedList = false;
350 yyextra->token.id = -1;
351 yyextra->token.indent = computeIndent(yytext,dashPos);
352 return Token::make_TK_LISTITEM();
353 }
lineCount
#define lineCount(s, len)
Definition doctokenizer.l:98
computeIndent
static int computeIndent(const char *str, size_t length)
Definition doctokenizer.l:127
ASSERT
#define ASSERT(x)
Definition message.h:142
354<St_Para>^{CLISTITEM} { /* checkbox item */
355 DString text=yytext;
356 size_t dashPos = text.rfind('-');
357 yyextra->token.isEnumList = false;
358 yyextra->token.isCheckedList = true;
359 if (text.find('x') != DString::npos) yyextra->token.id = DocAutoList::Checked_x;
360 else if (text.find('X') != DString::npos) yyextra->token.id = DocAutoList::Checked_X;
361 else yyextra->token.id = DocAutoList::Unchecked;
362 yyextra->token.indent = computeIndent(yytext,dashPos);
363 return Token::make_TK_LISTITEM();
364 }
DString::rfind
size_t rfind(char c, size_t pos=npos) const
Definition dstring.h:244
365<St_Para>^{MLISTITEM} { /* list item */
366 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
367 {
368 REJECT;
369 }
370 else
371 {
372 lineCount(yytext,yyleng);
373 std::string text(yytext);
374 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
375 reg::Match match;
376 reg::search(text,match,re);
377 size_t listPos = match.position();
378 ASSERT(listPos!=std::string::npos);
379 yyextra->token.isEnumList = false;
380 yyextra->token.isCheckedList = false;
381 yyextra->token.id = -1;
382 yyextra->token.indent = computeIndent(yytext,listPos);
383 return Token::make_TK_LISTITEM();
384 }
385 }
reg::Ex
Class representing a regular expression.
Definition regex.h:39
reg::Match
Object representing the matching results.
Definition regex.h:154
reg::search
bool search(std::string_view str, Match &match, const Ex &re, size_t pos)
Search in a given string str starting at position pos for a match against regular expression re.
Definition regex.cpp:850
reg::match
bool match(std::string_view str, Match &match, const Ex &re)
Matches a given string str for a match against regular expression re.
Definition regex.cpp:861
386<St_Para>^{OLISTITEM} { /* numbered list item */
387 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
388 {
389 REJECT;
390 }
391 else
392 {
393 std::string text(yytext);
394 static const reg::Ex re(R"(\d+)");
395 reg::Match match;
396 reg::search(text,match,re);
397 size_t markPos = match.position();
398 ASSERT(markPos!=std::string::npos);
399 yyextra->token.isEnumList = true;
400 yyextra->token.isCheckedList = false;
401 bool ok = false;
402 int id = DString(match.str()).toInt(&ok);
403 yyextra->token.id = ok ? id : -1;
404 if (!ok)
405 {
406 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '{}' ",match.str());
407 }
408 yyextra->token.indent = computeIndent(yytext,markPos);
409 return Token::make_TK_LISTITEM();
410 }
411 }
DString::toInt
int toInt(bool *ok=nullptr, int base=10) const
Definition dstring.cpp:191
412<St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM} { /* list item on next line */
413 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
414 lineCount(yytext,yyleng);
415 DString text=extractPartAfterNewLine(yytext);
416 size_t dashPos = text.rfind('-');
417 ASSERT(dashPos!=DString::npos);
418 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
419 yyextra->token.isCheckedList = false;
420 yyextra->token.id = -1;
421 yyextra->token.indent = computeIndent(text.data(),dashPos);
422 return Token::make_TK_LISTITEM();
423 }
DString::data
const char * data() const
Returns a pointer to the contents of the string in the form of a 0-terminated C string.
Definition dstring.h:157
extractPartAfterNewLine
DString extractPartAfterNewLine(const DString &text)
Definition doctokenizer.l:109
424<St_Para>{BLANK}*\n{CLISTITEM} { /* checkbox item on next line */
425 DString text=yytext;
426 text=text.mid(text.find('\n')+1);
427 size_t dashPos = text.rfind('-');
428 yyextra->token.isEnumList = false;
429 yyextra->token.isCheckedList = true;
430 if (text.find('x') != DString::npos) yyextra->token.id = DocAutoList::Checked_x;
431 else if (text.find('X') != DString::npos) yyextra->token.id = DocAutoList::Checked_X;
432 else yyextra->token.id = DocAutoList::Unchecked;
433 yyextra->token.indent = computeIndent(text.data(),dashPos);
434 return Token::make_TK_LISTITEM();
435 }
436<St_Para>{BLANK}*(\n|"\\ilinebr"){MLISTITEM} { /* list item on next line */
437 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
438 {
439 REJECT;
440 }
441 else
442 {
443 lineCount(yytext,yyleng);
444 std::string text=extractPartAfterNewLine(yytext).str();
445 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
446 reg::Match match;
447 reg::search(text,match,re);
448 size_t markPos = match.position();
449 ASSERT(markPos!=std::string::npos);
450 yyextra->token.isEnumList = false;
451 yyextra->token.isCheckedList = false;
452 yyextra->token.id = -1;
453 yyextra->token.indent = computeIndent(text.c_str(),markPos);
454 return Token::make_TK_LISTITEM();
455 }
456 }
DString::str
const std::string & str() const
Definition dstring.h:645
457<St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM} { /* list item on next line */
458 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
459 {
460 REJECT;
461 }
462 else
463 {
464 lineCount(yytext,yyleng);
465 std::string text=extractPartAfterNewLine(yytext).str();
466 static const reg::Ex re(R"(\d+)");
467 reg::Match match;
468 reg::search(text,match,re);
469 size_t markPos = match.position();
470 ASSERT(markPos!=std::string::npos);
471 yyextra->token.isEnumList = true;
472 yyextra->token.isCheckedList = false;
473 bool ok = false;
474 int id = DString(match.str()).toInt(&ok);
475 yyextra->token.id = ok ? id : -1;
476 if (!ok)
477 {
478 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '{}' ",match.str());
479 }
480 yyextra->token.indent = computeIndent(text.c_str(),markPos);
481 return Token::make_TK_LISTITEM();
482 }
483 }
484<St_Para>^{ENDLIST} { /* end list */
485 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
486 lineCount(yytext,yyleng);
487 size_t dotPos = DString(yytext).rfind('.');
488 yyextra->token.indent = computeIndent(yytext,dotPos);
489 return Token::make_TK_ENDLIST();
490 }
491<St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST} { /* end list on next line */
492 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
493 lineCount(yytext,yyleng);
494 DString text=extractPartAfterNewLine(yytext);
495 size_t dotPos = text.rfind('.');
496 yyextra->token.indent = computeIndent(text.data(),dotPos);
497 return Token::make_TK_ENDLIST();
498 }
499<St_Para>"{"{BLANK}*"@linkplain"/{WS}+ {
500 yyextra->token.name = "javalinkplain";
501 return Token::make_TK_COMMAND_AT();
502 }
503<St_Para>"{"{BLANK}*"@link"/{WS}+ {
504 yyextra->token.name = "javalink";
505 return Token::make_TK_COMMAND_AT();
506 }
507<St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
508 yyextra->token.name = "inheritdoc";
509 return Token::make_TK_COMMAND_AT();
510 }
511<St_Para>"@_fakenl" { // artificial new line
512 //yyextra->yyLineNr++;
513 }
514<St_Para>{SPCMD3} {
515 yyextra->token.name = "_form";
516 bool ok;
517 yyextra->token.id = DString(yytext).right((int)yyleng-7).toInt(&ok);
518 ASSERT(ok);
519 return Token::char_to_command(yytext[0]);
520 }
DString::right
DString right(size_t len) const
Definition dstring.h:311
Token::char_to_command
static Token char_to_command(char c)
Definition doctokenizer.h:108
521<St_Para>{CMD}"n"\n { /* \n followed by real newline */
522 lineCount(yytext,yyleng);
523 //yyextra->yyLineNr++;
524 yyextra->token.name = yytext+1;
525 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
526 yyextra->token.paramDir=TokenInfo::Unspecified;
527 return Token::char_to_command(yytext[0]);
528 }
529<St_Para>"\\ilinebr" {
530 }
531<St_Para>{SPCMD1} |
532<St_Para>{SPCMD2} |
533<St_Para>{SPCMD5} |
534<St_Para>{SPCMD4} { /* special command */
535 yyextra->token.name = yytext+1;
536 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
537 yyextra->token.paramDir=TokenInfo::Unspecified;
538 return Token::char_to_command(yytext[0]);
539 }
540<St_Para>{PARAMIO} { /* param [in,out] command */
541 yyextra->token.name = "param";
542 DString s(yytext);
543 bool isIn = s.find("in") !=DString::npos;
544 bool isOut = s.find("out")!=DString::npos;
545 if (isIn)
546 {
547 if (isOut)
548 {
549 yyextra->token.paramDir=TokenInfo::InOut;
550 }
551 else
552 {
553 yyextra->token.paramDir=TokenInfo::In;
554 }
555 }
556 else if (isOut)
557 {
558 yyextra->token.paramDir=TokenInfo::Out;
559 }
560 else
561 {
562 yyextra->token.paramDir=TokenInfo::Unspecified;
563 }
564 return Token::char_to_command(yytext[0]);
565 }
566<St_Para>{URLPROTOCOL}{URLMASK}/[,\.] { // URL, or URL.
567 yyextra->token.name=yytext;
568 yyextra->token.isEMailAddr=false;
569 return Token::make_TK_URL();
570 }
571<St_Para>{URLPROTOCOL}{URLMASK} { // URL
572 yyextra->token.name=yytext;
573 yyextra->token.isEMailAddr=false;
574 return Token::make_TK_URL();
575 }
576<St_Para>"<"{URLPROTOCOL}{URLMASK}">" { // URL
577 yyextra->token.name=yytext;
578 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
579 yyextra->token.isEMailAddr=false;
580 return Token::make_TK_URL();
581 }
582<St_Para>{MAILADDR} { // Mail address
583 yyextra->token.name=yytext;
584 yyextra->token.name.stripPrefix("mailto:");
585 yyextra->token.isEMailAddr=true;
586 return Token::make_TK_URL();
587 }
588<St_Para>"<"{MAILADDR}">" { // Mail address
589 yyextra->token.name=yytext;
590 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
591 yyextra->token.name.stripPrefix("mailto:");
592 yyextra->token.isEMailAddr=true;
593 return Token::make_TK_URL();
594 }
595<St_Para>"<"{MAILADDR2}">" { // anti spam mail address
596 yyextra->token.name=yytext;
597 return Token::make_TK_WORD();
598 }
599<St_Para>{RCSID} { /* RCS tag */
600 DString tagName(yytext+1);
601 size_t index=tagName.find(':');
602 if (index==DString::npos) index=0; // should never happen
603 yyextra->token.name = tagName.left(index);
604 size_t text_begin = index+2;
605 size_t text_end = tagName.length()-1;
606 if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
607 {
608 ++text_begin;
609 if (tagName[text_end-1]=='#')
610 {
611 --text_end;
612 }
613 }
614 yyextra->token.text = tagName.mid(text_begin,text_end-text_begin);
615 return Token::make_TK_RCSTAG();
616 }
617<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}")" | /* environment variable */
618<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}"("{ID}"))" { /* environment variable */
619 DString name(&yytext[2]);
620 name = name.left(static_cast<int>(name.length())-1);
621 DString value = Portable::getenv(name);
622 for (int i=static_cast<int>(value.length())-1;i>=0;i--) unput(value.at(i));
623 }
DString::length
size_t length() const
Returns the length of the string, not counting the 0-terminator.
Definition dstring.h:151
Portable::getenv
DString getenv(const DString &variable)
Definition portable.cpp:337
624<St_Para>"<blockquote>&zwj;" {
625 // for a markdown inserted block quote,
626 // tell flex that after putting the last indent
627 // back we are at the beginning of the line, see issue #11309
628 YY_CURRENT_BUFFER->yy_at_bol=1;
629 lineCount(yytext,yyleng);
630 handleHtmlTag(yyscanner,yytext);
631 return Token::make_TK_HTMLTAG();
632 }
handleHtmlTag
static void handleHtmlTag(yyscan_t yyscanner, const char *text)
Definition doctokenizer.l:1679
633<St_Para>{HTMLTAG} { /* html tag */
634 lineCount(yytext,yyleng);
635 handleHtmlTag(yyscanner,yytext);
636 return Token::make_TK_HTMLTAG();
637 }
638<St_Para,St_Text>"&"{ID}";" { /* special symbol */
639 yyextra->token.name = yytext;
640 return Token::make_TK_SYMBOL();
641 }
642
643 /********* patterns for linkable words ******************/
644
645<St_Para>{ID}/"<"{HTMLKEYW}">"+ { /* this rule is to prevent opening html
646 * tag to be recognized as a templated classes
647 */
648 yyextra->token.name = yytext;
649 return Token::make_TK_LNKWORD();
650 }
651<St_Para>{LNKWORDN}/("<"{HTMLKEYW}">")+ { // prevent <br> html tag to be parsed as template arguments
652 yyextra->token.name = yytext;
653 return Token::make_TK_LNKWORD();
654 }
655<St_Para>{LNKWORDN}"<"{HTMLKEYW}">"[^<]*"</"{HTMLKEYW}">" { // a word directly followed by a complete inline
656 // html span, e.g. one<tt>.h</tt> (from a markdown code span) or test<em>.h</em>,
657 // is not a template: return only the word and let the html tag rules handle the span
658 const std::string text(yytext);
659 const size_t closeTagPos = text.rfind('<');
660 const size_t openTagPos = text.rfind('<', closeTagPos - 1);
661 yyless(static_cast<int>(openTagPos));
662 yyextra->token.name = yytext;
663 return Token::make_TK_LNKWORD();
664 }
665<St_Para>{LNKWORD1} |
666<St_Para>{LNKWORD1}{FUNCARG} |
667<St_Para>{LNKWORD2} |
668<St_Para>{LNKWORD3} |
669<St_Para>{LNKWORD4} {
670 yyextra->token.name = yytext;
671 return Token::make_TK_LNKWORD();
672 }
673<St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
674 yyextra->token.name = yytext;
675 yyextra->token.name = yyextra->token.name.left(yyextra->token.name.length()-1);
676 unput(yytext[(int)yyleng-1]);
677 return Token::make_TK_LNKWORD();
678 }
679 /********* patterns for normal words ******************/
680
681<St_Para,St_Text>[\-+0-9] |
682<St_Para,St_Text>{WORD1} |
683<St_Para,St_Text>{WORD2} { /* function call */
684 if (DString(yytext).find("\\ilinebr")!=DString::npos) REJECT; // see issue #8311
685 lineCount(yytext,yyleng);
686 if (yytext[0]=='%') // strip % if present
687 yyextra->token.name = &yytext[1];
688 else
689 yyextra->token.name = yytext;
690 return Token::make_TK_WORD();
691 }
692<St_Text>({ID}".")+{ID} {
693 yyextra->token.name = yytext;
694 return Token::make_TK_WORD();
695 }
696<St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
697 // avoid interpretation as "operator <"
698 yyextra->token.name = yytext;
699 return Token::make_TK_WORD();
700 }
701
702 /*******************************************************/
703
704<St_Para,St_Text>{BLANK}+ |
705<St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
706 lineCount(yytext,yyleng);
707 yyextra->token.chars=yytext;
708 return Token::make_TK_WHITESPACE();
709 }
710<St_Text>[\\@<>&$#%~] {
711 yyextra->token.name = yytext;
712 return Token::char_to_command(yytext[0]);
713 }
714<St_Para>({BLANK}*\n)+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
715 if (yyextra->insidePre || yyextra->autoListLevel==0)
716 {
717 REJECT;
718 }
719 lineCount(yytext,yyleng);
720 }
721<St_Para>({BLANK}*\n)+{BLANK}*\n/{CLISTITEM} { /* skip trailing paragraph followed by new checkbox item */
722 if (yyextra->insidePre || yyextra->autoListLevel==0)
723 {
724 REJECT;
725 }
726 }
727<St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
728 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
729 {
730 REJECT;
731 }
732 lineCount(yytext,yyleng);
733 }
734<St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
735 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
736 {
737 REJECT;
738 }
739 lineCount(yytext,yyleng);
740 }
741<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}*/" \\ifile" | // we don't want to count the space before \ifile
742<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}* {
743 lineCount(yytext,yyleng);
744 if (yyextra->insidePre)
745 {
746 yyextra->token.chars=yytext;
747 return Token::make_TK_WHITESPACE();
748 }
749 else
750 {
751 yyextra->token.indent=computeIndent(yytext,yyleng);
752 int i;
753 // put back the indentation (needed for list items)
754 //printf("token.indent=%d\n",yyextra->token.indent);
755 for (i=0;i<yyextra->token.indent;i++)
756 {
757 unput(' ');
758 }
759 // tell flex that after putting the last indent
760 // back we are at the beginning of the line
761 YY_CURRENT_BUFFER->yy_at_bol=1;
762 // start of a new paragraph
763 return Token::make_TK_NEWPARA();
764 }
765 }
766<St_CodeOpt>{BLANK}*"{"(".")?{CODEID}"}" {
767 yyextra->token.name = yytext;
768 size_t i=yyextra->token.name.find('{'); /* } to keep vi happy */
769 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
770 BEGIN(St_Code);
771 }
772<St_iCodeOpt>{BLANK}*"{"(".")?{CODEID}"}" {
773 yyextra->token.name = yytext;
774 size_t i=yyextra->token.name.find('{'); /* } to keep vi happy */
775 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
776 BEGIN(St_iCode);
777 }
778<St_CodeOpt>"\\ilinebr" |
779<St_CodeOpt>\n |
780<St_CodeOpt>. {
781 unput_string(yytext,yyleng);
782 BEGIN(St_Code);
783 }
unput_string
#define unput_string(yytext, yyleng)
Definition doctokenizer.l:160
784<St_iCodeOpt>"\\ilinebr" |
785<St_iCodeOpt>\n |
786<St_iCodeOpt>. {
787 unput_string(yytext,yyleng);
788 BEGIN(St_iCode);
789 }
790<St_Code>{WS}*{CMD}"endcode" {
791 lineCount(yytext,yyleng);
792 return Token::make_RetVal_OK();
793 }
794<St_iCode>{WS}*{CMD}"endicode" {
795 lineCount(yytext,yyleng);
796 return Token::make_RetVal_OK();
797 }
798<St_XmlCode>{WS}*"</code>" {
799 lineCount(yytext,yyleng);
800 return Token::make_RetVal_OK();
801 }
802<St_Code,St_iCode,St_XmlCode>[^\\@\n<]+ |
803<St_Code,St_iCode,St_XmlCode>\n |
804<St_Code,St_iCode,St_XmlCode>. {
805 lineCount(yytext,yyleng);
806 yyextra->token.verb+=yytext;
807 }
808<St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
809 yyextra->token.name="block";
810 BEGIN(St_HtmlOnly);
811 }
812<St_HtmlOnlyOption>.|\n {
813 unput(*yytext);
814 BEGIN(St_HtmlOnly);
815 }
816<St_HtmlOnlyOption>"\\ilinebr" {
817 unput_string(yytext,yyleng);
818 BEGIN(St_HtmlOnly);
819 }
820<St_HtmlOnly>{CMD}"endhtmlonly" {
821 return Token::make_RetVal_OK();
822 }
823<St_HtmlOnly>[^\\@\n$]+ |
824<St_HtmlOnly>\n |
825<St_HtmlOnly>. {
826 lineCount(yytext,yyleng);
827 yyextra->token.verb+=yytext;
828 }
829<St_ManOnly>{CMD}"endmanonly" {
830 return Token::make_RetVal_OK();
831 }
832<St_ManOnly>[^\\@\n$]+ |
833<St_ManOnly>\n |
834<St_ManOnly>. {
835 lineCount(yytext,yyleng);
836 yyextra->token.verb+=yytext;
837 }
838<St_RtfOnly>{CMD}"endrtfonly" {
839 return Token::make_RetVal_OK();
840 }
841<St_RtfOnly>[^\\@\n$]+ |
842<St_RtfOnly>\n |
843<St_RtfOnly>. {
844 lineCount(yytext,yyleng);
845 yyextra->token.verb+=yytext;
846 }
847<St_LatexOnly>{CMD}"endlatexonly" {
848 return Token::make_RetVal_OK();
849 }
850<St_LatexOnly>[^\\@\n]+ |
851<St_LatexOnly>\n |
852<St_LatexOnly>. {
853 lineCount(yytext,yyleng);
854 yyextra->token.verb+=yytext;
855 }
856<St_XmlOnly>{CMD}"endxmlonly" {
857 return Token::make_RetVal_OK();
858 }
859<St_XmlOnly>[^\\@\n]+ |
860<St_XmlOnly>\n |
861<St_XmlOnly>. {
862 lineCount(yytext,yyleng);
863 yyextra->token.verb+=yytext;
864 }
865<St_DbOnly>{CMD}"enddocbookonly" {
866 return Token::make_RetVal_OK();
867 }
868<St_DbOnly>[^\\@\n]+ |
869<St_DbOnly>\n |
870<St_DbOnly>. {
871 lineCount(yytext,yyleng);
872 yyextra->token.verb+=yytext;
873 }
874<St_Verbatim>{CMD}"endverbatim" {
875 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
876 return Token::make_RetVal_OK();
877 }
878<St_ILiteral>{CMD}"endiliteral " { // note extra space as this is artificially added
879 // remove spaces that have been added
880 yyextra->token.verb = yyextra->token.verb.mid(1,yyextra->token.verb.length()-2);
881 return Token::make_RetVal_OK();
882 }
883<St_iVerbatim>{CMD}"endiverbatim" {
884 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
885 return Token::make_RetVal_OK();
886 }
887<St_Verbatim,St_iVerbatim,St_ILiteral>[^\\@\n]+ |
888<St_Verbatim,St_iVerbatim,St_ILiteral>\n |
889<St_Verbatim,St_iVerbatim,St_ILiteral>. { /* Verbatim text */
890 lineCount(yytext,yyleng);
891 yyextra->token.verb+=yytext;
892 }
893<St_ILiteralOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // option(s) present
894 yyextra->token.verb = DString(yytext).stripWhiteSpace();
895 return Token::make_RetVal_OK();
896 }
DString::stripWhiteSpace
DString stripWhiteSpace() const
returns a copy of this string with leading and trailing whitespace removed
Definition dstring.h:337
897<St_ILiteralOpt>"\\ilinebr" |
898<St_ILiteralOpt>"\n" |
899<St_ILiteralOpt>. {
900 yyextra->token.sectionId = "";
901 unput_string(yytext,yyleng);
902 return Token::make_RetVal_OK();
903 }
904<St_Dot>{CMD}"enddot" {
905 return Token::make_RetVal_OK();
906 }
907<St_Dot>[^\\@\n]+ |
908<St_Dot>\n |
909<St_Dot>. { /* dot text */
910 lineCount(yytext,yyleng);
911 yyextra->token.verb+=yytext;
912 }
913<St_Msc>{CMD}("endmsc") {
914 return Token::make_RetVal_OK();
915 }
916<St_Msc>[^\\@\n]+ |
917<St_Msc>\n |
918<St_Msc>. { /* msc text */
919 lineCount(yytext,yyleng);
920 yyextra->token.verb+=yytext;
921 }
922<St_PlantUMLOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
923 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
924 return Token::make_RetVal_OK();
925 }
926<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
927 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
928 return Token::make_RetVal_OK();
929 }
930<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
931 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
932 return Token::make_RetVal_OK();
933 }
934<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
935 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
936 return Token::make_RetVal_OK();
937 }
938<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
939 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
940 return Token::make_RetVal_OK();
941 }
942<St_PlantUMLOpt>"\\ilinebr" |
943<St_PlantUMLOpt>"\n" |
944<St_PlantUMLOpt>. {
945 yyextra->token.sectionId = "";
946 unput_string(yytext,yyleng);
947 return Token::make_RetVal_OK();
948 }
949<St_PlantUML>{CMD}"enduml" {
950 return Token::make_RetVal_OK();
951 }
952<St_PlantUML>[^\\@\n]+ |
953<St_PlantUML>\n |
954<St_PlantUML>. { /* plantuml text */
955 lineCount(yytext,yyleng);
956 yyextra->token.verb+=yytext;
957 }
958<St_MermaidOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
959 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
960 return Token::make_RetVal_OK();
961 }
962<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
963 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
964 return Token::make_RetVal_OK();
965 }
966<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
967 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
968 return Token::make_RetVal_OK();
969 }
970<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
971 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
972 return Token::make_RetVal_OK();
973 }
974<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
975 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
976 return Token::make_RetVal_OK();
977 }
978<St_MermaidOpt>"\\ilinebr" |
979<St_MermaidOpt>"\n" |
980<St_MermaidOpt>. {
981 yyextra->token.sectionId = "";
982 unput_string(yytext,yyleng);
983 return Token::make_RetVal_OK();
984 }
985<St_Mermaid>{CMD}"endmermaid" {
986 return Token::make_RetVal_OK();
987 }
988<St_Mermaid>[^\\@\n]+ |
989<St_Mermaid>\n |
990<St_Mermaid>. { /* mermaid text */
991 lineCount(yytext,yyleng);
992 yyextra->token.verb+=yytext;
993 }
994<St_Title>"\"" { // quoted title
995 BEGIN(St_TitleQ);
996 }
997<St_Title>[ \t]+ {
998 yyextra->token.chars=yytext;
999 return Token::make_TK_WHITESPACE();
1000 }
1001<St_Title>. { // non-quoted title
1002 unput(*yytext);
1003 BEGIN(St_TitleN);
1004 }
1005<St_Title>\n {
1006 unput(*yytext);
1007 return Token::make_TK_NONE();
1008 }
1009<St_Title>"\\ilinebr" {
1010 unput_string(yytext,yyleng);
1011 return Token::make_TK_NONE();
1012 }
1013<St_TitleN>"&"{ID}";" { /* symbol */
1014 yyextra->token.name = yytext;
1015 return Token::make_TK_SYMBOL();
1016 }
1017<St_TitleN>{HTMLTAG} {
1018 yyextra->token.name = yytext;
1019 handleHtmlTag(yyscanner,yytext);
1020 return Token::make_TK_HTMLTAG();
1021 }
1022<St_TitleN>\n { /* new line => end of title */
1023 unput(*yytext);
1024 return Token::make_TK_NONE();
1025 }
1026<St_TitleN>"\\ilinebr" { /* new line => end of title */
1027 unput_string(yytext,yyleng);
1028 return Token::make_TK_NONE();
1029 }
1030<St_TitleN>{SPCMD1} |
1031<St_TitleN>{SPCMD2} { /* special command */
1032 yyextra->token.name = yytext+1;
1033 yyextra->token.paramDir=TokenInfo::Unspecified;
1034 return Token::char_to_command(yytext[0]);
1035 }
1036<St_TitleN>{ID}"=" { /* attribute */
1037 if (yytext[0]=='%') // strip % if present
1038 yyextra->token.name = &yytext[1];
1039 else
1040 yyextra->token.name = yytext;
1041 return Token::make_TK_WORD();
1042 }
1043<St_TitleN>[\-+0-9] |
1044<St_TitleN>{WORD1} |
1045<St_TitleN>{WORD2} { /* word */
1046 if (DString(yytext).find("\\ilinebr")!=DString::npos) REJECT; // see issue #8311
1047 lineCount(yytext,yyleng);
1048 if (yytext[0]=='%') // strip % if present
1049 yyextra->token.name = &yytext[1];
1050 else
1051 yyextra->token.name = yytext;
1052 return Token::make_TK_WORD();
1053 }
1054<St_TitleN>[ \t]+ {
1055 yyextra->token.chars=yytext;
1056 return Token::make_TK_WHITESPACE();
1057 }
1058<St_TitleQ>"&"{ID}";" { /* symbol */
1059 yyextra->token.name = yytext;
1060 return Token::make_TK_SYMBOL();
1061 }
1062<St_TitleQ>(\n|"\\ilinebr") { /* new line => end of title */
1063 unput_string(yytext,yyleng);
1064 return Token::make_TK_NONE();
1065 }
1066<St_TitleQ>{SPCMD1} |
1067<St_TitleQ>{SPCMD2} { /* special command */
1068 yyextra->token.name = yytext+1;
1069 yyextra->token.paramDir=TokenInfo::Unspecified;
1070 return Token::char_to_command(yytext[0]);
1071 }
1072<St_TitleQ>{WORD1NQ} |
1073<St_TitleQ>{WORD2NQ} { /* word */
1074 yyextra->token.name = yytext;
1075 return Token::make_TK_WORD();
1076 }
1077<St_TitleQ>[ \t]+ {
1078 yyextra->token.chars=yytext;
1079 return Token::make_TK_WHITESPACE();
1080 }
1081<St_TitleQ>"\"" { /* closing quote => end of title */
1082 BEGIN(St_TitleA);
1083 return Token::make_TK_NONE();
1084 }
1085<St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
1086 yyextra->token.name = yytext;
1087 size_t pos = yyextra->token.name.find('=');
1088 if (pos==DString::npos) pos=0; // should never happen
1089 yyextra->token.name = yyextra->token.name.left(pos).stripWhiteSpace();
1090 BEGIN(St_TitleV);
1091 }
1092<St_TitleV>[^ \t\r\n]+ { // attribute value
1093 lineCount(yytext,yyleng);
1094 yyextra->token.chars = yytext;
1095 BEGIN(St_TitleN);
1096 return Token::make_TK_WORD();
1097 }
1098<St_TitleV,St_TitleA>. {
1099 unput(*yytext);
1100 return Token::make_TK_NONE();
1101 }
1102<St_TitleV,St_TitleA>(\n|"\\ilinebr") {
1103 unput_string(yytext,yyleng);
1104 return Token::make_TK_NONE();
1105 }
1106
1107<St_Anchor>({REQID}|{LABELID}){WS}? { // anchor
1108 lineCount(yytext,yyleng);
1109 yyextra->token.name = DString(yytext).stripWhiteSpace();
1110 return Token::make_TK_WORD();
1111 }
1112<St_Anchor>. {
1113 unput(*yytext);
1114 return Token::make_TK_NONE();
1115 }
1116<St_Cite>{CITEID} { // label to cite
1117 if (yytext[0] =='"')
1118 {
1119 yyextra->token.name=yytext+1;
1120 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-2);
1121 }
1122 else
1123 {
1124 yyextra->token.name=yytext;
1125 }
1126 return Token::make_TK_WORD();
1127 }
1128<St_Cite>{BLANK} { // white space
1129 unput(' ');
1130 return Token::make_TK_NONE();
1131 }
1132<St_Cite>(\n|"\\ilinebr") { // new line
1133 unput_string(yytext,yyleng);
1134 return Token::make_TK_NONE();
1135 }
1136<St_Cite>. { // any other character
1137 unput(*yytext);
1138 return Token::make_TK_NONE();
1139 }
1140<St_DoxyConfig>{DOXYCFG} { // config option
1141 yyextra->token.name=yytext;
1142 return Token::make_TK_WORD();
1143 }
1144<St_DoxyConfig>{BLANK} { // white space
1145 unput(' ');
1146 return Token::make_TK_NONE();
1147 }
1148<St_DoxyConfig>(\n|"\\ilinebr") { // new line
1149 unput_string(yytext,yyleng);
1150 return Token::make_TK_NONE();
1151 }
1152<St_DoxyConfig>. { // any other character
1153 unput(*yytext);
1154 return Token::make_TK_NONE();
1155 }
1156<St_Ref>{REFWORD_NOCV}/{BLANK}("const")[a-z_A-Z0-9] { // see bug776988
1157 yyextra->token.name=yytext;
1158 return Token::make_TK_WORD();
1159 }
1160<St_Ref>{REFWORD_NOCV}/{BLANK}("volatile")[a-z_A-Z0-9] { // see bug776988
1161 yyextra->token.name=yytext;
1162 return Token::make_TK_WORD();
1163 }
1164<St_Ref>{REFWORD} { // label to refer to
1165 yyextra->token.name=yytext;
1166 return Token::make_TK_WORD();
1167 }
1168<St_Ref>{BLANK} { // white space
1169 unput(' ');
1170 return Token::make_TK_NONE();
1171 }
1172<St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
1173 yyextra->expectQuote=true;
1174 lineCount(yytext,yyleng);
1175 BEGIN(St_Ref2);
1176 }
1177<St_Ref>(\n|"\\ilinebr") { // new line
1178 unput_string(yytext,yyleng);
1179 return Token::make_TK_NONE();
1180 }
1181<St_Ref>"\""[^"\n]+"\"" { // quoted first argument -> return without quotes
1182 yyextra->token.name=DString(yytext).mid(1,yyleng-2);
1183 return Token::make_TK_WORD();
1184 }
1185<St_Ref>. { // any other character
1186 unput(*yytext);
1187 return Token::make_TK_NONE();
1188 }
1189<St_IntRef>[A-Z_a-z0-9.:/#\-\+\‍(\‍)]+ {
1190 yyextra->token.name = yytext;
1191 return Token::make_TK_WORD();
1192 }
1193<St_IntRef>{BLANK}+"\"" {
1194 BEGIN(St_Ref2);
1195 }
1196<St_SetScope>({SCOPEMASK}|{ANONNS}){BLANK}|{FILEMASK} {
1197 yyextra->token.name = yytext;
1198 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1199 return Token::make_TK_WORD();
1200 }
1201<St_SetScope>{SCOPEMASK}"<" {
1202 yyextra->token.name = yytext;
1203 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1204 yyextra->sharpCount=1;
1205 BEGIN(St_SetScopeEnd);
1206 }
1207<St_SetScope>{BLANK} {
1208 }
1209<St_SetScopeEnd>"<" {
1210 yyextra->token.name += yytext;
1211 yyextra->sharpCount++;
1212 }
1213<St_SetScopeEnd>">" {
1214 yyextra->token.name += yytext;
1215 yyextra->sharpCount--;
1216 if (yyextra->sharpCount<=0)
1217 {
1218 return Token::make_TK_WORD();
1219 }
1220 }
1221<St_SetScopeEnd>. {
1222 yyextra->token.name += yytext;
1223 }
1224<St_Ref2>"&"{ID}";" { /* symbol */
1225 yyextra->token.name = yytext;
1226 return Token::make_TK_SYMBOL();
1227 }
1228<St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title? */
1229 lineCount(yytext,yyleng);
1230 if (!yyextra->expectQuote || yytext[0]=='"')
1231 {
1232 return Token::make_TK_NONE();
1233 }
1234 else
1235 {
1236 yyextra->token.name += yytext;
1237 }
1238 }
1239<St_Ref2>{HTMLTAG_STRICT} { /* html tag */
1240 lineCount(yytext,yyleng);
1241 handleHtmlTag(yyscanner,yytext);
1242 return Token::make_TK_HTMLTAG();
1243 }
1244<St_Ref2>{SPCMD1} |
1245<St_Ref2>{SPCMD2} { /* special command */
1246 yyextra->token.name = yytext+1;
1247 yyextra->token.paramDir=TokenInfo::Unspecified;
1248 return Token::char_to_command(yytext[0]);
1249 }
1250<St_Ref2>{WORD1NQ} |
1251<St_Ref2>{WORD2NQ} {
1252 /* word */
1253 yyextra->token.name = yytext;
1254 return Token::make_TK_WORD();
1255 }
1256<St_Ref2>[ \t]+ {
1257 yyextra->token.chars=yytext;
1258 return Token::make_TK_WHITESPACE();
1259 }
1260<St_XRefItem>{LABELID} {
1261 yyextra->token.name=yytext;
1262 }
1263<St_XRefItem>" " {
1264 BEGIN(St_XRefItem2);
1265 }
1266<St_XRefItem2>[0-9]+"." {
1267 DString numStr(yytext);
1268 numStr=numStr.left((int)yyleng-1);
1269 yyextra->token.id=numStr.toInt();
1270 return Token::make_RetVal_OK();
1271 }
1272<St_Para,St_Title,St_Ref2>"<!--" { /* html style comment block */
1273 yyextra->commentState = YY_START;
1274 BEGIN(St_Comment);
1275 }
1276<St_Param>"\""[^\n\"]+"\"" {
1277 yyextra->token.name = yytext+1;
1278 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1279 return Token::make_TK_WORD();
1280 }
1281<St_Param>({PHPTYPE}{BLANK}*("["{BLANK}*"]")*{BLANK}*"|"{BLANK}*)*{PHPTYPE}{BLANK}*("["{BLANK}*"]")*{WS}+("&")?"$"{LABELID} {
1282 lineCount(yytext,yyleng);
1283 DString params(yytext);
1284 size_t j = params.find('&');
1285 size_t i = params.find('$');
1286 if (i==DString::npos) i=0; // should never happen
1287 if (j!=DString::npos && j<i) i=j;
1288 DString types = params.left(i).stripWhiteSpace();
1289 yyextra->token.name = types+"#"+params.mid(i);
1290 return Token::make_TK_WORD();
1291 }
DString::left
DString left(size_t len) const
Definition dstring.h:306
1292<St_Param>[^ \t\n,@\\‍]+ {
1293 yyextra->token.name = yytext;
1294 if (yyextra->token.name.at(static_cast<uint32_t>(yyleng)-1)==':')
1295 {
1296 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-1);
1297 }
1298 return Token::make_TK_WORD();
1299 }
1300<St_Param>{WS}*","{WS}* /* param separator */
1301<St_Param>{WS} {
1302 lineCount(yytext,yyleng);
1303 yyextra->token.chars=yytext;
1304 return Token::make_TK_WHITESPACE();
1305 }
1306<St_Prefix>"\""[^\n\"]*"\"" {
1307 yyextra->token.name = yytext+1;
1308 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1309 return Token::make_TK_WORD();
1310 }
1311<St_Prefix>. {
1312 unput(*yytext);
1313 return Token::make_TK_NONE();
1314 }
1315<St_Options>{ID} {
1316 yyextra->token.name+=yytext;
1317 }
1318<St_Options>{WS}*":"{WS}* {
1319 lineCount(yytext,yyleng);
1320 yyextra->token.name+=":";
1321 }
1322<St_Options>{WS}*","{WS}* |
1323<St_Options>{WS} { /* option separator */
1324 lineCount(yytext,yyleng);
1325 yyextra->token.name+=",";
1326 }
1327<St_Options>"}" {
1328 return Token::make_TK_WORD();
1329 }
1330<St_Block>{ID} {
1331 yyextra->token.name+=yytext;
1332 }
1333<St_Block>"]" {
1334 return Token::make_TK_WORD();
1335 }
1336<St_Emoji>[:0-9_a-z+-]+ {
1337 yyextra->token.name=yytext;
1338 return Token::make_TK_WORD();
1339 }
1340<St_Emoji>. {
1341 unput(*yytext);
1342 return Token::make_TK_NONE();
1343 }
1344<St_QuotedString>"\"" {
1345 yyextra->token.name="";
1346 BEGIN(St_QuotedContent);
1347 }
1348<St_QuotedString>(\n|"\\ilinebr") {
1349 unput_string(yytext,yyleng);
1350 return Token::make_TK_NONE();
1351 }
1352<St_QuotedString>. {
1353 unput(*yytext);
1354 return Token::make_TK_NONE();
1355 }
1356<St_QuotedContent>"\"" {
1357 return Token::make_TK_WORD();
1358 }
1359<St_QuotedContent>. {
1360 yyextra->token.name+=yytext;
1361 }
1362<St_ShowDate>{WS}+{SHOWDATE} {
1363 lineCount(yytext,yyleng);
1364 yyextra->token.name=yytext;
1365 return Token::make_TK_WORD();
1366 }
1367<St_ShowDate>(\n|"\\ilinebr") {
1368 unput_string(yytext,yyleng);
1369 return Token::make_TK_NONE();
1370 }
1371<St_ShowDate>. {
1372 unput(*yytext);
1373 return Token::make_TK_NONE();
1374 }
1375<St_ILine>{LINENR}/[\\@\n\.] |
1376<St_ILine>{LINENR}{BLANK} {
1377 bool ok = false;
1378 int nr = DString(yytext).toInt(&ok);
1379 if (!ok)
1380 {
1381 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '{}' for iline command",yytext);
1382 }
1383 else
1384 {
1385 yyextra->yyLineNr = nr;
1386 }
1387 return Token::make_TK_WORD();
1388 }
1389<St_ILine>. {
1390 return Token::make_TK_NONE();
1391 }
1392<St_IFile>{BLANK}*{FILEMASK} {
1393 DString text(yytext);
1394 text = text.stripWhiteSpace();
1395 yyextra->fileName = text;
1396 yyextra->token.name = text;
1397 return Token::make_TK_WORD();
1398 }
1399<St_IFile>{BLANK}*"\""[^\n\"]+"\"" {
1400 DString text(yytext);
1401 text = text.stripWhiteSpace();
1402 yyextra->fileName = text.mid(1,text.length()-2);
1403 yyextra->token.name = text.mid(1,text.length()-2);
1404 return Token::make_TK_WORD();
1405 }
1406<St_File>{FILEMASK} {
1407 yyextra->token.name = yytext;
1408 if (yyextra->token.name.endsWith("\\ilinebr") ||yyextra->token.name.endsWith("@ilinebr"))
1409 {
1410 unput_string("\\ilinebr",8);
1411 yyextra->token.name = yyextra->token.name.left(yyleng-8);
1412 }
1413 return Token::make_TK_WORD();
1414 }
1415<St_File>"\""[^\n\"]+"\"" {
1416 DString text(yytext);
1417 yyextra->token.name = text.mid(1,text.length()-2);
1418 return Token::make_TK_WORD();
1419 }
1420<St_Pattern>[^\\\r\n]+ {
1421 yyextra->token.name += yytext;
1422 }
1423<St_Pattern>"\\ilinebr" {
1424 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1425 return Token::make_TK_WORD();
1426 }
1427<St_Pattern>\n {
1428 lineCount(yytext,yyleng);
1429 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1430 return Token::make_TK_WORD();
1431 }
1432<St_Pattern>. {
1433 yyextra->token.name += yytext;
1434 }
1435<St_Link>{LINKMASK}|{REFWORD} {
1436 yyextra->token.name = yytext;
1437 return Token::make_TK_WORD();
1438 }
1439<St_Comment>"-->" { /* end of html comment */
1440 BEGIN(yyextra->commentState);
1441 }
1442<St_Comment>[^-]+ /* inside html comment */
1443<St_Comment>. /* inside html comment */
1444
1445 /* State for skipping title (all chars until the end of the line) */
1446
1447<St_SkipTitle>.
1448<St_SkipTitle>(\n|"\\ilinebr") {
1449 if (*yytext == '\n') unput('\n');
1450 return Token::make_TK_NONE();
1451 }
1452
1453 /* State for the pass used to find the anchors and sections */
1454
1455<St_Sections>[^\n@\<]+
1456<St_Sections>{CMD}("<"|{CMD})
1457<St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
1458 lineCount(yytext,yyleng);
1459 DString tag(yytext);
1460 size_t s=tag.find("id=");
1461 if (s!=DString::npos) // command has id attribute
1462 {
1463 char c=tag[s+3];
1464 if (c=='\'' || c=='"') // valid start
1465 {
1466 size_t e=tag.find(c,s+4);
1467 if (e!=DString::npos) // found matching end
1468 {
1469 yyextra->secType = SectionType::Table;
1470 yyextra->secLabel=tag.mid(s+4,e-s-4); // extract id
1471 processSection(yyscanner);
1472 }
1473 }
1474 }
1475 }
SectionType::Table
static constexpr int Table
Definition section.h:41
processSection
static void processSection(yyscan_t yyscanner)
Definition doctokenizer.l:1658
1476<St_Sections>{CMD}"anchor"{BLANK}+ {
1477 yyextra->secType = SectionType::Anchor;
1478 BEGIN(St_SecLabel1);
1479 }
SectionType::Anchor
static constexpr int Anchor
Definition section.h:40
1480<St_Sections>{CMD}"ianchor"{BLANK}+ {
1481 yyextra->secType = SectionType::Anchor;
1482 BEGIN(St_SecLabel1);
1483 }
1484<St_Sections>{CMD}"section"{BLANK}+ {
1485 yyextra->secType = SectionType::Section;
1486 BEGIN(St_SecLabel2);
1487 }
SectionType::Section
static constexpr int Section
Definition section.h:33
1488<St_Sections>{CMD}"subsection"{BLANK}+ {
1489 yyextra->secType = SectionType::Subsection;
1490 BEGIN(St_SecLabel2);
1491 }
SectionType::Subsection
static constexpr int Subsection
Definition section.h:34
1492<St_Sections>{CMD}"subsubsection"{BLANK}+ {
1493 yyextra->secType = SectionType::Subsubsection;
1494 BEGIN(St_SecLabel2);
1495 }
SectionType::Subsubsection
static constexpr int Subsubsection
Definition section.h:35
1496<St_Sections>{CMD}"paragraph"{BLANK}+ {
1497 yyextra->secType = SectionType::Paragraph;
1498 BEGIN(St_SecLabel2);
1499 }
SectionType::Paragraph
static constexpr int Paragraph
Definition section.h:36
1500<St_Sections>{CMD}"subparagraph"{BLANK}+ {
1501 yyextra->secType = SectionType::Subparagraph;
1502 BEGIN(St_SecLabel2);
1503 }
SectionType::Subparagraph
static constexpr int Subparagraph
Definition section.h:37
1504<St_Sections>{CMD}"subsubparagraph"{BLANK}+ {
1505 yyextra->secType = SectionType::Subsubparagraph;
1506 BEGIN(St_SecLabel2);
1507 }
SectionType::Subsubparagraph
static constexpr int Subsubparagraph
Definition section.h:38
1508<St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9] {
1509 yyextra->endMarker="endverbatim";
1510 BEGIN(St_SecSkip);
1511 }
1512<St_Sections>{CMD}"iverbatim"/[^a-z_A-Z0-9] {
1513 yyextra->endMarker="endiverbatim";
1514 BEGIN(St_SecSkip);
1515 }
1516<St_Sections>{CMD}"iliteral"/[^a-z_A-Z0-9] {
1517 yyextra->endMarker="endiliteral";
1518 BEGIN(St_SecSkip);
1519 }
1520<St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1521 yyextra->endMarker="enddot";
1522 BEGIN(St_SecSkip);
1523 }
1524<St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1525 yyextra->endMarker="endmsc";
1526 BEGIN(St_SecSkip);
1527 }
1528<St_Sections>{CMD}"mermaid"/[^a-z_A-Z0-9] {
1529 yyextra->endMarker="endmermaid";
1530 BEGIN(St_SecSkip);
1531 }
1532<St_Sections>{CMD}"startuml"/[^a-z_A-Z0-9] {
1533 yyextra->endMarker="enduml";
1534 BEGIN(St_SecSkip);
1535 }
1536<St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1537 yyextra->endMarker="endhtmlonly";
1538 BEGIN(St_SecSkip);
1539 }
1540<St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1541 yyextra->endMarker="endlatexonly";
1542 BEGIN(St_SecSkip);
1543 }
1544<St_Sections>{CMD}"manonly"/[^a-z_A-Z0-9] {
1545 yyextra->endMarker="endmanonly";
1546 BEGIN(St_SecSkip);
1547 }
1548<St_Sections>{CMD}"rtfonly"/[^a-z_A-Z0-9] {
1549 yyextra->endMarker="endrtfonly";
1550 BEGIN(St_SecSkip);
1551 }
1552<St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1553 yyextra->endMarker="endxmlonly";
1554 BEGIN(St_SecSkip);
1555 }
1556<St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
1557 yyextra->endMarker="enddocbookonly";
1558 BEGIN(St_SecSkip);
1559 }
1560<St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1561 yyextra->endMarker="endcode";
1562 BEGIN(St_SecSkip);
1563 }
1564<St_Sections>{CMD}"icode"/[^a-z_A-Z0-9] {
1565 yyextra->endMarker="endicode";
1566 BEGIN(St_SecSkip);
1567 }
1568<St_Sections>"<!--" {
1569 yyextra->endMarker="-->";
1570 BEGIN(St_SecSkip);
1571 }
1572<St_SecSkip>{CMD}{ID} {
1573 if (yyextra->endMarker==yytext+1)
1574 {
1575 BEGIN(St_Sections);
1576 }
1577 }
1578<St_SecSkip>"-->" {
1579 if (yyextra->endMarker==yytext)
1580 {
1581 BEGIN(St_Sections);
1582 }
1583 }
1584<St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1585<St_SecSkip>.
1586<St_SecSkip>(\n|"\\ilinebr")
1587<St_Sections>.
1588<St_Sections>(\n|"\\ilinebr")
1589<St_SecLabel1>({REQID}|{LABELID}) {
1590 lineCount(yytext,yyleng);
1591 yyextra->secLabel = yytext;
1592 processSection(yyscanner);
1593 BEGIN(St_Sections);
1594 }
1595<St_SecLabel2>({REQID}|{LABELID}){BLANK}+ |
1596<St_SecLabel2>({REQID}|{LABELID}) {
1597 yyextra->secLabel = yytext;
1598 yyextra->secLabel = yyextra->secLabel.stripWhiteSpace();
1599 BEGIN(St_SecTitle);
1600 }
1601<St_SecTitle>[^\n]+ |
1602<St_SecTitle>[^\n]*\n {
1603 lineCount(yytext,yyleng);
1604 yyextra->secTitle = yytext;
1605 yyextra->secTitle = yyextra->secTitle.stripWhiteSpace();
1606 if (yyextra->secTitle.endsWith("\\ilinebr"))
1607 {
1608 yyextra->secTitle.left(yyextra->secTitle.length()-8);
1609 }
1610 processSection(yyscanner);
1611 BEGIN(St_Sections);
1612 }
1613<St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1614 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}' while looking for section label or title",yytext);
1615 }
1616
1617<St_Snippet>[^\\\n]+ {
1618 yyextra->token.name += yytext;
1619 }
1620<St_Snippet>"\\" {
1621 yyextra->token.name += yytext;
1622 }
1623<St_Snippet>(\n|"\\ilinebr") {
1624 unput_string(yytext,yyleng);
1625 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1626 return Token::make_TK_WORD();
1627 }
1628
1629 /* Generic rules that work for all states */
1630<*>\n {
1631 lineCount(yytext,yyleng);
1632 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected new line character");
1633 }
1634<*>"\\ilinebr" {
1635 }
1636<*>[\\@<>&$#%~"=] { /* unescaped special character */
1637 //warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}', assuming command \\{} was meant.",yytext,yytext);
1638 yyextra->token.name = yytext;
1639 return Token::char_to_command(yytext[0]);
1640 }
1641<*>. {
1642 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}'",yytext);
1643 }
1644%%

◆ yyread()

int yyread ( yyscan_t yyscanner,
char * buf,
int max_size )
static

Definition at line 1648 of file doctokenizer.l.

1649{
1650 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1651 int c=0;
1652 const char *p = yyextra->inputString + yyextra->inputPos;
1653 while ( c < max_size && *p ) { *buf++ = *p++; c++; }
1654 yyextra->inputPos+=c;
1655 return c;
1656}