Doxygen
Loading...
Searching...
No Matches
doctokenizer.l File Reference
#include <stdint.h>
#include "doctokenizer.h"
#include <cctype>
#include <stack>
#include <string>
#include "cmdmapper.h"
#include "config.h"
#include "debug.h"
#include "definition.h"
#include "docnode.h"
#include "message.h"
#include "portable.h"
#include "regex.h"
#include "section.h"
#include "stringutil.h"
#include "doxygen_lex.h"
#include "doctokenizer.l.h"
Include dependency graph for doctokenizer.l:

Go to the source code of this file.

Classes

struct  DocLexerContext
struct  doctokenizerYY_state
struct  DocTokenizer::Private

Macros

#define YY_TYPEDEF_YY_SCANNER_T
#define YY_NO_INPUT   1
#define YY_NO_UNISTD_H   1
#define lineCount(s, len)
#define unput_string(yytext, yyleng)
#define YY_INPUT(buf, result, max_size)
#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)
#define yyterminate()

Typedefs

typedef yyguts_t * yyscan_t

Functions

static const char * stateToString (int state)
static int yyread (yyscan_t yyscanner, char *buf, int max_size)
static void handleHtmlTag (yyscan_t yyscanner, const char *text)
static void processSection (yyscan_t yyscanner)
DString extractPartAfterNewLine (const DString &text)
static int computeIndent (const char *str, size_t length)
static const char * getLexerFILE ()
int yylex (yyscan_t yyscanner)

Macro Definition Documentation

◆ lineCount

#define lineCount ( s,
len )
Value:
do { for(int i=0;i<(int)len;i++) if (s[i]=='\n') yyextra->yyLineNr++; } while(0)

Definition at line 99 of file doctokenizer.l.

Referenced by endBrief(), initMethodProtection(), and CitationManager::insertCrossReferencesForBibFile().

◆ unput_string

#define unput_string ( yytext,
yyleng )
Value:
do { for (int i=(int)yyleng-1;i>=0;i--) unput(yytext[i]); } while(0)

Definition at line 161 of file doctokenizer.l.

◆ YY_DECL

#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)

Definition at line 172 of file doctokenizer.l.

◆ YY_INPUT

#define YY_INPUT ( buf,
result,
max_size )
Value:
result=yyread(yyscanner,buf,max_size);
static int yyread(yyscan_t yyscanner, char *buf, int max_size)

Definition at line 165 of file doctokenizer.l.

◆ YY_NO_INPUT

#define YY_NO_INPUT   1

Definition at line 53 of file doctokenizer.l.

◆ YY_NO_UNISTD_H

#define YY_NO_UNISTD_H   1

Definition at line 54 of file doctokenizer.l.

◆ YY_TYPEDEF_YY_SCANNER_T

#define YY_TYPEDEF_YY_SCANNER_T

Definition at line 26 of file doctokenizer.l.

◆ yyterminate

#define yyterminate ( )
Value:
return Token::make_TK_EOF()

Definition at line 175 of file doctokenizer.l.

Typedef Documentation

◆ yyscan_t

typedef yyguts_t* yyscan_t

Definition at line 28 of file doctokenizer.l.

Function Documentation

◆ computeIndent()

int computeIndent ( const char * str,
size_t length )
static

Definition at line 128 of file doctokenizer.l.

129{
130 if (str==0 || length==std::string::npos) return 0;
131 size_t i;
132 int indent=0;
133 int tabSize=Config_getInt(TAB_SIZE);
134 for (i=0;i<length;i++)
135 {
136 if (str[i]=='\t')
137 {
138 indent+=tabSize - (indent%tabSize);
139 }
140 else if (str[i]=='\n')
141 {
142 indent=0;
143 }
144 else if (literal_at(&str[i],"\\ilinebr"))
145 {
146 indent=0;
147 i+=7;
148 if (str[i+1]==' ') i++; // also eat space after \\ilinebr if present
149 }
150 else
151 {
152 indent++;
153 }
154 }
155 //printf("input('%s')=%d\n",str,indent);
156 return indent;
#define Config_getInt(name)
Definition config.h:34
bool literal_at(const char *data, const char(&str)[N])
returns true iff data points to a substring that matches string literal str
Definition stringutil.h:101
157}

References Config_getInt, and literal_at().

◆ extractPartAfterNewLine()

DString extractPartAfterNewLine ( const DString & text)

Definition at line 110 of file doctokenizer.l.

111{
112 size_t nl1 = text.find('\n');
113 size_t nl2 = text.find("\\ilinebr");
114 if (nl1!=DString::npos && nl2!=DString::npos && nl1<nl2)
115 {
116 return text.mid(nl1+1);
117 }
118 if (nl2!=DString::npos)
119 {
120 if (text.at(nl2+8)==' ') nl2++; // skip space after \\ilinebr
121 return text.mid(nl2+8);
122 }
123 return text;
DString mid(size_t index, size_t len=npos) const
Definition dstring.h:318
static constexpr size_t npos
value used to indicate 'not found' or 'to the end of the string', matching std::string::npos
Definition dstring.h:178
char & at(size_t i)
Returns a reference to the character at index i.
Definition dstring.h:686
size_t find(char c, size_t pos=0) const
Definition dstring.h:239
124}

References DString::at(), DString::find(), DString::mid(), and DString::npos.

◆ getLexerFILE()

const char * getLexerFILE ( )
inlinestatic

Definition at line 168 of file doctokenizer.l.

168{return __FILE__;}

◆ handleHtmlTag()

void handleHtmlTag ( yyscan_t yyscanner,
const char * text )
static

Definition at line 1690 of file doctokenizer.l.

1691{
1692 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1693
1694 DString tagText(text);
1695 yyextra->token.text = tagText;
1696 yyextra->token.attribs.clear();
1697 yyextra->token.endTag = false;
1698 yyextra->token.emptyTag = false;
A String class for use with Doxygen wrapping std::string and adding some additional functionality off...
Definition dstring.h:84
1699
1700 // Check for end tag
1701 int startNamePos=1;
1702 if (tagText.at(1)=='/')
1703 {
1704 yyextra->token.endTag = true;
1705 startNamePos++;
1706 }
1707
1708 // Parse the name portion
1709 int i = startNamePos;
1710 for (i=startNamePos; i < (int)yyleng; i++)
1711 {
1712 // Check for valid HTML/XML name chars (including namespaces)
1713 char c = tagText.at(i);
1714 if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
1715 }
1716 yyextra->token.name = tagText.mid(startNamePos,i-startNamePos);
1717
1718 // Parse the attributes. Each attribute is a name, value pair
1719 // The result is stored in yyextra->token.attribs.
1720 int startAttribList = i;
1721 while (i<(int)yyleng)
1722 {
1723 char c=tagText.at(i);
1724 // skip spaces
1725 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1726 // check for end of the tag
1727 if (c == '>') break;
1728 // Check for XML style "empty" tag.
1729 if (c == '/')
1730 {
1731 yyextra->token.emptyTag = true;
1732 break;
1733 }
1734 int startName=i;
1735 // search for end of name
1736 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='=' && c!= '>') { c=tagText.at(++i); }
1737 int endName=i;
1738 DString optName,optValue;
1739 optName = tagText.mid(startName,endName-startName).lower();
1740 // skip spaces
1741 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1742 if (tagText.at(i)=='=') // option has value
1743 {
1744 int startAttrib=0, endAttrib=0;
1745 c=tagText.at(++i);
1746 // skip spaces
1747 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1748 if (tagText.at(i)=='\'') // option '...'
1749 {
1750 c=tagText.at(++i);
1751 startAttrib=i;
DString lower() const
Definition dstring.h:326
1752
1753 // search for matching quote
1754 while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
1755 endAttrib=i;
1756 if (i<(int)yyleng) { c=tagText.at(++i);}
1757 }
1758 else if (tagText.at(i)=='"') // option "..."
1759 {
1760 c=tagText.at(++i);
1761 startAttrib=i;
1762 // search for matching quote
1763 while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
1764 endAttrib=i;
1765 if (i<(int)yyleng) { c=tagText.at(++i);}
1766 }
1767 else // value without any quotes
1768 {
1769 startAttrib=i;
1770 // search for separator or end symbol
1771 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='>') { c=tagText.at(++i); }
1772 endAttrib=i;
1773 if (i<(int)yyleng) { c=tagText.at(++i);}
1774 }
1775 optValue = tagText.mid(startAttrib,endAttrib-startAttrib);
1776 if (optName == "align") optValue = optValue.lower();
1777 else if (optName == "valign")
1778 {
1779 optValue = optValue.lower();
1780 if (optValue == "center") optValue="middle";
1781 }
1782 }
1783 else // start next option
1784 {
1785 }
1786 //printf("=====> Adding option name=<%s> value=<%s>\n",
1787 // qPrint(optName),qPrint(optValue));
1788 yyextra->token.attribs.emplace_back(optName,optValue);
1789 }
1790 yyextra->token.attribsStr = tagText.mid(startAttribList,i-startAttribList);
1791}

References DString::at(), DString::clear(), DString::lower(), and DString::mid().

◆ processSection()

void processSection ( yyscan_t yyscanner)
static

Definition at line 1669 of file doctokenizer.l.

1670{
1671 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1672 //printf("%s: found section/anchor with name '%s'\n",qPrint(yyextra->fileName),qPrint(yyextra->secLabel));
1673 DString file;
1674 if (yyextra->definition)
1675 {
1676 file = yyextra->definition->getOutputFileBase();
1677 }
1678 else
1679 {
1680 warn(yyextra->fileName,yyextra->yyLineNr,"Found section/anchor {} without context",yyextra->secLabel);
1681 }
1682 SectionInfo *si = SectionManager::instance().find(yyextra->secLabel);
1683 if (si)
1684 {
1685 si->setFileName(file);
1686 si->setType(yyextra->secType);
1687 }
const T * find(const std::string &key) const
Definition linkedmap.h:47
class that provide information about a section.
Definition section.h:58
void setType(SectionType t)
Definition section.h:81
void setFileName(const DString &fn)
Definition section.h:80
static SectionManager & instance()
returns a reference to the singleton
Definition section.h:179
#define warn(file, line, fmt,...)
Definition message.h:97
1688}

References LinkedMap< T, Hash, KeyEqual, Map >::find(), SectionManager::instance(), SectionInfo::setFileName(), SectionInfo::setType(), and warn.

◆ stateToString()

const char * stateToString ( int state)
static

◆ yylex()

int yylex ( yyscan_t yyscanner)

Definition at line 341 of file doctokenizer.l.

343 {LISTITEM} { /* list item */
344 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
345 lineCount(yytext,yyleng);
346 DString text(yytext);
347 size_t dashPos = text.rfind('-');
348 ASSERT(dashPos!=DString::npos);
349 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
350 yyextra->token.isCheckedList = false;
351 yyextra->token.id = -1;
352 yyextra->token.indent = computeIndent(yytext,dashPos);
353 return Token::make_TK_LISTITEM();
354 }
#define lineCount(s, len)
static int computeIndent(const char *str, size_t length)
#define ASSERT(x)
Definition message.h:142
355<St_Para>^{CLISTITEM} { /* checkbox item */
356 DString text=yytext;
357 size_t dashPos = text.rfind('-');
358 yyextra->token.isEnumList = false;
359 yyextra->token.isCheckedList = true;
360 if (text.find('x') != DString::npos) yyextra->token.id = DocAutoList::Checked_x;
361 else if (text.find('X') != DString::npos) yyextra->token.id = DocAutoList::Checked_X;
362 else yyextra->token.id = DocAutoList::Unchecked;
363 yyextra->token.indent = computeIndent(yytext,dashPos);
364 return Token::make_TK_LISTITEM();
365 }
size_t rfind(char c, size_t pos=npos) const
Definition dstring.h:244
366<St_Para>^{MLISTITEM} { /* list item */
367 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
368 {
369 REJECT;
370 }
371 else
372 {
373 lineCount(yytext,yyleng);
374 std::string text(yytext);
375 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
377 reg::search(text,match,re);
378 size_t listPos = match.position();
379 ASSERT(listPos!=std::string::npos);
380 yyextra->token.isEnumList = false;
381 yyextra->token.isCheckedList = false;
382 yyextra->token.id = -1;
383 yyextra->token.indent = computeIndent(yytext,listPos);
384 return Token::make_TK_LISTITEM();
385 }
386 }
Class representing a regular expression.
Definition regex.h:39
Object representing the matching results.
Definition regex.h:160
bool search(std::string_view str, Match &match, const Ex &re, size_t pos)
Search in a given string str starting at position pos for a match against regular expression re.
Definition regex.cpp:850
bool match(std::string_view str, Match &match, const Ex &re)
Matches a given string str for a match against regular expression re.
Definition regex.cpp:861
387<St_Para>^{OLISTITEM} { /* numbered list item */
388 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
389 {
390 REJECT;
391 }
392 else
393 {
394 std::string text(yytext);
395 static const reg::Ex re(R"(\d+)");
397 reg::search(text,match,re);
398 size_t markPos = match.position();
399 ASSERT(markPos!=std::string::npos);
400 yyextra->token.isEnumList = true;
401 yyextra->token.isCheckedList = false;
402 bool ok = false;
403 int id = DString(match.str()).toInt(&ok);
404 yyextra->token.id = ok ? id : -1;
405 if (!ok)
406 {
407 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '{}' ",match.str());
408 }
409 yyextra->token.indent = computeIndent(yytext,markPos);
410 return Token::make_TK_LISTITEM();
411 }
412 }
int toInt(bool *ok=nullptr, int base=10) const
Definition dstring.cpp:191
413<St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM} { /* list item on next line */
414 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
415 lineCount(yytext,yyleng);
416 DString text=extractPartAfterNewLine(yytext);
417 size_t dashPos = text.rfind('-');
418 ASSERT(dashPos!=DString::npos);
419 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
420 yyextra->token.isCheckedList = false;
421 yyextra->token.id = -1;
422 yyextra->token.indent = computeIndent(text.data(),dashPos);
423 return Token::make_TK_LISTITEM();
424 }
const char * data() const
Returns a pointer to the contents of the string in the form of a 0-terminated C string.
Definition dstring.h:157
DString extractPartAfterNewLine(const DString &text)
425<St_Para>{BLANK}*\n{CLISTITEM} { /* checkbox item on next line */
426 DString text=yytext;
427 text=text.mid(text.find('\n')+1);
428 lineCount(yytext,yyleng);
429 size_t dashPos = text.rfind('-');
430 yyextra->token.isEnumList = false;
431 yyextra->token.isCheckedList = true;
432 if (text.find('x') != DString::npos) yyextra->token.id = DocAutoList::Checked_x;
433 else if (text.find('X') != DString::npos) yyextra->token.id = DocAutoList::Checked_X;
434 else yyextra->token.id = DocAutoList::Unchecked;
435 yyextra->token.indent = computeIndent(text.data(),dashPos);
436 return Token::make_TK_LISTITEM();
437 }
438<St_Para>{BLANK}*(\n|"\\ilinebr"){MLISTITEM} { /* list item on next line */
439 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
440 {
441 REJECT;
442 }
443 else
444 {
445 lineCount(yytext,yyleng);
446 std::string text=extractPartAfterNewLine(yytext).str();
447 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
449 reg::search(text,match,re);
450 size_t markPos = match.position();
451 ASSERT(markPos!=std::string::npos);
452 yyextra->token.isEnumList = false;
453 yyextra->token.isCheckedList = false;
454 yyextra->token.id = -1;
455 yyextra->token.indent = computeIndent(text.c_str(),markPos);
456 return Token::make_TK_LISTITEM();
457 }
458 }
const std::string & str() const
Definition dstring.h:645
459<St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM} { /* list item on next line */
460 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
461 {
462 REJECT;
463 }
464 else
465 {
466 lineCount(yytext,yyleng);
467 std::string text=extractPartAfterNewLine(yytext).str();
468 static const reg::Ex re(R"(\d+)");
470 reg::search(text,match,re);
471 size_t markPos = match.position();
472 ASSERT(markPos!=std::string::npos);
473 yyextra->token.isEnumList = true;
474 yyextra->token.isCheckedList = false;
475 bool ok = false;
476 int id = DString(match.str()).toInt(&ok);
477 yyextra->token.id = ok ? id : -1;
478 if (!ok)
479 {
480 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '{}' ",match.str());
481 }
482 yyextra->token.indent = computeIndent(text.c_str(),markPos);
483 return Token::make_TK_LISTITEM();
484 }
485 }
486<St_Para>^{ENDLIST} { /* end list */
487 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
488 lineCount(yytext,yyleng);
489 size_t dotPos = DString(yytext).rfind('.');
490 yyextra->token.indent = computeIndent(yytext,dotPos);
491 return Token::make_TK_ENDLIST();
492 }
493<St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST} { /* end list on next line */
494 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
495 lineCount(yytext,yyleng);
496 DString text=extractPartAfterNewLine(yytext);
497 size_t dotPos = text.rfind('.');
498 yyextra->token.indent = computeIndent(text.data(),dotPos);
499 return Token::make_TK_ENDLIST();
500 }
501<St_Para>"{"{BLANK}*"@linkplain"/{WS}+ {
502 yyextra->token.name = "javalinkplain";
503 return Token::make_TK_COMMAND_AT();
504 }
505<St_Para>"{"{BLANK}*"@link"/{WS}+ {
506 yyextra->token.name = "javalink";
507 return Token::make_TK_COMMAND_AT();
508 }
509<St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
510 yyextra->token.name = "inheritdoc";
511 return Token::make_TK_COMMAND_AT();
512 }
513<St_Para>"@_fakenl" { // artificial new line
514 //yyextra->yyLineNr++;
515 }
516<St_Para>{SPCMD3} {
517 yyextra->token.name = "_form";
518 bool ok;
519 yyextra->token.id = DString(yytext).right((int)yyleng-7).toInt(&ok);
520 ASSERT(ok);
521 return Token::char_to_command(yytext[0]);
522 }
DString right(size_t len) const
Definition dstring.h:311
static Token char_to_command(char c)
523<St_Para>{CMD}"n"\n { /* \n followed by real newline */
524 lineCount(yytext,yyleng);
525 //yyextra->yyLineNr++;
526 yyextra->token.name = yytext+1;
527 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
528 yyextra->token.paramDir=TokenInfo::Unspecified;
529 return Token::char_to_command(yytext[0]);
530 }
531<St_Para>"\\ilinebr" {
532 }
533<St_Para>{SPCMD1} |
534<St_Para>{SPCMD2} |
535<St_Para>{SPCMD5} |
536<St_Para>{SPCMD4} { /* special command */
537 yyextra->token.name = yytext+1;
538 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
539 yyextra->token.paramDir=TokenInfo::Unspecified;
540 return Token::char_to_command(yytext[0]);
541 }
542<St_Para>{PARAMIO} { /* param [in,out] command */
543 yyextra->token.name = "param";
544 DString s(yytext);
545 bool isIn = s.find("in") !=DString::npos;
546 bool isOut = s.find("out")!=DString::npos;
547 if (isIn)
548 {
549 if (isOut)
550 {
551 yyextra->token.paramDir=TokenInfo::InOut;
552 }
553 else
554 {
555 yyextra->token.paramDir=TokenInfo::In;
556 }
557 }
558 else if (isOut)
559 {
560 yyextra->token.paramDir=TokenInfo::Out;
561 }
562 else
563 {
564 yyextra->token.paramDir=TokenInfo::Unspecified;
565 }
566 return Token::char_to_command(yytext[0]);
567 }
568<St_Para>{URLPROTOCOL}{URLMASK}/[,\.] { // URL, or URL.
569 yyextra->token.name=yytext;
570 yyextra->token.isEMailAddr=false;
571 return Token::make_TK_URL();
572 }
573<St_Para>{URLPROTOCOL}{URLMASK} { // URL
574 yyextra->token.name=yytext;
575 yyextra->token.isEMailAddr=false;
576 return Token::make_TK_URL();
577 }
578<St_Para>"<"{URLPROTOCOL}{URLMASK}">" { // URL
579 yyextra->token.name=yytext;
580 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
581 yyextra->token.isEMailAddr=false;
582 return Token::make_TK_URL();
583 }
584<St_Para>{MAILADDR} { // Mail address
585 yyextra->token.name=yytext;
586 yyextra->token.name.stripPrefix("mailto:");
587 yyextra->token.isEMailAddr=true;
588 return Token::make_TK_URL();
589 }
590<St_Para>"<"{MAILADDR}">" { // Mail address
591 yyextra->token.name=yytext;
592 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
593 yyextra->token.name.stripPrefix("mailto:");
594 yyextra->token.isEMailAddr=true;
595 return Token::make_TK_URL();
596 }
597<St_Para>"<"{MAILADDR2}">" { // anti spam mail address
598 yyextra->token.name=yytext;
599 return Token::make_TK_WORD();
600 }
601<St_Para>{RCSID} { /* RCS tag */
602 DString tagName(yytext+1);
603 size_t index=tagName.find(':');
604 if (index==DString::npos) index=0; // should never happen
605 yyextra->token.name = tagName.left(index);
606 size_t text_begin = index+2;
607 size_t text_end = tagName.length()-1;
608 if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
609 {
610 ++text_begin;
611 if (tagName[text_end-1]=='#')
612 {
613 --text_end;
614 }
615 }
616 yyextra->token.text = tagName.mid(text_begin,text_end-text_begin);
617 return Token::make_TK_RCSTAG();
618 }
619<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}")" | /* environment variable */
620<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}"("{ID}"))" { /* environment variable */
621 DString name(&yytext[2]);
622 name = name.left(static_cast<int>(name.length())-1);
623 DString value = Portable::getenv(name);
624 for (int i=static_cast<int>(value.length())-1;i>=0;i--) unput(value.at(i));
625 }
size_t length() const
Returns the length of the string, not counting the 0-terminator.
Definition dstring.h:151
const char * name(int level)
Definition latexgen.h:368
DString getenv(const DString &variable)
Definition portable.cpp:337
626<St_Para>"<blockquote>&zwj;" {
627 // for a markdown inserted block quote,
628 // tell flex that after putting the last indent
629 // back we are at the beginning of the line, see issue #11309
630 YY_CURRENT_BUFFER->yy_at_bol=1;
631 lineCount(yytext,yyleng);
632 handleHtmlTag(yyscanner,yytext);
633 return Token::make_TK_HTMLTAG();
634 }
static void handleHtmlTag(yyscan_t yyscanner, const char *text)
635<St_Para>{HTMLTAG} { /* html tag */
636 lineCount(yytext,yyleng);
637 handleHtmlTag(yyscanner,yytext);
638 return Token::make_TK_HTMLTAG();
639 }
640<St_Para,St_Text>"&"{ID}";" { /* special symbol */
641 yyextra->token.name = yytext;
642 return Token::make_TK_SYMBOL();
643 }
644
645 /********* patterns for linkable words ******************/
646
647<St_Para>{ID}/"<"{HTMLKEYW}">"+ { /* this rule is to prevent opening html
648 * tag to be recognized as a templated classes
649 */
650 yyextra->token.name = yytext;
651 return Token::make_TK_LNKWORD();
652 }
653<St_Para>{LNKWORDN}/("<"{HTMLKEYW}">")+ { // prevent <br> html tag to be parsed as template arguments
654 yyextra->token.name = yytext;
655 return Token::make_TK_LNKWORD();
656 }
657<St_Para>{LNKWORDN}"<"{HTMLKEYW}">"[^<]*"</"{HTMLKEYW}">" { // a word directly followed by a complete inline
658 // html span, e.g. one<tt>.h</tt> (from a markdown code span) or test<em>.h</em>,
659 // is not a template: return only the word and let the html tag rules handle the span
660 const std::string text(yytext);
661 const size_t closeTagPos = text.rfind('<');
662 const size_t openTagPos = text.rfind('<', closeTagPos - 1);
663 yyless(static_cast<int>(openTagPos));
664 yyextra->token.name = yytext;
665 return Token::make_TK_LNKWORD();
666 }
667<St_Para>{LNKWORD1} |
668<St_Para>{LNKWORD1}{FUNCARG} |
669<St_Para>{LNKWORD2} |
670<St_Para>{LNKWORD3} |
671<St_Para>{LNKWORD4} {
672 yyextra->token.name = yytext;
673 return Token::make_TK_LNKWORD();
674 }
675<St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
676 yyextra->token.name = yytext;
677 yyextra->token.name = yyextra->token.name.left(yyextra->token.name.length()-1);
678 unput(yytext[(int)yyleng-1]);
679 return Token::make_TK_LNKWORD();
680 }
681 /********* patterns for normal words ******************/
682
683<St_Para,St_Text>[\-+0-9] |
684<St_Para,St_Text>{WORD1} |
685<St_Para,St_Text>{WORD2} { /* function call */
686 if (DString(yytext).find("\\ilinebr")!=DString::npos) REJECT; // see issue #8311
687 lineCount(yytext,yyleng);
688 if (yytext[0]=='%') // strip % if present
689 yyextra->token.name = &yytext[1];
690 else
691 yyextra->token.name = yytext;
692 return Token::make_TK_WORD();
693 }
694<St_Text>({ID}".")+{ID} {
695 yyextra->token.name = yytext;
696 return Token::make_TK_WORD();
697 }
698<St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
699 // avoid interpretation as "operator <"
700 yyextra->token.name = yytext;
701 return Token::make_TK_WORD();
702 }
703
704 /*******************************************************/
705
706<St_Para,St_Text>{BLANK}+ |
707<St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
708 lineCount(yytext,yyleng);
709 yyextra->token.chars=yytext;
710 return Token::make_TK_WHITESPACE();
711 }
712<St_Text>[\\@<>&$#%~] {
713 yyextra->token.name = yytext;
714 return Token::char_to_command(yytext[0]);
715 }
716<St_Para>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
717 if (yyextra->insidePre || yyextra->autoListLevel==0)
718 {
719 REJECT;
720 }
721 lineCount(yytext,yyleng);
722 }
723<St_Para>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*\n/{CLISTITEM} { /* skip trailing paragraph followed by new checkbox item */
724 if (yyextra->insidePre || yyextra->autoListLevel==0)
725 {
726 REJECT;
727 }
728 lineCount(yytext,yyleng);
729 }
730<St_Para>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
731 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
732 {
733 REJECT;
734 }
735 lineCount(yytext,yyleng);
736 }
737<St_Para>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
738 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
739 {
740 REJECT;
741 }
742 lineCount(yytext,yyleng);
743 }
744<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}*/" \\ifile" | // we don't want to count the space before \ifile
745<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}* {
746 lineCount(yytext,yyleng);
747 if (yyextra->insidePre)
748 {
749 yyextra->token.chars=yytext;
750 return Token::make_TK_WHITESPACE();
751 }
752 else
753 {
754 yyextra->token.indent=computeIndent(yytext,yyleng);
755 int i;
756 // put back the indentation (needed for list items)
757 //printf("token.indent=%d\n",yyextra->token.indent);
758 for (i=0;i<yyextra->token.indent;i++)
759 {
760 unput(' ');
761 }
762 // tell flex that after putting the last indent
763 // back we are at the beginning of the line
764 YY_CURRENT_BUFFER->yy_at_bol=1;
765 // start of a new paragraph
766 return Token::make_TK_NEWPARA();
767 }
768 }
769<St_CodeOpt>{BLANK}*"{"(".")?{CODEID}"}" {
770 yyextra->token.name = yytext;
771 size_t i=yyextra->token.name.find('{'); /* } to keep vi happy */
772 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
773 BEGIN(St_Code);
774 }
775<St_iCodeOpt>{BLANK}*"{"(".")?{CODEID}"}" {
776 yyextra->token.name = yytext;
777 size_t i=yyextra->token.name.find('{'); /* } to keep vi happy */
778 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
779 BEGIN(St_iCode);
780 }
781<St_CodeOpt>"\\ilinebr" |
782<St_CodeOpt>\n |
783<St_CodeOpt>. {
784 unput_string(yytext,yyleng);
785 BEGIN(St_Code);
786 }
#define unput_string(yytext, yyleng)
787<St_iCodeOpt>"\\ilinebr" |
788<St_iCodeOpt>\n |
789<St_iCodeOpt>. {
790 unput_string(yytext,yyleng);
791 BEGIN(St_iCode);
792 }
793<St_Code>{WS}*{CMD}"endcode" {
794 lineCount(yytext,yyleng);
795 return Token::make_RetVal_OK();
796 }
797<St_iCode>{WS}*{CMD}"endicode" {
798 lineCount(yytext,yyleng);
799 return Token::make_RetVal_OK();
800 }
801<St_XmlCode>{WS}*"</code>" {
802 lineCount(yytext,yyleng);
803 return Token::make_RetVal_OK();
804 }
805<St_Code,St_iCode,St_XmlCode>[^\\@\n<]+ |
806<St_Code,St_iCode,St_XmlCode>\n |
807<St_Code,St_iCode,St_XmlCode>. {
808 lineCount(yytext,yyleng);
809 yyextra->token.verb+=yytext;
810 }
811<St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
812 yyextra->token.name="block";
813 BEGIN(St_HtmlOnly);
814 }
815<St_HtmlOnlyOption>.|\n {
816 unput(*yytext);
817 BEGIN(St_HtmlOnly);
818 }
819<St_HtmlOnlyOption>"\\ilinebr" {
820 unput_string(yytext,yyleng);
821 BEGIN(St_HtmlOnly);
822 }
823<St_HtmlOnly>{CMD}"endhtmlonly" {
824 return Token::make_RetVal_OK();
825 }
826<St_HtmlOnly>[^\\@\n$]+ |
827<St_HtmlOnly>\n |
828<St_HtmlOnly>. {
829 lineCount(yytext,yyleng);
830 yyextra->token.verb+=yytext;
831 }
832<St_ManOnly>{CMD}"endmanonly" {
833 return Token::make_RetVal_OK();
834 }
835<St_ManOnly>[^\\@\n$]+ |
836<St_ManOnly>\n |
837<St_ManOnly>. {
838 lineCount(yytext,yyleng);
839 yyextra->token.verb+=yytext;
840 }
841<St_RtfOnly>{CMD}"endrtfonly" {
842 return Token::make_RetVal_OK();
843 }
844<St_RtfOnly>[^\\@\n$]+ |
845<St_RtfOnly>\n |
846<St_RtfOnly>. {
847 lineCount(yytext,yyleng);
848 yyextra->token.verb+=yytext;
849 }
850<St_LatexOnly>{CMD}"endlatexonly" {
851 return Token::make_RetVal_OK();
852 }
853<St_LatexOnly>[^\\@\n]+ |
854<St_LatexOnly>\n |
855<St_LatexOnly>. {
856 lineCount(yytext,yyleng);
857 yyextra->token.verb+=yytext;
858 }
859<St_XmlOnly>{CMD}"endxmlonly" {
860 return Token::make_RetVal_OK();
861 }
862<St_XmlOnly>[^\\@\n]+ |
863<St_XmlOnly>\n |
864<St_XmlOnly>. {
865 lineCount(yytext,yyleng);
866 yyextra->token.verb+=yytext;
867 }
868<St_DbOnly>{CMD}"enddocbookonly" {
869 return Token::make_RetVal_OK();
870 }
871<St_DbOnly>[^\\@\n]+ |
872<St_DbOnly>\n |
873<St_DbOnly>. {
874 lineCount(yytext,yyleng);
875 yyextra->token.verb+=yytext;
876 }
877<St_Verbatim>{CMD}"endverbatim" {
878 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
879 return Token::make_RetVal_OK();
880 }
881<St_ILiteral>{CMD}"endiliteral " { // note extra space as this is artificially added
882 // remove spaces that have been added
883 yyextra->token.verb = yyextra->token.verb.mid(1,yyextra->token.verb.length()-2);
884 return Token::make_RetVal_OK();
885 }
886<St_iVerbatim>{CMD}"endiverbatim" {
887 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
888 return Token::make_RetVal_OK();
889 }
890<St_Verbatim,St_iVerbatim,St_ILiteral>[^\\@\n]+ |
891<St_Verbatim,St_iVerbatim,St_ILiteral>\n |
892<St_Verbatim,St_iVerbatim,St_ILiteral>. { /* Verbatim text */
893 lineCount(yytext,yyleng);
894 yyextra->token.verb+=yytext;
895 }
896<St_ILiteralOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // option(s) present
897 yyextra->token.verb = DString(yytext).stripWhiteSpace();
898 return Token::make_RetVal_OK();
899 }
DString stripWhiteSpace() const
returns a copy of this string with leading and trailing whitespace removed
Definition dstring.h:337
900<St_ILiteralOpt>"\\ilinebr" |
901<St_ILiteralOpt>"\n" |
902<St_ILiteralOpt>. {
903 yyextra->token.sectionId = "";
904 unput_string(yytext,yyleng);
905 return Token::make_RetVal_OK();
906 }
907<St_Dot>{CMD}"enddot" {
908 return Token::make_RetVal_OK();
909 }
910<St_Dot>[^\\@\n]+ |
911<St_Dot>\n |
912<St_Dot>. { /* dot text */
913 lineCount(yytext,yyleng);
914 yyextra->token.verb+=yytext;
915 }
916<St_Msc>{CMD}("endmsc") {
917 return Token::make_RetVal_OK();
918 }
919<St_Msc>[^\\@\n]+ |
920<St_Msc>\n |
921<St_Msc>. { /* msc text */
922 lineCount(yytext,yyleng);
923 yyextra->token.verb+=yytext;
924 }
925<St_PlantUMLOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
926 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
927 return Token::make_RetVal_OK();
928 }
929<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
930 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
931 return Token::make_RetVal_OK();
932 }
933<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
934 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
935 return Token::make_RetVal_OK();
936 }
937<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
938 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
939 return Token::make_RetVal_OK();
940 }
941<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
942 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
943 return Token::make_RetVal_OK();
944 }
945<St_PlantUMLOpt>"\\ilinebr" |
946<St_PlantUMLOpt>"\n" |
947<St_PlantUMLOpt>. {
948 yyextra->token.sectionId = "";
949 unput_string(yytext,yyleng);
950 return Token::make_RetVal_OK();
951 }
952<St_PlantUML>{CMD}"enduml" {
953 return Token::make_RetVal_OK();
954 }
955<St_PlantUML>[^\\@\n]+ |
956<St_PlantUML>\n |
957<St_PlantUML>. { /* plantuml text */
958 lineCount(yytext,yyleng);
959 yyextra->token.verb+=yytext;
960 }
961<St_MermaidOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
962 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
963 return Token::make_RetVal_OK();
964 }
965<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
966 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
967 return Token::make_RetVal_OK();
968 }
969<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
970 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
971 return Token::make_RetVal_OK();
972 }
973<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
974 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
975 return Token::make_RetVal_OK();
976 }
977<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
978 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
979 return Token::make_RetVal_OK();
980 }
981<St_MermaidOpt>"\\ilinebr" |
982<St_MermaidOpt>"\n" |
983<St_MermaidOpt>. {
984 yyextra->token.sectionId = "";
985 unput_string(yytext,yyleng);
986 return Token::make_RetVal_OK();
987 }
988<St_Mermaid>{CMD}"endmermaid" {
989 return Token::make_RetVal_OK();
990 }
991<St_Mermaid>[^\\@\n]+ |
992<St_Mermaid>\n |
993<St_Mermaid>. { /* mermaid text */
994 lineCount(yytext,yyleng);
995 yyextra->token.verb+=yytext;
996 }
997<St_Title>"\"" { // quoted title
998 BEGIN(St_TitleQ);
999 }
1000<St_Title>[ \t]+ {
1001 yyextra->token.chars=yytext;
1002 return Token::make_TK_WHITESPACE();
1003 }
1004<St_Title>. { // non-quoted title
1005 unput(*yytext);
1006 BEGIN(St_TitleN);
1007 }
1008<St_Title>\n {
1009 unput(*yytext);
1010 return Token::make_TK_NONE();
1011 }
1012<St_Title>"\\ilinebr" {
1013 unput_string(yytext,yyleng);
1014 return Token::make_TK_NONE();
1015 }
1016<St_TitleN>"&"{ID}";" { /* symbol */
1017 yyextra->token.name = yytext;
1018 return Token::make_TK_SYMBOL();
1019 }
1020<St_TitleN>{HTMLTAG} {
1021 yyextra->token.name = yytext;
1022 handleHtmlTag(yyscanner,yytext);
1023 return Token::make_TK_HTMLTAG();
1024 }
1025<St_TitleN>\n { /* new line => end of title */
1026 unput(*yytext);
1027 return Token::make_TK_NONE();
1028 }
1029<St_TitleN>"\\ilinebr" { /* new line => end of title */
1030 unput_string(yytext,yyleng);
1031 return Token::make_TK_NONE();
1032 }
1033<St_TitleN>{SPCMD1} |
1034<St_TitleN>{SPCMD2} { /* special command */
1035 yyextra->token.name = yytext+1;
1036 yyextra->token.paramDir=TokenInfo::Unspecified;
1037 return Token::char_to_command(yytext[0]);
1038 }
1039<St_TitleN>{ID}"=" { /* attribute */
1040 if (yytext[0]=='%') // strip % if present
1041 yyextra->token.name = &yytext[1];
1042 else
1043 yyextra->token.name = yytext;
1044 return Token::make_TK_WORD();
1045 }
1046<St_TitleN>[\-+0-9] |
1047<St_TitleN>{WORD1} |
1048<St_TitleN>{WORD2} { /* word */
1049 if (DString(yytext).find("\\ilinebr")!=DString::npos) REJECT; // see issue #8311
1050 lineCount(yytext,yyleng);
1051 if (yytext[0]=='%') // strip % if present
1052 yyextra->token.name = &yytext[1];
1053 else
1054 yyextra->token.name = yytext;
1055 return Token::make_TK_WORD();
1056 }
1057<St_TitleN>[ \t]+ {
1058 yyextra->token.chars=yytext;
1059 return Token::make_TK_WHITESPACE();
1060 }
1061<St_TitleQ>"&"{ID}";" { /* symbol */
1062 yyextra->token.name = yytext;
1063 return Token::make_TK_SYMBOL();
1064 }
1065<St_TitleQ>(\n|"\\ilinebr") { /* new line => end of title */
1066 unput_string(yytext,yyleng);
1067 return Token::make_TK_NONE();
1068 }
1069<St_TitleQ>{SPCMD1} |
1070<St_TitleQ>{SPCMD2} { /* special command */
1071 yyextra->token.name = yytext+1;
1072 yyextra->token.paramDir=TokenInfo::Unspecified;
1073 return Token::char_to_command(yytext[0]);
1074 }
1075<St_TitleQ>{WORD1NQ} |
1076<St_TitleQ>{WORD2NQ} { /* word */
1077 yyextra->token.name = yytext;
1078 return Token::make_TK_WORD();
1079 }
1080<St_TitleQ>[ \t]+ {
1081 yyextra->token.chars=yytext;
1082 return Token::make_TK_WHITESPACE();
1083 }
1084<St_TitleQ>"\"" { /* closing quote => end of title */
1085 BEGIN(St_TitleA);
1086 return Token::make_TK_NONE();
1087 }
1088<St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
1089 yyextra->token.name = yytext;
1090 size_t pos = yyextra->token.name.find('=');
1091 if (pos==DString::npos) pos=0; // should never happen
1092 yyextra->token.name = yyextra->token.name.left(pos).stripWhiteSpace();
1093 BEGIN(St_TitleV);
1094 }
1095<St_TitleV>[^ \t\r\n]+ { // attribute value
1096 lineCount(yytext,yyleng);
1097 yyextra->token.chars = yytext;
1098 BEGIN(St_TitleN);
1099 return Token::make_TK_WORD();
1100 }
1101<St_TitleV,St_TitleA>. {
1102 unput(*yytext);
1103 return Token::make_TK_NONE();
1104 }
1105<St_TitleV,St_TitleA>(\n|"\\ilinebr") {
1106 unput_string(yytext,yyleng);
1107 return Token::make_TK_NONE();
1108 }
1109
1110<St_Anchor>({REQID}|{LABELID}){WS}? { // anchor
1111 lineCount(yytext,yyleng);
1112 yyextra->token.name = DString(yytext).stripWhiteSpace();
1113 return Token::make_TK_WORD();
1114 }
1115<St_Anchor>. {
1116 unput(*yytext);
1117 return Token::make_TK_NONE();
1118 }
1119<St_Cite>{CITEID} { // label to cite
1120 if (yytext[0] =='"')
1121 {
1122 yyextra->token.name=yytext+1;
1123 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-2);
1124 }
1125 else
1126 {
1127 yyextra->token.name=yytext;
1128 }
1129 return Token::make_TK_WORD();
1130 }
1131<St_Cite>{BLANK} { // white space
1132 unput(' ');
1133 return Token::make_TK_NONE();
1134 }
1135<St_Cite>(\n|"\\ilinebr") { // new line
1136 unput_string(yytext,yyleng);
1137 return Token::make_TK_NONE();
1138 }
1139<St_Cite>. { // any other character
1140 unput(*yytext);
1141 return Token::make_TK_NONE();
1142 }
1143<St_DoxyConfig>{DOXYCFG} { // config option
1144 yyextra->token.name=yytext;
1145 return Token::make_TK_WORD();
1146 }
1147<St_DoxyConfig>{BLANK} { // white space
1148 unput(' ');
1149 return Token::make_TK_NONE();
1150 }
1151<St_DoxyConfig>(\n|"\\ilinebr") { // new line
1152 unput_string(yytext,yyleng);
1153 return Token::make_TK_NONE();
1154 }
1155<St_DoxyConfig>. { // any other character
1156 unput(*yytext);
1157 return Token::make_TK_NONE();
1158 }
1159<St_Ref>{REFWORD_NOCV}/{BLANK}("const")[a-z_A-Z0-9] { // see bug776988
1160 yyextra->token.name=yytext;
1161 return Token::make_TK_WORD();
1162 }
1163<St_Ref>{REFWORD_NOCV}/{BLANK}("volatile")[a-z_A-Z0-9] { // see bug776988
1164 yyextra->token.name=yytext;
1165 return Token::make_TK_WORD();
1166 }
1167<St_Ref>{REFWORD} { // label to refer to
1168 yyextra->token.name=yytext;
1169 return Token::make_TK_WORD();
1170 }
1171<St_Ref>{BLANK} { // white space
1172 unput(' ');
1173 return Token::make_TK_NONE();
1174 }
1175<St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
1176 yyextra->expectQuote=true;
1177 lineCount(yytext,yyleng);
1178 BEGIN(St_Ref2);
1179 }
1180<St_Ref>(\n|"\\ilinebr") { // new line
1181 unput_string(yytext,yyleng);
1182 return Token::make_TK_NONE();
1183 }
1184<St_Ref>"\""[^"\n]+"\"" { // quoted first argument -> return without quotes
1185 yyextra->token.name=DString(yytext).mid(1,yyleng-2);
1186 return Token::make_TK_WORD();
1187 }
1188<St_Ref>. { // any other character
1189 unput(*yytext);
1190 return Token::make_TK_NONE();
1191 }
1192<St_IntRef>[A-Z_a-z0-9.:/#\-\+\‍(\‍)]+ {
1193 yyextra->token.name = yytext;
1194 return Token::make_TK_WORD();
1195 }
1196<St_IntRef>{BLANK}+"\"" {
1197 BEGIN(St_Ref2);
1198 }
1199<St_SetScope>({SCOPEMASK}|{ANONNS}){BLANK}|{FILEMASK} {
1200 yyextra->token.name = yytext;
1201 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1202 return Token::make_TK_WORD();
1203 }
1204<St_SetScope>{SCOPEMASK}"<" {
1205 yyextra->token.name = yytext;
1206 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1207 yyextra->sharpCount=1;
1208 BEGIN(St_SetScopeEnd);
1209 }
1210<St_SetScope>{BLANK} {
1211 }
1212<St_SetScopeEnd>"<" {
1213 yyextra->token.name += yytext;
1214 yyextra->sharpCount++;
1215 }
1216<St_SetScopeEnd>">" {
1217 yyextra->token.name += yytext;
1218 yyextra->sharpCount--;
1219 if (yyextra->sharpCount<=0)
1220 {
1221 return Token::make_TK_WORD();
1222 }
1223 }
1224<St_SetScopeEnd>. {
1225 yyextra->token.name += yytext;
1226 }
1227<St_Ref2>"&"{ID}";" { /* symbol */
1228 yyextra->token.name = yytext;
1229 return Token::make_TK_SYMBOL();
1230 }
1231<St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title? */
1232 lineCount(yytext,yyleng);
1233 if (!yyextra->expectQuote || yytext[0]=='"')
1234 {
1235 return Token::make_TK_NONE();
1236 }
1237 else
1238 {
1239 yyextra->token.name += yytext;
1240 }
1241 }
1242<St_Ref2>{HTMLTAG_STRICT} { /* html tag */
1243 lineCount(yytext,yyleng);
1244 handleHtmlTag(yyscanner,yytext);
1245 return Token::make_TK_HTMLTAG();
1246 }
1247<St_Ref2>{SPCMD1} |
1248<St_Ref2>{SPCMD2} { /* special command */
1249 yyextra->token.name = yytext+1;
1250 yyextra->token.paramDir=TokenInfo::Unspecified;
1251 return Token::char_to_command(yytext[0]);
1252 }
1253<St_Ref2>{WORD1NQ} |
1254<St_Ref2>{WORD2NQ} {
1255 /* word */
1256 yyextra->token.name = yytext;
1257 return Token::make_TK_WORD();
1258 }
1259<St_Ref2>[ \t]+ {
1260 yyextra->token.chars=yytext;
1261 return Token::make_TK_WHITESPACE();
1262 }
1263<St_XRefItem>{LABELID} {
1264 yyextra->token.name=yytext;
1265 }
1266<St_XRefItem>" " {
1267 BEGIN(St_XRefItem2);
1268 }
1269<St_XRefItem2>[0-9]+"." {
1270 DString numStr(yytext);
1271 numStr=numStr.left((int)yyleng-1);
1272 yyextra->token.id=numStr.toInt();
1273 return Token::make_RetVal_OK();
1274 }
1275<St_Para,St_Title,St_Ref2>"<!--" { /* html style comment block */
1276 yyextra->commentState = YY_START;
1277 BEGIN(St_Comment);
1278 }
1279<St_Param>"\""[^\n\"]+"\"" {
1280 yyextra->token.name = yytext+1;
1281 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1282 return Token::make_TK_WORD();
1283 }
1284<St_Param>({PHPTYPE}{BLANK}*("["{BLANK}*"]")*{BLANK}*"|"{BLANK}*)*{PHPTYPE}{BLANK}*("["{BLANK}*"]")*{WS}+("&")?"$"{LABELID} {
1285 lineCount(yytext,yyleng);
1286 DString params(yytext);
1287 size_t j = params.find('&');
1288 size_t i = params.find('$');
1289 if (i==DString::npos) i=0; // should never happen
1290 if (j!=DString::npos && j<i) i=j;
1291 DString types = params.left(i).stripWhiteSpace();
1292 yyextra->token.name = types+"#"+params.mid(i);
1293 return Token::make_TK_WORD();
1294 }
DString left(size_t len) const
Definition dstring.h:306
1295<St_Param>[^ \t\n,@\\‍]+ {
1296 yyextra->token.name = yytext;
1297 if (yyextra->token.name.at(static_cast<uint32_t>(yyleng)-1)==':')
1298 {
1299 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-1);
1300 }
1301 return Token::make_TK_WORD();
1302 }
1303<St_Param>{WS}*","{WS}* /* param separator */
1304<St_Param>{WS} {
1305 lineCount(yytext,yyleng);
1306 yyextra->token.chars=yytext;
1307 return Token::make_TK_WHITESPACE();
1308 }
1309<St_Prefix>"\""[^\n\"]*"\"" {
1310 yyextra->token.name = yytext+1;
1311 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1312 return Token::make_TK_WORD();
1313 }
1314<St_Prefix>. {
1315 unput(*yytext);
1316 return Token::make_TK_NONE();
1317 }
1318<St_Options>{ID} {
1319 yyextra->token.name+=yytext;
1320 }
1321<St_Options>{WS}*":"{WS}* {
1322 lineCount(yytext,yyleng);
1323 yyextra->token.name+=":";
1324 }
1325<St_Options>{WS}*","{WS}* |
1326<St_Options>{WS} { /* option separator */
1327 lineCount(yytext,yyleng);
1328 yyextra->token.name+=",";
1329 }
1330<St_Options>"}" {
1331 return Token::make_TK_WORD();
1332 }
1333<St_Block>{ID} {
1334 yyextra->token.name+=yytext;
1335 }
1336<St_Block>"]" {
1337 return Token::make_TK_WORD();
1338 }
1339<St_Emoji>[:0-9_a-z+-]+ {
1340 yyextra->token.name=yytext;
1341 return Token::make_TK_WORD();
1342 }
1343<St_Emoji>. {
1344 unput(*yytext);
1345 return Token::make_TK_NONE();
1346 }
1347<St_QuotedString>"\"" {
1348 yyextra->token.name="";
1349 BEGIN(St_QuotedContent);
1350 }
1351<St_QuotedString>(\n|"\\ilinebr") {
1352 unput_string(yytext,yyleng);
1353 return Token::make_TK_NONE();
1354 }
1355<St_QuotedString>. {
1356 unput(*yytext);
1357 return Token::make_TK_NONE();
1358 }
1359<St_QuotedContent>"\"" {
1360 return Token::make_TK_WORD();
1361 }
1362<St_QuotedContent>. {
1363 yyextra->token.name+=yytext;
1364 }
1365<St_ShowDate>{WS}+{SHOWDATE} {
1366 lineCount(yytext,yyleng);
1367 yyextra->token.name=yytext;
1368 return Token::make_TK_WORD();
1369 }
1370<St_ShowDate>(\n|"\\ilinebr") {
1371 unput_string(yytext,yyleng);
1372 return Token::make_TK_NONE();
1373 }
1374<St_ShowDate>. {
1375 unput(*yytext);
1376 return Token::make_TK_NONE();
1377 }
1378<St_ILine>{LINENR}/[\\@\n\.] |
1379<St_ILine>{LINENR}{BLANK} {
1380 bool ok = false;
1381 int nr = DString(yytext).toInt(&ok);
1382 if (!ok)
1383 {
1384 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '{}' for iline command",yytext);
1385 }
1386 else
1387 {
1388 yyextra->yyLineNr = nr;
1389 }
1390 return Token::make_TK_WORD();
1391 }
1392<St_ILine>. {
1393 return Token::make_TK_NONE();
1394 }
1395<St_IFile>{BLANK}*{FILEMASK} {
1396 DString text(yytext);
1397 text = text.stripWhiteSpace();
1398 yyextra->fileName = text;
1399 yyextra->token.name = text;
1400 return Token::make_TK_WORD();
1401 }
1402<St_IFile>{BLANK}*"\""[^\n\"]+"\"" {
1403 DString text(yytext);
1404 text = text.stripWhiteSpace();
1405 yyextra->fileName = text.mid(1,text.length()-2);
1406 yyextra->token.name = text.mid(1,text.length()-2);
1407 return Token::make_TK_WORD();
1408 }
1409<St_File>{FILEMASK} {
1410 yyextra->token.name = yytext;
1411 if (yyextra->token.name.endsWith("\\ilinebr") ||yyextra->token.name.endsWith("@ilinebr"))
1412 {
1413 unput_string("\\ilinebr",8);
1414 yyextra->token.name = yyextra->token.name.left(yyleng-8);
1415 }
1416 return Token::make_TK_WORD();
1417 }
1418<St_File>"\""[^\n\"]+"\"" {
1419 DString text(yytext);
1420 yyextra->token.name = text.mid(1,text.length()-2);
1421 return Token::make_TK_WORD();
1422 }
1423<St_Pattern>[^\\\r\n]+ {
1424 yyextra->token.name += yytext;
1425 }
1426<St_Pattern>"\\ilinebr" {
1427 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1428 return Token::make_TK_WORD();
1429 }
1430<St_Pattern>\n {
1431 lineCount(yytext,yyleng);
1432 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1433 return Token::make_TK_WORD();
1434 }
1435<St_Pattern>. {
1436 yyextra->token.name += yytext;
1437 }
1438<St_Link>{LINKMASK}|{REFWORD} {
1439 yyextra->token.name = yytext;
1440 return Token::make_TK_WORD();
1441 }
1442<St_Comment>"-->" { /* end of html comment */
1443 BEGIN(yyextra->commentState);
1444 }
1445<St_Comment>[^-]+ /* inside html comment */
1446<St_Comment>. /* inside html comment */
1447
1448 /* State for skipping title (all chars until the end of the line) */
1449
1450<St_SkipTitle>.
1451<St_SkipTitle>(\n|"\\ilinebr") {
1452 if (*yytext == '\n') unput('\n');
1453 return Token::make_TK_NONE();
1454 }
1455
1456 /* State for the pass used to find the anchors and sections */
1457
1458<St_Sections>[^\n@\<]+
1459<St_Sections>{CMD}("<"|{CMD})
1460<St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
1461 lineCount(yytext,yyleng);
1462 DString tag(yytext);
1463 size_t s=tag.find("id=");
1464 if (s!=DString::npos) // command has id attribute
1465 {
1466 char c=tag[s+3];
1467 if (c=='\'' || c=='"') // valid start
1468 {
1469 size_t e=tag.find(c,s+4);
1470 if (e!=DString::npos) // found matching end
1471 {
1472 yyextra->secType = SectionType::Table;
1473 yyextra->secLabel=tag.mid(s+4,e-s-4); // extract id
1474 processSection(yyscanner);
1475 }
1476 }
1477 }
1478 }
static constexpr int Table
Definition section.h:41
static void processSection(yyscan_t yyscanner)
1479<St_Sections>{CMD}"anchor"{BLANK}+ {
1480 yyextra->secType = SectionType::Anchor;
1481 BEGIN(St_SecLabel1);
1482 }
static constexpr int Anchor
Definition section.h:40
1483<St_Sections>{CMD}"ianchor"{BLANK}+ {
1484 yyextra->secType = SectionType::Anchor;
1485 BEGIN(St_SecLabel1);
1486 }
1487<St_Sections>{CMD}"section"{BLANK}+ {
1488 yyextra->secType = SectionType::Section;
1489 BEGIN(St_SecLabel2);
1490 }
static constexpr int Section
Definition section.h:33
1491<St_Sections>{CMD}"subsection"{BLANK}+ {
1492 yyextra->secType = SectionType::Subsection;
1493 BEGIN(St_SecLabel2);
1494 }
static constexpr int Subsection
Definition section.h:34
1495<St_Sections>{CMD}"subsubsection"{BLANK}+ {
1496 yyextra->secType = SectionType::Subsubsection;
1497 BEGIN(St_SecLabel2);
1498 }
static constexpr int Subsubsection
Definition section.h:35
1499<St_Sections>{CMD}"paragraph"{BLANK}+ {
1500 yyextra->secType = SectionType::Paragraph;
1501 BEGIN(St_SecLabel2);
1502 }
static constexpr int Paragraph
Definition section.h:36
1503<St_Sections>{CMD}"subparagraph"{BLANK}+ {
1504 yyextra->secType = SectionType::Subparagraph;
1505 BEGIN(St_SecLabel2);
1506 }
static constexpr int Subparagraph
Definition section.h:37
1507<St_Sections>{CMD}"subsubparagraph"{BLANK}+ {
1508 yyextra->secType = SectionType::Subsubparagraph;
1509 BEGIN(St_SecLabel2);
1510 }
static constexpr int Subsubparagraph
Definition section.h:38
1511<St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9] {
1512 yyextra->endMarker="endverbatim";
1513 BEGIN(St_SecSkip);
1514 }
1515<St_Sections>{CMD}"iverbatim"/[^a-z_A-Z0-9] {
1516 yyextra->endMarker="endiverbatim";
1517 BEGIN(St_SecSkip);
1518 }
1519<St_Sections>{CMD}"iliteral"/[^a-z_A-Z0-9] {
1520 yyextra->endMarker="endiliteral";
1521 BEGIN(St_SecSkip);
1522 }
1523<St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1524 yyextra->endMarker="enddot";
1525 BEGIN(St_SecSkip);
1526 }
1527<St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1528 yyextra->endMarker="endmsc";
1529 BEGIN(St_SecSkip);
1530 }
1531<St_Sections>{CMD}"mermaid"/[^a-z_A-Z0-9] {
1532 yyextra->endMarker="endmermaid";
1533 BEGIN(St_SecSkip);
1534 }
1535<St_Sections>{CMD}"startuml"/[^a-z_A-Z0-9] {
1536 yyextra->endMarker="enduml";
1537 BEGIN(St_SecSkip);
1538 }
1539<St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1540 yyextra->endMarker="endhtmlonly";
1541 BEGIN(St_SecSkip);
1542 }
1543<St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1544 yyextra->endMarker="endlatexonly";
1545 BEGIN(St_SecSkip);
1546 }
1547<St_Sections>{CMD}"manonly"/[^a-z_A-Z0-9] {
1548 yyextra->endMarker="endmanonly";
1549 BEGIN(St_SecSkip);
1550 }
1551<St_Sections>{CMD}"rtfonly"/[^a-z_A-Z0-9] {
1552 yyextra->endMarker="endrtfonly";
1553 BEGIN(St_SecSkip);
1554 }
1555<St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1556 yyextra->endMarker="endxmlonly";
1557 BEGIN(St_SecSkip);
1558 }
1559<St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
1560 yyextra->endMarker="enddocbookonly";
1561 BEGIN(St_SecSkip);
1562 }
1563<St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1564 yyextra->endMarker="endcode";
1565 BEGIN(St_SecSkip);
1566 }
1567<St_Sections>{CMD}"icode"/[^a-z_A-Z0-9] {
1568 yyextra->endMarker="endicode";
1569 BEGIN(St_SecSkip);
1570 }
1571<St_Sections>"<!--" {
1572 yyextra->endMarker="-->";
1573 BEGIN(St_SecSkip);
1574 }
1575<St_SecSkip>{CMD}{ID} {
1576 if (yyextra->endMarker==yytext+1)
1577 {
1578 BEGIN(St_Sections);
1579 }
1580 }
1581<St_SecSkip>"-->" {
1582 if (yyextra->endMarker==yytext)
1583 {
1584 BEGIN(St_Sections);
1585 }
1586 }
1587<St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1588<St_SecSkip>.
1589<St_SecSkip>(\n|"\\ilinebr")
1590<St_Sections>.
1591<St_Sections>(\n|"\\ilinebr")
1592<St_SecLabel1>({REQID}|{LABELID}) {
1593 lineCount(yytext,yyleng);
1594 yyextra->secLabel = yytext;
1595 processSection(yyscanner);
1596 BEGIN(St_Sections);
1597 }
1598<St_SecLabel2>({REQID}|{LABELID}){BLANK}+ |
1599<St_SecLabel2>({REQID}|{LABELID}) {
1600 yyextra->secLabel = yytext;
1601 yyextra->secLabel = yyextra->secLabel.stripWhiteSpace();
1602 BEGIN(St_SecTitle);
1603 }
1604<St_SecTitle>[^\n]+ |
1605<St_SecTitle>[^\n]*\n {
1606 lineCount(yytext,yyleng);
1607 yyextra->secTitle = yytext;
1608 yyextra->secTitle = yyextra->secTitle.stripWhiteSpace();
1609 if (yyextra->secTitle.endsWith("\\ilinebr"))
1610 {
1611 yyextra->secTitle.left(yyextra->secTitle.length()-8);
1612 }
1613 processSection(yyscanner);
1614 BEGIN(St_Sections);
1615 }
1616<St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1617 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}' while looking for section label or title",yytext);
1618 }
1619
1620<St_Snippet>[^\\\n]+ {
1621 yyextra->token.name += yytext;
1622 }
1623<St_Snippet>"\\" {
1624 yyextra->token.name += yytext;
1625 }
1626<St_Snippet>(\n|"\\ilinebr") {
1627 if (yyextra->token.name.stripWhiteSpace().empty() && !yyextra->snippetRtn)
1628 {
1629 lineCount(yytext,yyleng);
1630 yyextra->snippetRtn++;
1631 }
1632 else
1633 {
1634 unput_string(yytext,yyleng);
1635 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1636 return Token::make_TK_WORD();
1637 }
1638 }
1639
1640 /* Generic rules that work for all states */
1641<*>\n {
1642 lineCount(yytext,yyleng);
1643 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected new line character");
1644 }
1645<*>"\\ilinebr" {
1646 }
1647<*>[\\@<>&$#%~"=] { /* unescaped special character */
1648 //warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}', assuming command \\{} was meant.",yytext,yytext);
1649 yyextra->token.name = yytext;
1650 return Token::char_to_command(yytext[0]);
1651 }
1652<*>. {
1653 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}'",yytext);
1654 }
1655%%

◆ yyread()

int yyread ( yyscan_t yyscanner,
char * buf,
int max_size )
static

Definition at line 1659 of file doctokenizer.l.

1660{
1661 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1662 int c=0;
1663 const char *p = yyextra->inputString + yyextra->inputPos;
1664 while ( c < max_size && *p ) { *buf++ = *p++; c++; }
1665 yyextra->inputPos+=c;
1666 return c;
1667}