Doxygen
Loading...
Searching...
No Matches
doctokenizer.l File Reference
#include <stdint.h>
#include "doctokenizer.h"
#include <cctype>
#include <stack>
#include <string>
#include "cmdmapper.h"
#include "config.h"
#include "debug.h"
#include "definition.h"
#include "docnode.h"
#include "message.h"
#include "portable.h"
#include "regex.h"
#include "section.h"
#include "stringutil.h"
#include "doxygen_lex.h"
#include "doctokenizer.l.h"
Include dependency graph for doctokenizer.l:

Go to the source code of this file.

Classes

struct  DocLexerContext
struct  doctokenizerYY_state
struct  DocTokenizer::Private

Macros

#define YY_TYPEDEF_YY_SCANNER_T
#define yyguts_t   doctokenizerYYguts_t
#define YY_NO_INPUT   1
#define YY_NO_UNISTD_H   1
#define lineCount(s, len)
#define unput_string(yytext, yyleng)
#define YY_INPUT(buf, result, max_size)
#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)
#define yyterminate()

Typedefs

typedef yyguts_t * yyscan_t

Functions

static const char * stateToString (int state)
static int yyread (yyscan_t yyscanner, char *buf, int max_size)
static void handleHtmlTag (yyscan_t yyscanner, const char *text)
static void processSection (yyscan_t yyscanner)
DString extractPartAfterNewLine (const DString &text)
static int computeIndent (const char *str, size_t length)
static const char * getLexerFILE ()
int yylex (yyscan_t yyscanner)

Macro Definition Documentation

◆ lineCount

#define lineCount ( s,
len )
Value:
do { for(int i=0;i<(int)len;i++) if (s[i]=='\n') yyextra->yyLineNr++; } while(0)

Definition at line 100 of file doctokenizer.l.

Referenced by endBrief(), initMethodProtection(), and CitationManager::insertCrossReferencesForBibFile().

◆ unput_string

#define unput_string ( yytext,
yyleng )
Value:
do { for (int i=(int)yyleng-1;i>=0;i--) unput(yytext[i]); } while(0)

Definition at line 162 of file doctokenizer.l.

◆ YY_DECL

#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)

Definition at line 173 of file doctokenizer.l.

◆ YY_INPUT

#define YY_INPUT ( buf,
result,
max_size )
Value:
result=yyread(yyscanner,buf,max_size);
static int yyread(yyscan_t yyscanner, char *buf, int max_size)

Definition at line 166 of file doctokenizer.l.

◆ YY_NO_INPUT

#define YY_NO_INPUT   1

Definition at line 54 of file doctokenizer.l.

◆ YY_NO_UNISTD_H

#define YY_NO_UNISTD_H   1

Definition at line 55 of file doctokenizer.l.

◆ YY_TYPEDEF_YY_SCANNER_T

#define YY_TYPEDEF_YY_SCANNER_T

Definition at line 26 of file doctokenizer.l.

◆ yyguts_t

#define yyguts_t   doctokenizerYYguts_t

Definition at line 27 of file doctokenizer.l.

◆ yyterminate

#define yyterminate ( )
Value:
return Token::make_TK_EOF()

Definition at line 176 of file doctokenizer.l.

Typedef Documentation

◆ yyscan_t

typedef yyguts_t* yyscan_t

Definition at line 29 of file doctokenizer.l.

Function Documentation

◆ computeIndent()

int computeIndent ( const char * str,
size_t length )
static

Definition at line 129 of file doctokenizer.l.

130{
131 if (str==0 || length==std::string::npos) return 0;
132 size_t i;
133 int indent=0;
134 int tabSize=Config_getInt(TAB_SIZE);
135 for (i=0;i<length;i++)
136 {
137 if (str[i]=='\t')
138 {
139 indent+=tabSize - (indent%tabSize);
140 }
141 else if (str[i]=='\n')
142 {
143 indent=0;
144 }
145 else if (literal_at(&str[i],"\\ilinebr"))
146 {
147 indent=0;
148 i+=7;
149 if (str[i+1]==' ') i++; // also eat space after \\ilinebr if present
150 }
151 else
152 {
153 indent++;
154 }
155 }
156 //printf("input('%s')=%d\n",str,indent);
157 return indent;
#define Config_getInt(name)
Definition config.h:34
bool literal_at(const char *data, const char(&str)[N])
returns true iff data points to a substring that matches string literal str
Definition stringutil.h:101
158}

References Config_getInt, and literal_at().

◆ extractPartAfterNewLine()

DString extractPartAfterNewLine ( const DString & text)

Definition at line 111 of file doctokenizer.l.

112{
113 size_t nl1 = text.find('\n');
114 size_t nl2 = text.find("\\ilinebr");
115 if (nl1!=DString::npos && nl2!=DString::npos && nl1<nl2)
116 {
117 return text.mid(nl1+1);
118 }
119 if (nl2!=DString::npos)
120 {
121 if (text.at(nl2+8)==' ') nl2++; // skip space after \\ilinebr
122 return text.mid(nl2+8);
123 }
124 return text;
DString mid(size_t index, size_t len=npos) const
Definition dstring.h:318
static constexpr size_t npos
value used to indicate 'not found' or 'to the end of the string', matching std::string::npos
Definition dstring.h:178
char & at(size_t i)
Returns a reference to the character at index i.
Definition dstring.h:686
size_t find(char c, size_t pos=0) const
Definition dstring.h:239
125}

References DString::at(), DString::find(), DString::mid(), and DString::npos.

◆ getLexerFILE()

const char * getLexerFILE ( )
inlinestatic

Definition at line 169 of file doctokenizer.l.

169{return __FILE__;}

◆ handleHtmlTag()

void handleHtmlTag ( yyscan_t yyscanner,
const char * text )
static

Definition at line 1691 of file doctokenizer.l.

1692{
1693 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
#define yyguts_t
Definition code.l:23
1694
1695 DString tagText(text);
1696 yyextra->token.text = tagText;
1697 yyextra->token.attribs.clear();
1698 yyextra->token.endTag = false;
1699 yyextra->token.emptyTag = false;
A String class for use with Doxygen wrapping std::string and adding some additional functionality off...
Definition dstring.h:84
1700
1701 // Check for end tag
1702 int startNamePos=1;
1703 if (tagText.at(1)=='/')
1704 {
1705 yyextra->token.endTag = true;
1706 startNamePos++;
1707 }
1708
1709 // Parse the name portion
1710 int i = startNamePos;
1711 for (i=startNamePos; i < (int)yyleng; i++)
1712 {
1713 // Check for valid HTML/XML name chars (including namespaces)
1714 char c = tagText.at(i);
1715 if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
1716 }
1717 yyextra->token.name = tagText.mid(startNamePos,i-startNamePos);
1718
1719 // Parse the attributes. Each attribute is a name, value pair
1720 // The result is stored in yyextra->token.attribs.
1721 int startAttribList = i;
1722 while (i<(int)yyleng)
1723 {
1724 char c=tagText.at(i);
1725 // skip spaces
1726 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1727 // check for end of the tag
1728 if (c == '>') break;
1729 // Check for XML style "empty" tag.
1730 if (c == '/')
1731 {
1732 yyextra->token.emptyTag = true;
1733 break;
1734 }
1735 int startName=i;
1736 // search for end of name
1737 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='=' && c!= '>') { c=tagText.at(++i); }
1738 int endName=i;
1739 DString optName,optValue;
1740 optName = tagText.mid(startName,endName-startName).lower();
1741 // skip spaces
1742 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1743 if (tagText.at(i)=='=') // option has value
1744 {
1745 int startAttrib=0, endAttrib=0;
1746 c=tagText.at(++i);
1747 // skip spaces
1748 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1749 if (tagText.at(i)=='\'') // option '...'
1750 {
1751 c=tagText.at(++i);
1752 startAttrib=i;
DString lower() const
Definition dstring.h:326
1753
1754 // search for matching quote
1755 while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
1756 endAttrib=i;
1757 if (i<(int)yyleng) { c=tagText.at(++i);}
1758 }
1759 else if (tagText.at(i)=='"') // option "..."
1760 {
1761 c=tagText.at(++i);
1762 startAttrib=i;
1763 // search for matching quote
1764 while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
1765 endAttrib=i;
1766 if (i<(int)yyleng) { c=tagText.at(++i);}
1767 }
1768 else // value without any quotes
1769 {
1770 startAttrib=i;
1771 // search for separator or end symbol
1772 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='>') { c=tagText.at(++i); }
1773 endAttrib=i;
1774 if (i<(int)yyleng) { c=tagText.at(++i);}
1775 }
1776 optValue = tagText.mid(startAttrib,endAttrib-startAttrib);
1777 if (optName == "align") optValue = optValue.lower();
1778 else if (optName == "valign")
1779 {
1780 optValue = optValue.lower();
1781 if (optValue == "center") optValue="middle";
1782 }
1783 }
1784 else // start next option
1785 {
1786 }
1787 //printf("=====> Adding option name=<%s> value=<%s>\n",
1788 // qPrint(optName),qPrint(optValue));
1789 yyextra->token.attribs.emplace_back(optName,optValue);
1790 }
1791 yyextra->token.attribsStr = tagText.mid(startAttribList,i-startAttribList);
1792}

References DString::at(), DString::clear(), DString::lower(), DString::mid(), and yyguts_t.

◆ processSection()

void processSection ( yyscan_t yyscanner)
static

Definition at line 1670 of file doctokenizer.l.

1671{
1672 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1673 //printf("%s: found section/anchor with name '%s'\n",qPrint(yyextra->fileName),qPrint(yyextra->secLabel));
1674 DString file;
1675 if (yyextra->definition)
1676 {
1677 file = yyextra->definition->getOutputFileBase();
1678 }
1679 else
1680 {
1681 warn(yyextra->fileName,yyextra->yyLineNr,"Found section/anchor {} without context",yyextra->secLabel);
1682 }
1683 SectionInfo *si = SectionManager::instance().find(yyextra->secLabel);
1684 if (si)
1685 {
1686 si->setFileName(file);
1687 si->setType(yyextra->secType);
1688 }
const T * find(const std::string &key) const
Definition linkedmap.h:47
class that provide information about a section.
Definition section.h:58
void setType(SectionType t)
Definition section.h:81
void setFileName(const DString &fn)
Definition section.h:80
static SectionManager & instance()
returns a reference to the singleton
Definition section.h:179
#define warn(file, line, fmt,...)
Definition message.h:97
1689}

References LinkedMap< T, Hash, KeyEqual, Map >::find(), SectionManager::instance(), SectionInfo::setFileName(), SectionInfo::setType(), warn, and yyguts_t.

◆ stateToString()

const char * stateToString ( int state)
static

◆ yylex()

int yylex ( yyscan_t yyscanner)

Definition at line 342 of file doctokenizer.l.

344 {LISTITEM} { /* list item */
345 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
346 lineCount(yytext,yyleng);
347 DString text(yytext);
348 size_t dashPos = text.rfind('-');
349 ASSERT(dashPos!=DString::npos);
350 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
351 yyextra->token.isCheckedList = false;
352 yyextra->token.id = -1;
353 yyextra->token.indent = computeIndent(yytext,dashPos);
354 return Token::make_TK_LISTITEM();
355 }
#define lineCount(s, len)
static int computeIndent(const char *str, size_t length)
#define ASSERT(x)
Definition message.h:142
356<St_Para>^{CLISTITEM} { /* checkbox item */
357 DString text=yytext;
358 size_t dashPos = text.rfind('-');
359 yyextra->token.isEnumList = false;
360 yyextra->token.isCheckedList = true;
361 if (text.find('x') != DString::npos) yyextra->token.id = DocAutoList::Checked_x;
362 else if (text.find('X') != DString::npos) yyextra->token.id = DocAutoList::Checked_X;
363 else yyextra->token.id = DocAutoList::Unchecked;
364 yyextra->token.indent = computeIndent(yytext,dashPos);
365 return Token::make_TK_LISTITEM();
366 }
size_t rfind(char c, size_t pos=npos) const
Definition dstring.h:244
367<St_Para>^{MLISTITEM} { /* list item */
368 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
369 {
370 REJECT;
371 }
372 else
373 {
374 lineCount(yytext,yyleng);
375 std::string text(yytext);
376 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
378 reg::search(text,match,re);
379 size_t listPos = match.position();
380 ASSERT(listPos!=std::string::npos);
381 yyextra->token.isEnumList = false;
382 yyextra->token.isCheckedList = false;
383 yyextra->token.id = -1;
384 yyextra->token.indent = computeIndent(yytext,listPos);
385 return Token::make_TK_LISTITEM();
386 }
387 }
Class representing a regular expression.
Definition regex.h:39
Object representing the matching results.
Definition regex.h:160
bool search(std::string_view str, Match &match, const Ex &re, size_t pos)
Search in a given string str starting at position pos for a match against regular expression re.
Definition regex.cpp:850
bool match(std::string_view str, Match &match, const Ex &re)
Matches a given string str for a match against regular expression re.
Definition regex.cpp:861
388<St_Para>^{OLISTITEM} { /* numbered list item */
389 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
390 {
391 REJECT;
392 }
393 else
394 {
395 std::string text(yytext);
396 static const reg::Ex re(R"(\d+)");
398 reg::search(text,match,re);
399 size_t markPos = match.position();
400 ASSERT(markPos!=std::string::npos);
401 yyextra->token.isEnumList = true;
402 yyextra->token.isCheckedList = false;
403 bool ok = false;
404 int id = DString(match.str()).toInt(&ok);
405 yyextra->token.id = ok ? id : -1;
406 if (!ok)
407 {
408 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '{}' ",match.str());
409 }
410 yyextra->token.indent = computeIndent(yytext,markPos);
411 return Token::make_TK_LISTITEM();
412 }
413 }
int toInt(bool *ok=nullptr, int base=10) const
Definition dstring.cpp:191
414<St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM} { /* list item on next line */
415 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
416 lineCount(yytext,yyleng);
417 DString text=extractPartAfterNewLine(yytext);
418 size_t dashPos = text.rfind('-');
419 ASSERT(dashPos!=DString::npos);
420 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
421 yyextra->token.isCheckedList = false;
422 yyextra->token.id = -1;
423 yyextra->token.indent = computeIndent(text.data(),dashPos);
424 return Token::make_TK_LISTITEM();
425 }
const char * data() const
Returns a pointer to the contents of the string in the form of a 0-terminated C string.
Definition dstring.h:157
DString extractPartAfterNewLine(const DString &text)
426<St_Para>{BLANK}*\n{CLISTITEM} { /* checkbox item on next line */
427 DString text=yytext;
428 text=text.mid(text.find('\n')+1);
429 lineCount(yytext,yyleng);
430 size_t dashPos = text.rfind('-');
431 yyextra->token.isEnumList = false;
432 yyextra->token.isCheckedList = true;
433 if (text.find('x') != DString::npos) yyextra->token.id = DocAutoList::Checked_x;
434 else if (text.find('X') != DString::npos) yyextra->token.id = DocAutoList::Checked_X;
435 else yyextra->token.id = DocAutoList::Unchecked;
436 yyextra->token.indent = computeIndent(text.data(),dashPos);
437 return Token::make_TK_LISTITEM();
438 }
439<St_Para>{BLANK}*(\n|"\\ilinebr"){MLISTITEM} { /* list item on next line */
440 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
441 {
442 REJECT;
443 }
444 else
445 {
446 lineCount(yytext,yyleng);
447 std::string text=extractPartAfterNewLine(yytext).str();
448 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
450 reg::search(text,match,re);
451 size_t markPos = match.position();
452 ASSERT(markPos!=std::string::npos);
453 yyextra->token.isEnumList = false;
454 yyextra->token.isCheckedList = false;
455 yyextra->token.id = -1;
456 yyextra->token.indent = computeIndent(text.c_str(),markPos);
457 return Token::make_TK_LISTITEM();
458 }
459 }
const std::string & str() const
Definition dstring.h:645
460<St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM} { /* list item on next line */
461 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
462 {
463 REJECT;
464 }
465 else
466 {
467 lineCount(yytext,yyleng);
468 std::string text=extractPartAfterNewLine(yytext).str();
469 static const reg::Ex re(R"(\d+)");
471 reg::search(text,match,re);
472 size_t markPos = match.position();
473 ASSERT(markPos!=std::string::npos);
474 yyextra->token.isEnumList = true;
475 yyextra->token.isCheckedList = false;
476 bool ok = false;
477 int id = DString(match.str()).toInt(&ok);
478 yyextra->token.id = ok ? id : -1;
479 if (!ok)
480 {
481 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '{}' ",match.str());
482 }
483 yyextra->token.indent = computeIndent(text.c_str(),markPos);
484 return Token::make_TK_LISTITEM();
485 }
486 }
487<St_Para>^{ENDLIST} { /* end list */
488 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
489 lineCount(yytext,yyleng);
490 size_t dotPos = DString(yytext).rfind('.');
491 yyextra->token.indent = computeIndent(yytext,dotPos);
492 return Token::make_TK_ENDLIST();
493 }
494<St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST} { /* end list on next line */
495 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
496 lineCount(yytext,yyleng);
497 DString text=extractPartAfterNewLine(yytext);
498 size_t dotPos = text.rfind('.');
499 yyextra->token.indent = computeIndent(text.data(),dotPos);
500 return Token::make_TK_ENDLIST();
501 }
502<St_Para>"{"{BLANK}*"@linkplain"/{WS}+ {
503 yyextra->token.name = "javalinkplain";
504 return Token::make_TK_COMMAND_AT();
505 }
506<St_Para>"{"{BLANK}*"@link"/{WS}+ {
507 yyextra->token.name = "javalink";
508 return Token::make_TK_COMMAND_AT();
509 }
510<St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
511 yyextra->token.name = "inheritdoc";
512 return Token::make_TK_COMMAND_AT();
513 }
514<St_Para>"@_fakenl" { // artificial new line
515 //yyextra->yyLineNr++;
516 }
517<St_Para>{SPCMD3} {
518 yyextra->token.name = "_form";
519 bool ok;
520 yyextra->token.id = DString(yytext).right((int)yyleng-7).toInt(&ok);
521 ASSERT(ok);
522 return Token::char_to_command(yytext[0]);
523 }
DString right(size_t len) const
Definition dstring.h:311
static Token char_to_command(char c)
524<St_Para>{CMD}"n"\n { /* \n followed by real newline */
525 lineCount(yytext,yyleng);
526 //yyextra->yyLineNr++;
527 yyextra->token.name = yytext+1;
528 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
529 yyextra->token.paramDir=TokenInfo::Unspecified;
530 return Token::char_to_command(yytext[0]);
531 }
532<St_Para>"\\ilinebr" {
533 }
534<St_Para>{SPCMD1} |
535<St_Para>{SPCMD2} |
536<St_Para>{SPCMD5} |
537<St_Para>{SPCMD4} { /* special command */
538 yyextra->token.name = yytext+1;
539 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
540 yyextra->token.paramDir=TokenInfo::Unspecified;
541 return Token::char_to_command(yytext[0]);
542 }
543<St_Para>{PARAMIO} { /* param [in,out] command */
544 yyextra->token.name = "param";
545 DString s(yytext);
546 bool isIn = s.find("in") !=DString::npos;
547 bool isOut = s.find("out")!=DString::npos;
548 if (isIn)
549 {
550 if (isOut)
551 {
552 yyextra->token.paramDir=TokenInfo::InOut;
553 }
554 else
555 {
556 yyextra->token.paramDir=TokenInfo::In;
557 }
558 }
559 else if (isOut)
560 {
561 yyextra->token.paramDir=TokenInfo::Out;
562 }
563 else
564 {
565 yyextra->token.paramDir=TokenInfo::Unspecified;
566 }
567 return Token::char_to_command(yytext[0]);
568 }
569<St_Para>{URLPROTOCOL}{URLMASK}/[,\.] { // URL, or URL.
570 yyextra->token.name=yytext;
571 yyextra->token.isEMailAddr=false;
572 return Token::make_TK_URL();
573 }
574<St_Para>{URLPROTOCOL}{URLMASK} { // URL
575 yyextra->token.name=yytext;
576 yyextra->token.isEMailAddr=false;
577 return Token::make_TK_URL();
578 }
579<St_Para>"<"{URLPROTOCOL}{URLMASK}">" { // URL
580 yyextra->token.name=yytext;
581 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
582 yyextra->token.isEMailAddr=false;
583 return Token::make_TK_URL();
584 }
585<St_Para>{MAILADDR} { // Mail address
586 yyextra->token.name=yytext;
587 yyextra->token.name.stripPrefix("mailto:");
588 yyextra->token.isEMailAddr=true;
589 return Token::make_TK_URL();
590 }
591<St_Para>"<"{MAILADDR}">" { // Mail address
592 yyextra->token.name=yytext;
593 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
594 yyextra->token.name.stripPrefix("mailto:");
595 yyextra->token.isEMailAddr=true;
596 return Token::make_TK_URL();
597 }
598<St_Para>"<"{MAILADDR2}">" { // anti spam mail address
599 yyextra->token.name=yytext;
600 return Token::make_TK_WORD();
601 }
602<St_Para>{RCSID} { /* RCS tag */
603 DString tagName(yytext+1);
604 size_t index=tagName.find(':');
605 if (index==DString::npos) index=0; // should never happen
606 yyextra->token.name = tagName.left(index);
607 size_t text_begin = index+2;
608 size_t text_end = tagName.length()-1;
609 if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
610 {
611 ++text_begin;
612 if (tagName[text_end-1]=='#')
613 {
614 --text_end;
615 }
616 }
617 yyextra->token.text = tagName.mid(text_begin,text_end-text_begin);
618 return Token::make_TK_RCSTAG();
619 }
620<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}")" | /* environment variable */
621<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}"("{ID}"))" { /* environment variable */
622 DString name(&yytext[2]);
623 name = name.left(static_cast<int>(name.length())-1);
624 DString value = Portable::getenv(name);
625 for (int i=static_cast<int>(value.length())-1;i>=0;i--) unput(value.at(i));
626 }
size_t length() const
Returns the length of the string, not counting the 0-terminator.
Definition dstring.h:151
const char * name(int level)
Definition latexgen.h:368
DString getenv(const DString &variable)
Definition portable.cpp:337
627<St_Para>"<blockquote>&zwj;" {
628 // for a markdown inserted block quote,
629 // tell flex that after putting the last indent
630 // back we are at the beginning of the line, see issue #11309
631 YY_CURRENT_BUFFER->yy_at_bol=1;
632 lineCount(yytext,yyleng);
633 handleHtmlTag(yyscanner,yytext);
634 return Token::make_TK_HTMLTAG();
635 }
static void handleHtmlTag(yyscan_t yyscanner, const char *text)
636<St_Para>{HTMLTAG} { /* html tag */
637 lineCount(yytext,yyleng);
638 handleHtmlTag(yyscanner,yytext);
639 return Token::make_TK_HTMLTAG();
640 }
641<St_Para,St_Text>"&"{ID}";" { /* special symbol */
642 yyextra->token.name = yytext;
643 return Token::make_TK_SYMBOL();
644 }
645
646 /********* patterns for linkable words ******************/
647
648<St_Para>{ID}/"<"{HTMLKEYW}">"+ { /* this rule is to prevent opening html
649 * tag to be recognized as a templated classes
650 */
651 yyextra->token.name = yytext;
652 return Token::make_TK_LNKWORD();
653 }
654<St_Para>{LNKWORDN}/("<"{HTMLKEYW}">")+ { // prevent <br> html tag to be parsed as template arguments
655 yyextra->token.name = yytext;
656 return Token::make_TK_LNKWORD();
657 }
658<St_Para>{LNKWORDN}"<"{HTMLKEYW}">"[^<]*"</"{HTMLKEYW}">" { // a word directly followed by a complete inline
659 // html span, e.g. one<tt>.h</tt> (from a markdown code span) or test<em>.h</em>,
660 // is not a template: return only the word and let the html tag rules handle the span
661 const std::string text(yytext);
662 const size_t closeTagPos = text.rfind('<');
663 const size_t openTagPos = text.rfind('<', closeTagPos - 1);
664 yyless(static_cast<int>(openTagPos));
665 yyextra->token.name = yytext;
666 return Token::make_TK_LNKWORD();
667 }
668<St_Para>{LNKWORD1} |
669<St_Para>{LNKWORD1}{FUNCARG} |
670<St_Para>{LNKWORD2} |
671<St_Para>{LNKWORD3} |
672<St_Para>{LNKWORD4} {
673 yyextra->token.name = yytext;
674 return Token::make_TK_LNKWORD();
675 }
676<St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
677 yyextra->token.name = yytext;
678 yyextra->token.name = yyextra->token.name.left(yyextra->token.name.length()-1);
679 unput(yytext[(int)yyleng-1]);
680 return Token::make_TK_LNKWORD();
681 }
682 /********* patterns for normal words ******************/
683
684<St_Para,St_Text>[\-+0-9] |
685<St_Para,St_Text>{WORD1} |
686<St_Para,St_Text>{WORD2} { /* function call */
687 if (DString(yytext).find("\\ilinebr")!=DString::npos) REJECT; // see issue #8311
688 lineCount(yytext,yyleng);
689 if (yytext[0]=='%') // strip % if present
690 yyextra->token.name = &yytext[1];
691 else
692 yyextra->token.name = yytext;
693 return Token::make_TK_WORD();
694 }
695<St_Text>({ID}".")+{ID} {
696 yyextra->token.name = yytext;
697 return Token::make_TK_WORD();
698 }
699<St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
700 // avoid interpretation as "operator <"
701 yyextra->token.name = yytext;
702 return Token::make_TK_WORD();
703 }
704
705 /*******************************************************/
706
707<St_Para,St_Text>{BLANK}+ |
708<St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
709 lineCount(yytext,yyleng);
710 yyextra->token.chars=yytext;
711 return Token::make_TK_WHITESPACE();
712 }
713<St_Text>[\\@<>&$#%~] {
714 yyextra->token.name = yytext;
715 return Token::char_to_command(yytext[0]);
716 }
717<St_Para>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
718 if (yyextra->insidePre || yyextra->autoListLevel==0)
719 {
720 REJECT;
721 }
722 lineCount(yytext,yyleng);
723 }
724<St_Para>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*\n/{CLISTITEM} { /* skip trailing paragraph followed by new checkbox item */
725 if (yyextra->insidePre || yyextra->autoListLevel==0)
726 {
727 REJECT;
728 }
729 lineCount(yytext,yyleng);
730 }
731<St_Para>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
732 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
733 {
734 REJECT;
735 }
736 lineCount(yytext,yyleng);
737 }
738<St_Para>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
739 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
740 {
741 REJECT;
742 }
743 lineCount(yytext,yyleng);
744 }
745<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}*/" \\ifile" | // we don't want to count the space before \ifile
746<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}* {
747 lineCount(yytext,yyleng);
748 if (yyextra->insidePre)
749 {
750 yyextra->token.chars=yytext;
751 return Token::make_TK_WHITESPACE();
752 }
753 else
754 {
755 yyextra->token.indent=computeIndent(yytext,yyleng);
756 int i;
757 // put back the indentation (needed for list items)
758 //printf("token.indent=%d\n",yyextra->token.indent);
759 for (i=0;i<yyextra->token.indent;i++)
760 {
761 unput(' ');
762 }
763 // tell flex that after putting the last indent
764 // back we are at the beginning of the line
765 YY_CURRENT_BUFFER->yy_at_bol=1;
766 // start of a new paragraph
767 return Token::make_TK_NEWPARA();
768 }
769 }
770<St_CodeOpt>{BLANK}*"{"(".")?{CODEID}"}" {
771 yyextra->token.name = yytext;
772 size_t i=yyextra->token.name.find('{'); /* } to keep vi happy */
773 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
774 BEGIN(St_Code);
775 }
776<St_iCodeOpt>{BLANK}*"{"(".")?{CODEID}"}" {
777 yyextra->token.name = yytext;
778 size_t i=yyextra->token.name.find('{'); /* } to keep vi happy */
779 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
780 BEGIN(St_iCode);
781 }
782<St_CodeOpt>"\\ilinebr" |
783<St_CodeOpt>\n |
784<St_CodeOpt>. {
785 unput_string(yytext,yyleng);
786 BEGIN(St_Code);
787 }
#define unput_string(yytext, yyleng)
788<St_iCodeOpt>"\\ilinebr" |
789<St_iCodeOpt>\n |
790<St_iCodeOpt>. {
791 unput_string(yytext,yyleng);
792 BEGIN(St_iCode);
793 }
794<St_Code>{WS}*{CMD}"endcode" {
795 lineCount(yytext,yyleng);
796 return Token::make_RetVal_OK();
797 }
798<St_iCode>{WS}*{CMD}"endicode" {
799 lineCount(yytext,yyleng);
800 return Token::make_RetVal_OK();
801 }
802<St_XmlCode>{WS}*"</code>" {
803 lineCount(yytext,yyleng);
804 return Token::make_RetVal_OK();
805 }
806<St_Code,St_iCode,St_XmlCode>[^\\@\n<]+ |
807<St_Code,St_iCode,St_XmlCode>\n |
808<St_Code,St_iCode,St_XmlCode>. {
809 lineCount(yytext,yyleng);
810 yyextra->token.verb+=yytext;
811 }
812<St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
813 yyextra->token.name="block";
814 BEGIN(St_HtmlOnly);
815 }
816<St_HtmlOnlyOption>.|\n {
817 unput(*yytext);
818 BEGIN(St_HtmlOnly);
819 }
820<St_HtmlOnlyOption>"\\ilinebr" {
821 unput_string(yytext,yyleng);
822 BEGIN(St_HtmlOnly);
823 }
824<St_HtmlOnly>{CMD}"endhtmlonly" {
825 return Token::make_RetVal_OK();
826 }
827<St_HtmlOnly>[^\\@\n$]+ |
828<St_HtmlOnly>\n |
829<St_HtmlOnly>. {
830 lineCount(yytext,yyleng);
831 yyextra->token.verb+=yytext;
832 }
833<St_ManOnly>{CMD}"endmanonly" {
834 return Token::make_RetVal_OK();
835 }
836<St_ManOnly>[^\\@\n$]+ |
837<St_ManOnly>\n |
838<St_ManOnly>. {
839 lineCount(yytext,yyleng);
840 yyextra->token.verb+=yytext;
841 }
842<St_RtfOnly>{CMD}"endrtfonly" {
843 return Token::make_RetVal_OK();
844 }
845<St_RtfOnly>[^\\@\n$]+ |
846<St_RtfOnly>\n |
847<St_RtfOnly>. {
848 lineCount(yytext,yyleng);
849 yyextra->token.verb+=yytext;
850 }
851<St_LatexOnly>{CMD}"endlatexonly" {
852 return Token::make_RetVal_OK();
853 }
854<St_LatexOnly>[^\\@\n]+ |
855<St_LatexOnly>\n |
856<St_LatexOnly>. {
857 lineCount(yytext,yyleng);
858 yyextra->token.verb+=yytext;
859 }
860<St_XmlOnly>{CMD}"endxmlonly" {
861 return Token::make_RetVal_OK();
862 }
863<St_XmlOnly>[^\\@\n]+ |
864<St_XmlOnly>\n |
865<St_XmlOnly>. {
866 lineCount(yytext,yyleng);
867 yyextra->token.verb+=yytext;
868 }
869<St_DbOnly>{CMD}"enddocbookonly" {
870 return Token::make_RetVal_OK();
871 }
872<St_DbOnly>[^\\@\n]+ |
873<St_DbOnly>\n |
874<St_DbOnly>. {
875 lineCount(yytext,yyleng);
876 yyextra->token.verb+=yytext;
877 }
878<St_Verbatim>{CMD}"endverbatim" {
879 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
880 return Token::make_RetVal_OK();
881 }
882<St_ILiteral>{CMD}"endiliteral " { // note extra space as this is artificially added
883 // remove spaces that have been added
884 yyextra->token.verb = yyextra->token.verb.mid(1,yyextra->token.verb.length()-2);
885 return Token::make_RetVal_OK();
886 }
887<St_iVerbatim>{CMD}"endiverbatim" {
888 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
889 return Token::make_RetVal_OK();
890 }
891<St_Verbatim,St_iVerbatim,St_ILiteral>[^\\@\n]+ |
892<St_Verbatim,St_iVerbatim,St_ILiteral>\n |
893<St_Verbatim,St_iVerbatim,St_ILiteral>. { /* Verbatim text */
894 lineCount(yytext,yyleng);
895 yyextra->token.verb+=yytext;
896 }
897<St_ILiteralOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // option(s) present
898 yyextra->token.verb = DString(yytext).stripWhiteSpace();
899 return Token::make_RetVal_OK();
900 }
DString stripWhiteSpace() const
returns a copy of this string with leading and trailing whitespace removed
Definition dstring.h:337
901<St_ILiteralOpt>"\\ilinebr" |
902<St_ILiteralOpt>"\n" |
903<St_ILiteralOpt>. {
904 yyextra->token.sectionId = "";
905 unput_string(yytext,yyleng);
906 return Token::make_RetVal_OK();
907 }
908<St_Dot>{CMD}"enddot" {
909 return Token::make_RetVal_OK();
910 }
911<St_Dot>[^\\@\n]+ |
912<St_Dot>\n |
913<St_Dot>. { /* dot text */
914 lineCount(yytext,yyleng);
915 yyextra->token.verb+=yytext;
916 }
917<St_Msc>{CMD}("endmsc") {
918 return Token::make_RetVal_OK();
919 }
920<St_Msc>[^\\@\n]+ |
921<St_Msc>\n |
922<St_Msc>. { /* msc text */
923 lineCount(yytext,yyleng);
924 yyextra->token.verb+=yytext;
925 }
926<St_PlantUMLOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
927 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
928 return Token::make_RetVal_OK();
929 }
930<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
931 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
932 return Token::make_RetVal_OK();
933 }
934<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
935 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
936 return Token::make_RetVal_OK();
937 }
938<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
939 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
940 return Token::make_RetVal_OK();
941 }
942<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
943 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
944 return Token::make_RetVal_OK();
945 }
946<St_PlantUMLOpt>"\\ilinebr" |
947<St_PlantUMLOpt>"\n" |
948<St_PlantUMLOpt>. {
949 yyextra->token.sectionId = "";
950 unput_string(yytext,yyleng);
951 return Token::make_RetVal_OK();
952 }
953<St_PlantUML>{CMD}"enduml" {
954 return Token::make_RetVal_OK();
955 }
956<St_PlantUML>[^\\@\n]+ |
957<St_PlantUML>\n |
958<St_PlantUML>. { /* plantuml text */
959 lineCount(yytext,yyleng);
960 yyextra->token.verb+=yytext;
961 }
962<St_MermaidOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
963 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
964 return Token::make_RetVal_OK();
965 }
966<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
967 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
968 return Token::make_RetVal_OK();
969 }
970<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
971 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
972 return Token::make_RetVal_OK();
973 }
974<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
975 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
976 return Token::make_RetVal_OK();
977 }
978<St_MermaidOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
979 yyextra->token.sectionId = DString(yytext).stripWhiteSpace();
980 return Token::make_RetVal_OK();
981 }
982<St_MermaidOpt>"\\ilinebr" |
983<St_MermaidOpt>"\n" |
984<St_MermaidOpt>. {
985 yyextra->token.sectionId = "";
986 unput_string(yytext,yyleng);
987 return Token::make_RetVal_OK();
988 }
989<St_Mermaid>{CMD}"endmermaid" {
990 return Token::make_RetVal_OK();
991 }
992<St_Mermaid>[^\\@\n]+ |
993<St_Mermaid>\n |
994<St_Mermaid>. { /* mermaid text */
995 lineCount(yytext,yyleng);
996 yyextra->token.verb+=yytext;
997 }
998<St_Title>"\"" { // quoted title
999 BEGIN(St_TitleQ);
1000 }
1001<St_Title>[ \t]+ {
1002 yyextra->token.chars=yytext;
1003 return Token::make_TK_WHITESPACE();
1004 }
1005<St_Title>. { // non-quoted title
1006 unput(*yytext);
1007 BEGIN(St_TitleN);
1008 }
1009<St_Title>\n {
1010 unput(*yytext);
1011 return Token::make_TK_NONE();
1012 }
1013<St_Title>"\\ilinebr" {
1014 unput_string(yytext,yyleng);
1015 return Token::make_TK_NONE();
1016 }
1017<St_TitleN>"&"{ID}";" { /* symbol */
1018 yyextra->token.name = yytext;
1019 return Token::make_TK_SYMBOL();
1020 }
1021<St_TitleN>{HTMLTAG} {
1022 yyextra->token.name = yytext;
1023 handleHtmlTag(yyscanner,yytext);
1024 return Token::make_TK_HTMLTAG();
1025 }
1026<St_TitleN>\n { /* new line => end of title */
1027 unput(*yytext);
1028 return Token::make_TK_NONE();
1029 }
1030<St_TitleN>"\\ilinebr" { /* new line => end of title */
1031 unput_string(yytext,yyleng);
1032 return Token::make_TK_NONE();
1033 }
1034<St_TitleN>{SPCMD1} |
1035<St_TitleN>{SPCMD2} { /* special command */
1036 yyextra->token.name = yytext+1;
1037 yyextra->token.paramDir=TokenInfo::Unspecified;
1038 return Token::char_to_command(yytext[0]);
1039 }
1040<St_TitleN>{ID}"=" { /* attribute */
1041 if (yytext[0]=='%') // strip % if present
1042 yyextra->token.name = &yytext[1];
1043 else
1044 yyextra->token.name = yytext;
1045 return Token::make_TK_WORD();
1046 }
1047<St_TitleN>[\-+0-9] |
1048<St_TitleN>{WORD1} |
1049<St_TitleN>{WORD2} { /* word */
1050 if (DString(yytext).find("\\ilinebr")!=DString::npos) REJECT; // see issue #8311
1051 lineCount(yytext,yyleng);
1052 if (yytext[0]=='%') // strip % if present
1053 yyextra->token.name = &yytext[1];
1054 else
1055 yyextra->token.name = yytext;
1056 return Token::make_TK_WORD();
1057 }
1058<St_TitleN>[ \t]+ {
1059 yyextra->token.chars=yytext;
1060 return Token::make_TK_WHITESPACE();
1061 }
1062<St_TitleQ>"&"{ID}";" { /* symbol */
1063 yyextra->token.name = yytext;
1064 return Token::make_TK_SYMBOL();
1065 }
1066<St_TitleQ>(\n|"\\ilinebr") { /* new line => end of title */
1067 unput_string(yytext,yyleng);
1068 return Token::make_TK_NONE();
1069 }
1070<St_TitleQ>{SPCMD1} |
1071<St_TitleQ>{SPCMD2} { /* special command */
1072 yyextra->token.name = yytext+1;
1073 yyextra->token.paramDir=TokenInfo::Unspecified;
1074 return Token::char_to_command(yytext[0]);
1075 }
1076<St_TitleQ>{WORD1NQ} |
1077<St_TitleQ>{WORD2NQ} { /* word */
1078 yyextra->token.name = yytext;
1079 return Token::make_TK_WORD();
1080 }
1081<St_TitleQ>[ \t]+ {
1082 yyextra->token.chars=yytext;
1083 return Token::make_TK_WHITESPACE();
1084 }
1085<St_TitleQ>"\"" { /* closing quote => end of title */
1086 BEGIN(St_TitleA);
1087 return Token::make_TK_NONE();
1088 }
1089<St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
1090 yyextra->token.name = yytext;
1091 size_t pos = yyextra->token.name.find('=');
1092 if (pos==DString::npos) pos=0; // should never happen
1093 yyextra->token.name = yyextra->token.name.left(pos).stripWhiteSpace();
1094 BEGIN(St_TitleV);
1095 }
1096<St_TitleV>[^ \t\r\n]+ { // attribute value
1097 lineCount(yytext,yyleng);
1098 yyextra->token.chars = yytext;
1099 BEGIN(St_TitleN);
1100 return Token::make_TK_WORD();
1101 }
1102<St_TitleV,St_TitleA>. {
1103 unput(*yytext);
1104 return Token::make_TK_NONE();
1105 }
1106<St_TitleV,St_TitleA>(\n|"\\ilinebr") {
1107 unput_string(yytext,yyleng);
1108 return Token::make_TK_NONE();
1109 }
1110
1111<St_Anchor>({REQID}|{LABELID}){WS}? { // anchor
1112 lineCount(yytext,yyleng);
1113 yyextra->token.name = DString(yytext).stripWhiteSpace();
1114 return Token::make_TK_WORD();
1115 }
1116<St_Anchor>. {
1117 unput(*yytext);
1118 return Token::make_TK_NONE();
1119 }
1120<St_Cite>{CITEID} { // label to cite
1121 if (yytext[0] =='"')
1122 {
1123 yyextra->token.name=yytext+1;
1124 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-2);
1125 }
1126 else
1127 {
1128 yyextra->token.name=yytext;
1129 }
1130 return Token::make_TK_WORD();
1131 }
1132<St_Cite>{BLANK} { // white space
1133 unput(' ');
1134 return Token::make_TK_NONE();
1135 }
1136<St_Cite>(\n|"\\ilinebr") { // new line
1137 unput_string(yytext,yyleng);
1138 return Token::make_TK_NONE();
1139 }
1140<St_Cite>. { // any other character
1141 unput(*yytext);
1142 return Token::make_TK_NONE();
1143 }
1144<St_DoxyConfig>{DOXYCFG} { // config option
1145 yyextra->token.name=yytext;
1146 return Token::make_TK_WORD();
1147 }
1148<St_DoxyConfig>{BLANK} { // white space
1149 unput(' ');
1150 return Token::make_TK_NONE();
1151 }
1152<St_DoxyConfig>(\n|"\\ilinebr") { // new line
1153 unput_string(yytext,yyleng);
1154 return Token::make_TK_NONE();
1155 }
1156<St_DoxyConfig>. { // any other character
1157 unput(*yytext);
1158 return Token::make_TK_NONE();
1159 }
1160<St_Ref>{REFWORD_NOCV}/{BLANK}("const")[a-z_A-Z0-9] { // see bug776988
1161 yyextra->token.name=yytext;
1162 return Token::make_TK_WORD();
1163 }
1164<St_Ref>{REFWORD_NOCV}/{BLANK}("volatile")[a-z_A-Z0-9] { // see bug776988
1165 yyextra->token.name=yytext;
1166 return Token::make_TK_WORD();
1167 }
1168<St_Ref>{REFWORD} { // label to refer to
1169 yyextra->token.name=yytext;
1170 return Token::make_TK_WORD();
1171 }
1172<St_Ref>{BLANK} { // white space
1173 unput(' ');
1174 return Token::make_TK_NONE();
1175 }
1176<St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
1177 yyextra->expectQuote=true;
1178 lineCount(yytext,yyleng);
1179 BEGIN(St_Ref2);
1180 }
1181<St_Ref>(\n|"\\ilinebr") { // new line
1182 unput_string(yytext,yyleng);
1183 return Token::make_TK_NONE();
1184 }
1185<St_Ref>"\""[^"\n]+"\"" { // quoted first argument -> return without quotes
1186 yyextra->token.name=DString(yytext).mid(1,yyleng-2);
1187 return Token::make_TK_WORD();
1188 }
1189<St_Ref>. { // any other character
1190 unput(*yytext);
1191 return Token::make_TK_NONE();
1192 }
1193<St_IntRef>[A-Z_a-z0-9.:/#\-\+\‍(\‍)]+ {
1194 yyextra->token.name = yytext;
1195 return Token::make_TK_WORD();
1196 }
1197<St_IntRef>{BLANK}+"\"" {
1198 BEGIN(St_Ref2);
1199 }
1200<St_SetScope>({SCOPEMASK}|{ANONNS}){BLANK}|{FILEMASK} {
1201 yyextra->token.name = yytext;
1202 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1203 return Token::make_TK_WORD();
1204 }
1205<St_SetScope>{SCOPEMASK}"<" {
1206 yyextra->token.name = yytext;
1207 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1208 yyextra->sharpCount=1;
1209 BEGIN(St_SetScopeEnd);
1210 }
1211<St_SetScope>{BLANK} {
1212 }
1213<St_SetScopeEnd>"<" {
1214 yyextra->token.name += yytext;
1215 yyextra->sharpCount++;
1216 }
1217<St_SetScopeEnd>">" {
1218 yyextra->token.name += yytext;
1219 yyextra->sharpCount--;
1220 if (yyextra->sharpCount<=0)
1221 {
1222 return Token::make_TK_WORD();
1223 }
1224 }
1225<St_SetScopeEnd>. {
1226 yyextra->token.name += yytext;
1227 }
1228<St_Ref2>"&"{ID}";" { /* symbol */
1229 yyextra->token.name = yytext;
1230 return Token::make_TK_SYMBOL();
1231 }
1232<St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title? */
1233 lineCount(yytext,yyleng);
1234 if (!yyextra->expectQuote || yytext[0]=='"')
1235 {
1236 return Token::make_TK_NONE();
1237 }
1238 else
1239 {
1240 yyextra->token.name += yytext;
1241 }
1242 }
1243<St_Ref2>{HTMLTAG_STRICT} { /* html tag */
1244 lineCount(yytext,yyleng);
1245 handleHtmlTag(yyscanner,yytext);
1246 return Token::make_TK_HTMLTAG();
1247 }
1248<St_Ref2>{SPCMD1} |
1249<St_Ref2>{SPCMD2} { /* special command */
1250 yyextra->token.name = yytext+1;
1251 yyextra->token.paramDir=TokenInfo::Unspecified;
1252 return Token::char_to_command(yytext[0]);
1253 }
1254<St_Ref2>{WORD1NQ} |
1255<St_Ref2>{WORD2NQ} {
1256 /* word */
1257 yyextra->token.name = yytext;
1258 return Token::make_TK_WORD();
1259 }
1260<St_Ref2>[ \t]+ {
1261 yyextra->token.chars=yytext;
1262 return Token::make_TK_WHITESPACE();
1263 }
1264<St_XRefItem>{LABELID} {
1265 yyextra->token.name=yytext;
1266 }
1267<St_XRefItem>" " {
1268 BEGIN(St_XRefItem2);
1269 }
1270<St_XRefItem2>[0-9]+"." {
1271 DString numStr(yytext);
1272 numStr=numStr.left((int)yyleng-1);
1273 yyextra->token.id=numStr.toInt();
1274 return Token::make_RetVal_OK();
1275 }
1276<St_Para,St_Title,St_Ref2>"<!--" { /* html style comment block */
1277 yyextra->commentState = YY_START;
1278 BEGIN(St_Comment);
1279 }
1280<St_Param>"\""[^\n\"]+"\"" {
1281 yyextra->token.name = yytext+1;
1282 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1283 return Token::make_TK_WORD();
1284 }
1285<St_Param>({PHPTYPE}{BLANK}*("["{BLANK}*"]")*{BLANK}*"|"{BLANK}*)*{PHPTYPE}{BLANK}*("["{BLANK}*"]")*{WS}+("&")?"$"{LABELID} {
1286 lineCount(yytext,yyleng);
1287 DString params(yytext);
1288 size_t j = params.find('&');
1289 size_t i = params.find('$');
1290 if (i==DString::npos) i=0; // should never happen
1291 if (j!=DString::npos && j<i) i=j;
1292 DString types = params.left(i).stripWhiteSpace();
1293 yyextra->token.name = types+"#"+params.mid(i);
1294 return Token::make_TK_WORD();
1295 }
DString left(size_t len) const
Definition dstring.h:306
1296<St_Param>[^ \t\n,@\\‍]+ {
1297 yyextra->token.name = yytext;
1298 if (yyextra->token.name.at(static_cast<uint32_t>(yyleng)-1)==':')
1299 {
1300 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-1);
1301 }
1302 return Token::make_TK_WORD();
1303 }
1304<St_Param>{WS}*","{WS}* /* param separator */
1305<St_Param>{WS} {
1306 lineCount(yytext,yyleng);
1307 yyextra->token.chars=yytext;
1308 return Token::make_TK_WHITESPACE();
1309 }
1310<St_Prefix>"\""[^\n\"]*"\"" {
1311 yyextra->token.name = yytext+1;
1312 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1313 return Token::make_TK_WORD();
1314 }
1315<St_Prefix>. {
1316 unput(*yytext);
1317 return Token::make_TK_NONE();
1318 }
1319<St_Options>{ID} {
1320 yyextra->token.name+=yytext;
1321 }
1322<St_Options>{WS}*":"{WS}* {
1323 lineCount(yytext,yyleng);
1324 yyextra->token.name+=":";
1325 }
1326<St_Options>{WS}*","{WS}* |
1327<St_Options>{WS} { /* option separator */
1328 lineCount(yytext,yyleng);
1329 yyextra->token.name+=",";
1330 }
1331<St_Options>"}" {
1332 return Token::make_TK_WORD();
1333 }
1334<St_Block>{ID} {
1335 yyextra->token.name+=yytext;
1336 }
1337<St_Block>"]" {
1338 return Token::make_TK_WORD();
1339 }
1340<St_Emoji>[:0-9_a-z+-]+ {
1341 yyextra->token.name=yytext;
1342 return Token::make_TK_WORD();
1343 }
1344<St_Emoji>. {
1345 unput(*yytext);
1346 return Token::make_TK_NONE();
1347 }
1348<St_QuotedString>"\"" {
1349 yyextra->token.name="";
1350 BEGIN(St_QuotedContent);
1351 }
1352<St_QuotedString>(\n|"\\ilinebr") {
1353 unput_string(yytext,yyleng);
1354 return Token::make_TK_NONE();
1355 }
1356<St_QuotedString>. {
1357 unput(*yytext);
1358 return Token::make_TK_NONE();
1359 }
1360<St_QuotedContent>"\"" {
1361 return Token::make_TK_WORD();
1362 }
1363<St_QuotedContent>. {
1364 yyextra->token.name+=yytext;
1365 }
1366<St_ShowDate>{WS}+{SHOWDATE} {
1367 lineCount(yytext,yyleng);
1368 yyextra->token.name=yytext;
1369 return Token::make_TK_WORD();
1370 }
1371<St_ShowDate>(\n|"\\ilinebr") {
1372 unput_string(yytext,yyleng);
1373 return Token::make_TK_NONE();
1374 }
1375<St_ShowDate>. {
1376 unput(*yytext);
1377 return Token::make_TK_NONE();
1378 }
1379<St_ILine>{LINENR}/[\\@\n\.] |
1380<St_ILine>{LINENR}{BLANK} {
1381 bool ok = false;
1382 int nr = DString(yytext).toInt(&ok);
1383 if (!ok)
1384 {
1385 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '{}' for iline command",yytext);
1386 }
1387 else
1388 {
1389 yyextra->yyLineNr = nr;
1390 }
1391 return Token::make_TK_WORD();
1392 }
1393<St_ILine>. {
1394 return Token::make_TK_NONE();
1395 }
1396<St_IFile>{BLANK}*{FILEMASK} {
1397 DString text(yytext);
1398 text = text.stripWhiteSpace();
1399 yyextra->fileName = text;
1400 yyextra->token.name = text;
1401 return Token::make_TK_WORD();
1402 }
1403<St_IFile>{BLANK}*"\""[^\n\"]+"\"" {
1404 DString text(yytext);
1405 text = text.stripWhiteSpace();
1406 yyextra->fileName = text.mid(1,text.length()-2);
1407 yyextra->token.name = text.mid(1,text.length()-2);
1408 return Token::make_TK_WORD();
1409 }
1410<St_File>{FILEMASK} {
1411 yyextra->token.name = yytext;
1412 if (yyextra->token.name.endsWith("\\ilinebr") ||yyextra->token.name.endsWith("@ilinebr"))
1413 {
1414 unput_string("\\ilinebr",8);
1415 yyextra->token.name = yyextra->token.name.left(yyleng-8);
1416 }
1417 return Token::make_TK_WORD();
1418 }
1419<St_File>"\""[^\n\"]+"\"" {
1420 DString text(yytext);
1421 yyextra->token.name = text.mid(1,text.length()-2);
1422 return Token::make_TK_WORD();
1423 }
1424<St_Pattern>[^\\\r\n]+ {
1425 yyextra->token.name += yytext;
1426 }
1427<St_Pattern>"\\ilinebr" {
1428 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1429 return Token::make_TK_WORD();
1430 }
1431<St_Pattern>\n {
1432 lineCount(yytext,yyleng);
1433 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1434 return Token::make_TK_WORD();
1435 }
1436<St_Pattern>. {
1437 yyextra->token.name += yytext;
1438 }
1439<St_Link>{LINKMASK}|{REFWORD} {
1440 yyextra->token.name = yytext;
1441 return Token::make_TK_WORD();
1442 }
1443<St_Comment>"-->" { /* end of html comment */
1444 BEGIN(yyextra->commentState);
1445 }
1446<St_Comment>[^-]+ /* inside html comment */
1447<St_Comment>. /* inside html comment */
1448
1449 /* State for skipping title (all chars until the end of the line) */
1450
1451<St_SkipTitle>.
1452<St_SkipTitle>(\n|"\\ilinebr") {
1453 if (*yytext == '\n') unput('\n');
1454 return Token::make_TK_NONE();
1455 }
1456
1457 /* State for the pass used to find the anchors and sections */
1458
1459<St_Sections>[^\n@\<]+
1460<St_Sections>{CMD}("<"|{CMD})
1461<St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
1462 lineCount(yytext,yyleng);
1463 DString tag(yytext);
1464 size_t s=tag.find("id=");
1465 if (s!=DString::npos) // command has id attribute
1466 {
1467 char c=tag[s+3];
1468 if (c=='\'' || c=='"') // valid start
1469 {
1470 size_t e=tag.find(c,s+4);
1471 if (e!=DString::npos) // found matching end
1472 {
1473 yyextra->secType = SectionType::Table;
1474 yyextra->secLabel=tag.mid(s+4,e-s-4); // extract id
1475 processSection(yyscanner);
1476 }
1477 }
1478 }
1479 }
static constexpr int Table
Definition section.h:41
static void processSection(yyscan_t yyscanner)
1480<St_Sections>{CMD}"anchor"{BLANK}+ {
1481 yyextra->secType = SectionType::Anchor;
1482 BEGIN(St_SecLabel1);
1483 }
static constexpr int Anchor
Definition section.h:40
1484<St_Sections>{CMD}"ianchor"{BLANK}+ {
1485 yyextra->secType = SectionType::Anchor;
1486 BEGIN(St_SecLabel1);
1487 }
1488<St_Sections>{CMD}"section"{BLANK}+ {
1489 yyextra->secType = SectionType::Section;
1490 BEGIN(St_SecLabel2);
1491 }
static constexpr int Section
Definition section.h:33
1492<St_Sections>{CMD}"subsection"{BLANK}+ {
1493 yyextra->secType = SectionType::Subsection;
1494 BEGIN(St_SecLabel2);
1495 }
static constexpr int Subsection
Definition section.h:34
1496<St_Sections>{CMD}"subsubsection"{BLANK}+ {
1497 yyextra->secType = SectionType::Subsubsection;
1498 BEGIN(St_SecLabel2);
1499 }
static constexpr int Subsubsection
Definition section.h:35
1500<St_Sections>{CMD}"paragraph"{BLANK}+ {
1501 yyextra->secType = SectionType::Paragraph;
1502 BEGIN(St_SecLabel2);
1503 }
static constexpr int Paragraph
Definition section.h:36
1504<St_Sections>{CMD}"subparagraph"{BLANK}+ {
1505 yyextra->secType = SectionType::Subparagraph;
1506 BEGIN(St_SecLabel2);
1507 }
static constexpr int Subparagraph
Definition section.h:37
1508<St_Sections>{CMD}"subsubparagraph"{BLANK}+ {
1509 yyextra->secType = SectionType::Subsubparagraph;
1510 BEGIN(St_SecLabel2);
1511 }
static constexpr int Subsubparagraph
Definition section.h:38
1512<St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9] {
1513 yyextra->endMarker="endverbatim";
1514 BEGIN(St_SecSkip);
1515 }
1516<St_Sections>{CMD}"iverbatim"/[^a-z_A-Z0-9] {
1517 yyextra->endMarker="endiverbatim";
1518 BEGIN(St_SecSkip);
1519 }
1520<St_Sections>{CMD}"iliteral"/[^a-z_A-Z0-9] {
1521 yyextra->endMarker="endiliteral";
1522 BEGIN(St_SecSkip);
1523 }
1524<St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1525 yyextra->endMarker="enddot";
1526 BEGIN(St_SecSkip);
1527 }
1528<St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1529 yyextra->endMarker="endmsc";
1530 BEGIN(St_SecSkip);
1531 }
1532<St_Sections>{CMD}"mermaid"/[^a-z_A-Z0-9] {
1533 yyextra->endMarker="endmermaid";
1534 BEGIN(St_SecSkip);
1535 }
1536<St_Sections>{CMD}"startuml"/[^a-z_A-Z0-9] {
1537 yyextra->endMarker="enduml";
1538 BEGIN(St_SecSkip);
1539 }
1540<St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1541 yyextra->endMarker="endhtmlonly";
1542 BEGIN(St_SecSkip);
1543 }
1544<St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1545 yyextra->endMarker="endlatexonly";
1546 BEGIN(St_SecSkip);
1547 }
1548<St_Sections>{CMD}"manonly"/[^a-z_A-Z0-9] {
1549 yyextra->endMarker="endmanonly";
1550 BEGIN(St_SecSkip);
1551 }
1552<St_Sections>{CMD}"rtfonly"/[^a-z_A-Z0-9] {
1553 yyextra->endMarker="endrtfonly";
1554 BEGIN(St_SecSkip);
1555 }
1556<St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1557 yyextra->endMarker="endxmlonly";
1558 BEGIN(St_SecSkip);
1559 }
1560<St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
1561 yyextra->endMarker="enddocbookonly";
1562 BEGIN(St_SecSkip);
1563 }
1564<St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1565 yyextra->endMarker="endcode";
1566 BEGIN(St_SecSkip);
1567 }
1568<St_Sections>{CMD}"icode"/[^a-z_A-Z0-9] {
1569 yyextra->endMarker="endicode";
1570 BEGIN(St_SecSkip);
1571 }
1572<St_Sections>"<!--" {
1573 yyextra->endMarker="-->";
1574 BEGIN(St_SecSkip);
1575 }
1576<St_SecSkip>{CMD}{ID} {
1577 if (yyextra->endMarker==yytext+1)
1578 {
1579 BEGIN(St_Sections);
1580 }
1581 }
1582<St_SecSkip>"-->" {
1583 if (yyextra->endMarker==yytext)
1584 {
1585 BEGIN(St_Sections);
1586 }
1587 }
1588<St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1589<St_SecSkip>.
1590<St_SecSkip>(\n|"\\ilinebr")
1591<St_Sections>.
1592<St_Sections>(\n|"\\ilinebr")
1593<St_SecLabel1>({REQID}|{LABELID}) {
1594 lineCount(yytext,yyleng);
1595 yyextra->secLabel = yytext;
1596 processSection(yyscanner);
1597 BEGIN(St_Sections);
1598 }
1599<St_SecLabel2>({REQID}|{LABELID}){BLANK}+ |
1600<St_SecLabel2>({REQID}|{LABELID}) {
1601 yyextra->secLabel = yytext;
1602 yyextra->secLabel = yyextra->secLabel.stripWhiteSpace();
1603 BEGIN(St_SecTitle);
1604 }
1605<St_SecTitle>[^\n]+ |
1606<St_SecTitle>[^\n]*\n {
1607 lineCount(yytext,yyleng);
1608 yyextra->secTitle = yytext;
1609 yyextra->secTitle = yyextra->secTitle.stripWhiteSpace();
1610 if (yyextra->secTitle.endsWith("\\ilinebr"))
1611 {
1612 yyextra->secTitle.left(yyextra->secTitle.length()-8);
1613 }
1614 processSection(yyscanner);
1615 BEGIN(St_Sections);
1616 }
1617<St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1618 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}' while looking for section label or title",yytext);
1619 }
1620
1621<St_Snippet>[^\\\n]+ {
1622 yyextra->token.name += yytext;
1623 }
1624<St_Snippet>"\\" {
1625 yyextra->token.name += yytext;
1626 }
1627<St_Snippet>(\n|"\\ilinebr") {
1628 if (yyextra->token.name.stripWhiteSpace().empty() && !yyextra->snippetRtn)
1629 {
1630 lineCount(yytext,yyleng);
1631 yyextra->snippetRtn++;
1632 }
1633 else
1634 {
1635 unput_string(yytext,yyleng);
1636 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1637 return Token::make_TK_WORD();
1638 }
1639 }
1640
1641 /* Generic rules that work for all states */
1642<*>\n {
1643 lineCount(yytext,yyleng);
1644 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected new line character");
1645 }
1646<*>"\\ilinebr" {
1647 }
1648<*>[\\@<>&$#%~"=] { /* unescaped special character */
1649 //warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}', assuming command \\{} was meant.",yytext,yytext);
1650 yyextra->token.name = yytext;
1651 return Token::char_to_command(yytext[0]);
1652 }
1653<*>. {
1654 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}'",yytext);
1655 }
1656%%

◆ yyread()

int yyread ( yyscan_t yyscanner,
char * buf,
int max_size )
static

Definition at line 1660 of file doctokenizer.l.

1661{
1662 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1663 int c=0;
1664 const char *p = yyextra->inputString + yyextra->inputPos;
1665 while ( c < max_size && *p ) { *buf++ = *p++; c++; }
1666 yyextra->inputPos+=c;
1667 return c;
1668}

References yyguts_t.