Doxygen
Loading...
Searching...
No Matches
doctokenizer.l File Reference
#include <stdint.h>
#include <ctype.h>
#include <stack>
#include <string>
#include <cassert>
#include "doctokenizer.h"
#include "cmdmapper.h"
#include "config.h"
#include "message.h"
#include "section.h"
#include "membergroup.h"
#include "definition.h"
#include "doxygen.h"
#include "portable.h"
#include "cite.h"
#include "regex.h"
#include "debug.h"
#include "docnode.h"
#include "stringutil.h"
#include "doxygen_lex.h"
#include "doctokenizer.l.h"
Include dependency graph for doctokenizer.l:

Go to the source code of this file.

Classes

struct  DocLexerContext
struct  doctokenizerYY_state
struct  DocTokenizer::Private

Macros

#define YY_TYPEDEF_YY_SCANNER_T
#define YY_NO_INPUT   1
#define YY_NO_UNISTD_H   1
#define lineCount(s, len)
#define unput_string(yytext, yyleng)
#define YY_INPUT(buf, result, max_size)
#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)
#define yyterminate()

Typedefs

typedef yyguts_t * yyscan_t

Functions

static const char * stateToString (int state)
static int yyread (yyscan_t yyscanner, char *buf, int max_size)
static void handleHtmlTag (yyscan_t yyscanner, const char *text)
static void processSection (yyscan_t yyscanner)
QCString extractPartAfterNewLine (const QCString &text)
static int computeIndent (const char *str, size_t length)
static const char * getLexerFILE ()
int yylex (yyscan_t yyscanner)

Macro Definition Documentation

◆ lineCount

#define lineCount ( s,
len )
Value:
do { for(int i=0;i<(int)len;i++) if (s[i]=='\n') yyextra->yyLineNr++; } while(0)

Definition at line 98 of file doctokenizer.l.

Referenced by CitationManager::insertCrossReferencesForBibFile().

◆ unput_string

#define unput_string ( yytext,
yyleng )
Value:
do { for (int i=(int)yyleng-1;i>=0;i--) unput(yytext[i]); } while(0)

Definition at line 160 of file doctokenizer.l.

◆ YY_DECL

#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)

Definition at line 171 of file doctokenizer.l.

◆ YY_INPUT

#define YY_INPUT ( buf,
result,
max_size )
Value:
result=yyread(yyscanner,buf,max_size);
yyread
static int yyread(yyscan_t yyscanner, char *buf, int max_size)
Definition code.l:3982

Definition at line 164 of file doctokenizer.l.

◆ YY_NO_INPUT

#define YY_NO_INPUT   1

Definition at line 53 of file doctokenizer.l.

◆ YY_NO_UNISTD_H

#define YY_NO_UNISTD_H   1

Definition at line 54 of file doctokenizer.l.

◆ YY_TYPEDEF_YY_SCANNER_T

#define YY_TYPEDEF_YY_SCANNER_T

Definition at line 26 of file doctokenizer.l.

◆ yyterminate

#define yyterminate ( )
Value:
return Token::make_TK_EOF()

Definition at line 174 of file doctokenizer.l.

Typedef Documentation

◆ yyscan_t

typedef yyguts_t* yyscan_t

Definition at line 28 of file doctokenizer.l.

Function Documentation

◆ computeIndent()

int computeIndent ( const char * str,
size_t length )
static

Definition at line 127 of file doctokenizer.l.

128{
129 if (str==0 || length==std::string::npos) return 0;
130 size_t i;
131 int indent=0;
132 int tabSize=Config_getInt(TAB_SIZE);
133 for (i=0;i<length;i++)
134 {
135 if (str[i]=='\t')
136 {
137 indent+=tabSize - (indent%tabSize);
138 }
139 else if (str[i]=='\n')
140 {
141 indent=0;
142 }
143 else if (literal_at(&str[i],"\\ilinebr"))
144 {
145 indent=0;
146 i+=7;
147 if (str[i+1]==' ') i++; // also eat space after \\ilinebr if present
148 }
149 else
150 {
151 indent++;
152 }
153 }
154 //printf("input('%s')=%d\n",str,indent);
155 return indent;
Config_getInt
#define Config_getInt(name)
Definition config.h:34
literal_at
bool literal_at(const char *data, const char(&str)[N])
returns TRUE iff data points to a substring that matches string literal str
Definition stringutil.h:98
156}

◆ extractPartAfterNewLine()

QCString extractPartAfterNewLine ( const QCString & text)

Definition at line 109 of file doctokenizer.l.

110{
111 int nl1 = text.find('\n');
112 int nl2 = text.find("\\ilinebr");
113 if (nl1!=-1 && nl1<nl2)
114 {
115 return text.mid(nl1+1);
116 }
117 if (nl2!=-1)
118 {
119 if (text.at(nl2+8)==' ') nl2++; // skip space after \\ilinebr
120 return text.mid(nl2+8);
121 }
122 return text;
QCString::find
int find(char c, int index=0, bool cs=TRUE) const
Definition qcstring.cpp:43
QCString::mid
QCString mid(size_t index, size_t len=static_cast< size_t >(-1)) const
Definition qcstring.h:241
QCString::at
char & at(size_t i)
Returns a reference to the character at index i.
Definition qcstring.h:593
123}

References QCString::mid().

◆ getLexerFILE()

const char * getLexerFILE ( )
inlinestatic

Definition at line 167 of file doctokenizer.l.

167{return __FILE__;}

◆ handleHtmlTag()

void handleHtmlTag ( yyscan_t yyscanner,
const char * text )
static

Definition at line 1623 of file doctokenizer.l.

1624{
1625 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1626
1627 QCString tagText(text);
1628 yyextra->token.text = tagText;
1629 yyextra->token.attribs.clear();
1630 yyextra->token.endTag = FALSE;
1631 yyextra->token.emptyTag = FALSE;
QCString
This is an alternative implementation of QCString.
Definition qcstring.h:101
FALSE
#define FALSE
Definition qcstring.h:34
1632
1633 // Check for end tag
1634 int startNamePos=1;
1635 if (tagText.at(1)=='/')
1636 {
1637 yyextra->token.endTag = TRUE;
1638 startNamePos++;
1639 }
TRUE
#define TRUE
Definition qcstring.h:37
1640
1641 // Parse the name portion
1642 int i = startNamePos;
1643 for (i=startNamePos; i < (int)yyleng; i++)
1644 {
1645 // Check for valid HTML/XML name chars (including namespaces)
1646 char c = tagText.at(i);
1647 if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
1648 }
1649 yyextra->token.name = tagText.mid(startNamePos,i-startNamePos);
1650
1651 // Parse the attributes. Each attribute is a name, value pair
1652 // The result is stored in yyextra->token.attribs.
1653 int startAttribList = i;
1654 while (i<(int)yyleng)
1655 {
1656 char c=tagText.at(i);
1657 // skip spaces
1658 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1659 // check for end of the tag
1660 if (c == '>') break;
1661 // Check for XML style "empty" tag.
1662 if (c == '/')
1663 {
1664 yyextra->token.emptyTag = TRUE;
1665 break;
1666 }
1667 int startName=i;
1668 // search for end of name
1669 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='=' && c!= '>') { c=tagText.at(++i); }
1670 int endName=i;
1671 QCString optName,optValue;
1672 optName = tagText.mid(startName,endName-startName).lower();
1673 // skip spaces
1674 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1675 if (tagText.at(i)=='=') // option has value
1676 {
1677 int startAttrib=0, endAttrib=0;
1678 c=tagText.at(++i);
1679 // skip spaces
1680 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1681 if (tagText.at(i)=='\'') // option '...'
1682 {
1683 c=tagText.at(++i);
1684 startAttrib=i;
QCString::lower
QCString lower() const
Definition qcstring.h:249
1685
1686 // search for matching quote
1687 while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
1688 endAttrib=i;
1689 if (i<(int)yyleng) { c=tagText.at(++i);}
1690 }
1691 else if (tagText.at(i)=='"') // option "..."
1692 {
1693 c=tagText.at(++i);
1694 startAttrib=i;
1695 // search for matching quote
1696 while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
1697 endAttrib=i;
1698 if (i<(int)yyleng) { c=tagText.at(++i);}
1699 }
1700 else // value without any quotes
1701 {
1702 startAttrib=i;
1703 // search for separator or end symbol
1704 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='>') { c=tagText.at(++i); }
1705 endAttrib=i;
1706 if (i<(int)yyleng) { c=tagText.at(++i);}
1707 }
1708 optValue = tagText.mid(startAttrib,endAttrib-startAttrib);
1709 if (optName == "align") optValue = optValue.lower();
1710 else if (optName == "valign")
1711 {
1712 optValue = optValue.lower();
1713 if (optValue == "center") optValue="middle";
1714 }
1715 }
1716 else // start next option
1717 {
1718 }
1719 //printf("=====> Adding option name=<%s> value=<%s>\n",
1720 // qPrint(optName),qPrint(optValue));
1721 yyextra->token.attribs.emplace_back(optName,optValue);
1722 }
1723 yyextra->token.attribsStr = tagText.mid(startAttribList,i-startAttribList);
1724}

References FALSE, and TRUE.

◆ processSection()

void processSection ( yyscan_t yyscanner)
static

Definition at line 1602 of file doctokenizer.l.

1603{
1604 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1605 //printf("%s: found section/anchor with name '%s'\n",qPrint(yyextra->fileName),qPrint(yyextra->secLabel));
1606 QCString file;
1607 if (yyextra->definition)
1608 {
1609 file = yyextra->definition->getOutputFileBase();
1610 }
1611 else
1612 {
1613 warn(yyextra->fileName,yyextra->yyLineNr,"Found section/anchor {} without context",yyextra->secLabel);
1614 }
1615 SectionInfo *si = SectionManager::instance().find(yyextra->secLabel);
1616 if (si)
1617 {
1618 si->setFileName(file);
1619 si->setType(yyextra->secType);
1620 }
LinkedMap::find
const T * find(const std::string &key) const
Definition linkedmap.h:47
SectionInfo
class that provide information about a section.
Definition section.h:58
SectionInfo::setType
void setType(SectionType t)
Definition section.h:81
SectionInfo::setFileName
void setFileName(const QCString &fn)
Definition section.h:80
SectionManager::instance
static SectionManager & instance()
returns a reference to the singleton
Definition section.h:179
warn
#define warn(file, line, fmt,...)
Definition message.h:97
1621}

◆ stateToString()

const char * stateToString ( int state)
static

◆ yylex()

int yylex ( yyscan_t yyscanner)

Definition at line 338 of file doctokenizer.l.

340 {LISTITEM} { /* list item */
341 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
342 lineCount(yytext,yyleng);
343 QCString text(yytext);
344 uint32_t dashPos = static_cast<uint32_t>(text.findRev('-'));
345 assert(dashPos!=static_cast<uint32_t>(-1));
346 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
347 yyextra->token.isCheckedList = false;
348 yyextra->token.id = -1;
349 yyextra->token.indent = computeIndent(yytext,dashPos);
350 return Token::make_TK_LISTITEM();
351 }
lineCount
#define lineCount(s, len)
Definition doctokenizer.l:98
computeIndent
static int computeIndent(const char *str, size_t length)
Definition doctokenizer.l:127
352<St_Para>^{CLISTITEM} { /* checkbox item */
353 QCString text=yytext;
354 int dashPos = text.findRev('-');
355 yyextra->token.isEnumList = false;
356 yyextra->token.isCheckedList = true;
357 if (text.find('x') != -1) yyextra->token.id = DocAutoList::Checked_x;
358 else if (text.find('X') != -1) yyextra->token.id = DocAutoList::Checked_X;
359 else yyextra->token.id = DocAutoList::Unchecked;
360 yyextra->token.indent = computeIndent(yytext,dashPos);
361 return Token::make_TK_LISTITEM();
362 }
QCString::findRev
int findRev(char c, int index=-1, bool cs=TRUE) const
Definition qcstring.cpp:96
363<St_Para>^{MLISTITEM} { /* list item */
364 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
365 {
366 REJECT;
367 }
368 else
369 {
370 lineCount(yytext,yyleng);
371 std::string text(yytext);
372 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
373 reg::Match match;
374 reg::search(text,match,re);
375 size_t listPos = match.position();
376 assert(listPos!=std::string::npos);
377 yyextra->token.isEnumList = false;
378 yyextra->token.isCheckedList = false;
379 yyextra->token.id = -1;
380 yyextra->token.indent = computeIndent(yytext,listPos);
381 return Token::make_TK_LISTITEM();
382 }
383 }
reg::Ex
Class representing a regular expression.
Definition regex.h:39
reg::Match
Object representing the matching results.
Definition regex.h:151
reg::search
bool search(std::string_view str, Match &match, const Ex &re, size_t pos)
Search in a given string str starting at position pos for a match against regular expression re.
Definition regex.cpp:844
reg::match
bool match(std::string_view str, Match &match, const Ex &re)
Matches a given string str for a match against regular expression re.
Definition regex.cpp:855
384<St_Para>^{OLISTITEM} { /* numbered list item */
385 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
386 {
387 REJECT;
388 }
389 else
390 {
391 std::string text(yytext);
392 static const reg::Ex re(R"(\d+)");
393 reg::Match match;
394 reg::search(text,match,re);
395 size_t markPos = match.position();
396 assert(markPos!=std::string::npos);
397 yyextra->token.isEnumList = true;
398 yyextra->token.isCheckedList = false;
399 bool ok = false;
400 int id = QCString(match.str()).toInt(&ok);
401 yyextra->token.id = ok ? id : -1;
402 if (!ok)
403 {
404 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '{}' ",match.str());
405 }
406 yyextra->token.indent = computeIndent(yytext,markPos);
407 return Token::make_TK_LISTITEM();
408 }
409 }
QCString::toInt
int toInt(bool *ok=nullptr, int base=10) const
Definition qcstring.cpp:254
410<St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM} { /* list item on next line */
411 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
412 lineCount(yytext,yyleng);
413 QCString text=extractPartAfterNewLine(yytext);
414 uint32_t dashPos = static_cast<uint32_t>(text.findRev('-'));
415 assert(dashPos!=static_cast<uint32_t>(-1));
416 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
417 yyextra->token.isCheckedList = false;
418 yyextra->token.id = -1;
419 yyextra->token.indent = computeIndent(text.data(),dashPos);
420 return Token::make_TK_LISTITEM();
421 }
QCString::data
const char * data() const
Returns a pointer to the contents of the string in the form of a 0-terminated C string.
Definition qcstring.h:172
extractPartAfterNewLine
QCString extractPartAfterNewLine(const QCString &text)
Definition doctokenizer.l:109
422<St_Para>{BLANK}*\n{CLISTITEM} { /* checkbox item on next line */
423 QCString text=yytext;
424 text=text.right(text.length()-text.find('\n')-1);
425 int dashPos = text.findRev('-');
426 yyextra->token.isEnumList = false;
427 yyextra->token.isCheckedList = true;
428 if (text.find('x') != -1) yyextra->token.id = DocAutoList::Checked_x;
429 else if (text.find('X') != -1) yyextra->token.id = DocAutoList::Checked_X;
430 else yyextra->token.id = DocAutoList::Unchecked;
431 yyextra->token.indent = computeIndent(text.data(),dashPos);
432 return Token::make_TK_LISTITEM();
433 }
QCString::length
size_t length() const
Returns the length of the string, not counting the 0-terminator.
Definition qcstring.h:166
QCString::right
QCString right(size_t len) const
Definition qcstring.h:234
434<St_Para>{BLANK}*(\n|"\\ilinebr"){MLISTITEM} { /* list item on next line */
435 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
436 {
437 REJECT;
438 }
439 else
440 {
441 lineCount(yytext,yyleng);
442 std::string text=extractPartAfterNewLine(yytext).str();
443 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
444 reg::Match match;
445 reg::search(text,match,re);
446 size_t markPos = match.position();
447 assert(markPos!=std::string::npos);
448 yyextra->token.isEnumList = FALSE;
449 yyextra->token.isCheckedList = false;
450 yyextra->token.id = -1;
451 yyextra->token.indent = computeIndent(text.c_str(),markPos);
452 return Token::make_TK_LISTITEM();
453 }
454 }
QCString::str
const std::string & str() const
Definition qcstring.h:552
455<St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM} { /* list item on next line */
456 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
457 {
458 REJECT;
459 }
460 else
461 {
462 lineCount(yytext,yyleng);
463 std::string text=extractPartAfterNewLine(yytext).str();
464 static const reg::Ex re(R"(\d+)");
465 reg::Match match;
466 reg::search(text,match,re);
467 size_t markPos = match.position();
468 assert(markPos!=std::string::npos);
469 yyextra->token.isEnumList = true;
470 yyextra->token.isCheckedList = false;
471 bool ok = false;
472 int id = QCString(match.str()).toInt(&ok);
473 yyextra->token.id = ok ? id : -1;
474 if (!ok)
475 {
476 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '{}' ",match.str());
477 }
478 yyextra->token.indent = computeIndent(text.c_str(),markPos);
479 return Token::make_TK_LISTITEM();
480 }
481 }
482<St_Para>^{ENDLIST} { /* end list */
483 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
484 lineCount(yytext,yyleng);
485 size_t dotPos = static_cast<size_t>(QCString(yytext).findRev('.'));
486 yyextra->token.indent = computeIndent(yytext,dotPos);
487 return Token::make_TK_ENDLIST();
488 }
489<St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST} { /* end list on next line */
490 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
491 lineCount(yytext,yyleng);
492 QCString text=extractPartAfterNewLine(yytext);
493 size_t dotPos = static_cast<size_t>(text.findRev('.'));
494 yyextra->token.indent = computeIndent(text.data(),dotPos);
495 return Token::make_TK_ENDLIST();
496 }
497<St_Para>"{"{BLANK}*"@linkplain"/{WS}+ {
498 yyextra->token.name = "javalinkplain";
499 return Token::make_TK_COMMAND_AT();
500 }
501<St_Para>"{"{BLANK}*"@link"/{WS}+ {
502 yyextra->token.name = "javalink";
503 return Token::make_TK_COMMAND_AT();
504 }
505<St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
506 yyextra->token.name = "inheritdoc";
507 return Token::make_TK_COMMAND_AT();
508 }
509<St_Para>"@_fakenl" { // artificial new line
510 //yyextra->yyLineNr++;
511 }
512<St_Para>{SPCMD3} {
513 yyextra->token.name = "_form";
514 bool ok;
515 yyextra->token.id = QCString(yytext).right((int)yyleng-7).toInt(&ok);
516 ASSERT(ok);
517 return Token::char_to_command(yytext[0]);
518 }
Token::char_to_command
static Token char_to_command(char c)
Definition doctokenizer.h:111
ASSERT
#define ASSERT(x)
Definition qcstring.h:39
519<St_Para>{CMD}"n"\n { /* \n followed by real newline */
520 lineCount(yytext,yyleng);
521 //yyextra->yyLineNr++;
522 yyextra->token.name = yytext+1;
523 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
524 yyextra->token.paramDir=TokenInfo::Unspecified;
525 return Token::char_to_command(yytext[0]);
526 }
527<St_Para>"\\ilinebr" {
528 }
529<St_Para>{SPCMD1} |
530<St_Para>{SPCMD2} |
531<St_Para>{SPCMD5} |
532<St_Para>{SPCMD4} { /* special command */
533 yyextra->token.name = yytext+1;
534 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
535 yyextra->token.paramDir=TokenInfo::Unspecified;
536 return Token::char_to_command(yytext[0]);
537 }
538<St_Para>{PARAMIO} { /* param [in,out] command */
539 yyextra->token.name = "param";
540 QCString s(yytext);
541 bool isIn = s.find("in")!=-1;
542 bool isOut = s.find("out")!=-1;
543 if (isIn)
544 {
545 if (isOut)
546 {
547 yyextra->token.paramDir=TokenInfo::InOut;
548 }
549 else
550 {
551 yyextra->token.paramDir=TokenInfo::In;
552 }
553 }
554 else if (isOut)
555 {
556 yyextra->token.paramDir=TokenInfo::Out;
557 }
558 else
559 {
560 yyextra->token.paramDir=TokenInfo::Unspecified;
561 }
562 return Token::char_to_command(yytext[0]);
563 }
564<St_Para>{URLPROTOCOL}{URLMASK}/[,\.] { // URL, or URL.
565 yyextra->token.name=yytext;
566 yyextra->token.isEMailAddr=FALSE;
567 return Token::make_TK_URL();
568 }
569<St_Para>{URLPROTOCOL}{URLMASK} { // URL
570 yyextra->token.name=yytext;
571 yyextra->token.isEMailAddr=FALSE;
572 return Token::make_TK_URL();
573 }
574<St_Para>"<"{URLPROTOCOL}{URLMASK}">" { // URL
575 yyextra->token.name=yytext;
576 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
577 yyextra->token.isEMailAddr=FALSE;
578 return Token::make_TK_URL();
579 }
580<St_Para>{MAILADDR} { // Mail address
581 yyextra->token.name=yytext;
582 yyextra->token.name.stripPrefix("mailto:");
583 yyextra->token.isEMailAddr=TRUE;
584 return Token::make_TK_URL();
585 }
586<St_Para>"<"{MAILADDR}">" { // Mail address
587 yyextra->token.name=yytext;
588 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
589 yyextra->token.name.stripPrefix("mailto:");
590 yyextra->token.isEMailAddr=TRUE;
591 return Token::make_TK_URL();
592 }
593<St_Para>"<"{MAILADDR2}">" { // anti spam mail address
594 yyextra->token.name=yytext;
595 return Token::make_TK_WORD();
596 }
597<St_Para>{RCSID} { /* RCS tag */
598 QCString tagName(yytext+1);
599 int index=tagName.find(':');
600 if (index<0) index=0; // should never happen
601 yyextra->token.name = tagName.left(index);
602 int text_begin = index+2;
603 int text_end = static_cast<int>(tagName.length())-1;
604 if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
605 {
606 ++text_begin;
607 if (tagName[text_end-1]=='#')
608 {
609 --text_end;
610 }
611 }
612 yyextra->token.text = tagName.mid(text_begin,text_end-text_begin);
613 return Token::make_TK_RCSTAG();
614 }
615<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}")" | /* environment variable */
616<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}"("{ID}"))" { /* environment variable */
617 QCString name(&yytext[2]);
618 name = name.left(static_cast<int>(name.length())-1);
619 QCString value = Portable::getenv(name);
620 for (int i=static_cast<int>(value.length())-1;i>=0;i--) unput(value.at(i));
621 }
Portable::getenv
QCString getenv(const QCString &variable)
Definition portable.cpp:322
622<St_Para>"<blockquote>&zwj;" {
623 // for a markdown inserted block quote,
624 // tell flex that after putting the last indent
625 // back we are at the beginning of the line, see issue #11309
626 YY_CURRENT_BUFFER->yy_at_bol=1;
627 lineCount(yytext,yyleng);
628 handleHtmlTag(yyscanner,yytext);
629 return Token::make_TK_HTMLTAG();
630 }
handleHtmlTag
static void handleHtmlTag(yyscan_t yyscanner, const char *text)
Definition doctokenizer.l:1623
631<St_Para>{HTMLTAG} { /* html tag */
632 lineCount(yytext,yyleng);
633 handleHtmlTag(yyscanner,yytext);
634 return Token::make_TK_HTMLTAG();
635 }
636<St_Para,St_Text>"&"{ID}";" { /* special symbol */
637 yyextra->token.name = yytext;
638 return Token::make_TK_SYMBOL();
639 }
640
641 /********* patterns for linkable words ******************/
642
643<St_Para>{ID}/"<"{HTMLKEYW}">"+ { /* this rule is to prevent opening html
644 * tag to be recognized as a templated classes
645 */
646 yyextra->token.name = yytext;
647 return Token::make_TK_LNKWORD();
648 }
649<St_Para>{LNKWORDN}/("<"{HTMLKEYW}">")+ { // prevent <br> html tag to be parsed as template arguments
650 yyextra->token.name = yytext;
651 return Token::make_TK_LNKWORD();
652 }
653<St_Para>{LNKWORD1} |
654<St_Para>{LNKWORD1}{FUNCARG} |
655<St_Para>{LNKWORD2} |
656<St_Para>{LNKWORD3} |
657<St_Para>{LNKWORD4} {
658 yyextra->token.name = yytext;
659 return Token::make_TK_LNKWORD();
660 }
661<St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
662 yyextra->token.name = yytext;
663 yyextra->token.name = yyextra->token.name.left(yyextra->token.name.length()-1);
664 unput(yytext[(int)yyleng-1]);
665 return Token::make_TK_LNKWORD();
666 }
667 /********* patterns for normal words ******************/
668
669<St_Para,St_Text>[\-+0-9] |
670<St_Para,St_Text>{WORD1} |
671<St_Para,St_Text>{WORD2} { /* function call */
672 if (QCString(yytext).find("\\ilinebr")!=-1) REJECT; // see issue #8311
673 lineCount(yytext,yyleng);
674 if (yytext[0]=='%') // strip % if present
675 yyextra->token.name = &yytext[1];
676 else
677 yyextra->token.name = yytext;
678 return Token::make_TK_WORD();
679 }
680<St_Text>({ID}".")+{ID} {
681 yyextra->token.name = yytext;
682 return Token::make_TK_WORD();
683 }
684<St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
685 // avoid interpretation as "operator <"
686 yyextra->token.name = yytext;
687 return Token::make_TK_WORD();
688 }
689
690 /*******************************************************/
691
692<St_Para,St_Text>{BLANK}+ |
693<St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
694 lineCount(yytext,yyleng);
695 yyextra->token.chars=yytext;
696 return Token::make_TK_WHITESPACE();
697 }
698<St_Text>[\\@<>&$#%~] {
699 yyextra->token.name = yytext;
700 return Token::char_to_command(yytext[0]);
701 }
702<St_Para>({BLANK}*\n)+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
703 if (yyextra->insidePre || yyextra->autoListLevel==0)
704 {
705 REJECT;
706 }
707 lineCount(yytext,yyleng);
708 }
709<St_Para>({BLANK}*\n)+{BLANK}*\n/{CLISTITEM} { /* skip trailing paragraph followed by new checkbox item */
710 if (yyextra->insidePre || yyextra->autoListLevel==0)
711 {
712 REJECT;
713 }
714 }
715<St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
716 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
717 {
718 REJECT;
719 }
720 lineCount(yytext,yyleng);
721 }
722<St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
723 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
724 {
725 REJECT;
726 }
727 lineCount(yytext,yyleng);
728 }
729<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}*/" \\ifile" | // we don't want to count the space before \ifile
730<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}* {
731 lineCount(yytext,yyleng);
732 if (yyextra->insidePre)
733 {
734 yyextra->token.chars=yytext;
735 return Token::make_TK_WHITESPACE();
736 }
737 else
738 {
739 yyextra->token.indent=computeIndent(yytext,yyleng);
740 int i;
741 // put back the indentation (needed for list items)
742 //printf("token.indent=%d\n",yyextra->token.indent);
743 for (i=0;i<yyextra->token.indent;i++)
744 {
745 unput(' ');
746 }
747 // tell flex that after putting the last indent
748 // back we are at the beginning of the line
749 YY_CURRENT_BUFFER->yy_at_bol=1;
750 // start of a new paragraph
751 return Token::make_TK_NEWPARA();
752 }
753 }
754<St_CodeOpt>{BLANK}*"{"(".")?{CODEID}"}" {
755 yyextra->token.name = yytext;
756 int i=yyextra->token.name.find('{'); /* } to keep vi happy */
757 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
758 BEGIN(St_Code);
759 }
760<St_iCodeOpt>{BLANK}*"{"(".")?{CODEID}"}" {
761 yyextra->token.name = yytext;
762 int i=yyextra->token.name.find('{'); /* } to keep vi happy */
763 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
764 BEGIN(St_iCode);
765 }
766<St_CodeOpt>"\\ilinebr" |
767<St_CodeOpt>\n |
768<St_CodeOpt>. {
769 unput_string(yytext,yyleng);
770 BEGIN(St_Code);
771 }
unput_string
#define unput_string(yytext, yyleng)
Definition doctokenizer.l:160
772<St_iCodeOpt>"\\ilinebr" |
773<St_iCodeOpt>\n |
774<St_iCodeOpt>. {
775 unput_string(yytext,yyleng);
776 BEGIN(St_iCode);
777 }
778<St_Code>{WS}*{CMD}"endcode" {
779 lineCount(yytext,yyleng);
780 return Token::make_RetVal_OK();
781 }
782<St_iCode>{WS}*{CMD}"endicode" {
783 lineCount(yytext,yyleng);
784 return Token::make_RetVal_OK();
785 }
786<St_XmlCode>{WS}*"</code>" {
787 lineCount(yytext,yyleng);
788 return Token::make_RetVal_OK();
789 }
790<St_Code,St_iCode,St_XmlCode>[^\\@\n<]+ |
791<St_Code,St_iCode,St_XmlCode>\n |
792<St_Code,St_iCode,St_XmlCode>. {
793 lineCount(yytext,yyleng);
794 yyextra->token.verb+=yytext;
795 }
796<St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
797 yyextra->token.name="block";
798 BEGIN(St_HtmlOnly);
799 }
800<St_HtmlOnlyOption>.|\n {
801 unput(*yytext);
802 BEGIN(St_HtmlOnly);
803 }
804<St_HtmlOnlyOption>"\\ilinebr" {
805 unput_string(yytext,yyleng);
806 BEGIN(St_HtmlOnly);
807 }
808<St_HtmlOnly>{CMD}"endhtmlonly" {
809 return Token::make_RetVal_OK();
810 }
811<St_HtmlOnly>[^\\@\n$]+ |
812<St_HtmlOnly>\n |
813<St_HtmlOnly>. {
814 lineCount(yytext,yyleng);
815 yyextra->token.verb+=yytext;
816 }
817<St_ManOnly>{CMD}"endmanonly" {
818 return Token::make_RetVal_OK();
819 }
820<St_ManOnly>[^\\@\n$]+ |
821<St_ManOnly>\n |
822<St_ManOnly>. {
823 lineCount(yytext,yyleng);
824 yyextra->token.verb+=yytext;
825 }
826<St_RtfOnly>{CMD}"endrtfonly" {
827 return Token::make_RetVal_OK();
828 }
829<St_RtfOnly>[^\\@\n$]+ |
830<St_RtfOnly>\n |
831<St_RtfOnly>. {
832 lineCount(yytext,yyleng);
833 yyextra->token.verb+=yytext;
834 }
835<St_LatexOnly>{CMD}"endlatexonly" {
836 return Token::make_RetVal_OK();
837 }
838<St_LatexOnly>[^\\@\n]+ |
839<St_LatexOnly>\n |
840<St_LatexOnly>. {
841 lineCount(yytext,yyleng);
842 yyextra->token.verb+=yytext;
843 }
844<St_XmlOnly>{CMD}"endxmlonly" {
845 return Token::make_RetVal_OK();
846 }
847<St_XmlOnly>[^\\@\n]+ |
848<St_XmlOnly>\n |
849<St_XmlOnly>. {
850 lineCount(yytext,yyleng);
851 yyextra->token.verb+=yytext;
852 }
853<St_DbOnly>{CMD}"enddocbookonly" {
854 return Token::make_RetVal_OK();
855 }
856<St_DbOnly>[^\\@\n]+ |
857<St_DbOnly>\n |
858<St_DbOnly>. {
859 lineCount(yytext,yyleng);
860 yyextra->token.verb+=yytext;
861 }
862<St_Verbatim>{CMD}"endverbatim" {
863 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
864 return Token::make_RetVal_OK();
865 }
866<St_ILiteral>{CMD}"endiliteral " { // note extra space as this is artificially added
867 // remove spaces that have been added
868 yyextra->token.verb = yyextra->token.verb.mid(1,yyextra->token.verb.length()-2);
869 return Token::make_RetVal_OK();
870 }
871<St_iVerbatim>{CMD}"endiverbatim" {
872 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
873 return Token::make_RetVal_OK();
874 }
875<St_Verbatim,St_iVerbatim,St_ILiteral>[^\\@\n]+ |
876<St_Verbatim,St_iVerbatim,St_ILiteral>\n |
877<St_Verbatim,St_iVerbatim,St_ILiteral>. { /* Verbatim text */
878 lineCount(yytext,yyleng);
879 yyextra->token.verb+=yytext;
880 }
881<St_ILiteralOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // option(s) present
882 yyextra->token.verb = QCString(yytext).stripWhiteSpace();
883 return Token::make_RetVal_OK();
884 }
QCString::stripWhiteSpace
QCString stripWhiteSpace() const
returns a copy of this string with leading and trailing whitespace removed
Definition qcstring.h:260
885<St_ILiteralOpt>"\\ilinebr" |
886<St_ILiteralOpt>"\n" |
887<St_ILiteralOpt>. {
888 yyextra->token.sectionId = "";
889 unput_string(yytext,yyleng);
890 return Token::make_RetVal_OK();
891 }
892<St_Dot>{CMD}"enddot" {
893 return Token::make_RetVal_OK();
894 }
895<St_Dot>[^\\@\n]+ |
896<St_Dot>\n |
897<St_Dot>. { /* dot text */
898 lineCount(yytext,yyleng);
899 yyextra->token.verb+=yytext;
900 }
901<St_Msc>{CMD}("endmsc") {
902 return Token::make_RetVal_OK();
903 }
904<St_Msc>[^\\@\n]+ |
905<St_Msc>\n |
906<St_Msc>. { /* msc text */
907 lineCount(yytext,yyleng);
908 yyextra->token.verb+=yytext;
909 }
910<St_PlantUMLOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
911 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
912 return Token::make_RetVal_OK();
913 }
914<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
915 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
916 return Token::make_RetVal_OK();
917 }
918<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
919 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
920 return Token::make_RetVal_OK();
921 }
922<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
923 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
924 return Token::make_RetVal_OK();
925 }
926<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
927 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
928 return Token::make_RetVal_OK();
929 }
930<St_PlantUMLOpt>"\\ilinebr" |
931<St_PlantUMLOpt>"\n" |
932<St_PlantUMLOpt>. {
933 yyextra->token.sectionId = "";
934 unput_string(yytext,yyleng);
935 return Token::make_RetVal_OK();
936 }
937<St_PlantUML>{CMD}"enduml" {
938 return Token::make_RetVal_OK();
939 }
940<St_PlantUML>[^\\@\n]+ |
941<St_PlantUML>\n |
942<St_PlantUML>. { /* plantuml text */
943 lineCount(yytext,yyleng);
944 yyextra->token.verb+=yytext;
945 }
946<St_Title>"\"" { // quoted title
947 BEGIN(St_TitleQ);
948 }
949<St_Title>[ \t]+ {
950 yyextra->token.chars=yytext;
951 return Token::make_TK_WHITESPACE();
952 }
953<St_Title>. { // non-quoted title
954 unput(*yytext);
955 BEGIN(St_TitleN);
956 }
957<St_Title>\n {
958 unput(*yytext);
959 return Token::make_TK_NONE();
960 }
961<St_Title>"\\ilinebr" {
962 unput_string(yytext,yyleng);
963 return Token::make_TK_NONE();
964 }
965<St_TitleN>"&"{ID}";" { /* symbol */
966 yyextra->token.name = yytext;
967 return Token::make_TK_SYMBOL();
968 }
969<St_TitleN>{HTMLTAG} {
970 yyextra->token.name = yytext;
971 handleHtmlTag(yyscanner,yytext);
972 return Token::make_TK_HTMLTAG();
973 }
974<St_TitleN>\n { /* new line => end of title */
975 unput(*yytext);
976 return Token::make_TK_NONE();
977 }
978<St_TitleN>"\\ilinebr" { /* new line => end of title */
979 unput_string(yytext,yyleng);
980 return Token::make_TK_NONE();
981 }
982<St_TitleN>{SPCMD1} |
983<St_TitleN>{SPCMD2} { /* special command */
984 yyextra->token.name = yytext+1;
985 yyextra->token.paramDir=TokenInfo::Unspecified;
986 return Token::char_to_command(yytext[0]);
987 }
988<St_TitleN>{ID}"=" { /* attribute */
989 if (yytext[0]=='%') // strip % if present
990 yyextra->token.name = &yytext[1];
991 else
992 yyextra->token.name = yytext;
993 return Token::make_TK_WORD();
994 }
995<St_TitleN>[\-+0-9] |
996<St_TitleN>{WORD1} |
997<St_TitleN>{WORD2} { /* word */
998 if (QCString(yytext).find("\\ilinebr")!=-1) REJECT; // see issue #8311
999 lineCount(yytext,yyleng);
1000 if (yytext[0]=='%') // strip % if present
1001 yyextra->token.name = &yytext[1];
1002 else
1003 yyextra->token.name = yytext;
1004 return Token::make_TK_WORD();
1005 }
1006<St_TitleN>[ \t]+ {
1007 yyextra->token.chars=yytext;
1008 return Token::make_TK_WHITESPACE();
1009 }
1010<St_TitleQ>"&"{ID}";" { /* symbol */
1011 yyextra->token.name = yytext;
1012 return Token::make_TK_SYMBOL();
1013 }
1014<St_TitleQ>(\n|"\\ilinebr") { /* new line => end of title */
1015 unput_string(yytext,yyleng);
1016 return Token::make_TK_NONE();
1017 }
1018<St_TitleQ>{SPCMD1} |
1019<St_TitleQ>{SPCMD2} { /* special command */
1020 yyextra->token.name = yytext+1;
1021 yyextra->token.paramDir=TokenInfo::Unspecified;
1022 return Token::char_to_command(yytext[0]);
1023 }
1024<St_TitleQ>{WORD1NQ} |
1025<St_TitleQ>{WORD2NQ} { /* word */
1026 yyextra->token.name = yytext;
1027 return Token::make_TK_WORD();
1028 }
1029<St_TitleQ>[ \t]+ {
1030 yyextra->token.chars=yytext;
1031 return Token::make_TK_WHITESPACE();
1032 }
1033<St_TitleQ>"\"" { /* closing quote => end of title */
1034 BEGIN(St_TitleA);
1035 return Token::make_TK_NONE();
1036 }
1037<St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
1038 yyextra->token.name = yytext;
1039 int pos = yyextra->token.name.find('=');
1040 if (pos<0) pos=0; // should never happen
1041 yyextra->token.name = yyextra->token.name.left(pos).stripWhiteSpace();
1042 BEGIN(St_TitleV);
1043 }
1044<St_TitleV>[^ \t\r\n]+ { // attribute value
1045 lineCount(yytext,yyleng);
1046 yyextra->token.chars = yytext;
1047 BEGIN(St_TitleN);
1048 return Token::make_TK_WORD();
1049 }
1050<St_TitleV,St_TitleA>. {
1051 unput(*yytext);
1052 return Token::make_TK_NONE();
1053 }
1054<St_TitleV,St_TitleA>(\n|"\\ilinebr") {
1055 unput_string(yytext,yyleng);
1056 return Token::make_TK_NONE();
1057 }
1058
1059<St_Anchor>({REQID}|{LABELID}){WS}? { // anchor
1060 lineCount(yytext,yyleng);
1061 yyextra->token.name = QCString(yytext).stripWhiteSpace();
1062 return Token::make_TK_WORD();
1063 }
1064<St_Anchor>. {
1065 unput(*yytext);
1066 return Token::make_TK_NONE();
1067 }
1068<St_Cite>{CITEID} { // label to cite
1069 if (yytext[0] =='"')
1070 {
1071 yyextra->token.name=yytext+1;
1072 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-2);
1073 }
1074 else
1075 {
1076 yyextra->token.name=yytext;
1077 }
1078 return Token::make_TK_WORD();
1079 }
1080<St_Cite>{BLANK} { // white space
1081 unput(' ');
1082 return Token::make_TK_NONE();
1083 }
1084<St_Cite>(\n|"\\ilinebr") { // new line
1085 unput_string(yytext,yyleng);
1086 return Token::make_TK_NONE();
1087 }
1088<St_Cite>. { // any other character
1089 unput(*yytext);
1090 return Token::make_TK_NONE();
1091 }
1092<St_DoxyConfig>{DOXYCFG} { // config option
1093 yyextra->token.name=yytext;
1094 return Token::make_TK_WORD();
1095 }
1096<St_DoxyConfig>{BLANK} { // white space
1097 unput(' ');
1098 return Token::make_TK_NONE();
1099 }
1100<St_DoxyConfig>(\n|"\\ilinebr") { // new line
1101 unput_string(yytext,yyleng);
1102 return Token::make_TK_NONE();
1103 }
1104<St_DoxyConfig>. { // any other character
1105 unput(*yytext);
1106 return Token::make_TK_NONE();
1107 }
1108<St_Ref>{REFWORD_NOCV}/{BLANK}("const")[a-z_A-Z0-9] { // see bug776988
1109 yyextra->token.name=yytext;
1110 return Token::make_TK_WORD();
1111 }
1112<St_Ref>{REFWORD_NOCV}/{BLANK}("volatile")[a-z_A-Z0-9] { // see bug776988
1113 yyextra->token.name=yytext;
1114 return Token::make_TK_WORD();
1115 }
1116<St_Ref>{REFWORD} { // label to refer to
1117 yyextra->token.name=yytext;
1118 return Token::make_TK_WORD();
1119 }
1120<St_Ref>{BLANK} { // white space
1121 unput(' ');
1122 return Token::make_TK_NONE();
1123 }
1124<St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
1125 yyextra->expectQuote=true;
1126 lineCount(yytext,yyleng);
1127 BEGIN(St_Ref2);
1128 }
1129<St_Ref>(\n|"\\ilinebr") { // new line
1130 unput_string(yytext,yyleng);
1131 return Token::make_TK_NONE();
1132 }
1133<St_Ref>"\""[^"\n]+"\"" { // quoted first argument -> return without quotes
1134 yyextra->token.name=QCString(yytext).mid(1,yyleng-2);
1135 return Token::make_TK_WORD();
1136 }
1137<St_Ref>. { // any other character
1138 unput(*yytext);
1139 return Token::make_TK_NONE();
1140 }
1141<St_IntRef>[A-Z_a-z0-9.:/#\-\+\‍(\‍)]+ {
1142 yyextra->token.name = yytext;
1143 return Token::make_TK_WORD();
1144 }
1145<St_IntRef>{BLANK}+"\"" {
1146 BEGIN(St_Ref2);
1147 }
1148<St_SetScope>({SCOPEMASK}|{ANONNS}){BLANK}|{FILEMASK} {
1149 yyextra->token.name = yytext;
1150 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1151 return Token::make_TK_WORD();
1152 }
1153<St_SetScope>{SCOPEMASK}"<" {
1154 yyextra->token.name = yytext;
1155 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1156 yyextra->sharpCount=1;
1157 BEGIN(St_SetScopeEnd);
1158 }
1159<St_SetScope>{BLANK} {
1160 }
1161<St_SetScopeEnd>"<" {
1162 yyextra->token.name += yytext;
1163 yyextra->sharpCount++;
1164 }
1165<St_SetScopeEnd>">" {
1166 yyextra->token.name += yytext;
1167 yyextra->sharpCount--;
1168 if (yyextra->sharpCount<=0)
1169 {
1170 return Token::make_TK_WORD();
1171 }
1172 }
1173<St_SetScopeEnd>. {
1174 yyextra->token.name += yytext;
1175 }
1176<St_Ref2>"&"{ID}";" { /* symbol */
1177 yyextra->token.name = yytext;
1178 return Token::make_TK_SYMBOL();
1179 }
1180<St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title? */
1181 lineCount(yytext,yyleng);
1182 if (!yyextra->expectQuote || yytext[0]=='"')
1183 {
1184 return Token::make_TK_NONE();
1185 }
1186 else
1187 {
1188 yyextra->token.name += yytext;
1189 }
1190 }
1191<St_Ref2>{HTMLTAG_STRICT} { /* html tag */
1192 lineCount(yytext,yyleng);
1193 handleHtmlTag(yyscanner,yytext);
1194 return Token::make_TK_HTMLTAG();
1195 }
1196<St_Ref2>{SPCMD1} |
1197<St_Ref2>{SPCMD2} { /* special command */
1198 yyextra->token.name = yytext+1;
1199 yyextra->token.paramDir=TokenInfo::Unspecified;
1200 return Token::char_to_command(yytext[0]);
1201 }
1202<St_Ref2>{WORD1NQ} |
1203<St_Ref2>{WORD2NQ} {
1204 /* word */
1205 yyextra->token.name = yytext;
1206 return Token::make_TK_WORD();
1207 }
1208<St_Ref2>[ \t]+ {
1209 yyextra->token.chars=yytext;
1210 return Token::make_TK_WHITESPACE();
1211 }
1212<St_XRefItem>{LABELID} {
1213 yyextra->token.name=yytext;
1214 }
1215<St_XRefItem>" " {
1216 BEGIN(St_XRefItem2);
1217 }
1218<St_XRefItem2>[0-9]+"." {
1219 QCString numStr(yytext);
1220 numStr=numStr.left((int)yyleng-1);
1221 yyextra->token.id=numStr.toInt();
1222 return Token::make_RetVal_OK();
1223 }
1224<St_Para,St_Title,St_Ref2>"<!--" { /* html style comment block */
1225 yyextra->commentState = YY_START;
1226 BEGIN(St_Comment);
1227 }
1228<St_Param>"\""[^\n\"]+"\"" {
1229 yyextra->token.name = yytext+1;
1230 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1231 return Token::make_TK_WORD();
1232 }
1233<St_Param>({PHPTYPE}{BLANK}*("["{BLANK}*"]")*{BLANK}*"|"{BLANK}*)*{PHPTYPE}{BLANK}*("["{BLANK}*"]")*{WS}+("&")?"$"{LABELID} {
1234 lineCount(yytext,yyleng);
1235 QCString params(yytext);
1236 int j = params.find('&');
1237 int i = params.find('$');
1238 if (i<0) i=0; // should never happen
1239 if (j<i && j>=0) i=j;
1240 QCString types = params.left(i).stripWhiteSpace();
1241 yyextra->token.name = types+"#"+params.mid(i);
1242 return Token::make_TK_WORD();
1243 }
QCString::left
QCString left(size_t len) const
Definition qcstring.h:229
1244<St_Param>[^ \t\n,@\\‍]+ {
1245 yyextra->token.name = yytext;
1246 if (yyextra->token.name.at(static_cast<uint32_t>(yyleng)-1)==':')
1247 {
1248 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-1);
1249 }
1250 return Token::make_TK_WORD();
1251 }
1252<St_Param>{WS}*","{WS}* /* param separator */
1253<St_Param>{WS} {
1254 lineCount(yytext,yyleng);
1255 yyextra->token.chars=yytext;
1256 return Token::make_TK_WHITESPACE();
1257 }
1258<St_Prefix>"\""[^\n\"]*"\"" {
1259 yyextra->token.name = yytext+1;
1260 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1261 return Token::make_TK_WORD();
1262 }
1263<St_Prefix>. {
1264 unput(*yytext);
1265 return Token::make_TK_NONE();
1266 }
1267<St_Options>{ID} {
1268 yyextra->token.name+=yytext;
1269 }
1270<St_Options>{WS}*":"{WS}* {
1271 lineCount(yytext,yyleng);
1272 yyextra->token.name+=":";
1273 }
1274<St_Options>{WS}*","{WS}* |
1275<St_Options>{WS} { /* option separator */
1276 lineCount(yytext,yyleng);
1277 yyextra->token.name+=",";
1278 }
1279<St_Options>"}" {
1280 return Token::make_TK_WORD();
1281 }
1282<St_Block>{ID} {
1283 yyextra->token.name+=yytext;
1284 }
1285<St_Block>"]" {
1286 return Token::make_TK_WORD();
1287 }
1288<St_Emoji>[:0-9_a-z+-]+ {
1289 yyextra->token.name=yytext;
1290 return Token::make_TK_WORD();
1291 }
1292<St_Emoji>. {
1293 unput(*yytext);
1294 return Token::make_TK_NONE();
1295 }
1296<St_QuotedString>"\"" {
1297 yyextra->token.name="";
1298 BEGIN(St_QuotedContent);
1299 }
1300<St_QuotedString>(\n|"\\ilinebr") {
1301 unput_string(yytext,yyleng);
1302 return Token::make_TK_NONE();
1303 }
1304<St_QuotedString>. {
1305 unput(*yytext);
1306 return Token::make_TK_NONE();
1307 }
1308<St_QuotedContent>"\"" {
1309 return Token::make_TK_WORD();
1310 }
1311<St_QuotedContent>. {
1312 yyextra->token.name+=yytext;
1313 }
1314<St_ShowDate>{WS}+{SHOWDATE} {
1315 lineCount(yytext,yyleng);
1316 yyextra->token.name=yytext;
1317 return Token::make_TK_WORD();
1318 }
1319<St_ShowDate>(\n|"\\ilinebr") {
1320 unput_string(yytext,yyleng);
1321 return Token::make_TK_NONE();
1322 }
1323<St_ShowDate>. {
1324 unput(*yytext);
1325 return Token::make_TK_NONE();
1326 }
1327<St_ILine>{LINENR}/[\\@\n\.] |
1328<St_ILine>{LINENR}{BLANK} {
1329 bool ok = false;
1330 int nr = QCString(yytext).toInt(&ok);
1331 if (!ok)
1332 {
1333 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '{}' for iline command",yytext);
1334 }
1335 else
1336 {
1337 yyextra->yyLineNr = nr;
1338 }
1339 return Token::make_TK_WORD();
1340 }
1341<St_ILine>. {
1342 return Token::make_TK_NONE();
1343 }
1344<St_IFile>{BLANK}*{FILEMASK} {
1345 yyextra->fileName = QCString(yytext).stripWhiteSpace();
1346 return Token::make_TK_WORD();
1347 }
1348<St_IFile>{BLANK}*"\""[^\n\"]+"\"" {
1349 QCString text(yytext);
1350 text = text.stripWhiteSpace();
1351 yyextra->fileName = text.mid(1,text.length()-2);
1352 return Token::make_TK_WORD();
1353 }
1354<St_File>{FILEMASK} {
1355 yyextra->token.name = yytext;
1356 if (yyextra->token.name.endsWith("\\ilinebr") ||yyextra->token.name.endsWith("@ilinebr"))
1357 {
1358 unput_string("\\ilinebr",8);
1359 yyextra->token.name = yyextra->token.name.left(yyleng-8);
1360 }
1361 return Token::make_TK_WORD();
1362 }
1363<St_File>"\""[^\n\"]+"\"" {
1364 QCString text(yytext);
1365 yyextra->token.name = text.mid(1,text.length()-2);
1366 return Token::make_TK_WORD();
1367 }
1368<St_Pattern>[^\\\r\n]+ {
1369 yyextra->token.name += yytext;
1370 }
1371<St_Pattern>"\\ilinebr" {
1372 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1373 return Token::make_TK_WORD();
1374 }
1375<St_Pattern>\n {
1376 lineCount(yytext,yyleng);
1377 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1378 return Token::make_TK_WORD();
1379 }
1380<St_Pattern>. {
1381 yyextra->token.name += yytext;
1382 }
1383<St_Link>{LINKMASK}|{REFWORD} {
1384 yyextra->token.name = yytext;
1385 return Token::make_TK_WORD();
1386 }
1387<St_Comment>"-->" { /* end of html comment */
1388 BEGIN(yyextra->commentState);
1389 }
1390<St_Comment>[^-]+ /* inside html comment */
1391<St_Comment>. /* inside html comment */
1392
1393 /* State for skipping title (all chars until the end of the line) */
1394
1395<St_SkipTitle>.
1396<St_SkipTitle>(\n|"\\ilinebr") {
1397 if (*yytext == '\n') unput('\n');
1398 return Token::make_TK_NONE();
1399 }
1400
1401 /* State for the pass used to find the anchors and sections */
1402
1403<St_Sections>[^\n@\<]+
1404<St_Sections>{CMD}("<"|{CMD})
1405<St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
1406 lineCount(yytext,yyleng);
1407 QCString tag(yytext);
1408 int s=tag.find("id=");
1409 if (s!=-1) // command has id attribute
1410 {
1411 char c=tag[s+3];
1412 if (c=='\'' || c=='"') // valid start
1413 {
1414 int e=tag.find(c,s+4);
1415 if (e!=-1) // found matching end
1416 {
1417 yyextra->secType = SectionType::Table;
1418 yyextra->secLabel=tag.mid(s+4,e-s-4); // extract id
1419 processSection(yyscanner);
1420 }
1421 }
1422 }
1423 }
SectionType::Table
static constexpr int Table
Definition section.h:41
processSection
static void processSection(yyscan_t yyscanner)
Definition doctokenizer.l:1602
1424<St_Sections>{CMD}"anchor"{BLANK}+ {
1425 yyextra->secType = SectionType::Anchor;
1426 BEGIN(St_SecLabel1);
1427 }
SectionType::Anchor
static constexpr int Anchor
Definition section.h:40
1428<St_Sections>{CMD}"ianchor"{BLANK}+ {
1429 yyextra->secType = SectionType::Anchor;
1430 BEGIN(St_SecLabel1);
1431 }
1432<St_Sections>{CMD}"section"{BLANK}+ {
1433 yyextra->secType = SectionType::Section;
1434 BEGIN(St_SecLabel2);
1435 }
SectionType::Section
static constexpr int Section
Definition section.h:33
1436<St_Sections>{CMD}"subsection"{BLANK}+ {
1437 yyextra->secType = SectionType::Subsection;
1438 BEGIN(St_SecLabel2);
1439 }
SectionType::Subsection
static constexpr int Subsection
Definition section.h:34
1440<St_Sections>{CMD}"subsubsection"{BLANK}+ {
1441 yyextra->secType = SectionType::Subsubsection;
1442 BEGIN(St_SecLabel2);
1443 }
SectionType::Subsubsection
static constexpr int Subsubsection
Definition section.h:35
1444<St_Sections>{CMD}"paragraph"{BLANK}+ {
1445 yyextra->secType = SectionType::Paragraph;
1446 BEGIN(St_SecLabel2);
1447 }
SectionType::Paragraph
static constexpr int Paragraph
Definition section.h:36
1448<St_Sections>{CMD}"subparagraph"{BLANK}+ {
1449 yyextra->secType = SectionType::Subparagraph;
1450 BEGIN(St_SecLabel2);
1451 }
SectionType::Subparagraph
static constexpr int Subparagraph
Definition section.h:37
1452<St_Sections>{CMD}"subsubparagraph"{BLANK}+ {
1453 yyextra->secType = SectionType::Subsubparagraph;
1454 BEGIN(St_SecLabel2);
1455 }
SectionType::Subsubparagraph
static constexpr int Subsubparagraph
Definition section.h:38
1456<St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9] {
1457 yyextra->endMarker="endverbatim";
1458 BEGIN(St_SecSkip);
1459 }
1460<St_Sections>{CMD}"iverbatim"/[^a-z_A-Z0-9] {
1461 yyextra->endMarker="endiverbatim";
1462 BEGIN(St_SecSkip);
1463 }
1464<St_Sections>{CMD}"iliteral"/[^a-z_A-Z0-9] {
1465 yyextra->endMarker="endiliteral";
1466 BEGIN(St_SecSkip);
1467 }
1468<St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1469 yyextra->endMarker="enddot";
1470 BEGIN(St_SecSkip);
1471 }
1472<St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1473 yyextra->endMarker="endmsc";
1474 BEGIN(St_SecSkip);
1475 }
1476<St_Sections>{CMD}"startuml"/[^a-z_A-Z0-9] {
1477 yyextra->endMarker="enduml";
1478 BEGIN(St_SecSkip);
1479 }
1480<St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1481 yyextra->endMarker="endhtmlonly";
1482 BEGIN(St_SecSkip);
1483 }
1484<St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1485 yyextra->endMarker="endlatexonly";
1486 BEGIN(St_SecSkip);
1487 }
1488<St_Sections>{CMD}"manonly"/[^a-z_A-Z0-9] {
1489 yyextra->endMarker="endmanonly";
1490 BEGIN(St_SecSkip);
1491 }
1492<St_Sections>{CMD}"rtfonly"/[^a-z_A-Z0-9] {
1493 yyextra->endMarker="endrtfonly";
1494 BEGIN(St_SecSkip);
1495 }
1496<St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1497 yyextra->endMarker="endxmlonly";
1498 BEGIN(St_SecSkip);
1499 }
1500<St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
1501 yyextra->endMarker="enddocbookonly";
1502 BEGIN(St_SecSkip);
1503 }
1504<St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1505 yyextra->endMarker="endcode";
1506 BEGIN(St_SecSkip);
1507 }
1508<St_Sections>{CMD}"icode"/[^a-z_A-Z0-9] {
1509 yyextra->endMarker="endicode";
1510 BEGIN(St_SecSkip);
1511 }
1512<St_Sections>"<!--" {
1513 yyextra->endMarker="-->";
1514 BEGIN(St_SecSkip);
1515 }
1516<St_SecSkip>{CMD}{ID} {
1517 if (yyextra->endMarker==yytext+1)
1518 {
1519 BEGIN(St_Sections);
1520 }
1521 }
1522<St_SecSkip>"-->" {
1523 if (yyextra->endMarker==yytext)
1524 {
1525 BEGIN(St_Sections);
1526 }
1527 }
1528<St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1529<St_SecSkip>.
1530<St_SecSkip>(\n|"\\ilinebr")
1531<St_Sections>.
1532<St_Sections>(\n|"\\ilinebr")
1533<St_SecLabel1>({REQID}|{LABELID}) {
1534 lineCount(yytext,yyleng);
1535 yyextra->secLabel = yytext;
1536 processSection(yyscanner);
1537 BEGIN(St_Sections);
1538 }
1539<St_SecLabel2>({REQID}|{LABELID}){BLANK}+ |
1540<St_SecLabel2>({REQID}|{LABELID}) {
1541 yyextra->secLabel = yytext;
1542 yyextra->secLabel = yyextra->secLabel.stripWhiteSpace();
1543 BEGIN(St_SecTitle);
1544 }
1545<St_SecTitle>[^\n]+ |
1546<St_SecTitle>[^\n]*\n {
1547 lineCount(yytext,yyleng);
1548 yyextra->secTitle = yytext;
1549 yyextra->secTitle = yyextra->secTitle.stripWhiteSpace();
1550 if (yyextra->secTitle.endsWith("\\ilinebr"))
1551 {
1552 yyextra->secTitle.left(yyextra->secTitle.length()-8);
1553 }
1554 processSection(yyscanner);
1555 BEGIN(St_Sections);
1556 }
1557<St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1558 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}' while looking for section label or title",yytext);
1559 }
1560
1561<St_Snippet>[^\\\n]+ {
1562 yyextra->token.name += yytext;
1563 }
1564<St_Snippet>"\\" {
1565 yyextra->token.name += yytext;
1566 }
1567<St_Snippet>(\n|"\\ilinebr") {
1568 unput_string(yytext,yyleng);
1569 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1570 return Token::make_TK_WORD();
1571 }
1572
1573 /* Generic rules that work for all states */
1574<*>\n {
1575 lineCount(yytext,yyleng);
1576 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected new line character");
1577 }
1578<*>"\\ilinebr" {
1579 }
1580<*>[\\@<>&$#%~"=] { /* unescaped special character */
1581 //warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}', assuming command \\{} was meant.",yytext,yytext);
1582 yyextra->token.name = yytext;
1583 return Token::char_to_command(yytext[0]);
1584 }
1585<*>. {
1586 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '{}'",yytext);
1587 }
1588%%

◆ yyread()

int yyread ( yyscan_t yyscanner,
char * buf,
int max_size )
static

Definition at line 1592 of file doctokenizer.l.

1593{
1594 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1595 int c=0;
1596 const char *p = yyextra->inputString + yyextra->inputPos;
1597 while ( c < max_size && *p ) { *buf++ = *p++; c++; }
1598 yyextra->inputPos+=c;
1599 return c;
1600}