Doxygen
Loading...
Searching...
No Matches
Classes | Macros | Typedefs | Functions
doctokenizer.l File Reference
#include <stdint.h>
#include <ctype.h>
#include <stack>
#include <string>
#include <cassert>
#include "doctokenizer.h"
#include "cmdmapper.h"
#include "config.h"
#include "message.h"
#include "section.h"
#include "membergroup.h"
#include "definition.h"
#include "doxygen.h"
#include "portable.h"
#include "cite.h"
#include "regex.h"
#include "debug.h"
#include "docnode.h"
#include "doxygen_lex.h"
#include "doctokenizer.l.h"
+ Include dependency graph for doctokenizer.l:

Go to the source code of this file.

Classes

struct  DocLexerContext
 
struct  doctokenizerYY_state
 
struct  DocTokenizer::Private
 

Macros

#define YY_TYPEDEF_YY_SCANNER_T
 
#define YY_NO_INPUT   1
 
#define YY_NO_UNISTD_H   1
 
#define lineCount(s, len)
 
#define unput_string(yytext, yyleng)
 
#define YY_INPUT(buf, result, max_size)
 
#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)
 
#define yyterminate()
 

Typedefs

typedef yyguts_t * yyscan_t
 

Functions

static const char * stateToString (int state)
 
static int yyread (yyscan_t yyscanner, char *buf, int max_size)
 
static void handleHtmlTag (yyscan_t yyscanner, const char *text)
 
static void processSection (yyscan_t yyscanner)
 
QCString extractPartAfterNewLine (const QCString &text)
 
static int computeIndent (const char *str, size_t length)
 
static const char * getLexerFILE ()
 
int yylex (yyscan_t yyscanner)
 

Macro Definition Documentation

◆ lineCount

#define lineCount ( s,
len )
Value:
do { for(int i=0;i<(int)len;i++) if (s[i]=='\n') yyextra->yyLineNr++; } while(0)

Definition at line 96 of file doctokenizer.l.

Referenced by endBrief(), initMethodProtection(), and CitationManager::insertCrossReferencesForBibFile().

◆ unput_string

#define unput_string ( yytext,
yyleng )
Value:
do { for (int i=(int)yyleng-1;i>=0;i--) unput(yytext[i]); } while(0)

Definition at line 158 of file doctokenizer.l.

◆ YY_DECL

#define YY_DECL   static Token doctokenizerYYlex(yyscan_t yyscanner)

Definition at line 169 of file doctokenizer.l.

◆ YY_INPUT

#define YY_INPUT ( buf,
result,
max_size )
Value:
result=yyread(yyscanner,buf,max_size);
yyread
static int yyread(yyscan_t yyscanner, char *buf, int max_size)
Definition doctokenizer.l:1564

Definition at line 162 of file doctokenizer.l.

◆ YY_NO_INPUT

#define YY_NO_INPUT   1

Definition at line 52 of file doctokenizer.l.

◆ YY_NO_UNISTD_H

#define YY_NO_UNISTD_H   1

Definition at line 53 of file doctokenizer.l.

◆ YY_TYPEDEF_YY_SCANNER_T

#define YY_TYPEDEF_YY_SCANNER_T

Definition at line 26 of file doctokenizer.l.

◆ yyterminate

#define yyterminate ( )
Value:
return Token::make_TK_EOF()

Definition at line 172 of file doctokenizer.l.

Typedef Documentation

◆ yyscan_t

typedef yyguts_t* yyscan_t

Definition at line 28 of file doctokenizer.l.

Function Documentation

◆ computeIndent()

static int computeIndent ( const char * str,
size_t length )
static

Definition at line 125 of file doctokenizer.l.

126{
127 if (str==0 || length==std::string::npos) return 0;
128 size_t i;
129 int indent=0;
130 int tabSize=Config_getInt(TAB_SIZE);
131 for (i=0;i<length;i++)
132 {
133 if (str[i]=='\t')
134 {
135 indent+=tabSize - (indent%tabSize);
136 }
137 else if (str[i]=='\n')
138 {
139 indent=0;
140 }
141 else if (str[i]=='\\' && qstrncmp(str+i+1,"ilinebr",7)==0)
142 {
143 indent=0;
144 i+=7;
145 if (str[i+1]==' ') i++; // also eat space after \\ilinebr if present
146 }
147 else
148 {
149 indent++;
150 }
151 }
152 //printf("input('%s')=%d\n",str,indent);
153 return indent;
Config_getInt
#define Config_getInt(name)
Definition config.h:34
qstrncmp
int qstrncmp(const char *str1, const char *str2, size_t len)
Definition qcstring.h:75
154}

References Config_getInt, and qstrncmp().

◆ extractPartAfterNewLine()

QCString extractPartAfterNewLine ( const QCString & text)

Definition at line 107 of file doctokenizer.l.

108{
109 int nl1 = text.find('\n');
110 int nl2 = text.find("\\ilinebr");
111 if (nl1!=-1 && nl1<nl2)
112 {
113 return text.mid(nl1+1);
114 }
115 if (nl2!=-1)
116 {
117 if (text.at(nl2+8)==' ') nl2++; // skip space after \\ilinebr
118 return text.mid(nl2+8);
119 }
120 return text;
QCString::find
int find(char c, int index=0, bool cs=TRUE) const
Definition qcstring.cpp:43
QCString::mid
QCString mid(size_t index, size_t len=static_cast< size_t >(-1)) const
Definition qcstring.h:226
QCString::at
char & at(size_t i)
Returns a reference to the character at index i.
Definition qcstring.h:578
121}

References QCString::at(), QCString::find(), and QCString::mid().

◆ getLexerFILE()

static const char * getLexerFILE ( )
inlinestatic

Definition at line 165 of file doctokenizer.l.

165{return __FILE__;}

◆ handleHtmlTag()

static void handleHtmlTag ( yyscan_t yyscanner,
const char * text )
static

Definition at line 1595 of file doctokenizer.l.

1596{
1597 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1598
1599 QCString tagText(text);
1600 yyextra->token.text = tagText;
1601 yyextra->token.attribs.clear();
1602 yyextra->token.endTag = FALSE;
1603 yyextra->token.emptyTag = FALSE;
QCString
This is an alternative implementation of QCString.
Definition qcstring.h:101
FALSE
#define FALSE
Definition qcstring.h:34
1604
1605 // Check for end tag
1606 int startNamePos=1;
1607 if (tagText.at(1)=='/')
1608 {
1609 yyextra->token.endTag = TRUE;
1610 startNamePos++;
1611 }
TRUE
#define TRUE
Definition qcstring.h:37
1612
1613 // Parse the name portion
1614 int i = startNamePos;
1615 for (i=startNamePos; i < (int)yyleng; i++)
1616 {
1617 // Check for valid HTML/XML name chars (including namespaces)
1618 char c = tagText.at(i);
1619 if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
1620 }
1621 yyextra->token.name = tagText.mid(startNamePos,i-startNamePos);
1622
1623 // Parse the attributes. Each attribute is a name, value pair
1624 // The result is stored in yyextra->token.attribs.
1625 int startAttribList = i;
1626 while (i<(int)yyleng)
1627 {
1628 char c=tagText.at(i);
1629 // skip spaces
1630 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1631 // check for end of the tag
1632 if (c == '>') break;
1633 // Check for XML style "empty" tag.
1634 if (c == '/')
1635 {
1636 yyextra->token.emptyTag = TRUE;
1637 break;
1638 }
1639 int startName=i;
1640 // search for end of name
1641 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='=' && c!= '>') { c=tagText.at(++i); }
1642 int endName=i;
1643 QCString optName,optValue;
1644 optName = tagText.mid(startName,endName-startName).lower();
1645 // skip spaces
1646 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1647 if (tagText.at(i)=='=') // option has value
1648 {
1649 int startAttrib=0, endAttrib=0;
1650 c=tagText.at(++i);
1651 // skip spaces
1652 while (i<(int)yyleng && isspace((uint8_t)c)) { c=tagText.at(++i); }
1653 if (tagText.at(i)=='\'') // option '...'
1654 {
1655 c=tagText.at(++i);
1656 startAttrib=i;
QCString::lower
QCString lower() const
Definition qcstring.h:234
1657
1658 // search for matching quote
1659 while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
1660 endAttrib=i;
1661 if (i<(int)yyleng) { c=tagText.at(++i);}
1662 }
1663 else if (tagText.at(i)=='"') // option "..."
1664 {
1665 c=tagText.at(++i);
1666 startAttrib=i;
1667 // search for matching quote
1668 while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
1669 endAttrib=i;
1670 if (i<(int)yyleng) { c=tagText.at(++i);}
1671 }
1672 else // value without any quotes
1673 {
1674 startAttrib=i;
1675 // search for separator or end symbol
1676 while (i<(int)yyleng && !isspace((uint8_t)c) && c!='>') { c=tagText.at(++i); }
1677 endAttrib=i;
1678 if (i<(int)yyleng) { c=tagText.at(++i);}
1679 }
1680 optValue = tagText.mid(startAttrib,endAttrib-startAttrib);
1681 if (optName == "align") optValue = optValue.lower();
1682 else if (optName == "valign")
1683 {
1684 optValue = optValue.lower();
1685 if (optValue == "center") optValue="middle";
1686 }
1687 }
1688 else // start next option
1689 {
1690 }
1691 //printf("=====> Adding option name=<%s> value=<%s>\n",
1692 // qPrint(optName),qPrint(optValue));
1693 yyextra->token.attribs.emplace_back(optName,optValue);
1694 }
1695 yyextra->token.attribsStr = tagText.mid(startAttribList,i-startAttribList);
1696}

References QCString::at(), QCString::clear(), FALSE, QCString::lower(), QCString::mid(), and TRUE.

◆ processSection()

static void processSection ( yyscan_t yyscanner)
static

Definition at line 1574 of file doctokenizer.l.

1575{
1576 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1577 //printf("%s: found section/anchor with name '%s'\n",qPrint(g_fileName),qPrint(g_secLabel));
1578 QCString file;
1579 if (yyextra->definition)
1580 {
1581 file = yyextra->definition->getOutputFileBase();
1582 }
1583 else
1584 {
1585 warn(yyextra->fileName,yyextra->yyLineNr,"Found section/anchor %s without context",qPrint(yyextra->secLabel));
1586 }
1587 SectionInfo *si = SectionManager::instance().find(yyextra->secLabel);
1588 if (si)
1589 {
1590 si->setFileName(file);
1591 si->setType(yyextra->secType);
1592 }
LinkedMap::find
const T * find(const std::string &key) const
Find an object given the key.
Definition linkedmap.h:47
SectionInfo
class that provide information about a section.
Definition section.h:57
SectionInfo::setType
void setType(SectionType t)
Definition section.h:80
SectionInfo::setFileName
void setFileName(const QCString &fn)
Definition section.h:79
SectionManager::instance
static SectionManager & instance()
returns a reference to the singleton
Definition section.h:175
warn
#define warn(file, line, fmt,...)
Definition message.h:59
qPrint
const char * qPrint(const char *s)
Definition qcstring.h:672
1593}

References LinkedMap< T, Hash, KeyEqual, Map >::find(), SectionManager::instance(), qPrint(), SectionInfo::setFileName(), SectionInfo::setType(), and warn.

◆ stateToString()

static const char * stateToString ( int state)
static

◆ yylex()

int yylex ( yyscan_t yyscanner)

Definition at line 333 of file doctokenizer.l.

335 {LISTITEM} { /* list item */
336 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
337 lineCount(yytext,yyleng);
338 QCString text(yytext);
339 uint32_t dashPos = static_cast<uint32_t>(text.findRev('-'));
340 assert(dashPos!=static_cast<uint32_t>(-1));
341 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
342 yyextra->token.isCheckedList = false;
343 yyextra->token.id = -1;
344 yyextra->token.indent = computeIndent(yytext,dashPos);
345 return Token::make_TK_LISTITEM();
346 }
lineCount
#define lineCount(s, len)
Definition doctokenizer.l:96
computeIndent
static int computeIndent(const char *str, size_t length)
Definition doctokenizer.l:125
347<St_Para>^{CLISTITEM} { /* checkbox item */
348 QCString text=yytext;
349 int dashPos = text.findRev('-');
350 yyextra->token.isEnumList = false;
351 yyextra->token.isCheckedList = true;
352 if (text.find('x') != -1) yyextra->token.id = DocAutoList::Checked_x;
353 else if (text.find('X') != -1) yyextra->token.id = DocAutoList::Checked_X;
354 else yyextra->token.id = DocAutoList::Unchecked;
355 yyextra->token.indent = computeIndent(yytext,dashPos);
356 return Token::make_TK_LISTITEM();
357 }
QCString::findRev
int findRev(char c, int index=-1, bool cs=TRUE) const
Definition qcstring.cpp:91
358<St_Para>^{MLISTITEM} { /* list item */
359 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
360 {
361 REJECT;
362 }
363 else
364 {
365 lineCount(yytext,yyleng);
366 std::string text(yytext);
367 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
368 reg::Match match;
369 reg::search(text,match,re);
370 size_t listPos = match.position();
371 assert(listPos!=std::string::npos);
372 yyextra->token.isEnumList = false;
373 yyextra->token.isCheckedList = false;
374 yyextra->token.id = -1;
375 yyextra->token.indent = computeIndent(yytext,listPos);
376 return Token::make_TK_LISTITEM();
377 }
378 }
reg::Ex
Class representing a regular expression.
Definition regex.h:39
reg::Match
Object representing the matching results.
Definition regex.h:153
reg::search
bool search(std::string_view str, Match &match, const Ex &re, size_t pos)
Search in a given string str starting at position pos for a match against regular expression re.
Definition regex.cpp:748
reg::match
bool match(std::string_view str, Match &match, const Ex &re)
Matches a given string str for a match against regular expression re.
Definition regex.cpp:759
379<St_Para>^{OLISTITEM} { /* numbered list item */
380 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
381 {
382 REJECT;
383 }
384 else
385 {
386 std::string text(yytext);
387 static const reg::Ex re(R"(\d+)");
388 reg::Match match;
389 reg::search(text,match,re);
390 size_t markPos = match.position();
391 assert(markPos!=std::string::npos);
392 yyextra->token.isEnumList = true;
393 yyextra->token.isCheckedList = false;
394 bool ok = false;
395 int id = QCString(match.str()).toInt(&ok);
396 yyextra->token.id = ok ? id : -1;
397 if (!ok)
398 {
399 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '%s' ",match.str().c_str());
400 }
401 yyextra->token.indent = computeIndent(yytext,markPos);
402 return Token::make_TK_LISTITEM();
403 }
404 }
QCString::toInt
int toInt(bool *ok=nullptr, int base=10) const
Definition qcstring.cpp:249
405<St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM} { /* list item on next line */
406 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
407 lineCount(yytext,yyleng);
408 QCString text=extractPartAfterNewLine(QCString(yytext));
409 uint32_t dashPos = static_cast<uint32_t>(text.findRev('-'));
410 assert(dashPos!=static_cast<uint32_t>(-1));
411 yyextra->token.isEnumList = text.at(dashPos+1)=='#';
412 yyextra->token.isCheckedList = false;
413 yyextra->token.id = -1;
414 yyextra->token.indent = computeIndent(text.data(),dashPos);
415 return Token::make_TK_LISTITEM();
416 }
QCString::data
const char * data() const
Returns a pointer to the contents of the string in the form of a 0-terminated C string.
Definition qcstring.h:159
extractPartAfterNewLine
QCString extractPartAfterNewLine(const QCString &text)
Definition doctokenizer.l:107
417<St_Para>{BLANK}*\n{CLISTITEM} { /* checkbox item on next line */
418 QCString text=yytext;
419 text=text.right(text.length()-text.find('\n')-1);
420 int dashPos = text.findRev('-');
421 yyextra->token.isEnumList = false;
422 yyextra->token.isCheckedList = true;
423 if (text.find('x') != -1) yyextra->token.id = DocAutoList::Checked_x;
424 else if (text.find('X') != -1) yyextra->token.id = DocAutoList::Checked_X;
425 else yyextra->token.id = DocAutoList::Unchecked;
426 yyextra->token.indent = computeIndent(text.data(),dashPos);
427 return Token::make_TK_LISTITEM();
428 }
QCString::length
size_t length() const
Returns the length of the string, not counting the 0-terminator.
Definition qcstring.h:153
QCString::right
QCString right(size_t len) const
Definition qcstring.h:219
429<St_Para>{BLANK}*(\n|"\\ilinebr"){MLISTITEM} { /* list item on next line */
430 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
431 {
432 REJECT;
433 }
434 else
435 {
436 lineCount(yytext,yyleng);
437 std::string text=extractPartAfterNewLine(QCString(yytext)).str();
438 static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
439 reg::Match match;
440 reg::search(text,match,re);
441 size_t markPos = match.position();
442 assert(markPos!=std::string::npos);
443 yyextra->token.isEnumList = FALSE;
444 yyextra->token.isCheckedList = false;
445 yyextra->token.id = -1;
446 yyextra->token.indent = computeIndent(text.c_str(),markPos);
447 return Token::make_TK_LISTITEM();
448 }
449 }
QCString::str
const std::string & str() const
Definition qcstring.h:537
450<St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM} { /* list item on next line */
451 if (yyextra->insideHtmlLink || !yyextra->markdownSupport || yyextra->insidePre)
452 {
453 REJECT;
454 }
455 else
456 {
457 lineCount(yytext,yyleng);
458 std::string text=extractPartAfterNewLine(QCString(yytext)).str();
459 static const reg::Ex re(R"(\d+)");
460 reg::Match match;
461 reg::search(text,match,re);
462 size_t markPos = match.position();
463 assert(markPos!=std::string::npos);
464 yyextra->token.isEnumList = true;
465 yyextra->token.isCheckedList = false;
466 bool ok = false;
467 int id = QCString(match.str()).toInt(&ok);
468 yyextra->token.id = ok ? id : -1;
469 if (!ok)
470 {
471 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '%s' ",match.str().c_str());
472 }
473 yyextra->token.indent = computeIndent(text.c_str(),markPos);
474 return Token::make_TK_LISTITEM();
475 }
476 }
477<St_Para>^{ENDLIST} { /* end list */
478 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
479 lineCount(yytext,yyleng);
480 size_t dotPos = static_cast<size_t>(QCString(yytext).findRev('.'));
481 yyextra->token.indent = computeIndent(yytext,dotPos);
482 return Token::make_TK_ENDLIST();
483 }
484<St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST} { /* end list on next line */
485 if (yyextra->insideHtmlLink || yyextra->insidePre) REJECT;
486 lineCount(yytext,yyleng);
487 QCString text=extractPartAfterNewLine(QCString(yytext));
488 size_t dotPos = static_cast<size_t>(text.findRev('.'));
489 yyextra->token.indent = computeIndent(text.data(),dotPos);
490 return Token::make_TK_ENDLIST();
491 }
492<St_Para>"{"{BLANK}*"@linkplain"/{WS}+ {
493 yyextra->token.name = "javalinkplain";
494 return Token::make_TK_COMMAND_AT();
495 }
496<St_Para>"{"{BLANK}*"@link"/{WS}+ {
497 yyextra->token.name = "javalink";
498 return Token::make_TK_COMMAND_AT();
499 }
500<St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
501 yyextra->token.name = "inheritdoc";
502 return Token::make_TK_COMMAND_AT();
503 }
504<St_Para>"@_fakenl" { // artificial new line
505 //yyextra->yyLineNr++;
506 }
507<St_Para>{SPCMD3} {
508 yyextra->token.name = "_form";
509 bool ok;
510 yyextra->token.id = QCString(yytext).right((int)yyleng-7).toInt(&ok);
511 ASSERT(ok);
512 return Token::char_to_command(yytext[0]);
513 }
Token::char_to_command
static Token char_to_command(char c)
Definition doctokenizer.h:109
ASSERT
#define ASSERT(x)
Definition qcstring.h:39
514<St_Para>{CMD}"n"\n { /* \n followed by real newline */
515 lineCount(yytext,yyleng);
516 //yyextra->yyLineNr++;
517 yyextra->token.name = yytext+1;
518 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
519 yyextra->token.paramDir=TokenInfo::Unspecified;
520 return Token::char_to_command(yytext[0]);
521 }
522<St_Para>"\\ilinebr" {
523 }
524<St_Para>{SPCMD1} |
525<St_Para>{SPCMD2} |
526<St_Para>{SPCMD5} |
527<St_Para>{SPCMD4} { /* special command */
528 yyextra->token.name = yytext+1;
529 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
530 yyextra->token.paramDir=TokenInfo::Unspecified;
531 return Token::char_to_command(yytext[0]);
532 }
533<St_Para>{PARAMIO} { /* param [in,out] command */
534 yyextra->token.name = "param";
535 QCString s(yytext);
536 bool isIn = s.find("in")!=-1;
537 bool isOut = s.find("out")!=-1;
538 if (isIn)
539 {
540 if (isOut)
541 {
542 yyextra->token.paramDir=TokenInfo::InOut;
543 }
544 else
545 {
546 yyextra->token.paramDir=TokenInfo::In;
547 }
548 }
549 else if (isOut)
550 {
551 yyextra->token.paramDir=TokenInfo::Out;
552 }
553 else
554 {
555 yyextra->token.paramDir=TokenInfo::Unspecified;
556 }
557 return Token::char_to_command(yytext[0]);
558 }
559<St_Para>{URLPROTOCOL}{URLMASK}/[,\.] { // URL, or URL.
560 yyextra->token.name=yytext;
561 yyextra->token.isEMailAddr=FALSE;
562 return Token::make_TK_URL();
563 }
564<St_Para>{URLPROTOCOL}{URLMASK} { // URL
565 yyextra->token.name=yytext;
566 yyextra->token.isEMailAddr=FALSE;
567 return Token::make_TK_URL();
568 }
569<St_Para>"<"{URLPROTOCOL}{URLMASK}">" { // URL
570 yyextra->token.name=yytext;
571 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
572 yyextra->token.isEMailAddr=FALSE;
573 return Token::make_TK_URL();
574 }
575<St_Para>{MAILADDR} { // Mail address
576 yyextra->token.name=yytext;
577 yyextra->token.name.stripPrefix("mailto:");
578 yyextra->token.isEMailAddr=TRUE;
579 return Token::make_TK_URL();
580 }
581<St_Para>"<"{MAILADDR}">" { // Mail address
582 yyextra->token.name=yytext;
583 yyextra->token.name = yyextra->token.name.mid(1,yyextra->token.name.length()-2);
584 yyextra->token.name.stripPrefix("mailto:");
585 yyextra->token.isEMailAddr=TRUE;
586 return Token::make_TK_URL();
587 }
588<St_Para>"<"{MAILADDR2}">" { // anti spam mail address
589 yyextra->token.name=yytext;
590 return Token::make_TK_WORD();
591 }
592<St_Para>{RCSID} { /* RCS tag */
593 QCString tagName(yytext+1);
594 int index=tagName.find(':');
595 if (index<0) index=0; // should never happen
596 yyextra->token.name = tagName.left(index);
597 int text_begin = index+2;
598 int text_end = static_cast<int>(tagName.length())-1;
599 if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
600 {
601 ++text_begin;
602 if (tagName[text_end-1]=='#')
603 {
604 --text_end;
605 }
606 }
607 yyextra->token.text = tagName.mid(text_begin,text_end-text_begin);
608 return Token::make_TK_RCSTAG();
609 }
610<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}")" | /* environment variable */
611<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}"("{ID}"))" { /* environment variable */
612 QCString name(&yytext[2]);
613 name = name.left(static_cast<int>(name.length())-1);
614 QCString value = Portable::getenv(name);
615 for (int i=static_cast<int>(value.length())-1;i>=0;i--) unput(value.at(i));
616 }
Portable::getenv
QCString getenv(const QCString &variable)
Definition portable.cpp:338
617<St_Para>{HTMLTAG} { /* html tag */
618 lineCount(yytext,yyleng);
619 handleHtmlTag(yyscanner,yytext);
620 return Token::make_TK_HTMLTAG();
621 }
handleHtmlTag
static void handleHtmlTag(yyscan_t yyscanner, const char *text)
Definition doctokenizer.l:1595
622<St_Para,St_Text>"&"{ID}";" { /* special symbol */
623 yyextra->token.name = yytext;
624 return Token::make_TK_SYMBOL();
625 }
626
627 /********* patterns for linkable words ******************/
628
629<St_Para>{ID}/"<"{HTMLKEYW}">"+ { /* this rule is to prevent opening html
630 * tag to be recognized as a templated classes
631 */
632 yyextra->token.name = yytext;
633 return Token::make_TK_LNKWORD();
634 }
635<St_Para>{LNKWORDN}/("<"{HTMLKEYW}">")+ { // prevent <br> html tag to be parsed as template arguments
636 yyextra->token.name = yytext;
637 return Token::make_TK_LNKWORD();
638 }
639<St_Para>{LNKWORD1} |
640<St_Para>{LNKWORD1}{FUNCARG} |
641<St_Para>{LNKWORD2} |
642<St_Para>{LNKWORD3} {
643 yyextra->token.name = yytext;
644 return Token::make_TK_LNKWORD();
645 }
646<St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
647 yyextra->token.name = yytext;
648 yyextra->token.name = yyextra->token.name.left(yyextra->token.name.length()-1);
649 unput(yytext[(int)yyleng-1]);
650 return Token::make_TK_LNKWORD();
651 }
652 /********* patterns for normal words ******************/
653
654<St_Para,St_Text>[\-+0-9] |
655<St_Para,St_Text>{WORD1} |
656<St_Para,St_Text>{WORD2} { /* function call */
657 if (QCString(yytext).find("\\ilinebr")!=-1) REJECT; // see issue #8311
658 lineCount(yytext,yyleng);
659 if (yytext[0]=='%') // strip % if present
660 yyextra->token.name = &yytext[1];
661 else
662 yyextra->token.name = yytext;
663 return Token::make_TK_WORD();
664 }
665<St_Text>({ID}".")+{ID} {
666 yyextra->token.name = yytext;
667 return Token::make_TK_WORD();
668 }
669<St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
670 // avoid interpretation as "operator <"
671 yyextra->token.name = yytext;
672 return Token::make_TK_WORD();
673 }
674
675 /*******************************************************/
676
677<St_Para,St_Text>{BLANK}+ |
678<St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
679 lineCount(yytext,yyleng);
680 yyextra->token.chars=yytext;
681 return Token::make_TK_WHITESPACE();
682 }
683<St_Text>[\\@<>&$#%~] {
684 yyextra->token.name = yytext;
685 return Token::char_to_command(yytext[0]);
686 }
687<St_Para>({BLANK}*\n)+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
688 if (yyextra->insidePre || yyextra->autoListLevel==0)
689 {
690 REJECT;
691 }
692 lineCount(yytext,yyleng);
693 }
694<St_Para>({BLANK}*\n)+{BLANK}*\n/{CLISTITEM} { /* skip trailing paragraph followed by new checkbox item */
695 if (yyextra->insidePre || yyextra->autoListLevel==0)
696 {
697 REJECT;
698 }
699 }
700<St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
701 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
702 {
703 REJECT;
704 }
705 lineCount(yytext,yyleng);
706 }
707<St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
708 if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
709 {
710 REJECT;
711 }
712 lineCount(yytext,yyleng);
713 }
714<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}*/" \\ifile" | // we don't want to count the space before \ifile
715<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}* {
716 lineCount(yytext,yyleng);
717 if (yyextra->insidePre)
718 {
719 yyextra->token.chars=yytext;
720 return Token::make_TK_WHITESPACE();
721 }
722 else
723 {
724 yyextra->token.indent=computeIndent(yytext,yyleng);
725 int i;
726 // put back the indentation (needed for list items)
727 //printf("token.indent=%d\n",yyextra->token.indent);
728 for (i=0;i<yyextra->token.indent;i++)
729 {
730 unput(' ');
731 }
732 // tell flex that after putting the last indent
733 // back we are at the beginning of the line
734 YY_CURRENT_BUFFER->yy_at_bol=1;
735 // start of a new paragraph
736 return Token::make_TK_NEWPARA();
737 }
738 }
739<St_CodeOpt>{BLANK}*"{"(".")?{LABELID}"}" {
740 yyextra->token.name = yytext;
741 int i=yyextra->token.name.find('{'); /* } to keep vi happy */
742 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
743 BEGIN(St_Code);
744 }
745<St_iCodeOpt>{BLANK}*"{"(".")?{LABELID}"}" {
746 yyextra->token.name = yytext;
747 int i=yyextra->token.name.find('{'); /* } to keep vi happy */
748 yyextra->token.name = yyextra->token.name.mid(i+1,yyextra->token.name.length()-i-2);
749 BEGIN(St_iCode);
750 }
751<St_CodeOpt>"\\ilinebr" |
752<St_CodeOpt>\n |
753<St_CodeOpt>. {
754 unput_string(yytext,yyleng);
755 BEGIN(St_Code);
756 }
unput_string
#define unput_string(yytext, yyleng)
Definition doctokenizer.l:158
757<St_iCodeOpt>"\\ilinebr" |
758<St_iCodeOpt>\n |
759<St_iCodeOpt>. {
760 unput_string(yytext,yyleng);
761 BEGIN(St_iCode);
762 }
763<St_Code>{WS}*{CMD}"endcode" {
764 lineCount(yytext,yyleng);
765 return Token::make_RetVal_OK();
766 }
767<St_iCode>{WS}*{CMD}"endicode" {
768 lineCount(yytext,yyleng);
769 return Token::make_RetVal_OK();
770 }
771<St_XmlCode>{WS}*"</code>" {
772 lineCount(yytext,yyleng);
773 return Token::make_RetVal_OK();
774 }
775<St_Code,St_iCode,St_XmlCode>[^\\@\n<]+ |
776<St_Code,St_iCode,St_XmlCode>\n |
777<St_Code,St_iCode,St_XmlCode>. {
778 lineCount(yytext,yyleng);
779 yyextra->token.verb+=yytext;
780 }
781<St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
782 yyextra->token.name="block";
783 BEGIN(St_HtmlOnly);
784 }
785<St_HtmlOnlyOption>.|\n {
786 unput(*yytext);
787 BEGIN(St_HtmlOnly);
788 }
789<St_HtmlOnlyOption>"\\ilinebr" {
790 unput_string(yytext,yyleng);
791 BEGIN(St_HtmlOnly);
792 }
793<St_HtmlOnly>{CMD}"endhtmlonly" {
794 return Token::make_RetVal_OK();
795 }
796<St_HtmlOnly>[^\\@\n$]+ |
797<St_HtmlOnly>\n |
798<St_HtmlOnly>. {
799 lineCount(yytext,yyleng);
800 yyextra->token.verb+=yytext;
801 }
802<St_ManOnly>{CMD}"endmanonly" {
803 return Token::make_RetVal_OK();
804 }
805<St_ManOnly>[^\\@\n$]+ |
806<St_ManOnly>\n |
807<St_ManOnly>. {
808 lineCount(yytext,yyleng);
809 yyextra->token.verb+=yytext;
810 }
811<St_RtfOnly>{CMD}"endrtfonly" {
812 return Token::make_RetVal_OK();
813 }
814<St_RtfOnly>[^\\@\n$]+ |
815<St_RtfOnly>\n |
816<St_RtfOnly>. {
817 lineCount(yytext,yyleng);
818 yyextra->token.verb+=yytext;
819 }
820<St_LatexOnly>{CMD}"endlatexonly" {
821 return Token::make_RetVal_OK();
822 }
823<St_LatexOnly>[^\\@\n]+ |
824<St_LatexOnly>\n |
825<St_LatexOnly>. {
826 lineCount(yytext,yyleng);
827 yyextra->token.verb+=yytext;
828 }
829<St_XmlOnly>{CMD}"endxmlonly" {
830 return Token::make_RetVal_OK();
831 }
832<St_XmlOnly>[^\\@\n]+ |
833<St_XmlOnly>\n |
834<St_XmlOnly>. {
835 lineCount(yytext,yyleng);
836 yyextra->token.verb+=yytext;
837 }
838<St_DbOnly>{CMD}"enddocbookonly" {
839 return Token::make_RetVal_OK();
840 }
841<St_DbOnly>[^\\@\n]+ |
842<St_DbOnly>\n |
843<St_DbOnly>. {
844 lineCount(yytext,yyleng);
845 yyextra->token.verb+=yytext;
846 }
847<St_Verbatim>{CMD}"endverbatim" {
848 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
849 return Token::make_RetVal_OK();
850 }
851<St_ILiteral>{CMD}"endiliteral " { // note extra space as this is artificially added
852 // remove spaces that have been added
853 yyextra->token.verb = yyextra->token.verb.mid(1,yyextra->token.verb.length()-2);
854 return Token::make_RetVal_OK();
855 }
856<St_iVerbatim>{CMD}"endiverbatim" {
857 yyextra->token.verb = yyextra->token.verb.stripLeadingAndTrailingEmptyLines();
858 return Token::make_RetVal_OK();
859 }
860<St_Verbatim,St_iVerbatim,St_ILiteral>[^\\@\n]+ |
861<St_Verbatim,St_iVerbatim,St_ILiteral>\n |
862<St_Verbatim,St_iVerbatim,St_ILiteral>. { /* Verbatim text */
863 lineCount(yytext,yyleng);
864 yyextra->token.verb+=yytext;
865 }
866<St_ILiteralOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // option(s) present
867 yyextra->token.verb = QCString(yytext).stripWhiteSpace();
868 return Token::make_RetVal_OK();
869 }
QCString::stripWhiteSpace
QCString stripWhiteSpace() const
returns a copy of this string with leading and trailing whitespace removed
Definition qcstring.h:245
870<St_ILiteralOpt>"\\ilinebr" |
871<St_ILiteralOpt>"\n" |
872<St_ILiteralOpt>. {
873 yyextra->token.sectionId = "";
874 unput_string(yytext,yyleng);
875 return Token::make_RetVal_OK();
876 }
877<St_Dot>{CMD}"enddot" {
878 return Token::make_RetVal_OK();
879 }
880<St_Dot>[^\\@\n]+ |
881<St_Dot>\n |
882<St_Dot>. { /* dot text */
883 lineCount(yytext,yyleng);
884 yyextra->token.verb+=yytext;
885 }
886<St_Msc>{CMD}("endmsc") {
887 return Token::make_RetVal_OK();
888 }
889<St_Msc>[^\\@\n]+ |
890<St_Msc>\n |
891<St_Msc>. { /* msc text */
892 lineCount(yytext,yyleng);
893 yyextra->token.verb+=yytext;
894 }
895<St_PlantUMLOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
896 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
897 return Token::make_RetVal_OK();
898 }
899<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
900 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
901 return Token::make_RetVal_OK();
902 }
903<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
904 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
905 return Token::make_RetVal_OK();
906 }
907<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
908 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
909 return Token::make_RetVal_OK();
910 }
911<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
912 yyextra->token.sectionId = QCString(yytext).stripWhiteSpace();
913 return Token::make_RetVal_OK();
914 }
915<St_PlantUMLOpt>"\\ilinebr" |
916<St_PlantUMLOpt>"\n" |
917<St_PlantUMLOpt>. {
918 yyextra->token.sectionId = "";
919 unput_string(yytext,yyleng);
920 return Token::make_RetVal_OK();
921 }
922<St_PlantUML>{CMD}"enduml" {
923 return Token::make_RetVal_OK();
924 }
925<St_PlantUML>[^\\@\n]+ |
926<St_PlantUML>\n |
927<St_PlantUML>. { /* plantuml text */
928 lineCount(yytext,yyleng);
929 yyextra->token.verb+=yytext;
930 }
931<St_Title>"\"" { // quoted title
932 BEGIN(St_TitleQ);
933 }
934<St_Title>[ \t]+ {
935 yyextra->token.chars=yytext;
936 return Token::make_TK_WHITESPACE();
937 }
938<St_Title>. { // non-quoted title
939 unput(*yytext);
940 BEGIN(St_TitleN);
941 }
942<St_Title>\n {
943 unput(*yytext);
944 return Token::make_TK_NONE();
945 }
946<St_Title>"\\ilinebr" {
947 unput_string(yytext,yyleng);
948 return Token::make_TK_NONE();
949 }
950<St_TitleN>"&"{ID}";" { /* symbol */
951 yyextra->token.name = yytext;
952 return Token::make_TK_SYMBOL();
953 }
954<St_TitleN>{HTMLTAG} {
955 yyextra->token.name = yytext;
956 handleHtmlTag(yyscanner,yytext);
957 return Token::make_TK_HTMLTAG();
958 }
959<St_TitleN>\n { /* new line => end of title */
960 unput(*yytext);
961 return Token::make_TK_NONE();
962 }
963<St_TitleN>"\\ilinebr" { /* new line => end of title */
964 unput_string(yytext,yyleng);
965 return Token::make_TK_NONE();
966 }
967<St_TitleN>{SPCMD1} |
968<St_TitleN>{SPCMD2} { /* special command */
969 yyextra->token.name = yytext+1;
970 yyextra->token.paramDir=TokenInfo::Unspecified;
971 return Token::char_to_command(yytext[0]);
972 }
973<St_TitleN>{ID}"=" { /* attribute */
974 if (yytext[0]=='%') // strip % if present
975 yyextra->token.name = &yytext[1];
976 else
977 yyextra->token.name = yytext;
978 return Token::make_TK_WORD();
979 }
980<St_TitleN>[\-+0-9] |
981<St_TitleN>{WORD1} |
982<St_TitleN>{WORD2} { /* word */
983 if (QCString(yytext).find("\\ilinebr")!=-1) REJECT; // see issue #8311
984 lineCount(yytext,yyleng);
985 if (yytext[0]=='%') // strip % if present
986 yyextra->token.name = &yytext[1];
987 else
988 yyextra->token.name = yytext;
989 return Token::make_TK_WORD();
990 }
991<St_TitleN>[ \t]+ {
992 yyextra->token.chars=yytext;
993 return Token::make_TK_WHITESPACE();
994 }
995<St_TitleQ>"&"{ID}";" { /* symbol */
996 yyextra->token.name = yytext;
997 return Token::make_TK_SYMBOL();
998 }
999<St_TitleQ>(\n|"\\ilinebr") { /* new line => end of title */
1000 unput_string(yytext,yyleng);
1001 return Token::make_TK_NONE();
1002 }
1003<St_TitleQ>{SPCMD1} |
1004<St_TitleQ>{SPCMD2} { /* special command */
1005 yyextra->token.name = yytext+1;
1006 yyextra->token.paramDir=TokenInfo::Unspecified;
1007 return Token::char_to_command(yytext[0]);
1008 }
1009<St_TitleQ>{WORD1NQ} |
1010<St_TitleQ>{WORD2NQ} { /* word */
1011 yyextra->token.name = yytext;
1012 return Token::make_TK_WORD();
1013 }
1014<St_TitleQ>[ \t]+ {
1015 yyextra->token.chars=yytext;
1016 return Token::make_TK_WHITESPACE();
1017 }
1018<St_TitleQ>"\"" { /* closing quote => end of title */
1019 BEGIN(St_TitleA);
1020 return Token::make_TK_NONE();
1021 }
1022<St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
1023 yyextra->token.name = yytext;
1024 int pos = yyextra->token.name.find('=');
1025 if (pos<0) pos=0; // should never happen
1026 yyextra->token.name = yyextra->token.name.left(pos).stripWhiteSpace();
1027 BEGIN(St_TitleV);
1028 }
1029<St_TitleV>[^ \t\r\n]+ { // attribute value
1030 lineCount(yytext,yyleng);
1031 yyextra->token.chars = yytext;
1032 BEGIN(St_TitleN);
1033 return Token::make_TK_WORD();
1034 }
1035<St_TitleV,St_TitleA>. {
1036 unput(*yytext);
1037 return Token::make_TK_NONE();
1038 }
1039<St_TitleV,St_TitleA>(\n|"\\ilinebr") {
1040 unput_string(yytext,yyleng);
1041 return Token::make_TK_NONE();
1042 }
1043
1044<St_Anchor>{LABELID}{WS}? { // anchor
1045 lineCount(yytext,yyleng);
1046 yyextra->token.name = QCString(yytext).stripWhiteSpace();
1047 return Token::make_TK_WORD();
1048 }
1049<St_Anchor>. {
1050 unput(*yytext);
1051 return Token::make_TK_NONE();
1052 }
1053<St_Cite>{CITEID} { // label to cite
1054 if (yytext[0] =='"')
1055 {
1056 yyextra->token.name=yytext+1;
1057 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-2);
1058 }
1059 else
1060 {
1061 yyextra->token.name=yytext;
1062 }
1063 return Token::make_TK_WORD();
1064 }
1065<St_Cite>{BLANK} { // white space
1066 unput(' ');
1067 return Token::make_TK_NONE();
1068 }
1069<St_Cite>(\n|"\\ilinebr") { // new line
1070 unput_string(yytext,yyleng);
1071 return Token::make_TK_NONE();
1072 }
1073<St_Cite>. { // any other character
1074 unput(*yytext);
1075 return Token::make_TK_NONE();
1076 }
1077<St_DoxyConfig>{DOXYCFG} { // config option
1078 yyextra->token.name=yytext;
1079 return Token::make_TK_WORD();
1080 }
1081<St_DoxyConfig>{BLANK} { // white space
1082 unput(' ');
1083 return Token::make_TK_NONE();
1084 }
1085<St_DoxyConfig>(\n|"\\ilinebr") { // new line
1086 unput_string(yytext,yyleng);
1087 return Token::make_TK_NONE();
1088 }
1089<St_DoxyConfig>. { // any other character
1090 unput(*yytext);
1091 return Token::make_TK_NONE();
1092 }
1093<St_Ref>{REFWORD_NOCV}/{BLANK}("const")[a-z_A-Z0-9] { // see bug776988
1094 yyextra->token.name=yytext;
1095 return Token::make_TK_WORD();
1096 }
1097<St_Ref>{REFWORD_NOCV}/{BLANK}("volatile")[a-z_A-Z0-9] { // see bug776988
1098 yyextra->token.name=yytext;
1099 return Token::make_TK_WORD();
1100 }
1101<St_Ref>{REFWORD} { // label to refer to
1102 yyextra->token.name=yytext;
1103 return Token::make_TK_WORD();
1104 }
1105<St_Ref>{BLANK} { // white space
1106 unput(' ');
1107 return Token::make_TK_NONE();
1108 }
1109<St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
1110 lineCount(yytext,yyleng);
1111 BEGIN(St_Ref2);
1112 }
1113<St_Ref>(\n|"\\ilinebr") { // new line
1114 unput_string(yytext,yyleng);
1115 return Token::make_TK_NONE();
1116 }
1117<St_Ref>"\""[^"\n]+"\"" { // quoted first argument -> return without quotes
1118 yyextra->token.name=QCString(yytext).mid(1,yyleng-2);
1119 return Token::make_TK_WORD();
1120 }
1121<St_Ref>. { // any other character
1122 unput(*yytext);
1123 return Token::make_TK_NONE();
1124 }
1125<St_IntRef>[A-Z_a-z0-9.:/#\-\+\‍(\‍)]+ {
1126 yyextra->token.name = yytext;
1127 return Token::make_TK_WORD();
1128 }
1129<St_IntRef>{BLANK}+"\"" {
1130 BEGIN(St_Ref2);
1131 }
1132<St_SetScope>({SCOPEMASK}|{ANONNS}){BLANK}|{FILEMASK} {
1133 yyextra->token.name = yytext;
1134 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1135 return Token::make_TK_WORD();
1136 }
1137<St_SetScope>{SCOPEMASK}"<" {
1138 yyextra->token.name = yytext;
1139 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1140 yyextra->sharpCount=1;
1141 BEGIN(St_SetScopeEnd);
1142 }
1143<St_SetScope>{BLANK} {
1144 }
1145<St_SetScopeEnd>"<" {
1146 yyextra->token.name += yytext;
1147 yyextra->sharpCount++;
1148 }
1149<St_SetScopeEnd>">" {
1150 yyextra->token.name += yytext;
1151 yyextra->sharpCount--;
1152 if (yyextra->sharpCount<=0)
1153 {
1154 return Token::make_TK_WORD();
1155 }
1156 }
1157<St_SetScopeEnd>. {
1158 yyextra->token.name += yytext;
1159 }
1160<St_Ref2>"&"{ID}";" { /* symbol */
1161 yyextra->token.name = yytext;
1162 return Token::make_TK_SYMBOL();
1163 }
1164<St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title */
1165 lineCount(yytext,yyleng);
1166 return Token::make_TK_NONE();
1167 }
1168<St_Ref2>{HTMLTAG_STRICT} { /* html tag */
1169 lineCount(yytext,yyleng);
1170 handleHtmlTag(yyscanner,yytext);
1171 return Token::make_TK_HTMLTAG();
1172 }
1173<St_Ref2>{SPCMD1} |
1174<St_Ref2>{SPCMD2} { /* special command */
1175 yyextra->token.name = yytext+1;
1176 yyextra->token.paramDir=TokenInfo::Unspecified;
1177 return Token::char_to_command(yytext[0]);
1178 }
1179<St_Ref2>{WORD1NQ} |
1180<St_Ref2>{WORD2NQ} {
1181 /* word */
1182 yyextra->token.name = yytext;
1183 return Token::make_TK_WORD();
1184 }
1185<St_Ref2>[ \t]+ {
1186 yyextra->token.chars=yytext;
1187 return Token::make_TK_WHITESPACE();
1188 }
1189<St_XRefItem>{LABELID} {
1190 yyextra->token.name=yytext;
1191 }
1192<St_XRefItem>" " {
1193 BEGIN(St_XRefItem2);
1194 }
1195<St_XRefItem2>[0-9]+"." {
1196 QCString numStr(yytext);
1197 numStr=numStr.left((int)yyleng-1);
1198 yyextra->token.id=numStr.toInt();
1199 return Token::make_RetVal_OK();
1200 }
1201<St_Para,St_Title,St_Ref2>"<!--" { /* html style comment block */
1202 yyextra->commentState = YY_START;
1203 BEGIN(St_Comment);
1204 }
1205<St_Param>"\""[^\n\"]+"\"" {
1206 yyextra->token.name = yytext+1;
1207 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1208 return Token::make_TK_WORD();
1209 }
1210<St_Param>({PHPTYPE}{BLANK}*("["{BLANK}*"]")*{BLANK}*"|"{BLANK}*)*{PHPTYPE}{BLANK}*("["{BLANK}*"]")*{WS}+("&")?"$"{LABELID} {
1211 lineCount(yytext,yyleng);
1212 QCString params(yytext);
1213 int j = params.find('&');
1214 int i = params.find('$');
1215 if (i<0) i=0; // should never happen
1216 if (j<i && j>=0) i=j;
1217 QCString types = params.left(i).stripWhiteSpace();
1218 yyextra->token.name = types+"#"+params.mid(i);
1219 return Token::make_TK_WORD();
1220 }
types
static const char types[][NUM_HTML_LIST_TYPES]
Definition htmldocvisitor.cpp:44
1221<St_Param>[^ \t\n,@\\]+ {
1222 yyextra->token.name = yytext;
1223 if (yyextra->token.name.at(static_cast<uint32_t>(yyleng)-1)==':')
1224 {
1225 yyextra->token.name=yyextra->token.name.left(static_cast<uint32_t>(yyleng)-1);
1226 }
1227 return Token::make_TK_WORD();
1228 }
1229<St_Param>{WS}*","{WS}* /* param separator */
1230<St_Param>{WS} {
1231 lineCount(yytext,yyleng);
1232 yyextra->token.chars=yytext;
1233 return Token::make_TK_WHITESPACE();
1234 }
1235<St_Prefix>"\""[^\n\"]*"\"" {
1236 yyextra->token.name = yytext+1;
1237 yyextra->token.name = yyextra->token.name.left((int)yyleng-2);
1238 return Token::make_TK_WORD();
1239 }
1240<St_Prefix>. {
1241 unput(*yytext);
1242 return Token::make_TK_NONE();
1243 }
1244<St_Options>{ID} {
1245 yyextra->token.name+=yytext;
1246 }
1247<St_Options>{WS}*":"{WS}* {
1248 lineCount(yytext,yyleng);
1249 yyextra->token.name+=":";
1250 }
1251<St_Options>{WS}*","{WS}* |
1252<St_Options>{WS} { /* option separator */
1253 lineCount(yytext,yyleng);
1254 yyextra->token.name+=",";
1255 }
1256<St_Options>"}" {
1257 return Token::make_TK_WORD();
1258 }
1259<St_Block>{ID} {
1260 yyextra->token.name+=yytext;
1261 }
1262<St_Block>"]" {
1263 return Token::make_TK_WORD();
1264 }
1265<St_Emoji>[:0-9_a-z+-]+ {
1266 yyextra->token.name=yytext;
1267 return Token::make_TK_WORD();
1268 }
1269<St_Emoji>. {
1270 unput(*yytext);
1271 return Token::make_TK_NONE();
1272 }
1273<St_QuotedString>"\"" {
1274 yyextra->token.name="";
1275 BEGIN(St_QuotedContent);
1276 }
1277<St_QuotedString>(\n|"\\ilinebr") {
1278 unput_string(yytext,yyleng);
1279 return Token::make_TK_NONE();
1280 }
1281<St_QuotedString>. {
1282 unput(*yytext);
1283 return Token::make_TK_NONE();
1284 }
1285<St_QuotedContent>"\"" {
1286 return Token::make_TK_WORD();
1287 }
1288<St_QuotedContent>. {
1289 yyextra->token.name+=yytext;
1290 }
1291<St_ShowDate>{WS}+{SHOWDATE} {
1292 lineCount(yytext,yyleng);
1293 yyextra->token.name=yytext;
1294 return Token::make_TK_WORD();
1295 }
1296<St_ShowDate>(\n|"\\ilinebr") {
1297 unput_string(yytext,yyleng);
1298 return Token::make_TK_NONE();
1299 }
1300<St_ShowDate>. {
1301 unput(*yytext);
1302 return Token::make_TK_NONE();
1303 }
1304<St_ILine>{LINENR}/[\\@\n\.] |
1305<St_ILine>{LINENR}{BLANK} {
1306 bool ok = false;
1307 int nr = QCString(yytext).toInt(&ok);
1308 if (!ok)
1309 {
1310 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '%s' for iline command",yytext);
1311 }
1312 else
1313 {
1314 yyextra->yyLineNr = nr;
1315 }
1316 return Token::make_TK_WORD();
1317 }
1318<St_ILine>. {
1319 return Token::make_TK_NONE();
1320 }
1321<St_IFile>{BLANK}*{FILEMASK} {
1322 yyextra->fileName = QCString(yytext).stripWhiteSpace();
1323 return Token::make_TK_WORD();
1324 }
1325<St_IFile>{BLANK}*"\""[^\n\"]+"\"" {
1326 QCString text(yytext);
1327 text = text.stripWhiteSpace();
1328 yyextra->fileName = text.mid(1,text.length()-2);
1329 return Token::make_TK_WORD();
1330 }
1331<St_File>{FILEMASK} {
1332 yyextra->token.name = yytext;
1333 return Token::make_TK_WORD();
1334 }
1335<St_File>"\""[^\n\"]+"\"" {
1336 QCString text(yytext);
1337 yyextra->token.name = text.mid(1,text.length()-2);
1338 return Token::make_TK_WORD();
1339 }
1340<St_Pattern>[^\\\r\n]+ {
1341 yyextra->token.name += yytext;
1342 }
1343<St_Pattern>"\\ilinebr" {
1344 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1345 return Token::make_TK_WORD();
1346 }
1347<St_Pattern>\n {
1348 lineCount(yytext,yyleng);
1349 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1350 return Token::make_TK_WORD();
1351 }
1352<St_Pattern>. {
1353 yyextra->token.name += yytext;
1354 }
1355<St_Link>{LINKMASK}|{REFWORD} {
1356 yyextra->token.name = yytext;
1357 return Token::make_TK_WORD();
1358 }
1359<St_Comment>"-->" { /* end of html comment */
1360 BEGIN(yyextra->commentState);
1361 }
1362<St_Comment>[^-]+ /* inside html comment */
1363<St_Comment>. /* inside html comment */
1364
1365 /* State for skipping title (all chars until the end of the line) */
1366
1367<St_SkipTitle>.
1368<St_SkipTitle>(\n|"\\ilinebr") {
1369 if (*yytext == '\n') unput('\n');
1370 return Token::make_TK_NONE();
1371 }
1372
1373 /* State for the pass used to find the anchors and sections */
1374
1375<St_Sections>[^\n@\<]+
1376<St_Sections>{CMD}("<"|{CMD})
1377<St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
1378 lineCount(yytext,yyleng);
1379 QCString tag(yytext);
1380 int s=tag.find("id=");
1381 if (s!=-1) // command has id attribute
1382 {
1383 char c=tag[s+3];
1384 if (c=='\'' || c=='"') // valid start
1385 {
1386 int e=tag.find(c,s+4);
1387 if (e!=-1) // found matching end
1388 {
1389 yyextra->secType = SectionType::Table;
1390 yyextra->secLabel=tag.mid(s+4,e-s-4); // extract id
1391 processSection(yyscanner);
1392 }
1393 }
1394 }
1395 }
SectionType::Table
static constexpr int Table
Definition section.h:41
processSection
static void processSection(yyscan_t yyscanner)
Definition doctokenizer.l:1574
1396<St_Sections>{CMD}"anchor"{BLANK}+ {
1397 yyextra->secType = SectionType::Anchor;
1398 BEGIN(St_SecLabel1);
1399 }
SectionType::Anchor
static constexpr int Anchor
Definition section.h:40
1400<St_Sections>{CMD}"ianchor"{BLANK}+ {
1401 yyextra->secType = SectionType::Anchor;
1402 BEGIN(St_SecLabel1);
1403 }
1404<St_Sections>{CMD}"section"{BLANK}+ {
1405 yyextra->secType = SectionType::Section;
1406 BEGIN(St_SecLabel2);
1407 }
SectionType::Section
static constexpr int Section
Definition section.h:33
1408<St_Sections>{CMD}"subsection"{BLANK}+ {
1409 yyextra->secType = SectionType::Subsection;
1410 BEGIN(St_SecLabel2);
1411 }
SectionType::Subsection
static constexpr int Subsection
Definition section.h:34
1412<St_Sections>{CMD}"subsubsection"{BLANK}+ {
1413 yyextra->secType = SectionType::Subsubsection;
1414 BEGIN(St_SecLabel2);
1415 }
SectionType::Subsubsection
static constexpr int Subsubsection
Definition section.h:35
1416<St_Sections>{CMD}"paragraph"{BLANK}+ {
1417 yyextra->secType = SectionType::Paragraph;
1418 BEGIN(St_SecLabel2);
1419 }
SectionType::Paragraph
static constexpr int Paragraph
Definition section.h:36
1420<St_Sections>{CMD}"subparagraph"{BLANK}+ {
1421 yyextra->secType = SectionType::Subparagraph;
1422 BEGIN(St_SecLabel2);
1423 }
SectionType::Subparagraph
static constexpr int Subparagraph
Definition section.h:37
1424<St_Sections>{CMD}"subsubparagraph"{BLANK}+ {
1425 yyextra->secType = SectionType::Subsubparagraph;
1426 BEGIN(St_SecLabel2);
1427 }
SectionType::Subsubparagraph
static constexpr int Subsubparagraph
Definition section.h:38
1428<St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9] {
1429 yyextra->endMarker="endverbatim";
1430 BEGIN(St_SecSkip);
1431 }
1432<St_Sections>{CMD}"iverbatim"/[^a-z_A-Z0-9] {
1433 yyextra->endMarker="endiverbatim";
1434 BEGIN(St_SecSkip);
1435 }
1436<St_Sections>{CMD}"iliteral"/[^a-z_A-Z0-9] {
1437 yyextra->endMarker="endiliteral";
1438 BEGIN(St_SecSkip);
1439 }
1440<St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1441 yyextra->endMarker="enddot";
1442 BEGIN(St_SecSkip);
1443 }
1444<St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1445 yyextra->endMarker="endmsc";
1446 BEGIN(St_SecSkip);
1447 }
1448<St_Sections>{CMD}"startuml"/[^a-z_A-Z0-9] {
1449 yyextra->endMarker="enduml";
1450 BEGIN(St_SecSkip);
1451 }
1452<St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1453 yyextra->endMarker="endhtmlonly";
1454 BEGIN(St_SecSkip);
1455 }
1456<St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1457 yyextra->endMarker="endlatexonly";
1458 BEGIN(St_SecSkip);
1459 }
1460<St_Sections>{CMD}"manonly"/[^a-z_A-Z0-9] {
1461 yyextra->endMarker="endmanonly";
1462 BEGIN(St_SecSkip);
1463 }
1464<St_Sections>{CMD}"rtfonly"/[^a-z_A-Z0-9] {
1465 yyextra->endMarker="endrtfonly";
1466 BEGIN(St_SecSkip);
1467 }
1468<St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1469 yyextra->endMarker="endxmlonly";
1470 BEGIN(St_SecSkip);
1471 }
1472<St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
1473 yyextra->endMarker="enddocbookonly";
1474 BEGIN(St_SecSkip);
1475 }
1476<St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1477 yyextra->endMarker="endcode";
1478 BEGIN(St_SecSkip);
1479 }
1480<St_Sections>{CMD}"icode"/[^a-z_A-Z0-9] {
1481 yyextra->endMarker="endicode";
1482 BEGIN(St_SecSkip);
1483 }
1484<St_Sections>"<!--" {
1485 yyextra->endMarker="-->";
1486 BEGIN(St_SecSkip);
1487 }
1488<St_SecSkip>{CMD}{ID} {
1489 if (yyextra->endMarker==yytext+1)
1490 {
1491 BEGIN(St_Sections);
1492 }
1493 }
1494<St_SecSkip>"-->" {
1495 if (yyextra->endMarker==yytext)
1496 {
1497 BEGIN(St_Sections);
1498 }
1499 }
1500<St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1501<St_SecSkip>.
1502<St_SecSkip>(\n|"\\ilinebr")
1503<St_Sections>.
1504<St_Sections>(\n|"\\ilinebr")
1505<St_SecLabel1>{LABELID} {
1506 lineCount(yytext,yyleng);
1507 yyextra->secLabel = yytext;
1508 processSection(yyscanner);
1509 BEGIN(St_Sections);
1510 }
1511<St_SecLabel2>{LABELID}{BLANK}+ |
1512<St_SecLabel2>{LABELID} {
1513 yyextra->secLabel = yytext;
1514 yyextra->secLabel = yyextra->secLabel.stripWhiteSpace();
1515 BEGIN(St_SecTitle);
1516 }
1517<St_SecTitle>[^\n]+ |
1518<St_SecTitle>[^\n]*\n {
1519 lineCount(yytext,yyleng);
1520 yyextra->secTitle = yytext;
1521 yyextra->secTitle = yyextra->secTitle.stripWhiteSpace();
1522 if (yyextra->secTitle.endsWith("\\ilinebr"))
1523 {
1524 yyextra->secTitle.left(yyextra->secTitle.length()-8);
1525 }
1526 processSection(yyscanner);
1527 BEGIN(St_Sections);
1528 }
1529<St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1530 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '%s' while looking for section label or title",yytext);
1531 }
1532
1533<St_Snippet>[^\\\n]+ {
1534 yyextra->token.name += yytext;
1535 }
1536<St_Snippet>"\\" {
1537 yyextra->token.name += yytext;
1538 }
1539<St_Snippet>(\n|"\\ilinebr") {
1540 unput_string(yytext,yyleng);
1541 yyextra->token.name = yyextra->token.name.stripWhiteSpace();
1542 return Token::make_TK_WORD();
1543 }
1544
1545 /* Generic rules that work for all states */
1546<*>\n {
1547 lineCount(yytext,yyleng);
1548 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected new line character");
1549 }
1550<*>"\\ilinebr" {
1551 }
1552<*>[\\@<>&$#%~"=] { /* unescaped special character */
1553 //warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '%s', assuming command \\%s was meant.",yytext,yytext);
1554 yyextra->token.name = yytext;
1555 return Token::char_to_command(yytext[0]);
1556 }
1557<*>. {
1558 warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '%s'",yytext);
1559 }
1560%%

◆ yyread()

static int yyread ( yyscan_t yyscanner,
char * buf,
int max_size )
static

Definition at line 1564 of file doctokenizer.l.

1565{
1566 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1567 int c=0;
1568 const char *p = yyextra->inputString + yyextra->inputPos;
1569 while ( c < max_size && *p ) { *buf++ = *p++; c++; }
1570 yyextra->inputPos+=c;
1571 return c;
1572}