Doxygen
Loading...
Searching...
No Matches
pyscanner.l File Reference
#include <stdint.h>
#include "pyscanner.h"
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "arguments.h"
#include "commentscan.h"
#include "config.h"
#include "debug.h"
#include "defargs.h"
#include "entry.h"
#include "fileinfo.h"
#include "markdown.h"
#include "message.h"
#include "stringutil.h"
#include "util.h"
#include "doxygen_lex.h"
#include "pyscanner.l.h"
Include dependency graph for pyscanner.l:

Go to the source code of this file.

Classes

struct  pyscannerYY_state
struct  PythonOutlineParser::Private

Macros

#define YY_TYPEDEF_YY_SCANNER_T
#define DBG_CTX(x)
#define YY_NO_INPUT   1
#define YY_NO_UNISTD_H   1
#define unput_string(yytext, yyleng)
#define YY_INPUT(buf, result, max_size)

Typedefs

typedef yyguts_t * yyscan_t

Functions

static const char * stateToString (int state)
static int computeIndent (const char *s)
static void initParser (yyscan_t yyscanner)
static void initEntry (yyscan_t yyscanner)
static void newEntry (yyscan_t yyscanner)
static void addEntry (yyscan_t yyscanner)
static void docVariable (yyscan_t yyscanner, const char *s)
static void newVariable (yyscan_t yyscanner)
static void newTypeAlias (yyscan_t yyscanner)
static void addVariable (yyscan_t yyscanner)
static void newFunction (yyscan_t yyscanner)
static DString findPackageScopeFromPath (yyscan_t yyscanner, const DString &path)
static void addFrom (yyscan_t yyscanner, bool all)
static void lineCount (yyscan_t yyscanner)
static void incLineNr (yyscan_t yyscanner)
static void startCommentBlock (yyscan_t yyscanner, bool brief)
static void handleCommentBlock (yyscan_t yyscanner, const DString &doc, bool brief)
static void endOfDef (yyscan_t yyscanner, int correction=0)
static void addToString (yyscan_t yyscanner, const char *s)
static void initTriDoubleQuoteBlock (yyscan_t yyscanner)
static void initTriSingleQuoteBlock (yyscan_t yyscanner)
static void initSpecialBlock (yyscan_t yyscanner)
static void searchFoundDef (yyscan_t yyscanner)
static void searchFoundClass (yyscan_t yyscanner)
static void searchFoundTypeAlias (yyscan_t yyscanner)
static DString findPackageScope (yyscan_t yyscanner, const DString &fileName)
static void setProtection (yyscan_t yyscanner)
static int yyread (yyscan_t yyscanner, char *buf, int max_size)
static const char * getLexerFILE ()
int yylex (yyscan_t yyscanner)
static void parseCompounds (yyscan_t yyscanner, std::shared_ptr< Entry > rt)
static void parseMain (yyscan_t yyscanner, const DString &fileName, const char *fileBuf, const std::shared_ptr< Entry > &rt)
static void parsePrototype (yyscan_t yyscanner, const DString &text)

Macro Definition Documentation

◆ DBG_CTX

#define DBG_CTX ( x)
Value:
do { } while(0)

Definition at line 60 of file pyscanner.l.

◆ unput_string

#define unput_string ( yytext,
yyleng )
Value:
do { for (int i=(int)yyleng-1;i>=0;i--) unput(yytext[i]); } while(0)

Definition at line 65 of file pyscanner.l.

◆ YY_INPUT

#define YY_INPUT ( buf,
result,
max_size )
Value:
result=yyread(yyscanner,buf,max_size);
yyread
static int yyread(yyscan_t yyscanner, char *buf, int max_size)
Definition commentcnv.l:2096

Definition at line 161 of file pyscanner.l.

◆ YY_NO_INPUT

#define YY_NO_INPUT   1

Definition at line 62 of file pyscanner.l.

◆ YY_NO_UNISTD_H

#define YY_NO_UNISTD_H   1

Definition at line 63 of file pyscanner.l.

◆ YY_TYPEDEF_YY_SCANNER_T

#define YY_TYPEDEF_YY_SCANNER_T

Definition at line 29 of file pyscanner.l.

Typedef Documentation

◆ yyscan_t

typedef yyguts_t* yyscan_t

Definition at line 31 of file pyscanner.l.

Function Documentation

◆ addEntry()

void addEntry ( yyscan_t yyscanner)
static

Definition at line 1910 of file pyscanner.l.

1911{
1912 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1913
1914 auto doc = yyextra->current->doc;
1915 auto docLine = yyextra->current->docLine;
1916 auto docFile = yyextra->current->docFile;
1917 auto brief = yyextra->current->brief;
1918 auto briefLine = yyextra->current->briefLine;
1919 auto briefFile = yyextra->current->briefFile;
1920
1921 yyextra->previous = yyextra->current;
1922 yyextra->current_root->moveToSubEntryAndRefresh(yyextra->current);
1923 initEntry(yyscanner);
initEntry
static void initEntry(yyscan_t yyscanner)
Definition pyscanner.l:1848
1924
1925 yyextra->current->doc = doc;
1926 yyextra->current->docLine = docLine;
1927 yyextra->current->docFile = docFile;
1928 yyextra->current->brief = brief;
1929 yyextra->current->briefLine = briefLine;
1930 yyextra->current->briefFile = briefFile;
1931}

References initEntry().

Referenced by addVariable().

◆ addFrom()

void addFrom ( yyscan_t yyscanner,
bool all )
static

Definition at line 2055 of file pyscanner.l.

2056{
2057 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2058 DString item=all ? yyextra->packageName : yyextra->packageName+"."+yytext;
2059 yyextra->current->name=removeRedundantWhiteSpace(substitute(item,".","::"));
2060 yyextra->current->fileName = yyextra->fileName;
2061 //printf("Adding using declaration: found:%s:%d name=%s\n",qPrint(yyextra->fileName),yyextra->yyLineNr,qPrint(yyextra->current->name));
2062 yyextra->current->section=all ? EntryType::makeUsingDir() : EntryType::makeUsingDecl();
2063 yyextra->current_root->moveToSubEntryAndRefresh(yyextra->current);
2064 initEntry(yyscanner);
DString
A String class for use with Doxygen wrapping std::string and adding some additional functionality off...
Definition dstring.h:84
EntryType
Wrapper class for the Entry type.
Definition types.h:856
substitute
DString substitute(const DString &s, const DString &src, const DString &dst)
substitute all occurrences of src in s by dst
Definition dstring.cpp:485
removeRedundantWhiteSpace
DString removeRedundantWhiteSpace(const DString &s)
Definition util.cpp:426
2065}

References initEntry(), removeRedundantWhiteSpace(), and substitute().

◆ addToString()

void addToString ( yyscan_t yyscanner,
const char * s )
inlinestatic

Definition at line 2169 of file pyscanner.l.

2170{
2171 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2172 if (yyextra->copyString) (*yyextra->copyString) << s;
2173}

◆ addVariable()

void addVariable ( yyscan_t yyscanner)
static

Definition at line 1980 of file pyscanner.l.

1981{
1982 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1983 setProtection(yyscanner);
1984 if (yyextra->current_root->section.isCompound()) // mark as class variable
1985 {
1986 yyextra->current->isStatic = true;
1987 }
1988 addEntry(yyscanner);
setProtection
static void setProtection(yyscan_t yyscanner)
Definition pyscanner.l:1933
addEntry
static void addEntry(yyscan_t yyscanner)
Definition pyscanner.l:1910
1989}

References addEntry(), and setProtection().

◆ computeIndent()

int computeIndent ( const char * s)
inlinestatic

Definition at line 2006 of file pyscanner.l.

2007{
2008 int col=0;
2009 int tabSize=Config_getInt(TAB_SIZE);
2010 const char *p=s;
2011 char c;
2012 while ((c=*p++))
2013 {
2014 if (c==' ') col++;
2015 else if (c=='\t') col+=tabSize-(col%tabSize);
2016 else break;
2017 }
2018 return col;
Config_getInt
#define Config_getInt(name)
Definition config.h:34
2019}

References Config_getInt.

◆ docVariable()

void docVariable ( yyscan_t yyscanner,
const char * s )
static

Definition at line 1949 of file pyscanner.l.

1950{
1951 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1952 yyextra->current->name = name;
1953 yyextra->current->section=EntryType::makeVariable();
1954 yyextra->current->fileName = yyextra->fileName;
1955 yyextra->current->startLine = yyextra->yyLineNr;
1956 yyextra->current->bodyLine = yyextra->yyLineNr;
1957 yyextra->current->type.clear();
1958 setProtection(yyscanner);
1959 yyextra->checkDupEntry = true;
1960}

References setProtection().

◆ endOfDef()

void endOfDef ( yyscan_t yyscanner,
int correction = 0 )
static

Definition at line 2156 of file pyscanner.l.

2157{
2158 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2159 //printf("endOfDef at=%d\n",yyextra->yyLineNr);
2160 if (yyextra->bodyEntry)
2161 {
2162 yyextra->bodyEntry->endBodyLine = yyextra->yyLineNr-correction;
2163 yyextra->bodyEntry = 0;
2164 }
2165 newEntry(yyscanner);
2166 //yyextra->insideConstructor = false;
newEntry
static void newEntry(yyscan_t yyscanner)
Definition pyscanner.l:1864
2167}

References newEntry().

◆ findPackageScope()

DString findPackageScope ( yyscan_t yyscanner,
const DString & fileName )
static

Definition at line 2048 of file pyscanner.l.

2049{
2050 if (fileName.empty()) return fileName;
2051 FileInfo fi(fileName.str());
2052 return findPackageScopeFromPath(yyscanner,fi.dirPath(true));
DString::empty
bool empty() const
Returns true iff the string is empty (std::string compatible alias for isEmpty()).
Definition dstring.h:148
DString::str
const std::string & str() const
Definition dstring.h:645
FileInfo
Minimal replacement for QFileInfo.
Definition fileinfo.h:26
findPackageScopeFromPath
static DString findPackageScopeFromPath(yyscan_t yyscanner, const DString &path)
Definition pyscanner.l:2021
2053}

References FileInfo::dirPath(), DString::empty(), findPackageScopeFromPath(), and DString::str().

Referenced by parseMain().

◆ findPackageScopeFromPath()

DString findPackageScopeFromPath ( yyscan_t yyscanner,
const DString & path )
static

Definition at line 2021 of file pyscanner.l.

2022{
2023 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2024 auto it = yyextra->packageNameCache.find(path.str());
2025 if (it!=yyextra->packageNameCache.end())
2026 {
2027 return it->second;
2028 }
2029 FileInfo pf(path.str()+"/__init__.py"); // found package initialization file
2030 if (pf.exists())
2031 {
2032 size_t i=path.rfind('/');
2033 if (i!=DString::npos)
2034 {
2035 DString scope = findPackageScopeFromPath(yyscanner,path.left(i));
2036 if (!scope.empty())
2037 {
2038 scope+="::";
2039 }
2040 scope+=path.mid(i+1);
2041 yyextra->packageNameCache.emplace(path.str(),scope.str());
2042 return scope;
2043 }
2044 }
2045 return "";
DString::rfind
size_t rfind(char c, size_t pos=npos) const
Definition dstring.h:244
DString::mid
DString mid(size_t index, size_t len=npos) const
Definition dstring.h:318
DString::npos
static constexpr size_t npos
value used to indicate 'not found' or 'to the end of the string', matching std::string::npos
Definition dstring.h:178
DString::left
DString left(size_t len) const
Definition dstring.h:306
2046}

References DString::empty(), FileInfo::exists(), DString::find(), findPackageScopeFromPath(), DString::left(), DString::mid(), DString::npos, DString::rfind(), and DString::str().

Referenced by findPackageScope(), and findPackageScopeFromPath().

◆ getLexerFILE()

const char * getLexerFILE ( )
inlinestatic

Definition at line 164 of file pyscanner.l.

164{return __FILE__;}

◆ handleCommentBlock()

void handleCommentBlock ( yyscan_t yyscanner,
const DString & doc,
bool brief )
static

Definition at line 2101 of file pyscanner.l.

2102{
2103 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2104 //printf("handleCommentBlock(doc=[%s] brief=%d yyextra->docBlockInBody=%d yyextra->docBlockJavaStyle=%d\n",
2105 // qPrint(doc),brief,yyextra->docBlockInBody,yyextra->docBlockJavaStyle);
2106
2107 // TODO: Fix me
2108 yyextra->docBlockInBody=false;
2109
2110 if (!yyextra->current->doc.empty())
2111 {
2112 yyextra->current->doc=yyextra->current->doc.stripWhiteSpace()+"\n\n";
2113 }
2114 if (yyextra->docBlockInBody && yyextra->previous && !yyextra->previous->doc.empty())
2115 {
2116 yyextra->previous->doc=yyextra->previous->doc.stripWhiteSpace()+"\n\n";
2117 }
2118
2119 int position = 0;
2120 bool needsEntry = false;
2121 int lineNr = brief ? yyextra->current->briefLine : yyextra->current->docLine;
2122 Markdown markdown(yyextra->fileName,lineNr);
2123 GuardedSectionStack guards;
2124 DString strippedDoc = stripIndentation(doc,true);
2125 //printf("stippedDoc=[%s]\n",qPrint(strippedDoc));
2126 DString processedDoc = Config_getBool(MARKDOWN_SUPPORT) ? markdown.process(strippedDoc,lineNr) : strippedDoc;
2127 while (yyextra->commentScanner.parseCommentBlock(
2128 yyextra->thisParser,
2129 (yyextra->docBlockInBody && yyextra->previous) ? yyextra->previous.get() : yyextra->current.get(),
2130 processedDoc, // text
2131 yyextra->fileName, // file
2132 lineNr,
2133 yyextra->docBlockInBody ? false : brief,
2134 yyextra->docBlockJavaStyle, // javadoc style // or false,
2135 yyextra->docBlockInBody,
2136 yyextra->protection,
2137 position,
2138 needsEntry,
2139 Config_getBool(MARKDOWN_SUPPORT),
2140 &guards
2141 )
2142 ) // need to start a new entry
2143 {
2144 if (needsEntry)
2145 {
2146 newEntry(yyscanner);
2147 }
2148 }
2149 if (needsEntry)
2150 {
2151 newEntry(yyscanner);
2152 }
Markdown
Helper class to process markdown formatted text.
Definition markdown.h:33
GuardedSectionStack
std::stack< GuardedSection > GuardedSectionStack
Definition commentscan.h:48
Config_getBool
#define Config_getBool(name)
Definition config.h:33
stripIndentation
DString stripIndentation(const DString &s, bool skipFirstLine)
Definition util.cpp:4683
2153
2154}

References Config_getBool, newEntry(), Markdown::process(), and stripIndentation().

◆ incLineNr()

void incLineNr ( yyscan_t yyscanner)
static

Definition at line 2078 of file pyscanner.l.

2079{
2080 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2081 DBG_CTX((stderr,"yyextra->yyLineNr=%d\n",yyextra->yyLineNr));
2082 yyextra->yyLineNr++;
DBG_CTX
#define DBG_CTX(x)
Definition pyscanner.l:60
2083}

References DBG_CTX.

◆ initEntry()

void initEntry ( yyscan_t yyscanner)
static

Definition at line 1848 of file pyscanner.l.

1849{
1850 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1851 //yyextra->current->python = true;
1852 yyextra->current->protection = yyextra->protection ;
1853 yyextra->current->mtype = yyextra->mtype;
1854 yyextra->current->virt = yyextra->virt;
1855 yyextra->current->isStatic = yyextra->isStatic;
1856 yyextra->current->lang = SrcLangExt::Python;
1857 yyextra->current->type.clear();
1858 yyextra->current->name.clear();
1859 yyextra->current->initializer.clear();
1860 yyextra->commentScanner.initGroupInfo(yyextra->current.get());
1861 yyextra->isStatic = false;
1862}

◆ initParser()

void initParser ( yyscan_t yyscanner)
static

Definition at line 1837 of file pyscanner.l.

1838{
1839 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1840 yyextra->protection = Protection::Public;
1841 yyextra->mtype = MethodTypes::Method;
1842 yyextra->isStatic = false;
1843 yyextra->virt = Specifier::Normal;
1844 yyextra->previous = 0;
1845 yyextra->packageCommentAllowed = true;
1846}

Referenced by parseMain().

◆ initSpecialBlock()

void initSpecialBlock ( yyscan_t yyscanner)
static

Definition at line 2201 of file pyscanner.l.

2202{
2203 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2204 yyextra->docBlockContext = YY_START;
2205 yyextra->docBlockInBody = false;
2206 yyextra->docBlockJavaStyle = true;
2207 yyextra->docBrief = true;
2208 yyextra->docBlock.clear();
2209 yyextra->commentIndent = yyextra->curIndent;
2210 startCommentBlock(yyscanner,false);
startCommentBlock
static void startCommentBlock(yyscan_t yyscanner, bool brief)
Definition pyscanner.l:2086
2211}

References startCommentBlock().

◆ initTriDoubleQuoteBlock()

void initTriDoubleQuoteBlock ( yyscan_t yyscanner)
static

Definition at line 2175 of file pyscanner.l.

2176{
2177 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2178 yyextra->docBlockContext = YY_START;
2179 yyextra->docBlockInBody = false;
2180 yyextra->docBlockJavaStyle = true;
2181 yyextra->docBlockSpecial = yytext[strlen(yytext) - 1]=='!' || !Config_getBool(PYTHON_DOCSTRING);
2182 yyextra->docBlock.clear();
2183 yyextra->commentIndent = yyextra->curIndent;
2184 yyextra->doubleQuote = true;
2185 startCommentBlock(yyscanner,false);
2186}

References Config_getBool, and startCommentBlock().

◆ initTriSingleQuoteBlock()

void initTriSingleQuoteBlock ( yyscan_t yyscanner)
static

Definition at line 2188 of file pyscanner.l.

2189{
2190 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2191 yyextra->docBlockContext = YY_START;
2192 yyextra->docBlockInBody = false;
2193 yyextra->docBlockJavaStyle = true;
2194 yyextra->docBlockSpecial = yytext[strlen(yytext) - 1]=='!' || !Config_getBool(PYTHON_DOCSTRING);
2195 yyextra->docBlock.clear();
2196 yyextra->commentIndent = yyextra->curIndent;
2197 yyextra->doubleQuote = false;
2198 startCommentBlock(yyscanner,false);
2199}

References Config_getBool, and startCommentBlock().

◆ lineCount()

void lineCount ( yyscan_t yyscanner)
static

Definition at line 2068 of file pyscanner.l.

2069{
2070 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2071 DBG_CTX((stderr,"yyextra->yyLineNr=%d\n",yyextra->yyLineNr));
2072 for (const char *p = yytext; *p; ++p)
2073 {
2074 yyextra->yyLineNr += (*p == '\n') ;
2075 }
2076}

References DBG_CTX.

◆ newEntry()

void newEntry ( yyscan_t yyscanner)
static

Definition at line 1864 of file pyscanner.l.

1865{
1866 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1867 if (yyextra->current->name.empty())
1868 {
1869 initEntry(yyscanner);
1870 return;
1871 }
1872 bool found = false;
1873 if (yyextra->checkDupEntry)
1874 {
1875 for (auto &v : yyextra->current_root->children())
1876 {
1877 //printf("candidate %s<->%s section=%s!\n",qPrint(v->name),qPrint(yyextra->current->name), qPrint(v->section.to_string()));
1878 if (v->name==yyextra->current->name && v->section==yyextra->current->section) // name is already added, don't add it again
1879 {
1880 if (v->doc.empty() && !yyextra->current->doc.empty())
1881 {
1882 v->doc = yyextra->current->doc;
1883 v->docLine = yyextra->current->docLine;
1884 v->docFile = yyextra->current->docFile;
1885 }
1886 if (v->brief.empty() && !yyextra->current->brief.empty())
1887 {
1888 v->brief = yyextra->current->brief;
1889 v->briefLine = yyextra->current->briefLine;
1890 v->briefFile = yyextra->current->briefFile;
1891 }
1892 if (v->type.empty() && !yyextra->current->type.empty())
1893 {
1894 //printf("copying type '%s' from '%s' to '%s'\n",qPrint(yyextra->current->type),qPrint(yyextra->current->name),qPrint(v->name));
1895 v->type = yyextra->current->type;
1896 }
1897 found=true;
1898 break;
1899 }
1900 }
1901 }
1902 if (!found)
1903 {
1904 yyextra->previous = yyextra->current;
1905 yyextra->current_root->moveToSubEntryAndRefresh(yyextra->current);
1906 }
1907 initEntry(yyscanner);
1908}

References initEntry().

Referenced by endOfDef(), handleCommentBlock(), newTypeAlias(), and newVariable().

◆ newFunction()

void newFunction ( yyscan_t yyscanner)
static

Definition at line 1991 of file pyscanner.l.

1992{
1993 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1994 if (yyextra->current->name.startsWith("__") && yyextra->current->name.endsWith("__"))
1995 {
1996 // special method name, see
1997 // http://docs.python.org/ref/specialnames.html
1998 yyextra->current->protection=Protection::Public;
1999 }
2000 else
2001 {
2002 setProtection(yyscanner);
2003 }
2004}

References setProtection().

◆ newTypeAlias()

void newTypeAlias ( yyscan_t yyscanner)
static

Definition at line 1973 of file pyscanner.l.

1974{
1975 //struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1976 //printf("*** newTypeAlias %s\n",qPrint(yyextra->current->name));
1977 newEntry(yyscanner);
1978}

References newEntry().

◆ newVariable()

void newVariable ( yyscan_t yyscanner)
static

Definition at line 1962 of file pyscanner.l.

1963{
1964 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1965 setProtection(yyscanner);
1966 if (yyextra->current_root->section.isCompound()) // mark as class variable
1967 {
1968 yyextra->current->isStatic = true;
1969 }
1970 newEntry(yyscanner);
1971}

References newEntry(), and setProtection().

◆ parseCompounds()

void parseCompounds ( yyscan_t yyscanner,
std::shared_ptr< Entry > rt )
static

Definition at line 2260 of file pyscanner.l.

2261{
2262 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2263 //printf("parseCompounds(%s)\n",qPrint(rt->name));
2264 for (size_t i=0; i<rt->children().size(); ++i)
2265 {
2266 std::shared_ptr<Entry> ce = rt->children()[i];
2267 if (!ce->program.empty())
2268 {
2269 //fprintf(stderr,"parseCompounds: -- %s (line %d) ---------\n%s\n---------------\n",
2270 // qPrint(ce->name), ce->bodyLine, qPrint(ce->program));
2271 // init scanner state
2272 yyextra->programStr = ce->program.str();
2273 yyextra->inputString = yyextra->programStr.data();
2274 yyextra->inputPosition = 0;
2275 yyextra->firstPass = false;
2276 pyscannerYYrestart( nullptr, yyscanner );
2277 if (ce->section.isCompound())
2278 {
2279 yyextra->specialBlock = false;
2280 yyextra->current_root = ce;
2281 BEGIN( Search );
2282 }
2283 else if (ce->parent())
2284 {
2285 yyextra->current_root = rt;
2286 //printf("Searching for member variables in %s parent=%s\n",
2287 // qPrint(ce->name),qPrint(ce->parent->name));
2288 BEGIN( SearchMemVars );
2289 }
2290 yyextra->fileName = ce->fileName;
2291 yyextra->yyLineNr = ce->bodyLine ;
2292 yyextra->current = std::make_shared<Entry>();
2293 initEntry(yyscanner);
2294
2295 yyextra->checkDupEntry = false;
2296
2297 DString name = ce->name;
2298 yyextra->commentScanner.enterCompound(yyextra->fileName,yyextra->yyLineNr,name);
2299
2300 pyscannerYYlex(yyscanner) ;
2301 yyextra->lexInit=true;
2302
2303 yyextra->programStr.clear();
2304 ce->program.str(std::string());
2305
2306 yyextra->commentScanner.leaveCompound(yyextra->fileName,yyextra->yyLineNr,name);
2307
2308 }
2309 parseCompounds(yyscanner,ce);
2310 }
parseCompounds
static void parseCompounds(yyscan_t yyscanner, std::shared_ptr< Entry > rt)
Definition pyscanner.l:2260
2311}

References initEntry(), parseCompounds(), and Search.

Referenced by parseCompounds(), and parseMain().

◆ parseMain()

void parseMain ( yyscan_t yyscanner,
const DString & fileName,
const char * fileBuf,
const std::shared_ptr< Entry > & rt )
static

Definition at line 2316 of file pyscanner.l.

2317{
2318 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2319 initParser(yyscanner);
initParser
static void initParser(yyscan_t yyscanner)
Definition pyscanner.l:1837
2320
2321 if (fileBuf==nullptr || fileBuf[0]=='\0') return;
2322
2323 yyextra->inputString = fileBuf;
2324 yyextra->inputPosition = 0;
2325
2326 yyextra->protection = Protection::Public;
2327 yyextra->mtype = MethodTypes::Method;
2328 yyextra->isStatic = false;
2329 yyextra->virt = Specifier::Normal;
2330 yyextra->current_root = rt;
2331 yyextra->specialBlock = false;
2332
2333 yyextra->yyLineNr = 1 ;
2334 yyextra->fileName = fileName;
2335 yyextra->checkDupEntry = false;
2336 yyextra->firstPass = true;
2337 //setContext();
2338 msg("Parsing file {}...\n",yyextra->fileName);
msg
#define msg(fmt,...)
Definition message.h:94
2339
2340 FileInfo fi(fileName.str());
2341 yyextra->moduleScope = findPackageScope(yyscanner,fileName);
2342 DString baseName=fi.baseName();
2343 if (baseName!="__init__") // package initializer file is not a package itself
2344 {
2345 if (!yyextra->moduleScope.empty())
2346 {
2347 yyextra->moduleScope+="::";
2348 }
2349 yyextra->moduleScope+=baseName;
2350 }
findPackageScope
static DString findPackageScope(yyscan_t yyscanner, const DString &fileName)
Definition pyscanner.l:2048
2351
2352 // add namespaces for each scope
2353 DString scope = yyextra->moduleScope;
2354 size_t startPos = 0;
2355 size_t pos = 0;
2356 do
2357 {
2358 pos = scope.find("::",startPos);
2359 if (pos==DString::npos)
2360 {
2361 startPos = pos = scope.length();
2362 }
2363 else
2364 {
2365 startPos = pos+2;
2366 }
2367 yyextra->current = std::make_shared<Entry>();
2368 initEntry(yyscanner);
2369 yyextra->current->name = scope.left(pos);
2370 yyextra->current->section = EntryType::makeNamespace();
2371 yyextra->current->type = "namespace";
2372 yyextra->current->fileName = yyextra->fileName;
2373 yyextra->current->startLine = yyextra->yyLineNr;
2374 yyextra->current->bodyLine = yyextra->yyLineNr;
2375 yyextra->current_root = yyextra->current;
2376 rt->moveToSubEntryAndRefresh(yyextra->current);
2377 } while (pos<scope.length());
DString::find
size_t find(char c, size_t pos=0) const
Definition dstring.h:239
DString::length
size_t length() const
Returns the length of the string, not counting the 0-terminator.
Definition dstring.h:151
2378
2379 initParser(yyscanner);
2380
2381 yyextra->commentScanner.enterFile(yyextra->fileName,yyextra->yyLineNr);
2382
2383 yyextra->current->reset();
2384 initEntry(yyscanner);
2385 pyscannerYYrestart(nullptr,yyscanner);
2386 BEGIN( Search );
2387 pyscannerYYlex(yyscanner);
2388 yyextra->lexInit=true;
2389
2390 yyextra->commentScanner.leaveFile(yyextra->fileName,yyextra->yyLineNr);
2391
2392 yyextra->programStr.clear();
2393 yyextra->current_root->program.str(std::string());
2394
2395 parseCompounds(yyscanner, yyextra->current_root);
2396}

References FileInfo::baseName(), DString::find(), findPackageScope(), initEntry(), initParser(), DString::left(), DString::length(), msg, DString::npos, parseCompounds(), Search, and DString::str().

Referenced by PythonOutlineParser::parseInput().

◆ parsePrototype()

void parsePrototype ( yyscan_t yyscanner,
const DString & text )
static

Definition at line 2400 of file pyscanner.l.

2401{
2402 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2403 //printf("**** parsePrototype(%s) begin\n",qPrint(text));
2404 if (text.empty())
2405 {
2406 warn(yyextra->fileName,yyextra->yyLineNr,"Empty prototype found!");
2407 return;
2408 }
warn
#define warn(file, line, fmt,...)
Definition message.h:97
2409
2410 yyextra->specialBlock = false;
2411 yyextra->packageCommentAllowed = false;
2412
2413 // save scanner state
2414 YY_BUFFER_STATE orgState = YY_CURRENT_BUFFER;
2415 yy_switch_to_buffer(yy_create_buffer(nullptr, YY_BUF_SIZE, yyscanner), yyscanner);
2416 const char *orgInputString = yyextra->inputString;
2417 int orgInputPosition = yyextra->inputPosition;
YY_BUF_SIZE
#define YY_BUF_SIZE
Definition commentcnv.l:19
2418
2419 // set new string
2420 yyextra->inputString = text.data();
2421 yyextra->inputPosition = 0;
2422 pyscannerYYrestart( nullptr, yyscanner );
DString::data
const char * data() const
Returns a pointer to the contents of the string in the form of a 0-terminated C string.
Definition dstring.h:157
2423
2424 BEGIN( FunctionDec );
2425
2426 pyscannerYYlex(yyscanner);
2427 yyextra->lexInit=true;
2428
2429 yyextra->current->name = yyextra->current->name.stripWhiteSpace();
2430 if (yyextra->current->section.isMemberDoc() && yyextra->current->args.empty())
2431 {
2432 yyextra->current->section = EntryType::makeVariableDoc();
2433 }
2434
2435 // restore original scanner state
2436
2437 yy_delete_buffer(YY_CURRENT_BUFFER, yyscanner);
2438 yy_switch_to_buffer(orgState, yyscanner);
2439
2440 yyextra->inputString = orgInputString;
2441 yyextra->inputPosition = orgInputPosition;
2442
2443 //printf("**** parsePrototype end\n");
2444}

References DString::data(), DString::empty(), warn, and YY_BUF_SIZE.

◆ searchFoundClass()

void searchFoundClass ( yyscan_t yyscanner)
static

Definition at line 2233 of file pyscanner.l.

2234{
2235 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2236 yyextra->current->section = EntryType::makeClass();
2237 yyextra->current->argList.clear();
2238 yyextra->current->type += "class" ;
2239 yyextra->current->fileName = yyextra->fileName;
2240 yyextra->current->startLine = yyextra->yyLineNr;
2241 yyextra->current->bodyLine = yyextra->yyLineNr;
2242 yyextra->packageCommentAllowed = false;
2243}

◆ searchFoundDef()

void searchFoundDef ( yyscan_t yyscanner)
static

Definition at line 2213 of file pyscanner.l.

2214{
2215 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2216 yyextra->current->fileName = yyextra->fileName;
2217 yyextra->current->startLine = yyextra->yyLineNr;
2218 yyextra->current->bodyLine = yyextra->yyLineNr;
2219 yyextra->current->section = EntryType::makeFunction();
2220 yyextra->current->lang = SrcLangExt::Python;
2221 yyextra->current->virt = Specifier::Normal;
2222 yyextra->current->isStatic = yyextra->isStatic;
2223 yyextra->current->mtype = yyextra->mtype = MethodTypes::Method;
2224 yyextra->current->type.clear();
2225 yyextra->current->name.clear();
2226 yyextra->current->args.clear();
2227 yyextra->current->argList.clear();
2228 yyextra->packageCommentAllowed = false;
2229 yyextra->isStatic=false;
2230 //printf("searchFoundDef at=%d\n",yyextra->yyLineNr);
2231}

◆ searchFoundTypeAlias()

void searchFoundTypeAlias ( yyscan_t yyscanner)
static

Definition at line 2245 of file pyscanner.l.

2246{
2247 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2248 yyextra->current->section = EntryType::makeVariable();
2249 yyextra->current->argList.clear();
2250 yyextra->current->type = "typedef " ;
2251 yyextra->current->spec.setAlias(true);
2252 yyextra->current->fileName = yyextra->fileName;
2253 yyextra->current->startLine = yyextra->yyLineNr;
2254 yyextra->current->bodyLine = yyextra->yyLineNr;
2255 yyextra->packageCommentAllowed = false;
2256}

◆ setProtection()

void setProtection ( yyscan_t yyscanner)
static

Definition at line 1933 of file pyscanner.l.

1934{
1935 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1936 if (!yyextra->current->name.empty() && yyextra->current->name.at(0)=='_')
1937 {
1938 if (yyextra->current->name.at(1)=='_') // mark as private
1939 {
1940 yyextra->current->protection=Protection::Private;
1941 }
1942 else // mark as protected
1943 {
1944 yyextra->current->protection=Protection::Protected;
1945 }
1946 }
1947}

Referenced by addVariable(), docVariable(), newFunction(), and newVariable().

◆ startCommentBlock()

void startCommentBlock ( yyscan_t yyscanner,
bool brief )
static

Definition at line 2086 of file pyscanner.l.

2087{
2088 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2089 if (brief)
2090 {
2091 yyextra->current->briefFile = yyextra->fileName;
2092 yyextra->current->briefLine = yyextra->yyLineNr;
2093 }
2094 else
2095 {
2096 yyextra->current->docFile = yyextra->fileName;
2097 yyextra->current->docLine = yyextra->yyLineNr;
2098 }
2099}

◆ stateToString()

const char * stateToString ( int state)
static

◆ yylex()

int yylex ( yyscan_t yyscanner)

Definition at line 278 of file pyscanner.l.

282 {
283
284 ^{B}"def"{BB} { // start of a function/method definition with indent
285 DBG_CTX((stderr,"Found def at %d\n",yyextra->yyLineNr));
286 yyextra->indent=computeIndent(yytext);
287 searchFoundDef(yyscanner);
288 BEGIN( FunctionDec );
289 }
290 ^{B}"async"{BB}"def"{BB} { // start of an async function/method definition with indent
291 DBG_CTX((stderr,"Found async def at %d\n",yyextra->yyLineNr));
292 yyextra->indent=computeIndent(yytext);
293 searchFoundDef(yyscanner);
294 BEGIN( FunctionDec );
295 }
296 "def"{BB} { // start of a function/method definition
297 searchFoundDef(yyscanner);
298 BEGIN( FunctionDec );
299 }
300 "async"{BB}"def"{BB} { // start of a function/method definition
301 searchFoundDef(yyscanner);
302 BEGIN( FunctionDec );
303 }
computeIndent
static int computeIndent(const char *s)
Definition pyscanner.l:2006
searchFoundDef
static void searchFoundDef(yyscan_t yyscanner)
Definition pyscanner.l:2213
304
305 ^{B}"class"{BB} { // start of a class definition with indent
306 DBG_CTX((stderr,"Found class at %d\n",yyextra->yyLineNr));
307 yyextra->indent=computeIndent(yytext);
308 searchFoundClass(yyscanner);
309 BEGIN( ClassDec ) ;
310 }
311 "class"{BB} { // start of a class definition
312 searchFoundClass(yyscanner);
313 BEGIN( ClassDec ) ;
314 }
315 ^{B}"type"{BB} {
316 yyextra->indent=computeIndent(yytext);
317 searchFoundTypeAlias(yyscanner);
318 BEGIN( TypeAlias );
319 }
320 "type"{BB} {
321 searchFoundTypeAlias(yyscanner);
322 BEGIN( TypeAlias );
323 }
324 ^{B}"from"{BB} |
325 "from"{BB} { // start of an from import
326 yyextra->packageCommentAllowed = false;
327 BEGIN( FromMod );
328 }
searchFoundTypeAlias
static void searchFoundTypeAlias(yyscan_t yyscanner)
Definition pyscanner.l:2245
searchFoundClass
static void searchFoundClass(yyscan_t yyscanner)
Definition pyscanner.l:2233
329
330 ^{B}"import"{BB} |
331 "import"{BB} { // start of an import statement
332 yyextra->packageCommentAllowed = false;
333 BEGIN( Import );
334 }
335 ^{B}{IDENTIFIER}/{B}"="{B}"property" { // property
336 yyextra->current->section = EntryType::makeVariable();
337 yyextra->current->mtype = MethodTypes::Property;
338 yyextra->current->name = DString(yytext).stripWhiteSpace();
339 yyextra->current->fileName = yyextra->fileName;
340 yyextra->current->startLine = yyextra->yyLineNr;
341 yyextra->current->bodyLine = yyextra->yyLineNr;
342 yyextra->packageCommentAllowed = false;
343 BEGIN(VariableDec);
344 }
345 ^{B}{IDENTIFIER}/{B}"="[^=] { // variable
346 if (yyextra->searchCount>0) REJECT;
347 yyextra->indent=computeIndent(yytext);
348 yyextra->current->section = EntryType::makeVariable();
349 yyextra->current->name = DString(yytext).stripWhiteSpace();
350 yyextra->current->fileName = yyextra->fileName;
351 yyextra->current->startLine = yyextra->yyLineNr;
352 yyextra->current->bodyLine = yyextra->yyLineNr;
353 yyextra->packageCommentAllowed = false;
354 BEGIN(VariableDec);
355 }
356 ^{B}{IDENTIFIER}/{B}":" { // variable
357 if (yyextra->searchCount>0) REJECT;
358 DString id = DString(yytext).stripWhiteSpace();
359 if (id =="try" || id == "else" || id == "except" || id == "finally") REJECT;
360 yyextra->indent=computeIndent(yytext);
361 yyextra->current->section = EntryType::makeVariable();
362 yyextra->current->name = id;
363 yyextra->current->fileName = yyextra->fileName;
364 yyextra->current->startLine = yyextra->yyLineNr;
365 yyextra->current->bodyLine = yyextra->yyLineNr;
366 yyextra->packageCommentAllowed = false;
367 BEGIN(VariableDec);
368 }
369 {B}{IDENTIFIER}/({B},{B}{IDENTIFIER})*{B}")"*{B}"="[^=] { // list of variables, we cannot place the default value
370 // so we will skip it later on in a general rule
371 // Also note ")" this is to catch also (a,b). the "("
372 // is caught in the rule: [(], the ")" will be handled in [)]
373 if (yyextra->searchCount>1) REJECT;
374 yyextra->indent=computeIndent(yytext);
375 yyextra->current->section = EntryType::makeVariable();
376 yyextra->current->name = DString(yytext).stripWhiteSpace();
377 yyextra->current->fileName = yyextra->fileName;
378 yyextra->current->startLine = yyextra->yyLineNr;
379 yyextra->current->bodyLine = yyextra->yyLineNr;
380 yyextra->packageCommentAllowed = false;
381 addVariable(yyscanner);
382 }
383 "'" { // start of a single quoted string
384 yyextra->stringContext=YY_START;
385 yyextra->copyString=nullptr;
386 yyextra->packageCommentAllowed = false;
387 BEGIN( SingleQuoteString );
388 }
389 "\"" { // start of a double quoted string
390 yyextra->stringContext=YY_START;
391 yyextra->copyString=nullptr;
392 yyextra->packageCommentAllowed = false;
393 BEGIN( DoubleQuoteString );
394 }
395 "@staticmethod" {
396 yyextra->isStatic=true;
397 }
398 "@"{SCOPE}"(" { // decorator
399 lineCount(yyscanner);
400 yyextra->decoratorRound = 1;
401 yyextra->copyString=nullptr;
402 BEGIN( Decorator );
403 }
404 "@"{SCOPE} { // decorator
405 lineCount(yyscanner);
406 }
407 {SCRIPTCOMMENT} { // Unix type script comment
408 if (yyextra->yyLineNr != 1) REJECT;
409 }
410 {POUNDCOMMENT} { // normal comment
411 // issue 9672
412 //yyextra->packageCommentAllowed = false;
413 }
414 {IDENTIFIER} { // some other identifier
415 yyextra->packageCommentAllowed = false;
416 }
417 ^{BB} {
418 yyextra->curIndent=computeIndent(yytext);
419 }
DString::stripWhiteSpace
DString stripWhiteSpace() const
returns a copy of this string with leading and trailing whitespace removed
Definition dstring.h:337
lineCount
#define lineCount(s, len)
Definition doctokenizer.l:98
addVariable
static void addVariable(yyscan_t yyscanner)
Definition pyscanner.l:1980
420
421 {NEWLINE}+ { // new line
422 lineCount(yyscanner);
423 }
424
425 {TRIDOUBLEQUOTE} { // start of a comment block
426 initTriDoubleQuoteBlock(yyscanner);
427 BEGIN(TripleComment);
428 }
initTriDoubleQuoteBlock
static void initTriDoubleQuoteBlock(yyscan_t yyscanner)
Definition pyscanner.l:2175
429
430 {TRISINGLEQUOTE} { // start of a comment block
431 initTriSingleQuoteBlock(yyscanner);
432 BEGIN(TripleComment);
433 }
initTriSingleQuoteBlock
static void initTriSingleQuoteBlock(yyscan_t yyscanner)
Definition pyscanner.l:2188
434
435 {B}{STARTDOCSYMS}/[^#] { // start of a special comment
436 yyextra->curIndent=computeIndent(yytext);
437 yyextra->packageCommentAllowed = false;
438 initSpecialBlock(yyscanner);
439 BEGIN(SpecialComment);
440 }
441 [(] { // we have to do something with (
442 yyextra->searchCount++;
443 }
444 [)] { // we have to do something with )
445 if (yyextra->searchCount>0)
446 {
447 yyextra->searchCount--;
448 }
449 }
450 "=" {
451 yyextra->current->doc.clear();
452 yyextra->current->brief.clear();
453 }
454 {IDENTIFIER} {
455 }
456 [^\n] { // any other character...
457 // This is the major default
458 // that should catch everything
459 // else in Body.
460 }
initSpecialBlock
static void initSpecialBlock(yyscan_t yyscanner)
Definition pyscanner.l:2201
461}
462
463<FromMod>{
464 "." { // python3 style imports
465 }
466 {IDENTIFIER}({B}"."{B}{IDENTIFIER})* { // from package import
467 yyextra->packageName=yytext;
468 }
469 "import"{B} {
470 BEGIN(FromModItem);
471 }
472 \n {
473 incLineNr(yyscanner);
474 BEGIN(Search);
475 }
476 {B} {
477 }
478 . {
479 unput(*yytext);
480 BEGIN(Search);
481 }
incLineNr
static void incLineNr(yyscan_t yyscanner)
Definition pyscanner.l:2078
482}
483
484<FromModItem>{
485 "*" { // import all
486 addFrom(yyscanner,true);
487 BEGIN(Search);
488 }
489 {IDENTIFIER}/{B}","{B} {
490 addFrom(yyscanner,false);
491 }
492 {IDENTIFIER}/{B}")" {
493 addFrom(yyscanner,false);
494 }
495 {IDENTIFIER} {
496 addFrom(yyscanner,false);
497 if (!yyextra->importTuple)
498 {
499 BEGIN(Search);
500 }
501 }
502 \n {
503 incLineNr(yyscanner);
504 if (!yyextra->importTuple)
505 {
506 BEGIN(Search);
507 }
508 }
509 {B} {
510 }
511 "(" {
512 yyextra->importTuple=true;
513 }
514 ")" {
515 yyextra->importTuple=false;
516 BEGIN(Search);
517 }
518 "," {
519 }
520 "\\"{B}\n { // line continuation
521 incLineNr(yyscanner);
522 }
523 . {
524 unput(*yytext);
525 BEGIN(Search);
526 }
addFrom
static void addFrom(yyscan_t yyscanner, bool all)
Definition pyscanner.l:2055
527}
528
529<Import>{
530 {IDENTIFIER}({B}"."{B}{IDENTIFIER})* {
531 yyextra->current->name=removeRedundantWhiteSpace(substitute(yytext,".","::"));
532 yyextra->current->fileName = yyextra->fileName;
533 //printf("Adding using declaration: found:%s:%d name=%s\n",qPrint(yyextra->fileName),yyextra->yyLineNr,qPrint(yyextra->current->name));
534 yyextra->current->section=EntryType::makeUsingDecl();
535 yyextra->current_root->moveToSubEntryAndRefresh(yyextra->current);
536 initEntry(yyscanner);
537 BEGIN(Search);
538 }
539 \n {
540 incLineNr(yyscanner);
541 BEGIN(Search);
542 }
543 {B} {
544 }
545 . {
546 unput(*yytext);
547 BEGIN(Search);
548 }
549}
550
551<SearchMemVars>{
552 ("cls"|"self")"."{IDENTIFIER}/{B}[,)] {
553 const char *s = strchr(yytext,'.'); s++;
554 DBG_CTX((stderr,"Found instance method variable %s in %s at %d\n",s,qPrint(yyextra->current_root->name.data(),yyextra->yyLineNr)));
555 docVariable(yyscanner,s);
556 addEntry(yyscanner);
557 }
558 ("cls"|"self")"."{IDENTIFIER}/{B}"=" {
559 const char *s = strchr(yytext,'.'); s++;
560 DBG_CTX((stderr,"Found instance method variable %s in %s at %d\n",s,qPrint(yyextra->current_root->name.data(),yyextra->yyLineNr)));
561 docVariable(yyscanner,s);
562 BEGIN( SearchSkipValue );
563 }
564 ("cls"|"self")"."{IDENTIFIER}/{B}":" { // type hint
565 const char *s = strchr(yytext,'.'); s++;
566 DBG_CTX((stderr,"Found instance method variable %s in %s at %d\n",s,qPrint(yyextra->current_root->name.data(),yyextra->yyLineNr)));
567 docVariable(yyscanner,s);
568 BEGIN(TypeHint);
569 }
570 {TRIDOUBLEQUOTE} { // start of a comment block
571 initTriDoubleQuoteBlock(yyscanner);
572 BEGIN(TripleComment);
573 }
qPrint
const char * qPrint(const char *s)
Definition dstring.h:783
docVariable
static void docVariable(yyscan_t yyscanner, const char *s)
Definition pyscanner.l:1949
574
575 {TRISINGLEQUOTE} { // start of a comment block
576 initTriSingleQuoteBlock(yyscanner);
577 BEGIN(TripleComment);
578 }
579
580 {STARTDOCSYMS}/[^#] { // start of a special comment
581 initSpecialBlock(yyscanner);
582 BEGIN(SpecialComment);
583 }
584 {POUNDCOMMENT} { // #
585 }
586 "'" { // start of a single quoted string
587 yyextra->stringContext=YY_START;
588 yyextra->copyString=nullptr;
589 BEGIN( SingleQuoteString );
590 }
591 "\"" { // start of a double quoted string
592 yyextra->stringContext=YY_START;
593 yyextra->copyString=nullptr;
594 BEGIN( DoubleQuoteString );
595 }
596 \n { incLineNr(yyscanner); }
597 "=" {
598 yyextra->current->doc.clear();
599 yyextra->current->brief.clear();
600 unput(*yytext);
601 BEGIN( SearchSkipValue );
602 }
603 {IDENTIFIER} // identifiers
604 [^'"\.#a-z_A-Z=\n]+ // other uninteresting stuff
605 . // anything else
606}
607
608<TypeHint>{
609 ":" { // skip over start of type hint
610 yyextra->braceCount=0;
611 }
612 "("|"["|"{" {
613 yyextra->current->type+=*yytext;
614 yyextra->braceCount++;
615 }
616 ")"|"]"|"}" {
617 yyextra->current->type+=*yytext;
618 yyextra->braceCount--;
619 }
620 "'" { // start of a single quoted string
621 yyextra->stringContext=YY_START;
622 yyextra->copyString=nullptr;
623 BEGIN( SingleQuoteString );
624 }
625 "\"" { // start of a double quoted string
626 yyextra->stringContext=YY_START;
627 yyextra->copyString=nullptr;
628 BEGIN( DoubleQuoteString );
629 }
630 "=" {
631 if (yyextra->braceCount==0)
632 { // end of the type hint
633 yyextra->current->type = yyextra->current->type.stripWhiteSpace();
634 unput(*yytext);
635 BEGIN(SearchSkipValue);
636 }
637 else
638 {
639 yyextra->current->type+=*yytext;
640 }
641 }
642 \n { // end of the type hint
643 yyextra->current->type = yyextra->current->type.stripWhiteSpace();
644 incLineNr(yyscanner);
645 newEntry(yyscanner);
646 BEGIN(SearchMemVars);
647 }
648 "\\\n" {
649 yyextra->current->type+=' ';
650 incLineNr(yyscanner);
651 }
652 . {
653 yyextra->current->type+=*yytext;
654 }
655}
656
657<SearchSkipValue,VariableDec,TypeValue>{
658 "=" { // the assignment operator
659 //printf("====== VariableDec at line %d\n",yyextra->yyLineNr);
660 yyextra->startInit = true;
661 if (YY_START!=TypeValue)
662 {
663 yyextra->current->initializer.str(yytext);
664 yyextra->current->initializer << " ";
665 }
666 }
667 {B} { // spaces
668 yyextra->current->initializer << yytext;
669 }
670
671 {INTNUMBER} { // integer value
672 if (yyextra->current->type.empty()) yyextra->current->type = "int";
673 yyextra->current->initializer << yytext;
674 }
675 {FLOATNUMBER} { // floating point value
676 if (yyextra->current->type.empty()) yyextra->current->type = "float";
677 yyextra->current->initializer << yytext;
678 }
679 {BOOL} { // boolean value
680 if (yyextra->current->type.empty()) yyextra->current->type = "bool";
681 yyextra->current->initializer << yytext;
682 }
683 {STRINGPREFIX}?"'" { // string
684 if (yyextra->current->type.empty()) yyextra->current->type = "str";
685 yyextra->current->initializer << yytext;
686 yyextra->copyString=&yyextra->current->initializer;
687 yyextra->stringContext=YY_START;
688 BEGIN( SingleQuoteString );
689 }
690 {STRINGPREFIX}?"\"" { // string
691 if (yyextra->current->type.empty()) yyextra->current->type = "str";
692 yyextra->current->initializer << yytext;
693 yyextra->copyString=&yyextra->current->initializer;
694 yyextra->stringContext=YY_START;
695 BEGIN( DoubleQuoteString );
696 }
697 {TRIDOUBLEQUOTE} { // start of a comment block
698 if (yyextra->current->type.empty()) yyextra->current->type = "str";
699 yyextra->current->initializer << yytext;
700 yyextra->doubleQuote=true;
701 yyextra->copyString=&yyextra->current->initializer;
702 yyextra->stringContext=YY_START;
703 BEGIN(TripleString);
704 }
705
706 {TRISINGLEQUOTE} { // start of a comment block
707 if (yyextra->current->type.empty()) yyextra->current->type = "str";
708 yyextra->current->initializer << yytext;
709 yyextra->doubleQuote=false;
710 yyextra->copyString=&yyextra->current->initializer;
711 yyextra->stringContext=YY_START;
712 BEGIN(TripleString);
713 }
714 "(" { // tuple, only when direct after =
715 if (yyextra->current->mtype!=MethodTypes::Property && yyextra->startInit)
716 {
717 yyextra->current->type = "tuple";
718 }
719 yyextra->current->initializer << *yytext;
720 yyextra->atomStart='(';
721 yyextra->atomEnd=')';
722 yyextra->atomCount=1;
723 yyextra->atomContext=YY_START;
724 BEGIN( VariableAtom );
725 }
726 "[" { // list
727 if (yyextra->startInit) yyextra->current->type = "list";
728 yyextra->current->initializer << *yytext;
729 yyextra->atomStart='[';
730 yyextra->atomEnd=']';
731 yyextra->atomCount=1;
732 yyextra->atomContext=YY_START;
733 BEGIN( VariableAtom );
734 }
735 "{" { // dictionary
736 if (yyextra->startInit) yyextra->current->type = "dict";
737 yyextra->current->initializer << *yytext;
738 yyextra->atomStart='{';
739 yyextra->atomEnd='}';
740 yyextra->atomCount=1;
741 yyextra->atomContext=YY_START;
742 BEGIN( VariableAtom );
743 }
744 "\\\n" {
745 yyextra->current->initializer << yytext;
746 incLineNr(yyscanner);
747 }
748 {IDENTIFIER} {
749 // do something based on the type of the IDENTIFIER
750 if (yyextra->current->type.empty())
751 {
752 for (const auto &child : yyextra->current_root->children())
753 {
754 if (child->name == yytext)
755 {
756 yyextra->current->type = child->type;
757 break;
758 }
759 }
760 }
761 yyextra->startInit = false;
762 yyextra->current->initializer << yytext;
763 }
764 . {
765 yyextra->startInit = false;
766 yyextra->current->initializer << *yytext;
767 }
768}
769<SearchSkipValue>{
770 {STARTDOCSYMS}/[^#] { // start of a special comment
771 initSpecialBlock(yyscanner);
772 BEGIN(SpecialComment);
773 }
774 {POUNDCOMMENT} { // #
775 }
776 \n { incLineNr(yyscanner);
777 newEntry(yyscanner);
778 BEGIN(SearchMemVars);
779 }
780 <<EOF>> { incLineNr(yyscanner);
781 newEntry(yyscanner);
782 BEGIN(SearchMemVars);
783 }
784}
785
786<FunctionBody>{
787 \n{B}/{IDENTIFIER}[^{LETTER}{DIGIT}_] {
788 DBG_CTX((stderr,"indent %d<=%d\n",computeIndent(&yytext[1]),yyextra->indent));
789 if (computeIndent(&yytext[1])<=yyextra->indent)
790 {
791 unput_string(yytext,yyleng);
792 endOfDef(yyscanner);
793 //YY_CURRENT_BUFFER->yy_at_bol=true;
794 BEGIN(Search);
795 }
796 else
797 {
798 incLineNr(yyscanner);
799 yyextra->current->program << yytext;
800 }
801 }
802 \n{B}/"##" {
803 if (computeIndent(&yytext[1])<=yyextra->indent)
804 {
805 unput_string(yytext,yyleng);
806 endOfDef(yyscanner);
807 //YY_CURRENT_BUFFER->yy_at_bol=true;
808 BEGIN(Search);
809 }
810 else
811 {
812 incLineNr(yyscanner);
813 yyextra->current->program << yytext;
814 }
815 }
816 <<EOF>> {
817 endOfDef(yyscanner);
818 yyterminate();
819 }
820 ^{BB}/\n { // skip empty line
821 yyextra->current->program << yytext;
822 }
823 ^{BB} { // something at indent >0
824 yyextra->curIndent = computeIndent(yytext);
825 if (yyextra->curIndent<=yyextra->indent)
826 // jumped out of the function
827 {
828 endOfDef(yyscanner,1);
829 BEGIN(Search);
830 }
831 else
832 {
833 yyextra->current->program << yytext;
834 }
835 }
836 "'" { // start of a single quoted string
837 yyextra->current->program << yytext;
838 yyextra->stringContext=YY_START;
839 yyextra->specialBlock = false;
840 yyextra->copyString=&yyextra->current->program;
841 BEGIN( SingleQuoteString );
842 }
843 "\"" { // start of a double quoted string
844 yyextra->current->program << yytext;
845 yyextra->stringContext=YY_START;
846 yyextra->specialBlock = false;
847 yyextra->copyString=&yyextra->current->program;
848 BEGIN( DoubleQuoteString );
849 }
850 [^ \t\n#'".]+ { // non-special stuff
851 yyextra->current->program << yytext;
852 yyextra->specialBlock = false;
853 }
854 ^{POUNDCOMMENT} { // normal comment
855 yyextra->current->program << yytext;
856 }
857 "#".* { // comment half way
858 yyextra->current->program << yytext;
859 }
860 {NEWLINE} {
861 incLineNr(yyscanner);
862 yyextra->current->program << yytext;
863 }
864 . { // any character
865 yyextra->current->program << *yytext;
866 yyextra->specialBlock = false;
867 }
yyterminate
#define yyterminate()
Definition doctokenizer.l:174
vhdl::parser::DIGIT
const int DIGIT
RegularExpression Id.
Definition VhdlParserConstants.h:378
vhdl::parser::LETTER
const int LETTER
RegularExpression Id.
Definition VhdlParserConstants.h:368
endOfDef
static void endOfDef(yyscan_t yyscanner, int correction=0)
Definition pyscanner.l:2156
unput_string
#define unput_string(yytext, yyleng)
Definition pyscanner.l:65
868
869 {TRIDOUBLEQUOTE} { // start of a comment block
870 yyextra->current->program << yytext;
871 initTriDoubleQuoteBlock(yyscanner);
872 BEGIN(TripleComment);
873 }
874
875 {TRISINGLEQUOTE} { // start of a comment block
876 yyextra->current->program << yytext;
877 initTriSingleQuoteBlock(yyscanner);
878 BEGIN(TripleComment);
879 }
880
881}
882
883<FunctionDec>{
884 {IDENTIFIER} {
885 //found function name
886 yyextra->current->name = yytext;
887 yyextra->current->name = yyextra->current->name.stripWhiteSpace();
888 newFunction(yyscanner);
889 }
890 {B}":"{B} { // function without arguments
891 yyextra->specialBlock = true; // expecting a docstring
892 yyextra->bodyEntry = yyextra->current;
893 yyextra->current->bodyLine = yyextra->yyLineNr;
894 BEGIN(FunctionBody);
895 }
896 "[" { // python 3.12+ style generics
897 yyextra->genericsContext = YY_START;
898 yyextra->bracketDepth = 1;
899 yyextra->genericsValue = "[";
900 BEGIN(Generics);
901 }
902 "->" {
903 yyextra->defVal.str(std::string());
904 yyextra->braceCount = 0;
905 BEGIN(FunctionTypeAnnotation);
906 }
907 {B}"(" {
908 yyextra->funcParamsEnd = false;
909 yyextra->current->bodyLine = yyextra->yyLineNr;
910 BEGIN(FunctionParams);
911 }
912 ")" { // end of parameter list
913 if (yyextra->current->argList.empty())
914 {
915 yyextra->current->argList.setNoParameters(true);
916 }
917 yyextra->current->args = argListToString(yyextra->current->argList);
918 yyextra->funcParamsEnd = true;
919 }
newFunction
static void newFunction(yyscan_t yyscanner)
Definition pyscanner.l:1991
argListToString
DString argListToString(const ArgumentList &al, bool useCanonicalType, bool showDefVals)
Definition util.cpp:859
920}
921
922<Generics>{
923 {IDENTIFIER} {
924 yyextra->genericsValue+=yytext;
925 }
926 "[" {
927 yyextra->genericsValue+=yytext;
928 yyextra->bracketDepth++;
929 }
930 "]" {
931 yyextra->genericsValue+=yytext;
932 yyextra->bracketDepth--;
933 if (yyextra->bracketDepth==0)
934 {
935 BEGIN(yyextra->genericsContext);
936 }
937 }
938 . {
939 yyextra->genericsValue+=yytext;
940 }
941}
942
943<FunctionParams>{
944 {BB} {
945 }
946
947 "," {
948 if (!yyextra->argType.empty())
949 {
950 Argument a;
951 a.name = "";
952 a.type = yyextra->argType;
953 yyextra->current->argList.push_back(a);
954 yyextra->argType = "";
955 }
956 }
Argument
This class contains the information about the argument of a function or template.
Definition arguments.h:27
Argument::name
DString name
Definition arguments.h:45
Argument::type
DString type
Definition arguments.h:43
DString::push_back
void push_back(char c)
Definition dstring.h:202
957
958 [\*]+ {
959 yyextra->argType = yytext;
960 }
961 {IDENTIFIER} { // Name of parameter
962 lineCount(yyscanner);
963 Argument a;
964 a.name = DString(yytext).stripWhiteSpace();
965 a.type = yyextra->argType;
966 yyextra->current->argList.push_back(a);
967 yyextra->argType = "";
968 }
969 "=" { // default value
970 // TODO: this rule is too simple, need to be able to
971 // match things like =")" as well!
972 yyextra->defVal.str(std::string());
973 yyextra->braceCount = 0;
974 BEGIN(FunctionParamDefVal);
975 }
976 ")" {
977 if (!yyextra->argType.empty())
978 {
979 Argument a;
980 a.name = "";
981 a.type = yyextra->argType;
982 yyextra->current->argList.push_back(a);
983 yyextra->argType = "";
984 }
985 unput(*yytext);
986 BEGIN(FunctionDec);
987 }
988 ":"{B} {
989 yyextra->defVal.str(std::string());
990 yyextra->braceCount = 0;
991 BEGIN(FunctionAnnotation);
992 }
993 {POUNDCOMMENT} { // a comment
994 }
995 {PARAMNONEMPTY} { // Default rule inside arguments.
996 }
997
998}
999
1000<FunctionTypeAnnotation>{
1001 "{" |
1002 "[" |
1003 "(" {
1004 ++yyextra->braceCount;
1005 yyextra->defVal << *yytext;
1006 }
1007 "}" |
1008 "]" |
1009 ")" {
1010 --yyextra->braceCount;
1011 yyextra->defVal << *yytext;
1012 }
1013 ":" {
1014 if (yyextra->braceCount == 0)
1015 {
1016 yyextra->current->type = yyextra->defVal.str();
1017 unput(*yytext);
1018 BEGIN(FunctionDec);
1019 }
1020 else
1021 yyextra->defVal << *yytext;
1022 }
1023 "'" {
1024 yyextra->defVal << *yytext;
1025 yyextra->copyString=&yyextra->defVal;
1026 yyextra->stringContext=FunctionTypeAnnotation;
1027 BEGIN(SingleQuoteString);
1028 }
1029 "\"" {
1030 yyextra->defVal << *yytext;
1031 yyextra->copyString=&yyextra->defVal;
1032 yyextra->stringContext=FunctionTypeAnnotation;
1033 BEGIN(DoubleQuoteString);
1034 }
1035 \n {
1036 yyextra->defVal << *yytext;
1037 incLineNr(yyscanner);
1038 }
1039 . {
1040 yyextra->defVal << *yytext;
1041 }
1042}
1043
1044<FunctionAnnotation>{
1045 "{" |
1046 "[" |
1047 "(" {
1048 ++yyextra->braceCount;
1049 yyextra->defVal << *yytext;
1050 }
1051 "}" |
1052 "]" {
1053 --yyextra->braceCount;
1054 yyextra->defVal << *yytext;
1055 }
1056 ")" |
1057 "=" |
1058 "," {
1059 if (yyextra->braceCount == 0)
1060 {
1061 if (!yyextra->current->argList.empty())
1062 yyextra->current->argList.back().type += yyextra->defVal.str();
1063 if (*yytext != ',')
1064 unput(*yytext);
1065 BEGIN(FunctionParams);
1066 }
1067 else
1068 {
1069 if (*yytext == ')')
1070 --yyextra->braceCount;
1071 yyextra->defVal << *yytext;
1072 }
1073 }
1074 "'" {
1075 yyextra->defVal << *yytext;
1076 yyextra->copyString=&yyextra->defVal;
1077 yyextra->stringContext=FunctionAnnotation;
1078 BEGIN(SingleQuoteString);
1079 }
1080 "\"" {
1081 yyextra->defVal << *yytext;
1082 yyextra->copyString=&yyextra->defVal;
1083 yyextra->stringContext=FunctionAnnotation;
1084 BEGIN(DoubleQuoteString);
1085 }
1086 \n {
1087 yyextra->defVal << *yytext;
1088 incLineNr(yyscanner);
1089 }
1090 . {
1091 yyextra->defVal << *yytext;
1092 }
1093}
1094
1095<FunctionParamDefVal>{
1096 "{" |
1097 "[" |
1098 "(" { // internal opening brace, assumption is that we have correct code so braces do match
1099 ++yyextra->braceCount;
1100 yyextra->defVal << *yytext;
1101 }
1102 "}" |
1103 "]" {
1104 --yyextra->braceCount;
1105 yyextra->defVal << *yytext;
1106 }
1107 ")" |
1108 "," {
1109 if (yyextra->braceCount == 0)
1110 {
1111 if (!yyextra->current->argList.empty())
1112 yyextra->current->argList.back().defval=DString(yyextra->defVal.str()).stripWhiteSpace();
1113 if (*yytext == ')')
1114 unput(*yytext);
1115 BEGIN(FunctionParams);
1116 }
1117 else
1118 {
1119 if (*yytext == ')')
1120 --yyextra->braceCount;
1121 yyextra->defVal << *yytext;
1122 }
1123 }
1124
1125 "'" {
1126 yyextra->defVal << *yytext;
1127 yyextra->copyString=&yyextra->defVal;
1128 yyextra->stringContext=FunctionParamDefVal;
1129 BEGIN( SingleQuoteString );
1130 }
1131 "\"" {
1132 yyextra->defVal << *yytext;
1133 yyextra->copyString=&yyextra->defVal;
1134 yyextra->stringContext=FunctionParamDefVal;
1135 BEGIN( DoubleQuoteString );
1136 }
1137 \n {
1138 yyextra->defVal << *yytext;
1139 incLineNr(yyscanner);
1140 }
1141 . {
1142 yyextra->defVal << *yytext;
1143 }
1144}
1145
1146
1147<ClassBody>{
1148 \n/{IDENTIFIER}{BB} { // new def at indent 0
1149 if (computeIndent(&yytext[1])<=yyextra->indent)
1150 {
1151 int i;
1152 for (i=(int)yyleng-1;i>=0;i--)
1153 {
1154 unput(yytext[i]);
1155 }
1156 endOfDef(yyscanner);
1157 //YY_CURRENT_BUFFER->yy_at_bol=true;
1158 BEGIN(Search);
1159 }
1160 else
1161 {
1162 incLineNr(yyscanner);
1163 yyextra->current->program << yytext;
1164 }
1165 }
1166 \n/"##"[^#] { // start of a special comment at indent 0
1167 if (computeIndent(&yytext[1])<=yyextra->indent)
1168 {
1169 int i;
1170 for (i=(int)yyleng-1;i>=0;i--)
1171 {
1172 unput(yytext[i]);
1173 }
1174 endOfDef(yyscanner);
1175 //YY_CURRENT_BUFFER->yy_at_bol=true;
1176 BEGIN(Search);
1177 }
1178 else
1179 {
1180 incLineNr(yyscanner);
1181 yyextra->current->program << yytext;
1182 }
1183 }
1184 ^{BB}/\n { // skip empty line
1185 yyextra->current->program << yytext;
1186 }
1187 <<EOF>> {
1188 endOfDef(yyscanner);
1189 yyterminate();
1190 }
1191 ^{BB} { // something at indent >0
1192 yyextra->curIndent=computeIndent(yytext);
1193 DBG_CTX((stderr,"yyextra->curIndent=%d yyextra->indent=%d\n",yyextra->curIndent,yyextra->indent));
1194 if (yyextra->curIndent<=yyextra->indent)
1195 // jumped out of the class/method
1196 {
1197 endOfDef(yyscanner,1);
1198 yyextra->indent=yyextra->curIndent;
1199 // make sure the next rule matches ^...
1200 //YY_CURRENT_BUFFER->yy_at_bol=true;
1201 //yyextra->hideClassDocs = false;
1202 BEGIN(Search);
1203 }
1204 else
1205 {
1206 yyextra->current->program << yytext;
1207 }
1208 }
1209 "'" { // start of a single quoted string
1210 yyextra->current->program << *yytext;
1211 yyextra->stringContext=YY_START;
1212 yyextra->specialBlock = false;
1213 yyextra->copyString=&yyextra->current->program;
1214 BEGIN( SingleQuoteString );
1215 }
1216 "\"" { // start of a double quoted string
1217 yyextra->current->program << *yytext;
1218 yyextra->stringContext=YY_START;
1219 yyextra->specialBlock = false;
1220 yyextra->copyString=&yyextra->current->program;
1221 BEGIN( DoubleQuoteString );
1222 }
1223 [^ \t\n#'"]+ { // non-special stuff
1224 yyextra->current->program << yytext;
1225 yyextra->specialBlock = false;
1226 //yyextra->hideClassDocs = false;
1227 }
1228 {NEWLINE} {
1229 yyextra->current->program << *yytext;
1230 incLineNr(yyscanner);
1231 }
1232 {POUNDCOMMENT} { // normal comment
1233 yyextra->current->program << yytext;
1234 }
1235 . { // any character
1236 yyextra->specialBlock = false;
1237 yyextra->current->program << *yytext;
1238 }
1239 {TRIDOUBLEQUOTE} { // start of a comment block
1240 //if (!yyextra->hideClassDocs)
1241 yyextra->current->program << yytext;
1242 initTriDoubleQuoteBlock(yyscanner);
1243 BEGIN(TripleComment);
1244 }
1245
1246 {TRISINGLEQUOTE} { // start of a comment block
1247 //if (!yyextra->hideClassDocs)
1248 yyextra->current->program << yytext;
1249 initTriSingleQuoteBlock(yyscanner);
1250 BEGIN(TripleComment);
1251 }
1252}
1253
1254<TypeAlias>{
1255 {IDENTIFIER} {
1256 yyextra->current->name = yytext;
1257 }
1258 "[" {
1259 yyextra->bracketDepth=1;
1260 yyextra->genericsContext = YY_START;
1261 yyextra->genericsValue = "[";
1262 BEGIN(Generics);
1263 }
1264 "=" {
1265 yyextra->current->args = yyextra->genericsValue;
1266 BEGIN(TypeValue);
1267 }
1268}
1269<TypeValue>{
1270 \n {
1271 incLineNr(yyscanner);
1272 if (!stripWhiteSpace(yyextra->current->initializer.str()).empty())
1273 {
1274 newTypeAlias(yyscanner);
1275 }
1276 BEGIN(Search);
1277 }
1278 . {
1279 unput(*yytext);
1280 newTypeAlias(yyscanner);
1281 BEGIN(Search);
1282 }
1283 <<EOF>> { yyterminate();
1284 }
newTypeAlias
static void newTypeAlias(yyscan_t yyscanner)
Definition pyscanner.l:1973
stripWhiteSpace
std::string_view stripWhiteSpace(std::string_view s)
Given a string view s, returns a new, narrower view on that string, skipping over any leading or trai...
Definition stringutil.h:75
1285}
1286
1287<ClassDec>{IDENTIFIER} {
1288 if (yyextra->current->type.empty())
1289 {
1290 yyextra->current->type = "class";
1291 }
1292
1293 yyextra->current->section = EntryType::makeClass();
1294 yyextra->current->name = yytext;
1295 // we need to set the protectiion based on the "local" class name
1296 setProtection(yyscanner);
1297
1298 // prepend scope in case of nested classes
1299 if (yyextra->current_root->section.isScope())
1300 {
1301 //printf("*** Prepending scope %s to class %s\n",qPrint(yyextra->current_root->name),qPrint(yyextra->current->name));
1302 yyextra->current->name.prepend(yyextra->current_root->name+"::");
1303 }
1304
1305 yyextra->current->name = yyextra->current->name.stripWhiteSpace();
1306 yyextra->current->fileName = yyextra->fileName;
1307 yyextra->docBlockContext = YY_START;
1308 yyextra->docBlockInBody = false;
1309 yyextra->docBlockJavaStyle = false;
1310 yyextra->docBlock.clear();
1311 yyextra->bracketDepth = 0;
1312
1313 BEGIN(ClassInheritance);
1314 }
1315
1316<ClassInheritance>{
1317 ({BB}|[\‍(,\‍)]) { // syntactic sugar for the list
1318 }
1319 "[" {
1320 yyextra->bracketDepth=1;
1321 yyextra->genericsContext = YY_START;
1322 yyextra->genericsValue = "[";
1323 BEGIN(Generics);
1324 }
1325 ":" { // begin of the class definition
1326 yyextra->specialBlock = true; // expecting a docstring
1327 yyextra->current->bodyLine = yyextra->yyLineNr;
1328 yyextra->current->program.str(std::string());
1329 BEGIN(ClassCaptureIndent);
1330 }
1331
1332 {SCOPE} {
1333 yyextra->current->extends.emplace_back(
1334 substitute(yytext,".","::"),Protection::Public,Specifier::Normal
1335 );
1336 }
1337 "'" { // start of a single quoted string
1338 yyextra->stringContext=YY_START;
1339 BEGIN( SingleQuoteStringIgnore );
1340 }
1341 "\"" { // start of a double quoted string
1342 yyextra->stringContext=YY_START;
1343 BEGIN( DoubleQuoteStringIgnore );
1344 }
1345}
1346
1347<SingleQuoteStringIgnore>{
1348 "'" { // end of a single quoted string
1349 BEGIN(yyextra->stringContext);
1350 }
1351 . { }
1352}
1353<DoubleQuoteStringIgnore>{
1354 "\"" { // end of a double quoted string
1355 BEGIN(yyextra->stringContext);
1356 }
1357 . { }
1358}
1359
1360<ClassCaptureIndent>{
1361 "\n"|({BB}"\n") {
1362 // Blankline - ignore, keep looking for indentation.
1363 lineCount(yyscanner);
1364 yyextra->current->program << yytext;
1365 }
1366
1367 {TRIDOUBLEQUOTE} { // start of a comment block
1368 initTriDoubleQuoteBlock(yyscanner);
1369 yyextra->current->program << yytext;
1370 BEGIN(TripleComment);
1371 }
1372 {TRISINGLEQUOTE} { // start of a comment block
1373 initTriSingleQuoteBlock(yyscanner);
1374 yyextra->current->program << yytext;
1375 BEGIN(TripleComment);
1376 }
1377 {STARTDOCSYMS}[#]* { // start of a special comment
1378 initSpecialBlock(yyscanner);
1379 BEGIN(SpecialComment);
1380 }
1381 {POUNDCOMMENT} { // ignore comment with just one #
1382 }
1383 ^{BB} {
1384 yyextra->current->program << yytext;
1385 //yyextra->current->startLine = yyextra->yyLineNr;
1386 yyextra->curIndent=computeIndent(yytext);
1387 yyextra->bodyEntry = yyextra->current;
1388 DBG_CTX((stderr,"setting indent %d\n",yyextra->curIndent));
1389 //printf("yyextra->current->program=[%s]\n",qPrint(yyextra->current->program));
1390 //yyextra->hideClassDocs = true;
1391 BEGIN(ClassBody);
1392 }
1393
1394 ""/({NONEMPTY}|{EXPCHAR}) {
1395 // Just pushback an empty class, and
1396 // resume parsing the body.
1397 newEntry(yyscanner);
1398 yyextra->current->program << yytext;
1399
1400 // printf("Failed to find indent - skipping!");
1401 BEGIN( Search );
1402 }
1403}
1404
1405
1406<VariableDec>{
1407 ":"{B}{IDENTIFIER} { //typing
1408 yyextra->startInit = false;
1409 yyextra->current->type = substitute(yytext,":","");
1410 }
1411 {STARTDOCSYMS}"<"/.* { // start of a special comment
1412 yyextra->curIndent=computeIndent(yytext);
1413 yyextra->packageCommentAllowed = false;
1414 initSpecialBlock(yyscanner);
1415 yyextra->docBlockContext = VariableEnd;
1416 BEGIN(SpecialComment);
1417 }
1418 "#".* { // comment
1419 BEGIN( VariableEnd );
1420 }
1421 \n {
1422 unput('\n');
1423 BEGIN( VariableEnd );
1424 }
1425}
1426
1427<VariableAtom>{
1428 [\‍(\[\{] {
1429 yyextra->current->initializer << *yytext;
1430 if (yyextra->atomStart==*yytext)
1431 {
1432 yyextra->atomCount++;
1433 }
1434 }
1435 [\‍)\]\}] {
1436 yyextra->current->initializer << *yytext;
1437 if (yyextra->atomEnd==*yytext)
1438 {
1439 yyextra->atomCount--;
1440 }
1441 if (yyextra->atomCount==0)
1442 {
1443 yyextra->startInit = false;
1444 BEGIN(yyextra->atomContext);
1445 }
1446 }
1447 {TRIDOUBLEQUOTE} { // start of a comment block
1448 yyextra->specialBlock = false;
1449 yyextra->current->program << yytext;
1450 initTriDoubleQuoteBlock(yyscanner);
1451 BEGIN(TripleComment);
1452 }
1453
1454 {TRISINGLEQUOTE} { // start of a comment block
1455 yyextra->specialBlock = false;
1456 yyextra->current->program << yytext;
1457 initTriSingleQuoteBlock(yyscanner);
1458 BEGIN(TripleComment);
1459 }
1460 "'" {
1461 yyextra->stringContext=YY_START;
1462 yyextra->current->initializer << "'";
1463 yyextra->copyString=&yyextra->current->initializer;
1464 BEGIN( SingleQuoteString );
1465 }
1466 "\"" {
1467 yyextra->stringContext=YY_START;
1468 yyextra->current->initializer << "\"";
1469 yyextra->copyString=&yyextra->current->initializer;
1470 BEGIN( DoubleQuoteString );
1471 }
1472 {IDENTIFIER} {
1473 yyextra->current->initializer << yytext;
1474 }
1475 {POUNDCOMMENT} { // normal comment
1476 yyextra->current->initializer << yytext;
1477 }
1478 . {
1479 yyextra->current->initializer << *yytext;
1480 }
1481 \n {
1482 yyextra->current->initializer << *yytext;
1483 incLineNr(yyscanner);
1484 }
1485
1486}
1487
1488<VariableEnd>{
1489 \n {
1490 incLineNr(yyscanner);
1491 if (!stripWhiteSpace(yyextra->current->initializer.str()).empty())
1492 {
1493 newVariable(yyscanner);
1494 }
1495 BEGIN(Search);
1496 }
1497 . {
1498 unput(*yytext);
1499 newVariable(yyscanner);
1500 BEGIN(Search);
1501 }
1502 <<EOF>> { yyterminate();
1503 }
newVariable
static void newVariable(yyscan_t yyscanner)
Definition pyscanner.l:1962
1504}
1505
1506<TripleComment>{
1507 {ENDTRIDOUBLEQUOTE} |
1508 {ENDTRISINGLEQUOTE} {
1509 // printf("Expected module block %d special=%d\n",yyextra->expectModuleDocs,yyextra->specialBlock);
1510 if (yyextra->doubleQuote==(yytext[0]=='"'))
1511 {
1512 if (yyextra->specialBlock) // expecting a docstring
1513 {
1514 DString actualDoc=yyextra->docBlock;
1515 if (!yyextra->docBlockSpecial) // legacy unformatted docstring
1516 {
1517 if (!actualDoc.empty())
1518 {
1519 stripIndentationVerbatim(actualDoc,yyextra->commentIndent);
1520 actualDoc.prepend("@iverbatim\n");
1521 actualDoc.append("@endiverbatim ");
1522 }
1523 }
1524 //printf("-------> yyextra->current=%p yyextra->bodyEntry=%p\n",yyextra->current,yyextra->bodyEntry);
1525 handleCommentBlock(yyscanner, actualDoc, false);
1526 }
1527 else if (yyextra->packageCommentAllowed) // expecting module docs
1528 {
1529 DString actualDoc=yyextra->docBlock;
1530 if (!yyextra->docBlockSpecial) // legacy unformatted docstring
1531 {
1532 if (!actualDoc.empty())
1533 {
1534 stripIndentationVerbatim(actualDoc,yyextra->commentIndent);
1535 actualDoc.prepend("@iverbatim\n");
1536 actualDoc.append("@endiverbatim ");
1537 }
1538 }
1539 if (yyextra->moduleScope.startsWith("__") && yyextra->moduleScope.endsWith("__"))
1540 {
1541 actualDoc.prepend("\\namespace \\"+yyextra->moduleScope+" ");
1542 }
1543 else
1544 {
1545 actualDoc.prepend("\\namespace "+yyextra->moduleScope+" ");
1546 }
1547 handleCommentBlock(yyscanner, actualDoc, false);
1548 }
1549 if ((yyextra->docBlockContext==ClassBody /*&& !yyextra->hideClassDocs*/) ||
1550 yyextra->docBlockContext==FunctionBody)
1551 {
1552 yyextra->current->program << yyextra->docBlock;
1553 yyextra->current->program << yytext;
1554 }
1555 //if (yyextra->hideClassDocs)
1556 //{
1557 // yyextra->current->startLine = yyextra->yyLineNr;
1558 //}
1559 //yyextra->hideClassDocs=false;
1560 BEGIN(yyextra->docBlockContext);
1561 }
1562 else
1563 {
1564 yyextra->docBlock += yytext;
1565 }
1566 yyextra->packageCommentAllowed = false;
1567 }
DString::append
DString & append(char c)
Definition dstring.h:489
DString::prepend
DString & prepend(const char *s)
Definition dstring.h:515
handleCommentBlock
static void handleCommentBlock(yyscan_t yyscanner, const DString &doc, bool brief)
Definition pyscanner.l:2101
stripIndentationVerbatim
void stripIndentationVerbatim(DString &doc, size_t indentationLevel, bool skipFirstLine)
Definition util.cpp:4772
1568
1569
1570 ^{BB} { // leading whitespace, compensate for """! / '''!
1571 if (yyextra->firstPass && yyextra->docBlockSpecial && yyleng >= yyextra->curIndent)
1572 yyextra->docBlock += yytext + yyextra->curIndent;
1573 else
1574 yyextra->docBlock += yytext;
1575 }
1576 [^"'\n \t\\@]+ {
1577 yyextra->docBlock += yytext;
1578 }
1579 \n {
1580 incLineNr(yyscanner);
1581 yyextra->docBlock += yytext;
1582 }
1583 {CMD}"ifile"{B}+"\""[^\n\"]+"\"" {
1584 yyextra->fileName = &yytext[6];
1585 yyextra->fileName = yyextra->fileName.stripWhiteSpace();
1586 yyextra->fileName = yyextra->fileName.mid(1,yyextra->fileName.length()-2);
1587 yyextra->docBlock+=yytext;
1588 }
1589 {CMD}"ifile"{B}+{FILEMASK} {
1590 yyextra->fileName = &yytext[6];
1591 yyextra->fileName = yyextra->fileName.stripWhiteSpace();
1592 yyextra->docBlock+=yytext;
1593 }
1594 {CMD}"iline"{LINENR}/[\n\.] |
1595 {CMD}"iline"{LINENR}{B} {
1596 bool ok = false;
1597 int nr = DString(&yytext[6]).toInt(&ok);
1598 if (!ok)
1599 {
1600 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '{}' for iline command",yytext);
1601 }
1602 else
1603 {
1604 yyextra->yyLineNr = nr;
1605 }
1606 yyextra->docBlock+=yytext;
1607 }
1608 ({CMD}{CMD}){ID}/[^a-z_A-Z0-9] { // escaped command
1609 yyextra->docBlock+=yytext;
1610 }
1611 \\. { // escaped char TO be extended
1612 yyextra->docBlock += yytext;
1613 }
1614 . {
1615 yyextra->docBlock += yytext;
1616 }
DString::toInt
int toInt(bool *ok=nullptr, int base=10) const
Definition dstring.cpp:191
1617}
1618
1619<SpecialComment>{
1620 ^{B}"#"("#")* { // skip leading hashes
1621 }
1622 \n/{B}"#" { // continuation of the comment on the next line
1623 yyextra->docBlock+='\n';
1624 yyextra->docBrief = false;
1625 incLineNr(yyscanner);
1626 }
1627 {CMD}"ifile"{B}+"\""[^\n\"]+"\"" {
1628 yyextra->fileName = &yytext[6];
1629 yyextra->fileName = yyextra->fileName.stripWhiteSpace();
1630 yyextra->fileName = yyextra->fileName.mid(1,yyextra->fileName.length()-2);
1631 yyextra->docBlock+=yytext;
1632 }
1633 {CMD}"ifile"{B}+{FILEMASK} {
1634 yyextra->fileName = &yytext[6];
1635 yyextra->fileName = yyextra->fileName.stripWhiteSpace();
1636 yyextra->docBlock+=yytext;
1637 }
1638 {CMD}"iline"{LINENR}/[\n\.] |
1639 {CMD}"iline"{LINENR}{B} {
1640 bool ok = false;
1641 int nr = DString(&yytext[6]).toInt(&ok);
1642 if (!ok)
1643 {
1644 warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '{}' for iline command",yytext);
1645 }
1646 else
1647 {
1648 yyextra->yyLineNr = nr;
1649 }
1650 yyextra->docBlock+=yytext;
1651 }
1652 ({CMD}{CMD}){ID}/[^a-z_A-Z0-9] { // escaped command
1653 yyextra->docBlock+=yytext;
1654 }
1655 "\\ilinebr "{B}* {
1656 DString indent;
1657 int extraSpaces = std::max(0,static_cast<int>(yyleng-9-yyextra->curIndent-2));
1658 indent.fill(' ',extraSpaces);
1659 //printf("extraSpaces=%d\n",extraSpaces);
1660 yyextra->docBlock += "\\ilinebr ";
1661 yyextra->docBlock += indent;
1662 }
1663 [^#\\@\n]+ { // any other stuff
1664 yyextra->docBlock+=yytext;
1665 }
1666 \n { // new line that ends the comment
1667 handleCommentBlock(yyscanner, yyextra->docBlock, yyextra->docBrief);
1668 if (yyextra->docBlockContext == VariableEnd)
1669 {
1670 unput(*yytext);
1671 }
1672 else
1673 {
1674 incLineNr(yyscanner);
1675 }
1676 BEGIN(yyextra->docBlockContext);
1677 }
1678 . { // anything we missed
1679 yyextra->docBlock+=*yytext;
1680 }
DString::fill
DString fill(char c, size_t len)
Fills a string with a predefined character.
Definition dstring.h:278
1681}
1682
1683<SingleQuoteString>{
1684 \\{B}\n { // line continuation
1685 addToString(yyscanner,yytext);
1686 incLineNr(yyscanner);
1687 }
1688 \\. { // escaped char
1689 addToString(yyscanner,yytext);
1690 }
1691 "\"\"\"" { // triple double quotes
1692 addToString(yyscanner,yytext);
1693 }
1694 "'" { // end of the string
1695 addToString(yyscanner,yytext);
1696 BEGIN(yyextra->stringContext);
1697 }
1698 [^"'\n\\]+ { // normal chars
1699 addToString(yyscanner,yytext);
1700 }
1701 . { // normal char
1702 addToString(yyscanner,yytext);
1703 }
addToString
static void addToString(yyscan_t yyscanner, const char *s)
Definition pyscanner.l:2169
1704}
1705
1706<DoubleQuoteString>{
1707 \\{B}\n { // line continuation
1708 addToString(yyscanner,yytext);
1709 incLineNr(yyscanner);
1710 }
1711 \\. { // escaped char
1712 addToString(yyscanner,yytext);
1713 }
1714 "'''" { // triple single quotes
1715 addToString(yyscanner,yytext);
1716 }
1717 "\"" { // end of the string
1718 addToString(yyscanner,yytext);
1719 BEGIN(yyextra->stringContext);
1720 }
1721 [^"'\n\\]+ { // normal chars
1722 addToString(yyscanner,yytext);
1723 }
1724 . { // normal char
1725 addToString(yyscanner,yytext);
1726 }
1727}
1728
1729<TripleString>{
1730 {ENDTRIDOUBLEQUOTE} |
1731 {ENDTRISINGLEQUOTE} {
1732 *yyextra->copyString << yytext;
1733 if (yyextra->doubleQuote==(yytext[0]=='"'))
1734 {
1735 BEGIN(yyextra->stringContext);
1736 }
1737 }
1738
1739
1740 ({LONGSTRINGBLOCK}) {
1741 lineCount(yyscanner);
1742 *yyextra->copyString << yytext;
1743 }
1744 \n {
1745 incLineNr(yyscanner);
1746 *yyextra->copyString << yytext;
1747 }
1748 . {
1749 *yyextra->copyString << *yytext;
1750 }
1751}
1752
1753<Decorator>{
1754 {TRIDOUBLEQUOTE} { // start of a comment block
1755 yyextra->doubleQuote=true;
1756 yyextra->decoratorCommentStr.str(std::string());
1757 yyextra->copyString=&yyextra->decoratorCommentStr;
1758 yyextra->stringContext=YY_START;
1759 BEGIN(TripleString);
1760 }
1761
1762 {TRISINGLEQUOTE} { // start of a comment block
1763 yyextra->doubleQuote=false;
1764 yyextra->decoratorCommentStr.str(std::string());
1765 yyextra->copyString=&yyextra->decoratorCommentStr;
1766 yyextra->stringContext=YY_START;
1767 BEGIN(TripleString);
1768 }
1769 "'" {
1770 yyextra->stringContext=YY_START;
1771 yyextra->decoratorCommentStr.str(std::string());
1772 yyextra->copyString=&yyextra->decoratorCommentStr;
1773 BEGIN( SingleQuoteString );
1774 }
1775 "\"" {
1776 yyextra->stringContext=YY_START;
1777 yyextra->decoratorCommentStr.str(std::string());
1778 yyextra->copyString=&yyextra->decoratorCommentStr;
1779 BEGIN( DoubleQuoteString );
1780 }
1781 "(" {
1782 yyextra->decoratorRound++;
1783 }
1784 ")" {
1785 yyextra->decoratorRound--;
1786 if (!yyextra->decoratorRound) BEGIN( Search );
1787 }
1788 \n {
1789 incLineNr(yyscanner);
1790 }
1791 . { }
1792}
1793
1794 /* ------------ End rules -------------- */
1795
1796 /*
1797<*>({NONEMPTY}|{EXPCHAR}|{BB}) { // This should go one character at a time.
1798 // printf("[pyscanner] '%s' [ state %d ] [line %d] no match\n",
1799 // yytext, YY_START, yyextra->yyLineNr);
1800
1801 }
1802 */
1803
1804<*>{NEWLINE} {
1805 //printf("[pyscanner] %d NEWLINE [line %d] no match\n",
1806 // YY_START, yyextra->yyLineNr);
1807
1808 lineCount(yyscanner);
1809 }
1810
1811<*>"'" {
1812 //fprintf(stderr,"Quote: %d\n",YY_START);
1813 }
1814
1815<*>. {
1816 //printf("[pyscanner] '%s' [ state %d ] [line %d] no match\n",
1817 // yytext, YY_START, yyextra->yyLineNr);
1818
1819 }
1820
1821
1822%%

◆ yyread()

int yyread ( yyscan_t yyscanner,
char * buf,
int max_size )
static

Definition at line 1827 of file pyscanner.l.

1828{
1829 struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1830 int c=0;
1831 const char *p = yyextra->inputString + yyextra->inputPosition;
1832 while ( c < max_size && *p ) { *buf++ = *p++; c++; }
1833 yyextra->inputPosition+=c;
1834 return c;
1835}