Branch: refs/heads/master Author: Jiří Techet techet@gmail.com Committer: GitHub noreply@github.com Date: Mon, 14 Mar 2022 19:39:39 UTC Commit: 378e5e8a0db18364cd9ec848aa81f5eded508da5 https://github.com/geany/geany/commit/378e5e8a0db18364cd9ec848aa81f5eded508d...
Log Message: ----------- Merge pull request #3031 from techee/python_sync
Use python parser from uctags
Modified Paths: -------------- ctags/Makefile.am ctags/parsers/geany_python.c ctags/parsers/python.c src/tagmanager/tm_parser.c tests/ctags/cython_sample2.pyx.tags tests/ctags/py_constructor_arglist.py.tags tests/ctags/simple.py.tags tests/ctags/tabindent.py.tags
Modified: ctags/Makefile.am 2 lines changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -81,7 +81,7 @@ parsers = \ parsers/perl.h \ parsers/php.c \ parsers/powershell.c \ - parsers/geany_python.c \ + parsers/python.c \ parsers/r.c \ parsers/r.h \ parsers/rst.c \
Modified: ctags/parsers/geany_python.c 862 lines changed, 0 insertions(+), 862 deletions(-) =================================================================== @@ -1,862 +0,0 @@ -/* -* Copyright (c) 2000-2003, Darren Hiebert -* -* This source code is released for free distribution under the terms of the -* GNU General Public License version 2 or (at your option) any later version. -* -* This module contains functions for generating tags for Python language -* files. -*/ -/* -* INCLUDE FILES -*/ -#include "general.h" /* must always come first */ - -#include <string.h> - -#include "entry.h" -#include "nestlevel.h" -#include "options.h" -#include "read.h" -#include "parse.h" -#include "vstring.h" -#include "routines.h" -#include "debug.h" -#include "xtag.h" - -/* -* DATA DEFINITIONS -*/ - -struct corkInfo { - int index; -}; - -struct nestingLevelUserData { - int indentation; -}; -#define PY_NL_INDENTATION(nl) ((struct nestingLevelUserData *)nestingLevelGetUserData(nl))->indentation - - -typedef enum { - K_CLASS, K_FUNCTION, K_METHOD, K_VARIABLE, K_IMPORT -} pythonKind; - -static kindDefinition PythonKinds[] = { - {true, 'c', "class", "classes"}, - {true, 'f', "function", "functions"}, - {true, 'm', "member", "class members"}, - {true, 'v', "variable", "variables"}, - {true, 'x', "unknown", "name referring a classe/variable/function/module defined in other module"} -}; - -typedef enum { - A_PUBLIC, A_PRIVATE, A_PROTECTED -} pythonAccess; - -static const char *const PythonAccesses[] = { - "public", "private", "protected" -}; - -static char const * const singletriple = "'''"; -static char const * const doubletriple = """""; - -/* -* FUNCTION DEFINITIONS -*/ - -static bool isIdentifierFirstCharacter (int c) -{ - return (bool) (isalpha (c) || c == '_'); -} - -static bool isIdentifierCharacter (int c) -{ - return (bool) (isalnum (c) || c == '_'); -} - -/* follows PEP-8, and always reports single-underscores as protected - * See: - * - http://www.python.org/dev/peps/pep-0008/#method-names-and-instance-variables - * - http://www.python.org/dev/peps/pep-0008/#designing-for-inheritance - */ -static pythonAccess accessFromIdentifier (const vString *const ident, - pythonKind kind, bool has_parent, bool parent_is_class) -{ - const char *const p = vStringValue (ident); - const size_t len = vStringLength (ident); - - /* inside a function/method, private */ - if (has_parent && !parent_is_class) - return A_PRIVATE; - /* not starting with "_", public */ - else if (len < 1 || p[0] != '_') - return A_PUBLIC; - /* "__...__": magic methods */ - else if (kind == K_METHOD && parent_is_class && - len > 3 && p[1] == '_' && p[len - 2] == '_' && p[len - 1] == '_') - return A_PUBLIC; - /* "__...": name mangling */ - else if (parent_is_class && len > 1 && p[1] == '_') - return A_PRIVATE; - /* "_...": suggested as non-public, but easily accessible */ - else - return A_PROTECTED; -} - -static void addAccessFields (tagEntryInfo *const entry, - const vString *const ident, pythonKind kind, - bool has_parent, bool parent_is_class) -{ - pythonAccess access; - - access = accessFromIdentifier (ident, kind, has_parent, parent_is_class); - entry->extensionFields.access = PythonAccesses [access]; - /* FIXME: should we really set isFileScope in addition to access? */ - if (access == A_PRIVATE) - entry->isFileScope = true; -} - -/* Given a string with the contents of a line directly after the "def" keyword, - * extract all relevant information and create a tag. - */ -static struct corkInfo makeFunctionTag (vString *const function, - vString *const parent, int is_class_parent, const char *arglist) -{ - tagEntryInfo tag; - int corkIndex; - struct corkInfo info; - - if (vStringLength (parent) > 0) - { - if (is_class_parent) - { - initTagEntry (&tag, vStringValue (function), K_METHOD); - tag.extensionFields.scopeKindIndex = K_CLASS; - } - else - { - initTagEntry (&tag, vStringValue (function), K_FUNCTION); - tag.extensionFields.scopeKindIndex = K_FUNCTION; - } - tag.extensionFields.scopeName = vStringValue (parent); - } - else - initTagEntry (&tag, vStringValue (function), K_FUNCTION); - - tag.extensionFields.signature = arglist; - - addAccessFields (&tag, function, is_class_parent ? K_METHOD : K_FUNCTION, - vStringLength (parent) > 0, is_class_parent); - - corkIndex = makeTagEntry (&tag); - - info.index = corkIndex; - return info; -} - -/* Given a string with the contents of the line directly after the "class" - * keyword, extract all necessary information and create a tag. - */ -static int makeClassTag (vString *const class, vString *const inheritance, - vString *const parent, int is_class_parent) -{ - tagEntryInfo tag; - initTagEntry (&tag, vStringValue (class), K_CLASS); - if (vStringLength (parent) > 0) - { - if (is_class_parent) - { - tag.extensionFields.scopeKindIndex = K_CLASS; - tag.extensionFields.scopeName = vStringValue (parent); - } - else - { - tag.extensionFields.scopeKindIndex = K_FUNCTION; - tag.extensionFields.scopeName = vStringValue (parent); - } - } - tag.extensionFields.inheritance = vStringValue (inheritance); - addAccessFields (&tag, class, K_CLASS, vStringLength (parent) > 0, - is_class_parent); - return makeTagEntry (&tag); -} - -static void makeVariableTag (vString *const var, vString *const parent, - bool is_class_parent) -{ - tagEntryInfo tag; - initTagEntry (&tag, vStringValue (var), K_VARIABLE); - if (vStringLength (parent) > 0) - { - tag.extensionFields.scopeKindIndex = K_CLASS; - tag.extensionFields.scopeName = vStringValue (parent); - } - addAccessFields (&tag, var, K_VARIABLE, vStringLength (parent) > 0, - is_class_parent); - makeTagEntry (&tag); -} - -/* Skip a single or double quoted string. */ -static const char *skipString (const char *cp) -{ - const char *start = cp; - int escaped = 0; - for (cp++; *cp; cp++) - { - if (escaped) - escaped--; - else if (*cp == '\') - escaped++; - else if (*cp == *start) - return cp + 1; - } - return cp; -} - -/* Skip everything up to an identifier start. */ -static const char *skipEverything (const char *cp) -{ - int match; - for (; *cp; cp++) - { - if (*cp == '#') - return strchr(cp, '\0'); - - match = 0; - if (*cp == '"' || *cp == ''') - match = 1; - - /* these checks find unicode, binary (Python 3) and raw strings */ - if (!match) - { - bool r_first = (*cp == 'r' || *cp == 'R'); - - /* "r" | "R" | "u" | "U" | "b" | "B" */ - if (r_first || *cp == 'u' || *cp == 'U' || *cp == 'b' || *cp == 'B') - { - unsigned int i = 1; - - /* r_first -> "rb" | "rB" | "Rb" | "RB" - !r_first -> "ur" | "UR" | "Ur" | "uR" | "br" | "Br" | "bR" | "BR" */ - if (( r_first && (cp[i] == 'b' || cp[i] == 'B')) || - (!r_first && (cp[i] == 'r' || cp[i] == 'R'))) - i++; - - if (cp[i] == ''' || cp[i] == '"') - { - match = 1; - cp += i; - } - } - } - if (match) - { - cp = skipString(cp); - if (!*cp) break; - } - if (isIdentifierFirstCharacter ((int) *cp)) - return cp; - if (match) - cp--; /* avoid jumping over the character after a skipped string */ - } - return cp; -} - -/* Skip an identifier. */ -static const char *skipIdentifier (const char *cp) -{ - while (isIdentifierCharacter ((int) *cp)) - cp++; - return cp; -} - -static const char *findDefinitionOrClass (const char *cp) -{ - while (*cp) - { - cp = skipEverything (cp); - if (!strncmp(cp, "def", 3) || !strncmp(cp, "class", 5) || - !strncmp(cp, "cdef", 4) || !strncmp(cp, "cpdef", 5)) - { - return cp; - } - cp = skipIdentifier (cp); - } - return NULL; -} - -static const char *skipSpace (const char *cp) -{ - while (isspace ((int) *cp)) - ++cp; - return cp; -} - -/* Starting at ''cp'', parse an identifier into ''identifier''. */ -static const char *parseIdentifier (const char *cp, vString *const identifier) -{ - vStringClear (identifier); - while (isIdentifierCharacter ((int) *cp)) - { - vStringPut (identifier, (int) *cp); - ++cp; - } - return cp; -} - -static int parseClass (const char *cp, vString *const class, - vString *const parent, int is_class_parent) -{ - int corkIndex; - vString *const inheritance = vStringNew (); - vStringClear (inheritance); - cp = parseIdentifier (cp, class); - cp = skipSpace (cp); - if (*cp == '(') - { - ++cp; - while (*cp != ')') - { - if (*cp == '\0') - { - /* Closing parenthesis can be in follow up line. */ - cp = (const char *) readLineFromInputFile (); - if (!cp) break; - vStringPut (inheritance, ' '); - continue; - } - vStringPut (inheritance, *cp); - ++cp; - } - } - corkIndex = makeClassTag (class, inheritance, parent, is_class_parent); - vStringDelete (inheritance); - return corkIndex; -} - -static void parseImports (const char *cp) -{ - const char *pos; - vString *name, *name_next; - - cp = skipEverything (cp); - - if ((pos = strstr (cp, "import")) == NULL) - return; - - cp = pos + 6; - - /* continue only if there is some space between the keyword and the identifier */ - if (! isspace (*cp)) - return; - - cp++; - cp = skipSpace (cp); - - name = vStringNew (); - name_next = vStringNew (); - - cp = skipEverything (cp); - while (*cp) - { - cp = parseIdentifier (cp, name); - - cp = skipEverything (cp); - /* we parse the next possible import statement as well to be able to ignore 'foo' in - * 'import foo as bar' */ - parseIdentifier (cp, name_next); - - /* take the current tag only if the next one is not "as" */ - if (strcmp (vStringValue (name_next), "as") != 0 && - strcmp (vStringValue (name), "as") != 0) - { - makeSimpleTag (name, K_IMPORT); - } - } - vStringDelete (name); - vStringDelete (name_next); -} - -/* modified from lcpp.c getArglistFromStr(). - * warning: terminates rest of string past arglist! - * note: does not ignore brackets inside strings! */ -static char *parseArglist(const char *buf) -{ - char *start, *end; - int level; - if (NULL == buf) - return NULL; - if (NULL == (start = strchr(buf, '('))) - return NULL; - for (level = 1, end = start + 1; level > 0; ++end) - { - if ('\0' == *end) - break; - else if ('(' == *end) - ++ level; - else if (')' == *end) - -- level; - } - *end = '\0'; - return strdup(start); -} - -static struct corkInfo parseFunction (const char *cp, vString *const def, - vString *const parent, int is_class_parent) -{ - char *arglist; - struct corkInfo info; - - cp = parseIdentifier (cp, def); - arglist = parseArglist (cp); - info = makeFunctionTag (def, parent, is_class_parent, arglist); - if (arglist != NULL) - eFree (arglist); - return info; -} - -/* Get the combined name of a nested symbol. Classes are separated with ".", - * functions with "/". For example this code: - * class MyClass: - * def myFunction: - * def SubFunction: - * class SubClass: - * def Method: - * pass - * Would produce this string: - * MyClass.MyFunction/SubFunction/SubClass.Method - */ -static bool constructParentString(NestingLevels *nls, int indent, - vString *result) -{ - int i; - NestingLevel *prev = NULL; - int is_class = false; - vStringClear (result); - for (i = 0; i < nls->n; i++) - { - NestingLevel *nl = nestingLevelsGetNthFromRoot (nls, i); - tagEntryInfo *e; - - if (indent <= PY_NL_INDENTATION(nl)) - break; - if (prev) - { - vStringCatS(result, "."); /* make Geany symbol list grouping work properly */ -/* - if (prev->kindIndex == K_CLASS) - vStringCatS(result, "."); - else - vStringCatS(result, "/"); -*/ - } - - e = getEntryOfNestingLevel (nl); - if (e) - { - vStringCatS(result, e->name); - is_class = (e->kindIndex == K_CLASS); - } - else - is_class = false; - - prev = nl; - } - return is_class; -} - -/* Check indentation level and truncate nesting levels accordingly */ -static void checkIndent(NestingLevels *nls, int indent) -{ - int i; - NestingLevel *n; - - for (i = 0; i < nls->n; i++) - { - n = nestingLevelsGetNthFromRoot (nls, i); - if (n && indent <= PY_NL_INDENTATION(n)) - { - /* truncate levels */ - nls->n = i; - break; - } - } -} - -static void addNestingLevel(NestingLevels *nls, int indentation, struct corkInfo *info) -{ - int i; - NestingLevel *nl = NULL; - - for (i = 0; i < nls->n; i++) - { - nl = nestingLevelsGetNthFromRoot(nls, i); - if (indentation <= PY_NL_INDENTATION(nl)) break; - } - if (i == nls->n) - nl = nestingLevelsPush(nls, info->index); - else - /* reuse existing slot */ - nl = nestingLevelsTruncate (nls, i + 1, info->index); - - PY_NL_INDENTATION(nl) = indentation; -} - -/* Return a pointer to the start of the next triple string, or NULL. Store - * the kind of triple string in "which" if the return is not NULL. - */ -static char const *find_triple_start(char const *string, char const **which) -{ - char const *cp = string; - - for (; *cp; cp++) - { - if (*cp == '#') - break; - if (*cp == '"' || *cp == ''') - { - if (strncmp(cp, doubletriple, 3) == 0) - { - *which = doubletriple; - return cp; - } - if (strncmp(cp, singletriple, 3) == 0) - { - *which = singletriple; - return cp; - } - cp = skipString(cp); - if (!*cp) break; - cp--; /* avoid jumping over the character after a skipped string */ - } - } - return NULL; -} - -/* Find the end of a triple string as pointed to by "which", and update "which" - * with any other triple strings following in the given string. - */ -static void find_triple_end(char const *string, char const **which) -{ - char const *s = string; - while (1) - { - /* Check if the string ends in the same line. */ - s = strstr (s, *which); - if (!s) break; - s += 3; - *which = NULL; - /* If yes, check if another one starts in the same line. */ - s = find_triple_start(s, which); - if (!s) break; - s += 3; - } -} - -static const char *findVariable(const char *line) -{ - /* Parse global and class variable names (C.x) from assignment statements. - * Object attributes (obj.x) are ignored. - * Assignment to a tuple 'x, y = 2, 3' not supported. - * TODO: ignore duplicate tags from reassignment statements. */ - const char *cp, *sp, *eq, *start; - - cp = strstr(line, "="); - if (!cp) - return NULL; - eq = cp + 1; - while (*eq) - { - if (*eq == '=') - return NULL; /* ignore '==' operator and 'x=5,y=6)' function lines */ - if (*eq == '(' || *eq == '#') - break; /* allow 'x = func(b=2,y=2,' lines and comments at the end of line */ - eq++; - } - - /* go backwards to the start of the line, checking we have valid chars */ - start = cp - 1; - while (start >= line && isspace ((int) *start)) - --start; - while (start >= line && isIdentifierCharacter ((int) *start)) - --start; - if (!isIdentifierFirstCharacter(*(start + 1))) - return NULL; - sp = start; - while (sp >= line && isspace ((int) *sp)) - --sp; - if ((sp + 1) != line) /* the line isn't a simple variable assignment */ - return NULL; - /* the line is valid, parse the variable name */ - ++start; - return start; -} - -/* Skip type declaration that optionally follows a cdef/cpdef */ -static const char *skipTypeDecl (const char *cp, bool *is_class) -{ - const char *lastStart = cp, *ptr = cp; - int loopCount = 0; - ptr = skipSpace(cp); - if (!strncmp("extern", ptr, 6)) { - ptr += 6; - ptr = skipSpace(ptr); - if (!strncmp("from", ptr, 4)) { return NULL; } - } - if (!strncmp("class", ptr, 5)) { - ptr += 5 ; - *is_class = true; - ptr = skipSpace(ptr); - return ptr; - } - /* limit so that we don't pick off "int item=obj()" */ - while (*ptr && loopCount++ < 2) { - while (*ptr && *ptr != '=' && *ptr != '(' && !isspace(*ptr)) { - /* skip over e.g. 'cpdef numpy.ndarray[dtype=double, ndim=1]' */ - if(*ptr == '[') { - while (*ptr && *ptr != ']') ptr++; - if (*ptr) ptr++; - } else { - ptr++; - } - } - if (!*ptr || *ptr == '=') return NULL; - if (*ptr == '(') { - return lastStart; /* if we stopped on a '(' we are done */ - } - ptr = skipSpace(ptr); - lastStart = ptr; - while (*lastStart == '*') lastStart++; /* cdef int *identifier */ - } - return NULL; -} - -/* checks if there is a lambda at position of cp, and return its argument list - * if so. - * We don't return the lambda name since it is useless for now since we already - * know it when we call this function, and it would be a little slower. */ -static bool varIsLambda (const char *cp, char **arglist) -{ - bool is_lambda = false; - - cp = skipSpace (cp); - cp = skipIdentifier (cp); /* skip the lambda's name */ - cp = skipSpace (cp); - if (*cp == '=') - { - cp++; - cp = skipSpace (cp); - if (strncmp (cp, "lambda", 6) == 0) - { - const char *tmp; - - cp += 6; /* skip the lambda */ - tmp = skipSpace (cp); - /* check if there is a space after lambda to detect assignations - * starting with 'lambdaXXX' */ - if (tmp != cp) - { - vString *args = vStringNew (); - - cp = tmp; - vStringPut (args, '('); - for (; *cp != 0 && *cp != ':'; cp++) - vStringPut (args, *cp); - vStringPut (args, ')'); - if (arglist) - *arglist = strdup (vStringValue (args)); - vStringDelete (args); - is_lambda = true; - } - } - } - return is_lambda; -} - -/* checks if @p cp has keyword @p keyword at the start, and fills @p cp_n with - * the position of the next non-whitespace after the keyword */ -static bool matchKeyword (const char *keyword, const char *cp, const char **cp_n) -{ - size_t kw_len = strlen (keyword); - if (strncmp (cp, keyword, kw_len) == 0 && isspace (cp[kw_len])) - { - *cp_n = skipSpace (&cp[kw_len + 1]); - return true; - } - return false; -} - -static void findPythonTags (void) -{ - vString *const continuation = vStringNew (); - vString *const name = vStringNew (); - vString *const parent = vStringNew(); - - NestingLevels *const nesting_levels = nestingLevelsNew(sizeof (struct nestingLevelUserData)); - - const char *line; - int line_skip = 0; - char const *longStringLiteral = NULL; - - while ((line = (const char *) readLineFromInputFile ()) != NULL) - { - const char *cp = line, *candidate; - char const *longstring; - char const *keyword, *variable; - int indent; - - cp = skipSpace (cp); - - if (*cp == '\0') /* skip blank line */ - continue; - - /* Skip comment if we are not inside a multi-line string. */ - if (*cp == '#' && !longStringLiteral) - continue; - - /* Deal with line continuation. */ - if (!line_skip) vStringClear(continuation); - vStringCatS(continuation, line); - vStringStripTrailing(continuation); - if (vStringLast(continuation) == '\') - { - vStringChop(continuation); - vStringCatS(continuation, " "); - line_skip = 1; - continue; - } - cp = line = vStringValue(continuation); - cp = skipSpace (cp); - indent = cp - line; - line_skip = 0; - - /* Deal with multiline string ending. */ - if (longStringLiteral) - { - find_triple_end(cp, &longStringLiteral); - continue; - } - - checkIndent(nesting_levels, indent); - - /* Find global and class variables */ - variable = findVariable(line); - if (variable) - { - const char *start = variable; - char *arglist; - bool parent_is_class; - - vStringClear (name); - while (isIdentifierCharacter ((int) *start)) - { - vStringPut (name, (int) *start); - ++start; - } - - parent_is_class = constructParentString(nesting_levels, indent, parent); - if (varIsLambda (variable, &arglist)) - { - /* show class members or top-level script lambdas only */ - if (parent_is_class || vStringLength(parent) == 0) - makeFunctionTag (name, parent, parent_is_class, arglist); - eFree (arglist); - } - else - { - /* skip variables in methods */ - if (parent_is_class || vStringLength(parent) == 0) - makeVariableTag (name, parent, parent_is_class); - } - } - - /* Deal with multiline string start. */ - longstring = find_triple_start(cp, &longStringLiteral); - if (longstring) - { - longstring += 3; - find_triple_end(longstring, &longStringLiteral); - /* We don't parse for any tags in the rest of the line. */ - continue; - } - - /* Deal with def and class keywords. */ - keyword = findDefinitionOrClass (cp); - if (keyword) - { - bool found = false; - bool is_class = false; - if (matchKeyword ("def", keyword, &cp)) - { - found = true; - } - else if (matchKeyword ("class", keyword, &cp)) - { - found = true; - is_class = true; - } - else if (matchKeyword ("cdef", keyword, &cp)) - { - candidate = skipTypeDecl (cp, &is_class); - if (candidate) - { - found = true; - cp = candidate; - } - - } - else if (matchKeyword ("cpdef", keyword, &cp)) - { - candidate = skipTypeDecl (cp, &is_class); - if (candidate) - { - found = true; - cp = candidate; - } - } - - if (found) - { - bool is_parent_class; - struct corkInfo info; - - is_parent_class = - constructParentString(nesting_levels, indent, parent); - - if (is_class) - { - info.index = parseClass (cp, name, parent, is_parent_class); - } - else - info = parseFunction(cp, name, parent, is_parent_class); - - addNestingLevel(nesting_levels, indent, &info); - } - continue; - } - /* Find and parse imports */ - parseImports(line); - } - - /* Force popping all nesting levels. */ - checkIndent(nesting_levels, 0); - - /* Clean up all memory we allocated. */ - vStringDelete (parent); - vStringDelete (name); - vStringDelete (continuation); - nestingLevelsFree (nesting_levels); -} - -extern parserDefinition *PythonParser (void) -{ - static const char *const extensions[] = { "py", "pyx", "pxd", "pxi" ,"scons", NULL }; - parserDefinition *def = parserNew ("Python"); - def->kindTable = PythonKinds; - def->kindCount = ARRAY_SIZE (PythonKinds); - def->extensions = extensions; - def->parser = findPythonTags; - def->useCork = CORK_QUEUE; - return def; -}
Modified: ctags/parsers/python.c 1597 lines changed, 1597 insertions(+), 0 deletions(-) =================================================================== @@ -0,0 +1,1597 @@ +/* +* Copyright (c) 2000-2003, Darren Hiebert +* Copyright (c) 2014-2016, Colomban Wendling ban@herbesfolles.org +* +* This source code is released for free distribution under the terms of the +* GNU General Public License version 2 or (at your option) any later version. +* +* This module contains functions for generating tags for Python language +* files. +*/ + +#include "general.h" /* must always come first */ + +#include <string.h> + +#include "entry.h" +#include "nestlevel.h" +#include "read.h" +#include "parse.h" +#include "vstring.h" +#include "keyword.h" +#include "routines.h" +#include "debug.h" +#include "xtag.h" +#include "objpool.h" + +#define isIdentifierChar(c) \ + (isalnum (c) || (c) == '_' || (c) >= 0x80) +#define newToken() (objPoolGet (TokenPool)) +#define deleteToken(t) (objPoolPut (TokenPool, (t))) + +enum { + KEYWORD_as, + KEYWORD_async, + KEYWORD_cdef, + KEYWORD_class, + KEYWORD_cpdef, + KEYWORD_def, + KEYWORD_extern, + KEYWORD_from, + KEYWORD_import, + KEYWORD_inline, + KEYWORD_lambda, + KEYWORD_pass, + KEYWORD_return, +}; +typedef int keywordId; /* to allow KEYWORD_NONE */ + +typedef enum { + ACCESS_PRIVATE, + ACCESS_PROTECTED, + ACCESS_PUBLIC, + COUNT_ACCESS +} accessType; + +static const char *const PythonAccesses[COUNT_ACCESS] = { + "private", + "protected", + "public" +}; + +typedef enum { + F_DECORATORS, + F_NAMEREF, + COUNT_FIELD +} pythonField; + +static fieldDefinition PythonFields[COUNT_FIELD] = { + { .name = "decorators", + .description = "decorators on functions and classes", + .enabled = false }, + { .name = "nameref", + .description = "the original name for the tag", + .enabled = true }, +}; + +typedef enum { + K_CLASS, + K_FUNCTION, + K_METHOD, + K_VARIABLE, + K_NAMESPACE, + K_MODULE, + K_UNKNOWN, + K_PARAMETER, + K_LOCAL_VARIABLE, + COUNT_KIND +} pythonKind; + +typedef enum { + PYTHON_MODULE_IMPORTED, + PYTHON_MODULE_NAMESPACE, + PYTHON_MODULE_INDIRECTLY_IMPORTED, +} pythonModuleRole; + +typedef enum { + PYTHON_UNKNOWN_IMPORTED, + PYTHON_UNKNOWN_INDIRECTLY_IMPORTED, +} pythonUnknownRole; + +/* Roles related to `import' + * ========================== + * import X X = (kind:module, role:imported) + * + * import X as Y X = (kind:module, role:indirectlyImported), + * Y = (kind:namespace, nameref:module:X) + * ------------------------------------------------ + * Don't confuse the kind of Y with namespace role of module kind. + * + * from X import * X = (kind:module, role:namespace) + * + * from X import Y X = (kind:module, role:namespace), + * Y = (kind:unknown, role:imported, scope:module:X) + * + * from X import Y as Z X = (kind:module, role:namespace), + * Y = (kind:unknown, role:indirectlyImported, scope:module:X) + * Z = (kind:unknown, nameref:unknown:Y) */ + +static roleDefinition PythonModuleRoles [] = { + { true, "imported", + "imported modules" }, + { true, "namespace", + "namespace from where classes/variables/functions are imported" }, + { true, "indirectlyImported", + "module imported in alternative name" }, +}; + +static roleDefinition PythonUnknownRoles [] = { + { true, "imported", "imported from the other module" }, + { true, "indirectlyImported", + "classes/variables/functions/modules imported in alternative name" }, +}; + +static kindDefinition PythonKinds[COUNT_KIND] = { + {true, 'c', "class", "classes"}, + {true, 'f', "function", "functions"}, + {true, 'm', "member", "class members"}, + {true, 'v', "variable", "variables"}, + {true, 'I', "namespace", "name referring a module defined in other file"}, + {true, 'i', "module", "modules", + .referenceOnly = true, ATTACH_ROLES(PythonModuleRoles)}, + {true, 'x', "unknown", "name referring a class/variable/function/module defined in other module", + .referenceOnly = false, ATTACH_ROLES(PythonUnknownRoles)}, + {false, 'z', "parameter", "function parameters" }, + {false, 'l', "local", "local variables" }, +}; + +static const keywordTable PythonKeywordTable[] = { + /* keyword keyword ID */ + { "as", KEYWORD_as }, + { "async", KEYWORD_async }, + { "cdef", KEYWORD_cdef }, + { "cimport", KEYWORD_import }, + { "class", KEYWORD_class }, + { "cpdef", KEYWORD_cpdef }, + { "def", KEYWORD_def }, + { "extern", KEYWORD_extern }, + { "from", KEYWORD_from }, + { "import", KEYWORD_import }, + { "inline", KEYWORD_inline }, + { "lambda", KEYWORD_lambda }, + { "pass", KEYWORD_pass }, + { "return", KEYWORD_return }, +}; + +typedef enum eTokenType { + /* 0..255 are the byte's value */ + TOKEN_EOF = 256, + TOKEN_UNDEFINED, + TOKEN_INDENT, + TOKEN_KEYWORD, + TOKEN_OPERATOR, + TOKEN_IDENTIFIER, + TOKEN_STRING, + TOKEN_ARROW, /* -> */ + TOKEN_WHITESPACE, +} tokenType; + +typedef struct { + int type; + keywordId keyword; + vString * string; + int indent; + unsigned long lineNumber; + MIOPos filePosition; +} tokenInfo; + +struct pythonNestingLevelUserData { + int indentation; +}; +#define PY_NL(nl) ((struct pythonNestingLevelUserData *) nestingLevelGetUserData (nl)) + +static langType Lang_python; +static unsigned int TokenContinuationDepth = 0; +static tokenInfo *NextToken = NULL; +static NestingLevels *PythonNestingLevels = NULL; +static objPool *TokenPool = NULL; + + +/* follows PEP-8, and always reports single-underscores as protected + * See: + * - http://www.python.org/dev/peps/pep-0008/#method-names-and-instance-variables + * - http://www.python.org/dev/peps/pep-0008/#designing-for-inheritance + */ +static accessType accessFromIdentifier (const vString *const ident, + pythonKind kind, int parentKind) +{ + const char *const p = vStringValue (ident); + const size_t len = vStringLength (ident); + + /* inside a function/method, private */ + if (parentKind != -1 && parentKind != K_CLASS) + return ACCESS_PRIVATE; + /* not starting with "_", public */ + else if (len < 1 || p[0] != '_') + return ACCESS_PUBLIC; + /* "__...__": magic methods */ + else if (kind == K_FUNCTION && parentKind == K_CLASS && + len > 3 && p[1] == '_' && p[len - 2] == '_' && p[len - 1] == '_') + return ACCESS_PUBLIC; + /* "__...": name mangling */ + else if (parentKind == K_CLASS && len > 1 && p[1] == '_') + return ACCESS_PRIVATE; + /* "_...": suggested as non-public, but easily accessible */ + else + return ACCESS_PROTECTED; +} + +static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token, + const pythonKind kind) +{ + accessType access; + int parentKind = -1; + NestingLevel *nl; + + initTagEntry (e, vStringValue (token->string), kind); + + e->lineNumber = token->lineNumber; + e->filePosition = token->filePosition; + + nl = nestingLevelsGetCurrent (PythonNestingLevels); + if (nl) + { + tagEntryInfo *nlEntry = getEntryOfNestingLevel (nl); + + e->extensionFields.scopeIndex = nl->corkIndex; + + /* nlEntry can be NULL if a kind was disabled. But what can we do + * here? Even disabled kinds should count for the hierarchy I + * guess -- as it'd otherwise be wrong -- but with cork we're + * fucked up as there's nothing to look up. Damn. */ + if (nlEntry) + { + parentKind = nlEntry->kindIndex; + + /* functions directly inside classes are methods, fix it up */ + if (kind == K_FUNCTION && parentKind == K_CLASS) + e->kindIndex = K_METHOD; + } + } + + access = accessFromIdentifier (token->string, kind, parentKind); + e->extensionFields.access = PythonAccesses[access]; + /* FIXME: should we really set isFileScope in addition to access? */ + if (access == ACCESS_PRIVATE) + e->isFileScope = true; +} + +static int makeClassTag (const tokenInfo *const token, + const vString *const inheritance, + const vString *const decorators) +{ + if (PythonKinds[K_CLASS].enabled) + { + tagEntryInfo e; + + initPythonEntry (&e, token, K_CLASS); + + e.extensionFields.inheritance = inheritance ? vStringValue (inheritance) : ""; + if (decorators && vStringLength (decorators) > 0) + { + attachParserField (&e, false, PythonFields[F_DECORATORS].ftype, + vStringValue (decorators)); + } + + return makeTagEntry (&e); + } + + return CORK_NIL; +} + +static int makeFunctionTag (const tokenInfo *const token, + const vString *const arglist, + const vString *const decorators) +{ + if (PythonKinds[K_FUNCTION].enabled) + { + tagEntryInfo e; + + initPythonEntry (&e, token, K_FUNCTION); + + if (arglist) + e.extensionFields.signature = vStringValue (arglist); + if (decorators && vStringLength (decorators) > 0) + { + attachParserField (&e, false, PythonFields[F_DECORATORS].ftype, + vStringValue (decorators)); + } + + return makeTagEntry (&e); + } + + return CORK_NIL; +} + +static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const kind) +{ + if (PythonKinds[kind].enabled) + { + tagEntryInfo e; + + initPythonEntry (&e, token, kind); + return makeTagEntry (&e); + } + + return CORK_NIL; +} + +static int makeSimplePythonRefTag (const tokenInfo *const token, + const vString *const altName, + pythonKind const kind, + int roleIndex, xtagType xtag) +{ + if (isXtagEnabled (XTAG_REFERENCE_TAGS) && + PythonKinds[kind].roles[roleIndex].enabled) + { + tagEntryInfo e; + + initRefTagEntry (&e, vStringValue (altName ? altName : token->string), + kind, roleIndex); + + e.lineNumber = token->lineNumber; + e.filePosition = token->filePosition; + + if (xtag != XTAG_UNKNOWN) + markTagExtraBit (&e, xtag); + + return makeTagEntry (&e); + } + + return CORK_NIL; +} + +static void *newPoolToken (void *createArg CTAGS_ATTR_UNUSED) +{ + tokenInfo *token = xMalloc (1, tokenInfo); + token->string = vStringNew (); + return token; +} + +static void deletePoolToken (void *data) +{ + tokenInfo *token = data; + vStringDelete (token->string); + eFree (token); +} + +static void clearPoolToken (void *data) +{ + tokenInfo *token = data; + + token->type = TOKEN_UNDEFINED; + token->keyword = KEYWORD_NONE; + token->indent = 0; + token->lineNumber = getInputLineNumber (); + token->filePosition = getInputFilePosition (); + vStringClear (token->string); +} + +static void copyToken (tokenInfo *const dest, const tokenInfo *const src) +{ + dest->lineNumber = src->lineNumber; + dest->filePosition = src->filePosition; + dest->type = src->type; + dest->keyword = src->keyword; + dest->indent = src->indent; + vStringCopy(dest->string, src->string); +} + +/* Skip a single or double quoted string. */ +static void readString (vString *const string, const int delimiter) +{ + int escaped = 0; + int c; + + while ((c = getcFromInputFile ()) != EOF) + { + if (escaped) + { + vStringPut (string, c); + escaped--; + } + else if (c == '\') + escaped++; + else if (c == delimiter || c == '\n' || c == '\r') + { + if (c != delimiter) + ungetcToInputFile (c); + break; + } + else + vStringPut (string, c); + } +} + +/* Skip a single or double triple quoted string. */ +static void readTripleString (vString *const string, const int delimiter) +{ + int c; + int escaped = 0; + int n = 0; + while ((c = getcFromInputFile ()) != EOF) + { + if (c == delimiter && ! escaped) + { + if (++n >= 3) + break; + } + else + { + for (; n > 0; n--) + vStringPut (string, delimiter); + if (c != '\' || escaped) + vStringPut (string, c); + n = 0; + } + + if (escaped) + escaped--; + else if (c == '\') + escaped++; + } +} + +static void readIdentifier (vString *const string, const int firstChar) +{ + int c = firstChar; + do + { + vStringPut (string, (char) c); + c = getcFromInputFile (); + } + while (isIdentifierChar (c)); + ungetcToInputFile (c); +} + +static void ungetToken (tokenInfo *const token) +{ + Assert (NextToken == NULL); + NextToken = newToken (); + copyToken (NextToken, token); +} + +static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) +{ + int c; + int n; + + /* if we've got a token held back, emit it */ + if (NextToken) + { + copyToken (token, NextToken); + deleteToken (NextToken); + NextToken = NULL; + return; + } + + token->type = TOKEN_UNDEFINED; + token->keyword = KEYWORD_NONE; + vStringClear (token->string); + +getNextChar: + + n = 0; + do + { + c = getcFromInputFile (); + n++; + } + while (c == ' ' || c == '\t' || c == '\f'); + + token->lineNumber = getInputLineNumber (); + token->filePosition = getInputFilePosition (); + + if (inclWhitespaces && n > 1 && c != '\r' && c != '\n') + { + ungetcToInputFile (c); + vStringPut (token->string, ' '); + token->type = TOKEN_WHITESPACE; + return; + } + + switch (c) + { + case EOF: + token->type = TOKEN_EOF; + break; + + case ''': + case '"': + { + int d = getcFromInputFile (); + token->type = TOKEN_STRING; + vStringPut (token->string, c); + if (d != c) + { + ungetcToInputFile (d); + readString (token->string, c); + } + else if ((d = getcFromInputFile ()) == c) + readTripleString (token->string, c); + else /* empty string */ + ungetcToInputFile (d); + vStringPut (token->string, c); + token->lineNumber = getInputLineNumber (); + token->filePosition = getInputFilePosition (); + break; + } + + case '=': + { + int d = getcFromInputFile (); + vStringPut (token->string, c); + if (d == c) + { + vStringPut (token->string, d); + token->type = TOKEN_OPERATOR; + } + else + { + ungetcToInputFile (d); + token->type = c; + } + break; + } + + case '-': + { + int d = getcFromInputFile (); + if (d == '>') + { + vStringPut (token->string, c); + vStringPut (token->string, d); + token->type = TOKEN_ARROW; + break; + } + ungetcToInputFile (d); + /* fall through */ + } + case '+': + case '*': + case '%': + case '<': + case '>': + case '/': + { + int d = getcFromInputFile (); + vStringPut (token->string, c); + if (d != '=') + { + ungetcToInputFile (d); + token->type = c; + } + else + { + vStringPut (token->string, d); + token->type = TOKEN_OPERATOR; + } + break; + } + + /* eats newline to implement line continuation */ + case '\': + { + int d = getcFromInputFile (); + if (d == '\r') + d = getcFromInputFile (); + if (d != '\n') + ungetcToInputFile (d); + goto getNextChar; + } + + case '#': /* comment */ + case '\r': /* newlines for indent */ + case '\n': + { + int indent = 0; + do + { + if (c == '#') + { + do + c = getcFromInputFile (); + while (c != EOF && c != '\r' && c != '\n'); + } + if (c == '\r') + { + int d = getcFromInputFile (); + if (d != '\n') + ungetcToInputFile (d); + } + indent = 0; + while ((c = getcFromInputFile ()) == ' ' || c == '\t' || c == '\f') + { + if (c == '\t') + indent += 8 - (indent % 8); + else if (c == '\f') /* yeah, it's weird */ + indent = 0; + else + indent++; + } + } /* skip completely empty lines, so retry */ + while (c == '\r' || c == '\n' || c == '#'); + ungetcToInputFile (c); + if (TokenContinuationDepth > 0) + { + if (inclWhitespaces) + { + vStringPut (token->string, ' '); + token->type = TOKEN_WHITESPACE; + } + else + goto getNextChar; + } + else + { + token->type = TOKEN_INDENT; + token->indent = indent; + } + break; + } + + default: + if (! isIdentifierChar (c)) + { + vStringPut (token->string, c); + token->type = c; + } + else + { + /* FIXME: handle U, B, R and F string prefixes? */ + readIdentifier (token->string, c); + token->keyword = lookupKeyword (vStringValue (token->string), Lang_python); + if (token->keyword == KEYWORD_NONE) + token->type = TOKEN_IDENTIFIER; + else + token->type = TOKEN_KEYWORD; + } + break; + } + + /* handle implicit continuation lines not to emit INDENT inside brackets + * https://docs.python.org/3.6/reference/lexical_analysis.html#implicit-line-jo... */ + if (token->type == '(' || + token->type == '{' || + token->type == '[') + { + TokenContinuationDepth ++; + } + else if (TokenContinuationDepth > 0 && + (token->type == ')' || + token->type == '}' || + token->type == ']')) + { + TokenContinuationDepth --; + } +} + +static void readToken (tokenInfo *const token) +{ + readTokenFull (token, false); +} + +/*================================= parsing =================================*/ + + +static void reprCat (vString *const repr, const tokenInfo *const token) +{ + if (token->type != TOKEN_INDENT && + token->type != TOKEN_WHITESPACE) + { + vStringCat (repr, token->string); + } + else if (vStringLength (repr) > 0 && vStringLast (repr) != ' ') + { + vStringPut (repr, ' '); + } +} + +static bool skipOverPair (tokenInfo *const token, int tOpen, int tClose, + vString *const repr, bool reprOuterPair) +{ + if (token->type == tOpen) + { + int depth = 1; + + if (repr && reprOuterPair) + reprCat (repr, token); + do + { + readTokenFull (token, true); + if (repr && (reprOuterPair || token->type != tClose || depth > 1)) + { + reprCat (repr, token); + } + if (token->type == tOpen) + depth ++; + else if (token->type == tClose) + depth --; + } + while (token->type != TOKEN_EOF && depth > 0); + } + + return token->type == tClose; +} + +static bool skipLambdaArglist (tokenInfo *const token, vString *const repr) +{ + while (token->type != TOKEN_EOF && token->type != ':' && + /* avoid reading too much, just in case */ + token->type != TOKEN_INDENT) + { + bool readNext = true; + + if (token->type == '(') + readNext = skipOverPair (token, '(', ')', repr, true); + else if (token->type == '[') + readNext = skipOverPair (token, '[', ']', repr, true); + else if (token->type == '{') + readNext = skipOverPair (token, '{', '}', repr, true); + else if (token->keyword == KEYWORD_lambda) + { /* handle lambdas in a default value */ + if (repr) + reprCat (repr, token); + readTokenFull (token, true); + readNext = skipLambdaArglist (token, repr); + if (token->type == ':') + readNext = true; + if (readNext && repr) + reprCat (repr, token); + } + else if (repr) + { + reprCat (repr, token); + } + + if (readNext) + readTokenFull (token, true); + } + return false; +} + +static void readQualifiedName (tokenInfo *const nameToken) +{ + readToken (nameToken); + + if (nameToken->type == TOKEN_IDENTIFIER || + nameToken->type == '.') + { + vString *qualifiedName = vStringNew (); + tokenInfo *token = newToken (); + + while (nameToken->type == TOKEN_IDENTIFIER || + nameToken->type == '.') + { + vStringCat (qualifiedName, nameToken->string); + copyToken (token, nameToken); + + readToken (nameToken); + } + /* put the last, non-matching, token back */ + ungetToken (nameToken); + + copyToken (nameToken, token); + nameToken->type = TOKEN_IDENTIFIER; + vStringCopy (nameToken->string, qualifiedName); + + deleteToken (token); + vStringDelete (qualifiedName); + } +} + +static bool readCDefName (tokenInfo *const token, pythonKind *kind) +{ + readToken (token); + + if (token->keyword == KEYWORD_extern || + token->keyword == KEYWORD_import) + { + readToken (token); + if (token->keyword == KEYWORD_from) + return false; + } + + if (token->keyword == KEYWORD_class) + { + *kind = K_CLASS; + readToken (token); + } + else + { + /* skip the optional type declaration -- everything on the same line + * until an identifier followed by "(". */ + tokenInfo *candidate = newToken (); + + while (token->type != TOKEN_EOF && + token->type != TOKEN_INDENT && + token->type != '=' && + token->type != ',' && + token->type != ':') + { + if (token->type == '[') + { + if (skipOverPair (token, '[', ']', NULL, false)) + readToken (token); + } + else if (token->type == '(') + { + if (skipOverPair (token, '(', ')', NULL, false)) + readToken (token); + } + else if (token->type == TOKEN_IDENTIFIER) + { + copyToken (candidate, token); + readToken (token); + if (token->type == '(') + { /* okay, we really found a function, use this */ + *kind = K_FUNCTION; + ungetToken (token); + copyToken (token, candidate); + break; + } + } + else + readToken (token); + } + + deleteToken (candidate); + } + + return token->type == TOKEN_IDENTIFIER; +} + +static vString *parseParamTypeAnnotation (tokenInfo *const token, + vString *arglist) +{ + readToken (token); + if (token->type != ':') + { + ungetToken (token); + return NULL; + } + + reprCat (arglist, token); + int depth = 0; + vString *t = vStringNew (); + while (true) + { + readTokenFull (token, true); + if (token->type == TOKEN_WHITESPACE) + { + reprCat (arglist, token); + continue; + } + else if (token->type == TOKEN_EOF) + break; + + if (token->type == '(' || + token->type == '[' || + token->type == '{') + depth ++; + else if (token->type == ')' || + token->type == ']' || + token->type == '}') + depth --; + + if (depth < 0 + || (depth == 0 && (token->type == '=' + || token->type == ','))) + { + ungetToken (token); + return t; + } + reprCat (arglist, token); + reprCat (t, token); + } + vStringDelete (t); + return NULL; +} + +static vString *parseReturnTypeAnnotation (tokenInfo *const token) +{ + readToken (token); + if (token->type != TOKEN_ARROW) + { + ungetToken (token); + return NULL; + } + + int depth = 0; + vString *t = vStringNew (); + while (true) + { + readToken (token); + if (token->type == TOKEN_EOF) + break; + + if (token->type == '(' || + token->type == '[' || + token->type == '{') + depth ++; + else if (token->type == ')' || + token->type == ']' || + token->type == '}') + depth --; + if (depth == 0 && token->type == ':') + { + ungetToken (token); + return t; + } + else + reprCat (t, token); + } + vStringDelete (t); + return NULL; +} + +static bool parseClassOrDef (tokenInfo *const token, + const vString *const decorators, + pythonKind kind, bool isCDef) +{ + vString *arglist = NULL; + tokenInfo *name = NULL; + tokenInfo *parameterTokens[16] = { NULL }; + vString *parameterTypes [ARRAY_SIZE(parameterTokens)] = { NULL }; + unsigned int parameterCount = 0; + NestingLevel *lv; + int corkIndex; + + if (isCDef) + { + if (! readCDefName (token, &kind)) + return false; + } + else + { + readToken (token); + if (token->type != TOKEN_IDENTIFIER) + return false; + } + + name = newToken (); + copyToken (name, token); + + readToken (token); + /* collect parameters or inheritance */ + if (token->type == '(') + { + int prevTokenType = token->type; + int depth = 1; + + arglist = vStringNew (); + if (kind != K_CLASS) + reprCat (arglist, token); + + do + { + if (token->type != TOKEN_WHITESPACE && + /* for easy `*args` and `**kwargs` support, we also ignore + * `*`, which anyway can't otherwise screw us up */ + token->type != '*') + { + prevTokenType = token->type; + } + + readTokenFull (token, true); + if (kind != K_CLASS || token->type != ')' || depth > 1) + reprCat (arglist, token); + + if (token->type == '(' || + token->type == '[' || + token->type == '{') + depth ++; + else if (token->type == ')' || + token->type == ']' || + token->type == '}') + depth --; + else if (kind != K_CLASS && depth == 1 && + token->type == TOKEN_IDENTIFIER && + (prevTokenType == '(' || prevTokenType == ',') && + parameterCount < ARRAY_SIZE (parameterTokens) && + PythonKinds[K_PARAMETER].enabled) + { + tokenInfo *parameterName = newToken (); + + copyToken (parameterName, token); + parameterTokens[parameterCount] = parameterName; + parameterTypes [parameterCount++] = parseParamTypeAnnotation (token, arglist); + } + } + while (token->type != TOKEN_EOF && depth > 0); + } + + if (kind == K_CLASS) + corkIndex = makeClassTag (name, arglist, decorators); + else + corkIndex = makeFunctionTag (name, arglist, decorators); + + lv = nestingLevelsPush (PythonNestingLevels, corkIndex); + PY_NL (lv)->indentation = token->indent; + + deleteToken (name); + vStringDelete (arglist); + + if (parameterCount > 0) + { + unsigned int i; + + for (i = 0; i < parameterCount; i++) + { + int paramCorkIndex = makeSimplePythonTag (parameterTokens[i], K_PARAMETER); + deleteToken (parameterTokens[i]); + tagEntryInfo *e = getEntryInCorkQueue (paramCorkIndex); + if (e && parameterTypes[i]) + { + e->extensionFields.typeRef [0] = eStrdup ("typename"); + e->extensionFields.typeRef [1] = vStringDeleteUnwrap (parameterTypes[i]); + parameterTypes[i] = NULL; + } + vStringDelete (parameterTypes[i]); /* NULL is acceptable. */ + } + } + + tagEntryInfo *e; + vString *t; + if (kind != K_CLASS + && (e = getEntryInCorkQueue (corkIndex)) + && (t = parseReturnTypeAnnotation (token))) + { + e->extensionFields.typeRef [0] = eStrdup ("typename"); + e->extensionFields.typeRef [1] = vStringDeleteUnwrap (t); + } + + return true; +} + +static bool parseImport (tokenInfo *const token) +{ + tokenInfo *fromModule = NULL; + + if (token->keyword == KEYWORD_from) + { + readQualifiedName (token); + if (token->type == TOKEN_IDENTIFIER) + { + fromModule = newToken (); + copyToken (fromModule, token); + readToken (token); + } + } + + if (token->keyword == KEYWORD_import) + { + bool parenthesized = false; + int moduleIndex; + + if (fromModule) + { + /* from X import ... + * -------------------- + * X = (kind:module, role:namespace) */ + moduleIndex = makeSimplePythonRefTag (fromModule, NULL, K_MODULE, + PYTHON_MODULE_NAMESPACE, + XTAG_UNKNOWN); + } + + do + { + readQualifiedName (token); + + /* support for `from x import (...)` */ + if (fromModule && ! parenthesized && token->type == '(') + { + parenthesized = true; + readQualifiedName (token); + } + + if (token->type == TOKEN_IDENTIFIER) + { + tokenInfo *name = newToken (); + + copyToken (name, token); + readToken (token); + /* if there is an "as", use it as the name */ + if (token->keyword == KEYWORD_as) + { + readToken (token); + if (token->type == TOKEN_IDENTIFIER) + { + if (fromModule) + { + /* from x import Y as Z + * ---------------------------- + * x = (kind:module, role:namespace), + * Y = (kind:unknown, role:indirectlyImported, scope:module:X), + * Z = (kind:unknown, nameref:unknown:Y) */ + int index; + + /* Y */ + index = makeSimplePythonRefTag (name, NULL, K_UNKNOWN, + PYTHON_UNKNOWN_INDIRECTLY_IMPORTED, + XTAG_UNKNOWN); + /* fill the scope field for Y */ + tagEntryInfo *e = getEntryInCorkQueue (index); + if (e) + e->extensionFields.scopeIndex = moduleIndex; + + /* Z */ + index = makeSimplePythonTag (token, K_UNKNOWN); + /* fill the nameref filed for Y */ + if (PythonFields[F_NAMEREF].enabled) + { + vString *nameref = vStringNewInit (PythonKinds [K_UNKNOWN].name); + vStringPut (nameref, ':'); + vStringCat (nameref, name->string); + attachParserFieldToCorkEntry (index, PythonFields[F_NAMEREF].ftype, + vStringValue (nameref)); + vStringDelete (nameref); + } + } + else + { + /* import x as Y + * ---------------------------- + * x = (kind:module, role:indirectlyImported) + * Y = (kind:namespace, nameref:module:x)*/ + /* x */ + makeSimplePythonRefTag (name, NULL, K_MODULE, + PYTHON_MODULE_INDIRECTLY_IMPORTED, + XTAG_UNKNOWN); + /* Y */ + int index = makeSimplePythonTag (token, K_NAMESPACE); + /* fill the nameref filed for Y */ + if (PythonFields[F_NAMEREF].enabled) + { + vString *nameref = vStringNewInit (PythonKinds [K_MODULE].name); + vStringPut (nameref, ':'); + vStringCat (nameref, name->string); + attachParserFieldToCorkEntry (index, PythonFields[F_NAMEREF].ftype, + vStringValue (nameref)); + vStringDelete (nameref); + } + } + + copyToken (name, token); + readToken (token); + } + } + else + { + if (fromModule) + { + /* from x import Y + -------------- + x = (kind:module, role:namespace), + Y = (kind:unknown, role:imported, scope:module:x) */ + /* Y */ + int index = makeSimplePythonRefTag (name, NULL, K_UNKNOWN, + PYTHON_UNKNOWN_IMPORTED, + XTAG_UNKNOWN); + /* fill the scope field for Y */ + tagEntryInfo *e = getEntryInCorkQueue (index); + if (e) + e->extensionFields.scopeIndex = moduleIndex; + } + else + { + /* import X + -------------- + X = (kind:module, role:imported) */ + makeSimplePythonRefTag (name, NULL, K_MODULE, + PYTHON_MODULE_IMPORTED, + XTAG_UNKNOWN); + } + } + + deleteToken (name); + } + } + while (token->type == ','); + + if (parenthesized && token->type == ')') + readToken (token); + } + + if (fromModule) + deleteToken (fromModule); + + return false; +} + +/* this only handles the most common cases, but an annotation can be any + * expression in theory. + * this function assumes there must be an annotation, and doesn't do any check + * on the token on which it is called: the caller should do that part. */ +static bool skipVariableTypeAnnotation (tokenInfo *const token, vString *const repr) +{ + bool readNext = true; + + readToken (token); + switch (token->type) + { + case '[': readNext = skipOverPair (token, '[', ']', repr, true); break; + case '(': readNext = skipOverPair (token, '(', ')', repr, true); break; + case '{': readNext = skipOverPair (token, '{', '}', repr, true); break; + default: reprCat (repr, token); + } + if (readNext) + readToken (token); + /* skip subscripts and calls */ + while (token->type == '[' || token->type == '(' || token->type == '.' || token->type == '|') + { + switch (token->type) + { + case '[': readNext = skipOverPair (token, '[', ']', repr, true); break; + case '(': readNext = skipOverPair (token, '(', ')', repr, true); break; + case '|': + reprCat (repr, token); + skipVariableTypeAnnotation (token, repr); + readNext = false; + break; + case '.': + reprCat (repr, token); + readToken (token); + readNext = token->type == TOKEN_IDENTIFIER; + if (readNext) + reprCat (repr, token); + break; + default: readNext = false; break; + } + if (readNext) + readToken (token); + } + + return false; +} + +static bool parseVariable (tokenInfo *const token, const pythonKind kind) +{ + /* In order to support proper tag type for lambdas in multiple + * assignations, we first collect all the names, and then try and map + * an assignation to it */ + tokenInfo *nameTokens[8] = { NULL }; + vString *nameTypes [ARRAY_SIZE (nameTokens)] = { NULL }; + unsigned int nameCount = 0; + vString *type = vStringNew(); + + /* first, collect variable name tokens */ + while (token->type == TOKEN_IDENTIFIER && + nameCount < ARRAY_SIZE (nameTokens)) + { + unsigned int i; + tokenInfo *name = newToken (); + copyToken (name, token); + + readToken (token); + if (token->type == '.') + { + /* FIXME: what to do with dotted names? We currently ignore them + * as we need to do something not to break the whole + * declaration, but the expected behavior is questionable */ + deleteToken (name); + name = NULL; + + do + { + readToken (token); + } + while (token->type == TOKEN_IDENTIFIER || + token->type == '.'); + } + + i = nameCount++; + nameTokens[i] = name; + + /* (parse and) skip annotations. we need not to be too permissive because we + * aren't yet sure we're actually parsing a variable. */ + if (token->type == ':' && skipVariableTypeAnnotation (token, type)) + readToken (token); + + if (vStringLength (type) > 0) + { + nameTypes[i] = type; + type = vStringNew (); + } + + if (token->type == ',') + readToken (token); + else + break; + } + vStringDelete (type); + + /* then, if it's a proper assignation, try and map assignations so that + * we catch lambdas and alike */ + if (token->type == '=') + { + unsigned int i = 0; + + do + { + const tokenInfo *const nameToken = nameTokens[i]; + vString **type = &(nameTypes[i++]); + + readToken (token); + + if (! nameToken) + /* nothing */; + else if (token->keyword != KEYWORD_lambda) + { + int index = makeSimplePythonTag (nameToken, kind); + tagEntryInfo *e = getEntryInCorkQueue (index); + if (e && *type) + { + e->extensionFields.typeRef [0] = eStrdup ("typename"); + e->extensionFields.typeRef [1] = vStringDeleteUnwrap (*type); + *type = NULL; + } + } + else + { + tokenInfo *anon = NULL; + vString *arglist = vStringNew (); + if (*type) + { + anon = newToken (); + copyToken (anon, token); + } + readToken (token); + vStringPut (arglist, '('); + skipLambdaArglist (token, arglist); + vStringPut (arglist, ')'); + if (*type) + { + /* How to handle lambda assigned to a variable + * -------------------------------------------- + * + * input.py: + * + * id = lambda var: var + * id_t: Callable[[int], int] = lambda var: var + * + * `id' is tagged as a function kind object like: + * + * id input.py /^id = lambda var: var$/;" function + * + * For `id_t' we cannot do the same as `id'. + * + * We should not store `Callable[[int], int]' to typeref + * field of the tag of `id_t' if the tag has "function" as + * its kind because users expect the typeref field of a + * function kind represents a type for the value returned + * from the function (return type). + * + * the unexpected tag: + * + * id_t input.py /^id_t: Callable[[int], int] = lambda var: var$/;" function \ + * typeref:typename:Callable[[int], int] + * + * If we make a tag for `id_t' as a function, we should + * attach `typeref:typename:int' and `signature:(int)'. To + * achieve this, we have to make ctags analyze + * `Callable[[int], int]'. However, we want to avoid the + * level of analyzing. + * + * For recording `Callable[[int], int]', a valuable + * information in the input, we use indirection. + * + * id_t input.py /^id_t: Callable[[int], int] = lambda var: var$/;" variable \ + * typeref:typename:Callable[[int], int] nameref:function:anonFuncNNN + * anonFuncNNN input.py /^id_t: Callable[[int], int] = lambda var: var$/;" function \ + * extras:anonymous + */ + int vindex = makeSimplePythonTag (nameToken, kind); + vStringClear (anon->string); + anonGenerate (anon->string, "anonFunc", K_FUNCTION); + int findex = makeFunctionTag (anon, arglist, NULL); + tagEntryInfo *fe = getEntryInCorkQueue (findex); + if (fe) + markTagExtraBit (fe, XTAG_ANONYMOUS); + + tagEntryInfo *ve = getEntryInCorkQueue (vindex); + if (ve) + { + ve->extensionFields.typeRef [0] = eStrdup ("typename"); + ve->extensionFields.typeRef [1] = vStringDeleteUnwrap (*type); + *type = NULL; + vString *nameref = vStringNewInit (PythonKinds [K_FUNCTION].name); + vStringPut (nameref, ':'); + vStringCat (nameref, anon->string); + attachParserField (ve, true, PythonFields[F_NAMEREF].ftype, + vStringValue (nameref)); + vStringDelete (nameref); + } + if (anon) + deleteToken (anon); + } + else + makeFunctionTag (nameToken, arglist, NULL); + vStringDelete (arglist); + } + + /* skip until next initializer */ + while ((TokenContinuationDepth > 0 || token->type != ',') && + token->type != TOKEN_EOF && + token->type != ';' && + token->type != TOKEN_INDENT) + { + readToken (token); + } + } + while (token->type == ',' && i < nameCount); + + /* if we got leftover to initialize, just make variables out of them. + * This handles cases like `a, b, c = (c, d, e)` -- or worse */ + for (; i < nameCount; i++) + { + if (nameTokens[i]) + makeSimplePythonTag (nameTokens[i], kind); + } + } + + while (nameCount > 0) + { + if (nameTokens[--nameCount]) + deleteToken (nameTokens[nameCount]); + vStringDelete (nameTypes[nameCount]); /* NULL is acceptable. */ + } + + return false; +} + +/* pops any level >= to indent */ +static void setIndent (tokenInfo *const token) +{ + NestingLevel *lv = nestingLevelsGetCurrent (PythonNestingLevels); + + while (lv && PY_NL (lv)->indentation >= token->indent) + { + tagEntryInfo *e = getEntryInCorkQueue (lv->corkIndex); + if (e) + e->extensionFields.endLine = token->lineNumber; + + nestingLevelsPop (PythonNestingLevels); + lv = nestingLevelsGetCurrent (PythonNestingLevels); + } +} + +static void findPythonTags (void) +{ + tokenInfo *const token = newToken (); + vString *decorators = vStringNew (); + bool atStatementStart = true; + + TokenContinuationDepth = 0; + NextToken = NULL; + PythonNestingLevels = nestingLevelsNew (sizeof (struct pythonNestingLevelUserData)); + + readToken (token); + while (token->type != TOKEN_EOF) + { + tokenType iterationTokenType = token->type; + bool readNext = true; + + /* skip async keyword that confuses decorator parsing before a def */ + if (token->keyword == KEYWORD_async) + readToken (token); + + if (token->type == TOKEN_INDENT) + setIndent (token); + else if (token->keyword == KEYWORD_class || + token->keyword == KEYWORD_def) + { + pythonKind kind = token->keyword == KEYWORD_class ? K_CLASS : K_FUNCTION; + + readNext = parseClassOrDef (token, decorators, kind, false); + } + else if (token->keyword == KEYWORD_cdef || + token->keyword == KEYWORD_cpdef) + { + readNext = parseClassOrDef (token, decorators, K_FUNCTION, true); + } + else if (token->keyword == KEYWORD_from || + token->keyword == KEYWORD_import) + { + readNext = parseImport (token); + } + else if (token->type == '(') + { /* skip parentheses to avoid finding stuff inside them */ + readNext = skipOverPair (token, '(', ')', NULL, false); + } + else if (token->type == TOKEN_IDENTIFIER && atStatementStart) + { + NestingLevel *lv = nestingLevelsGetCurrent (PythonNestingLevels); + tagEntryInfo *lvEntry = getEntryOfNestingLevel (lv); + pythonKind kind = K_VARIABLE; + + if (lvEntry && lvEntry->kindIndex != K_CLASS) + kind = K_LOCAL_VARIABLE; + + readNext = parseVariable (token, kind); + } + else if (token->type == '@' && atStatementStart && + PythonFields[F_DECORATORS].enabled) + { + /* collect decorators */ + readQualifiedName (token); + if (token->type != TOKEN_IDENTIFIER) + readNext = false; + else + { + if (vStringLength (decorators) > 0) + vStringPut (decorators, ','); + vStringCat (decorators, token->string); + readToken (token); + readNext = skipOverPair (token, '(', ')', decorators, true); + } + } + + /* clear collected decorators for any non-decorator tokens non-indent + * token. decorator collection takes care of skipping the possible + * argument list, so we should never hit here parsing a decorator */ + if (iterationTokenType != TOKEN_INDENT && + iterationTokenType != '@' && + PythonFields[F_DECORATORS].enabled) + { + vStringClear (decorators); + } + + atStatementStart = (token->type == TOKEN_INDENT || token->type == ';'); + + if (readNext) + readToken (token); + } + + nestingLevelsFree (PythonNestingLevels); + vStringDelete (decorators); + deleteToken (token); + Assert (NextToken == NULL); +} + +static void initialize (const langType language) +{ + Lang_python = language; + + TokenPool = objPoolNew (16, newPoolToken, deletePoolToken, clearPoolToken, NULL); +} + +static void finalize (langType language CTAGS_ATTR_UNUSED, bool initialized) +{ + if (!initialized) + return; + + objPoolDelete (TokenPool); +} + +extern parserDefinition* PythonParser (void) +{ + static const char *const extensions[] = { "py", "pyx", "pxd", "pxi", "scons", + "wsgi", NULL }; + static const char *const aliases[] = { "python[23]*", "scons", NULL }; + parserDefinition *def = parserNew ("Python"); + def->kindTable = PythonKinds; + def->kindCount = ARRAY_SIZE (PythonKinds); + def->extensions = extensions; + def->aliases = aliases; + def->parser = findPythonTags; + def->initialize = initialize; + def->finalize = finalize; + def->keywordTable = PythonKeywordTable; + def->keywordCount = ARRAY_SIZE (PythonKeywordTable); + def->fieldTable = PythonFields; + def->fieldCount = ARRAY_SIZE (PythonFields); + def->useCork = CORK_QUEUE; + def->requestAutomaticFQTag = true; + return def; +}
Modified: src/tagmanager/tm_parser.c 4 lines changed, 4 insertions(+), 0 deletions(-) =================================================================== @@ -136,9 +136,13 @@ static TMParserMapEntry map_PYTHON[] = { {'f', tm_tag_function_t}, {'m', tm_tag_method_t}, {'v', tm_tag_variable_t}, + {'I', tm_tag_externvar_t}, + {'i', tm_tag_externvar_t}, /* defined as externvar to get those excluded as forward type in symbols.c:goto_tag() * so we can jump to the real implementation (if known) instead of to the import statement */ {'x', tm_tag_externvar_t}, + {'z', tm_tag_undef_t}, + {'l', tm_tag_undef_t}, };
/* different parser than tex.c from universal-ctags */
Modified: tests/ctags/cython_sample2.pyx.tags 1 lines changed, 1 insertions(+), 0 deletions(-) =================================================================== @@ -1,3 +1,4 @@ # format=tagmanager my_fun�16�(np.ndarray[dtype=double, ndim=1] x)�0 np�32768�0 +numpy�32768�0
Modified: tests/ctags/py_constructor_arglist.py.tags 33 lines changed, 21 insertions(+), 12 deletions(-) =================================================================== @@ -1,16 +1,25 @@ # format=tagmanager -HttpResponse�32768�0 -HttpResponseBadRequest�32768�0 -InterfaceDataValidationError�32768�0 -RequestContext�32768�0 +HttpResponse�32768�django.http�0 +HttpResponseBadRequest�32768�django.http�0 +InterfaceDataValidationError�32768�vnstat.api.error�0 +RequestContext�32768�django.template.context�0 SomeClass�1�0 __init__�128�(self, filename, pathsep='', treegap=64)�SomeClass�0 -btopen�32768�0 -csrf_exempt�32768�0 -csrf_exempt2�32768�0 -login_required�32768�0 -permission_required�32768�0 -render_to_response�32768�0 -require_POST�32768�0 -simplejson�32768�0 +bsddb�32768�0 +btopen�32768�bsddb�0 +csrf_exempt�32768�django.views.decorators.csrf�0 +csrf_exempt2�32768�django.views.decorators.csrf�0 +django.contrib.auth.decorators�32768�0 +django.http�32768�0 +django.shortcuts�32768�0 +django.template.context�32768�0 +django.utils�32768�0 +django.views.decorators.csrf�32768�0 +django.views.decorators.http�32768�0 +login_required�32768�django.contrib.auth.decorators�0 +permission_required�32768�django.contrib.auth.decorators�0 +render_to_response�32768�django.shortcuts�0 +require_POST�32768�django.views.decorators.http�0 +simplejson�32768�django.utils�0 sys�32768�0 +vnstat.api.error�32768�0
Modified: tests/ctags/simple.py.tags 6 lines changed, 4 insertions(+), 2 deletions(-) =================================================================== @@ -7,11 +7,13 @@ TREEID VERSION�16384�0 __init__�128�(self, filename, pathsep='', treegap=64)�one�0 __private_function__�128�(self, key, data)�one�0 +_pack�128�(self, i, s)�one�0 _test�16�(test, code, outcome, exception)�0 -btopen�32768�0 +bsddb�32768�0 +btopen�32768�bsddb�0 deeply_nested�1�_test.ignored_function.more_nesting�0 even_more�128�()�_test.ignored_function.more_nesting.deeply_nested�0 -foo�16�(�0 +foo�16�( x , y, z)�0 ignored_function�16�()�_test�0 more_nesting�16�()�_test.ignored_function�0 one�1�0
Modified: tests/ctags/tabindent.py.tags 2 lines changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -1,2 +1,2 @@ # format=tagmanager -find_heading�16�(self, position=0, direction=Direction.FORWARD, heading=Heading, connect_with_document=True)�0 +find_heading�16�(self, position=0, direction=Direction.FORWARD, heading=Heading, connect_with_document=True)�0
-------------- This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).