2 * Copyright (c) 2000-2003, Darren Hiebert
4 * This source code is released for free distribution under the terms of the
5 * GNU General Public License version 2 or (at your option) any later version.
7 * This module contains functions for generating tags for Python language
13 #include "general.h" /* must always come first */
18 #include "nestlevel.h"
35 struct nestingLevelUserData
{
38 #define PY_NL_INDENTATION(nl) ((struct nestingLevelUserData *)nestingLevelGetUserData(nl))->indentation
42 K_CLASS
, K_FUNCTION
, K_METHOD
, K_VARIABLE
, K_IMPORT
45 static kindDefinition PythonKinds
[] = {
46 {true, 'c', "class", "classes"},
47 {true, 'f', "function", "functions"},
48 {true, 'm', "member", "class members"},
49 {true, 'v', "variable", "variables"},
50 {true, 'x', "unknown", "name referring a classe/variable/function/module defined in other module"}
54 A_PUBLIC
, A_PRIVATE
, A_PROTECTED
57 static const char *const PythonAccesses
[] = {
58 "public", "private", "protected"
61 static char const * const singletriple
= "'''";
62 static char const * const doubletriple
= "\"\"\"";
65 * FUNCTION DEFINITIONS
68 static bool isIdentifierFirstCharacter (int c
)
70 return (bool) (isalpha (c
) || c
== '_');
73 static bool isIdentifierCharacter (int c
)
75 return (bool) (isalnum (c
) || c
== '_');
78 /* follows PEP-8, and always reports single-underscores as protected
80 * - http://www.python.org/dev/peps/pep-0008/#method-names-and-instance-variables
81 * - http://www.python.org/dev/peps/pep-0008/#designing-for-inheritance
83 static pythonAccess
accessFromIdentifier (const vString
*const ident
,
84 pythonKind kind
, bool has_parent
, bool parent_is_class
)
86 const char *const p
= vStringValue (ident
);
87 const size_t len
= vStringLength (ident
);
89 /* inside a function/method, private */
90 if (has_parent
&& !parent_is_class
)
92 /* not starting with "_", public */
93 else if (len
< 1 || p
[0] != '_')
95 /* "__...__": magic methods */
96 else if (kind
== K_METHOD
&& parent_is_class
&&
97 len
> 3 && p
[1] == '_' && p
[len
- 2] == '_' && p
[len
- 1] == '_')
99 /* "__...": name mangling */
100 else if (parent_is_class
&& len
> 1 && p
[1] == '_')
102 /* "_...": suggested as non-public, but easily accessible */
107 static void addAccessFields (tagEntryInfo
*const entry
,
108 const vString
*const ident
, pythonKind kind
,
109 bool has_parent
, bool parent_is_class
)
113 access
= accessFromIdentifier (ident
, kind
, has_parent
, parent_is_class
);
114 entry
->extensionFields
.access
= PythonAccesses
[access
];
115 /* FIXME: should we really set isFileScope in addition to access? */
116 if (access
== A_PRIVATE
)
117 entry
->isFileScope
= true;
120 /* Given a string with the contents of a line directly after the "def" keyword,
121 * extract all relevant information and create a tag.
123 static struct corkInfo
makeFunctionTag (vString
*const function
,
124 vString
*const parent
, int is_class_parent
, const char *arglist
)
128 struct corkInfo info
;
130 if (vStringLength (parent
) > 0)
134 initTagEntry (&tag
, vStringValue (function
), K_METHOD
);
135 tag
.extensionFields
.scopeKindIndex
= K_CLASS
;
139 initTagEntry (&tag
, vStringValue (function
), K_FUNCTION
);
140 tag
.extensionFields
.scopeKindIndex
= K_FUNCTION
;
142 tag
.extensionFields
.scopeName
= vStringValue (parent
);
145 initTagEntry (&tag
, vStringValue (function
), K_FUNCTION
);
147 tag
.extensionFields
.signature
= arglist
;
149 addAccessFields (&tag
, function
, is_class_parent
? K_METHOD
: K_FUNCTION
,
150 vStringLength (parent
) > 0, is_class_parent
);
152 corkIndex
= makeTagEntry (&tag
);
154 info
.index
= corkIndex
;
158 /* Given a string with the contents of the line directly after the "class"
159 * keyword, extract all necessary information and create a tag.
161 static int makeClassTag (vString
*const class, vString
*const inheritance
,
162 vString
*const parent
, int is_class_parent
)
165 initTagEntry (&tag
, vStringValue (class), K_CLASS
);
166 if (vStringLength (parent
) > 0)
170 tag
.extensionFields
.scopeKindIndex
= K_CLASS
;
171 tag
.extensionFields
.scopeName
= vStringValue (parent
);
175 tag
.extensionFields
.scopeKindIndex
= K_FUNCTION
;
176 tag
.extensionFields
.scopeName
= vStringValue (parent
);
179 tag
.extensionFields
.inheritance
= vStringValue (inheritance
);
180 addAccessFields (&tag
, class, K_CLASS
, vStringLength (parent
) > 0,
182 return makeTagEntry (&tag
);
185 static void makeVariableTag (vString
*const var
, vString
*const parent
,
186 bool is_class_parent
)
189 initTagEntry (&tag
, vStringValue (var
), K_VARIABLE
);
190 if (vStringLength (parent
) > 0)
192 tag
.extensionFields
.scopeKindIndex
= K_CLASS
;
193 tag
.extensionFields
.scopeName
= vStringValue (parent
);
195 addAccessFields (&tag
, var
, K_VARIABLE
, vStringLength (parent
) > 0,
200 /* Skip a single or double quoted string. */
201 static const char *skipString (const char *cp
)
203 const char *start
= cp
;
205 for (cp
++; *cp
; cp
++)
209 else if (*cp
== '\\')
211 else if (*cp
== *start
)
217 /* Skip everything up to an identifier start. */
218 static const char *skipEverything (const char *cp
)
224 return strchr(cp
, '\0');
227 if (*cp
== '"' || *cp
== '\'')
230 /* these checks find unicode, binary (Python 3) and raw strings */
233 bool r_first
= (*cp
== 'r' || *cp
== 'R');
235 /* "r" | "R" | "u" | "U" | "b" | "B" */
236 if (r_first
|| *cp
== 'u' || *cp
== 'U' || *cp
== 'b' || *cp
== 'B')
240 /* r_first -> "rb" | "rB" | "Rb" | "RB"
241 !r_first -> "ur" | "UR" | "Ur" | "uR" | "br" | "Br" | "bR" | "BR" */
242 if (( r_first
&& (cp
[i
] == 'b' || cp
[i
] == 'B')) ||
243 (!r_first
&& (cp
[i
] == 'r' || cp
[i
] == 'R')))
246 if (cp
[i
] == '\'' || cp
[i
] == '"')
258 if (isIdentifierFirstCharacter ((int) *cp
))
261 cp
--; /* avoid jumping over the character after a skipped string */
266 /* Skip an identifier. */
267 static const char *skipIdentifier (const char *cp
)
269 while (isIdentifierCharacter ((int) *cp
))
274 static const char *findDefinitionOrClass (const char *cp
)
278 cp
= skipEverything (cp
);
279 if (!strncmp(cp
, "def", 3) || !strncmp(cp
, "class", 5) ||
280 !strncmp(cp
, "cdef", 4) || !strncmp(cp
, "cpdef", 5))
284 cp
= skipIdentifier (cp
);
289 static const char *skipSpace (const char *cp
)
291 while (isspace ((int) *cp
))
296 /* Starting at ''cp'', parse an identifier into ''identifier''. */
297 static const char *parseIdentifier (const char *cp
, vString
*const identifier
)
299 vStringClear (identifier
);
300 while (isIdentifierCharacter ((int) *cp
))
302 vStringPut (identifier
, (int) *cp
);
308 static int parseClass (const char *cp
, vString
*const class,
309 vString
*const parent
, int is_class_parent
)
312 vString
*const inheritance
= vStringNew ();
313 vStringClear (inheritance
);
314 cp
= parseIdentifier (cp
, class);
323 /* Closing parenthesis can be in follow up line. */
324 cp
= (const char *) readLineFromInputFile ();
326 vStringPut (inheritance
, ' ');
329 vStringPut (inheritance
, *cp
);
333 corkIndex
= makeClassTag (class, inheritance
, parent
, is_class_parent
);
334 vStringDelete (inheritance
);
338 static void parseImports (const char *cp
)
341 vString
*name
, *name_next
;
343 cp
= skipEverything (cp
);
345 if ((pos
= strstr (cp
, "import")) == NULL
)
350 /* continue only if there is some space between the keyword and the identifier */
357 name
= vStringNew ();
358 name_next
= vStringNew ();
360 cp
= skipEverything (cp
);
363 cp
= parseIdentifier (cp
, name
);
365 cp
= skipEverything (cp
);
366 /* we parse the next possible import statement as well to be able to ignore 'foo' in
367 * 'import foo as bar' */
368 parseIdentifier (cp
, name_next
);
370 /* take the current tag only if the next one is not "as" */
371 if (strcmp (vStringValue (name_next
), "as") != 0 &&
372 strcmp (vStringValue (name
), "as") != 0)
374 makeSimpleTag (name
, K_IMPORT
);
377 vStringDelete (name
);
378 vStringDelete (name_next
);
381 /* modified from lcpp.c getArglistFromStr().
382 * warning: terminates rest of string past arglist!
383 * note: does not ignore brackets inside strings! */
384 static char *parseArglist(const char *buf
)
390 if (NULL
== (start
= strchr(buf
, '(')))
392 for (level
= 1, end
= start
+ 1; level
> 0; ++end
)
396 else if ('(' == *end
)
398 else if (')' == *end
)
402 return strdup(start
);
405 static struct corkInfo
parseFunction (const char *cp
, vString
*const def
,
406 vString
*const parent
, int is_class_parent
)
409 struct corkInfo info
;
411 cp
= parseIdentifier (cp
, def
);
412 arglist
= parseArglist (cp
);
413 info
= makeFunctionTag (def
, parent
, is_class_parent
, arglist
);
419 /* Get the combined name of a nested symbol. Classes are separated with ".",
420 * functions with "/". For example this code:
427 * Would produce this string:
428 * MyClass.MyFunction/SubFunction/SubClass.Method
430 static bool constructParentString(NestingLevels
*nls
, int indent
,
434 NestingLevel
*prev
= NULL
;
435 int is_class
= false;
436 vStringClear (result
);
437 for (i
= 0; i
< nls
->n
; i
++)
439 NestingLevel
*nl
= nestingLevelsGetNthFromRoot (nls
, i
);
442 if (indent
<= PY_NL_INDENTATION(nl
))
446 vStringCatS(result
, "."); /* make Geany symbol list grouping work properly */
448 if (prev->kindIndex == K_CLASS)
449 vStringCatS(result, ".");
451 vStringCatS(result, "/");
455 e
= getEntryOfNestingLevel (nl
);
458 vStringCatS(result
, e
->name
);
459 is_class
= (e
->kindIndex
== K_CLASS
);
469 /* Check indentation level and truncate nesting levels accordingly */
470 static void checkIndent(NestingLevels
*nls
, int indent
)
475 for (i
= 0; i
< nls
->n
; i
++)
477 n
= nestingLevelsGetNthFromRoot (nls
, i
);
478 if (n
&& indent
<= PY_NL_INDENTATION(n
))
480 /* truncate levels */
487 static void addNestingLevel(NestingLevels
*nls
, int indentation
, struct corkInfo
*info
)
490 NestingLevel
*nl
= NULL
;
492 for (i
= 0; i
< nls
->n
; i
++)
494 nl
= nestingLevelsGetNthFromRoot(nls
, i
);
495 if (indentation
<= PY_NL_INDENTATION(nl
)) break;
498 nl
= nestingLevelsPush(nls
, info
->index
);
500 /* reuse existing slot */
501 nl
= nestingLevelsTruncate (nls
, i
+ 1, info
->index
);
503 PY_NL_INDENTATION(nl
) = indentation
;
506 /* Return a pointer to the start of the next triple string, or NULL. Store
507 * the kind of triple string in "which" if the return is not NULL.
509 static char const *find_triple_start(char const *string
, char const **which
)
511 char const *cp
= string
;
517 if (*cp
== '"' || *cp
== '\'')
519 if (strncmp(cp
, doubletriple
, 3) == 0)
521 *which
= doubletriple
;
524 if (strncmp(cp
, singletriple
, 3) == 0)
526 *which
= singletriple
;
531 cp
--; /* avoid jumping over the character after a skipped string */
537 /* Find the end of a triple string as pointed to by "which", and update "which"
538 * with any other triple strings following in the given string.
540 static void find_triple_end(char const *string
, char const **which
)
542 char const *s
= string
;
545 /* Check if the string ends in the same line. */
546 s
= strstr (s
, *which
);
550 /* If yes, check if another one starts in the same line. */
551 s
= find_triple_start(s
, which
);
557 static const char *findVariable(const char *line
)
559 /* Parse global and class variable names (C.x) from assignment statements.
560 * Object attributes (obj.x) are ignored.
561 * Assignment to a tuple 'x, y = 2, 3' not supported.
562 * TODO: ignore duplicate tags from reassignment statements. */
563 const char *cp
, *sp
, *eq
, *start
;
565 cp
= strstr(line
, "=");
572 return NULL
; /* ignore '==' operator and 'x=5,y=6)' function lines */
573 if (*eq
== '(' || *eq
== '#')
574 break; /* allow 'x = func(b=2,y=2,' lines and comments at the end of line */
578 /* go backwards to the start of the line, checking we have valid chars */
580 while (start
>= line
&& isspace ((int) *start
))
582 while (start
>= line
&& isIdentifierCharacter ((int) *start
))
584 if (!isIdentifierFirstCharacter(*(start
+ 1)))
587 while (sp
>= line
&& isspace ((int) *sp
))
589 if ((sp
+ 1) != line
) /* the line isn't a simple variable assignment */
591 /* the line is valid, parse the variable name */
596 /* Skip type declaration that optionally follows a cdef/cpdef */
597 static const char *skipTypeDecl (const char *cp
, bool *is_class
)
599 const char *lastStart
= cp
, *ptr
= cp
;
602 if (!strncmp("extern", ptr
, 6)) {
604 ptr
= skipSpace(ptr
);
605 if (!strncmp("from", ptr
, 4)) { return NULL
; }
607 if (!strncmp("class", ptr
, 5)) {
610 ptr
= skipSpace(ptr
);
613 /* limit so that we don't pick off "int item=obj()" */
614 while (*ptr
&& loopCount
++ < 2) {
615 while (*ptr
&& *ptr
!= '=' && *ptr
!= '(' && !isspace(*ptr
)) {
616 /* skip over e.g. 'cpdef numpy.ndarray[dtype=double, ndim=1]' */
618 while (*ptr
&& *ptr
!= ']') ptr
++;
624 if (!*ptr
|| *ptr
== '=') return NULL
;
626 return lastStart
; /* if we stopped on a '(' we are done */
628 ptr
= skipSpace(ptr
);
630 while (*lastStart
== '*') lastStart
++; /* cdef int *identifier */
635 /* checks if there is a lambda at position of cp, and return its argument list
637 * We don't return the lambda name since it is useless for now since we already
638 * know it when we call this function, and it would be a little slower. */
639 static bool varIsLambda (const char *cp
, char **arglist
)
641 bool is_lambda
= false;
644 cp
= skipIdentifier (cp
); /* skip the lambda's name */
650 if (strncmp (cp
, "lambda", 6) == 0)
654 cp
+= 6; /* skip the lambda */
655 tmp
= skipSpace (cp
);
656 /* check if there is a space after lambda to detect assignations
657 * starting with 'lambdaXXX' */
660 vString
*args
= vStringNew ();
663 vStringPut (args
, '(');
664 for (; *cp
!= 0 && *cp
!= ':'; cp
++)
665 vStringPut (args
, *cp
);
666 vStringPut (args
, ')');
668 *arglist
= strdup (vStringValue (args
));
669 vStringDelete (args
);
677 /* checks if @p cp has keyword @p keyword at the start, and fills @p cp_n with
678 * the position of the next non-whitespace after the keyword */
679 static bool matchKeyword (const char *keyword
, const char *cp
, const char **cp_n
)
681 size_t kw_len
= strlen (keyword
);
682 if (strncmp (cp
, keyword
, kw_len
) == 0 && isspace (cp
[kw_len
]))
684 *cp_n
= skipSpace (&cp
[kw_len
+ 1]);
690 static void findPythonTags (void)
692 vString
*const continuation
= vStringNew ();
693 vString
*const name
= vStringNew ();
694 vString
*const parent
= vStringNew();
696 NestingLevels
*const nesting_levels
= nestingLevelsNew(sizeof (struct nestingLevelUserData
));
700 char const *longStringLiteral
= NULL
;
702 while ((line
= (const char *) readLineFromInputFile ()) != NULL
)
704 const char *cp
= line
, *candidate
;
705 char const *longstring
;
706 char const *keyword
, *variable
;
711 if (*cp
== '\0') /* skip blank line */
714 /* Skip comment if we are not inside a multi-line string. */
715 if (*cp
== '#' && !longStringLiteral
)
718 /* Deal with line continuation. */
719 if (!line_skip
) vStringClear(continuation
);
720 vStringCatS(continuation
, line
);
721 vStringStripTrailing(continuation
);
722 if (vStringLast(continuation
) == '\\')
724 vStringChop(continuation
);
725 vStringCatS(continuation
, " ");
729 cp
= line
= vStringValue(continuation
);
734 /* Deal with multiline string ending. */
735 if (longStringLiteral
)
737 find_triple_end(cp
, &longStringLiteral
);
741 checkIndent(nesting_levels
, indent
);
743 /* Find global and class variables */
744 variable
= findVariable(line
);
747 const char *start
= variable
;
749 bool parent_is_class
;
752 while (isIdentifierCharacter ((int) *start
))
754 vStringPut (name
, (int) *start
);
758 parent_is_class
= constructParentString(nesting_levels
, indent
, parent
);
759 if (varIsLambda (variable
, &arglist
))
761 /* show class members or top-level script lambdas only */
762 if (parent_is_class
|| vStringLength(parent
) == 0)
763 makeFunctionTag (name
, parent
, parent_is_class
, arglist
);
768 /* skip variables in methods */
769 if (parent_is_class
|| vStringLength(parent
) == 0)
770 makeVariableTag (name
, parent
, parent_is_class
);
774 /* Deal with multiline string start. */
775 longstring
= find_triple_start(cp
, &longStringLiteral
);
779 find_triple_end(longstring
, &longStringLiteral
);
780 /* We don't parse for any tags in the rest of the line. */
784 /* Deal with def and class keywords. */
785 keyword
= findDefinitionOrClass (cp
);
789 bool is_class
= false;
790 if (matchKeyword ("def", keyword
, &cp
))
794 else if (matchKeyword ("class", keyword
, &cp
))
799 else if (matchKeyword ("cdef", keyword
, &cp
))
801 candidate
= skipTypeDecl (cp
, &is_class
);
809 else if (matchKeyword ("cpdef", keyword
, &cp
))
811 candidate
= skipTypeDecl (cp
, &is_class
);
821 bool is_parent_class
;
822 struct corkInfo info
;
825 constructParentString(nesting_levels
, indent
, parent
);
829 info
.index
= parseClass (cp
, name
, parent
, is_parent_class
);
832 info
= parseFunction(cp
, name
, parent
, is_parent_class
);
834 addNestingLevel(nesting_levels
, indent
, &info
);
838 /* Find and parse imports */
842 /* Force popping all nesting levels. */
843 checkIndent(nesting_levels
, 0);
845 /* Clean up all memory we allocated. */
846 vStringDelete (parent
);
847 vStringDelete (name
);
848 vStringDelete (continuation
);
849 nestingLevelsFree (nesting_levels
);
852 extern parserDefinition
*PythonParser (void)
854 static const char *const extensions
[] = { "py", "pyx", "pxd", "pxi" ,"scons", NULL
};
855 parserDefinition
*def
= parserNew ("Python");
856 def
->kindTable
= PythonKinds
;
857 def
->kindCount
= ARRAY_SIZE (PythonKinds
);
858 def
->extensions
= extensions
;
859 def
->parser
= findPythonTags
;
860 def
->useCork
= CORK_QUEUE
;