diff options
| author | Xavier Morel <xmo@odoo.com> | 2017-09-18 10:13:54 +0200 |
|---|---|---|
| committer | Géry Debongnie <ged@odoo.com> | 2017-09-18 11:54:36 +0200 |
| commit | 313f69e9487e991e4db33c75682342b185da5c73 (patch) | |
| tree | dfe8918e95a5f27888319e7a5c1a5187cd8c8751 /doc/_extensions/autojsdoc/parser | |
| parent | 8defed9536711a05f147f28916b7109854f9932c (diff) | |
[ADD] doc: JS document extraction system
* Fix a bunch of ill-documented/incomplete/incorrect method docs
* add start of Sphinx extension to extract & integrate jsdoc into
Sphinx documentation:
- parse JS files (and don't blow up), uses a fork of pyjsparser as
the project currently does not parse comments
- extract cross-module dependency information
- parse JsDoc comments using pyjsdoc and infer structure from code &
jsdoc
- ``ast`` CLI printing a simplified AST of the input files
- ``dependencies`` creating a dependency graph of either all modules
in the provided input files or the modules matching the specified
filters (warning: will not work if missing dependencies),
generates a .dot file
- ``extractor`` generating a plain text module documentation (mix of
rst and markdown styles, not anything formal)
* sphinx extension with an "automodule" directive taking a module name
and generating the documentation for it
Diffstat (limited to 'doc/_extensions/autojsdoc/parser')
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/__init__.py | 1 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/jsdoc.py | 324 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/parser.py | 552 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/tests/README.rst | 6 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/tests/support.py | 73 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/tests/test_class.py | 227 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/tests/test_crap.py | 75 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/tests/test_module.py | 173 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/tests/test_namespace.py | 177 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/tests/test_params.py | 116 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/tests/test_typespec.py | 81 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/types.py | 191 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/utils.py | 295 | ||||
| -rw-r--r-- | doc/_extensions/autojsdoc/parser/visitor.py | 126 |
14 files changed, 2417 insertions, 0 deletions
diff --git a/doc/_extensions/autojsdoc/parser/__init__.py b/doc/_extensions/autojsdoc/parser/__init__.py new file mode 100644 index 00000000000..40a96afc6ff --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/doc/_extensions/autojsdoc/parser/jsdoc.py b/doc/_extensions/autojsdoc/parser/jsdoc.py new file mode 100644 index 00000000000..00406df42c1 --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/jsdoc.py @@ -0,0 +1,324 @@ +# -*- coding: utf-8 -*- +import re + +import collections + +import pyjsdoc + +def strip_stars(doc_comment): + """ + Version of jsdoc.strip_stars which always removes 1 space after * if + one is available. + """ + return re.sub('\n\s*?\*[\t ]?', '\n', doc_comment[3:-2]).strip() + +class ParamDoc(pyjsdoc.ParamDoc): + """ + Replace ParamDoc because FunctionDoc doesn't properly handle optional + params or default values (TODO: or compounds) if guessed_params is used + + => augment paramdoc with "required" and "default" items to clean up name + """ + def __init__(self, text): + super(ParamDoc, self).__init__(text) + # param name and doc can be separated by - or :, strip it + self.doc = self.doc.strip().lstrip('-:').lstrip() + self.optional = False + self.default = None + # there may not be a space between the param name and the :, in which + # case the : gets attached to the name, strip *again* + # TODO: formal @param/@property parser to handle this crap properly once and for all + self.name = self.name.strip().rstrip(':') + if self.name.startswith('['): + self.name = self.name.strip('[]') + self.optional = True + if '=' in self.name: + self.name, self.default = self.name.rsplit('=', 1) + def to_dict(self): + d = super(ParamDoc, self).to_dict() + d['optional'] = self.optional + d['default'] = self.default + return d +pyjsdoc.ParamDoc = ParamDoc + +class CommentDoc(pyjsdoc.CommentDoc): + namekey = object() + is_constructor = False + + @property + def name(self): + return self[self.namekey] or self['name'] or self['guessed_name'] + def set_name(self, name): + # not great... + if name != '<exports>': + self.parsed['guessed_name'] = name + + @property + def is_private(self): + return 'private' in self.parsed + + def to_dict(self): + d = super(CommentDoc, self).to_dict() + d['name'] = self.name + return d + +class PropertyDoc(CommentDoc): + @classmethod + def from_param(cls, s, sourcemodule=None): + parsed = ParamDoc(s).to_dict() + parsed['sourcemodule'] = sourcemodule + return cls(parsed) + + @property + def type(self): + return self['type'].strip('{}') + + def to_dict(self): + d = super(PropertyDoc, self).to_dict() + d['type'] = self.type + d['is_private'] = self.is_private + return d + +class InstanceDoc(CommentDoc): + @property + def cls(self): + return self['cls'] + + def to_dict(self): + return dict(super(InstanceDoc, self).to_dict(), cls=self.cls) + +class LiteralDoc(CommentDoc): + @property + def type(self): + if self['type']: + return self['type'] + valtype = type(self['value']) + if valtype is bool: + return 'Boolean' + elif valtype is float: + return 'Number' + elif valtype is type(u''): + return 'String' + return '' + + @property + def value(self): + return self['value'] + + def to_dict(self): + d = super(LiteralDoc, self).to_dict() + d['type'] = self.type + d['value'] = self.value + return d + +class FunctionDoc(CommentDoc): + type = 'Function' + namekey = 'function' + + @property + def is_constructor(self): + return self.name == 'init' + + @property + def params(self): + tag_texts = self.get_as_list('param') + if self.get('guessed_params') is None: + return [ParamDoc(text) for text in tag_texts] + else: + param_dict = {} + for text in tag_texts: + param = ParamDoc(text) + param_dict[param.name] = param + return [param_dict.get(name) or ParamDoc('{} ' + name) + for name in self.get('guessed_params')] + @property + def return_val(self): + ret = self.get('return') or self.get('returns') + type = self.get('type') + if '{' in ret and '}' in ret: + if not '} ' in ret: + # Ensure that name is empty + ret = re.sub(r'\}\s*', '} ', ret) + return ParamDoc(ret) + if ret and type: + return ParamDoc('{%s} %s' % (type, ret)) + return ParamDoc(ret) + + def to_dict(self): + d = super(FunctionDoc, self).to_dict() + d['name'] = self.name + d['params'] = [param.to_dict() for param in self.params] + d['return_val']= self.return_val.to_dict() + return d + +class NSDoc(CommentDoc): + namekey = 'namespace' + def __init__(self, parsed_comment): + super(NSDoc, self).__init__(parsed_comment) + self.members = collections.OrderedDict() + def add_member(self, name, member): + """ + :type name: str + :type member: CommentDoc + """ + member.set_name(name) + self.members[name] = member + + @property + def properties(self): + if self.get('property'): + return [ + (p.name, p) + for p in ( + PropertyDoc.from_param(p, self['sourcemodule']) + for p in self.get_as_list('property') + ) + ] + return self.members.items() or self['_members'] or [] + + def has_property(self, name): + return self.get_property(name) is not None + + def get_property(self, name): + return next((p for n, p in self.properties if n == name), None) + + def to_dict(self): + d = super(NSDoc, self).to_dict() + d['properties'] = [(n, p.to_dict()) for n, p in self.properties] + return d + +class MixinDoc(NSDoc): + namekey = 'mixin' + +class ModuleDoc(NSDoc): + namekey = 'module' + def __init__(self, parsed_comment): + super(ModuleDoc, self).__init__(parsed_comment) + #: callbacks to run with the modules mapping once every module is resolved + self._post_process = [] + + def post_process(self, modules): + for callback in self._post_process: + callback(modules) + + @property + def module(self): + return self # lol + + @property + def dependencies(self): + """ + Returns the immediate dependencies of a module (only those explicitly + declared/used). + """ + return self.get('dependency', None) or set() + + @property + def exports(self): + """ + Returns the actual item exported from the AMD module, can be a + namespace, a class, a function, an instance, ... + """ + return self.get_property('<exports>') + + def to_dict(self): + vars = super(ModuleDoc, self).to_dict() + vars['dependencies'] = self.dependencies + vars['exports'] = self.exports + return vars + +class ClassDoc(NSDoc): + namekey = 'class' + @property + def constructor(self): + return self.get_property('init') + + @property + def superclass(self): + return self['extends'] or self['base'] + + def get_property(self, method_name): + if method_name == 'extend': + return FunctionDoc({ + 'doc': 'Create subclass for %s' % self.name, + 'guessed_function': 'extend', + }) + # FIXME: should ideally be a proxy namespace + if method_name == 'prototype': + return self + return super(ClassDoc, self).get_property(method_name) + + @property + def mixins(self): + return self.get_as_list('mixes') + + def to_dict(self): + d = super(ClassDoc, self).to_dict() + d['mixins'] = self.mixins + return d + +class UnknownNS(NSDoc): + def get_property(self, name): + return super(UnknownNS, self).get_property(name) or \ + UnknownNS({'name': '{}.{}'.format(self.name, name)}) + +class Unknown(CommentDoc): + @classmethod + def from_(cls, source): + def builder(parsed): + inst = cls(parsed) + inst.parsed['source'] = source + return inst + return builder + + @property + def name(self): + return self['name'] + ' ' + self['source'] + + @property + def type(self): + return "Unknown" + + def get_property(self, p): + return Unknown(dict(self.parsed, source=self.name, name=p + '<')) + +def parse_comments(comments, doctype=None): + # find last comment which starts with a * + docstring = next(( + c['value'] + for c in reversed(comments or []) + if c['value'].startswith(u'*') + ), None) or u"" + + # \n prefix necessary otherwise parse_comment fails to take first + # block comment parser strips delimiters, but strip_stars fails without + # them + extract = '\n' + strip_stars('/*' + docstring + '\n*/') + parsed = pyjsdoc.parse_comment(extract, u'') + + if doctype == 'FunctionExpression': + doctype = FunctionDoc + elif doctype == 'ObjectExpression' or doctype is None: + doctype = guess + + if doctype is guess: + return doctype(parsed) + + # in case a specific doctype is given, allow overriding it anyway + return guess(parsed, default=doctype) + +def guess(parsed, default=NSDoc): + if 'class' in parsed: + return ClassDoc(parsed) + if 'function' in parsed: + return FunctionDoc(parsed) + if 'mixin' in parsed: + return MixinDoc(parsed) + if 'namespace' in parsed: + return NSDoc(parsed) + if 'module' in parsed: + return ModuleDoc(parsed) + if 'type' in parsed: + return PropertyDoc(parsed) + + return default(parsed) diff --git a/doc/_extensions/autojsdoc/parser/parser.py b/doc/_extensions/autojsdoc/parser/parser.py new file mode 100644 index 00000000000..d91b9f22596 --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/parser.py @@ -0,0 +1,552 @@ +# -*- coding: utf-8 -*- +import collections + +import pyjsdoc +from attr import attrs, Factory, attr, astuple + +from . import jsdoc +from . import utils +from .visitor import Visitor, SKIP + +DECLARATOR_INIT_TO_REF = ('Literal', 'Identifier', 'MemberExpression') + + +class ModuleMatcher(Visitor): + """Looks for structures of the form:: + + odoo.define($string, function ($name) { + + These are *Odoo module definitions*, upon encountering one the + matcher: + + * creates a module entry, optionally associated with the module comment + * spawns off a :class:`ModuleBodyMatcher` on the function's body with the + module name and the $name as "require" function + """ + def __init__(self, filename): + super(ModuleMatcher, self).__init__() + self.filename = filename + self.result = [] + def enter_Program(self, node): + pass # allows visiting toplevel + def enter_ExpressionStatement(self, node): + # we're interested in expression statements (toplevel call) + if utils.match(node, {'expression': { + 'callee': { + 'object': {'name': 'odoo'}, + 'property': {'name': 'define'}, + }, + }}): + [module, func] = node['expression']['arguments'] + mod = jsdoc.parse_comments(node.get('comments'), jsdoc.ModuleDoc) + # set module name + mod.set_name(module['value']) + mod.parsed['sourcefile'] = self.filename + self.result.append(mod) + + # get name of require parameter + require = None # a module can have no dependencies + if func['params']: + require = func['params'][0]['name'] + mod.parsed['dependency'], post = ModuleExtractor(mod, require).visit(func['body']) + mod._post_process.extend(post) + # don't recurse since we've fired off a sub-visitor for the + # bits we're interested in + return SKIP + def enter_generic(self, node): + # skip all other toplevel statements + return SKIP + +ref = collections.namedtuple('Ref', 'object property') +def _name(r): + bits = [] + while isinstance(r, ref): + bits.append(r.property) + r = r.object + return '.'.join(reversed(bits)) +def deref(item, prop=None): + assert isinstance(item, ref) + + while isinstance(item, ref): + obj = item.object + if isinstance(obj, ref): + obj = deref(obj) + + if isinstance(obj, (jsdoc.NSDoc, jsdoc.Unknown)): + item = obj.get_property(item.property) + elif isinstance(obj, dict): + item = obj[item.property] + elif isinstance(obj, jsdoc.PropertyDoc): + # f'n dynamic crap + item = jsdoc.Unknown(obj.to_dict()).get_property(item.property) + else: + raise ValueError("%r (%s) should be a dict or namespace" % (obj, type(obj))) + return item + +def m2r(me, scope): + # create a ref to the scope in case it's a hoisted function declaration, + # this may false-positive on other hoisting but w/e + if me['type'] == 'Literal': + return jsdoc.LiteralDoc({'value': me['value']}) + if me['type'] == 'Identifier': + return ref(scope, me['name']) + if me['type'] != 'MemberExpression': + raise ValueError(me) + return ref( + m2r(me['object'], scope), + utils._value(me['property'], strict=True) + ) + +@attrs(slots=True) +class Declaration(object): + id = attr(default=None) + comments = attr(default=Factory(list)) +@attrs(slots=True) +class ModuleContent(object): + dependencies = attr(default=Factory(set)) + post = attr(default=Factory(list)) + def __iter__(self): + yield self.dependencies + yield self.post + +@attrs # needs dict as it's replacing a ModuleProxy +class Nothing(object): + name = attr() + def __bool__(self): + return False + __nonzero__ = __bool__ + +class RefProxy(object): + def __init__(self, r): + self._ref = r + def become(self, modules): + s = self + other = deref(self._ref) or Nothing(_name(self._ref)) + # ??? shouldn't all previous refs have been resolved? + if isinstance(other, (RefProxy, ModuleProxy)): + other = other.become(modules) + self.__class__ = other.__class__ + self.__dict__ = other.__dict__ + return self + def set_name(self, name): + pass # ??? +class ModuleProxy(object): + def __init__(self, name): + self._name = name + + # replace the ModuleProxy by the module's exports + def become(self, modules): + s = self + m = modules[self._name].get_property('<exports>') or Nothing(self._name) + self.__class__ = m.__class__ + self.__dict__ = m.__dict__ + return self + + def set_name(self, name): + pass # FIXME: ??? + +jq = jsdoc.UnknownNS({ + 'name': u'jQuery', + 'doc': u'<jQuery>', +}) +window = jsdoc.UnknownNS({ + 'doc': '<window>', + 'name': 'window', +}) +BASE_SCOPE = { + '_': jsdoc.UnknownNS({'doc': u'<underscore.js>', 'name': u'_'}), + '$': jq, 'jQuery': jq, + 'nv': jsdoc.UnknownNS({ + 'doc': u'<nvd3>', + 'name': u'nv', + }), + 'window': window, + 'document': window.get_property('document'), + 'Date': jsdoc.ClassDoc({ + 'name': u'Date', + 'doc': u'', + }), + 'Backbone': jsdoc.UnknownNS({ + '_members': [ + ('Model', jsdoc.ClassDoc({ + 'name': u'Model', + 'doc': u'', + })), + ('Collection', jsdoc.ClassDoc({ + 'name': u'Collection', + 'doc': u'', + })), + ] + }), + 'odoo': jsdoc.UnknownNS({ + 'name': u'odoo', + 'doc': u"Odoo", + '_members': [ + ('name', jsdoc.PropertyDoc({'name': u'csrf_token', 'type': u'{String}'})), + ] + }), + 'undefined': jsdoc.LiteralDoc({'name': u'undefined', 'value': None}) +} + +class Scope(object): + """ + Add hoc scope versioning/SSA such that rebinding a symbol in a module + scope does not screw everything up e.g. "Foo = Foo.extend({})" should not + have the final Foo extending itself... + """ + def __init__(self, mapping): + self._namemap = {} + self._targets = [] + for k, v in mapping.items(): + self[k] = v + + def __setitem__(self, k, v): + self._namemap[k] = len(self._targets) + self._targets.append(v) + + def freeze(self): + return { + k: self._targets[v] + for k, v in self._namemap.items() + } + +class ModuleExtractor(Visitor): + def __init__(self, module, requirefunc): + super(ModuleExtractor, self).__init__() + self.module = module + self.requirefunc = requirefunc + self.result = ModuleContent() + self.scope = Scope(BASE_SCOPE) + self.declaration = None + + def enter_BlockStatement(self, node): + Hoistifier(self).visit(node) + def exit_BlockStatement(self, node): + for k, v in self.scope._namemap.items(): + if k not in BASE_SCOPE: + self.module.add_member(k, self.scope._targets[v]) + for t in TypedefMatcher(self).visit(node): + self.module.add_member(t.name, t) + + def enter_VariableDeclaration(self, node): + self.declaration = Declaration(comments=node.get('comments')) + def enter_VariableDeclarator(self, node): + # otherwise we've already hoisted the declaration so the variable + # already exist initialised to undefined, and ValueExtractor does not + # handle a None input node so it returns None which makes no sense + if node['init']: + self.declaration.id = node['id']['name'] + self.scope[self.declaration.id] = ValueExtractor( + self, self.declaration + ).visit(node['init'] or []) + self.declaration.id = None + return SKIP + def exit_VariableDeclaration(self, node): + self.declaration = None + + # as the name denotes, AssignmentExpression is an *expression*, which + # means at the module toplevel it is wrapped into an ExpressionStatement + # which is where the comments get attached + def enter_ExpressionStatement(self, node): + self.declaration = Declaration(comments=node.get('comments')) + def exit_ExpressionStatement(self, node): + self.declaration = None + def enter_AssignmentExpression(self, node): + target = node['left'] + if target['type'] == 'Identifier': + self.declaration.id = target['name'] + self.scope[self.declaration.id] = ValueExtractor( + self, self.declaration + ).visit(node['right']) + self.declaration.id = None + return SKIP + + if target['type'] != 'MemberExpression': + raise ValueError("Unhandled assign to %s" % target['type']) + + # only assign to straight a.b.c patterns (OK and trivial literals) + if target['computed'] and target['property']['type'] != 'Literal': + return SKIP + + name = utils._value(target['property'], strict=True) + if isinstance(name, type(u'')) and name.endswith('.extend'): + return SKIP # ignore overwrite of .extend (WTF) + self.declaration.id = name + it = ValueExtractor( + self, self.declaration + ).visit(node['right']) + self.declaration.id = None + assert it, "assigned a non-value from %s to %s" % (node['right'], name) + + @self.result.post.append + def _augment_module(modules): + try: + t = deref(m2r(target['object'], self.scope.freeze())) + except ValueError: + return # f'n extension of global libraries garbage + if not isinstance(t, jsdoc.NSDoc): + # function Foo(){}; Foo.prototype = bar + # fuck that yo + return + # TODO: note which module added this + m = it + if isinstance(m, jsdoc.LiteralDoc): + m = jsdoc.PropertyDoc(m.to_dict()) + t.add_member(name, m) + return SKIP + + def enter_FunctionDeclaration(self, node): + """ Already processed by hoistitifier + """ + return SKIP + + def enter_ReturnStatement(self, node): + self.declaration = Declaration(comments=node.get('comments')) + if node['argument']: + export = ValueExtractor(self, self.declaration).visit(node['argument']) + if isinstance(export, RefProxy): + self.module.parsed['exports'] = _name(export._ref) + self.scope['<exports>'] = export + self.declaration = None + return SKIP + + def enter_CallExpression(self, node): + if utils.match(node, { + 'callee': { + 'type': 'MemberExpression', + 'object': lambda n: ( + n['type'] in ('Identifier', 'MemberExpression') + # _.str.include + and not utils._name(n).startswith('_') + ), + 'property': {'name': 'include'}, + } + }): + target = RefProxy(m2r(node['callee']['object'], self.scope.freeze())) + target_name = utils._name(node['callee']['object']) + items = ClassProcessor(self).visit(node['arguments']) + @self.result.post.append + def resolve_extension(modules): + t = target.become(modules) + if not isinstance(t, jsdoc.ClassDoc): + raise ValueError("include() subjects should be classes, %s is %s" % (target_name, type(t))) + # TODO: note which module added these + for it in items: + if isinstance(it, dict): + for n, member in it.items(): + t.add_member(n, member) + else: + t.parsed.setdefault('mixes', []).append(it.become(modules)) + + return SKIP + + def refify(self, node, also=None): + it = m2r(node, self.scope.freeze()) + assert isinstance(it, ref), "Expected ref, got {}".format(it) + px = RefProxy(it) + @self.result.post.append + def resolve(modules): + p = px.become(modules) + if also: also(p) + return px + +class ValueExtractor(Visitor): + def __init__(self, parent, declaration=None): + super(ValueExtractor, self).__init__() + self.parent = parent + self.declaration = declaration or Declaration() + + def enter_generic(self, node): + self.result = jsdoc.parse_comments( + self.declaration.comments, + jsdoc.Unknown.from_(node['type']) + ) + self._update_result_meta() + return SKIP + + def _update_result_meta(self, name=None): + self.result.parsed['sourcemodule'] = self.parent.module + n = name or self.declaration.id + if n: + self.result.set_name(n) + + def enter_Literal(self, node): + self.result = jsdoc.parse_comments( + self.declaration.comments, jsdoc.LiteralDoc) + self._update_result_meta() + self.result.parsed['value'] = node['value'] + return SKIP + def enter_Identifier(self, node): + self.result = self.parent.refify(node) + return SKIP + + def enter_MemberExpression(self, node): + self.result = RefProxy(ref( + ValueExtractor(self.parent).visit(node['object']), + utils._value(node['property'], strict=True) + )) + self.parent.result.post.append(self.result.become) + return SKIP + + def enter_FunctionExpression(self, node): + name, comments = astuple(self.declaration) + self.result = jsdoc.parse_comments(comments, jsdoc.FunctionDoc) + self.result.parsed['name'] = node['id'] and node['id']['name'] + self._update_result_meta() + self.result.parsed['guessed_params'] = [p['name'] for p in node['params']] + return SKIP + + def enter_NewExpression(self, node): + comments = self.declaration.comments if self.declaration else node.get('comments') + self.result = ns = jsdoc.parse_comments(comments, jsdoc.InstanceDoc) + self._update_result_meta() + + def _update_contents(cls): + if not isinstance(cls, jsdoc.ClassDoc): + return + ns.parsed['cls'] = cls + self.parent.refify(node['callee'], also=_update_contents) + return SKIP + + def enter_ObjectExpression(self, node): + self.result = obj = jsdoc.parse_comments(self.declaration.comments) + self._update_result_meta() + for n, p in MemberExtractor(parent=self.parent).visit(node['properties']).items(): + obj.add_member(n, p) + return SKIP + + def enter_CallExpression(self, node): + # require(a_module_name) + if utils.match(node, {'callee': {'type': 'Identifier', 'name': self.parent.requirefunc}}): + depname = node['arguments'][0]['value'] + # TODO: clean this up + self.parent.result.dependencies.add(depname) + self.result = ModuleProxy(name=depname) + self.parent.result.post.append(self.result.become) + + # Class.extend(..mixins, {}) + elif utils.match(node, { + 'callee': { + 'type': 'MemberExpression', + 'object': lambda n: ( + n['type'] in ('Identifier', 'MemberExpression') + # $.extend, $.fn.extend, _.extend + and not utils._name(n).startswith(('_', '$')) + ), + 'property': {'name': 'extend'}, + }, + }): # creates a new class, but may not actually return it + obj = node['callee']['object'] + _, comments = astuple(self.declaration) + self.result = cls = jsdoc.parse_comments(comments, jsdoc.ClassDoc) + cls.parsed['extends'] = self.parent.refify(obj) + self._update_result_meta() + items = ClassProcessor(self.parent).visit(node['arguments']) + + @self.parent.result.post.append + def add_to_class(modules): + for item in items: + if isinstance(item, dict): + # methods/attributes + for n, method in item.items(): + cls.add_member(n, method) + else: + cls.parsed.setdefault('mixes', []).append(item) + + # other function calls + else: + self.result = jsdoc.parse_comments(self.declaration.comments, jsdoc.guess) + self._update_result_meta() + + return SKIP + +class ClassProcessor(Visitor): + def __init__(self, parent): + super(ClassProcessor, self).__init__() + self.result = [] + self.parent = parent + + def enter_generic(self, node): + self.result.append(self.parent.refify(node)) + return SKIP + def enter_ObjectExpression(self, node): + self.result.append(MemberExtractor(parent=self.parent, for_class=True).visit(node['properties'])) + return SKIP + +String = type(u'') +class MemberExtractor(Visitor): + def __init__(self, parent, for_class=False): + super(MemberExtractor, self).__init__() + self.result = collections.OrderedDict() + self.parent = parent + self.for_class = for_class + + def enter_Property(self, node): + name = utils._value(node['key']) + prop = ValueExtractor( + self.parent, + Declaration(id=name, comments=node.get('comments')) + ).visit(node['value']) + + if isinstance(prop, jsdoc.LiteralDoc): + prop = jsdoc.PropertyDoc(prop.to_dict()) + + # ValueExtractor can return a Ref, maybe this should be sent as part + # of the decl/comments? + if name.startswith('_') and hasattr(prop, 'parsed'): + prop.parsed['private'] = True + self.result[name] = prop + + return SKIP + +class Hoistifier(Visitor): + """ + Processor for variable and function declarations properly hoisting them + to the "top" of a module such that they are available with the relevant + value afterwards. + """ + def __init__(self, parent): + super(Hoistifier, self).__init__() + self.parent = parent + + def enter_generic(self, node): + return SKIP + + # nodes to straight recurse into, others are just skipped + enter_BlockStatement = enter_VariableDeclaration = lambda self, node: None + + def enter_VariableDeclarator(self, node): + self.parent.scope[node['id']['name']] = BASE_SCOPE['undefined'] + + def enter_FunctionDeclaration(self, node): + funcname = node['id']['name'] + self.parent.scope[funcname] = fn = jsdoc.parse_comments( + node.get('comments'), + jsdoc.FunctionDoc, + ) + fn.parsed['sourcemodule'] = self.parent.module + fn.parsed['name'] = funcname + fn.parsed['guessed_params'] = [p['name'] for p in node['params']] + return SKIP + +class TypedefMatcher(Visitor): + def __init__(self, parent): + super(TypedefMatcher, self).__init__() + self.parent = parent + self.result = [] + + enter_BlockStatement = lambda self, node: None + def enter_generic(self, node): + # just traverse all top-level statements, check their comments, and + # bail + for comment in node.get('comments') or []: + if '@typedef' in comment['value']: + extract = '\n' + jsdoc.strip_stars('/*' + comment['value'] + '\n*/') + parsed = pyjsdoc.parse_comment(extract, u'') + p = jsdoc.ParamDoc(parsed['typedef']) + parsed['name'] = p.name + parsed['sourcemodule'] = self.parent.module + # TODO: add p.type as superclass somehow? Builtin types not in scope :( + self.result.append(jsdoc.ClassDoc(parsed)) + + return SKIP diff --git a/doc/_extensions/autojsdoc/parser/tests/README.rst b/doc/_extensions/autojsdoc/parser/tests/README.rst new file mode 100644 index 00000000000..dd862beae66 --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/tests/README.rst @@ -0,0 +1,6 @@ +:orphan: + +These files should be run via pytest_, simply install pytest and from the top +of the Odoo project run ``pytest doc/_extensions``. + +.. _pytest: https://pytest.org/ diff --git a/doc/_extensions/autojsdoc/parser/tests/support.py b/doc/_extensions/autojsdoc/parser/tests/support.py new file mode 100644 index 00000000000..f533f98d90c --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/tests/support.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +import operator + +import pyjsparser + +from autojsdoc.parser import jsdoc, parser + +params = operator.attrgetter('name', 'type', 'doc') + + +def parse(s, source=None): + tree = pyjsparser.parse(s) + mods = parser.ModuleMatcher(source).visit(tree) + post(mods) + return mods + +def post(mods): + modules = dict(BASE_MODULES) + modules.update((m.name, m) for m in mods) + + for mod in mods: + mod.post_process(modules) + +BASE_MODULES = { + 'other': jsdoc.ModuleDoc({ + 'module': 'other', + '_members': [ + ('<exports>', jsdoc.LiteralDoc({'name': 'value', 'value': "ok"})), + ], + }), + 'dep2': jsdoc.ModuleDoc({ + 'module': 'dep2', + '_members': [ + ('<exports>', jsdoc.LiteralDoc({'value': 42.})), + ], + }), + 'dep3': jsdoc.ModuleDoc({ + 'module': 'dep3', + '_members': [ + ('<exports>', jsdoc.LiteralDoc({'value': 56.})), + ], + }), + 'Class': jsdoc.ModuleDoc({ + 'module': 'Class', + '_members': [ + ('<exports>', jsdoc.ClassDoc({ + 'name': 'Class', + 'doc': "Base Class" + })), + ], + }), + 'mixins': jsdoc.ModuleDoc({ + 'module': 'mixins', + '_members': [ + ('<exports>', jsdoc.NSDoc({ + 'name': 'mixins', + '_members': [ + ('Bob', jsdoc.ClassDoc({'class': "Bob"})), + ] + })), + ], + }), + 'Mixin': jsdoc.ModuleDoc({ + 'module': 'Mixin', + '_members': [ + ('<exports>', jsdoc.MixinDoc({ + '_members': [ + ('a', jsdoc.FunctionDoc({'function': 'a'})), + ] + })), + ], + }) +} diff --git a/doc/_extensions/autojsdoc/parser/tests/test_class.py b/doc/_extensions/autojsdoc/parser/tests/test_class.py new file mode 100644 index 00000000000..403ae9b48b7 --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/tests/test_class.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +import pytest + +from autojsdoc.parser import jsdoc +from support import params, parse + +def test_classvar(): + [mod] = parse(""" + odoo.define('a.A', function(require) { + var Class = require('Class'); + /** + * This is my class-kai + */ + var A = Class.extend({}); + return A; + }); + """) + cls = mod.exports + assert type(cls) == jsdoc.ClassDoc + + assert type(cls.superclass) == jsdoc.ClassDoc + assert cls.superclass.name == 'Class' + + assert cls.name == 'A' + assert cls.constructor is None + assert cls.properties == [] + assert cls['doc'] == 'This is my class-kai' + +def test_classret(): + [mod] = parse(""" + odoo.define('a.A', function(require) { + var Class = require('Class'); + /** + * This is my class-kai + */ + return Class.extend({}); + }); + """) + cls = mod.exports + assert type(cls) == jsdoc.ClassDoc + + assert cls.name == '' + assert cls.constructor is None + assert cls.properties == [] + assert cls['doc'] == 'This is my class-kai' + +def test_methods(): + [mod] = parse(""" + odoo.define('a.A', function(require) { + var Class = require('Class'); + return Class.extend({ + /** + * @param {Widget} parent + */ + init: function (parent) {}, + /** + * @returns {Widget} + */ + itself: function () { return this; }, + /** + * @param {MouseEvent} e + */ + _onValidate: function (e) {}, + }); + }); + """) + cls = mod.exports + assert len(cls.properties) == 3 + assert cls.constructor + # assume methods are in source order + [_, init] = cls.properties[0] + assert init == cls.constructor + assert init.name == 'init' + assert not init.is_private + assert init.is_constructor + [param] = init.params + assert params(param) == ('parent', 'Widget', '') + + [_, itself] = cls.properties[1] + assert itself.name == 'itself' + assert not itself.is_private + assert not itself.is_constructor + assert not itself.params + assert params(itself.return_val) == ('', 'Widget', '') + + [_, _on] = cls.properties[2] + assert _on.name == '_onValidate' + assert _on.is_private + assert not _on.is_constructor + [param] = _on.params + assert params(param) == ('e', 'MouseEvent', '') + +def test_mixin_explicit(): + [mod] = parse(""" + odoo.define('a.A', function (require) { + var Class = require('Class'); + var mixins = require('mixins'); + /** + * This is my class-kai + * @mixes mixins.Bob + */ + return Class.extend({}); + }); + """) + cls = mod.exports + # FIXME: ClassDoc may want to m2r(mixin, scope)? + assert cls.mixins == ['mixins.Bob'] + +def test_mixin_implicit(): + [mod] = parse(""" + odoo.define('a.A', function(require) { + var Class = require('Class'); + var Mixin = require('Mixin'); + /** + * This is my class-kai + */ + return Class.extend(Mixin, { foo: function() {} }); + }); + """) + cls = mod.exports + [mixin] = cls.mixins + assert type(mixin) == jsdoc.MixinDoc + assert params(mixin.properties[0][1]) == ('a', 'Function', '') + assert params(mixin.get_property('a')) == ('a', 'Function', '') + + assert params(cls.get_property('foo')) == ('foo', 'Function', '') + +def test_instanciation(): + [A, a] = parse(""" + odoo.define('A', function (r) { + var Class = r('Class'); + /** + * @class A + */ + return Class.extend({ + foo: function () {} + }); + }); + odoo.define('a', function (r) { + var A = r('A'); + var a = new A; + return a; + }); + """) + assert type(a.exports) == jsdoc.InstanceDoc + assert a.exports.cls.name == A.exports.name + +def test_non_function_properties(): + [A] = parse(""" + odoo.define('A', function (r) { + var Class = r('Class'); + return Class.extend({ + template: 'thing', + a_prop: [1, 2, 3], + 'other': {a: 7} + }); + }); + """) + t = A.exports.get_property('template') + assert type(t) == jsdoc.PropertyDoc + assert params(t) == ('template', 'String', '') + assert not t.is_private + +def test_non_extend_classes(): + [mod] = parse(""" + odoo.define('A', function () { + /** + * @class + */ + var Class = function () {} + return Class; + }); + """) + assert type(mod.exports) == jsdoc.ClassDoc + +def test_extend(): + [a, _] = parse(""" + odoo.define('A', function (require) { + var Class = require('Class'); + return Class.extend({}); + }); + odoo.define('B', function (require) { + var A = require('A'); + A.include({ + /** A property */ + a: 3, + /** A method */ + b: function () {} + }); + }); + """) + cls = a.exports + assert type(cls) == jsdoc.ClassDoc + a = cls.get_property('a') + assert type(a) == jsdoc.PropertyDoc + assert params(a) == ('a', 'Number', 'A property') + b = cls.get_property('b') + assert type(b) == jsdoc.FunctionDoc + assert params(b) == ('b', 'Function', 'A method') + +# TODO: also support virtual members? +# TODO: computed properties? +@pytest.mark.skip(reason="Need to implement member/var-parsing?") +def test_members(): + [mod] = parse(""" + odoo.define('A', function (r) { + var Class = r('Class'); + return Class.extend({ + init: function () { + /** + * This is bob + * @var {Foo} + */ + this.foo = 3; + this.bar = 42; + /** + * @member {Baz} + */ + this.baz = null; + } + }); + }); + """) + cls = mod.exports + assert params(cls.members[0]) == ('foo', 'Foo', 'This is bob') + assert params(cls.members[1]) == ('bar', '', '') + assert params(cls.members[2]) == ('baz', 'Baz', '') diff --git a/doc/_extensions/autojsdoc/parser/tests/test_crap.py b/doc/_extensions/autojsdoc/parser/tests/test_crap.py new file mode 100644 index 00000000000..70063e6dd47 --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/tests/test_crap.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +Test various crap patterns found in Odoo code to ensure they don't blow up +the parser thingie +""" +from autojsdoc.parser import jsdoc +from support import parse + +def test_export_external(): + [mod] = parse(""" + odoo.define('module', function () { + return $.Deferred().reject(); + }); + """) + assert isinstance(mod.exports, jsdoc.CommentDoc) + assert mod.exports.doc == '' + +def test_extend_jq(): + parse(""" + odoo.define('a', function (r) { + $.extend($.expr[':'], { a: function () {} }); + $.fn.extend({ a: function () {} }); + }); + """) + +def test_extend_dynamic(): + parse(""" + odoo.define('a', function () { + foo.bar.baz[qux + '_external'] = function () {}; + }); + """) + +def test_extend_deep(): + parse(""" + odoo.define('a', function () { + var eventHandler = $.summernote.eventHandler; + var dom = $.summernote.core.dom; + dom.thing = function () {}; + + var fn_editor_currentstyle = eventHandler.modules.editor.currentStyle; + eventHandler.modules.editor.currentStyle = function () {} + }); + """) + +def test_arbitrary(): + parse(""" + odoo.define('bob', function () { + var page = window.location.href.replace(/^.*\/\/[^\/]+/, ''); + var mailWidgets = ['mail_followers', 'mail_thread', 'mail_activity', 'kanban_activity']; + var bob; + var fldj = foo.getTemplate().baz; + }); + """) + +def test_prototype(): + [A, B] = parse(""" + odoo.define('mod1', function () { + var exports = {}; + exports.Foo = Backbone.Model.extend({}); + exports.Bar = Backbone.Model.extend({}); + var BarCollection = Backbone.Collection.extend({ + model: exports.Bar, + }); + exports.Baz = Backbone.Model.extend({}); + return exports; + }); + odoo.define('mod2', function (require) { + var models = require('mod1'); + var _super_orderline = models.Bar.prototype; + models.Foo = models.Bar.extend({}); + var _super_order = models.Baz.prototype; + models.Bar = models.Baz.extend({}); + }); + """) + diff --git a/doc/_extensions/autojsdoc/parser/tests/test_module.py b/doc/_extensions/autojsdoc/parser/tests/test_module.py new file mode 100644 index 00000000000..2a63cc1754c --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/tests/test_module.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- + +from autojsdoc.parser import jsdoc +from support import params, parse, BASE_MODULES + + +def test_single(): + [mod] = parse(""" + /** + * This is a super module! + */ + odoo.define('supermodule', function (req) { + var other = req('other'); + }); + """) + assert mod.name == 'supermodule' + assert mod.dependencies == {'other'} + assert mod.exports is None + assert mod.doc == "This is a super module!" + m = mod.get_property('other') + assert m.name == 'value', "property is exported value not imported module" + +def test_multiple(): + [mod1, mod2, mod3] = parse(""" + odoo.define('module1', function (req) { + return 1; + }); + odoo.define('module2', function (req) { + return req('dep2'); + }); + odoo.define('module3', function (req) { + var r = req('dep3'); + return r; + }); + """) + assert mod1.name == 'module1' + assert mod1.dependencies == set() + assert isinstance(mod1.exports, jsdoc.LiteralDoc) + assert mod1.exports.value == 1.0 + assert mod1.exports['sourcemodule'] is mod1 + assert mod1.doc == "" + + assert mod2.name == 'module2' + assert mod2.dependencies == {'dep2'} + assert isinstance(mod2.exports, jsdoc.LiteralDoc) + assert mod2.exports.value == 42.0 + assert mod2.doc == "" + + assert mod3.name == 'module3' + assert mod3.dependencies == {'dep3'} + assert isinstance(mod3.exports, jsdoc.LiteralDoc) + assert mod3.exports.value == 56.0 + assert mod3.doc == '' + +def test_func(): + [mod] = parse(""" + odoo.define('module', function (d) { + /** + * @param {Foo} bar this is a bar + * @param {Baz} qux this is a qux + */ + return function (bar, qux) { + return 42; + } + }); + """) + exports = mod.exports + assert type(exports) == jsdoc.FunctionDoc + assert exports['sourcemodule'] is mod + + assert exports.name == '' + assert exports.is_constructor == False + assert exports.is_private == False + + assert params(exports.params[0]) == ('bar', 'Foo', 'this is a bar') + assert params(exports.params[1]) == ('qux', 'Baz', 'this is a qux') + assert params(exports.return_val) == ('', '', '') + +def test_hoist(): + [mod] = parse(""" + odoo.define('module', function() { + return foo; + /** + * @param a_thing + */ + function foo(a_thing) { + return 42; + } + }); + """) + actual = mod.exports + assert type(actual) == jsdoc.FunctionDoc + [param] = actual.params + assert params(param) == ('a_thing', '', '') + +def test_export_instance(): + [mod] = parse(""" + odoo.define('module', function (require) { + var Class = require('Class'); + /** + * Provides an instance of Class + */ + return new Class(); + }); + """) + assert type(mod.exports) == jsdoc.InstanceDoc + assert mod.exports.doc == 'Provides an instance of Class' + assert mod.exports['sourcemodule'] is mod + +def test_bounce(): + [m2, m1] = parse(""" + odoo.define('m2', function (require) { + var Item = require('m1'); + return { + Item: Item + }; + }); + odoo.define('m1', function (require) { + var Class = require('Class'); + var Item = Class.extend({}); + return Item; + }); + """) + assert type(m2.exports) == jsdoc.NSDoc + it = m2.exports.get_property('Item') + assert type(it) == jsdoc.ClassDoc + assert it['sourcemodule'] is m1 + assert sorted([n for n, _ in m1.properties]) == ['<exports>', 'Class', 'Item'] + +def test_reassign(): + [m] = parse(""" + odoo.define('m', function (require) { + var Class = require('Class'); + /** local class */ + var Class = Class.extend({}); + return Class + }); + """) + assert m.exports.doc == 'local class' + # can't use equality or identity so use class comment... + assert m.exports.superclass.doc == 'Base Class' + +def test_attr(): + [m1, m2] = parse(""" + odoo.define('m1', function (require) { + var Class = require('Class'); + var Item = Class.extend({}); + return {Item: Item}; + }); + odoo.define('m2', function (require) { + var Item = require('m1').Item; + Item.include({}); + return Item.extend({}); + }); + """) + assert type(m2.exports) == jsdoc.ClassDoc + # that these two are resolved separately may be an issue at one point (?) + assert m2.exports.superclass.to_dict() == m1.exports.get_property('Item').to_dict() + +def test_nothing_implicit(): + [m] = parse(""" + odoo.define('m', function () { + }); + """) + assert m.exports is None + +def test_nothing_explicit(): + [m] = parse(""" + odoo.define('m', function () { + return; + }); + """) + assert m.exports is None diff --git a/doc/_extensions/autojsdoc/parser/tests/test_namespace.py b/doc/_extensions/autojsdoc/parser/tests/test_namespace.py new file mode 100644 index 00000000000..54cbcf8cbb4 --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/tests/test_namespace.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +from autojsdoc.parser import jsdoc +from support import parse, params + + +def test_empty(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + return {}; + }); + """) + assert type(mod.exports) == jsdoc.NSDoc + assert mod.exports.properties == [] + +def test_inline(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + return { + /** + * a thing + * @type {Boolean} + */ + a: true + } + }); + """) + assert isinstance(mod.exports, jsdoc.NSDoc) + [(n, p)] = mod.exports.properties + assert n == 'a' + assert params(p) == ('a', 'Boolean', 'a thing') + +def test_header(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + /** + * @property {Boolean} a a thing + */ + return { a: true } + }); + """) + assert isinstance(mod.exports, jsdoc.NSDoc) + [(_, p)] = mod.exports.properties + assert params(p) == ('a', 'Boolean', 'a thing') + +def test_header_conflict(): + """ should the header or the inline comment take precedence? """ + [mod] = parse(""" + odoo.define('a.ns', function (r) { + /** + * @property {Boolean} a a thing + */ + return { + /** @type {String} */ + a: true + } + }); + """) + assert isinstance(mod.exports, jsdoc.NSDoc) + [(_, p)] = mod.exports.properties + assert params(p) == ('a', 'Boolean', 'a thing') + +def test_mixin(): + [mod] = parse(""" + odoo.define('a.mixin', function (r) { + /** + * @mixin + */ + return { + /** + * @returns {Number} a number + */ + do_thing: function other() { return 42; } + } + }); + """) + assert isinstance(mod.exports, jsdoc.MixinDoc) + [(n, p)] = mod.exports.properties + assert n == 'do_thing' + assert params(p) == ('other', 'Function', '') + assert params(p.return_val) == ('', 'Number', 'a number') + +def test_literal(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + /** whop whop */ + return { + 'a': 1, + /** wheee */ + 'b': 2, + }; + }); + """) + assert mod.exports.doc == 'whop whop' + [(_1, a), (_2, b)] = mod.exports.properties + assert params(a) == ('a', 'Number', '') + assert params(b) == ('b', 'Number', 'wheee') + +def test_fill_ns(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + var Class = r('Class'); + var ns = {}; + /** ok */ + ns.a = 1; + /** @type {String} */ + ns['b'] = 2; + /** Ike */ + ns.c = Class.extend({}); + ns.d = function () {} + return ns; + }); + """) + ns = mod.exports + assert type(ns) == jsdoc.NSDoc + [(_a, a), (_b, b), (_c, c), (_d, d)] = ns.properties + assert params(a) == ('a', 'Number', 'ok') + assert params(b) == ('b', 'String', '') + assert type(c) == jsdoc.ClassDoc + assert type(d) == jsdoc.FunctionDoc + +def test_extend_other(): + [o, b] = parse(""" + odoo.define('a.ns', function () { + /** @name outer */ + return { + /** @name inner */ + a: {} + }; + }); + odoo.define('b', function (r) { + var o = r('a.ns'); + var Class = r('Class'); + /** Class 1 */ + o.a.b = Class.extend({m_b: 1}); + /** Class 2 */ + o.a['c'] = Class.extend({m_c: 1}); + }); + """) + [(_, m)] = o.exports.properties + assert type(m) == jsdoc.NSDoc + + b = m.get_property('b') + assert type(b) == jsdoc.ClassDoc + assert b.get_property('m_b') + assert b.doc == 'Class 1' + + c = m.get_property('c') + assert type(c) == jsdoc.ClassDoc + assert c.get_property('m_c') + assert c.doc == 'Class 2' + +def test_ns_variables(): + [mod] = parse(""" + odoo.define('A', function (r) { + var Class = r('Class'); + var Thing = Class.extend({}); + return { + Thing: Thing + }; + }); + """) + p = mod.exports.get_property('Thing') + assert type(p) == jsdoc.ClassDoc + +def test_diff(): + """ Have the NS key and the underlying object differ + """ + [mod] = parse(""" + odoo.define('mod', function (r) { + var Class = r('Class'); + var Foo = Class.extend({}); + return { Class: Foo }; + }); + """) + c = mod.exports.get_property('Class') + assert type(c) == jsdoc.ClassDoc + assert c.name == 'Foo' diff --git a/doc/_extensions/autojsdoc/parser/tests/test_params.py b/doc/_extensions/autojsdoc/parser/tests/test_params.py new file mode 100644 index 00000000000..c51139b8cb3 --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/tests/test_params.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +from autojsdoc.parser.jsdoc import ParamDoc + +"Lorem ipsum dolor sit amet, consectetur adipiscing elit." + +def test_basic(): + d = ParamDoc("Lorem ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'Lorem', + 'type': '', + 'optional': False, + 'default': None, + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + + d = ParamDoc("{Lorem} ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'ipsum', + 'type': 'Lorem', + 'optional': False, + 'default': None, + 'doc': 'dolor sit amet, consectetur adipiscing elit.', + } + +def test_optional(): + d = ParamDoc("[Lorem] ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'Lorem', + 'type': '', + 'optional': True, + 'default': None, + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + + d = ParamDoc("[Lorem=42] ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'Lorem', + 'type': '', + 'optional': True, + 'default': "42", + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + + d = ParamDoc("{Lorem} [ipsum] dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'ipsum', + 'type': 'Lorem', + 'optional': True, + 'default': None, + 'doc': 'dolor sit amet, consectetur adipiscing elit.', + } + + d = ParamDoc("{Lorem} [ipsum=42] dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'ipsum', + 'type': 'Lorem', + 'optional': True, + 'default': '42', + 'doc': 'dolor sit amet, consectetur adipiscing elit.', + } + +def test_returns(): + d = ParamDoc("{} ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': '', + 'type': '', + 'optional': False, + 'default': None, + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + d = ParamDoc("{Lorem} ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': '', + 'type': 'Lorem', + 'optional': False, + 'default': None, + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + +def test_odd(): + d = ParamDoc("{jQuery} [$target] the node where content will be prepended").to_dict() + assert d == { + 'name': '$target', + 'type': 'jQuery', + 'optional': True, + 'default': None, + 'doc': 'the node where content will be prepended', + } + + d = ParamDoc("""{htmlString} [content] DOM element, + array of elements, HTML string or jQuery object to prepend to $target""").to_dict() + assert d == { + 'name': 'content', + 'type': 'htmlString', + 'optional': True, + 'default': None, + 'doc': "DOM element,\n array of elements, HTML string or jQuery object to prepend to $target", + } + + d = ParamDoc("{Boolean} [options.in_DOM] true if $target is in the DOM").to_dict() + assert d == { + 'name': 'options.in_DOM', + 'type': 'Boolean', + 'optional': True, + 'default': None, + 'doc': 'true if $target is in the DOM', + } + + d = ParamDoc("Lorem\n ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d['doc'] == 'ipsum dolor sit amet, consectetur adipiscing elit.' + + d = ParamDoc("Lorem - ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d['doc'] == 'ipsum dolor sit amet, consectetur adipiscing elit.' + + d = ParamDoc("Lorem : ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d['doc'] == 'ipsum dolor sit amet, consectetur adipiscing elit.' diff --git a/doc/_extensions/autojsdoc/parser/tests/test_typespec.py b/doc/_extensions/autojsdoc/parser/tests/test_typespec.py new file mode 100644 index 00000000000..d90fe7995dd --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/tests/test_typespec.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import pytest + +from autojsdoc.parser import types + +def test_parser(): + assert types.parse("MouseEvent|TouchEvent") == types.Alt([ + types.Type('MouseEvent', []), + types.Type('TouchEvent', []), + ]) + + assert types.parse('Deferred<Object>') == types.Alt([ + types.Type('Deferred', [ + types.Alt([types.Type('Object', [])]) + ]) + ]) + + assert types.parse('Object|undefined') == types.Alt([ + types.Type('Object', []), + types.Literal('undefined'), + ]) + assert types.parse('any[]') == types.Alt([ + types.Type('Array', [types.Type('any', [])]) + ]) + assert types.parse("'xml' | 'less'") == types.Alt([ + types.Literal('xml'), + types.Literal('less'), + ]) + assert types.parse('Foo<Bar | Bar[] | null>') == types.Alt([ + types.Type('Foo', [types.Alt([ + types.Type('Bar', []), + types.Type('Array', [types.Type('Bar', [])]), + types.Literal('null'), + ])]) + ]) + +def test_tokens(): + toks = list(types.tokenize('A')) + assert toks == [(types.NAME, 'A')] + + toks = list(types.tokenize('"foo" | "bar"')) + assert toks == [(types.LITERAL, 'foo'), (types.OP, '|'), (types.LITERAL, 'bar')] + + toks = list(types.tokenize('1 or 2')) + assert toks == [(types.LITERAL, 1), (types.OP, '|'), (types.LITERAL, 2)] + + toks = list(types.tokenize('a.b.c.d')) + assert toks == [ + (types.NAME, 'a'), (types.OP, '.'), + (types.NAME, 'b'), (types.OP, '.'), + (types.NAME, 'c'), (types.OP, '.'), + (types.NAME, 'd'), + ] + + toks = list(types.tokenize('Function<String, Object>')) + assert toks == [ + (types.NAME, 'Function'), + (types.OP, '<'), + (types.NAME, 'String'), + (types.OP, ','), + (types.NAME, 'Object'), + (types.OP, '>') + ] + +def test_peekable(): + p = types.Peekable(range(5)) + + assert p.peek() == 0 + assert next(p) == 0, "consuming should yield previously peeked value" + next(p) + next(p) + assert next(p) == 3 + assert p.peek() == 4 + next(p) + assert p.peek(None) is None + with pytest.raises(StopIteration): + p.peek() + with pytest.raises(StopIteration): + next(p) diff --git a/doc/_extensions/autojsdoc/parser/types.py b/doc/_extensions/autojsdoc/parser/types.py new file mode 100644 index 00000000000..ecbccfe8651 --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/types.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +import ast +import collections +import io +import token +from tokenize import generate_tokens + +LITERAL_NAME = ('null', 'undefined', 'true', 'false') + +__all__ = ['tokenize', 'parse', 'iterate', 'Alt', 'Type', 'Literal'] + +class _Marker(object): + __slots__ = ['name'] + def __init__(self, name): self.name = name + def __repr__(self): return '<%s>' % self.name + +OP = _Marker('OP') +NAME = _Marker('NAME') +LITERAL = _Marker('LITERAL') +def tokenize(typespec): + typespec = typespec.replace('$', 'jQuery') + for toktype, string, _, _, _ in generate_tokens(io.StringIO(typespec).readline): + if toktype == token.NAME: + if string == 'or': # special case "A or B" + # TODO: deprecation warning + yield (OP, '|') + else: + yield (NAME, string) + elif toktype == token.OP: + if string in '|<>[].,': + yield (OP, string) + elif string == '*': # maybe? + yield (NAME, 'any') + elif string in '()': + # TODO: deprecation warning + # seems useless unless we add support for "..." + continue + else: + raise ValueError("Unknown typespec operator %r" % string) + elif toktype in (token.STRING, token.NUMBER): # enum-ish + yield (LITERAL, ast.literal_eval(string)) + elif toktype == token.ENDMARKER: + return + else: + raise ValueError("Unknown typespec token %s" % token.tok_name[toktype]) + +Alt = collections.namedtuple('Alt', ['params']) +Literal = collections.namedtuple('Literal', ['value']) +Type = collections.namedtuple('Type', ['name', 'params']) + +def iterate(t, fortype, forliteral): + if isinstance(t, Alt): + ps = Peekable(t.params) + for param in ps: + for it in iterate(param, fortype, forliteral): + yield it + if ps.peek(None): + yield forliteral('|') + elif isinstance(t, Type): + yield fortype(t.name) + if not t.params: + return + yield forliteral('<') + ps = Peekable(t.params) + for param in ps: + for it in iterate(param, fortype, forliteral): + yield it + if ps.peek(None): + yield forliteral(',') + yield forliteral('>') + elif isinstance(t, Literal): + yield forliteral(t.value) + else: + raise ValueError('Unknown item %s' % t) + +def parse(typespec): + tokens = Peekable(tokenize(typespec)) + return parse_alt(tokens) + +# alt = param +# | param '|' alt +def parse_alt(tokens): + alt = Alt([]) + while True: + alt.params.append(parse_param(tokens)) + if not next_is(tokens, (OP, '|')): + break + return alt + +# param = 'null' | 'undefined' | 'true' | 'false' +# | number +# | string +# | typename '[]' +# X '[' alt ']' +# | typename '<' params '>' +# params = alt | alt ',' params +def parse_param(tokens): + t, v = tokens.peek() + if t == LITERAL or (t == NAME and v in LITERAL_NAME): + next(tokens) + return Literal(str(v)) + + # # [typespec] # should this be Array<T> or an n-uple allowing multiple items? + # if tok == (OP, '['): + # next(tokens) + # t = Type('Array', [parse_alt(tokens)]) + # rest = next(tokens) + # expect(rest, (OP, ']')) + # return t + + t = Type(parse_typename(tokens), []) + # type[] + peek = tokens.peek(None) + if peek == (OP, '['): + next(tokens) # skip peeked + expect(next(tokens), (OP, ']')) + return Type('Array', [t]) + # type<typespec, ...> + if peek == (OP, '<'): + next(tokens) # skip peeked + while True: + t.params.append(parse_alt(tokens)) + n = next(tokens) + if n == (OP, ','): + continue + if n == (OP, '>'): + break + raise ValueError("Expected OP ',' or OP ',', got %s '%s'" % n) + return t + +# typename = name | name '.' typename +def parse_typename(tokens): + typename = [] + while True: + (t, n) = next(tokens) + if t != NAME: + raise ValueError("Expected a name token, got %s '%s'" % (t, n)) + if n in LITERAL_NAME: + raise ValueError("Expected a type name, got literal %s" % n) + typename.append(n) + if not next_is(tokens, (OP, '.')): + break + return '.'.join(typename) + +def next_is(tokens, expected): + """ + Consumes the next token if it's `expected` otherwise does not touch the + tokens. + + Returns whether it consumed a token + """ + if tokens.peek(None) == expected: + next(tokens) + return True + return False + +def expect(actual, expected): + """ Raises ValueError if `actual` and `expected` are different + + :type actual: (object, str) + :type expected: (object, str) + """ + if actual != expected: + raise ValueError("Expected %s '%s', got %s '%s'" % (expected + actual)) + + +NONE = object() +class Peekable(object): + __slots__ = ['_head', '_it'] + def __init__(self, iterable): + self._head = NONE + self._it = iter(iterable) + + def __iter__(self): + return self + def __next__(self): + if self._head is not NONE: + r, self._head = self._head, NONE + return r + return next(self._it) + next = __next__ + + def peek(self, default=NONE): + if self._head is NONE: + try: + self._head = next(self._it) + except StopIteration: + if default is not NONE: + return default + raise + return self._head diff --git a/doc/_extensions/autojsdoc/parser/utils.py b/doc/_extensions/autojsdoc/parser/utils.py new file mode 100644 index 00000000000..b8b1a97e7aa --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/utils.py @@ -0,0 +1,295 @@ +# -*- coding: utf-8 -*- +def _name(node): + if node['type'] == 'Identifier': + return node['name'] + if node['type'] == 'MemberExpression': + return "%s.%s" % (_name(node['object']), _name(node['property'])) + raise ValueError("Unnamable node type %s" % node['type']) + +def _value(node, strict=False): + t = node['type'] + if t == 'Identifier': + return node['name'] + elif t == 'Literal': + return node['value'] + msg = '<%s has no value>' % _identify(node, {}) + if strict: + raise ValueError(msg) + return msg + +def _identify(valnode, ns): + if valnode is None: + return "None" + # already identified and re-set in the ns? + if isinstance(valnode, Printable): + return valnode + + # check other non-empty returns + t = valnode['type'] + if t == "Literal": + if valnode.get('regex'): + return valnode['regex']['pattern'] + return valnode['value'] + elif t == "Identifier": + n = valnode['name'] + return ns.get(n, Global(n)) + elif t == "NewExpression": + return Instance(_identify(valnode['callee'], ns)) + elif t == "ObjectExpression": + return Namespace({ + _value(prop['key']): _identify(prop['value'], ns) + for prop in valnode['properties'] + }) + elif t == "MemberExpression": + return Deref( + _identify(valnode['object'], ns), + _identify(valnode['property'], ns) if valnode['computed'] else valnode['property']['name'], + ) + elif t == "CallExpression": + return Call( + _identify(valnode['callee'], ns), + [_identify(arg, ns) for arg in valnode['arguments']], + ) + elif t == 'BinaryExpression': + return "%s %s %s" % ( + _identify(valnode['left'], ns), + valnode['operator'], + _identify(valnode['right'], ns), + ) + else: + return t + +class Counter(object): + __slots__ = ['_number'] + def __init__(self): + self._number = 0 + + def __bool__(self): + return bool(self._number) + __nonzero__ = __bool__ + def __int__(self): + return self._number + + def get(self): + return self._number + def increment(self): + self._number += 1 + return self._number + def decrement(self): + self._number -= 1 + return self._number + + def __enter__(self): + self._number += 1 + return self._number + def __exit__(self, *args): + self._number -= 1 + +class Printable(object): + __slots__ = [] + depth = Counter() + def __repr__(self): + with self.depth as i: + if i > 2: + return "%s(...)" % type(self).__name__ + return "%s(%s)" % (type(self).__name__, ', '.join( + "%s=%r" % (k, getattr(self, k)) + for k in self.__slots__ + if not k.startswith('_') + if getattr(self, k) + )) + + +def resolve(obj, store): + if obj is None: + return None + + if isinstance(obj, (unicode, bool, int, float)): + return obj + + if isinstance(obj, ModuleProxy): + return resolve(obj.get(), store) + + try: + if getattr(obj, 'resolved', False): + return obj + return obj.resolve(store) + except AttributeError: + raise TypeError("Unresolvable {!r}".format(obj)) + +class Resolvable(Printable): + """ + For types resolved in place + """ + __slots__ = ['resolved'] + + def __init__(self): + super(Resolvable, self).__init__() + self.resolved = False + def resolve(self, store): + self.resolved = True + return self + +class Namespace(Resolvable): + __slots__ = ['attrs'] + def __init__(self, attrs): + super(Namespace, self).__init__() + self.attrs = attrs + def resolve(self, store): + r = super(Namespace, self).resolve(store) + self.attrs = { + k: resolve(v, store) + for k, v in self.attrs.iteritems() + } + return r + def __getitem__(self, key): + assert isinstance(key, unicode), "%r is not a namespace key" % key + try: + return self.attrs[key] + except KeyError: + return Global(self)[key] + +class Module(Resolvable): + __slots__ = ('name', 'comments', 'exports', 'augments', 'dependencies', '_store') + def __init__(self, name, store, comments=()): + super(Module, self).__init__() + self.name = name + self._store = store + self.comments = tuple(comments) + self.exports = None + self.augments = [] + self.dependencies = set() + store[name] = self + def add_dependency(self, depname): + dep = ModuleProxy(depname, self._store) + self.dependencies.add(dep) + return dep + def get(self): + return self + def resolve(self, store=None): + r = super(Module, self).resolve(store) + self.exports = resolve(self.exports, self._store) + self.augments = [resolve(a, self._store) for a in self.augments] + self.dependencies = { + resolve(d, self._store) + for d in self.dependencies + } + return r + def __getitem__(self, k): + try: + return self.exports[k] + except KeyError: + return Global(self)[k] + +class ModuleProxy(object): + def __init__(self, name, store): + self.name = name + self.store = store + def get(self): + # web.web_client is not an actual module + return self.store.get(self.name, '<unstored %s>' % self.name) + def __repr__(self): + return repr(self.get()) + def __hash__(self): + return hash(self.name) + def __eq__(self, other): + return self.name == other.name + +class Class(Resolvable): + __slots__ = ['name', 'members', 'extends', 'mixins', 'comments'] + def __init__(self, name, extends): + super(Class, self).__init__() + self.name = name + self.extends = extends + self.members = {} + self.mixins = [] + self.comments = [] + + def resolve(self, store): + r = super(Class, self).resolve(store) + self.extends = resolve(self.extends, store) + self.mixins = [resolve(m, store) for m in self.mixins] + return r + +class Instance(Resolvable): + __slots__ = ['type'] + def __init__(self, type): + super(Instance, self).__init__() + self.type = type + + def resolve(self, store): + r = super(Instance, self).resolve(store) + self.type = resolve(self.type, store) + if isinstance(self.type, Module): + self.type = self.type.exports + return r + + def __getitem__(self, key): + return Global(self)['key'] + +class Function(Resolvable): + __slots__ = ['comments'] + def __init__(self, comments): + super(Function, self).__init__() + self.comments = comments + +class Deref(Printable): + __slots__ = ['object', 'property'] + def __init__(self, object, property): + self.object = object + self.property = property + def resolve(self, store): + return resolve(self.object, store)[self.property] + +class Call(Resolvable): + __slots__ = ['callable'] + def __init__(self, callable, arguments): + super(Call, self).__init__() + self.callable = callable + def resolve(self, store): + r = super(Call, self).resolve(store) + self.callable = resolve(self.callable, store) + return r + def __getitem__(self, item): + return Global(self)[item] + +def get(node, it, *path): + if not path: + return node[it] + return get(node[it], *path) + + +def match(node, pattern): + """Checks that ``pattern`` is a subset of ``node``. + + If a sub-pattern is a callable it will be called with the current + sub-node and its result is the match's. Other sub-patterns are + checked by equality against the correspoding sub-node. + + Can be used to quickly check the descendants of a node of suitable + type. + + TODO: support checking list content? Would be convenient to + constraint e.g. CallExpression based on their parameters + + """ + if node is None and pattern is not None: + return + if callable(pattern): + return pattern(node) + if isinstance(node, list): + return all(len(node) > i and match(node[i], v) for i, v in pattern.items()) + if isinstance(pattern, dict): + return all(match(node.get(k), v) for k, v in pattern.items()) + return node == pattern + + +class Global(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return '<external %s>' % self.name + def __getitem__(self, name): + return Global('%s.%s' % (self.name, name)) + def resolve(self, store): + return self diff --git a/doc/_extensions/autojsdoc/parser/visitor.py b/doc/_extensions/autojsdoc/parser/visitor.py new file mode 100644 index 00000000000..e5529d511dc --- /dev/null +++ b/doc/_extensions/autojsdoc/parser/visitor.py @@ -0,0 +1,126 @@ +from pyjsparser.pyjsparserdata import Syntax + +_binary = lambda n: [n['left'], n['right']] +_function = lambda n: n['params'] + n['defaults'] + [n['body']] +# yields children in visitation order +_children = { + Syntax.ArrayExpression: lambda n: n['elements'], + Syntax.ArrayPattern: lambda n: n['elements'], + Syntax.ArrowFunctionExpression: _function, + Syntax.AssignmentExpression: _binary, + Syntax.AssignmentPattern: _binary, + Syntax.BinaryExpression: _binary, + Syntax.BlockStatement: lambda n: n['body'], + Syntax.BreakStatement: lambda n: [], + Syntax.CallExpression: lambda n: [n['callee']] + n['arguments'], + Syntax.CatchClause: lambda n: [n['param'], n['body']], + Syntax.ClassBody: lambda n: [n['body']], + Syntax.ClassDeclaration: lambda n: [n['superClass'], n['body']], + Syntax.ClassExpression: lambda n: [n['superClass'], n['body']], + Syntax.ConditionalExpression: lambda n: [n['test'], n['consequent'], n['alternate']], + Syntax.ContinueStatement: lambda n: [], + Syntax.DebuggerStatement: lambda n: [], + Syntax.DoWhileStatement: lambda n: [n['body'], n['test']], + Syntax.EmptyStatement: lambda n: [], + Syntax.ExportAllDeclaration: lambda n: [n['source']], + Syntax.ExportDefaultDeclaration: lambda n: [n['declaration']], + Syntax.ExportNamedDeclaration: lambda n: ([n['declaration']] if n['declaration'] else n['specifiers']) + [n['source']], + Syntax.ExportSpecifier: lambda n: [n['local'], n['exported']], + Syntax.ExpressionStatement: lambda n: [n['expression']], + Syntax.ForStatement: lambda n: [n['init'], n['test'], n['update'], n['body']], + Syntax.ForInStatement: lambda n: [n['left'], n['right'], n['body']], + Syntax.FunctionDeclaration: _function, + Syntax.FunctionExpression: _function, + Syntax.Identifier: lambda n: [], + Syntax.IfStatement: lambda n: [n['test'], n['consequent'], n['alternate']], + Syntax.ImportDeclaration: lambda n: n['specifiers'] + [n['source']], + Syntax.ImportDefaultSpecifier: lambda n: [n['local']], + Syntax.ImportNamespaceSpecifier: lambda n: [n['local']], + Syntax.ImportSpecifier: lambda n: [n['local'], n['imported']], + Syntax.LabeledStatement: lambda n: [n['body']], + Syntax.Literal: lambda n: [], + Syntax.LogicalExpression: _binary, + Syntax.MemberExpression: lambda n: [n['object'], n['property']], + #Syntax.MethodDefinition: lambda n: [], + Syntax.NewExpression: lambda n: [n['callee']] + n['arguments'], + Syntax.ObjectExpression: lambda n: n['properties'], + Syntax.ObjectPattern: lambda n: n['properties'], + Syntax.Program: lambda n: n['body'], + Syntax.Property: lambda n: [n['key'], n['value']], + Syntax.RestElement: lambda n: [n['argument']], + Syntax.ReturnStatement: lambda n: [n['argument']], + Syntax.SequenceExpression: lambda n: n['expressions'], + Syntax.SpreadElement: lambda n: [n['argument']], + Syntax.Super: lambda n: [], + Syntax.SwitchCase: lambda n: [n['test'], n['consequent']], + Syntax.SwitchStatement: lambda n: [n['discriminant']] + n['cases'], + Syntax.TaggedTemplateExpression: lambda n: [n['tag'], n['quasi']], + Syntax.TemplateElement: lambda n: [], + Syntax.TemplateLiteral: lambda n: n['quasis'] + n['expressions'], + Syntax.ThisExpression: lambda n: [], + Syntax.ThrowStatement: lambda n: [n['argument']], + Syntax.TryStatement: lambda n: [n['block'], n['handler'], n['finalizer']], + Syntax.UnaryExpression: lambda n: [n['argument']], + Syntax.UpdateExpression: lambda n: [n['argument']], + Syntax.VariableDeclaration: lambda n: n['declarations'], + Syntax.VariableDeclarator: lambda n: [n['id'], n['init']], + Syntax.WhileStatement: lambda n: [n['test'], n['body']], + Syntax.WithStatement: lambda n: [n['object'], n['body']], +} + +SKIP = object() +class Visitor(object): + """ + Generic visitor for the pyjsparser AST. + + Visitation is driven by the ``visit`` method, which iterates the tree in + depth-first pre-order. + + For each node, calls ``enter_$NODETYPE``, visits the children then calls + ``exit_$NODETYPE``. If the enter or exit methods are not present on the + visitor, falls back on ``enter_generic`` and ``exit_generic``. + + Any ``enter_`` method can return ``SKIP`` to suppress both the traversal + of the subtree *and* the call to the corresponding ``exit_`` method + (whether generic or specific). + + For convenience, ``visit`` will return whatever is set as the visitor's + ``result`` attribute, ``None`` by default. + + ``visit`` can be given multiple root nodes, and it can be called multiple + times. The ``result`` attribute is cleared at each call but not between + two roots of the same ``visit`` call. + """ + def __init__(self): + super(Visitor, self).__init__() + self.result = None + + def enter_generic(self, node): pass + def exit_generic(self, node): pass + + def visit(self, nodes): + if isinstance(nodes, dict): + nodes = [nodes] + # if multiple nodes are passed in, we need to reverse the order in + # order to traverse front-to-back rather than the other way around + nodes = list(reversed(nodes)) + + while nodes: + node = nodes.pop() + # should probably filter None descendants in _children... + if node is None: + continue + node_type = node['type'] + if node_type == '_exit': + node = node['node'] + getattr(self, 'exit_' + node['type'], self.exit_generic)(node) + continue + + if getattr(self, 'enter_' + node_type, self.enter_generic)(node) is SKIP: + continue + + nodes.append({'type': '_exit', 'node': node}) + nodes.extend(reversed(_children[node_type](node))) + + return self.result + |
