From b26c534b70935d54487781234a7b3731d2e6fd96 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Tue, 14 Feb 2017 13:59:37 -0800 Subject: [PATCH] Re-switch order of arguments; defer handling semantics to typeanal --- mypy/exprtotype.py | 12 ++-- mypy/fastparse.py | 17 +++-- mypy/messages.py | 2 +- mypy/parsetype.py | 107 +++++----------------------- mypy/sharedparse.py | 12 ---- mypy/typeanal.py | 26 +++++-- mypy/types.py | 16 +---- test-data/unit/check-functions.test | 25 +++---- 8 files changed, 66 insertions(+), 151 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 9b271ede0a697..08b9c1a6a2349 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -3,9 +3,7 @@ from mypy.nodes import ( Expression, NameExpr, MemberExpr, IndexExpr, TupleExpr, ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, - ARG_POS, ARG_NAMED, ) -from mypy.sharedparse import ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS from mypy.parsetype import parse_str_as_type, TypeParseError from mypy.types import Type, UnboundType, ArgumentList, EllipsisType, AnyType, Optional @@ -56,14 +54,14 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: elif isinstance(expr, ListExpr): types = [] # type: List[Type] names = [] # type: List[Optional[str]] - constructors = [] # type: List[UnboundArgumentConstructor] + constructors = [] # type: List[Optional[str]] for it in expr.items: if isinstance(it, CallExpr): callee = it.callee if isinstance(callee, NameExpr): - constructor = UnboundArgumentConstructor(callee.name) - elif isinstance(it.callee, MemberExpr): - constructor = UnboundArgumentConstructor(get_member_expr_fullname(callee)) + constructor = callee.name + elif isinstance(callee, MemberExpr): + constructor = get_member_expr_fullname(callee) else: raise TypeTranslationError() name = None @@ -90,7 +88,7 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: else: types.append(expr_to_unanalyzed_type(it)) names.append(None) - constructors.append(UnboundArgumentConstructor(None)) + constructors.append(None) return ArgumentList(types, names, constructors, line=expr.line, column=expr.column) elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 53c41e8c251dc..8ff226a02c7f8 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -4,7 +4,6 @@ from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List, Set from mypy.sharedparse import ( special_function_elide_names, argument_elide_name, - ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS, ) from mypy.nodes import ( MypyFile, Node, ImportBase, Import, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, @@ -953,22 +952,21 @@ def translate_expr_list(self, l: Sequence[ast3.AST]) -> List[Type]: def translate_argument_list(self, l: Sequence[ast3.AST]) -> ArgumentList: types = [] # type: List[Type] names = [] # type: List[Optional[str]] - constructors = [] # type: List[UnboundArgumentConstructor] + constructors = [] # type: List[Optional[str]] for e in l: - constructor = UnboundArgumentConstructor(None) + constructor = None # type: Optional[str] if isinstance(e, ast3.Call): name = None # type: Optional[str] # Parse the arg constructor f = e.func if isinstance(f, ast3.Name): - constructor = UnboundArgumentConstructor(f.id, line=self.line) + constructor = f.id elif isinstance(f, ast3.NameConstant): - constructor = UnboundArgumentConstructor(str(f.value), line=self.line) + constructor = str(f.value) elif isinstance(f, ast3.Attribute): before_dot = self.visit(f.value) if isinstance(before_dot, UnboundType): - constructor = UnboundArgumentConstructor( - "{}.{}".format(before_dot.name, f.attr), line = self.line) + constructor = "{}.{}".format(before_dot.name, f.attr) typ = AnyType(implicit=True) # type: Type if len(e.args) >= 1: @@ -980,7 +978,7 @@ def translate_argument_list(self, l: Sequence[ast3.AST]) -> ArgumentList: f.lineno, getattr(f, 'col_offset', -1)) for k in e.keywords: value = k.value - if k.arg == "name" and not star: + if k.arg == "name": name = self._extract_str(value) elif k.arg == "typ": typ = self.visit(value) @@ -1052,7 +1050,7 @@ def visit_Ellipsis(self, n: ast3.Ellipsis) -> Type: def visit_List(self, n: ast3.List) -> Type: return self.translate_argument_list(n.elts) - def _extract_str(arg: ast3.expr) -> Optional[str]: + def _extract_str(self, arg: ast3.expr) -> Optional[str]: if isinstance(arg, ast3.Name) and arg.id == 'None': return None elif isinstance(arg, ast3.NameConstant) and arg.value is None: @@ -1061,3 +1059,4 @@ def _extract_str(arg: ast3.expr) -> Optional[str]: return arg.s else: self.fail("Bad type for name of argument", arg.lineno, arg.col_offset) + return None diff --git a/mypy/messages.py b/mypy/messages.py index e6491a1df2b78..e598abbe0beff 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -209,7 +209,7 @@ def format(self, typ: Type, verbosity: int = 0) -> str: verbosity = max(verbosity - 1, 0)))) else: constructor = ARG_CONSTRUCTOR_NAMES[arg_kind] - if arg_kind in (ARG_STAR, ARG_STAR2): + if arg_kind in (ARG_STAR, ARG_STAR2) or arg_name is None: arg_strings.append("{}({})".format( constructor, strip_quotes(self.format(arg_type)))) diff --git a/mypy/parsetype.py b/mypy/parsetype.py index fbf6376226f5b..354f2ed79c852 100644 --- a/mypy/parsetype.py +++ b/mypy/parsetype.py @@ -1,20 +1,14 @@ """Type parser""" -from typing import List, Tuple, Union, Optional, TypeVar, cast - -import typing +from typing import List, Tuple, Union, Optional from mypy.types import ( Type, UnboundType, TupleType, ArgumentList, CallableType, StarType, - EllipsisType, AnyType, ArgNameException, ArgKindException + EllipsisType ) - -from mypy.sharedparse import ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS - from mypy.lex import Token, Name, StrLit, lex from mypy import nodes -T = TypeVar('T', bound=Token) none = Token('') # Empty token @@ -108,80 +102,23 @@ def parse_types(self) -> Type: type = TupleType(items, None, type.line, implicit=True) return type - def parse_argument_spec(self) -> Tuple[Type, Optional[str], int]: - current = self.current_token() - nxt = self.next_token() - # This is a small recreation of a subset of parsing a CallExpr; just - # enough to parse what happens in an arugment list. - # TODO: Doesn't handle an explicit name of None yet. - if isinstance(current, Name) and nxt is not None and nxt.string == '(': - arg_const = self.expect_type(Name).string - name = None # type: Optional[str] - typ = AnyType(implicit=True) # type: Type - try: - kind = ARG_KINDS_BY_CONSTRUCTOR[arg_const] - except KeyError: - raise self.parse_error("Unknown argument constructor {}".format(arg_const)) - name, typ = self.parse_arg_args(read_name = arg_const not in STAR_ARG_CONSTRUCTORS) - return typ, name, kind - else: - return self.parse_type(), None, nodes.ARG_POS - - def parse_arg_args(self, *, read_name: bool) -> Tuple[Optional[str], Optional[Type]]: - self.expect('(') - name = None # type: Optional[str] - typ = AnyType(implicit=True) # type: Type - i = 0 - while self.current_token_str() != ')': - if i > 0: - self.expect(',') - if self.next_token() and self.next_token().string == '=': - arg_arg_name = self.current_token_str() - if arg_arg_name == 'name' and read_name: - self.expect('name') - self.expect('=') - if self.current_token_str() == 'None': - self.expect('None') - else: - name = self.expect_type(StrLit).parsed() - elif arg_arg_name == 'typ': - self.expect('typ') - self.expect('=') - typ = self.parse_type() - else: - raise self.parse_error( - 'Unexpected argument "{}" for argument constructor'.format(arg_arg_name)) - elif i == 0 and read_name: - if self.current_token_str() == 'None': - self.expect('None') - else: - name = self.expect_type(StrLit).parsed() - elif i == 0 and not read_name or i == 1 and read_name: - typ = self.parse_type() - else: - raise self.parse_error("Unexpected argument for argument constructor") - i += 1 - self.expect(')') - return name, typ - def parse_argument_list(self) -> ArgumentList: """Parse type list [t, ...].""" lbracket = self.expect('[') commas = [] # type: List[Token] items = [] # type: List[Type] - names = [] # type: List[Optional[str]] - kinds = [] # type: List[int] while self.current_token_str() != ']': - t, name, kind = self.parse_argument_spec() + t = self.parse_type() items.append(t) - names.append(name) - kinds.append(kind) - if self.current_token_str() != ',': break commas.append(self.skip()) self.expect(']') - return ArgumentList(items, names, kinds, line=lbracket.line) + return ArgumentList( + items, + [None] * len(items), + [None] * len(items), + line=lbracket.line) def parse_named_type(self) -> Type: line = self.current_token().line @@ -235,26 +172,21 @@ def expect(self, string: str) -> Token: else: raise self.parse_error() - def expect_type(self, typ: typing.Type[T]) -> T: + def expect_type(self, typ: type) -> Token: if isinstance(self.current_token(), typ): self.ind += 1 - return cast(T, self.tok[self.ind - 1]) + return self.tok[self.ind - 1] else: raise self.parse_error() def current_token(self) -> Token: return self.tok[self.ind] - def next_token(self) -> Optional[Token]: - if self.ind + 1 >= len(self.tok): - return None - return self.tok[self.ind + 1] - def current_token_str(self) -> str: return self.current_token().string - def parse_error(self, message: Optional[str] = None) -> TypeParseError: - return TypeParseError(self.tok[self.ind], self.ind, message=message) + def parse_error(self) -> TypeParseError: + return TypeParseError(self.tok[self.ind], self.ind) def parse_str_as_type(typestr: str, line: int) -> Type: @@ -279,8 +211,6 @@ def parse_signature(tokens: List[Token]) -> Tuple[CallableType, int]: i = 0 if tokens[i].string != '(': raise TypeParseError(tokens[i], i) - begin = tokens[i] - begin_idx = i i += 1 arg_types = [] # type: List[Type] arg_kinds = [] # type: List[int] @@ -316,11 +246,8 @@ def parse_signature(tokens: List[Token]) -> Tuple[CallableType, int]: raise TypeParseError(tokens[i], i) i += 1 ret_type, i = parse_type(tokens, i) - try: - return CallableType(arg_types, - arg_kinds, - [None] * len(arg_types), - ret_type, None, - is_ellipsis_args=encountered_ellipsis), i - except (ArgKindException, ArgNameException) as e: - raise TypeParseError(begin, begin_idx, e.message) + return CallableType(arg_types, + arg_kinds, + [None] * len(arg_types), + ret_type, None, + is_ellipsis_args=encountered_ellipsis), i diff --git a/mypy/sharedparse.py b/mypy/sharedparse.py index 148b151f12607..cc515298d8bf2 100644 --- a/mypy/sharedparse.py +++ b/mypy/sharedparse.py @@ -94,18 +94,6 @@ MAGIC_METHODS_POS_ARGS_ONLY = MAGIC_METHODS - MAGIC_METHODS_ALLOWING_KWARGS -ARG_KINDS_BY_CONSTRUCTOR = { - 'Arg': ARG_POS, - 'DefaultArg': ARG_OPT, - 'NamedArg': ARG_NAMED, - 'DefaultNamedArg': ARG_NAMED_OPT, - 'StarArg': ARG_STAR, - 'KwArg': ARG_STAR2, -} - -STAR_ARG_CONSTRUCTORS = {'StarArg', 'KwArg'} - - def special_function_elide_names(name: str) -> bool: return name in MAGIC_METHODS_POS_ARGS_ONLY diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 0f55288b1535d..a2a121daccf4e 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -342,19 +342,18 @@ def visit_type_type(self, t: TypeType) -> Type: def argument_list_to_kinds(self, a: ArgumentList) -> List[int]: res = [] # type: List[int] for constructor in a.constructors: - if constructor.name is None: + if constructor is None: res.append(nodes.ARG_POS) else: - sym = self.lookup(constructor.name, constructor) - if sym.node is None: - assert sym.kind == UNBOUND_IMPORTED - self.fail("Argument constructor not found: {}".format(constructor.name), - constructor) + sym = self.lookup(constructor, a) + if sym is None or sym.node is None: + self.fail("Argument constructor not found: {}".format(constructor), + a) res.append(nodes.ARG_POS) continue fullname = sym.node.fullname() if fullname not in ARG_KINDS_BY_CONSTRUCTOR: - self.fail("Not an argument constructor: {}".format(fullname), constructor) + self.fail("Unknown argument constructor {}".format(constructor), a) res.append(nodes.ARG_POS) else: res.append(ARG_KINDS_BY_CONSTRUCTOR[fullname]) @@ -375,6 +374,19 @@ def analyze_callable_type(self, t: UnboundType) -> Type: if isinstance(t.args[0], ArgumentList): # Callable[[ARG, ...], RET] (ordinary callable type) kinds = self.argument_list_to_kinds(t.args[0]) + names = t.args[0].names + seen_names = set() # type: Set[str] + for name, kind, const in zip(names, kinds, t.args[0].constructors): + if name is not None: + if kind in {nodes.ARG_STAR, nodes.ARG_STAR2}: + self.fail( + "{} should not have a specified name".format(const), + t.args[0]) + if name in seen_names: + self.fail( + "duplicate argument '{}' in callable type".format(name), + t.args[0]) + seen_names.add(name) args = t.args[0].types try: return CallableType(self.anal_array(args), diff --git a/mypy/types.py b/mypy/types.py index a399f8a2eab27..6ee9e6eb26680 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -241,12 +241,12 @@ class ArgumentList(Type): types = None # type: List[Type] names = None # type: List[Optional[str]] - constructors = None # type: List[UnboundArgumentConstructor] + constructors = None # type: List[Optional[str]] def __init__(self, types: List[Type], names: List[Optional[str]], - constructors: List[UnboundArgumentConstructor], + constructors: List[Optional[str]], line: int = -1, column: int = -1) -> None: super().__init__(line, column) @@ -268,7 +268,7 @@ def deserialize(cls, data: JsonDict) -> 'ArgumentList': assert data['.class'] == 'ArgumentList' or data['.class'] == 'TypeList' types = [Type.deserialize(t) for t in data['items']] names = cast(List[Optional[str]], data.get('names', [None] * len(types))) - constructors = [UnboundArgumentConstructor.deserialize(c) for c in data['constructors']] + constructors = data['constructors'] return ArgumentList(types=types, names=names, constructors=constructors) @@ -625,7 +625,6 @@ def __init__(self, special_sig: Optional[str] = None, ) -> None: self._process_kinds_on_init(arg_kinds) - self._process_names_on_init(arg_names) if variables is None: variables = [] assert len(arg_types) == len(arg_kinds) @@ -644,15 +643,6 @@ def __init__(self, self.special_sig = special_sig super().__init__(line, column) - def _process_names_on_init(self, arg_names: Iterable[str]) -> None: - seen = set() # type: Set[str] - for name in arg_names: - if name is None: - continue - if name in seen: - raise ArgNameException('Duplicate argument name "{}"'.format(name)) - seen.add(name) - def _process_kinds_on_init(self, arg_kinds: Iterable[int]) -> None: self.is_var_arg = False self.is_kw_arg = False diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 90ca3f08fed8d..f610fd853d61b 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1416,9 +1416,9 @@ def WrongArg(x, y): return y # and a consequence of Callable being set to an int in the test stub. We can't set it to # something else sensible, because other tests require the stub not have anything # that looks like a function call. -F = Callable[[WrongArg(int, 'x')], int] # E: Invalid type alias # E: Value of type "int" is not indexable +F = Callable[[WrongArg(int, 'x')], int] # E: Unknown argument constructor WrongArg G = Callable[[Arg(1, 'x')], int] # E: Invalid type alias # E: Value of type "int" is not indexable -H = Callable[[StarArg(int, 'x')], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: Too many arguments for "StarArg" +H = Callable[[StarArg(int, 'x')], int] # E: StarArg should not have a specified name I = Callable[[StarArg(int)], int] # ok J = Callable[[StarArg(), KwArg()], int] # ok K = Callable[[StarArg(), int], int] # E: Required positional args may not appear after default, named or star args @@ -1435,13 +1435,14 @@ from mypy_extensions import Arg, StarArg, KwArg def WrongArg(x, y): return y -def a(f: Callable[[WrongArg(int, 'x')], int]): pass # E: Parse error before (: Unknown argument constructor WrongArg # E: Parse error before end of line -def b(f: Callable[[Arg(1, 'x')], int]): pass # E: Parse error before numeric literal # E: Parse error before end of line -def c(f: Callable[[StarArg('x', int)], int]): pass # E: Parse error before "int": Unexpected argument for argument constructor # E: Parse error before end of line +# This test only shows parse-time errors +def a(f: Callable[[WrongArg(int, 'x')], int]): pass # typeanal-time error +def b(f: Callable[[Arg(1, 'x')], int]): pass # E: invalid type comment or annotation +def c(f: Callable[[StarArg(int, 'x')], int]): pass # typeanal-time error def d(f: Callable[[StarArg(int)], int]): pass # ok def e(f: Callable[[StarArg(), KwArg()], int]): pass # ok def g(f: Callable[[Arg(name='x', typ=int)], int]): pass # ok -def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Parse error before "gnome": Unexpected argument "gnome" for argument constructor # E: Parse error before end of line +def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok [builtins fixtures/dict.pyi] @@ -1457,20 +1458,20 @@ def a(f: Callable[[WrongArg(int, 'x')], int]): pass # E: Unknown argument constr # flags: --fast-parser from typing import Callable from mypy_extensions import Arg -def b(f: Callable[[Arg('x', 1)], int]): pass # E: Bad type for callable argument +def b(f: Callable[[Arg(1, 'x')], int]): pass # E: invalid type comment or annotation [builtins fixtures/dict.pyi] [case testCallableFastParseTooManyStarArg] # flags: --fast-parser from typing import Callable from mypy_extensions import StarArg -def c(f: Callable[[StarArg(int, 'x')], int]): pass # E: Too many arguments for argument constructor +def c(f: Callable[[StarArg(int, 'x')], int]): pass # E: StarArg should not have a specified name [builtins fixtures/dict.pyi] [case testCallableFastParseGood] # flags: --fast-parser from typing import Callable -from mypy_extensions import StarArg, Arg +from mypy_extensions import StarArg, Arg, KwArg def d(f: Callable[[StarArg(int)], int]): pass # ok def e(f: Callable[[StarArg(), KwArg()], int]): pass # ok def g(f: Callable[[Arg(name='x', typ=int)], int]): pass # ok @@ -1486,13 +1487,13 @@ def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Unexpected argumen [case testCallableKindsOrdering] from typing import Callable -from mypy_extensions import Arg, StarArg, KwArg, DefaultArg +from mypy_extensions import Arg, StarArg, KwArg, DefaultArg, NamedArg def f(f: Callable[[StarArg(), int], int]): pass # E: Required positional args may not appear after default, named or star args def g(f: Callable[[StarArg(), StarArg()], int]): pass # E: Star args may not appear after named or star args def h(f: Callable[[KwArg(), KwArg()], int]): pass # E: You may only have one **kwargs argument def i(f: Callable[[DefaultArg(), int], int]): pass # E: Required positional args may not appear after default, named or star args -def j(f: Callable[[NamedArg('x'), DefaultArg(int, 'y')], int]): pass # E: Positional default args may not appear after named or star args +def j(f: Callable[[NamedArg(int, 'x'), DefaultArg(int, 'y')], int]): pass # E: Positional default args may not appear after named or star args [builtins fixtures/dict.pyi] @@ -1500,7 +1501,7 @@ def j(f: Callable[[NamedArg('x'), DefaultArg(int, 'y')], int]): pass # E: Positi from typing import Callable from mypy_extensions import Arg, StarArg, KwArg, DefaultArg -def f(f: Callable[[Arg(int, 'x'), int, Arg(int, 'x')], int]): pass # E: Duplicate argument name "x" +def f(f: Callable[[Arg(int, 'x'), int, Arg(int, 'x')], int]): pass # E: duplicate argument 'x' in callable type [builtins fixtures/dict.pyi]