Merge pull request #1505 from google/google_sync

Google sync

CHANGELOG

@@ -1,3 +1,22 @@
+Version 2023.09.19:
+
+Updates:
+* Update typeshed pin to commit ce222e5 from Jun 12.
+* pytype-single: Add --no-validate-version flag to disable Python version
+  checks. This makes testing not-yet-supported versions easier.
+* Remove max version requirement for installing pytype. This prevents accidental
+  installation of very old versions of pytype that don't have this requirement.
+  Pytype itself still emits an error upon encountering an unsupported version.
+* Use pycnite for bytecode processing.
+
+Bug fixes:
+* Fix a corner case in ParamSpec matching for a class with a __call__ method.
+* Python 3.11:
+  * Fix implementations of several 3.11 opcodes.
+  * op.next should not point backwards even if the op is a backwards jump.
+* Track undecorated functions for better handling of TypeVars in signatures.
+* Fix 'TypeVar not in scope' check for imported TypeVars.
+
 Version 2023.09.11:
 
 Updates:

pytype/__version__.py

@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2023.09.11'
+__version__ = '2023.09.19'

pytype/abstract/_classes.py

@@ -136,12 +136,22 @@ def update_method_type_params(self):
     if not self.template:
       return
     # For function type parameters check
-    methods = set()
+    methods = []
+    # members of self._undecorated_methods that will be ignored for updating
+    # signature scope.
+    skip = set()
     for mbr in self.members.values():
       for m in mbr.data:
-        if _isinstance(m, "Function"):
-          methods.add(m)
-    methods.update(self._undecorated_methods)
+        if not _isinstance(m, "Function"):
+          continue
+        methods.append(m)
+        # We don't need to update the same method twice.
+        skip.add(m)
+        if m.__class__.__name__ == "StaticMethodInstance":
+          # TypeVars in staticmethods should not be treated as bound to the
+          # current class.
+          skip.update(m.func.data)
+    methods.extend(m for m in self._undecorated_methods if m not in skip)
     for m in methods:
       m.update_signature_scope(self)
 

pytype/annotation_utils.py

@@ -4,7 +4,7 @@
 import dataclasses
 import itertools
 
-from typing import AbstractSet, Any, Dict, Optional, Sequence, Tuple
+from typing import Any, Dict, Optional, Sequence, Set, Tuple
 
 from pytype import state
 from pytype import utils
@@ -397,7 +397,7 @@ def apply_annotation(self, node, op, name, value):
 
   def extract_annotation(
       self, node, var, name, stack,
-      allowed_type_params: Optional[AbstractSet[str]] = None):
+      allowed_type_params: Optional[Set[str]] = None):
     """Returns an annotation extracted from 'var'.
 
     Args:
@@ -421,8 +421,10 @@ def extract_annotation(
     if typ.formal and allowed_type_params is not None:
       allowed_type_params = (allowed_type_params |
                              self.get_callable_type_parameter_names(typ))
-      illegal_params = [x.name for x in self.get_type_parameters(typ)
-                        if x.name not in allowed_type_params]
+      illegal_params = []
+      for x in self.get_type_parameters(typ):
+        if not allowed_type_params.intersection([x.name, x.full_name]):
+          illegal_params.append(x.name)
       if illegal_params:
         details = "TypeVar(s) %s not in scope" % ", ".join(
             repr(p) for p in utils.unique_list(illegal_params))

pytype/pyc/opcodes.py

@@ -999,10 +999,6 @@ class POP_JUMP_BACKWARD_IF_TRUE(OpcodeWithArg):
   __slots__ = ()
 
 
-def _is_backward_jump(opcls):
-  return "JUMP_BACKWARD" in opcls.__name__
-
-
 def dis(code) -> List[Opcode]:
   """Disassemble a string into a list of Opcode instances."""
   ret = []
@@ -1024,5 +1020,5 @@ def dis(code) -> List[Opcode]:
       op.target = ret[op.arg]
     get_code = lambda j: ret[j] if 0 <= j < len(ret) else None
     op.prev = get_code(i - 1)
-    op.next = get_code(i +(-1 if _is_backward_jump(op.__class__) else 1))
+    op.next = get_code(i + 1)
   return ret

pytype/pyc/pyc_test.py

@@ -57,31 +57,40 @@ def test_lineno(self):
                         )
     self.assertIn("a", code.co_names)
     op_and_line = [(op.name, op.line) for op in opcodes.dis(code)]
-    self.assertEqual([("LOAD_CONST", 1),
-                      ("STORE_NAME", 1),
-                      ("LOAD_NAME", 3),
-                      ("LOAD_CONST", 3),
-                      ("BINARY_ADD", 3),
-                      ("STORE_NAME", 3),
-                      ("LOAD_CONST", 3),
-                      ("RETURN_VALUE", 3)], op_and_line)
+    expected = [("LOAD_CONST", 1),
+                ("STORE_NAME", 1),
+                ("LOAD_NAME", 3),
+                ("LOAD_CONST", 3),
+                ("BINARY_ADD", 3),
+                ("STORE_NAME", 3),
+                ("LOAD_CONST", 3),
+                ("RETURN_VALUE", 3)]
+    if self.python_version >= (3, 11):
+      expected = [("RESUME", 0)] + expected
+      expected[5] = ("BINARY_OP", 3)  # this was BINARY_ADD in 3.10-
+    self.assertEqual(expected, op_and_line)
 
   def test_mode(self):
     code = self._compile("foo", mode="eval")
     self.assertIn("foo", code.co_names)
     ops = [op.name for op in opcodes.dis(code)]
-    self.assertEqual(["LOAD_NAME",
-                      "RETURN_VALUE"], ops)
+    expected = ["LOAD_NAME", "RETURN_VALUE"]
+    if self.python_version >= (3, 11):
+      expected = ["RESUME"] + expected
+    self.assertEqual(expected, ops)
 
   def test_singlelineno(self):
     code = self._compile("a = 1\n"      # line 1
                         )
     self.assertIn("a", code.co_names)
     op_and_line = [(op.name, op.line) for op in opcodes.dis(code)]
-    self.assertEqual([("LOAD_CONST", 1),
-                      ("STORE_NAME", 1),
-                      ("LOAD_CONST", 1),
-                      ("RETURN_VALUE", 1)], op_and_line)
+    expected = [("LOAD_CONST", 1),
+                ("STORE_NAME", 1),
+                ("LOAD_CONST", 1),
+                ("RETURN_VALUE", 1)]
+    if self.python_version >= (3, 11):
+      expected = [("RESUME", 0)] + expected
+    self.assertEqual(expected, op_and_line)
 
   def test_singlelinenowithspace(self):
     code = self._compile("\n"
@@ -90,10 +99,13 @@ def test_singlelinenowithspace(self):
                         )
     self.assertIn("a", code.co_names)
     op_and_line = [(op.name, op.line) for op in opcodes.dis(code)]
-    self.assertEqual([("LOAD_CONST", 3),
-                      ("STORE_NAME", 3),
-                      ("LOAD_CONST", 3),
-                      ("RETURN_VALUE", 3)], op_and_line)
+    expected = [("LOAD_CONST", 3),
+                ("STORE_NAME", 3),
+                ("LOAD_CONST", 3),
+                ("RETURN_VALUE", 3)]
+    if self.python_version >= (3, 11):
+      expected = [("RESUME", 0)] + expected
+    self.assertEqual(expected, op_and_line)
 
 
 if __name__ == "__main__":

pytype/pyi/parser_test.py

@@ -960,8 +960,12 @@ def test_star_params(self):
     # Various illegal uses of * args.
     self.check_error("def foo(*) -> int: ...", 1,
                      "named arguments must follow bare *")
-    self.check_error("def foo(*x, *y) -> int: ...", 1, "invalid syntax")
-    self.check_error("def foo(**x, *y) -> int: ...", 1, "invalid syntax")
+    if self.python_version >= (3, 11):
+      expected_error = "ParseError"
+    else:
+      expected_error = "invalid syntax"
+    self.check_error("def foo(*x, *y) -> int: ...", 1, expected_error)
+    self.check_error("def foo(**x, *y) -> int: ...", 1, expected_error)
 
   def test_typeignore(self):
     self.check("def foo() -> int:  # type: ignore\n  ...",

pytype/tests/test_generic2.py

@@ -1333,6 +1333,31 @@ def f() -> str:
            return foo.A.f('')
       """)
 
+  def test_generic_staticmethod(self):
+    # Regression test for a crash caused by
+    # InterpreterClass.update_method_type_params treating static methods as
+    # instance methods.
+    self.Check("""
+      from typing import Any, Callable, Generic, TypeVar, Union
+
+      T = TypeVar('T')
+
+      class Expr(Generic[T]):
+
+        def __call__(self, *args: Any, **kwargs: Any) -> T:
+          return __any_object__
+
+        @staticmethod
+        def make_unbound(
+            init: Union[Callable[..., T], 'Expr[T]'],
+        ) -> 'Expr[T]':
+          return Expr()
+
+
+      def expr_var(initial_expr: Expr[T]) -> Expr[T]:
+        return Expr.make_unbound(init=initial_expr)
+    """)
+
 
 if __name__ == "__main__":
   test_base.main()

pytype/tests/test_typevar2.py

@@ -595,6 +595,18 @@ def test_future_annotations(self):
       x: Callable[[T], T] = lambda x: x
     """)
 
+  def test_imported_typevar_in_scope(self):
+    with self.DepTree([("foo.pyi", """
+      from typing import TypeVar
+      T = TypeVar('T')
+    """)]):
+      self.Check("""
+        import foo
+        def f(x: foo.T) -> foo.T:
+          y: foo.T = x
+          return y
+      """)
+
 
 class GenericTypeAliasTest(test_base.BaseTest):
   """Tests for generic type aliases ("type macros")."""