Convert the plugin to a package (#519)

Author: AlexWaygood

.github/workflows/typeshed_primer.yml

@@ -3,7 +3,7 @@ name: typeshed_primer
 on:
   pull_request:
     paths:
-      - "pyi.py"
+      - "flake8_pyi/**/*"
       - ".github/**/*"
   workflow_dispatch:
 

CHANGELOG.md

@@ -13,6 +13,7 @@ Other changes:
   [dependency groups](https://packaging.python.org/en/latest/specifications/dependency-groups/)
   rather than
   [optional dependencies](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#dependencies-and-requirements).
+* The plugin now exists as a `flake8_pyi` package rather than a single `pyi.py` file.
 * Declare support for Python 3.14
 
 ## 24.9.0

CONTRIBUTING.md

@@ -7,8 +7,9 @@ end will be warmly received.
 
 ## Guide to the codebase
 
-The plugin consists of a single file: `pyi.py`. Tests are run using `pytest`, and can be
-found in the `tests` folder.
+The plugin consists of a single package: `flake8_pyi`. Most of the logic lives in the
+`flake8_pyi/visitor.py` file. Tests are run using `pytest`, and can be found in the `tests`
+folder.
 
 PRs that make user-visible changes should generally add a short description of the change
 to the `CHANGELOG.md` file in the repository root.
@@ -29,8 +30,8 @@ however, we advise setting up a virtual environment first:
 
 To format your code with `isort` and `black`, run:
 
-    $ isort pyi.py
-    $ black pyi.py
+    $ isort flake8_pyi
+    $ black flake8_pyi
 
 If you want, you can also run locally the commands that GitHub Actions runs.
 Look in `.github/workflows/` to find the commands.

flake8_pyi/__init__.py

@@ -0,0 +1,3 @@
+from .checker import PyiTreeChecker
+
+__all__ = ["PyiTreeChecker"]

flake8_pyi/checker.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import argparse
+import ast
+import logging
+import re
+from dataclasses import dataclass
+from typing import Any, ClassVar, Iterator, Literal
+
+from flake8 import checker
+from flake8.options.manager import OptionManager
+from flake8.plugins.finder import LoadedPlugin
+from flake8.plugins.pyflakes import FlakesChecker
+from pyflakes.checker import ModuleScope
+
+from . import errors, visitor
+
+LOG = logging.getLogger("flake8.pyi")
+
+
+class PyflakesPreProcessor(ast.NodeTransformer):
+    """Transform AST prior to passing it to pyflakes.
+
+    This reduces false positives on recursive class definitions.
+    """
+
+    def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef:
+        self.generic_visit(node)
+        node.bases = [
+            # Remove the subscript to prevent F821 errors from being emitted
+            # for (valid) recursive definitions: Foo[Bar] --> Foo
+            base.value if isinstance(base, ast.Subscript) else base
+            for base in node.bases
+        ]
+        return node
+
+
+class PyiAwareFlakesChecker(FlakesChecker):
+    def __init__(self, tree: ast.AST, *args: Any, **kwargs: Any) -> None:
+        super().__init__(PyflakesPreProcessor().visit(tree), *args, **kwargs)
+
+    @property
+    def annotationsFutureEnabled(self) -> Literal[True]:
+        """Always allow forward references in `.pyi` files.
+
+        Pyflakes can already handle forward refs for annotations,
+        but only via `from __future__ import annotations`.
+        In a stub file, `from __future__ import annotations` is unnecessary,
+        so we pretend to pyflakes that it's always present when linting a `.pyi` file.
+        """
+        return True
+
+    @annotationsFutureEnabled.setter
+    def annotationsFutureEnabled(self, value: bool) -> None:
+        """Does nothing, as we always want this property to be `True`."""
+
+    def ASSIGN(
+        self, tree: ast.Assign, omit: str | tuple[str, ...] | None = None
+    ) -> None:
+        """Defer evaluation of assignments in the module scope.
+
+        This is a custom implementation of ASSIGN, originally derived from
+        handleChildren() in pyflakes 1.3.0.
+
+        This reduces false positives for:
+          - TypeVars bound or constrained to forward references
+          - Assignments to forward references that are not explicitly
+            demarcated as type aliases.
+        """
+        if not isinstance(self.scope, ModuleScope):
+            super().ASSIGN(tree)
+            return
+
+        for target in tree.targets:
+            self.handleNode(target, tree)
+
+        self.deferFunction(lambda: self.handleNode(tree.value, tree))
+
+    def handleNodeDelete(self, node: ast.AST) -> None:
+        """Null implementation.
+
+        Lets users use `del` in stubs to denote private names.
+        """
+        return
+
+
+class PyiAwareFileChecker(checker.FileChecker):
+    def run_check(self, plugin: LoadedPlugin, **kwargs: Any) -> Any:
+        if plugin.obj is FlakesChecker:
+            if self.filename == "-":
+                filename = self.options.stdin_display_name
+            else:
+                filename = self.filename
+
+            if filename.endswith(".pyi"):
+                LOG.info(
+                    f"Replacing FlakesChecker with PyiAwareFlakesChecker while "
+                    f"checking {filename!r}"
+                )
+                plugin = plugin._replace(obj=PyiAwareFlakesChecker)
+        return super().run_check(plugin, **kwargs)
+
+
+_TYPE_COMMENT_REGEX = re.compile(r"#\s*type:\s*(?!\s?ignore)([^#]+)(\s*#.*?)?$")
+
+
+def _check_for_type_comments(lines: list[str]) -> Iterator[errors.Error]:
+    for lineno, line in enumerate(lines, start=1):
+        cleaned_line = line.strip()
+
+        if cleaned_line.startswith("#"):
+            continue
+
+        if match := _TYPE_COMMENT_REGEX.search(cleaned_line):
+            type_comment = match.group(1).strip()
+
+            try:
+                ast.parse(type_comment)
+            except SyntaxError:
+                continue
+
+            yield errors.Error(lineno, 0, errors.Y033, PyiTreeChecker)
+
+
+@dataclass
+class PyiTreeChecker:
+    name: ClassVar[str] = "flake8-pyi"
+    tree: ast.Module
+    lines: list[str]
+    filename: str = "(none)"
+
+    def run(self) -> Iterator[errors.Error]:
+        if self.filename.endswith(".pyi"):
+            yield from _check_for_type_comments(self.lines)
+            yield from visitor.PyiVisitor(filename=self.filename).run(self.tree)
+
+    @staticmethod
+    def add_options(parser: OptionManager) -> None:
+        """This is brittle, there's multiple levels of caching of defaults."""
+        parser.parser.set_defaults(filename="*.py,*.pyi")
+        parser.extend_default_ignore(errors.DISABLED_BY_DEFAULT)
+        parser.add_option(
+            "--no-pyi-aware-file-checker",
+            default=False,
+            action="store_true",
+            parse_from_config=True,
+            help="don't patch flake8 with .pyi-aware file checker",
+        )
+
+    @staticmethod
+    def parse_options(options: argparse.Namespace) -> None:
+        if not options.no_pyi_aware_file_checker:
+            checker.FileChecker = PyiAwareFileChecker

flake8_pyi/errors.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, NamedTuple
+
+if TYPE_CHECKING:
+    # Import is only needed for type annotations,
+    # and causes a circular import if it's imported at runtime.
+    from .checker import PyiTreeChecker
+
+
+class Error(NamedTuple):
+    lineno: int
+    col: int
+    message: str
+    type: type[PyiTreeChecker]
+
+
+# Please keep error code lists in ERRORCODES and CHANGELOG up to date
+Y001 = "Y001 Name of private {} must start with _"
+Y002 = (
+    "Y002 If test must be a simple comparison against sys.platform or sys.version_info"
+)
+Y003 = "Y003 Unrecognized sys.version_info check"
+Y004 = "Y004 Version comparison must use only major and minor version"
+Y005 = "Y005 Version comparison must be against a length-{n} tuple"
+Y006 = "Y006 Use only < and >= for version comparisons"
+Y007 = "Y007 Unrecognized sys.platform check"
+Y008 = 'Y008 Unrecognized platform "{platform}"'
+Y009 = 'Y009 Empty body should contain "...", not "pass"'
+Y010 = 'Y010 Function body must contain only "..."'
+Y011 = "Y011 Only simple default values allowed for typed arguments"
+Y012 = 'Y012 Class body must not contain "pass"'
+Y013 = 'Y013 Non-empty class body must not contain "..."'
+Y014 = "Y014 Only simple default values allowed for arguments"
+Y015 = "Y015 Only simple default values are allowed for assignments"
+Y016 = 'Y016 Duplicate union member "{}"'
+Y017 = "Y017 Only simple assignments allowed"
+Y018 = 'Y018 {typevarlike_cls} "{typevar_name}" is not used'
+Y019 = (
+    'Y019 Use "typing_extensions.Self" instead of "{typevar_name}", e.g. "{new_syntax}"'
+)
+Y020 = "Y020 Quoted annotations should never be used in stubs"
+Y021 = "Y021 Docstrings should not be included in stubs"
+Y022 = "Y022 Use {good_syntax} instead of {bad_syntax} (PEP 585 syntax)"
+Y023 = "Y023 Use {good_syntax} instead of {bad_syntax}"
+Y024 = 'Y024 Use "typing.NamedTuple" instead of "collections.namedtuple"'
+Y025 = (
+    'Y025 Use "from collections.abc import Set as AbstractSet" '
+    'to avoid confusion with "builtins.set"'
+)
+Y026 = 'Y026 Use typing_extensions.TypeAlias for type aliases, e.g. "{suggestion}"'
+Y028 = "Y028 Use class-based syntax for NamedTuples"
+Y029 = "Y029 Defining __repr__ or __str__ in a stub is almost always redundant"
+Y030 = "Y030 Multiple Literal members in a union. {suggestion}"
+Y031 = "Y031 Use class-based syntax for TypedDicts where possible"
+Y032 = (
+    'Y032 Prefer "object" to "Any" for the second parameter in "{method_name}" methods'
+)
+Y033 = (
+    "Y033 Do not use type comments in stubs "
+    '(e.g. use "x: int" instead of "x = ... # type: int")'
+)
+Y034 = (
+    'Y034 {methods} usually return "self" at runtime. '
+    'Consider using "typing_extensions.Self" in "{method_name}", '
+    'e.g. "{suggested_syntax}"'
+)
+Y035 = (
+    'Y035 "{var}" in a stub file must have a value, '
+    'as it has the same semantics as "{var}" at runtime.'
+)
+Y036 = "Y036 Badly defined {method_name} method: {details}"
+Y037 = "Y037 Use PEP 604 union types instead of {old_syntax} (e.g. {example})."
+Y038 = (
+    'Y038 Use "from collections.abc import Set as AbstractSet" '
+    'instead of "from {module} import AbstractSet" (PEP 585 syntax)'
+)
+Y039 = 'Y039 Use "str" instead of "{module}.Text"'
+Y040 = 'Y040 Do not inherit from "object" explicitly, as it is redundant in Python 3'
+Y041 = (
+    'Y041 Use "{implicit_supertype}" '
+    'instead of "{implicit_subtype} | {implicit_supertype}" '
+    '(see "The numeric tower" in PEP 484)'
+)
+Y042 = "Y042 Type aliases should use the CamelCase naming convention"
+Y043 = 'Y043 Bad name for a type alias (the "T" suffix implies a TypeVar)'
+Y044 = 'Y044 "from __future__ import annotations" has no effect in stub files.'
+Y045 = 'Y045 "{iter_method}" methods should return an {good_cls}, not an {bad_cls}'
+Y046 = 'Y046 Protocol "{protocol_name}" is not used'
+Y047 = 'Y047 Type alias "{alias_name}" is not used'
+Y048 = "Y048 Function body should contain exactly one statement"
+Y049 = 'Y049 TypedDict "{typeddict_name}" is not used'
+Y050 = (
+    'Y050 Use "typing_extensions.Never" instead of "NoReturn" for argument annotations'
+)
+Y051 = 'Y051 "{literal_subtype}" is redundant in a union with "{builtin_supertype}"'
+Y052 = 'Y052 Need type annotation for "{variable}"'
+Y053 = "Y053 String and bytes literals >50 characters long are not permitted"
+Y054 = (
+    "Y054 Numeric literals with a string representation "
+    ">10 characters long are not permitted"
+)
+Y055 = 'Y055 Multiple "type[Foo]" members in a union. {suggestion}'
+Y056 = (
+    'Y056 Calling "{method}" on "__all__" may not be supported by all type checkers '
+    "(use += instead)"
+)
+Y057 = (
+    "Y057 Do not use {module}.ByteString, which has unclear semantics and is deprecated"
+)
+Y058 = (
+    'Y058 Use "{good_cls}" as the return value for simple "{iter_method}" methods, '
+    'e.g. "{example}"'
+)
+Y059 = 'Y059 "Generic[]" should always be the last base class'
+Y060 = (
+    'Y060 Redundant inheritance from "{redundant_base}"; '
+    "class would be inferred as generic anyway"
+)
+Y061 = 'Y061 None inside "Literal[]" expression. Replace with "{suggestion}"'
+Y062 = 'Y062 Duplicate "Literal[]" member "{}"'
+Y063 = "Y063 Use PEP-570 syntax to indicate positional-only arguments"
+Y064 = 'Y064 Use "{suggestion}" instead of "{original}"'
+Y065 = 'Y065 Leave {what} unannotated rather than using "Incomplete"'
+Y066 = (
+    "Y066 When using if/else with sys.version_info, "
+    'put the code for new Python versions first, e.g. "{new_syntax}"'
+)
+Y067 = 'Y067 Use "=None" instead of "Incomplete | None = None"'
+Y090 = (
+    'Y090 "{original}" means '
+    '"a tuple of length 1, in which the sole element is of type {typ!r}". '
+    'Perhaps you meant "{new}"?'
+)
+Y091 = (
+    'Y091 Argument "{arg}" to protocol method "{method}" should probably not be positional-or-keyword. '
+    "Make it positional-only, since usually you don't want to mandate a specific argument name"
+)
+
+DISABLED_BY_DEFAULT = ["Y090", "Y091"]