test_typing.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182
  1. import importlib.util
  2. import itertools
  3. import os
  4. import re
  5. from collections import defaultdict
  6. from typing import Optional
  7. import pytest
  8. try:
  9. from mypy import api
  10. except ImportError:
  11. NO_MYPY = True
  12. else:
  13. NO_MYPY = False
  14. DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
  15. PASS_DIR = os.path.join(DATA_DIR, "pass")
  16. FAIL_DIR = os.path.join(DATA_DIR, "fail")
  17. REVEAL_DIR = os.path.join(DATA_DIR, "reveal")
  18. MYPY_INI = os.path.join(DATA_DIR, "mypy.ini")
  19. CACHE_DIR = os.path.join(DATA_DIR, ".mypy_cache")
  20. def get_test_cases(directory):
  21. for root, _, files in os.walk(directory):
  22. for fname in files:
  23. if os.path.splitext(fname)[-1] == ".py":
  24. fullpath = os.path.join(root, fname)
  25. # Use relative path for nice py.test name
  26. relpath = os.path.relpath(fullpath, start=directory)
  27. yield pytest.param(
  28. fullpath,
  29. # Manually specify a name for the test
  30. id=relpath,
  31. )
  32. @pytest.mark.slow
  33. @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed")
  34. @pytest.mark.parametrize("path", get_test_cases(PASS_DIR))
  35. def test_success(path):
  36. stdout, stderr, exitcode = api.run([
  37. "--config-file",
  38. MYPY_INI,
  39. "--cache-dir",
  40. CACHE_DIR,
  41. path,
  42. ])
  43. assert exitcode == 0, stdout
  44. assert re.match(r"Success: no issues found in \d+ source files?", stdout.strip())
  45. @pytest.mark.slow
  46. @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed")
  47. @pytest.mark.parametrize("path", get_test_cases(FAIL_DIR))
  48. def test_fail(path):
  49. __tracebackhide__ = True
  50. stdout, stderr, exitcode = api.run([
  51. "--config-file",
  52. MYPY_INI,
  53. "--cache-dir",
  54. CACHE_DIR,
  55. path,
  56. ])
  57. assert exitcode != 0
  58. with open(path) as fin:
  59. lines = fin.readlines()
  60. errors = defaultdict(lambda: "")
  61. error_lines = stdout.rstrip("\n").split("\n")
  62. assert re.match(
  63. r"Found \d+ errors? in \d+ files? \(checked \d+ source files?\)",
  64. error_lines[-1].strip(),
  65. )
  66. for error_line in error_lines[:-1]:
  67. error_line = error_line.strip()
  68. if not error_line:
  69. continue
  70. match = re.match(
  71. r"^.+\.py:(?P<lineno>\d+): (error|note): .+$",
  72. error_line,
  73. )
  74. if match is None:
  75. raise ValueError(f"Unexpected error line format: {error_line}")
  76. lineno = int(match.group('lineno'))
  77. errors[lineno] += error_line
  78. for i, line in enumerate(lines):
  79. lineno = i + 1
  80. if line.startswith('#') or (" E:" not in line and lineno not in errors):
  81. continue
  82. target_line = lines[lineno - 1]
  83. if "# E:" in target_line:
  84. marker = target_line.split("# E:")[-1].strip()
  85. expected_error = errors.get(lineno)
  86. _test_fail(path, marker, expected_error, lineno)
  87. else:
  88. pytest.fail(f"Error {repr(errors[lineno])} not found")
  89. _FAIL_MSG1 = """Extra error at line {}
  90. Extra error: {!r}
  91. """
  92. _FAIL_MSG2 = """Error mismatch at line {}
  93. Expected error: {!r}
  94. Observed error: {!r}
  95. """
  96. def _test_fail(path: str, error: str, expected_error: Optional[str], lineno: int) -> None:
  97. if expected_error is None:
  98. raise AssertionError(_FAIL_MSG1.format(lineno, error))
  99. elif error not in expected_error:
  100. raise AssertionError(_FAIL_MSG2.format(lineno, expected_error, error))
  101. @pytest.mark.slow
  102. @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed")
  103. @pytest.mark.parametrize("path", get_test_cases(REVEAL_DIR))
  104. def test_reveal(path):
  105. __tracebackhide__ = True
  106. stdout, stderr, exitcode = api.run([
  107. "--config-file",
  108. MYPY_INI,
  109. "--cache-dir",
  110. CACHE_DIR,
  111. path,
  112. ])
  113. with open(path) as fin:
  114. lines = fin.read().replace('*', '').split("\n")
  115. stdout_list = stdout.replace('*', '').split("\n")
  116. for error_line in stdout_list:
  117. error_line = error_line.strip()
  118. if not error_line:
  119. continue
  120. match = re.match(
  121. r"^.+\.py:(?P<lineno>\d+): note: .+$",
  122. error_line,
  123. )
  124. if match is None:
  125. raise ValueError(f"Unexpected reveal line format: {error_line}")
  126. lineno = int(match.group('lineno')) - 1
  127. assert "Revealed type is" in error_line
  128. marker = lines[lineno].split("# E:")[-1].strip()
  129. _test_reveal(path, marker, error_line, 1 + lineno)
  130. _REVEAL_MSG = """Reveal mismatch at line {}
  131. Expected reveal: {!r}
  132. Observed reveal: {!r}
  133. """
  134. def _test_reveal(path: str, reveal: str, expected_reveal: str, lineno: int) -> None:
  135. if reveal not in expected_reveal:
  136. raise AssertionError(_REVEAL_MSG.format(lineno, expected_reveal, reveal))
  137. @pytest.mark.slow
  138. @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed")
  139. @pytest.mark.parametrize("path", get_test_cases(PASS_DIR))
  140. def test_code_runs(path):
  141. path_without_extension, _ = os.path.splitext(path)
  142. dirname, filename = path.split(os.sep)[-2:]
  143. spec = importlib.util.spec_from_file_location(f"{dirname}.{filename}", path)
  144. test_module = importlib.util.module_from_spec(spec)
  145. spec.loader.exec_module(test_module)