convert.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332
  1. from __future__ import annotations
  2. import os.path
  3. import re
  4. from abc import ABCMeta, abstractmethod
  5. from collections import defaultdict
  6. from collections.abc import Iterator
  7. from email.message import Message
  8. from email.parser import Parser
  9. from email.policy import EmailPolicy
  10. from glob import iglob
  11. from pathlib import Path
  12. from textwrap import dedent
  13. from zipfile import ZipFile
  14. from .. import __version__
  15. from ..metadata import generate_requirements
  16. from ..vendored.packaging.tags import parse_tag
  17. from ..wheelfile import WheelFile
  18. egg_filename_re = re.compile(
  19. r"""
  20. (?P<name>.+?)-(?P<ver>.+?)
  21. (-(?P<pyver>py\d\.\d+)
  22. (-(?P<arch>.+?))?
  23. )?.egg$""",
  24. re.VERBOSE,
  25. )
  26. egg_info_re = re.compile(
  27. r"""
  28. ^(?P<name>.+?)-(?P<ver>.+?)
  29. (-(?P<pyver>py\d\.\d+)
  30. )?.egg-info/""",
  31. re.VERBOSE,
  32. )
  33. wininst_re = re.compile(
  34. r"\.(?P<platform>win32|win-amd64)(?:-(?P<pyver>py\d\.\d))?\.exe$"
  35. )
  36. pyd_re = re.compile(r"\.(?P<abi>[a-z0-9]+)-(?P<platform>win32|win_amd64)\.pyd$")
  37. serialization_policy = EmailPolicy(
  38. utf8=True,
  39. mangle_from_=False,
  40. max_line_length=0,
  41. )
  42. GENERATOR = f"wheel {__version__}"
  43. def convert_requires(requires: str, metadata: Message) -> None:
  44. extra: str | None = None
  45. requirements: dict[str | None, list[str]] = defaultdict(list)
  46. for line in requires.splitlines():
  47. line = line.strip()
  48. if not line:
  49. continue
  50. if line.startswith("[") and line.endswith("]"):
  51. extra = line[1:-1]
  52. continue
  53. requirements[extra].append(line)
  54. for key, value in generate_requirements(requirements):
  55. metadata.add_header(key, value)
  56. def convert_pkg_info(pkginfo: str, metadata: Message):
  57. parsed_message = Parser().parsestr(pkginfo)
  58. for key, value in parsed_message.items():
  59. key_lower = key.lower()
  60. if value == "UNKNOWN":
  61. continue
  62. if key_lower == "description":
  63. description_lines = value.splitlines()
  64. value = "\n".join(
  65. (
  66. description_lines[0].lstrip(),
  67. dedent("\n".join(description_lines[1:])),
  68. "\n",
  69. )
  70. )
  71. metadata.set_payload(value)
  72. elif key_lower == "home-page":
  73. metadata.add_header("Project-URL", f"Homepage, {value}")
  74. elif key_lower == "download-url":
  75. metadata.add_header("Project-URL", f"Download, {value}")
  76. else:
  77. metadata.add_header(key, value)
  78. metadata.replace_header("Metadata-Version", "2.4")
  79. def normalize(name: str) -> str:
  80. return re.sub(r"[-_.]+", "-", name).lower().replace("-", "_")
  81. class ConvertSource(metaclass=ABCMeta):
  82. name: str
  83. version: str
  84. pyver: str = "py2.py3"
  85. abi: str = "none"
  86. platform: str = "any"
  87. metadata: Message
  88. @property
  89. def dist_info_dir(self) -> str:
  90. return f"{self.name}-{self.version}.dist-info"
  91. @abstractmethod
  92. def generate_contents(self) -> Iterator[tuple[str, bytes]]:
  93. pass
  94. class EggFileSource(ConvertSource):
  95. def __init__(self, path: Path):
  96. if not (match := egg_filename_re.match(path.name)):
  97. raise ValueError(f"Invalid egg file name: {path.name}")
  98. # Binary wheels are assumed to be for CPython
  99. self.path = path
  100. self.name = normalize(match.group("name"))
  101. self.version = match.group("ver")
  102. if pyver := match.group("pyver"):
  103. self.pyver = pyver.replace(".", "")
  104. if arch := match.group("arch"):
  105. self.abi = self.pyver.replace("py", "cp")
  106. self.platform = normalize(arch)
  107. self.metadata = Message()
  108. def generate_contents(self) -> Iterator[tuple[str, bytes]]:
  109. with ZipFile(self.path, "r") as zip_file:
  110. for filename in sorted(zip_file.namelist()):
  111. # Skip pure directory entries
  112. if filename.endswith("/"):
  113. continue
  114. # Handle files in the egg-info directory specially, selectively moving
  115. # them to the dist-info directory while converting as needed
  116. if filename.startswith("EGG-INFO/"):
  117. if filename == "EGG-INFO/requires.txt":
  118. requires = zip_file.read(filename).decode("utf-8")
  119. convert_requires(requires, self.metadata)
  120. elif filename == "EGG-INFO/PKG-INFO":
  121. pkginfo = zip_file.read(filename).decode("utf-8")
  122. convert_pkg_info(pkginfo, self.metadata)
  123. elif filename == "EGG-INFO/entry_points.txt":
  124. yield (
  125. f"{self.dist_info_dir}/entry_points.txt",
  126. zip_file.read(filename),
  127. )
  128. continue
  129. # For any other file, just pass it through
  130. yield filename, zip_file.read(filename)
  131. class EggDirectorySource(EggFileSource):
  132. def generate_contents(self) -> Iterator[tuple[str, bytes]]:
  133. for dirpath, _, filenames in os.walk(self.path):
  134. for filename in sorted(filenames):
  135. path = Path(dirpath, filename)
  136. if path.parent.name == "EGG-INFO":
  137. if path.name == "requires.txt":
  138. requires = path.read_text("utf-8")
  139. convert_requires(requires, self.metadata)
  140. elif path.name == "PKG-INFO":
  141. pkginfo = path.read_text("utf-8")
  142. convert_pkg_info(pkginfo, self.metadata)
  143. if name := self.metadata.get("Name"):
  144. self.name = normalize(name)
  145. if version := self.metadata.get("Version"):
  146. self.version = version
  147. elif path.name == "entry_points.txt":
  148. yield (
  149. f"{self.dist_info_dir}/entry_points.txt",
  150. path.read_bytes(),
  151. )
  152. continue
  153. # For any other file, just pass it through
  154. yield str(path.relative_to(self.path)), path.read_bytes()
  155. class WininstFileSource(ConvertSource):
  156. """
  157. Handles distributions created with ``bdist_wininst``.
  158. The egginfo filename has the format::
  159. name-ver(-pyver)(-arch).egg-info
  160. The installer filename has the format::
  161. name-ver.arch(-pyver).exe
  162. Some things to note:
  163. 1. The installer filename is not definitive. An installer can be renamed
  164. and work perfectly well as an installer. So more reliable data should
  165. be used whenever possible.
  166. 2. The egg-info data should be preferred for the name and version, because
  167. these come straight from the distutils metadata, and are mandatory.
  168. 3. The pyver from the egg-info data should be ignored, as it is
  169. constructed from the version of Python used to build the installer,
  170. which is irrelevant - the installer filename is correct here (even to
  171. the point that when it's not there, any version is implied).
  172. 4. The architecture must be taken from the installer filename, as it is
  173. not included in the egg-info data.
  174. 5. Architecture-neutral installers still have an architecture because the
  175. installer format itself (being executable) is architecture-specific. We
  176. should therefore ignore the architecture if the content is pure-python.
  177. """
  178. def __init__(self, path: Path):
  179. self.path = path
  180. self.metadata = Message()
  181. # Determine the initial architecture and Python version from the file name
  182. # (if possible)
  183. if match := wininst_re.search(path.name):
  184. self.platform = normalize(match.group("platform"))
  185. if pyver := match.group("pyver"):
  186. self.pyver = pyver.replace(".", "")
  187. # Look for an .egg-info directory and any .pyd files for more precise info
  188. egg_info_found = pyd_found = False
  189. with ZipFile(self.path) as zip_file:
  190. for filename in zip_file.namelist():
  191. prefix, filename = filename.split("/", 1)
  192. if not egg_info_found and (match := egg_info_re.match(filename)):
  193. egg_info_found = True
  194. self.name = normalize(match.group("name"))
  195. self.version = match.group("ver")
  196. if pyver := match.group("pyver"):
  197. self.pyver = pyver.replace(".", "")
  198. elif not pyd_found and (match := pyd_re.search(filename)):
  199. pyd_found = True
  200. self.abi = match.group("abi")
  201. self.platform = match.group("platform")
  202. if egg_info_found and pyd_found:
  203. break
  204. def generate_contents(self) -> Iterator[tuple[str, bytes]]:
  205. dist_info_dir = f"{self.name}-{self.version}.dist-info"
  206. data_dir = f"{self.name}-{self.version}.data"
  207. with ZipFile(self.path, "r") as zip_file:
  208. for filename in sorted(zip_file.namelist()):
  209. # Skip pure directory entries
  210. if filename.endswith("/"):
  211. continue
  212. # Handle files in the egg-info directory specially, selectively moving
  213. # them to the dist-info directory while converting as needed
  214. prefix, target_filename = filename.split("/", 1)
  215. if egg_info_re.search(target_filename):
  216. basename = target_filename.rsplit("/", 1)[-1]
  217. if basename == "requires.txt":
  218. requires = zip_file.read(filename).decode("utf-8")
  219. convert_requires(requires, self.metadata)
  220. elif basename == "PKG-INFO":
  221. pkginfo = zip_file.read(filename).decode("utf-8")
  222. convert_pkg_info(pkginfo, self.metadata)
  223. elif basename == "entry_points.txt":
  224. yield (
  225. f"{dist_info_dir}/entry_points.txt",
  226. zip_file.read(filename),
  227. )
  228. continue
  229. elif prefix == "SCRIPTS":
  230. target_filename = f"{data_dir}/scripts/{target_filename}"
  231. # For any other file, just pass it through
  232. yield target_filename, zip_file.read(filename)
  233. def convert(files: list[str], dest_dir: str, verbose: bool) -> None:
  234. for pat in files:
  235. for archive in iglob(pat):
  236. path = Path(archive)
  237. if path.suffix == ".egg":
  238. if path.is_dir():
  239. source: ConvertSource = EggDirectorySource(path)
  240. else:
  241. source = EggFileSource(path)
  242. else:
  243. source = WininstFileSource(path)
  244. if verbose:
  245. print(f"{archive}...", flush=True, end="")
  246. dest_path = Path(dest_dir) / (
  247. f"{source.name}-{source.version}-{source.pyver}-{source.abi}"
  248. f"-{source.platform}.whl"
  249. )
  250. with WheelFile(dest_path, "w") as wheelfile:
  251. for name_or_zinfo, contents in source.generate_contents():
  252. wheelfile.writestr(name_or_zinfo, contents)
  253. # Write the METADATA file
  254. wheelfile.writestr(
  255. f"{source.dist_info_dir}/METADATA",
  256. source.metadata.as_string(policy=serialization_policy).encode(
  257. "utf-8"
  258. ),
  259. )
  260. # Write the WHEEL file
  261. wheel_message = Message()
  262. wheel_message.add_header("Wheel-Version", "1.0")
  263. wheel_message.add_header("Generator", GENERATOR)
  264. wheel_message.add_header(
  265. "Root-Is-Purelib", str(source.platform == "any").lower()
  266. )
  267. tags = parse_tag(f"{source.pyver}-{source.abi}-{source.platform}")
  268. for tag in sorted(tags, key=lambda tag: tag.interpreter):
  269. wheel_message.add_header("Tag", str(tag))
  270. wheelfile.writestr(
  271. f"{source.dist_info_dir}/WHEEL",
  272. wheel_message.as_string(policy=serialization_policy).encode(
  273. "utf-8"
  274. ),
  275. )
  276. if verbose:
  277. print("OK")