portalocker.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444
  1. # pyright: reportUnknownMemberType=false, reportAttributeAccessIssue=false
  2. """Module portalocker.
  3. This module provides cross-platform file locking functionality.
  4. The Windows implementation now supports two variants:
  5. 1. A default method using the Win32 API (win32file.LockFileEx/UnlockFileEx).
  6. 2. An alternative that uses msvcrt.locking for exclusive locks (shared
  7. locks still use the Win32 API).
  8. This version uses classes to encapsulate locking logic, while maintaining
  9. the original external API, including the LOCKER constant for specific
  10. backwards compatibility (POSIX) and Windows behavior.
  11. """
  12. import io
  13. import os
  14. import typing
  15. from typing import (
  16. Any,
  17. Callable,
  18. Optional,
  19. Union,
  20. cast,
  21. )
  22. from . import constants, exceptions, types
  23. # Alias for readability
  24. LockFlags = constants.LockFlags
  25. # Define a protocol for callable lockers
  26. class LockCallable(typing.Protocol):
  27. def __call__(
  28. self, file_obj: types.FileArgument, flags: LockFlags
  29. ) -> None: ...
  30. class UnlockCallable(typing.Protocol):
  31. def __call__(self, file_obj: types.FileArgument) -> None: ...
  32. class BaseLocker:
  33. """Base class for locker implementations."""
  34. def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None:
  35. """Lock the file."""
  36. raise NotImplementedError
  37. def unlock(self, file_obj: types.FileArgument) -> None:
  38. """Unlock the file."""
  39. raise NotImplementedError
  40. # Define refined LockerType with more specific types
  41. LockerType = Union[
  42. # POSIX-style fcntl.flock callable
  43. Callable[[Union[int, types.HasFileno], int], Any],
  44. # Tuple of lock and unlock functions
  45. tuple[LockCallable, UnlockCallable],
  46. # BaseLocker instance
  47. BaseLocker,
  48. # BaseLocker class
  49. type[BaseLocker],
  50. ]
  51. LOCKER: LockerType
  52. if os.name == 'nt': # pragma: not-posix
  53. # Windows-specific helper functions
  54. def _prepare_windows_file(
  55. file_obj: types.FileArgument,
  56. ) -> tuple[int, Optional[typing.IO[Any]], Optional[int]]:
  57. """Prepare file for Windows: get fd, optionally seek and save pos."""
  58. if isinstance(file_obj, int):
  59. # Plain file descriptor
  60. return file_obj, None, None
  61. # Full IO objects (have tell/seek) -> preserve and restore position
  62. if isinstance(file_obj, io.IOBase):
  63. fd: int = file_obj.fileno()
  64. original_pos = file_obj.tell()
  65. if original_pos != 0:
  66. file_obj.seek(0)
  67. return fd, typing.cast(typing.IO[Any], file_obj), original_pos
  68. # cast satisfies mypy: IOBase -> IO[Any]
  69. # Fallback: an object that only implements fileno() (HasFileno)
  70. fd = typing.cast(types.HasFileno, file_obj).fileno() # type: ignore[redundant-cast]
  71. return fd, None, None
  72. def _restore_windows_file_pos(
  73. file_io_obj: Optional[typing.IO[Any]],
  74. original_pos: Optional[int],
  75. ) -> None:
  76. """Restore file position if it was an IO object and pos was saved."""
  77. if file_io_obj and original_pos is not None and original_pos != 0:
  78. file_io_obj.seek(original_pos)
  79. class Win32Locker(BaseLocker):
  80. """Locker using Win32 API (LockFileEx/UnlockFileEx)."""
  81. _overlapped: Any # pywintypes.OVERLAPPED
  82. _lock_bytes_low: int = -0x10000
  83. def __init__(self) -> None:
  84. try:
  85. import pywintypes
  86. except ImportError as e:
  87. raise ImportError(
  88. 'pywintypes is required for Win32Locker but not '
  89. 'found. Please install pywin32.'
  90. ) from e
  91. self._overlapped = pywintypes.OVERLAPPED()
  92. def _get_os_handle(self, fd: int) -> int:
  93. try:
  94. import msvcrt
  95. except ImportError as e:
  96. raise ImportError(
  97. 'msvcrt is required for _get_os_handle on Windows '
  98. 'but not found.'
  99. ) from e
  100. return cast(int, msvcrt.get_osfhandle(fd)) # type: ignore[attr-defined,redundant-cast]
  101. def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None:
  102. import pywintypes
  103. import win32con
  104. import win32file
  105. import winerror
  106. fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj)
  107. os_fh = self._get_os_handle(fd)
  108. mode = 0
  109. if flags & LockFlags.NON_BLOCKING:
  110. mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY
  111. if flags & LockFlags.EXCLUSIVE:
  112. mode |= win32con.LOCKFILE_EXCLUSIVE_LOCK
  113. try:
  114. win32file.LockFileEx(
  115. os_fh, mode, 0, self._lock_bytes_low, self._overlapped
  116. )
  117. except pywintypes.error as exc_value: # type: ignore[misc]
  118. if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION:
  119. raise exceptions.AlreadyLocked(
  120. exceptions.LockException.LOCK_FAILED,
  121. exc_value.strerror,
  122. fh=file_obj, # Pass original file_obj
  123. ) from exc_value
  124. else:
  125. raise
  126. finally:
  127. _restore_windows_file_pos(io_obj_ctx, pos_ctx)
  128. def unlock(self, file_obj: types.FileArgument) -> None:
  129. import pywintypes
  130. import win32file
  131. import winerror
  132. fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj)
  133. os_fh = self._get_os_handle(fd)
  134. try:
  135. win32file.UnlockFileEx(
  136. os_fh, 0, self._lock_bytes_low, self._overlapped
  137. )
  138. except pywintypes.error as exc: # type: ignore[misc]
  139. if exc.winerror != winerror.ERROR_NOT_LOCKED:
  140. raise exceptions.LockException(
  141. exceptions.LockException.LOCK_FAILED,
  142. exc.strerror,
  143. fh=file_obj, # Pass original file_obj
  144. ) from exc
  145. except OSError as exc:
  146. raise exceptions.LockException(
  147. exceptions.LockException.LOCK_FAILED,
  148. exc.strerror,
  149. fh=file_obj, # Pass original file_obj
  150. ) from exc
  151. finally:
  152. _restore_windows_file_pos(io_obj_ctx, pos_ctx)
  153. class MsvcrtLocker(BaseLocker):
  154. _win32_locker: Win32Locker
  155. _msvcrt_lock_length: int = 0x10000
  156. def __init__(self) -> None:
  157. self._win32_locker = Win32Locker()
  158. try:
  159. import msvcrt
  160. except ImportError as e:
  161. raise ImportError(
  162. 'msvcrt is required for MsvcrtLocker but not found.'
  163. ) from e
  164. attrs = ['LK_LOCK', 'LK_RLCK', 'LK_NBLCK', 'LK_UNLCK', 'LK_NBRLCK']
  165. defaults = [0, 1, 2, 3, 2] # LK_NBRLCK often same as LK_NBLCK (2)
  166. for attr, default_val in zip(attrs, defaults):
  167. if not hasattr(msvcrt, attr):
  168. setattr(msvcrt, attr, default_val)
  169. def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None:
  170. import msvcrt
  171. if flags & LockFlags.SHARED:
  172. win32_api_flags = LockFlags(0)
  173. if flags & LockFlags.NON_BLOCKING:
  174. win32_api_flags |= LockFlags.NON_BLOCKING
  175. self._win32_locker.lock(file_obj, win32_api_flags)
  176. return
  177. fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj)
  178. mode = (
  179. msvcrt.LK_NBLCK # type: ignore[attr-defined]
  180. if flags & LockFlags.NON_BLOCKING
  181. else msvcrt.LK_LOCK # type: ignore[attr-defined]
  182. )
  183. try:
  184. msvcrt.locking( # type: ignore[attr-defined]
  185. fd,
  186. mode,
  187. self._msvcrt_lock_length,
  188. )
  189. except OSError as exc_value:
  190. if exc_value.errno in (13, 16, 33, 36):
  191. raise exceptions.AlreadyLocked(
  192. exceptions.LockException.LOCK_FAILED,
  193. str(exc_value),
  194. fh=file_obj, # Pass original file_obj
  195. ) from exc_value
  196. raise exceptions.LockException(
  197. exceptions.LockException.LOCK_FAILED,
  198. str(exc_value),
  199. fh=file_obj, # Pass original file_obj
  200. ) from exc_value
  201. finally:
  202. _restore_windows_file_pos(io_obj_ctx, pos_ctx)
  203. def unlock(self, file_obj: types.FileArgument) -> None:
  204. import msvcrt
  205. fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj)
  206. took_fallback_path = False
  207. try:
  208. msvcrt.locking( # type: ignore[attr-defined]
  209. fd,
  210. msvcrt.LK_UNLCK, # type: ignore[attr-defined]
  211. self._msvcrt_lock_length,
  212. )
  213. except OSError as exc:
  214. if exc.errno == 13: # EACCES (Permission denied)
  215. took_fallback_path = True
  216. # Restore position before calling win32_locker,
  217. # as it will re-prepare.
  218. _restore_windows_file_pos(io_obj_ctx, pos_ctx)
  219. try:
  220. self._win32_locker.unlock(
  221. file_obj
  222. ) # win32_locker handles its own seeking
  223. except exceptions.LockException as win32_exc:
  224. raise exceptions.LockException(
  225. exceptions.LockException.LOCK_FAILED,
  226. f'msvcrt unlock failed ({exc.strerror}), and '
  227. f'win32 fallback failed ({win32_exc.strerror})',
  228. fh=file_obj,
  229. ) from win32_exc
  230. except Exception as final_exc:
  231. raise exceptions.LockException(
  232. exceptions.LockException.LOCK_FAILED,
  233. f'msvcrt unlock failed ({exc.strerror}), and '
  234. f'win32 fallback failed with unexpected error: '
  235. f'{final_exc!s}',
  236. fh=file_obj,
  237. ) from final_exc
  238. else:
  239. raise exceptions.LockException(
  240. exceptions.LockException.LOCK_FAILED,
  241. exc.strerror,
  242. fh=file_obj,
  243. ) from exc
  244. finally:
  245. if not took_fallback_path:
  246. _restore_windows_file_pos(io_obj_ctx, pos_ctx)
  247. _locker_instances: dict[type[BaseLocker], BaseLocker] = dict()
  248. LOCKER = MsvcrtLocker # type: ignore[reportConstantRedefinition]
  249. def lock(file: types.FileArgument, flags: LockFlags) -> None:
  250. if isinstance(LOCKER, BaseLocker):
  251. # If LOCKER is a BaseLocker instance, use its lock method
  252. locker: Callable[[types.FileArgument, LockFlags], None] = (
  253. LOCKER.lock
  254. )
  255. elif isinstance(LOCKER, tuple):
  256. locker = LOCKER[0] # type: ignore[reportUnknownVariableType]
  257. elif issubclass(LOCKER, BaseLocker): # type: ignore[unreachable,arg-type] # pyright: ignore [reportUnnecessaryIsInstance]
  258. locker_instance = _locker_instances.get(LOCKER) # type: ignore[arg-type]
  259. if locker_instance is None:
  260. # Create an instance of the locker class if not already done
  261. _locker_instances[LOCKER] = locker_instance = LOCKER() # type: ignore[ignore,index,call-arg]
  262. locker = locker_instance.lock
  263. else:
  264. raise TypeError(
  265. f'LOCKER must be a BaseLocker instance, a tuple of lock and '
  266. f'unlock functions, or a subclass of BaseLocker, '
  267. f'got {type(LOCKER)}.'
  268. )
  269. locker(file, flags)
  270. def unlock(file: types.FileArgument) -> None:
  271. if isinstance(LOCKER, BaseLocker):
  272. # If LOCKER is a BaseLocker instance, use its lock method
  273. unlocker: Callable[[types.FileArgument], None] = LOCKER.unlock
  274. elif isinstance(LOCKER, tuple):
  275. unlocker = LOCKER[1] # type: ignore[reportUnknownVariableType]
  276. elif issubclass(LOCKER, BaseLocker): # type: ignore[unreachable,arg-type] # pyright: ignore [reportUnnecessaryIsInstance]
  277. locker_instance = _locker_instances.get(LOCKER) # type: ignore[arg-type]
  278. if locker_instance is None:
  279. # Create an instance of the locker class if not already done
  280. _locker_instances[LOCKER] = locker_instance = LOCKER() # type: ignore[ignore,index,call-arg]
  281. unlocker = locker_instance.unlock
  282. else:
  283. raise TypeError(
  284. f'LOCKER must be a BaseLocker instance, a tuple of lock and '
  285. f'unlock functions, or a subclass of BaseLocker, '
  286. f'got {type(LOCKER)}.'
  287. )
  288. unlocker(file)
  289. else: # pragma: not-nt
  290. import errno
  291. import fcntl
  292. # PosixLocker methods accept FileArgument | HasFileno
  293. PosixFileArgument = Union[types.FileArgument, types.HasFileno]
  294. class PosixLocker(BaseLocker):
  295. """Locker implementation using the `LOCKER` constant"""
  296. _locker: Optional[
  297. Callable[[Union[int, types.HasFileno], int], Any]
  298. ] = None
  299. @property
  300. def locker(self) -> Callable[[Union[int, types.HasFileno], int], Any]:
  301. if self._locker is None:
  302. # On POSIX systems ``LOCKER`` is a callable (fcntl.flock) but
  303. # mypy also sees the Windows-only tuple assignment. Explicitly
  304. # cast so mypy knows we are returning the callable variant
  305. # here.
  306. return cast(
  307. Callable[[Union[int, types.HasFileno], int], Any], LOCKER
  308. ) # pyright: ignore[reportUnnecessaryCast]
  309. # mypy does not realise ``self._locker`` is non-None after the
  310. # check
  311. assert self._locker is not None
  312. return self._locker
  313. def _get_fd(self, file_obj: PosixFileArgument) -> int:
  314. if isinstance(file_obj, int):
  315. return file_obj
  316. # Check for fileno() method; covers typing.IO and HasFileno
  317. elif hasattr(file_obj, 'fileno') and callable(file_obj.fileno):
  318. return file_obj.fileno()
  319. else:
  320. # Should not be reached if PosixFileArgument is correct.
  321. # isinstance(file_obj, io.IOBase) could be an
  322. # alternative check
  323. # but hasattr is more general for HasFileno.
  324. raise TypeError(
  325. "Argument 'file_obj' must be an int, an IO object "
  326. 'with fileno(), or implement HasFileno.'
  327. )
  328. def lock(self, file_obj: PosixFileArgument, flags: LockFlags) -> None:
  329. if (flags & LockFlags.NON_BLOCKING) and not flags & (
  330. LockFlags.SHARED | LockFlags.EXCLUSIVE
  331. ):
  332. raise RuntimeError(
  333. 'When locking in non-blocking mode on POSIX, '
  334. 'the SHARED or EXCLUSIVE flag must be specified as well.'
  335. )
  336. fd = self._get_fd(file_obj)
  337. try:
  338. self.locker(fd, flags)
  339. except OSError as exc_value:
  340. if exc_value.errno in (errno.EACCES, errno.EAGAIN):
  341. raise exceptions.AlreadyLocked(
  342. exc_value,
  343. strerror=str(exc_value),
  344. fh=file_obj, # Pass original file_obj
  345. ) from exc_value
  346. else:
  347. raise exceptions.LockException(
  348. exc_value,
  349. strerror=str(exc_value),
  350. fh=file_obj, # Pass original file_obj
  351. ) from exc_value
  352. except EOFError as exc_value: # NFS specific
  353. raise exceptions.LockException(
  354. exc_value,
  355. strerror=str(exc_value),
  356. fh=file_obj, # Pass original file_obj
  357. ) from exc_value
  358. def unlock(self, file_obj: PosixFileArgument) -> None:
  359. fd = self._get_fd(file_obj)
  360. self.locker(fd, LockFlags.UNBLOCK)
  361. class FlockLocker(PosixLocker):
  362. """FlockLocker is a PosixLocker implementation using fcntl.flock."""
  363. LOCKER = fcntl.flock # type: ignore[attr-defined]
  364. class LockfLocker(PosixLocker):
  365. """LockfLocker is a PosixLocker implementation using fcntl.lockf."""
  366. LOCKER = fcntl.lockf # type: ignore[attr-defined]
  367. # LOCKER constant for POSIX is fcntl.flock for backward compatibility.
  368. # Type matches: Callable[[Union[int, HasFileno], int], Any]
  369. LOCKER = fcntl.flock # type: ignore[attr-defined,reportConstantRedefinition]
  370. _posix_locker_instance = PosixLocker()
  371. # Public API for POSIX uses the PosixLocker instance
  372. def lock(file: types.FileArgument, flags: LockFlags) -> None:
  373. _posix_locker_instance.lock(file, flags)
  374. def unlock(file: types.FileArgument) -> None:
  375. _posix_locker_instance.unlock(file)