__init__.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525
  1. # coding: utf-8
  2. import codecs
  3. import errno
  4. import logging
  5. import os
  6. import random
  7. import shutil
  8. import six
  9. import stat
  10. import sys
  11. import library.python.func
  12. import library.python.strings
  13. import library.python.windows
  14. logger = logging.getLogger(__name__)
  15. try:
  16. WindowsError
  17. except NameError:
  18. WindowsError = None
  19. _diehard_win_tries = 10
  20. errorfix_win = library.python.windows.errorfix
  21. class CustomFsError(OSError):
  22. def __init__(self, errno, message='', filename=None):
  23. super(CustomFsError, self).__init__(message)
  24. self.errno = errno
  25. self.strerror = os.strerror(errno)
  26. self.filename = filename
  27. # Directories creation
  28. # If dst is already exists and is a directory - does nothing
  29. # Throws OSError
  30. @errorfix_win
  31. def ensure_dir(path):
  32. try:
  33. os.makedirs(path)
  34. except OSError as e:
  35. if e.errno != errno.EEXIST or not os.path.isdir(path):
  36. raise
  37. # Directories creation
  38. # If dst is already exists and is a directory - does nothing
  39. # Returns path
  40. # Throws OSError
  41. @errorfix_win
  42. def create_dirs(path):
  43. ensure_dir(path)
  44. return path
  45. # Atomic file/directory move (rename)
  46. # Doesn't guarantee dst replacement
  47. # Atomic if no device boundaries are crossed
  48. # Depends on ctypes on Windows
  49. # Throws OSError
  50. # On Unix, if dst exists:
  51. # if dst is file or empty dir - replaces it
  52. # if src is dir and dst is not dir - throws OSError (errno ENOTDIR)
  53. # if src is dir and dst is non-empty dir - throws OSError (errno ENOTEMPTY)
  54. # if src is file and dst is dir - throws OSError (errno EISDIR)
  55. # On Windows, if dst exists - throws OSError (errno EEXIST)
  56. @errorfix_win
  57. @library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries)
  58. def move(src, dst):
  59. os.rename(src, dst)
  60. # Atomic replacing file move (rename)
  61. # Replaces dst if exists and not a dir
  62. # Doesn't guarantee dst dir replacement
  63. # Atomic if no device boundaries are crossed
  64. # Depends on ctypes on Windows
  65. # Throws OSError
  66. # On Unix, if dst exists:
  67. # if dst is file - replaces it
  68. # if dst is dir - throws OSError (errno EISDIR)
  69. # On Windows, if dst exists:
  70. # if dst is file - replaces it
  71. # if dst is dir - throws OSError (errno EACCES)
  72. @errorfix_win
  73. @library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries)
  74. def replace_file(src, dst):
  75. if library.python.windows.on_win():
  76. library.python.windows.replace_file(src, dst)
  77. else:
  78. os.rename(src, dst)
  79. # File/directory replacing move (rename)
  80. # Removes dst if exists
  81. # Non-atomic
  82. # Depends on ctypes on Windows
  83. # Throws OSError
  84. @errorfix_win
  85. def replace(src, dst):
  86. try:
  87. move(src, dst)
  88. except OSError as e:
  89. if e.errno not in (errno.EEXIST, errno.EISDIR, errno.ENOTDIR, errno.ENOTEMPTY):
  90. raise
  91. remove_tree(dst)
  92. move(src, dst)
  93. # Atomic file remove
  94. # Throws OSError
  95. @errorfix_win
  96. @library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries)
  97. def remove_file(path):
  98. os.remove(path)
  99. # Atomic empty directory remove
  100. # Throws OSError
  101. @errorfix_win
  102. @library.python.windows.diehard(library.python.windows.RETRIABLE_DIR_ERRORS, tries=_diehard_win_tries)
  103. def remove_dir(path):
  104. os.rmdir(path)
  105. def fix_path_encoding(path):
  106. return library.python.strings.to_str(path, library.python.strings.fs_encoding())
  107. # File/directory remove
  108. # Non-atomic
  109. # Throws OSError, AssertionError
  110. @errorfix_win
  111. def remove_tree(path):
  112. @library.python.windows.diehard(library.python.windows.RETRIABLE_DIR_ERRORS, tries=_diehard_win_tries)
  113. def rmtree(path):
  114. if library.python.windows.on_win():
  115. library.python.windows.rmtree(path)
  116. else:
  117. shutil.rmtree(fix_path_encoding(path))
  118. st = os.lstat(path)
  119. if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode):
  120. remove_file(path)
  121. elif stat.S_ISDIR(st.st_mode):
  122. rmtree(path)
  123. else:
  124. assert False
  125. # File/directory remove ignoring errors
  126. # Non-atomic
  127. @errorfix_win
  128. def remove_tree_safe(path):
  129. try:
  130. st = os.lstat(path)
  131. if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode):
  132. os.remove(path)
  133. elif stat.S_ISDIR(st.st_mode):
  134. shutil.rmtree(fix_path_encoding(path), ignore_errors=True)
  135. # XXX
  136. except UnicodeDecodeError as e:
  137. logging.exception(u'remove_tree_safe with argument %s raise exception: %s', path, e)
  138. raise
  139. except OSError:
  140. pass
  141. # File/directory remove
  142. # If path doesn't exist - does nothing
  143. # Non-atomic
  144. # Throws OSError, AssertionError
  145. @errorfix_win
  146. def ensure_removed(path):
  147. try:
  148. remove_tree(path)
  149. except OSError as e:
  150. if e.errno != errno.ENOENT:
  151. raise
  152. # Atomic file hardlink
  153. # Dst must not exist
  154. # Depends on ctypes on Windows
  155. # Throws OSError
  156. # If dst exists - throws OSError (errno EEXIST)
  157. @errorfix_win
  158. def hardlink(src, lnk):
  159. if library.python.windows.on_win():
  160. library.python.windows.hardlink(src, lnk)
  161. else:
  162. os.link(src, lnk)
  163. # Atomic file/directory symlink (Unix only)
  164. # Dst must not exist
  165. # Throws OSError
  166. # If dst exists - throws OSError (errno EEXIST)
  167. @errorfix_win
  168. def symlink(src, lnk):
  169. if library.python.windows.on_win():
  170. library.python.windows.run_disabled(src, lnk)
  171. else:
  172. os.symlink(src, lnk)
  173. # shutil.copy2 with follow_symlinks=False parameter (Unix only)
  174. def copy2(src, lnk, follow_symlinks=True):
  175. if six.PY3:
  176. shutil.copy2(src, lnk, follow_symlinks=follow_symlinks)
  177. return
  178. if follow_symlinks or not os.path.islink(src):
  179. shutil.copy2(src, lnk)
  180. return
  181. symlink(os.readlink(src), lnk)
  182. def copy2_safe(src, lnk, follow_symlinks=True):
  183. try:
  184. copy2(src, lnk, follow_symlinks)
  185. except shutil.SameFileError:
  186. pass
  187. @errorfix_win
  188. def hardlink_or_copy(src, lnk, copy_function=copy2):
  189. def should_fallback_to_copy(exc):
  190. if WindowsError is not None and isinstance(exc, WindowsError) and exc.winerror == 1142: # too many hardlinks
  191. return True
  192. # cross-device hardlink or too many hardlinks, or some known WSL error
  193. if isinstance(exc, OSError) and exc.errno in (
  194. errno.EXDEV,
  195. errno.EMLINK,
  196. errno.EINVAL,
  197. errno.EACCES,
  198. errno.EPERM,
  199. ):
  200. return True
  201. return False
  202. try:
  203. hardlink(src, lnk)
  204. except Exception as e:
  205. logger.debug('Failed to hardlink %s to %s with error %s, will copy it', src, lnk, repr(e))
  206. if should_fallback_to_copy(e):
  207. copy_function(src, lnk, follow_symlinks=False)
  208. else:
  209. raise
  210. # Recursively hardlink directory
  211. # Uses plain hardlink for files
  212. # Dst must not exist
  213. # Non-atomic
  214. # Throws OSError
  215. @errorfix_win
  216. def hardlink_tree(src, dst, hardlink_function=hardlink, mkdir_function=os.mkdir):
  217. if not os.path.exists(src):
  218. raise CustomFsError(errno.ENOENT, filename=src)
  219. if os.path.isfile(src):
  220. hardlink_function(src, dst)
  221. return
  222. for dirpath, _, filenames in walk_relative(src):
  223. src_dirpath = os.path.join(src, dirpath) if dirpath != '.' else src
  224. dst_dirpath = os.path.join(dst, dirpath) if dirpath != '.' else dst
  225. mkdir_function(dst_dirpath)
  226. for filename in filenames:
  227. hardlink_function(os.path.join(src_dirpath, filename), os.path.join(dst_dirpath, filename))
  228. # File copy
  229. # throws EnvironmentError (OSError, IOError)
  230. @errorfix_win
  231. def copy_file(src, dst, copy_function=shutil.copy2):
  232. if os.path.isdir(dst):
  233. raise CustomFsError(errno.EISDIR, filename=dst)
  234. copy_function(src, dst)
  235. # File/directory copy
  236. # throws EnvironmentError (OSError, IOError, shutil.Error)
  237. @errorfix_win
  238. def copy_tree(src, dst, copy_function=shutil.copy2):
  239. if os.path.isfile(src):
  240. copy_file(src, dst, copy_function=copy_function)
  241. return
  242. copytree3(src, dst, copy_function=copy_function)
  243. # File read
  244. # Throws OSError
  245. @errorfix_win
  246. def read_file(path, binary=True):
  247. kwargs = {}
  248. if not binary and six.PY3:
  249. kwargs['encoding'] = sys.getfilesystemencoding()
  250. with open(path, 'r' + ('b' if binary else ''), **kwargs) as f:
  251. return f.read()
  252. # Text file read
  253. # Throws OSError
  254. @errorfix_win
  255. def read_text(path):
  256. return read_file(path, binary=False)
  257. # Decoding file read
  258. # Throws OSError
  259. @errorfix_win
  260. def read_file_unicode(path, binary=True, enc='utf-8'):
  261. if not binary:
  262. if six.PY2:
  263. with open(path, 'r') as f:
  264. return library.python.strings.to_unicode(f.read(), enc)
  265. else:
  266. with open(path, 'r', encoding=enc) as f:
  267. return f.read()
  268. # codecs.open is always binary
  269. with codecs.open(path, 'r', encoding=enc, errors=library.python.strings.ENCODING_ERRORS_POLICY) as f:
  270. return f.read()
  271. @errorfix_win
  272. def open_file(*args, **kwargs):
  273. return (
  274. library.python.windows.open_file(*args, **kwargs) if library.python.windows.on_win() else open(*args, **kwargs)
  275. )
  276. # Atomic file write
  277. # Throws OSError
  278. @errorfix_win
  279. def write_file(path, data, binary=True):
  280. dir_path = os.path.dirname(path)
  281. if dir_path:
  282. ensure_dir(dir_path)
  283. tmp_path = path + '.tmp.' + str(random.random())
  284. with open_file(tmp_path, 'w' + ('b' if binary else '')) as f:
  285. if not isinstance(data, bytes) and binary:
  286. data = data.encode('UTF-8')
  287. f.write(data)
  288. replace_file(tmp_path, path)
  289. # Atomic text file write
  290. # Throws OSError
  291. @errorfix_win
  292. def write_text(path, data):
  293. write_file(path, data, binary=False)
  294. # File size
  295. # Throws OSError
  296. @errorfix_win
  297. def get_file_size(path):
  298. return os.path.getsize(path)
  299. # File/directory size
  300. # Non-recursive mode for directory counts size for immediates
  301. # While raise_all_errors is set to False, file size fallbacks to zero in case of getsize errors
  302. # Throws OSError
  303. @errorfix_win
  304. def get_tree_size(path, recursive=False, raise_all_errors=False):
  305. if os.path.isfile(path):
  306. return get_file_size(path)
  307. total_size = 0
  308. for dir_path, _, files in os.walk(path):
  309. for f in files:
  310. fp = os.path.join(dir_path, f)
  311. try:
  312. total_size += get_file_size(fp)
  313. except OSError as e:
  314. if raise_all_errors:
  315. raise
  316. logger.debug("Cannot calculate file size: %s", e)
  317. if not recursive:
  318. break
  319. return total_size
  320. # Directory copy ported from Python 3
  321. def copytree3(
  322. src,
  323. dst,
  324. symlinks=False,
  325. ignore=None,
  326. copy_function=shutil.copy2,
  327. ignore_dangling_symlinks=False,
  328. dirs_exist_ok=False,
  329. ):
  330. """Recursively copy a directory tree.
  331. The copytree3 is a port of shutil.copytree function from python-3.2.
  332. It has additional useful parameters and may be helpful while we are
  333. on python-2.x. It has to be removed as soon as we have moved to
  334. python-3.2 or higher.
  335. The destination directory must not already exist.
  336. If exception(s) occur, an Error is raised with a list of reasons.
  337. If the optional symlinks flag is true, symbolic links in the
  338. source tree result in symbolic links in the destination tree; if
  339. it is false, the contents of the files pointed to by symbolic
  340. links are copied. If the file pointed by the symlink doesn't
  341. exist, an exception will be added in the list of errors raised in
  342. an Error exception at the end of the copy process.
  343. You can set the optional ignore_dangling_symlinks flag to true if you
  344. want to silence this exception. Notice that this has no effect on
  345. platforms that don't support os.symlink.
  346. The optional ignore argument is a callable. If given, it
  347. is called with the `src` parameter, which is the directory
  348. being visited by copytree3(), and `names` which is the list of
  349. `src` contents, as returned by os.listdir():
  350. callable(src, names) -> ignored_names
  351. Since copytree3() is called recursively, the callable will be
  352. called once for each directory that is copied. It returns a
  353. list of names relative to the `src` directory that should
  354. not be copied.
  355. The optional copy_function argument is a callable that will be used
  356. to copy each file. It will be called with the source path and the
  357. destination path as arguments. By default, copy2() is used, but any
  358. function that supports the same signature (like copy()) can be used.
  359. """
  360. names = os.listdir(src)
  361. if ignore is not None:
  362. ignored_names = ignore(src, names)
  363. else:
  364. ignored_names = set()
  365. if not (dirs_exist_ok and os.path.isdir(dst)):
  366. os.makedirs(dst)
  367. errors = []
  368. for name in names:
  369. if name in ignored_names:
  370. continue
  371. srcname = os.path.join(src, name)
  372. dstname = os.path.join(dst, name)
  373. try:
  374. if os.path.islink(srcname):
  375. linkto = os.readlink(srcname)
  376. if symlinks:
  377. # We can't just leave it to `copy_function` because legacy
  378. # code with a custom `copy_function` may rely on copytree3
  379. # doing the right thing.
  380. os.symlink(linkto, dstname)
  381. else:
  382. # ignore dangling symlink if the flag is on
  383. if not os.path.exists(linkto) and ignore_dangling_symlinks:
  384. continue
  385. # otherwise let the copy occurs. copy2 will raise an error
  386. copy_function(srcname, dstname)
  387. elif os.path.isdir(srcname):
  388. copytree3(srcname, dstname, symlinks, ignore, copy_function, dirs_exist_ok=dirs_exist_ok)
  389. else:
  390. # Will raise a SpecialFileError for unsupported file types
  391. copy_function(srcname, dstname)
  392. # catch the Error from the recursive copytree3 so that we can
  393. # continue with other files
  394. except shutil.Error as err:
  395. errors.extend(err.args[0])
  396. except EnvironmentError as why:
  397. errors.append((srcname, dstname, str(why)))
  398. try:
  399. shutil.copystat(src, dst)
  400. except OSError as why:
  401. if WindowsError is not None and isinstance(why, WindowsError):
  402. # Copying file access times may fail on Windows
  403. pass
  404. else:
  405. errors.extend((src, dst, str(why)))
  406. if errors:
  407. raise shutil.Error(errors)
  408. def walk_relative(path, topdown=True, onerror=None, followlinks=False):
  409. for dirpath, dirnames, filenames in os.walk(path, topdown=topdown, onerror=onerror, followlinks=followlinks):
  410. yield os.path.relpath(dirpath, path), dirnames, filenames
  411. def supports_clone():
  412. if 'darwin' in sys.platform:
  413. import platform
  414. return list(map(int, platform.mac_ver()[0].split('.'))) >= [10, 13]
  415. return False
  416. def commonpath(paths):
  417. assert paths
  418. if len(paths) == 1:
  419. return next(iter(paths))
  420. split_paths = [path.split(os.sep) for path in paths]
  421. smin = min(split_paths)
  422. smax = max(split_paths)
  423. common = smin
  424. for i, c in enumerate(smin):
  425. if c != smax[i]:
  426. common = smin[:i]
  427. break
  428. return os.path.sep.join(common)
  429. def set_execute_bits(filename):
  430. stm = os.stat(filename).st_mode
  431. exe = stm | 0o111
  432. if stm != exe:
  433. os.chmod(filename, exe)