lockfile.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. import base64
  2. import binascii
  3. import yaml
  4. import os
  5. import io
  6. import re
  7. from six.moves.urllib import parse as urlparse
  8. from six import iteritems
  9. from ..base import PackageJson, BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError
  10. LOCKFILE_VERSION = "lockfileVersion"
  11. class PnpmLockfile(BaseLockfile):
  12. IMPORTER_KEYS = PackageJson.DEP_KEYS + ("specifiers",)
  13. def read(self):
  14. with io.open(self.path, "rb") as f:
  15. self.data = yaml.load(f, Loader=yaml.CSafeLoader) or {LOCKFILE_VERSION: "6.0"}
  16. version_in_data = LOCKFILE_VERSION in self.data
  17. r = re.compile('^[56]\\.\\d$')
  18. if not version_in_data or not r.match(str(self.data[LOCKFILE_VERSION])):
  19. raise Exception(
  20. 'Error of project configuration: {} has lockfileVersion: {}. '.format(
  21. self.path, self.data[LOCKFILE_VERSION] if version_in_data else "<no-version>"
  22. )
  23. + 'This version is not supported. Please, delete pnpm-lock.yaml and regenerate it using "ya tool nots --clean update-lockfile"'
  24. )
  25. def write(self, path=None):
  26. """
  27. :param path: path to store lockfile, defaults to original path
  28. :type path: str
  29. """
  30. if path is None:
  31. path = self.path
  32. with open(path, "w") as f:
  33. yaml.dump(self.data, f, Dumper=yaml.CSafeDumper)
  34. def get_packages_meta(self):
  35. """
  36. Extracts packages meta from lockfile.
  37. :rtype: list of LockfilePackageMeta
  38. """
  39. packages = self.data.get("packages", {})
  40. return map(lambda x: _parse_package_meta(*x), iteritems(packages))
  41. def update_tarball_resolutions(self, fn):
  42. """
  43. :param fn: maps `LockfilePackageMeta` instance to new `resolution.tarball` value
  44. :type fn: lambda
  45. """
  46. packages = self.data.get("packages", {})
  47. for key, meta in iteritems(packages):
  48. meta["resolution"]["tarball"] = fn(_parse_package_meta(key, meta, allow_file_protocol=True))
  49. packages[key] = meta
  50. def get_importers(self):
  51. """
  52. Returns "importers" section from the lockfile or creates similar structure from "dependencies" and "specifiers".
  53. :rtype: dict of dict of dict of str
  54. """
  55. importers = self.data.get("importers")
  56. if importers is not None:
  57. return importers
  58. importer = {k: self.data[k] for k in self.IMPORTER_KEYS if k in self.data}
  59. return {".": importer} if importer else {}
  60. def merge(self, lf):
  61. """
  62. Merges two lockfiles:
  63. 1. Converts the lockfile to monorepo-like lockfile with "importers" section instead of "dependencies" and "specifiers".
  64. 2. Merges `lf`'s dependencies and specifiers to importers.
  65. 3. Merges `lf`'s packages to the lockfile.
  66. :param lf: lockfile to merge
  67. :type lf: PnpmLockfile
  68. """
  69. importers = self.get_importers()
  70. build_path = os.path.dirname(self.path)
  71. for [importer, imports] in iteritems(lf.get_importers()):
  72. importer_path = os.path.normpath(os.path.join(os.path.dirname(lf.path), importer))
  73. importer_rel_path = os.path.relpath(importer_path, build_path)
  74. importers[importer_rel_path] = imports
  75. self.data["importers"] = importers
  76. for k in self.IMPORTER_KEYS:
  77. self.data.pop(k, None)
  78. packages = self.data.get("packages", {})
  79. for k, v in iteritems(lf.data.get("packages", {})):
  80. if k not in packages:
  81. packages[k] = v
  82. self.data["packages"] = packages
  83. def validate_has_addons_flags(self):
  84. packages = self.data.get("packages", {})
  85. invalid_keys = []
  86. for key, meta in iteritems(packages):
  87. if meta.get("requiresBuild") and "hasAddons" not in meta:
  88. invalid_keys.append(key)
  89. return (not invalid_keys, invalid_keys)
  90. def _parse_package_meta(key, meta, allow_file_protocol=False):
  91. """
  92. :param key: uniq package key from lockfile
  93. :type key: string
  94. :param meta: package meta dict from lockfile
  95. :type meta: dict
  96. :rtype: LockfilePackageMetaInvalidError
  97. """
  98. try:
  99. tarball_url = _parse_tarball_url(meta["resolution"]["tarball"], allow_file_protocol)
  100. sky_id = _parse_sky_id_from_tarball_url(meta["resolution"]["tarball"])
  101. integrity_algorithm, integrity = _parse_package_integrity(meta["resolution"]["integrity"])
  102. except KeyError as e:
  103. raise TypeError("Invalid package meta for key {}, missing {} key".format(key, e))
  104. except LockfilePackageMetaInvalidError as e:
  105. raise TypeError("Invalid package meta for key {}, parse error: {}".format(key, e))
  106. return LockfilePackageMeta(key, tarball_url, sky_id, integrity, integrity_algorithm)
  107. def _parse_tarball_url(tarball_url, allow_file_protocol):
  108. if tarball_url.startswith("file:") and not allow_file_protocol:
  109. raise LockfilePackageMetaInvalidError("tarball cannot point to a file, got {}".format(tarball_url))
  110. return tarball_url.split("?")[0]
  111. def _parse_sky_id_from_tarball_url(tarball_url):
  112. """
  113. :param tarball_url: tarball url
  114. :type tarball_url: string
  115. :rtype: string
  116. """
  117. if tarball_url.startswith("file:"):
  118. return ""
  119. rbtorrent_param = urlparse.parse_qs(urlparse.urlparse(tarball_url).query).get("rbtorrent")
  120. if rbtorrent_param is None:
  121. return ""
  122. return "rbtorrent:{}".format(rbtorrent_param[0])
  123. def _parse_package_integrity(integrity):
  124. """
  125. Returns tuple of algorithm and hash (hex).
  126. :param integrity: package integrity in format "{algo}-{base64_of_hash}"
  127. :type integrity: string
  128. :rtype: (str, str)
  129. """
  130. algo, hash_b64 = integrity.split("-", 1)
  131. try:
  132. hash_hex = binascii.hexlify(base64.b64decode(hash_b64))
  133. except TypeError as e:
  134. raise LockfilePackageMetaInvalidError(
  135. "Invalid package integrity encoding, integrity: {}, error: {}".format(integrity, e)
  136. )
  137. return (algo, hash_hex)