update_symbols.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. # Copyright 2020 Google LLC
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. """Quick and dirty utility to download latest icon assets for github."""
  15. from absl import app
  16. from absl import flags
  17. import icons
  18. import json
  19. from pathlib import Path
  20. import re
  21. import requests
  22. import time
  23. from typing import NamedTuple, Set, Sequence, Tuple
  24. from zipfile import ZipFile
  25. from joblib import Parallel, delayed, wrap_non_picklable_objects
  26. from fontTools.ttLib import woff2
  27. import os
  28. FLAGS = flags.FLAGS
  29. flags.DEFINE_bool("fetch", True, "Whether we can attempt to download assets.")
  30. flags.DEFINE_integer("icon_limit", 0, "If > 0, the max # of icons to process.")
  31. _METADATA_URL = "http://fonts.google.com/metadata/icons?incomplete=1&key=material_symbols"
  32. class Asset(NamedTuple):
  33. src_url_pattern: str
  34. dest_dir_pattern: str
  35. class Fetch(NamedTuple):
  36. src_url: str
  37. dest_file: Path
  38. class Icon(NamedTuple):
  39. name: str
  40. version: int
  41. stylistic_sets: Set[str]
  42. _ICON_ASSETS = (
  43. Asset(
  44. "https://{host}/s/i/short-term/release/{stylistic_set_snake}/{icon.name}/{style}/{size_px}px.svg",
  45. "symbols/web/{icon.name}/{stylistic_set_snake}/{icon.name}{style_suffix}_{size_px}px.svg",
  46. ),
  47. Asset(
  48. "https://{host}/s/i/short-term/release/{stylistic_set_snake}/{icon.name}/{style}/{size_px}px.xml",
  49. "symbols/android/{icon.name}/{stylistic_set_snake}/{icon.name}{style_suffix}_{size_px}px.xml",
  50. ),
  51. )
  52. # no wght variants for apple symbols.
  53. _ICON_IOS_ASSETS = (
  54. Asset(
  55. "https://{host}/s/i/short-term/release/{stylistic_set_snake}/{icon.name}/{style}/{icon.name}{style_suffix}_symbol.svg",
  56. "symbols/ios/{icon.name}/{stylistic_set_snake}/{icon.name}{style_suffix}_symbol.svg"
  57. ),
  58. )
  59. _HEADERS = {
  60. 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
  61. }
  62. _SET_ASSETS = (
  63. # Fonts are acquired by abusing the Google Fonts web api. Nobody tell them :D
  64. Asset(
  65. "https://fonts.googleapis.com/css2?family={stylistic_set_url}:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200",
  66. "variablefont/{stylistic_set_font}.css",
  67. ),
  68. )
  69. def _latest_metadata():
  70. resp = requests.get(_METADATA_URL)
  71. resp.raise_for_status()
  72. raw_json = resp.text[5:]
  73. return json.loads(raw_json)
  74. def _current_versions():
  75. return Path("current_versions.json")
  76. def _version_key(icon: Icon):
  77. return f"symbols::{icon.name}"
  78. def _symbol_families(metadata):
  79. return set(s for s in set(metadata["families"]) if "Symbols" in s)
  80. def _icons(metadata):
  81. all_sets = _symbol_families(metadata)
  82. for raw_icon in metadata["icons"]:
  83. unsupported = set(raw_icon["unsupported_families"])
  84. yield Icon(
  85. raw_icon["name"],
  86. raw_icon["version"],
  87. all_sets - unsupported,
  88. )
  89. def _create_fetch(asset, args):
  90. src_url = asset.src_url_pattern.format(**args)
  91. dest_file = asset.dest_dir_pattern.format(**args)
  92. dest_file = (Path(__file__) / "../.." / dest_file).resolve()
  93. return Fetch(src_url, dest_file)
  94. @delayed
  95. def _do_fetch_delayed(src_url, dest_file, i, total):
  96. _do_fetch(src_url, dest_file)
  97. if i % 5000 == 0:
  98. print("%d/%d complete" % (i, total))
  99. def _do_fetch(src_url, dest_file):
  100. try :
  101. resp = requests.get(src_url, headers = _HEADERS)
  102. resp.raise_for_status()
  103. dest_file.parent.mkdir(parents=True, exist_ok=True)
  104. dest_file.write_bytes(resp.content)
  105. except Exception as e:
  106. print(str(e))
  107. def _do_fetches(fetches):
  108. print(f"Starting {len(fetches)} fetches")
  109. total = len(fetches)
  110. Parallel(n_jobs=50)(_do_fetch_delayed(f.src_url, f.dest_file, i, total) for i,f in enumerate(fetches))
  111. if total:
  112. print("%d/%d complete" % (total, total))
  113. def decompress(infilepath: Path, outfilepath: Path):
  114. with infilepath.open(mode='rb') as infile:
  115. with outfilepath.open(mode='wb') as outfile:
  116. woff2.decompress(infile, outfile)
  117. def _fetch_fonts(css_files: Sequence[Path]):
  118. for css_file in css_files:
  119. css = css_file.read_text()
  120. url = re.search(r"src:\s+url\(([^)]+)\)", css).group(1)
  121. assert url.endswith(".woff2")
  122. woff2_file = css_file.parent / (css_file.stem + ".woff2")
  123. dest_file = css_file.parent / (css_file.stem + ".ttf")
  124. _do_fetch(url, woff2_file)
  125. decompress(woff2_file, dest_file)
  126. css_file.unlink()
  127. with open(dest_file.with_suffix(".codepoints"), "w") as f:
  128. for name, codepoint in sorted(icons.enumerate(dest_file)):
  129. f.write(f"{name} {codepoint:04x}\n")
  130. def _is_css(p: Path):
  131. return p.suffix == ".css"
  132. def _files(fetches: Sequence[Fetch], pred):
  133. return [f.dest_file for f in fetches if pred(f.dest_file)]
  134. def _should_skip(fetch: Fetch):
  135. return fetch.dest_file.is_file()
  136. def _pattern_args(metadata, stylistic_set):
  137. return {
  138. "host": metadata["host"],
  139. "stylistic_set_snake": stylistic_set.replace(" ", "").lower(),
  140. "stylistic_set_url": stylistic_set.replace(" ", "+"),
  141. "stylistic_set_font": stylistic_set.replace(" ", "") + "[FILL,GRAD,opsz,wght]",
  142. }
  143. def _create_fetches(style, opsz, pattern_args, fetches, skips, assets):
  144. pattern_args["style"] = style if style else "default"
  145. pattern_args["style_suffix"] = f"_{style}" if style else ""
  146. pattern_args["size_px"] = str(opsz)
  147. for asset in assets:
  148. fetch = _create_fetch(asset, pattern_args)
  149. if _should_skip(fetch):
  150. skips.append(fetch)
  151. else:
  152. fetches.append(fetch)
  153. def main(_):
  154. current_versions = json.loads(_current_versions().read_text())
  155. metadata = _latest_metadata()
  156. stylistic_sets = _symbol_families(metadata)
  157. fetches = []
  158. skips = []
  159. num_changed = 0
  160. icons = tuple(_icons(metadata))
  161. if FLAGS.icon_limit > 0:
  162. icons = icons[: FLAGS.icon_limit]
  163. for icon in icons:
  164. ver_key = _version_key(icon)
  165. if icon.version <= current_versions.get(ver_key, 0):
  166. continue
  167. current_versions[ver_key] = icon.version
  168. num_changed += 1
  169. for stylistic_set in stylistic_sets:
  170. if stylistic_set not in icon.stylistic_sets:
  171. continue
  172. pattern_args = _pattern_args(metadata, stylistic_set)
  173. pattern_args["icon"] = icon
  174. for opsz in [20,24,40,48] :
  175. for fill in ["","fill1"] :
  176. for grad in ["gradN25","","grad200"]:
  177. _create_fetches(grad + fill, opsz, pattern_args, fetches, skips, _ICON_IOS_ASSETS)
  178. for wght in ["wght100","wght200","wght300","","wght500","wght600","wght700"] :
  179. _create_fetches(wght + grad + fill, opsz, pattern_args, fetches, skips, _ICON_ASSETS)
  180. for stylistic_set in stylistic_sets:
  181. for asset in _SET_ASSETS:
  182. pattern_args = _pattern_args(metadata, stylistic_set)
  183. fetch = _create_fetch(asset, pattern_args)
  184. fetches.append(fetch)
  185. print(f"{num_changed}/{len(icons)} icons have changed")
  186. if skips:
  187. print(f"{len(skips)} fetches skipped because assets exist")
  188. if fetches:
  189. if FLAGS.fetch:
  190. _do_fetches(fetches)
  191. else:
  192. print(f"fetch disabled; not fetching {len(fetches)} assets")
  193. _fetch_fonts(_files(fetches + skips, _is_css))
  194. with open(_current_versions(), "w") as f:
  195. json.dump(current_versions, f, indent=4, sort_keys=True)
  196. if __name__ == "__main__":
  197. app.run(main)