jsinterp.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871
  1. import collections
  2. import contextlib
  3. import itertools
  4. import json
  5. import math
  6. import operator
  7. import re
  8. from .utils import (
  9. NO_DEFAULT,
  10. ExtractorError,
  11. function_with_repr,
  12. js_to_json,
  13. remove_quotes,
  14. truncate_string,
  15. unified_timestamp,
  16. write_string,
  17. )
  18. def _js_bit_op(op):
  19. def zeroise(x):
  20. if x in (None, JS_Undefined):
  21. return 0
  22. with contextlib.suppress(TypeError):
  23. if math.isnan(x): # NB: NaN cannot be checked by membership
  24. return 0
  25. return x
  26. def wrapped(a, b):
  27. return op(zeroise(a), zeroise(b)) & 0xffffffff
  28. return wrapped
  29. def _js_arith_op(op):
  30. def wrapped(a, b):
  31. if JS_Undefined in (a, b):
  32. return float('nan')
  33. return op(a or 0, b or 0)
  34. return wrapped
  35. def _js_div(a, b):
  36. if JS_Undefined in (a, b) or not (a or b):
  37. return float('nan')
  38. return (a or 0) / b if b else float('inf')
  39. def _js_mod(a, b):
  40. if JS_Undefined in (a, b) or not b:
  41. return float('nan')
  42. return (a or 0) % b
  43. def _js_exp(a, b):
  44. if not b:
  45. return 1 # even 0 ** 0 !!
  46. elif JS_Undefined in (a, b):
  47. return float('nan')
  48. return (a or 0) ** b
  49. def _js_eq_op(op):
  50. def wrapped(a, b):
  51. if {a, b} <= {None, JS_Undefined}:
  52. return op(a, a)
  53. return op(a, b)
  54. return wrapped
  55. def _js_comp_op(op):
  56. def wrapped(a, b):
  57. if JS_Undefined in (a, b):
  58. return False
  59. if isinstance(a, str) or isinstance(b, str):
  60. return op(str(a or 0), str(b or 0))
  61. return op(a or 0, b or 0)
  62. return wrapped
  63. def _js_ternary(cndn, if_true=True, if_false=False):
  64. """Simulate JS's ternary operator (cndn?if_true:if_false)"""
  65. if cndn in (False, None, 0, '', JS_Undefined):
  66. return if_false
  67. with contextlib.suppress(TypeError):
  68. if math.isnan(cndn): # NB: NaN cannot be checked by membership
  69. return if_false
  70. return if_true
  71. # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence
  72. _OPERATORS = { # None => Defined in JSInterpreter._operator
  73. '?': None,
  74. '??': None,
  75. '||': None,
  76. '&&': None,
  77. '|': _js_bit_op(operator.or_),
  78. '^': _js_bit_op(operator.xor),
  79. '&': _js_bit_op(operator.and_),
  80. '===': operator.is_,
  81. '!==': operator.is_not,
  82. '==': _js_eq_op(operator.eq),
  83. '!=': _js_eq_op(operator.ne),
  84. '<=': _js_comp_op(operator.le),
  85. '>=': _js_comp_op(operator.ge),
  86. '<': _js_comp_op(operator.lt),
  87. '>': _js_comp_op(operator.gt),
  88. '>>': _js_bit_op(operator.rshift),
  89. '<<': _js_bit_op(operator.lshift),
  90. '+': _js_arith_op(operator.add),
  91. '-': _js_arith_op(operator.sub),
  92. '*': _js_arith_op(operator.mul),
  93. '%': _js_mod,
  94. '/': _js_div,
  95. '**': _js_exp,
  96. }
  97. _COMP_OPERATORS = {'===', '!==', '==', '!=', '<=', '>=', '<', '>'}
  98. _NAME_RE = r'[a-zA-Z_$][\w$]*'
  99. _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
  100. _QUOTES = '\'"/'
  101. class JS_Undefined:
  102. pass
  103. class JS_Break(ExtractorError):
  104. def __init__(self):
  105. ExtractorError.__init__(self, 'Invalid break')
  106. class JS_Continue(ExtractorError):
  107. def __init__(self):
  108. ExtractorError.__init__(self, 'Invalid continue')
  109. class JS_Throw(ExtractorError):
  110. def __init__(self, e):
  111. self.error = e
  112. ExtractorError.__init__(self, f'Uncaught exception {e}')
  113. class LocalNameSpace(collections.ChainMap):
  114. def __setitem__(self, key, value):
  115. for scope in self.maps:
  116. if key in scope:
  117. scope[key] = value
  118. return
  119. self.maps[0][key] = value
  120. def __delitem__(self, key):
  121. raise NotImplementedError('Deleting is not supported')
  122. class Debugger:
  123. import sys
  124. ENABLED = False and 'pytest' in sys.modules
  125. @staticmethod
  126. def write(*args, level=100):
  127. write_string(f'[debug] JS: {" " * (100 - level)}'
  128. f'{" ".join(truncate_string(str(x), 50, 50) for x in args)}\n')
  129. @classmethod
  130. def wrap_interpreter(cls, f):
  131. def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs):
  132. if cls.ENABLED and stmt.strip():
  133. cls.write(stmt, level=allow_recursion)
  134. try:
  135. ret, should_ret = f(self, stmt, local_vars, allow_recursion, *args, **kwargs)
  136. except Exception as e:
  137. if cls.ENABLED:
  138. if isinstance(e, ExtractorError):
  139. e = e.orig_msg
  140. cls.write('=> Raises:', e, '<-|', stmt, level=allow_recursion)
  141. raise
  142. if cls.ENABLED and stmt.strip():
  143. if should_ret or repr(ret) != stmt:
  144. cls.write(['->', '=>'][should_ret], repr(ret), '<-|', stmt, level=allow_recursion)
  145. return ret, should_ret
  146. return interpret_statement
  147. class JSInterpreter:
  148. __named_object_counter = 0
  149. _RE_FLAGS = {
  150. # special knowledge: Python's re flags are bitmask values, current max 128
  151. # invent new bitmask values well above that for literal parsing
  152. # TODO: new pattern class to execute matches with these flags
  153. 'd': 1024, # Generate indices for substring matches
  154. 'g': 2048, # Global search
  155. 'i': re.I, # Case-insensitive search
  156. 'm': re.M, # Multi-line search
  157. 's': re.S, # Allows . to match newline characters
  158. 'u': re.U, # Treat a pattern as a sequence of unicode code points
  159. 'y': 4096, # Perform a "sticky" search that matches starting at the current position in the target string
  160. }
  161. def __init__(self, code, objects=None):
  162. self.code, self._functions = code, {}
  163. self._objects = {} if objects is None else objects
  164. class Exception(ExtractorError): # noqa: A001
  165. def __init__(self, msg, expr=None, *args, **kwargs):
  166. if expr is not None:
  167. msg = f'{msg.rstrip()} in: {truncate_string(expr, 50, 50)}'
  168. super().__init__(msg, *args, **kwargs)
  169. def _named_object(self, namespace, obj):
  170. self.__named_object_counter += 1
  171. name = f'__yt_dlp_jsinterp_obj{self.__named_object_counter}'
  172. if callable(obj) and not isinstance(obj, function_with_repr):
  173. obj = function_with_repr(obj, f'F<{self.__named_object_counter}>')
  174. namespace[name] = obj
  175. return name
  176. @classmethod
  177. def _regex_flags(cls, expr):
  178. flags = 0
  179. if not expr:
  180. return flags, expr
  181. for idx, ch in enumerate(expr): # noqa: B007
  182. if ch not in cls._RE_FLAGS:
  183. break
  184. flags |= cls._RE_FLAGS[ch]
  185. return flags, expr[idx + 1:]
  186. @staticmethod
  187. def _separate(expr, delim=',', max_split=None):
  188. OP_CHARS = '+-*/%&|^=<>!,;{}:['
  189. if not expr:
  190. return
  191. counters = {k: 0 for k in _MATCHING_PARENS.values()}
  192. start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
  193. in_quote, escaping, after_op, in_regex_char_group = None, False, True, False
  194. for idx, char in enumerate(expr):
  195. if not in_quote and char in _MATCHING_PARENS:
  196. counters[_MATCHING_PARENS[char]] += 1
  197. elif not in_quote and char in counters:
  198. # Something's wrong if we get negative, but ignore it anyway
  199. if counters[char]:
  200. counters[char] -= 1
  201. elif not escaping:
  202. if char in _QUOTES and in_quote in (char, None):
  203. if in_quote or after_op or char != '/':
  204. in_quote = None if in_quote and not in_regex_char_group else char
  205. elif in_quote == '/' and char in '[]':
  206. in_regex_char_group = char == '['
  207. escaping = not escaping and in_quote and char == '\\'
  208. in_unary_op = (not in_quote and not in_regex_char_group
  209. and after_op not in (True, False) and char in '-+')
  210. after_op = char if (not in_quote and char in OP_CHARS) else (char.isspace() and after_op)
  211. if char != delim[pos] or any(counters.values()) or in_quote or in_unary_op:
  212. pos = 0
  213. continue
  214. elif pos != delim_len:
  215. pos += 1
  216. continue
  217. yield expr[start: idx - delim_len]
  218. start, pos = idx + 1, 0
  219. splits += 1
  220. if max_split and splits >= max_split:
  221. break
  222. yield expr[start:]
  223. @classmethod
  224. def _separate_at_paren(cls, expr, delim=None):
  225. if delim is None:
  226. delim = expr and _MATCHING_PARENS[expr[0]]
  227. separated = list(cls._separate(expr, delim, 1))
  228. if len(separated) < 2:
  229. raise cls.Exception(f'No terminating paren {delim}', expr)
  230. return separated[0][1:].strip(), separated[1].strip()
  231. def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
  232. if op in ('||', '&&'):
  233. if (op == '&&') ^ _js_ternary(left_val):
  234. return left_val # short circuiting
  235. elif op == '??':
  236. if left_val not in (None, JS_Undefined):
  237. return left_val
  238. elif op == '?':
  239. right_expr = _js_ternary(left_val, *self._separate(right_expr, ':', 1))
  240. right_val = self.interpret_expression(right_expr, local_vars, allow_recursion)
  241. if not _OPERATORS.get(op):
  242. return right_val
  243. try:
  244. return _OPERATORS[op](left_val, right_val)
  245. except Exception as e:
  246. raise self.Exception(f'Failed to evaluate {left_val!r} {op} {right_val!r}', expr, cause=e)
  247. def _index(self, obj, idx, allow_undefined=False):
  248. if idx == 'length':
  249. return len(obj)
  250. try:
  251. return obj[int(idx)] if isinstance(obj, list) else obj[idx]
  252. except Exception as e:
  253. if allow_undefined:
  254. return JS_Undefined
  255. raise self.Exception(f'Cannot get index {idx}', repr(obj), cause=e)
  256. def _dump(self, obj, namespace):
  257. try:
  258. return json.dumps(obj)
  259. except TypeError:
  260. return self._named_object(namespace, obj)
  261. @Debugger.wrap_interpreter
  262. def interpret_statement(self, stmt, local_vars, allow_recursion=100):
  263. if allow_recursion < 0:
  264. raise self.Exception('Recursion limit reached')
  265. allow_recursion -= 1
  266. should_return = False
  267. sub_statements = list(self._separate(stmt, ';')) or ['']
  268. expr = stmt = sub_statements.pop().strip()
  269. for sub_stmt in sub_statements:
  270. ret, should_return = self.interpret_statement(sub_stmt, local_vars, allow_recursion)
  271. if should_return:
  272. return ret, should_return
  273. m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|(?=["\'])|$)|(?P<throw>throw\s+)', stmt)
  274. if m:
  275. expr = stmt[len(m.group(0)):].strip()
  276. if m.group('throw'):
  277. raise JS_Throw(self.interpret_expression(expr, local_vars, allow_recursion))
  278. should_return = not m.group('var')
  279. if not expr:
  280. return None, should_return
  281. if expr[0] in _QUOTES:
  282. inner, outer = self._separate(expr, expr[0], 1)
  283. if expr[0] == '/':
  284. flags, outer = self._regex_flags(outer)
  285. # We don't support regex methods yet, so no point compiling it
  286. inner = f'{inner}/{flags}'
  287. # Avoid https://github.com/python/cpython/issues/74534
  288. # inner = re.compile(inner[1:].replace('[[', r'[\['), flags=flags)
  289. else:
  290. inner = json.loads(js_to_json(f'{inner}{expr[0]}', strict=True))
  291. if not outer:
  292. return inner, should_return
  293. expr = self._named_object(local_vars, inner) + outer
  294. if expr.startswith('new '):
  295. obj = expr[4:]
  296. if obj.startswith('Date('):
  297. left, right = self._separate_at_paren(obj[4:])
  298. date = unified_timestamp(
  299. self.interpret_expression(left, local_vars, allow_recursion), False)
  300. if date is None:
  301. raise self.Exception(f'Failed to parse date {left!r}', expr)
  302. expr = self._dump(int(date * 1000), local_vars) + right
  303. else:
  304. raise self.Exception(f'Unsupported object {obj}', expr)
  305. if expr.startswith('void '):
  306. left = self.interpret_expression(expr[5:], local_vars, allow_recursion)
  307. return None, should_return
  308. if expr.startswith('{'):
  309. inner, outer = self._separate_at_paren(expr)
  310. # try for object expression (Map)
  311. sub_expressions = [list(self._separate(sub_expr.strip(), ':', 1)) for sub_expr in self._separate(inner)]
  312. if all(len(sub_expr) == 2 for sub_expr in sub_expressions):
  313. def dict_item(key, val):
  314. val = self.interpret_expression(val, local_vars, allow_recursion)
  315. if re.match(_NAME_RE, key):
  316. return key, val
  317. return self.interpret_expression(key, local_vars, allow_recursion), val
  318. return dict(dict_item(k, v) for k, v in sub_expressions), should_return
  319. inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
  320. if not outer or should_abort:
  321. return inner, should_abort or should_return
  322. else:
  323. expr = self._dump(inner, local_vars) + outer
  324. if expr.startswith('('):
  325. inner, outer = self._separate_at_paren(expr)
  326. inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
  327. if not outer or should_abort:
  328. return inner, should_abort or should_return
  329. else:
  330. expr = self._dump(inner, local_vars) + outer
  331. if expr.startswith('['):
  332. inner, outer = self._separate_at_paren(expr)
  333. name = self._named_object(local_vars, [
  334. self.interpret_expression(item, local_vars, allow_recursion)
  335. for item in self._separate(inner)])
  336. expr = name + outer
  337. m = re.match(r'''(?x)
  338. (?P<try>try)\s*\{|
  339. (?P<if>if)\s*\(|
  340. (?P<switch>switch)\s*\(|
  341. (?P<for>for)\s*\(
  342. ''', expr)
  343. md = m.groupdict() if m else {}
  344. if md.get('if'):
  345. cndn, expr = self._separate_at_paren(expr[m.end() - 1:])
  346. if_expr, expr = self._separate_at_paren(expr.lstrip())
  347. # TODO: "else if" is not handled
  348. else_expr = None
  349. m = re.match(r'else\s*{', expr)
  350. if m:
  351. else_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
  352. cndn = _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion))
  353. ret, should_abort = self.interpret_statement(
  354. if_expr if cndn else else_expr, local_vars, allow_recursion)
  355. if should_abort:
  356. return ret, True
  357. if md.get('try'):
  358. try_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
  359. err = None
  360. try:
  361. ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
  362. if should_abort:
  363. return ret, True
  364. except Exception as e:
  365. # XXX: This works for now, but makes debugging future issues very hard
  366. err = e
  367. pending = (None, False)
  368. m = re.match(fr'catch\s*(?P<err>\(\s*{_NAME_RE}\s*\))?\{{', expr)
  369. if m:
  370. sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
  371. if err:
  372. catch_vars = {}
  373. if m.group('err'):
  374. catch_vars[m.group('err')] = err.error if isinstance(err, JS_Throw) else err
  375. catch_vars = local_vars.new_child(catch_vars)
  376. err, pending = None, self.interpret_statement(sub_expr, catch_vars, allow_recursion)
  377. m = re.match(r'finally\s*\{', expr)
  378. if m:
  379. sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
  380. ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
  381. if should_abort:
  382. return ret, True
  383. ret, should_abort = pending
  384. if should_abort:
  385. return ret, True
  386. if err:
  387. raise err
  388. elif md.get('for'):
  389. constructor, remaining = self._separate_at_paren(expr[m.end() - 1:])
  390. if remaining.startswith('{'):
  391. body, expr = self._separate_at_paren(remaining)
  392. else:
  393. switch_m = re.match(r'switch\s*\(', remaining) # FIXME: ?
  394. if switch_m:
  395. switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:])
  396. body, expr = self._separate_at_paren(remaining, '}')
  397. body = 'switch(%s){%s}' % (switch_val, body)
  398. else:
  399. body, expr = remaining, ''
  400. start, cndn, increment = self._separate(constructor, ';')
  401. self.interpret_expression(start, local_vars, allow_recursion)
  402. while True:
  403. if not _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
  404. break
  405. try:
  406. ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion)
  407. if should_abort:
  408. return ret, True
  409. except JS_Break:
  410. break
  411. except JS_Continue:
  412. pass
  413. self.interpret_expression(increment, local_vars, allow_recursion)
  414. elif md.get('switch'):
  415. switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:])
  416. switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
  417. body, expr = self._separate_at_paren(remaining, '}')
  418. items = body.replace('default:', 'case default:').split('case ')[1:]
  419. for default in (False, True):
  420. matched = False
  421. for item in items:
  422. case, stmt = (i.strip() for i in self._separate(item, ':', 1))
  423. if default:
  424. matched = matched or case == 'default'
  425. elif not matched:
  426. matched = (case != 'default'
  427. and switch_val == self.interpret_expression(case, local_vars, allow_recursion))
  428. if not matched:
  429. continue
  430. try:
  431. ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion)
  432. if should_abort:
  433. return ret
  434. except JS_Break:
  435. break
  436. if matched:
  437. break
  438. if md:
  439. ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
  440. return ret, should_abort or should_return
  441. # Comma separated statements
  442. sub_expressions = list(self._separate(expr))
  443. if len(sub_expressions) > 1:
  444. for sub_expr in sub_expressions:
  445. ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
  446. if should_abort:
  447. return ret, True
  448. return ret, False
  449. for m in re.finditer(rf'''(?x)
  450. (?P<pre_sign>\+\+|--)(?P<var1>{_NAME_RE})|
  451. (?P<var2>{_NAME_RE})(?P<post_sign>\+\+|--)''', expr):
  452. var = m.group('var1') or m.group('var2')
  453. start, end = m.span()
  454. sign = m.group('pre_sign') or m.group('post_sign')
  455. ret = local_vars[var]
  456. local_vars[var] += 1 if sign[0] == '+' else -1
  457. if m.group('pre_sign'):
  458. ret = local_vars[var]
  459. expr = expr[:start] + self._dump(ret, local_vars) + expr[end:]
  460. if not expr:
  461. return None, should_return
  462. m = re.match(fr'''(?x)
  463. (?P<assign>
  464. (?P<out>{_NAME_RE})(?:\[(?P<index>[^\]]+?)\])?\s*
  465. (?P<op>{"|".join(map(re.escape, set(_OPERATORS) - _COMP_OPERATORS))})?
  466. =(?!=)(?P<expr>.*)$
  467. )|(?P<return>
  468. (?!if|return|true|false|null|undefined|NaN)(?P<name>{_NAME_RE})$
  469. )|(?P<indexing>
  470. (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
  471. )|(?P<attribute>
  472. (?P<var>{_NAME_RE})(?:(?P<nullish>\?)?\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
  473. )|(?P<function>
  474. (?P<fname>{_NAME_RE})\((?P<args>.*)\)$
  475. )''', expr)
  476. if m and m.group('assign'):
  477. left_val = local_vars.get(m.group('out'))
  478. if not m.group('index'):
  479. local_vars[m.group('out')] = self._operator(
  480. m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
  481. return local_vars[m.group('out')], should_return
  482. elif left_val in (None, JS_Undefined):
  483. raise self.Exception(f'Cannot index undefined variable {m.group("out")}', expr)
  484. idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
  485. if not isinstance(idx, (int, float)):
  486. raise self.Exception(f'List index {idx} must be integer', expr)
  487. idx = int(idx)
  488. left_val[idx] = self._operator(
  489. m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion)
  490. return left_val[idx], should_return
  491. elif expr.isdigit():
  492. return int(expr), should_return
  493. elif expr == 'break':
  494. raise JS_Break
  495. elif expr == 'continue':
  496. raise JS_Continue
  497. elif expr == 'undefined':
  498. return JS_Undefined, should_return
  499. elif expr == 'NaN':
  500. return float('NaN'), should_return
  501. elif m and m.group('return'):
  502. return local_vars.get(m.group('name'), JS_Undefined), should_return
  503. with contextlib.suppress(ValueError):
  504. return json.loads(js_to_json(expr, strict=True)), should_return
  505. if m and m.group('indexing'):
  506. val = local_vars[m.group('in')]
  507. idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
  508. return self._index(val, idx), should_return
  509. for op in _OPERATORS:
  510. separated = list(self._separate(expr, op))
  511. right_expr = separated.pop()
  512. while True:
  513. if op in '?<>*-' and len(separated) > 1 and not separated[-1].strip():
  514. separated.pop()
  515. elif not (separated and op == '?' and right_expr.startswith('.')):
  516. break
  517. right_expr = f'{op}{right_expr}'
  518. if op != '-':
  519. right_expr = f'{separated.pop()}{op}{right_expr}'
  520. if not separated:
  521. continue
  522. left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
  523. return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), should_return
  524. if m and m.group('attribute'):
  525. variable, member, nullish = m.group('var', 'member', 'nullish')
  526. if not member:
  527. member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
  528. arg_str = expr[m.end():]
  529. if arg_str.startswith('('):
  530. arg_str, remaining = self._separate_at_paren(arg_str)
  531. else:
  532. arg_str, remaining = None, arg_str
  533. def assertion(cndn, msg):
  534. """ assert, but without risk of getting optimized out """
  535. if not cndn:
  536. raise self.Exception(f'{member} {msg}', expr)
  537. def eval_method():
  538. nonlocal member
  539. if (variable, member) == ('console', 'debug'):
  540. if Debugger.ENABLED:
  541. Debugger.write(self.interpret_expression(f'[{arg_str}]', local_vars, allow_recursion))
  542. return
  543. types = {
  544. 'String': str,
  545. 'Math': float,
  546. 'Array': list,
  547. }
  548. obj = local_vars.get(variable, types.get(variable, NO_DEFAULT))
  549. if obj is NO_DEFAULT:
  550. if variable not in self._objects:
  551. try:
  552. self._objects[variable] = self.extract_object(variable)
  553. except self.Exception:
  554. if not nullish:
  555. raise
  556. obj = self._objects.get(variable, JS_Undefined)
  557. if nullish and obj is JS_Undefined:
  558. return JS_Undefined
  559. # Member access
  560. if arg_str is None:
  561. return self._index(obj, member, nullish)
  562. # Function call
  563. argvals = [
  564. self.interpret_expression(v, local_vars, allow_recursion)
  565. for v in self._separate(arg_str)]
  566. # Fixup prototype call
  567. if isinstance(obj, type) and member.startswith('prototype.'):
  568. new_member, _, func_prototype = member.partition('.')[2].partition('.')
  569. assertion(argvals, 'takes one or more arguments')
  570. assertion(isinstance(argvals[0], obj), f'needs binding to type {obj}')
  571. if func_prototype == 'call':
  572. obj, *argvals = argvals
  573. elif func_prototype == 'apply':
  574. assertion(len(argvals) == 2, 'takes two arguments')
  575. obj, argvals = argvals
  576. assertion(isinstance(argvals, list), 'second argument needs to be a list')
  577. else:
  578. raise self.Exception(f'Unsupported Function method {func_prototype}', expr)
  579. member = new_member
  580. if obj is str:
  581. if member == 'fromCharCode':
  582. assertion(argvals, 'takes one or more arguments')
  583. return ''.join(map(chr, argvals))
  584. raise self.Exception(f'Unsupported String method {member}', expr)
  585. elif obj is float:
  586. if member == 'pow':
  587. assertion(len(argvals) == 2, 'takes two arguments')
  588. return argvals[0] ** argvals[1]
  589. raise self.Exception(f'Unsupported Math method {member}', expr)
  590. if member == 'split':
  591. assertion(argvals, 'takes one or more arguments')
  592. assertion(len(argvals) == 1, 'with limit argument is not implemented')
  593. return obj.split(argvals[0]) if argvals[0] else list(obj)
  594. elif member == 'join':
  595. assertion(isinstance(obj, list), 'must be applied on a list')
  596. assertion(len(argvals) == 1, 'takes exactly one argument')
  597. return argvals[0].join(obj)
  598. elif member == 'reverse':
  599. assertion(not argvals, 'does not take any arguments')
  600. obj.reverse()
  601. return obj
  602. elif member == 'slice':
  603. assertion(isinstance(obj, (list, str)), 'must be applied on a list or string')
  604. assertion(len(argvals) <= 2, 'takes between 0 and 2 arguments')
  605. return obj[slice(*argvals, None)]
  606. elif member == 'splice':
  607. assertion(isinstance(obj, list), 'must be applied on a list')
  608. assertion(argvals, 'takes one or more arguments')
  609. index, how_many = map(int, ([*argvals, len(obj)])[:2])
  610. if index < 0:
  611. index += len(obj)
  612. add_items = argvals[2:]
  613. res = []
  614. for _ in range(index, min(index + how_many, len(obj))):
  615. res.append(obj.pop(index))
  616. for i, item in enumerate(add_items):
  617. obj.insert(index + i, item)
  618. return res
  619. elif member == 'unshift':
  620. assertion(isinstance(obj, list), 'must be applied on a list')
  621. assertion(argvals, 'takes one or more arguments')
  622. for item in reversed(argvals):
  623. obj.insert(0, item)
  624. return obj
  625. elif member == 'pop':
  626. assertion(isinstance(obj, list), 'must be applied on a list')
  627. assertion(not argvals, 'does not take any arguments')
  628. if not obj:
  629. return
  630. return obj.pop()
  631. elif member == 'push':
  632. assertion(argvals, 'takes one or more arguments')
  633. obj.extend(argvals)
  634. return obj
  635. elif member == 'forEach':
  636. assertion(argvals, 'takes one or more arguments')
  637. assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
  638. f, this = ([*argvals, ''])[:2]
  639. return [f((item, idx, obj), {'this': this}, allow_recursion) for idx, item in enumerate(obj)]
  640. elif member == 'indexOf':
  641. assertion(argvals, 'takes one or more arguments')
  642. assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
  643. idx, start = ([*argvals, 0])[:2]
  644. try:
  645. return obj.index(idx, start)
  646. except ValueError:
  647. return -1
  648. elif member == 'charCodeAt':
  649. assertion(isinstance(obj, str), 'must be applied on a string')
  650. assertion(len(argvals) == 1, 'takes exactly one argument')
  651. idx = argvals[0] if isinstance(argvals[0], int) else 0
  652. if idx >= len(obj):
  653. return None
  654. return ord(obj[idx])
  655. idx = int(member) if isinstance(obj, list) else member
  656. return obj[idx](argvals, allow_recursion=allow_recursion)
  657. if remaining:
  658. ret, should_abort = self.interpret_statement(
  659. self._named_object(local_vars, eval_method()) + remaining,
  660. local_vars, allow_recursion)
  661. return ret, should_return or should_abort
  662. else:
  663. return eval_method(), should_return
  664. elif m and m.group('function'):
  665. fname = m.group('fname')
  666. argvals = [self.interpret_expression(v, local_vars, allow_recursion)
  667. for v in self._separate(m.group('args'))]
  668. if fname in local_vars:
  669. return local_vars[fname](argvals, allow_recursion=allow_recursion), should_return
  670. elif fname not in self._functions:
  671. self._functions[fname] = self.extract_function(fname)
  672. return self._functions[fname](argvals, allow_recursion=allow_recursion), should_return
  673. raise self.Exception(
  674. f'Unsupported JS expression {truncate_string(expr, 20, 20) if expr != stmt else ""}', stmt)
  675. def interpret_expression(self, expr, local_vars, allow_recursion):
  676. ret, should_return = self.interpret_statement(expr, local_vars, allow_recursion)
  677. if should_return:
  678. raise self.Exception('Cannot return from an expression', expr)
  679. return ret
  680. def extract_object(self, objname):
  681. _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
  682. obj = {}
  683. obj_m = re.search(
  684. r'''(?x)
  685. (?<!\.)%s\s*=\s*{\s*
  686. (?P<fields>(%s\s*:\s*function\s*\(.*?\)\s*{.*?}(?:,\s*)?)*)
  687. }\s*;
  688. ''' % (re.escape(objname), _FUNC_NAME_RE),
  689. self.code)
  690. if not obj_m:
  691. raise self.Exception(f'Could not find object {objname}')
  692. fields = obj_m.group('fields')
  693. # Currently, it only supports function definitions
  694. fields_m = re.finditer(
  695. r'''(?x)
  696. (?P<key>%s)\s*:\s*function\s*\((?P<args>(?:%s|,)*)\){(?P<code>[^}]+)}
  697. ''' % (_FUNC_NAME_RE, _NAME_RE),
  698. fields)
  699. for f in fields_m:
  700. argnames = f.group('args').split(',')
  701. name = remove_quotes(f.group('key'))
  702. obj[name] = function_with_repr(self.build_function(argnames, f.group('code')), f'F<{name}>')
  703. return obj
  704. def extract_function_code(self, funcname):
  705. """ @returns argnames, code """
  706. func_m = re.search(
  707. r'''(?xs)
  708. (?:
  709. function\s+%(name)s|
  710. [{;,]\s*%(name)s\s*=\s*function|
  711. (?:var|const|let)\s+%(name)s\s*=\s*function
  712. )\s*
  713. \((?P<args>[^)]*)\)\s*
  714. (?P<code>{.+})''' % {'name': re.escape(funcname)},
  715. self.code)
  716. if func_m is None:
  717. raise self.Exception(f'Could not find JS function "{funcname}"')
  718. code, _ = self._separate_at_paren(func_m.group('code'))
  719. return [x.strip() for x in func_m.group('args').split(',')], code
  720. def extract_function(self, funcname):
  721. return function_with_repr(
  722. self.extract_function_from_code(*self.extract_function_code(funcname)),
  723. f'F<{funcname}>')
  724. def extract_function_from_code(self, argnames, code, *global_stack):
  725. local_vars = {}
  726. while True:
  727. mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
  728. if mobj is None:
  729. break
  730. start, body_start = mobj.span()
  731. body, remaining = self._separate_at_paren(code[body_start - 1:])
  732. name = self._named_object(local_vars, self.extract_function_from_code(
  733. [x.strip() for x in mobj.group('args').split(',')],
  734. body, local_vars, *global_stack))
  735. code = code[:start] + name + remaining
  736. return self.build_function(argnames, code, local_vars, *global_stack)
  737. def call_function(self, funcname, *args):
  738. return self.extract_function(funcname)(args)
  739. def build_function(self, argnames, code, *global_stack):
  740. global_stack = list(global_stack) or [{}]
  741. argnames = tuple(argnames)
  742. def resf(args, kwargs={}, allow_recursion=100):
  743. global_stack[0].update(itertools.zip_longest(argnames, args, fillvalue=None))
  744. global_stack[0].update(kwargs)
  745. var_stack = LocalNameSpace(*global_stack)
  746. ret, should_abort = self.interpret_statement(code.replace('\n', ' '), var_stack, allow_recursion - 1)
  747. if should_abort:
  748. return ret
  749. return resf