TokensTest.php 56 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005
  1. <?php
  2. declare(strict_types=1);
  3. /*
  4. * This file is part of PHP CS Fixer.
  5. *
  6. * (c) Fabien Potencier <fabien@symfony.com>
  7. * Dariusz Rumiński <dariusz.ruminski@gmail.com>
  8. *
  9. * This source file is subject to the MIT license that is bundled
  10. * with this source code in the file LICENSE.
  11. */
  12. namespace PhpCsFixer\Tests\Tokenizer;
  13. use PhpCsFixer\Tests\Test\Assert\AssertTokensTrait;
  14. use PhpCsFixer\Tests\TestCase;
  15. use PhpCsFixer\Tokenizer\Analyzer\Analysis\NamespaceAnalysis;
  16. use PhpCsFixer\Tokenizer\CT;
  17. use PhpCsFixer\Tokenizer\Token;
  18. use PhpCsFixer\Tokenizer\Tokens;
  19. /**
  20. * @author Dariusz Rumiński <dariusz.ruminski@gmail.com>
  21. *
  22. * @internal
  23. *
  24. * @covers \PhpCsFixer\Tokenizer\Tokens
  25. */
  26. final class TokensTest extends TestCase
  27. {
  28. use AssertTokensTrait;
  29. public function testReadFromCacheAfterClearing(): void
  30. {
  31. $code = '<?php echo 1;';
  32. $tokens = Tokens::fromCode($code);
  33. $countBefore = $tokens->count();
  34. for ($i = 0; $i < $countBefore; ++$i) {
  35. $tokens->clearAt($i);
  36. }
  37. $tokens = Tokens::fromCode($code);
  38. self::assertCount($countBefore, $tokens);
  39. }
  40. /**
  41. * @param null|array<int, Token> $expected
  42. * @param list<array{0: int, 1?: string}|string|Token> $sequence
  43. * @param bool|list<bool> $caseSensitive
  44. *
  45. * @dataProvider provideFindSequenceCases
  46. */
  47. public function testFindSequence(
  48. string $source,
  49. ?array $expected,
  50. array $sequence,
  51. int $start = 0,
  52. ?int $end = null,
  53. $caseSensitive = true
  54. ): void {
  55. $tokens = Tokens::fromCode($source);
  56. self::assertEqualsTokensArray(
  57. $expected,
  58. $tokens->findSequence(
  59. $sequence,
  60. $start,
  61. $end,
  62. $caseSensitive
  63. )
  64. );
  65. }
  66. public static function provideFindSequenceCases(): iterable
  67. {
  68. yield [
  69. '<?php $x = 1;',
  70. null,
  71. [
  72. new Token(';'),
  73. ],
  74. 7,
  75. ];
  76. yield [
  77. '<?php $x = 2;',
  78. null,
  79. [
  80. [T_OPEN_TAG],
  81. [T_VARIABLE, '$y'],
  82. ],
  83. ];
  84. yield [
  85. '<?php $x = 3;',
  86. [
  87. 0 => new Token([T_OPEN_TAG, '<?php ']),
  88. 1 => new Token([T_VARIABLE, '$x']),
  89. ],
  90. [
  91. [T_OPEN_TAG],
  92. [T_VARIABLE, '$x'],
  93. ],
  94. ];
  95. yield [
  96. '<?php $x = 4;',
  97. [
  98. 3 => new Token('='),
  99. 5 => new Token([T_LNUMBER, '4']),
  100. 6 => new Token(';'),
  101. ],
  102. [
  103. '=',
  104. [T_LNUMBER, '4'],
  105. ';',
  106. ],
  107. ];
  108. yield [
  109. '<?php $x = 5;',
  110. [
  111. 0 => new Token([T_OPEN_TAG, '<?php ']),
  112. 1 => new Token([T_VARIABLE, '$x']),
  113. ],
  114. [
  115. [T_OPEN_TAG],
  116. [T_VARIABLE, '$x'],
  117. ],
  118. 0,
  119. ];
  120. yield [
  121. '<?php $x = 6;',
  122. null,
  123. [
  124. [T_OPEN_TAG],
  125. [T_VARIABLE, '$x'],
  126. ],
  127. 1,
  128. ];
  129. yield [
  130. '<?php $x = 7;',
  131. [
  132. 3 => new Token('='),
  133. 5 => new Token([T_LNUMBER, '7']),
  134. 6 => new Token(';'),
  135. ],
  136. [
  137. '=',
  138. [T_LNUMBER, '7'],
  139. ';',
  140. ],
  141. 3,
  142. 6,
  143. ];
  144. yield [
  145. '<?php $x = 8;',
  146. null,
  147. [
  148. '=',
  149. [T_LNUMBER, '8'],
  150. ';',
  151. ],
  152. 4,
  153. 6,
  154. ];
  155. yield [
  156. '<?php $x = 9;',
  157. null,
  158. [
  159. '=',
  160. [T_LNUMBER, '9'],
  161. ';',
  162. ],
  163. 3,
  164. 5,
  165. ];
  166. yield [
  167. '<?php $x = 10;',
  168. [
  169. 0 => new Token([T_OPEN_TAG, '<?php ']),
  170. 1 => new Token([T_VARIABLE, '$x']),
  171. ],
  172. [
  173. [T_OPEN_TAG],
  174. [T_VARIABLE, '$x'],
  175. ],
  176. 0,
  177. 1,
  178. true,
  179. ];
  180. yield [
  181. '<?php $x = 11;',
  182. null,
  183. [
  184. [T_OPEN_TAG],
  185. [T_VARIABLE, '$X'],
  186. ],
  187. 0,
  188. 1,
  189. true,
  190. ];
  191. yield [
  192. '<?php $x = 12;',
  193. null,
  194. [
  195. [T_OPEN_TAG],
  196. [T_VARIABLE, '$X'],
  197. ],
  198. 0,
  199. 1,
  200. [1 => true],
  201. ];
  202. yield [
  203. '<?php $x = 13;',
  204. [
  205. 0 => new Token([T_OPEN_TAG, '<?php ']),
  206. 1 => new Token([T_VARIABLE, '$x']),
  207. ],
  208. [
  209. [T_OPEN_TAG],
  210. [T_VARIABLE, '$X'],
  211. ],
  212. 0,
  213. 1,
  214. false,
  215. ];
  216. yield [
  217. '<?php $x = 14;',
  218. [
  219. 0 => new Token([T_OPEN_TAG, '<?php ']),
  220. 1 => new Token([T_VARIABLE, '$x']),
  221. ],
  222. [
  223. [T_OPEN_TAG],
  224. [T_VARIABLE, '$X'],
  225. ],
  226. 0,
  227. 1,
  228. [1 => false],
  229. ];
  230. yield [
  231. '<?php $x = 15;',
  232. [
  233. 0 => new Token([T_OPEN_TAG, '<?php ']),
  234. 1 => new Token([T_VARIABLE, '$x']),
  235. ],
  236. [
  237. [T_OPEN_TAG],
  238. [T_VARIABLE, '$X'],
  239. ],
  240. 0,
  241. 1,
  242. [1 => false],
  243. ];
  244. yield [
  245. '<?php $x = 16;',
  246. null,
  247. [
  248. [T_OPEN_TAG],
  249. [T_VARIABLE, '$X'],
  250. ],
  251. 0,
  252. 1,
  253. [2 => false],
  254. ];
  255. yield [
  256. '<?php $x = 17;',
  257. null,
  258. [
  259. [T_VARIABLE, '$X'],
  260. '=',
  261. ],
  262. 0,
  263. 10,
  264. ];
  265. }
  266. /**
  267. * @param list<mixed> $sequence sequence of token prototypes
  268. *
  269. * @dataProvider provideFindSequenceExceptionCases
  270. */
  271. public function testFindSequenceException(string $message, array $sequence): void
  272. {
  273. $tokens = Tokens::fromCode('<?php $x = 1;');
  274. $this->expectException(\InvalidArgumentException::class);
  275. $this->expectExceptionMessage($message);
  276. $tokens->findSequence($sequence);
  277. }
  278. public static function provideFindSequenceExceptionCases(): iterable
  279. {
  280. $emptyToken = new Token('');
  281. yield ['Invalid sequence.', []];
  282. yield [
  283. 'Non-meaningful token at position: "0".',
  284. [[T_WHITESPACE, ' ']],
  285. ];
  286. yield [
  287. 'Non-meaningful token at position: "1".',
  288. ['{', [T_COMMENT, '// Foo'], '}'],
  289. ];
  290. yield [
  291. 'Non-meaningful (empty) token at position: "2".',
  292. ['{', '!', $emptyToken, '}'],
  293. ];
  294. }
  295. public function testClearRange(): void
  296. {
  297. $source = <<<'PHP'
  298. <?php
  299. class FooBar
  300. {
  301. public function foo()
  302. {
  303. return 'bar';
  304. }
  305. public function bar()
  306. {
  307. return 'foo';
  308. }
  309. }
  310. PHP;
  311. $tokens = Tokens::fromCode($source);
  312. [$fooIndex, $barIndex] = array_keys($tokens->findGivenKind(T_PUBLIC));
  313. $tokens->clearRange($fooIndex, $barIndex - 1);
  314. $newPublicIndexes = array_keys($tokens->findGivenKind(T_PUBLIC));
  315. self::assertSame($barIndex, reset($newPublicIndexes));
  316. for ($i = $fooIndex; $i < $barIndex; ++$i) {
  317. self::assertTrue($tokens[$i]->isWhitespace());
  318. }
  319. }
  320. /**
  321. * @dataProvider provideMonolithicPhpDetectionCases
  322. */
  323. public function testMonolithicPhpDetection(bool $isMonolithic, string $source): void
  324. {
  325. $tokens = Tokens::fromCode($source);
  326. self::assertSame($isMonolithic, $tokens->isMonolithicPhp());
  327. }
  328. /**
  329. * @return iterable<array{bool, string}>
  330. */
  331. public static function provideMonolithicPhpDetectionCases(): iterable
  332. {
  333. yield [true, "<?php\n"];
  334. yield [true, "<?php\n?>"];
  335. yield [false, "#!\n<?php\n"];
  336. yield [false, "#!/usr/bin/bash\ncat <?php\n"];
  337. yield [false, "#!/usr/bin/env bash\ncat <?php\n"];
  338. yield [true, "#!/usr/bin/php\n<?php\n"];
  339. yield [true, "#!/usr/bin/php7.4-cli\n<?php\n"];
  340. yield [false, "#!/usr/bin/php\n\n<?php\n"]; // empty line after shebang would be printed to console before PHP executes
  341. yield [true, "#!/usr/bin/php8\n<?php\n"];
  342. yield [true, "#!/usr/bin/env php\n<?php\n"];
  343. yield [true, "#!/usr/bin/env php7.4\n<?php\n"];
  344. yield [true, "#!/usr/bin/env php7.4-cli\n<?php\n"];
  345. yield [false, "#!/usr/bin/env this-is\ntoo-much\n<?php\n"];
  346. yield [false, "#!/usr/bin/php\nFoo bar<?php\n"];
  347. yield [false, "#!/usr/bin/env php -n \nFoo bar\n<?php\n"];
  348. yield [false, ''];
  349. yield [false, ' '];
  350. yield [false, " <?php\n"];
  351. yield [false, "<?php\n?> "];
  352. yield [false, "<?php\n?><?php\n"];
  353. yield [false, 'Hello<?php echo "World!"; ?>'];
  354. yield [false, '<?php echo "Hello"; ?> World!'];
  355. // short open tag
  356. yield [(bool) \ini_get('short_open_tag'), "<?\n"];
  357. yield [(bool) \ini_get('short_open_tag'), "<?\n?>"];
  358. yield [false, " <?\n"];
  359. yield [false, "<?\n?> "];
  360. yield [false, "<?\n?><?\n"];
  361. yield [false, "<?\n?><?php\n"];
  362. yield [false, "<?\n?><?=' ';\n"];
  363. yield [false, "<?php\n?><?\n"];
  364. yield [false, "<?=' '\n?><?\n"];
  365. // short open tag echo
  366. yield [true, "<?=' ';\n"];
  367. yield [true, "<?=' '?>"];
  368. yield [false, " <?=' ';\n"];
  369. yield [false, "<?=' '?> "];
  370. yield [false, "<?php\n?><?=' ';\n"];
  371. yield [false, "<?=' '\n?><?php\n"];
  372. yield [false, "<?=' '\n?><?=' ';\n"];
  373. }
  374. public function testTokenKindsFound(): void
  375. {
  376. $code = <<<'EOF'
  377. <?php
  378. class Foo
  379. {
  380. public $foo;
  381. }
  382. if (!function_exists('bar')) {
  383. function bar()
  384. {
  385. return 'bar';
  386. }
  387. }
  388. EOF;
  389. $tokens = Tokens::fromCode($code);
  390. self::assertTrue($tokens->isTokenKindFound(T_CLASS));
  391. self::assertTrue($tokens->isTokenKindFound(T_RETURN));
  392. self::assertFalse($tokens->isTokenKindFound(T_INTERFACE));
  393. self::assertFalse($tokens->isTokenKindFound(T_ARRAY));
  394. self::assertTrue($tokens->isAllTokenKindsFound([T_CLASS, T_RETURN]));
  395. self::assertFalse($tokens->isAllTokenKindsFound([T_CLASS, T_INTERFACE]));
  396. self::assertTrue($tokens->isAnyTokenKindsFound([T_CLASS, T_RETURN]));
  397. self::assertTrue($tokens->isAnyTokenKindsFound([T_CLASS, T_INTERFACE]));
  398. self::assertFalse($tokens->isAnyTokenKindsFound([T_INTERFACE, T_ARRAY]));
  399. }
  400. public function testFindGivenKind(): void
  401. {
  402. $source = <<<'PHP'
  403. <?php
  404. class FooBar
  405. {
  406. public function foo()
  407. {
  408. return 'bar';
  409. }
  410. public function bar()
  411. {
  412. return 'foo';
  413. }
  414. }
  415. PHP;
  416. $tokens = Tokens::fromCode($source);
  417. /** @var array<int, Token> $found */
  418. $found = $tokens->findGivenKind(T_CLASS);
  419. self::assertCount(1, $found);
  420. self::assertArrayHasKey(1, $found);
  421. self::assertSame(T_CLASS, $found[1]->getId());
  422. $found = $tokens->findGivenKind([T_CLASS, T_FUNCTION]);
  423. self::assertCount(2, $found);
  424. self::assertArrayHasKey(T_CLASS, $found);
  425. self::assertIsArray($found[T_CLASS]);
  426. self::assertCount(1, $found[T_CLASS]);
  427. self::assertArrayHasKey(1, $found[T_CLASS]);
  428. self::assertSame(T_CLASS, $found[T_CLASS][1]->getId());
  429. self::assertArrayHasKey(T_FUNCTION, $found);
  430. self::assertIsArray($found[T_FUNCTION]);
  431. self::assertCount(2, $found[T_FUNCTION]);
  432. self::assertArrayHasKey(9, $found[T_FUNCTION]);
  433. self::assertSame(T_FUNCTION, $found[T_FUNCTION][9]->getId());
  434. self::assertArrayHasKey(26, $found[T_FUNCTION]);
  435. self::assertSame(T_FUNCTION, $found[T_FUNCTION][26]->getId());
  436. // test offset and limits of the search
  437. $found = $tokens->findGivenKind([T_CLASS, T_FUNCTION], 10);
  438. self::assertArrayHasKey(T_CLASS, $found);
  439. self::assertCount(0, $found[T_CLASS]);
  440. self::assertArrayHasKey(T_FUNCTION, $found);
  441. self::assertCount(1, $found[T_FUNCTION]);
  442. self::assertArrayHasKey(26, $found[T_FUNCTION]);
  443. $found = $tokens->findGivenKind([T_CLASS, T_FUNCTION], 2, 10);
  444. self::assertArrayHasKey(T_CLASS, $found);
  445. self::assertCount(0, $found[T_CLASS]);
  446. self::assertArrayHasKey(T_FUNCTION, $found);
  447. self::assertCount(1, $found[T_FUNCTION]);
  448. self::assertArrayHasKey(9, $found[T_FUNCTION]);
  449. }
  450. /**
  451. * @param list<int> $indexes to clear
  452. * @param list<Token> $expected tokens
  453. *
  454. * @dataProvider provideClearTokenAndMergeSurroundingWhitespaceCases
  455. */
  456. public function testClearTokenAndMergeSurroundingWhitespace(string $source, array $indexes, array $expected): void
  457. {
  458. $this->doTestClearTokens($source, $indexes, $expected);
  459. if (\count($indexes) > 1) {
  460. $this->doTestClearTokens($source, array_reverse($indexes), $expected);
  461. }
  462. }
  463. public static function provideClearTokenAndMergeSurroundingWhitespaceCases(): iterable
  464. {
  465. $clearToken = new Token('');
  466. yield [
  467. '<?php if($a){}else{}',
  468. [7, 8, 9],
  469. [
  470. new Token([T_OPEN_TAG, '<?php ']),
  471. new Token([T_IF, 'if']),
  472. new Token('('),
  473. new Token([T_VARIABLE, '$a']),
  474. new Token(')'),
  475. new Token('{'),
  476. new Token('}'),
  477. $clearToken,
  478. $clearToken,
  479. $clearToken,
  480. ],
  481. ];
  482. yield [
  483. '<?php $a;/**/;',
  484. [2],
  485. [
  486. // <?php $a /**/;
  487. new Token([T_OPEN_TAG, '<?php ']),
  488. new Token([T_VARIABLE, '$a']),
  489. $clearToken,
  490. new Token([T_COMMENT, '/**/']),
  491. new Token(';'),
  492. ],
  493. ];
  494. yield [
  495. '<?php ; ; ;',
  496. [3],
  497. [
  498. // <?php ; ;
  499. new Token([T_OPEN_TAG, '<?php ']),
  500. new Token(';'),
  501. new Token([T_WHITESPACE, ' ']),
  502. $clearToken,
  503. $clearToken,
  504. new Token(';'),
  505. ],
  506. ];
  507. yield [
  508. '<?php ; ; ;',
  509. [1, 5],
  510. [
  511. // <?php ;
  512. new Token([T_OPEN_TAG, '<?php ']),
  513. new Token([T_WHITESPACE, ' ']),
  514. $clearToken,
  515. new Token(';'),
  516. new Token([T_WHITESPACE, ' ']),
  517. $clearToken,
  518. ],
  519. ];
  520. yield [
  521. '<?php ; ; ;',
  522. [1, 3],
  523. [
  524. // <?php ;
  525. new Token([T_OPEN_TAG, '<?php ']),
  526. new Token([T_WHITESPACE, ' ']),
  527. $clearToken,
  528. $clearToken,
  529. $clearToken,
  530. new Token(';'),
  531. ],
  532. ];
  533. yield [
  534. '<?php ; ; ;',
  535. [1],
  536. [
  537. // <?php ; ;
  538. new Token([T_OPEN_TAG, '<?php ']),
  539. new Token([T_WHITESPACE, ' ']),
  540. $clearToken,
  541. new Token(';'),
  542. new Token([T_WHITESPACE, ' ']),
  543. new Token(';'),
  544. ],
  545. ];
  546. }
  547. /**
  548. * @param -1|1 $direction
  549. * @param list<array{int}|string|Token> $findTokens
  550. *
  551. * @dataProvider provideTokenOfKindSiblingCases
  552. */
  553. public function testTokenOfKindSibling(
  554. ?int $expectedIndex,
  555. int $direction,
  556. int $index,
  557. array $findTokens,
  558. bool $caseSensitive = true
  559. ): void {
  560. $source =
  561. '<?php
  562. $a = function ($b) {
  563. return $b;
  564. };
  565. echo $a(1);
  566. // test
  567. return 123;';
  568. Tokens::clearCache();
  569. $tokens = Tokens::fromCode($source);
  570. if (1 === $direction) {
  571. self::assertSame($expectedIndex, $tokens->getNextTokenOfKind($index, $findTokens, $caseSensitive));
  572. } else {
  573. self::assertSame($expectedIndex, $tokens->getPrevTokenOfKind($index, $findTokens, $caseSensitive));
  574. }
  575. self::assertSame($expectedIndex, $tokens->getTokenOfKindSibling($index, $direction, $findTokens, $caseSensitive));
  576. }
  577. public static function provideTokenOfKindSiblingCases(): iterable
  578. {
  579. // find next cases
  580. yield [
  581. 35, 1, 34, [';'],
  582. ];
  583. yield [
  584. 14, 1, 0, [[T_RETURN]],
  585. ];
  586. yield [
  587. 32, 1, 14, [[T_RETURN]],
  588. ];
  589. yield [
  590. 6, 1, 0, [[T_RETURN], [T_FUNCTION]],
  591. ];
  592. // find previous cases
  593. yield [
  594. 14, -1, 32, [[T_RETURN], [T_FUNCTION]],
  595. ];
  596. yield [
  597. 6, -1, 7, [[T_FUNCTION]],
  598. ];
  599. yield [
  600. null, -1, 6, [[T_FUNCTION]],
  601. ];
  602. }
  603. /**
  604. * @dataProvider provideFindBlockEndCases
  605. *
  606. * @param Tokens::BLOCK_TYPE_* $type
  607. */
  608. public function testFindBlockEnd(int $expectedIndex, string $source, int $type, int $searchIndex): void
  609. {
  610. self::assertFindBlockEnd($expectedIndex, $source, $type, $searchIndex);
  611. }
  612. /**
  613. * @return iterable<array{int, string, int, int}>
  614. */
  615. public static function provideFindBlockEndCases(): iterable
  616. {
  617. yield [4, '<?php ${$bar};', Tokens::BLOCK_TYPE_DYNAMIC_VAR_BRACE, 2];
  618. yield [4, '<?php test(1);', Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 2];
  619. yield [4, '<?php $a[1];', Tokens::BLOCK_TYPE_INDEX_SQUARE_BRACE, 2];
  620. yield [6, '<?php [1, "foo"];', Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 1];
  621. yield [5, '<?php $foo->{$bar};', Tokens::BLOCK_TYPE_DYNAMIC_PROP_BRACE, 3];
  622. yield [4, '<?php list($a) = $b;', Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 2];
  623. yield [6, '<?php if($a){}?>', Tokens::BLOCK_TYPE_CURLY_BRACE, 5];
  624. yield [11, '<?php $foo = (new Foo());', Tokens::BLOCK_TYPE_BRACE_CLASS_INSTANTIATION, 5];
  625. yield [10, '<?php $object->{"set_{$name}"}(42);', Tokens::BLOCK_TYPE_DYNAMIC_PROP_BRACE, 3];
  626. yield [19, '<?php $foo = (new class () implements Foo {});', Tokens::BLOCK_TYPE_BRACE_CLASS_INSTANTIATION, 5];
  627. yield [10, '<?php use a\{ClassA, ClassB};', Tokens::BLOCK_TYPE_GROUP_IMPORT_BRACE, 5];
  628. yield [3, '<?php [$a] = $array;', Tokens::BLOCK_TYPE_DESTRUCTURING_SQUARE_BRACE, 1];
  629. yield [8, '<?php "start__${array[key]}__end";', Tokens::BLOCK_TYPE_COMPLEX_STRING_VARIABLE, 3];
  630. }
  631. /**
  632. * @requires PHP 8.0
  633. *
  634. * @dataProvider provideFindBlockEnd80Cases
  635. *
  636. * @param Tokens::BLOCK_TYPE_* $type
  637. */
  638. public function testFindBlockEnd80(int $expectedIndex, string $source, int $type, int $searchIndex): void
  639. {
  640. self::assertFindBlockEnd($expectedIndex, $source, $type, $searchIndex);
  641. }
  642. /**
  643. * @return iterable<array{int, string, int, int}>
  644. */
  645. public static function provideFindBlockEnd80Cases(): iterable
  646. {
  647. yield [
  648. 9,
  649. '<?php class Foo {
  650. #[Required]
  651. public $bar;
  652. }',
  653. Tokens::BLOCK_TYPE_ATTRIBUTE,
  654. 7,
  655. ];
  656. }
  657. /**
  658. * @requires PHP 8.2
  659. *
  660. * @dataProvider provideFindBlockEnd82Cases
  661. *
  662. * @param Tokens::BLOCK_TYPE_* $type
  663. */
  664. public function testFindBlockEnd82(int $expectedIndex, string $source, int $type, int $searchIndex): void
  665. {
  666. self::assertFindBlockEnd($expectedIndex, $source, $type, $searchIndex);
  667. }
  668. /**
  669. * @return iterable<array{int, string, int, int}>
  670. */
  671. public static function provideFindBlockEnd82Cases(): iterable
  672. {
  673. yield [
  674. 11,
  675. '<?php function foo(A|(B&C) $x) {}',
  676. Tokens::BLOCK_TYPE_DISJUNCTIVE_NORMAL_FORM_TYPE_PARENTHESIS,
  677. 7,
  678. ];
  679. yield [
  680. 11,
  681. '<?php function foo((A&B&C)|D $x) {}',
  682. Tokens::BLOCK_TYPE_DISJUNCTIVE_NORMAL_FORM_TYPE_PARENTHESIS,
  683. 5,
  684. ];
  685. foreach ([7 => 11, 19 => 23, 27 => 35] as $openIndex => $closeIndex) {
  686. yield [
  687. $closeIndex,
  688. '<?php function foo(A|(B&C)|D $x): (A&B)|bool|(C&D&E&F) {}',
  689. Tokens::BLOCK_TYPE_DISJUNCTIVE_NORMAL_FORM_TYPE_PARENTHESIS,
  690. $openIndex,
  691. ];
  692. }
  693. }
  694. /**
  695. * @requires PHP 8.3
  696. *
  697. * @dataProvider provideFindBlockEnd83Cases
  698. *
  699. * @param Tokens::BLOCK_TYPE_* $type
  700. */
  701. public function testFindBlockEnd83(int $expectedIndex, string $source, int $type, int $searchIndex): void
  702. {
  703. self::assertFindBlockEnd($expectedIndex, $source, $type, $searchIndex);
  704. }
  705. /**
  706. * @return iterable<string, array{int, string, int, int}>
  707. */
  708. public static function provideFindBlockEnd83Cases(): iterable
  709. {
  710. yield 'simple dynamic class constant fetch' => [
  711. 7,
  712. '<?php echo Foo::{$bar};',
  713. Tokens::BLOCK_TYPE_DYNAMIC_CLASS_CONSTANT_FETCH_CURLY_BRACE,
  714. 5,
  715. ];
  716. foreach ([[5, 7], [9, 11]] as $startEnd) {
  717. yield 'chained dynamic class constant fetch: '.$startEnd[0] => [
  718. $startEnd[1],
  719. "<?php echo Foo::{'BAR'}::{'BLA'}::{static_method}(1,2) ?>",
  720. Tokens::BLOCK_TYPE_DYNAMIC_CLASS_CONSTANT_FETCH_CURLY_BRACE,
  721. $startEnd[0],
  722. ];
  723. }
  724. }
  725. /**
  726. * @param Tokens::BLOCK_TYPE_* $type
  727. *
  728. * @dataProvider provideFindBlockEndPre84Cases
  729. *
  730. * @requires PHP <8.4
  731. */
  732. public function testFindBlockEndPre84(int $expectedIndex, string $source, int $type, int $searchIndex): void
  733. {
  734. self::assertFindBlockEnd($expectedIndex, $source, $type, $searchIndex);
  735. }
  736. /**
  737. * @return iterable<array{int, string, int, int}>
  738. */
  739. public static function provideFindBlockEndPre84Cases(): iterable
  740. {
  741. yield [4, '<?php $a{1};', Tokens::BLOCK_TYPE_ARRAY_INDEX_CURLY_BRACE, 2];
  742. }
  743. public function testFindBlockEndInvalidType(): void
  744. {
  745. Tokens::clearCache();
  746. $tokens = Tokens::fromCode('<?php ');
  747. $this->expectException(\InvalidArgumentException::class);
  748. $this->expectExceptionMessageMatches('/^Invalid param type: "-1"\.$/');
  749. // @phpstan-ignore-next-line
  750. $tokens->findBlockEnd(-1, 0);
  751. }
  752. public function testFindBlockEndInvalidStart(): void
  753. {
  754. Tokens::clearCache();
  755. $tokens = Tokens::fromCode('<?php ');
  756. $this->expectException(\InvalidArgumentException::class);
  757. $this->expectExceptionMessageMatches('/^Invalid param \$startIndex - not a proper block "start"\.$/');
  758. $tokens->findBlockEnd(Tokens::BLOCK_TYPE_DYNAMIC_VAR_BRACE, 0);
  759. }
  760. public function testFindBlockEndCalledMultipleTimes(): void
  761. {
  762. Tokens::clearCache();
  763. $tokens = Tokens::fromCode('<?php foo(1, 2);');
  764. self::assertSame(7, $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 2));
  765. $this->expectException(\InvalidArgumentException::class);
  766. $this->expectExceptionMessageMatches('/^Invalid param \$startIndex - not a proper block "start"\.$/');
  767. $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 7);
  768. }
  769. public function testFindBlockStartEdgeCalledMultipleTimes(): void
  770. {
  771. Tokens::clearCache();
  772. $tokens = Tokens::fromCode('<?php foo(1, 2);');
  773. self::assertSame(2, $tokens->findBlockStart(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 7));
  774. $this->expectException(\InvalidArgumentException::class);
  775. $this->expectExceptionMessageMatches('/^Invalid param \$startIndex - not a proper block "end"\.$/');
  776. $tokens->findBlockStart(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, 2);
  777. }
  778. public function testEmptyTokens(): void
  779. {
  780. $code = '';
  781. $tokens = Tokens::fromCode($code);
  782. self::assertCount(0, $tokens);
  783. self::assertFalse($tokens->isTokenKindFound(T_OPEN_TAG));
  784. }
  785. public function testEmptyTokensMultiple(): void
  786. {
  787. $code = '';
  788. $tokens = Tokens::fromCode($code);
  789. self::assertFalse($tokens->isChanged());
  790. $tokens->insertAt(0, new Token([T_WHITESPACE, ' ']));
  791. self::assertCount(1, $tokens);
  792. self::assertFalse($tokens->isTokenKindFound(T_OPEN_TAG));
  793. self::assertTrue($tokens->isChanged());
  794. $tokens2 = Tokens::fromCode($code);
  795. self::assertCount(0, $tokens2);
  796. self::assertFalse($tokens->isTokenKindFound(T_OPEN_TAG));
  797. }
  798. public function testFromArray(): void
  799. {
  800. $code = '<?php echo 1;';
  801. $tokens1 = Tokens::fromCode($code);
  802. $tokens2 = Tokens::fromArray($tokens1->toArray());
  803. self::assertTrue($tokens1->isTokenKindFound(T_OPEN_TAG));
  804. self::assertTrue($tokens2->isTokenKindFound(T_OPEN_TAG));
  805. self::assertSame($tokens1->getCodeHash(), $tokens2->getCodeHash());
  806. }
  807. public function testFromArrayEmpty(): void
  808. {
  809. $tokens = Tokens::fromArray([]);
  810. self::assertFalse($tokens->isTokenKindFound(T_OPEN_TAG));
  811. }
  812. /**
  813. * @dataProvider provideIsEmptyCases
  814. */
  815. public function testIsEmpty(Token $token, bool $isEmpty): void
  816. {
  817. $tokens = Tokens::fromArray([$token]);
  818. Tokens::clearCache();
  819. self::assertSame($isEmpty, $tokens->isEmptyAt(0), $token->toJson());
  820. }
  821. /**
  822. * @return iterable<array{Token, bool}>
  823. */
  824. public static function provideIsEmptyCases(): iterable
  825. {
  826. yield [new Token(''), true];
  827. yield [new Token('('), false];
  828. yield [new Token([T_WHITESPACE, ' ']), false];
  829. }
  830. public function testClone(): void
  831. {
  832. $code = '<?php echo 1;';
  833. $tokens = Tokens::fromCode($code);
  834. $tokensClone = clone $tokens;
  835. self::assertTrue($tokens->isTokenKindFound(T_OPEN_TAG));
  836. self::assertTrue($tokensClone->isTokenKindFound(T_OPEN_TAG));
  837. $count = \count($tokens);
  838. self::assertCount($count, $tokensClone);
  839. for ($i = 0; $i < $count; ++$i) {
  840. self::assertTrue($tokens[$i]->equals($tokensClone[$i]));
  841. self::assertNotSame($tokens[$i], $tokensClone[$i]);
  842. }
  843. }
  844. /**
  845. * @dataProvider provideEnsureWhitespaceAtIndexCases
  846. */
  847. public function testEnsureWhitespaceAtIndex(string $expected, string $input, int $index, int $offset, string $whiteSpace): void
  848. {
  849. $tokens = Tokens::fromCode($input);
  850. $tokens->ensureWhitespaceAtIndex($index, $offset, $whiteSpace);
  851. $tokens->clearEmptyTokens();
  852. self::assertTokens(Tokens::fromCode($expected), $tokens);
  853. }
  854. /**
  855. * @return iterable<array{string, string, int, int, string}>
  856. */
  857. public static function provideEnsureWhitespaceAtIndexCases(): iterable
  858. {
  859. yield [
  860. '<?php echo 1;',
  861. '<?php echo 1;',
  862. 1,
  863. 1,
  864. ' ',
  865. ];
  866. yield [
  867. '<?php echo 7;',
  868. '<?php echo 7;',
  869. 1,
  870. 1,
  871. ' ',
  872. ];
  873. yield [
  874. '<?php ',
  875. '<?php ',
  876. 1,
  877. 1,
  878. ' ',
  879. ];
  880. yield [
  881. '<?php $a. $b;',
  882. '<?php $a.$b;',
  883. 2,
  884. 1,
  885. ' ',
  886. ];
  887. yield [
  888. '<?php $a .$b;',
  889. '<?php $a.$b;',
  890. 2,
  891. 0,
  892. ' ',
  893. ];
  894. yield [
  895. "<?php\r\n",
  896. '<?php ',
  897. 0,
  898. 1,
  899. "\r\n",
  900. ];
  901. yield [
  902. '<?php $a.$b;',
  903. '<?php $a.$b;',
  904. 2,
  905. -1,
  906. ' ',
  907. ];
  908. yield [
  909. "<?php\t ",
  910. "<?php\n",
  911. 0,
  912. 1,
  913. "\t ",
  914. ];
  915. yield [
  916. '<?php ',
  917. '<?php ',
  918. 0,
  919. 1,
  920. ' ',
  921. ];
  922. yield [
  923. "<?php\n",
  924. '<?php ',
  925. 0,
  926. 1,
  927. "\n",
  928. ];
  929. yield [
  930. "<?php\t",
  931. '<?php ',
  932. 0,
  933. 1,
  934. "\t",
  935. ];
  936. yield [
  937. '<?php
  938. //
  939. echo $a;',
  940. '<?php
  941. //
  942. echo $a;',
  943. 2,
  944. 1,
  945. "\n ",
  946. ];
  947. yield [
  948. '<?php
  949. echo $a;',
  950. '<?php
  951. echo $a;',
  952. 0,
  953. 1,
  954. "\n ",
  955. ];
  956. yield [
  957. '<?php
  958. echo $a;',
  959. '<?php echo $a;',
  960. 0,
  961. 1,
  962. "\n",
  963. ];
  964. yield [
  965. "<?php\techo \$a;",
  966. '<?php echo $a;',
  967. 0,
  968. 1,
  969. "\t",
  970. ];
  971. }
  972. public function testAssertTokensAfterChanging(): void
  973. {
  974. $template =
  975. '<?php class SomeClass {
  976. %s//
  977. public function __construct($name)
  978. {
  979. $this->name = $name;
  980. }
  981. }';
  982. $tokens = Tokens::fromCode(\sprintf($template, ''));
  983. $commentIndex = $tokens->getNextTokenOfKind(0, [[T_COMMENT]]);
  984. $tokens->insertAt(
  985. $commentIndex,
  986. [
  987. new Token([T_PRIVATE, 'private']),
  988. new Token([T_WHITESPACE, ' ']),
  989. new Token([T_VARIABLE, '$name']),
  990. new Token(';'),
  991. ]
  992. );
  993. self::assertTrue($tokens->isChanged());
  994. $expected = Tokens::fromCode(\sprintf($template, 'private $name;'));
  995. self::assertFalse($expected->isChanged());
  996. self::assertTokens($expected, $tokens);
  997. }
  998. /**
  999. * @dataProvider provideRemoveLeadingWhitespaceCases
  1000. */
  1001. public function testRemoveLeadingWhitespace(int $index, ?string $whitespaces, string $expected, ?string $input = null): void
  1002. {
  1003. Tokens::clearCache();
  1004. $tokens = Tokens::fromCode($input ?? $expected);
  1005. $tokens->removeLeadingWhitespace($index, $whitespaces);
  1006. self::assertSame($expected, $tokens->generateCode());
  1007. }
  1008. /**
  1009. * @return iterable<array{0: int, 1: null|string, 2: string, 3?: string}>
  1010. */
  1011. public static function provideRemoveLeadingWhitespaceCases(): iterable
  1012. {
  1013. yield [
  1014. 7,
  1015. null,
  1016. "<?php echo 1;//\necho 2;",
  1017. ];
  1018. yield [
  1019. 7,
  1020. null,
  1021. "<?php echo 1;//\necho 2;",
  1022. "<?php echo 1;//\n echo 2;",
  1023. ];
  1024. yield [
  1025. 7,
  1026. null,
  1027. "<?php echo 1;//\r\necho 2;",
  1028. "<?php echo 1;//\r\n echo 2;",
  1029. ];
  1030. yield [
  1031. 7,
  1032. " \t",
  1033. "<?php echo 1;//\n//",
  1034. "<?php echo 1;//\n //",
  1035. ];
  1036. yield [
  1037. 6,
  1038. "\t ",
  1039. '<?php echo 1;//',
  1040. "<?php echo 1;\t \t \t //",
  1041. ];
  1042. yield [
  1043. 8,
  1044. null,
  1045. '<?php $a = 1;//',
  1046. '<?php $a = 1; //',
  1047. ];
  1048. yield [
  1049. 6,
  1050. null,
  1051. '<?php echo 1;echo 2;',
  1052. "<?php echo 1; \n \n \n \necho 2;",
  1053. ];
  1054. yield [
  1055. 8,
  1056. null,
  1057. "<?php echo 1; // 1\necho 2;",
  1058. "<?php echo 1; // 1\n \n \n \necho 2;",
  1059. ];
  1060. }
  1061. /**
  1062. * @dataProvider provideRemoveTrailingWhitespaceCases
  1063. */
  1064. public function testRemoveTrailingWhitespace(int $index, ?string $whitespaces, string $expected, ?string $input = null): void
  1065. {
  1066. Tokens::clearCache();
  1067. $tokens = Tokens::fromCode($input ?? $expected);
  1068. $tokens->removeTrailingWhitespace($index, $whitespaces);
  1069. self::assertSame($expected, $tokens->generateCode());
  1070. }
  1071. /**
  1072. * @return iterable<array{0: int, 1: null|string, 2: string, 3?: string}>
  1073. */
  1074. public static function provideRemoveTrailingWhitespaceCases(): iterable
  1075. {
  1076. $leadingCases = self::provideRemoveLeadingWhitespaceCases();
  1077. foreach ($leadingCases as $leadingCase) {
  1078. $leadingCase[0] -= 2;
  1079. yield $leadingCase;
  1080. }
  1081. }
  1082. public function testRemovingLeadingWhitespaceWithEmptyTokenInCollection(): void
  1083. {
  1084. $code = "<?php\n /* I will be removed */MY_INDEX_IS_THREE;foo();";
  1085. $tokens = Tokens::fromCode($code);
  1086. $tokens->clearAt(2);
  1087. $tokens->removeLeadingWhitespace(3);
  1088. $tokens->clearEmptyTokens();
  1089. self::assertTokens(Tokens::fromCode("<?php\nMY_INDEX_IS_THREE;foo();"), $tokens);
  1090. }
  1091. public function testRemovingTrailingWhitespaceWithEmptyTokenInCollection(): void
  1092. {
  1093. $code = "<?php\nMY_INDEX_IS_ONE/* I will be removed */ ;foo();";
  1094. $tokens = Tokens::fromCode($code);
  1095. $tokens->clearAt(2);
  1096. $tokens->removeTrailingWhitespace(1);
  1097. $tokens->clearEmptyTokens();
  1098. self::assertTokens(Tokens::fromCode("<?php\nMY_INDEX_IS_ONE;foo();"), $tokens);
  1099. }
  1100. /**
  1101. * Action that begins with the word "remove" should not change the size of collection.
  1102. */
  1103. public function testRemovingLeadingWhitespaceWillNotIncreaseTokensCount(): void
  1104. {
  1105. $tokens = Tokens::fromCode('<?php
  1106. // Foo
  1107. $bar;');
  1108. $originalCount = $tokens->count();
  1109. $tokens->removeLeadingWhitespace(4);
  1110. self::assertCount($originalCount, $tokens);
  1111. self::assertSame(
  1112. '<?php
  1113. // Foo
  1114. $bar;',
  1115. $tokens->generateCode()
  1116. );
  1117. }
  1118. /**
  1119. * Action that begins with the word "remove" should not change the size of collection.
  1120. */
  1121. public function testRemovingTrailingWhitespaceWillNotIncreaseTokensCount(): void
  1122. {
  1123. $tokens = Tokens::fromCode('<?php
  1124. // Foo
  1125. $bar;');
  1126. $originalCount = $tokens->count();
  1127. $tokens->removeTrailingWhitespace(2);
  1128. self::assertCount($originalCount, $tokens);
  1129. self::assertSame(
  1130. '<?php
  1131. // Foo
  1132. $bar;',
  1133. $tokens->generateCode()
  1134. );
  1135. }
  1136. /**
  1137. * @param null|array{type: Tokens::BLOCK_TYPE_*, isStart: bool} $expected
  1138. *
  1139. * @dataProvider provideDetectBlockTypeCases
  1140. */
  1141. public function testDetectBlockType(?array $expected, string $code, int $index): void
  1142. {
  1143. $tokens = Tokens::fromCode($code);
  1144. self::assertSame($expected, Tokens::detectBlockType($tokens[$index]));
  1145. }
  1146. public static function provideDetectBlockTypeCases(): iterable
  1147. {
  1148. yield [
  1149. [
  1150. 'type' => Tokens::BLOCK_TYPE_CURLY_BRACE,
  1151. 'isStart' => true,
  1152. ],
  1153. '<?php { echo 1; }',
  1154. 1,
  1155. ];
  1156. yield [
  1157. null,
  1158. '<?php { echo 1;}',
  1159. 0,
  1160. ];
  1161. }
  1162. public function testOverrideRangeTokens(): void
  1163. {
  1164. $expected = [
  1165. new Token([T_OPEN_TAG, '<?php ']),
  1166. new Token([T_FUNCTION, 'function']),
  1167. new Token([T_WHITESPACE, ' ']),
  1168. new Token([T_STRING, 'foo']),
  1169. new Token('('),
  1170. new Token([T_ARRAY, 'array']),
  1171. new Token([T_WHITESPACE, ' ']),
  1172. new Token([T_VARIABLE, '$bar']),
  1173. new Token(')'),
  1174. new Token('{'),
  1175. new Token('}'),
  1176. ];
  1177. $code = '<?php function foo(array $bar){}';
  1178. $indexStart = 5;
  1179. $indexEnd = 5;
  1180. $items = Tokens::fromArray([new Token([T_ARRAY, 'array'])]);
  1181. $tokens = Tokens::fromCode($code);
  1182. $tokens->overrideRange($indexStart, $indexEnd, $items);
  1183. $tokens->clearEmptyTokens();
  1184. self::assertTokens(Tokens::fromArray($expected), $tokens);
  1185. }
  1186. /**
  1187. * @param list<Token> $expected
  1188. * @param array<int, Token> $items
  1189. *
  1190. * @dataProvider provideOverrideRangeCases
  1191. */
  1192. public function testOverrideRange(array $expected, string $code, int $indexStart, int $indexEnd, array $items): void
  1193. {
  1194. $tokens = Tokens::fromCode($code);
  1195. $tokens->overrideRange($indexStart, $indexEnd, $items);
  1196. $tokens->clearEmptyTokens();
  1197. self::assertTokens(Tokens::fromArray($expected), $tokens);
  1198. }
  1199. public static function provideOverrideRangeCases(): iterable
  1200. {
  1201. // typically done by transformers, here we test the reverse
  1202. yield 'override different tokens but same content' => [
  1203. [
  1204. new Token([T_OPEN_TAG, '<?php ']),
  1205. new Token([T_FUNCTION, 'function']),
  1206. new Token([T_WHITESPACE, ' ']),
  1207. new Token([T_STRING, 'foo']),
  1208. new Token('('),
  1209. new Token([T_ARRAY, 'array']),
  1210. new Token([T_WHITESPACE, ' ']),
  1211. new Token([T_VARIABLE, '$bar']),
  1212. new Token(')'),
  1213. new Token('{'),
  1214. new Token('}'),
  1215. ],
  1216. '<?php function foo(array $bar){}',
  1217. 5,
  1218. 5,
  1219. [new Token([T_ARRAY, 'array'])],
  1220. ];
  1221. yield 'add more item than in range' => [
  1222. [
  1223. new Token([T_OPEN_TAG, "<?php\n"]),
  1224. new Token([T_COMMENT, '// test']),
  1225. new Token([T_WHITESPACE, "\n"]),
  1226. new Token([T_COMMENT, '// test']),
  1227. new Token([T_WHITESPACE, "\n"]),
  1228. new Token([T_COMMENT, '// test']),
  1229. new Token([T_WHITESPACE, "\n"]),
  1230. ],
  1231. "<?php\n#comment",
  1232. 1,
  1233. 1,
  1234. [
  1235. new Token([T_COMMENT, '// test']),
  1236. new Token([T_WHITESPACE, "\n"]),
  1237. new Token([T_COMMENT, '// test']),
  1238. new Token([T_WHITESPACE, "\n"]),
  1239. new Token([T_COMMENT, '// test']),
  1240. new Token([T_WHITESPACE, "\n"]),
  1241. ],
  1242. ];
  1243. yield [
  1244. [
  1245. new Token([T_OPEN_TAG, "<?php\n"]),
  1246. new Token([T_COMMENT, '#comment1']),
  1247. new Token([T_WHITESPACE, "\n"]),
  1248. new Token([T_COMMENT, '// test 1']),
  1249. new Token([T_WHITESPACE, "\n"]),
  1250. new Token([T_COMMENT, '#comment5']),
  1251. new Token([T_WHITESPACE, "\n"]),
  1252. new Token([T_COMMENT, '#comment6']),
  1253. ],
  1254. "<?php\n#comment1\n#comment2\n#comment3\n#comment4\n#comment5\n#comment6",
  1255. 3,
  1256. 7,
  1257. [
  1258. new Token([T_COMMENT, '// test 1']),
  1259. ],
  1260. ];
  1261. yield [
  1262. [
  1263. new Token([T_OPEN_TAG, "<?php\n"]),
  1264. new Token([T_COMMENT, '// test']),
  1265. ],
  1266. "<?php\n#comment1\n#comment2\n#comment3\n#comment4\n#comment5\n#comment6\n#comment7",
  1267. 1,
  1268. 13,
  1269. [
  1270. new Token([T_COMMENT, '// test']),
  1271. ],
  1272. ];
  1273. yield [
  1274. [
  1275. new Token([T_OPEN_TAG, "<?php\n"]),
  1276. new Token([T_COMMENT, '// test']),
  1277. ],
  1278. "<?php\n#comment",
  1279. 1,
  1280. 1,
  1281. [
  1282. new Token([T_COMMENT, '// test']),
  1283. ],
  1284. ];
  1285. }
  1286. public function testInitialChangedState(): void
  1287. {
  1288. $tokens = Tokens::fromCode("<?php\n");
  1289. self::assertFalse($tokens->isChanged());
  1290. $tokens = Tokens::fromArray(
  1291. [
  1292. new Token([T_OPEN_TAG, "<?php\n"]),
  1293. new Token([T_STRING, 'Foo']),
  1294. new Token(';'),
  1295. ]
  1296. );
  1297. self::assertFalse($tokens->isChanged());
  1298. }
  1299. /**
  1300. * @param -1|1 $direction
  1301. *
  1302. * @dataProvider provideGetMeaningfulTokenSiblingCases
  1303. */
  1304. public function testGetMeaningfulTokenSibling(?int $expectIndex, int $index, int $direction, string $source): void
  1305. {
  1306. Tokens::clearCache();
  1307. $tokens = Tokens::fromCode($source);
  1308. self::assertSame($expectIndex, $tokens->getMeaningfulTokenSibling($index, $direction));
  1309. }
  1310. /**
  1311. * @return iterable<int|string, array{null|int, int, int, string}>
  1312. */
  1313. public static function provideGetMeaningfulTokenSiblingCases(): iterable
  1314. {
  1315. yield [null, 0, 1, '<?php '];
  1316. yield [null, 1, 1, '<?php /**/ /**/ /**/ /**/#'];
  1317. for ($i = 0; $i < 3; ++$i) {
  1318. yield '>'.$i => [3, $i, 1, '<?php /**/ foo();'];
  1319. }
  1320. yield '>>' => [4, 3, 1, '<?php /**/ foo();'];
  1321. yield '@ end' => [null, 6, 1, '<?php /**/ foo();'];
  1322. yield 'over end' => [null, 888, 1, '<?php /**/ foo();'];
  1323. yield [0, 3, -1, '<?php /**/ foo();'];
  1324. yield [4, 5, -1, '<?php /**/ foo();'];
  1325. yield [5, 6, -1, '<?php /**/ foo();'];
  1326. yield [null, 0, -1, '<?php /**/ foo();'];
  1327. }
  1328. /**
  1329. * @dataProvider provideInsertSlicesAtMultiplePlacesCases
  1330. *
  1331. * @param array<int, Token> $slices
  1332. */
  1333. public function testInsertSlicesAtMultiplePlaces(string $expected, array $slices): void
  1334. {
  1335. $input = <<<'EOF'
  1336. <?php
  1337. $after = get_class($after);
  1338. $before = get_class($before);
  1339. EOF;
  1340. $tokens = Tokens::fromCode($input);
  1341. $tokens->insertSlices([
  1342. 16 => $slices,
  1343. 6 => $slices,
  1344. ]);
  1345. self::assertTokens(Tokens::fromCode($expected), $tokens);
  1346. }
  1347. public static function provideInsertSlicesAtMultiplePlacesCases(): iterable
  1348. {
  1349. yield 'one slice count' => [
  1350. <<<'EOF'
  1351. <?php
  1352. $after = /*foo*/get_class($after);
  1353. $before = /*foo*/get_class($before);
  1354. EOF,
  1355. [new Token([T_COMMENT, '/*foo*/'])],
  1356. ];
  1357. yield 'two slice count' => [
  1358. <<<'EOF'
  1359. <?php
  1360. $after = (string) get_class($after);
  1361. $before = (string) get_class($before);
  1362. EOF,
  1363. [new Token([T_STRING_CAST, '(string)']), new Token([T_WHITESPACE, ' '])],
  1364. ];
  1365. yield 'three slice count' => [
  1366. <<<'EOF'
  1367. <?php
  1368. $after = !(bool) get_class($after);
  1369. $before = !(bool) get_class($before);
  1370. EOF,
  1371. [new Token('!'), new Token([T_BOOL_CAST, '(bool)']), new Token([T_WHITESPACE, ' '])],
  1372. ];
  1373. }
  1374. public function testInsertSlicesChangesState(): void
  1375. {
  1376. $tokens = Tokens::fromCode('<?php echo 1234567890;');
  1377. self::assertFalse($tokens->isChanged());
  1378. self::assertFalse($tokens->isTokenKindFound(T_COMMENT));
  1379. self::assertSame(5, $tokens->getSize());
  1380. $tokens->insertSlices([1 => new Token([T_COMMENT, '/* comment */'])]);
  1381. self::assertTrue($tokens->isChanged());
  1382. self::assertTrue($tokens->isTokenKindFound(T_COMMENT));
  1383. self::assertSame(6, $tokens->getSize());
  1384. }
  1385. /**
  1386. * @param array<int, list<Token>|Token|Tokens> $slices
  1387. *
  1388. * @dataProvider provideInsertSlicesCases
  1389. */
  1390. public function testInsertSlices(Tokens $expected, Tokens $tokens, array $slices): void
  1391. {
  1392. $tokens->insertSlices($slices);
  1393. self::assertTokens($expected, $tokens);
  1394. }
  1395. public static function provideInsertSlicesCases(): iterable
  1396. {
  1397. // basic insert of single token at 3 different locations including appending as new token
  1398. $template = "<?php\n%s\n/* single token test header */%s\necho 1;\n%s";
  1399. $commentContent = '/* test */';
  1400. $commentToken = new Token([T_COMMENT, $commentContent]);
  1401. $from = Tokens::fromCode(\sprintf($template, '', '', ''));
  1402. yield 'single insert @ 1' => [
  1403. Tokens::fromCode(\sprintf($template, $commentContent, '', '')),
  1404. clone $from,
  1405. [1 => $commentToken],
  1406. ];
  1407. yield 'single insert @ 3' => [
  1408. Tokens::fromCode(\sprintf($template, '', $commentContent, '')),
  1409. clone $from,
  1410. [3 => Tokens::fromArray([$commentToken])],
  1411. ];
  1412. yield 'single insert @ 9' => [
  1413. Tokens::fromCode(\sprintf($template, '', '', $commentContent)),
  1414. clone $from,
  1415. [9 => [$commentToken]],
  1416. ];
  1417. // basic tests for single token, array of that token and tokens object with that token
  1418. $openTagToken = new Token([T_OPEN_TAG, "<?php\n"]);
  1419. $expected = Tokens::fromArray([$openTagToken]);
  1420. $slices = [
  1421. [0 => $openTagToken],
  1422. [0 => [clone $openTagToken]],
  1423. [0 => clone Tokens::fromArray([$openTagToken])],
  1424. ];
  1425. foreach ($slices as $i => $slice) {
  1426. yield 'insert open tag @ 0 into empty collection '.$i => [$expected, new Tokens(), $slice];
  1427. }
  1428. // test insert lists of tokens, index out of order
  1429. $setOne = [
  1430. new Token([T_ECHO, 'echo']),
  1431. new Token([T_WHITESPACE, ' ']),
  1432. new Token([T_CONSTANT_ENCAPSED_STRING, '"new"']),
  1433. new Token(';'),
  1434. ];
  1435. $setTwo = [
  1436. new Token([T_WHITESPACE, ' ']),
  1437. new Token([T_COMMENT, '/* new comment */']),
  1438. ];
  1439. $setThree = Tokens::fromArray([
  1440. new Token([T_VARIABLE, '$new']),
  1441. new Token([T_WHITESPACE, ' ']),
  1442. new Token('='),
  1443. new Token([T_WHITESPACE, ' ']),
  1444. new Token([T_LNUMBER, '8899']),
  1445. new Token(';'),
  1446. new Token([T_WHITESPACE, "\n"]),
  1447. ]);
  1448. $template = "<?php\n%s\n/* header */%s\necho 789;\n%s";
  1449. $expected = Tokens::fromCode(
  1450. \sprintf(
  1451. $template,
  1452. 'echo "new";',
  1453. ' /* new comment */',
  1454. "\$new = 8899;\n"
  1455. )
  1456. );
  1457. $from = Tokens::fromCode(\sprintf($template, '', '', ''));
  1458. yield 'insert 3 token collections' => [$expected, $from, [9 => $setThree, 1 => $setOne, 3 => $setTwo]];
  1459. $sets = [];
  1460. for ($j = 0; $j < 4; ++$j) {
  1461. $set = ['tokens' => [], 'content' => ''];
  1462. for ($i = 0; $i < 10; ++$i) {
  1463. $content = \sprintf('/* new %d|%s */', $j, $i);
  1464. $set['tokens'][] = new Token([T_COMMENT, $content]);
  1465. $set['content'] .= $content;
  1466. }
  1467. $sets[$j] = $set;
  1468. }
  1469. yield 'overlapping inserts of bunch of comments ' => [
  1470. Tokens::fromCode(\sprintf("<?php\n%s/* line #1 */\n%s/* line #2 */\n%s/* line #3 */%s", $sets[0]['content'], $sets[1]['content'], $sets[2]['content'], $sets[3]['content'])),
  1471. Tokens::fromCode("<?php\n/* line #1 */\n/* line #2 */\n/* line #3 */"),
  1472. [1 => $sets[0]['tokens'], 3 => $sets[1]['tokens'], 5 => $sets[2]['tokens'], 6 => $sets[3]['tokens']],
  1473. ];
  1474. }
  1475. public function testBlockEdgeCachingOffsetSet(): void
  1476. {
  1477. $tokens = $this->getBlockEdgeCachingTestTokens();
  1478. $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5);
  1479. self::assertSame(9, $endIndex);
  1480. $tokens->offsetSet(5, new Token('('));
  1481. $tokens->offsetSet(9, new Token('('));
  1482. $this->expectException(\InvalidArgumentException::class);
  1483. $this->expectExceptionMessage('Invalid param $startIndex - not a proper block "start".');
  1484. $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5);
  1485. }
  1486. public function testBlockEdgeCachingOffsetSetPruneEvenIfTokenEquals(): void
  1487. {
  1488. $tokens = Tokens::fromArray([
  1489. new Token([T_OPEN_TAG, '<?php ']),
  1490. new Token([T_VARIABLE, '$a']),
  1491. new Token('='),
  1492. new Token([T_WHITESPACE, ' ']),
  1493. new Token([CT::T_ARRAY_SQUARE_BRACE_OPEN, '[']),
  1494. new Token([T_WHITESPACE, ' ']),
  1495. new Token([CT::T_ARRAY_SQUARE_BRACE_CLOSE, ']']),
  1496. new Token(';'),
  1497. ]);
  1498. self::assertSame(6, $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 4));
  1499. self::assertSame(4, $tokens->findBlockStart(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 6));
  1500. $tokens->overrideRange(3, 6, [
  1501. new Token([CT::T_ARRAY_SQUARE_BRACE_OPEN, '[']),
  1502. $tokens[4],
  1503. new Token([CT::T_ARRAY_SQUARE_BRACE_CLOSE, ']']),
  1504. $tokens[6],
  1505. ]);
  1506. self::assertSame(5, $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 4));
  1507. self::assertSame(4, $tokens->findBlockStart(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5));
  1508. self::assertSame(6, $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 3));
  1509. self::assertSame(3, $tokens->findBlockStart(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 6));
  1510. }
  1511. public function testBlockEdgeCachingClearAt(): void
  1512. {
  1513. $tokens = $this->getBlockEdgeCachingTestTokens();
  1514. $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5);
  1515. self::assertSame(9, $endIndex);
  1516. $tokens->clearAt(7); // note: offsetUnset doesn't work here
  1517. $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5);
  1518. self::assertSame(9, $endIndex);
  1519. $tokens->clearEmptyTokens();
  1520. $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5);
  1521. self::assertSame(8, $endIndex);
  1522. }
  1523. public function testBlockEdgeCachingInsertSlices(): void
  1524. {
  1525. $tokens = $this->getBlockEdgeCachingTestTokens();
  1526. $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5);
  1527. self::assertSame(9, $endIndex);
  1528. $tokens->insertSlices([6 => [new Token([T_COMMENT, '/* A */'])], new Token([T_COMMENT, '/* B */'])]);
  1529. $endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, 5);
  1530. self::assertSame(11, $endIndex);
  1531. }
  1532. public function testNamespaceDeclarations(): void
  1533. {
  1534. $code = '<?php // no namespaces';
  1535. $tokens = Tokens::fromCode($code);
  1536. self::assertSame(
  1537. serialize([
  1538. new NamespaceAnalysis(
  1539. '',
  1540. '',
  1541. 0,
  1542. 0,
  1543. 0,
  1544. 1
  1545. ),
  1546. ]),
  1547. serialize($tokens->getNamespaceDeclarations())
  1548. );
  1549. $newNS = '<?php namespace Foo\Bar;';
  1550. $tokens->insertAt(2, Tokens::fromCode($newNS));
  1551. self::assertSame(
  1552. serialize([
  1553. new NamespaceAnalysis(
  1554. 'Foo\Bar',
  1555. 'Bar',
  1556. 3,
  1557. 8,
  1558. 3,
  1559. 8
  1560. ),
  1561. ]),
  1562. serialize($tokens->getNamespaceDeclarations())
  1563. );
  1564. }
  1565. public function testFindingToken(): void
  1566. {
  1567. $tokens = Tokens::fromCode('<?php $x;');
  1568. self::assertTrue($tokens->isTokenKindFound(T_VARIABLE));
  1569. $tokens->offsetUnset(1);
  1570. $tokens->offsetUnset(1); // 2nd unset of the same index should not crash anything
  1571. self::assertFalse($tokens->isTokenKindFound(T_VARIABLE));
  1572. $tokens[1] = new Token([T_VARIABLE, '$x']);
  1573. self::assertTrue($tokens->isTokenKindFound(T_VARIABLE));
  1574. }
  1575. public function testSettingSizeThrowsException(): void
  1576. {
  1577. $tokens = new Tokens();
  1578. $this->expectException(\RuntimeException::class);
  1579. $this->expectExceptionMessage('Changing tokens collection size explicitly is not allowed.');
  1580. $tokens->setSize(3);
  1581. }
  1582. public function testSettingSizeInTryCatchBlockDoesNotChangeSize(): void
  1583. {
  1584. $tokens = Tokens::fromCode('<?php $x = true;');
  1585. $size = $tokens->getSize();
  1586. try {
  1587. $tokens->setSize(5);
  1588. } catch (\RuntimeException $exception) {
  1589. self::assertSame('Changing tokens collection size explicitly is not allowed.', $exception->getMessage());
  1590. }
  1591. self::assertSame($size, $tokens->getSize());
  1592. }
  1593. private function getBlockEdgeCachingTestTokens(): Tokens
  1594. {
  1595. Tokens::clearCache();
  1596. return Tokens::fromArray([
  1597. new Token([T_OPEN_TAG, '<?php ']),
  1598. new Token([T_VARIABLE, '$a']),
  1599. new Token([T_WHITESPACE, ' ']),
  1600. new Token('='),
  1601. new Token([T_WHITESPACE, ' ']),
  1602. new Token([CT::T_ARRAY_SQUARE_BRACE_OPEN, '[']),
  1603. new Token([T_WHITESPACE, ' ']),
  1604. new Token([T_COMMENT, '/* foo */']),
  1605. new Token([T_WHITESPACE, ' ']),
  1606. new Token([CT::T_ARRAY_SQUARE_BRACE_CLOSE, ']']),
  1607. new Token(';'),
  1608. new Token([T_WHITESPACE, "\n"]),
  1609. ]);
  1610. }
  1611. /**
  1612. * @param Tokens::BLOCK_TYPE_* $type
  1613. */
  1614. private static function assertFindBlockEnd(int $expectedIndex, string $source, int $type, int $searchIndex): void
  1615. {
  1616. Tokens::clearCache();
  1617. $tokens = Tokens::fromCode($source);
  1618. self::assertSame($expectedIndex, $tokens->findBlockEnd($type, $searchIndex));
  1619. self::assertSame($searchIndex, $tokens->findBlockStart($type, $expectedIndex));
  1620. $detectedType = Tokens::detectBlockType($tokens[$searchIndex]);
  1621. self::assertIsArray($detectedType);
  1622. self::assertArrayHasKey('type', $detectedType);
  1623. self::assertArrayHasKey('isStart', $detectedType);
  1624. self::assertSame($type, $detectedType['type']);
  1625. self::assertTrue($detectedType['isStart']);
  1626. $detectedType = Tokens::detectBlockType($tokens[$expectedIndex]);
  1627. self::assertIsArray($detectedType);
  1628. self::assertArrayHasKey('type', $detectedType);
  1629. self::assertArrayHasKey('isStart', $detectedType);
  1630. self::assertSame($type, $detectedType['type']);
  1631. self::assertFalse($detectedType['isStart']);
  1632. }
  1633. /**
  1634. * @param null|array<int, Token> $expected
  1635. * @param null|array<int, Token> $input
  1636. */
  1637. private static function assertEqualsTokensArray(?array $expected = null, ?array $input = null): void
  1638. {
  1639. if (null === $expected) {
  1640. self::assertNull($input);
  1641. return;
  1642. }
  1643. if (null === $input) {
  1644. self::fail('While "input" is <null>, "expected" is not.');
  1645. }
  1646. self::assertSame(array_keys($expected), array_keys($input), 'Both arrays need to have same keys.');
  1647. foreach ($expected as $index => $expectedToken) {
  1648. self::assertTrue(
  1649. $expectedToken->equals($input[$index]),
  1650. \sprintf('The token at index %d should be %s, got %s', $index, $expectedToken->toJson(), $input[$index]->toJson())
  1651. );
  1652. }
  1653. }
  1654. /**
  1655. * @param list<int> $indexes
  1656. * @param list<Token> $expected
  1657. */
  1658. private function doTestClearTokens(string $source, array $indexes, array $expected): void
  1659. {
  1660. Tokens::clearCache();
  1661. $tokens = Tokens::fromCode($source);
  1662. foreach ($indexes as $index) {
  1663. $tokens->clearTokenAndMergeSurroundingWhitespace($index);
  1664. }
  1665. self::assertSameSize($expected, $tokens);
  1666. foreach ($expected as $index => $expectedToken) {
  1667. $token = $tokens[$index];
  1668. $expectedPrototype = $expectedToken->getPrototype();
  1669. self::assertTrue($token->equals($expectedPrototype), \sprintf('The token at index %d should be %s, got %s', $index, json_encode($expectedPrototype, JSON_THROW_ON_ERROR), $token->toJson()));
  1670. }
  1671. }
  1672. }