decode_test.go 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955
  1. // Copyright 2019 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package text_test
  5. import (
  6. "fmt"
  7. "math"
  8. "strings"
  9. "testing"
  10. "unicode/utf8"
  11. "github.com/google/go-cmp/cmp"
  12. "google.golang.org/protobuf/internal/encoding/text"
  13. "google.golang.org/protobuf/internal/flags"
  14. )
  15. var eofErr = text.ErrUnexpectedEOF.Error()
  16. type R struct {
  17. // K is expected Kind of the returned Token object from calling Decoder.Read.
  18. K text.Kind
  19. // E is expected error substring from calling Decoder.Read if set.
  20. E string
  21. // T contains NT (if K is Name) or ST (if K is Scalar) or nil (others)
  22. T interface{}
  23. // P is expected Token.Pos if set > 0.
  24. P int
  25. // RS is expected result from Token.RawString() if not empty.
  26. RS string
  27. }
  28. // NT contains data for checking against a name token.
  29. type NT struct {
  30. K text.NameKind
  31. // Sep is true if name token should have separator character, else false.
  32. Sep bool
  33. // If K is IdentName or TypeName, invoke corresponding getter and compare against this field.
  34. S string
  35. // If K is FieldNumber, invoke getter and compare against this field.
  36. N int32
  37. }
  38. // ST contains data for checking against a scalar token.
  39. type ST struct {
  40. // checker that is expected to return OK.
  41. ok checker
  42. // checker that is expected to return not OK.
  43. nok checker
  44. }
  45. // checker provides API for the token wrapper API call types Str, Enum, Bool,
  46. // Uint64, Uint32, Int64, Int32, Float64, Float32.
  47. type checker interface {
  48. // checkOk checks and expects for token API call to return ok and compare
  49. // against implementation-stored value. Returns empty string if success,
  50. // else returns error message describing the error.
  51. checkOk(text.Token) string
  52. // checkNok checks and expects for token API call to return not ok. Returns
  53. // empty string if success, else returns error message describing the error.
  54. checkNok(text.Token) string
  55. }
  56. type Str struct {
  57. val string
  58. }
  59. func (s Str) checkOk(tok text.Token) string {
  60. got, ok := tok.String()
  61. if !ok {
  62. return fmt.Sprintf("Token.String() returned not OK for token: %v", tok.RawString())
  63. }
  64. if got != s.val {
  65. return fmt.Sprintf("Token.String() got %q want %q for token: %v", got, s.val, tok.RawString())
  66. }
  67. return ""
  68. }
  69. func (s Str) checkNok(tok text.Token) string {
  70. if _, ok := tok.String(); ok {
  71. return fmt.Sprintf("Token.String() returned OK for token: %v", tok.RawString())
  72. }
  73. return ""
  74. }
  75. type Enum struct {
  76. val string
  77. }
  78. func (e Enum) checkOk(tok text.Token) string {
  79. got, ok := tok.Enum()
  80. if !ok {
  81. return fmt.Sprintf("Token.Enum() returned not OK for token: %v", tok.RawString())
  82. }
  83. if got != e.val {
  84. return fmt.Sprintf("Token.Enum() got %q want %q for token: %v", got, e.val, tok.RawString())
  85. }
  86. return ""
  87. }
  88. func (e Enum) checkNok(tok text.Token) string {
  89. if _, ok := tok.Enum(); ok {
  90. return fmt.Sprintf("Token.Enum() returned OK for token: %v", tok.RawString())
  91. }
  92. return ""
  93. }
  94. type Bool struct {
  95. val bool
  96. }
  97. func (b Bool) checkOk(tok text.Token) string {
  98. got, ok := tok.Bool()
  99. if !ok {
  100. return fmt.Sprintf("Token.Bool() returned not OK for token: %v", tok.RawString())
  101. }
  102. if got != b.val {
  103. return fmt.Sprintf("Token.Bool() got %v want %v for token: %v", got, b.val, tok.RawString())
  104. }
  105. return ""
  106. }
  107. func (b Bool) checkNok(tok text.Token) string {
  108. if _, ok := tok.Bool(); ok {
  109. return fmt.Sprintf("Token.Bool() returned OK for token: %v", tok.RawString())
  110. }
  111. return ""
  112. }
  113. type Uint64 struct {
  114. val uint64
  115. }
  116. func (n Uint64) checkOk(tok text.Token) string {
  117. got, ok := tok.Uint64()
  118. if !ok {
  119. return fmt.Sprintf("Token.Uint64() returned not OK for token: %v", tok.RawString())
  120. }
  121. if got != n.val {
  122. return fmt.Sprintf("Token.Uint64() got %v want %v for token: %v", got, n.val, tok.RawString())
  123. }
  124. return ""
  125. }
  126. func (n Uint64) checkNok(tok text.Token) string {
  127. if _, ok := tok.Uint64(); ok {
  128. return fmt.Sprintf("Token.Uint64() returned OK for token: %v", tok.RawString())
  129. }
  130. return ""
  131. }
  132. type Uint32 struct {
  133. val uint32
  134. }
  135. func (n Uint32) checkOk(tok text.Token) string {
  136. got, ok := tok.Uint32()
  137. if !ok {
  138. return fmt.Sprintf("Token.Uint32() returned not OK for token: %v", tok.RawString())
  139. }
  140. if got != n.val {
  141. return fmt.Sprintf("Token.Uint32() got %v want %v for token: %v", got, n.val, tok.RawString())
  142. }
  143. return ""
  144. }
  145. func (n Uint32) checkNok(tok text.Token) string {
  146. if _, ok := tok.Uint32(); ok {
  147. return fmt.Sprintf("Token.Uint32() returned OK for token: %v", tok.RawString())
  148. }
  149. return ""
  150. }
  151. type Int64 struct {
  152. val int64
  153. }
  154. func (n Int64) checkOk(tok text.Token) string {
  155. got, ok := tok.Int64()
  156. if !ok {
  157. return fmt.Sprintf("Token.Int64() returned not OK for token: %v", tok.RawString())
  158. }
  159. if got != n.val {
  160. return fmt.Sprintf("Token.Int64() got %v want %v for token: %v", got, n.val, tok.RawString())
  161. }
  162. return ""
  163. }
  164. func (n Int64) checkNok(tok text.Token) string {
  165. if _, ok := tok.Int64(); ok {
  166. return fmt.Sprintf("Token.Int64() returned OK for token: %v", tok.RawString())
  167. }
  168. return ""
  169. }
  170. type Int32 struct {
  171. val int32
  172. }
  173. func (n Int32) checkOk(tok text.Token) string {
  174. got, ok := tok.Int32()
  175. if !ok {
  176. return fmt.Sprintf("Token.Int32() returned not OK for token: %v", tok.RawString())
  177. }
  178. if got != n.val {
  179. return fmt.Sprintf("Token.Int32() got %v want %v for token: %v", got, n.val, tok.RawString())
  180. }
  181. return ""
  182. }
  183. func (n Int32) checkNok(tok text.Token) string {
  184. if _, ok := tok.Int32(); ok {
  185. return fmt.Sprintf("Token.Int32() returned OK for token: %v", tok.RawString())
  186. }
  187. return ""
  188. }
  189. type Float64 struct {
  190. val float64
  191. }
  192. func (n Float64) checkOk(tok text.Token) string {
  193. got, ok := tok.Float64()
  194. if !ok {
  195. return fmt.Sprintf("Token.Float64() returned not OK for token: %v", tok.RawString())
  196. }
  197. if math.Float64bits(got) != math.Float64bits(n.val) {
  198. return fmt.Sprintf("Token.Float64() got %v want %v for token: %v", got, n.val, tok.RawString())
  199. }
  200. return ""
  201. }
  202. func (n Float64) checkNok(tok text.Token) string {
  203. if _, ok := tok.Float64(); ok {
  204. return fmt.Sprintf("Token.Float64() returned OK for token: %v", tok.RawString())
  205. }
  206. return ""
  207. }
  208. type Float32 struct {
  209. val float32
  210. }
  211. func (n Float32) checkOk(tok text.Token) string {
  212. got, ok := tok.Float32()
  213. if !ok {
  214. return fmt.Sprintf("Token.Float32() returned not OK for token: %v", tok.RawString())
  215. }
  216. if math.Float32bits(got) != math.Float32bits(n.val) {
  217. return fmt.Sprintf("Token.Float32() got %v want %v for token: %v", got, n.val, tok.RawString())
  218. }
  219. return ""
  220. }
  221. func (n Float32) checkNok(tok text.Token) string {
  222. if _, ok := tok.Float32(); ok {
  223. return fmt.Sprintf("Token.Float32() returned OK for token: %v", tok.RawString())
  224. }
  225. return ""
  226. }
  227. func TestDecoder(t *testing.T) {
  228. const space = " \n\r\t"
  229. tests := []struct {
  230. in string
  231. // want is a list of expected Tokens returned from calling Decoder.Read.
  232. // An item makes the test code invoke Decoder.Read and compare against
  233. // R.K and R.E. If R.K is Name, it compares
  234. want []R
  235. }{
  236. {
  237. in: "",
  238. want: []R{{K: text.EOF}},
  239. },
  240. {
  241. in: "# comment",
  242. want: []R{{K: text.EOF}},
  243. },
  244. {
  245. in: space + "# comment" + space,
  246. want: []R{{K: text.EOF}},
  247. },
  248. {
  249. in: space,
  250. want: []R{{K: text.EOF, P: len(space)}},
  251. },
  252. {
  253. // Calling Read after EOF will keep returning EOF for
  254. // succeeding Read calls.
  255. in: space,
  256. want: []R{
  257. {K: text.EOF},
  258. {K: text.EOF},
  259. {K: text.EOF},
  260. },
  261. },
  262. {
  263. // NUL is an invalid whitespace since C++ uses C-strings.
  264. in: "\x00",
  265. want: []R{{E: "invalid field name: \x00"}},
  266. },
  267. // Field names.
  268. {
  269. in: "name",
  270. want: []R{
  271. {K: text.Name, T: NT{K: text.IdentName, S: "name"}, RS: "name"},
  272. {E: eofErr},
  273. },
  274. },
  275. {
  276. in: space + "name:" + space,
  277. want: []R{
  278. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
  279. {E: eofErr},
  280. },
  281. },
  282. {
  283. in: space + "name" + space + ":" + space,
  284. want: []R{
  285. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
  286. {E: eofErr},
  287. },
  288. },
  289. {
  290. in: "name # comment",
  291. want: []R{
  292. {K: text.Name, T: NT{K: text.IdentName, S: "name"}},
  293. {E: eofErr},
  294. },
  295. },
  296. {
  297. // Comments only extend until the newline.
  298. in: "# comment \nname",
  299. want: []R{
  300. {K: text.Name, T: NT{K: text.IdentName, S: "name"}, P: 11},
  301. },
  302. },
  303. {
  304. in: "name # comment \n:",
  305. want: []R{
  306. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
  307. },
  308. },
  309. {
  310. in: "name123",
  311. want: []R{
  312. {K: text.Name, T: NT{K: text.IdentName, S: "name123"}},
  313. },
  314. },
  315. {
  316. in: "name_123",
  317. want: []R{
  318. {K: text.Name, T: NT{K: text.IdentName, S: "name_123"}},
  319. },
  320. },
  321. {
  322. in: "_123",
  323. want: []R{
  324. {K: text.Name, T: NT{K: text.IdentName, S: "_123"}},
  325. },
  326. },
  327. {
  328. in: ":",
  329. want: []R{{E: "syntax error (line 1:1): invalid field name: :"}},
  330. },
  331. {
  332. in: "\n\n\n {",
  333. want: []R{{E: "syntax error (line 4:2): invalid field name: {"}},
  334. },
  335. {
  336. in: "123name",
  337. want: []R{{E: "invalid field name: 123name"}},
  338. },
  339. {
  340. in: `/`,
  341. want: []R{{E: `invalid field name: /`}},
  342. },
  343. {
  344. in: `δΈ–η•Œ`,
  345. want: []R{{E: `invalid field name: δΈ–`}},
  346. },
  347. {
  348. in: `1a/b`,
  349. want: []R{{E: `invalid field name: 1a`}},
  350. },
  351. {
  352. in: `1c\d`,
  353. want: []R{{E: `invalid field name: 1c`}},
  354. },
  355. {
  356. in: "\x84f",
  357. want: []R{{E: "invalid field name: \x84"}},
  358. },
  359. {
  360. in: "\uFFFDxxx",
  361. want: []R{{E: "invalid field name: \uFFFD"}},
  362. },
  363. {
  364. in: "-a234567890123456789012345678901234567890abc",
  365. want: []R{{E: "invalid field name: -a2345678901234567890123456789012…"}},
  366. },
  367. {
  368. in: "[type]",
  369. want: []R{
  370. {K: text.Name, T: NT{K: text.TypeName, S: "type"}, RS: "[type]"},
  371. },
  372. },
  373. {
  374. // V1 allows this syntax. C++ does not, however, C++ also fails if
  375. // field is Any and does not contain '/'.
  376. in: "[/type]",
  377. want: []R{
  378. {K: text.Name, T: NT{K: text.TypeName, S: "/type"}},
  379. },
  380. },
  381. {
  382. in: "[.type]",
  383. want: []R{{E: "invalid type URL/extension field name: [.type]"}},
  384. },
  385. {
  386. in: "[pkg.Foo.extension_field]",
  387. want: []R{
  388. {K: text.Name, T: NT{K: text.TypeName, S: "pkg.Foo.extension_field"}},
  389. },
  390. },
  391. {
  392. in: "[domain.com/type]",
  393. want: []R{
  394. {K: text.Name, T: NT{K: text.TypeName, S: "domain.com/type"}},
  395. },
  396. },
  397. {
  398. in: "[domain.com/pkg.type]",
  399. want: []R{
  400. {K: text.Name, T: NT{K: text.TypeName, S: "domain.com/pkg.type"}},
  401. },
  402. },
  403. {
  404. in: "[sub.domain.com\x2fpath\x2fto\x2fproto.package.name]",
  405. want: []R{
  406. {
  407. K: text.Name,
  408. T: NT{
  409. K: text.TypeName,
  410. S: "sub.domain.com/path/to/proto.package.name",
  411. },
  412. RS: "[sub.domain.com\x2fpath\x2fto\x2fproto.package.name]",
  413. },
  414. },
  415. },
  416. {
  417. // V2 no longer allows a quoted string for the Any type URL.
  418. in: `["domain.com/pkg.type"]`,
  419. want: []R{{E: `invalid type URL/extension field name: ["`}},
  420. },
  421. {
  422. // V2 no longer allows a quoted string for the Any type URL.
  423. in: `['domain.com/pkg.type']`,
  424. want: []R{{E: `invalid type URL/extension field name: ['`}},
  425. },
  426. {
  427. in: "[pkg.Foo.extension_field:",
  428. want: []R{{E: "invalid type URL/extension field name: [pkg.Foo.extension_field:"}},
  429. },
  430. {
  431. // V2 no longer allows whitespace within identifier "word".
  432. in: "[proto.packa ge.field]",
  433. want: []R{{E: "invalid type URL/extension field name: [proto.packa g"}},
  434. },
  435. {
  436. // V2 no longer allows comments within identifier "word".
  437. in: "[proto.packa # comment\n ge.field]",
  438. want: []R{{E: "invalid type URL/extension field name: [proto.packa # comment\n g"}},
  439. },
  440. {
  441. in: "[proto.package.]",
  442. want: []R{{E: "invalid type URL/extension field name: [proto.package."}},
  443. },
  444. {
  445. in: "[proto.package/]",
  446. want: []R{{E: "invalid type URL/extension field name: [proto.package/"}},
  447. },
  448. {
  449. in: `message_field{[bad@]`,
  450. want: []R{
  451. {K: text.Name},
  452. {K: text.MessageOpen},
  453. {E: `invalid type URL/extension field name: [bad@`},
  454. },
  455. },
  456. {
  457. in: `message_field{[invalid//type]`,
  458. want: []R{
  459. {K: text.Name},
  460. {K: text.MessageOpen},
  461. {E: `invalid type URL/extension field name: [invalid//`},
  462. },
  463. },
  464. {
  465. in: `message_field{[proto.package.]`,
  466. want: []R{
  467. {K: text.Name},
  468. {K: text.MessageOpen},
  469. {E: `invalid type URL/extension field name: [proto.package.`},
  470. },
  471. },
  472. {
  473. in: "[proto.package",
  474. want: []R{{E: eofErr}},
  475. },
  476. {
  477. in: "[" + space + "type" + space + "]" + space + ":",
  478. want: []R{
  479. {
  480. K: text.Name,
  481. T: NT{
  482. K: text.TypeName,
  483. Sep: true,
  484. S: "type",
  485. },
  486. RS: "[" + space + "type" + space + "]",
  487. },
  488. },
  489. },
  490. {
  491. // Whitespaces/comments are only allowed betweeb
  492. in: "[" + space + "domain" + space + "." + space + "com # comment\n" +
  493. "/" + "pkg" + space + "." + space + "type" + space + "]",
  494. want: []R{
  495. {K: text.Name, T: NT{K: text.TypeName, S: "domain.com/pkg.type"}},
  496. },
  497. },
  498. {
  499. in: "42",
  500. want: []R{
  501. {K: text.Name, T: NT{K: text.FieldNumber, N: 42}},
  502. },
  503. },
  504. {
  505. in: "0x42:",
  506. want: []R{{E: "invalid field number: 0x42"}},
  507. },
  508. {
  509. in: "042:",
  510. want: []R{{E: "invalid field number: 042"}},
  511. },
  512. {
  513. in: "123.456:",
  514. want: []R{{E: "invalid field number: 123.456"}},
  515. },
  516. {
  517. in: "-123",
  518. want: []R{{E: "invalid field number: -123"}},
  519. },
  520. {
  521. in: "- \t 123.321e6",
  522. want: []R{{E: "invalid field number: -123.321e6"}},
  523. },
  524. {
  525. in: "-",
  526. want: []R{{E: "invalid field name: -"}},
  527. },
  528. {
  529. in: "- ",
  530. want: []R{{E: "invalid field name: -"}},
  531. },
  532. {
  533. in: "- # negative\n 123",
  534. want: []R{{E: "invalid field number: -123"}},
  535. },
  536. {
  537. // Field number > math.MaxInt32.
  538. in: "2147483648:",
  539. want: []R{{E: "invalid field number: 2147483648"}},
  540. },
  541. // String field value. More string parsing specific testing in
  542. // TestUnmarshalString.
  543. {
  544. in: `name: "hello world"`,
  545. want: []R{
  546. {K: text.Name},
  547. {
  548. K: text.Scalar,
  549. T: ST{ok: Str{"hello world"}, nok: Enum{}},
  550. RS: `"hello world"`,
  551. },
  552. {K: text.EOF},
  553. },
  554. },
  555. {
  556. in: `name: 'hello'`,
  557. want: []R{
  558. {K: text.Name},
  559. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  560. },
  561. },
  562. {
  563. in: `name: "hello'`,
  564. want: []R{
  565. {K: text.Name},
  566. {E: eofErr},
  567. },
  568. },
  569. {
  570. in: `name: 'hello`,
  571. want: []R{
  572. {K: text.Name},
  573. {E: eofErr},
  574. },
  575. },
  576. {
  577. // Field name without separator is ok. prototext package will need
  578. // to determine that this is not valid for scalar values.
  579. in: space + `name` + space + `"hello"` + space,
  580. want: []R{
  581. {K: text.Name},
  582. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  583. },
  584. },
  585. {
  586. in: `name'hello'`,
  587. want: []R{
  588. {K: text.Name},
  589. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  590. },
  591. },
  592. {
  593. in: `name: ` + space + `"hello"` + space + `,`,
  594. want: []R{
  595. {K: text.Name},
  596. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  597. {K: text.EOF},
  598. },
  599. },
  600. {
  601. in: `name` + space + `:` + `"hello"` + space + `;` + space,
  602. want: []R{
  603. {K: text.Name},
  604. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  605. {K: text.EOF},
  606. },
  607. },
  608. {
  609. in: `name:"hello" , ,`,
  610. want: []R{
  611. {K: text.Name},
  612. {K: text.Scalar},
  613. {E: "(line 1:16): invalid field name: ,"},
  614. },
  615. },
  616. {
  617. in: `name:"hello" , ;`,
  618. want: []R{
  619. {K: text.Name},
  620. {K: text.Scalar},
  621. {E: "(line 1:16): invalid field name: ;"},
  622. },
  623. },
  624. {
  625. in: `name:"hello" name:'world'`,
  626. want: []R{
  627. {K: text.Name},
  628. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  629. {K: text.Name},
  630. {K: text.Scalar, T: ST{ok: Str{"world"}}},
  631. {K: text.EOF},
  632. },
  633. },
  634. {
  635. in: `name:"hello", name:"world"`,
  636. want: []R{
  637. {K: text.Name},
  638. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  639. {K: text.Name},
  640. {K: text.Scalar, T: ST{ok: Str{"world"}}},
  641. {K: text.EOF},
  642. },
  643. },
  644. {
  645. in: `name:"hello"; name:"world",`,
  646. want: []R{
  647. {K: text.Name},
  648. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  649. {K: text.Name},
  650. {K: text.Scalar, T: ST{ok: Str{"world"}}},
  651. {K: text.EOF},
  652. },
  653. },
  654. {
  655. in: `foo:"hello"bar:"world"`,
  656. want: []R{
  657. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "foo"}},
  658. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  659. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "bar"}},
  660. {K: text.Scalar, T: ST{ok: Str{"world"}}},
  661. {K: text.EOF},
  662. },
  663. },
  664. {
  665. in: `foo:"hello"[bar]:"world"`,
  666. want: []R{
  667. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "foo"}},
  668. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  669. {K: text.Name, T: NT{K: text.TypeName, Sep: true, S: "bar"}},
  670. {K: text.Scalar, T: ST{ok: Str{"world"}}},
  671. {K: text.EOF},
  672. },
  673. },
  674. {
  675. in: `name:"foo"` + space + `"bar"` + space + `'qux'`,
  676. want: []R{
  677. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
  678. {K: text.Scalar, T: ST{ok: Str{"foobarqux"}}},
  679. {K: text.EOF},
  680. },
  681. },
  682. {
  683. in: `name:"foo"'bar'"qux"`,
  684. want: []R{
  685. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
  686. {K: text.Scalar, T: ST{ok: Str{"foobarqux"}}},
  687. {K: text.EOF},
  688. },
  689. },
  690. {
  691. in: `name:"foo"` + space + `"bar" # comment` + "\n'qux' # comment",
  692. want: []R{
  693. {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
  694. {K: text.Scalar, T: ST{ok: Str{"foobarqux"}}},
  695. {K: text.EOF},
  696. },
  697. },
  698. // Lists.
  699. {
  700. in: `name: [`,
  701. want: []R{
  702. {K: text.Name},
  703. {K: text.ListOpen},
  704. {E: eofErr},
  705. },
  706. },
  707. {
  708. in: `name: []`,
  709. want: []R{
  710. {K: text.Name},
  711. {K: text.ListOpen},
  712. {K: text.ListClose},
  713. {K: text.EOF},
  714. },
  715. },
  716. {
  717. in: `name []`,
  718. want: []R{
  719. {K: text.Name},
  720. {K: text.ListOpen},
  721. {K: text.ListClose},
  722. {K: text.EOF},
  723. },
  724. },
  725. {
  726. in: `name: [,`,
  727. want: []R{
  728. {K: text.Name},
  729. {K: text.ListOpen},
  730. {E: `(line 1:8): invalid scalar value: ,`},
  731. },
  732. },
  733. {
  734. in: `name: [0`,
  735. want: []R{
  736. {K: text.Name},
  737. {K: text.ListOpen},
  738. {K: text.Scalar},
  739. {E: eofErr},
  740. },
  741. },
  742. {
  743. in: `name: [` + space + `"hello"` + space + `]` + space,
  744. want: []R{
  745. {K: text.Name},
  746. {K: text.ListOpen},
  747. {K: text.Scalar, T: ST{ok: Str{"hello"}}, P: len(space) + 7},
  748. {K: text.ListClose},
  749. {K: text.EOF},
  750. },
  751. },
  752. {
  753. in: `name: ["hello",]`,
  754. want: []R{
  755. {K: text.Name},
  756. {K: text.ListOpen},
  757. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  758. {E: `invalid scalar value: ]`},
  759. },
  760. },
  761. {
  762. in: `name: ["foo"` + space + `'bar' "qux"]`,
  763. want: []R{
  764. {K: text.Name},
  765. {K: text.ListOpen},
  766. {K: text.Scalar, T: ST{ok: Str{"foobarqux"}}},
  767. {K: text.ListClose},
  768. {K: text.EOF},
  769. },
  770. },
  771. {
  772. in: `name:` + space + `["foo",` + space + "'bar', # comment\n\n" + `"qux"]`,
  773. want: []R{
  774. {K: text.Name},
  775. {K: text.ListOpen},
  776. {K: text.Scalar, T: ST{ok: Str{"foo"}}},
  777. {K: text.Scalar, T: ST{ok: Str{"bar"}}},
  778. {K: text.Scalar, T: ST{ok: Str{"qux"}}},
  779. {K: text.ListClose},
  780. {K: text.EOF},
  781. },
  782. },
  783. {
  784. // List within list is not allowed.
  785. in: `name: [[]]`,
  786. want: []R{
  787. {K: text.Name},
  788. {K: text.ListOpen},
  789. {E: `syntax error (line 1:8): invalid scalar value: [`},
  790. },
  791. },
  792. {
  793. // List items need to be separated by ,.
  794. in: `name: ["foo" true]`,
  795. want: []R{
  796. {K: text.Name},
  797. {K: text.ListOpen},
  798. {K: text.Scalar, T: ST{ok: Str{"foo"}}},
  799. {E: `syntax error (line 1:14): unexpected character 't'`},
  800. },
  801. },
  802. {
  803. in: `name: ["foo"; "bar"]`,
  804. want: []R{
  805. {K: text.Name},
  806. {K: text.ListOpen},
  807. {K: text.Scalar, T: ST{ok: Str{"foo"}}},
  808. {E: `syntax error (line 1:13): unexpected character ';'`},
  809. },
  810. },
  811. {
  812. in: `name: ["foo", true, ENUM, 1.0]`,
  813. want: []R{
  814. {K: text.Name},
  815. {K: text.ListOpen},
  816. {K: text.Scalar, T: ST{ok: Str{"foo"}}},
  817. {K: text.Scalar, T: ST{ok: Enum{"true"}}},
  818. {K: text.Scalar, T: ST{ok: Enum{"ENUM"}}},
  819. {K: text.Scalar, T: ST{ok: Float32{1.0}}},
  820. {K: text.ListClose},
  821. },
  822. },
  823. // Boolean literal values.
  824. {
  825. in: `name: True`,
  826. want: []R{
  827. {K: text.Name},
  828. {
  829. K: text.Scalar,
  830. T: ST{ok: Bool{true}},
  831. },
  832. {K: text.EOF},
  833. },
  834. },
  835. {
  836. in: `name false`,
  837. want: []R{
  838. {K: text.Name},
  839. {
  840. K: text.Scalar,
  841. T: ST{ok: Bool{false}},
  842. },
  843. {K: text.EOF},
  844. },
  845. },
  846. {
  847. in: `name: [t, f, True, False, true, false, 1, 0, 0x01, 0x00, 01, 00]`,
  848. want: []R{
  849. {K: text.Name},
  850. {K: text.ListOpen},
  851. {K: text.Scalar, T: ST{ok: Bool{true}}},
  852. {K: text.Scalar, T: ST{ok: Bool{false}}},
  853. {K: text.Scalar, T: ST{ok: Bool{true}}},
  854. {K: text.Scalar, T: ST{ok: Bool{false}}},
  855. {K: text.Scalar, T: ST{ok: Bool{true}}},
  856. {K: text.Scalar, T: ST{ok: Bool{false}}},
  857. {K: text.Scalar, T: ST{ok: Bool{true}}},
  858. {K: text.Scalar, T: ST{ok: Bool{false}}},
  859. {K: text.Scalar, T: ST{ok: Bool{true}}},
  860. {K: text.Scalar, T: ST{ok: Bool{false}}},
  861. {K: text.Scalar, T: ST{ok: Bool{true}}},
  862. {K: text.Scalar, T: ST{ok: Bool{false}}},
  863. {K: text.ListClose},
  864. },
  865. },
  866. {
  867. // Looks like boolean but not.
  868. in: `name: [tRUe, falSE, -1, -0, -0x01, -0x00, -01, -00, 0.0]`,
  869. want: []R{
  870. {K: text.Name},
  871. {K: text.ListOpen},
  872. {K: text.Scalar, T: ST{nok: Bool{}}},
  873. {K: text.Scalar, T: ST{nok: Bool{}}},
  874. {K: text.Scalar, T: ST{nok: Bool{}}},
  875. {K: text.Scalar, T: ST{nok: Bool{}}},
  876. {K: text.Scalar, T: ST{nok: Bool{}}},
  877. {K: text.Scalar, T: ST{nok: Bool{}}},
  878. {K: text.Scalar, T: ST{nok: Bool{}}},
  879. {K: text.Scalar, T: ST{nok: Bool{}}},
  880. {K: text.Scalar, T: ST{nok: Bool{}}},
  881. {K: text.ListClose},
  882. },
  883. },
  884. {
  885. in: `foo: true[bar] false`,
  886. want: []R{
  887. {K: text.Name},
  888. {K: text.Scalar, T: ST{ok: Bool{true}}},
  889. {K: text.Name},
  890. {K: text.Scalar, T: ST{ok: Bool{false}}},
  891. },
  892. },
  893. // Enum field values.
  894. {
  895. in: space + `name: ENUM`,
  896. want: []R{
  897. {K: text.Name},
  898. {K: text.Scalar, T: ST{ok: Enum{"ENUM"}}},
  899. },
  900. },
  901. {
  902. in: space + `name:[TRUE, FALSE, T, F, t, f]`,
  903. want: []R{
  904. {K: text.Name},
  905. {K: text.ListOpen},
  906. {K: text.Scalar, T: ST{ok: Enum{"TRUE"}}},
  907. {K: text.Scalar, T: ST{ok: Enum{"FALSE"}}},
  908. {K: text.Scalar, T: ST{ok: Enum{"T"}}},
  909. {K: text.Scalar, T: ST{ok: Enum{"F"}}},
  910. {K: text.Scalar, T: ST{ok: Enum{"t"}}},
  911. {K: text.Scalar, T: ST{ok: Enum{"f"}}},
  912. {K: text.ListClose},
  913. },
  914. },
  915. {
  916. in: `foo: Enum1[bar]:Enum2`,
  917. want: []R{
  918. {K: text.Name},
  919. {K: text.Scalar, T: ST{ok: Enum{"Enum1"}}},
  920. {K: text.Name},
  921. {K: text.Scalar, T: ST{ok: Enum{"Enum2"}}},
  922. },
  923. },
  924. {
  925. // Invalid enum values.
  926. in: `name: [-inf, -foo, "string", 42, 1.0, 0x47]`,
  927. want: []R{
  928. {K: text.Name},
  929. {K: text.ListOpen},
  930. {K: text.Scalar, T: ST{nok: Enum{}}},
  931. {K: text.Scalar, T: ST{nok: Enum{}}},
  932. {K: text.Scalar, T: ST{nok: Enum{}}},
  933. {K: text.Scalar, T: ST{nok: Enum{}}},
  934. {K: text.Scalar, T: ST{nok: Enum{}}},
  935. {K: text.Scalar, T: ST{nok: Enum{}}},
  936. {K: text.ListClose},
  937. },
  938. },
  939. {
  940. in: `name: true.`,
  941. want: []R{
  942. {K: text.Name},
  943. {E: `invalid scalar value: true.`},
  944. },
  945. },
  946. // Numeric values.
  947. {
  948. in: `nums:42 nums:0x2A nums:052`,
  949. want: []R{
  950. {K: text.Name},
  951. {K: text.Scalar, T: ST{ok: Uint64{42}}},
  952. {K: text.Name},
  953. {K: text.Scalar, T: ST{ok: Uint64{42}}},
  954. {K: text.Name},
  955. {K: text.Scalar, T: ST{ok: Uint64{42}}},
  956. },
  957. },
  958. {
  959. in: `nums:[-42, -0x2a, -052]`,
  960. want: []R{
  961. {K: text.Name},
  962. {K: text.ListOpen},
  963. {K: text.Scalar, T: ST{nok: Uint64{}}},
  964. {K: text.Scalar, T: ST{nok: Uint64{}}},
  965. {K: text.Scalar, T: ST{nok: Uint64{}}},
  966. {K: text.ListClose},
  967. },
  968. },
  969. {
  970. in: `nums:[-42, -0x2a, -052]`,
  971. want: []R{
  972. {K: text.Name},
  973. {K: text.ListOpen},
  974. {K: text.Scalar, T: ST{ok: Int64{-42}}},
  975. {K: text.Scalar, T: ST{ok: Int64{-42}}},
  976. {K: text.Scalar, T: ST{ok: Int64{-42}}},
  977. {K: text.ListClose},
  978. },
  979. },
  980. {
  981. in: `nums: [0,0x0,00,-9876543210,9876543210,0x0123456789abcdef,-0x0123456789abcdef,01234567,-01234567]`,
  982. want: []R{
  983. {K: text.Name},
  984. {K: text.ListOpen},
  985. {K: text.Scalar, T: ST{ok: Uint64{0}}},
  986. {K: text.Scalar, T: ST{ok: Int64{0}}},
  987. {K: text.Scalar, T: ST{ok: Uint64{0}}},
  988. {K: text.Scalar, T: ST{ok: Int64{-9876543210}}},
  989. {K: text.Scalar, T: ST{ok: Uint64{9876543210}}},
  990. {K: text.Scalar, T: ST{ok: Uint64{0x0123456789abcdef}}},
  991. {K: text.Scalar, T: ST{ok: Int64{-0x0123456789abcdef}}},
  992. {K: text.Scalar, T: ST{ok: Uint64{01234567}}},
  993. {K: text.Scalar, T: ST{ok: Int64{-01234567}}},
  994. {K: text.ListClose},
  995. },
  996. },
  997. {
  998. in: `nums: [0,0x0,00,-876543210,876543210,0x01234,-0x01234,01234567,-01234567]`,
  999. want: []R{
  1000. {K: text.Name},
  1001. {K: text.ListOpen},
  1002. {K: text.Scalar, T: ST{ok: Uint32{0}}},
  1003. {K: text.Scalar, T: ST{ok: Int32{0}}},
  1004. {K: text.Scalar, T: ST{ok: Uint32{0}}},
  1005. {K: text.Scalar, T: ST{ok: Int32{-876543210}}},
  1006. {K: text.Scalar, T: ST{ok: Uint32{876543210}}},
  1007. {K: text.Scalar, T: ST{ok: Uint32{0x01234}}},
  1008. {K: text.Scalar, T: ST{ok: Int32{-0x01234}}},
  1009. {K: text.Scalar, T: ST{ok: Uint32{01234567}}},
  1010. {K: text.Scalar, T: ST{ok: Int32{-01234567}}},
  1011. {K: text.ListClose},
  1012. },
  1013. },
  1014. {
  1015. in: `nums: [` +
  1016. fmt.Sprintf("%d", uint64(math.MaxUint64)) + `,` +
  1017. fmt.Sprintf("%d", uint32(math.MaxUint32)) + `,` +
  1018. fmt.Sprintf("%d", int64(math.MaxInt64)) + `,` +
  1019. fmt.Sprintf("%d", int64(math.MinInt64)) + `,` +
  1020. fmt.Sprintf("%d", int32(math.MaxInt32)) + `,` +
  1021. fmt.Sprintf("%d", int32(math.MinInt32)) +
  1022. `]`,
  1023. want: []R{
  1024. {K: text.Name},
  1025. {K: text.ListOpen},
  1026. {K: text.Scalar, T: ST{ok: Uint64{math.MaxUint64}}},
  1027. {K: text.Scalar, T: ST{ok: Uint32{math.MaxUint32}}},
  1028. {K: text.Scalar, T: ST{ok: Int64{math.MaxInt64}}},
  1029. {K: text.Scalar, T: ST{ok: Int64{math.MinInt64}}},
  1030. {K: text.Scalar, T: ST{ok: Int32{math.MaxInt32}}},
  1031. {K: text.Scalar, T: ST{ok: Int32{math.MinInt32}}},
  1032. {K: text.ListClose},
  1033. },
  1034. },
  1035. {
  1036. // Integer exceeds range.
  1037. in: `nums: [` +
  1038. `18446744073709551616,` + // max uint64 + 1
  1039. fmt.Sprintf("%d", uint64(math.MaxUint32+1)) + `,` +
  1040. fmt.Sprintf("%d", uint64(math.MaxInt64+1)) + `,` +
  1041. `-9223372036854775809,` + // min int64 - 1
  1042. fmt.Sprintf("%d", uint64(math.MaxInt32+1)) + `,` +
  1043. fmt.Sprintf("%d", int64(math.MinInt32-1)) + `` +
  1044. `]`,
  1045. want: []R{
  1046. {K: text.Name},
  1047. {K: text.ListOpen},
  1048. {K: text.Scalar, T: ST{nok: Uint64{}}},
  1049. {K: text.Scalar, T: ST{nok: Uint32{}}},
  1050. {K: text.Scalar, T: ST{nok: Int64{}}},
  1051. {K: text.Scalar, T: ST{nok: Int64{}}},
  1052. {K: text.Scalar, T: ST{nok: Int32{}}},
  1053. {K: text.Scalar, T: ST{nok: Int32{}}},
  1054. {K: text.ListClose},
  1055. },
  1056. },
  1057. {
  1058. in: `nums: [0xbeefbeef, 0xbeefbeefbeefbeef]`,
  1059. want: []R{
  1060. {K: text.Name},
  1061. {K: text.ListOpen},
  1062. {
  1063. K: text.Scalar,
  1064. T: func() ST {
  1065. if flags.ProtoLegacy {
  1066. return ST{ok: Int32{-1091584273}}
  1067. }
  1068. return ST{nok: Int32{}}
  1069. }(),
  1070. },
  1071. {
  1072. K: text.Scalar,
  1073. T: func() ST {
  1074. if flags.ProtoLegacy {
  1075. return ST{ok: Int64{-4688318750159552785}}
  1076. }
  1077. return ST{nok: Int64{}}
  1078. }(),
  1079. },
  1080. {K: text.ListClose},
  1081. },
  1082. },
  1083. {
  1084. in: `nums: [0.,0f,1f,10f,-0f,-1f,-10f,1.0,0.1e-3,1.5e+5,1e10,.0]`,
  1085. want: []R{
  1086. {K: text.Name},
  1087. {K: text.ListOpen},
  1088. {K: text.Scalar, T: ST{ok: Float64{0.0}}},
  1089. {K: text.Scalar, T: ST{ok: Float64{0.0}}},
  1090. {K: text.Scalar, T: ST{ok: Float64{1.0}}},
  1091. {K: text.Scalar, T: ST{ok: Float64{10.0}}},
  1092. {K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}},
  1093. {K: text.Scalar, T: ST{ok: Float64{-1.0}}},
  1094. {K: text.Scalar, T: ST{ok: Float64{-10.0}}},
  1095. {K: text.Scalar, T: ST{ok: Float64{1.0}}},
  1096. {K: text.Scalar, T: ST{ok: Float64{0.1e-3}}},
  1097. {K: text.Scalar, T: ST{ok: Float64{1.5e+5}}},
  1098. {K: text.Scalar, T: ST{ok: Float64{1.0e+10}}},
  1099. {K: text.Scalar, T: ST{ok: Float64{0.0}}},
  1100. {K: text.ListClose},
  1101. },
  1102. },
  1103. {
  1104. in: `nums: [0.,0f,1f,10f,-0f,-1f,-10f,1.0,0.1e-3,1.5e+5,1e10,.0]`,
  1105. want: []R{
  1106. {K: text.Name},
  1107. {K: text.ListOpen},
  1108. {K: text.Scalar, T: ST{ok: Float32{0.0}}},
  1109. {K: text.Scalar, T: ST{ok: Float32{0.0}}},
  1110. {K: text.Scalar, T: ST{ok: Float32{1.0}}},
  1111. {K: text.Scalar, T: ST{ok: Float32{10.0}}},
  1112. {K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}},
  1113. {K: text.Scalar, T: ST{ok: Float32{-1.0}}},
  1114. {K: text.Scalar, T: ST{ok: Float32{-10.0}}},
  1115. {K: text.Scalar, T: ST{ok: Float32{1.0}}},
  1116. {K: text.Scalar, T: ST{ok: Float32{0.1e-3}}},
  1117. {K: text.Scalar, T: ST{ok: Float32{1.5e+5}}},
  1118. {K: text.Scalar, T: ST{ok: Float32{1.0e+10}}},
  1119. {K: text.Scalar, T: ST{ok: Float32{0.0}}},
  1120. {K: text.ListClose},
  1121. },
  1122. },
  1123. {
  1124. in: `nums: [0.,1f,10F,1e1,1.10]`,
  1125. want: []R{
  1126. {K: text.Name},
  1127. {K: text.ListOpen},
  1128. {K: text.Scalar, T: ST{nok: Int64{}}},
  1129. {K: text.Scalar, T: ST{nok: Int64{}}},
  1130. {K: text.Scalar, T: ST{nok: Int64{}}},
  1131. {K: text.Scalar, T: ST{nok: Int64{}}},
  1132. {K: text.Scalar, T: ST{nok: Int64{}}},
  1133. {K: text.ListClose},
  1134. },
  1135. },
  1136. {
  1137. in: `nums: [0.,1f,10F,1e1,1.10]`,
  1138. want: []R{
  1139. {K: text.Name},
  1140. {K: text.ListOpen},
  1141. {K: text.Scalar, T: ST{nok: Int32{}}},
  1142. {K: text.Scalar, T: ST{nok: Int32{}}},
  1143. {K: text.Scalar, T: ST{nok: Int32{}}},
  1144. {K: text.Scalar, T: ST{nok: Int32{}}},
  1145. {K: text.Scalar, T: ST{nok: Int32{}}},
  1146. {K: text.ListClose},
  1147. },
  1148. },
  1149. {
  1150. in: `nums: [0.,1f,10F,1e1,1.10]`,
  1151. want: []R{
  1152. {K: text.Name},
  1153. {K: text.ListOpen},
  1154. {K: text.Scalar, T: ST{nok: Uint64{}}},
  1155. {K: text.Scalar, T: ST{nok: Uint64{}}},
  1156. {K: text.Scalar, T: ST{nok: Uint64{}}},
  1157. {K: text.Scalar, T: ST{nok: Uint64{}}},
  1158. {K: text.Scalar, T: ST{nok: Uint64{}}},
  1159. {K: text.ListClose},
  1160. },
  1161. },
  1162. {
  1163. in: `nums: [0.,1f,10F,1e1,1.10]`,
  1164. want: []R{
  1165. {K: text.Name},
  1166. {K: text.ListOpen},
  1167. {K: text.Scalar, T: ST{nok: Uint32{}}},
  1168. {K: text.Scalar, T: ST{nok: Uint32{}}},
  1169. {K: text.Scalar, T: ST{nok: Uint32{}}},
  1170. {K: text.Scalar, T: ST{nok: Uint32{}}},
  1171. {K: text.Scalar, T: ST{nok: Uint32{}}},
  1172. {K: text.ListClose},
  1173. },
  1174. },
  1175. {
  1176. in: `nums: [` +
  1177. fmt.Sprintf("%g", math.MaxFloat32) + `,` +
  1178. fmt.Sprintf("%g", -math.MaxFloat32) + `,` +
  1179. fmt.Sprintf("%g", math.MaxFloat32*2) + `,` +
  1180. fmt.Sprintf("%g", -math.MaxFloat32*2) + `,` +
  1181. `3.59539e+308,` + // math.MaxFloat64 * 2
  1182. `-3.59539e+308,` + // -math.MaxFloat64 * 2
  1183. fmt.Sprintf("%d000", uint64(math.MaxUint64)) +
  1184. `]`,
  1185. want: []R{
  1186. {K: text.Name},
  1187. {K: text.ListOpen},
  1188. {K: text.Scalar, T: ST{ok: Float32{float32(math.MaxFloat32)}}},
  1189. {K: text.Scalar, T: ST{ok: Float32{float32(-math.MaxFloat32)}}},
  1190. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1191. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1192. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1193. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1194. {K: text.Scalar, T: ST{ok: Float32{float32(math.MaxUint64) * 1000}}},
  1195. {K: text.ListClose},
  1196. },
  1197. },
  1198. {
  1199. in: `nums: [` +
  1200. fmt.Sprintf("%g", math.MaxFloat64) + `,` +
  1201. fmt.Sprintf("%g", -math.MaxFloat64) + `,` +
  1202. `3.59539e+308,` + // math.MaxFloat64 * 2
  1203. `-3.59539e+308,` + // -math.MaxFloat64 * 2
  1204. fmt.Sprintf("%d000", uint64(math.MaxUint64)) +
  1205. `]`,
  1206. want: []R{
  1207. {K: text.Name},
  1208. {K: text.ListOpen},
  1209. {K: text.Scalar, T: ST{ok: Float64{math.MaxFloat64}}},
  1210. {K: text.Scalar, T: ST{ok: Float64{-math.MaxFloat64}}},
  1211. {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1212. {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1213. {K: text.Scalar, T: ST{ok: Float64{float64(math.MaxUint64) * 1000}}},
  1214. {K: text.ListClose},
  1215. },
  1216. },
  1217. {
  1218. // -0 is only valid for signed types. It is not valid for unsigned types.
  1219. in: `num: [-0, -0]`,
  1220. want: []R{
  1221. {K: text.Name},
  1222. {K: text.ListOpen},
  1223. {K: text.Scalar, T: ST{nok: Uint32{}}},
  1224. {K: text.Scalar, T: ST{nok: Uint64{}}},
  1225. {K: text.ListClose},
  1226. },
  1227. },
  1228. {
  1229. // -0 is only valid for signed types. It is not valid for unsigned types.
  1230. in: `num: [-0, -0]`,
  1231. want: []R{
  1232. {K: text.Name},
  1233. {K: text.ListOpen},
  1234. {K: text.Scalar, T: ST{ok: Int32{0}}},
  1235. {K: text.Scalar, T: ST{ok: Int64{0}}},
  1236. {K: text.ListClose},
  1237. },
  1238. },
  1239. {
  1240. // Negative zeros on float64 should preserve sign bit.
  1241. in: `num: [-0, -.0]`,
  1242. want: []R{
  1243. {K: text.Name},
  1244. {K: text.ListOpen},
  1245. {K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}},
  1246. {K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}},
  1247. {K: text.ListClose},
  1248. },
  1249. },
  1250. {
  1251. // Negative zeros on float32 should preserve sign bit.
  1252. in: `num: [-0, -.0]`,
  1253. want: []R{
  1254. {K: text.Name},
  1255. {K: text.ListOpen},
  1256. {K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}},
  1257. {K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}},
  1258. {K: text.ListClose},
  1259. },
  1260. },
  1261. {
  1262. in: `num: +0`,
  1263. want: []R{
  1264. {K: text.Name},
  1265. {E: `invalid scalar value: +`},
  1266. },
  1267. },
  1268. {
  1269. in: `num: 01.1234`,
  1270. want: []R{
  1271. {K: text.Name},
  1272. {E: `invalid scalar value: 01.1234`},
  1273. },
  1274. },
  1275. {
  1276. in: `num: 0x`,
  1277. want: []R{
  1278. {K: text.Name},
  1279. {E: `invalid scalar value: 0x`},
  1280. },
  1281. },
  1282. {
  1283. in: `num: 0xX`,
  1284. want: []R{
  1285. {K: text.Name},
  1286. {E: `invalid scalar value: 0xX`},
  1287. },
  1288. },
  1289. {
  1290. in: `num: 0800`,
  1291. want: []R{
  1292. {K: text.Name},
  1293. {E: `invalid scalar value: 0800`},
  1294. },
  1295. },
  1296. {
  1297. in: `num: 1.`,
  1298. want: []R{
  1299. {K: text.Name},
  1300. {K: text.Scalar, T: ST{ok: Float32{1.0}}},
  1301. },
  1302. },
  1303. {
  1304. in: `num: -.`,
  1305. want: []R{
  1306. {K: text.Name},
  1307. {E: `invalid scalar value: -.`},
  1308. },
  1309. },
  1310. // Float special literal values, case-insensitive match.
  1311. {
  1312. in: `name:[nan, NaN, Nan, NAN]`,
  1313. want: []R{
  1314. {K: text.Name},
  1315. {K: text.ListOpen},
  1316. {K: text.Scalar, T: ST{ok: Float64{math.NaN()}}},
  1317. {K: text.Scalar, T: ST{ok: Float64{math.NaN()}}},
  1318. {K: text.Scalar, T: ST{ok: Float64{math.NaN()}}},
  1319. {K: text.Scalar, T: ST{ok: Float64{math.NaN()}}},
  1320. {K: text.ListClose},
  1321. },
  1322. },
  1323. {
  1324. in: `name:[inf, INF, infinity, Infinity, INFinity]`,
  1325. want: []R{
  1326. {K: text.Name},
  1327. {K: text.ListOpen},
  1328. {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1329. {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1330. {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1331. {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1332. {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1333. {K: text.ListClose},
  1334. },
  1335. },
  1336. {
  1337. in: `name:[-inf, -INF, -infinity, -Infinity, -INFinity]`,
  1338. want: []R{
  1339. {K: text.Name},
  1340. {K: text.ListOpen},
  1341. {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1342. {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1343. {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1344. {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1345. {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1346. {K: text.ListClose},
  1347. },
  1348. },
  1349. {
  1350. in: `name:[nan, NaN, Nan, NAN]`,
  1351. want: []R{
  1352. {K: text.Name},
  1353. {K: text.ListOpen},
  1354. {K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}},
  1355. {K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}},
  1356. {K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}},
  1357. {K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}},
  1358. {K: text.ListClose},
  1359. },
  1360. },
  1361. {
  1362. in: `name:[inf, INF, infinity, Infinity, INFinity]`,
  1363. want: []R{
  1364. {K: text.Name},
  1365. {K: text.ListOpen},
  1366. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1367. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1368. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1369. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1370. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1371. {K: text.ListClose},
  1372. },
  1373. },
  1374. {
  1375. in: `name:[-inf, -INF, -infinity, -Infinity, -INFinity]`,
  1376. want: []R{
  1377. {K: text.Name},
  1378. {K: text.ListOpen},
  1379. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1380. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1381. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1382. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1383. {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1384. {K: text.ListClose},
  1385. },
  1386. },
  1387. {
  1388. // C++ permits this, but we currently reject this. It is easy to add
  1389. // if needed.
  1390. in: `name: -nan`,
  1391. want: []R{
  1392. {K: text.Name},
  1393. {K: text.Scalar, T: ST{nok: Float64{}}},
  1394. },
  1395. },
  1396. // Messages.
  1397. {
  1398. in: `m: {}`,
  1399. want: []R{
  1400. {K: text.Name},
  1401. {K: text.MessageOpen},
  1402. {K: text.MessageClose},
  1403. {K: text.EOF},
  1404. },
  1405. },
  1406. {
  1407. in: `m: <>`,
  1408. want: []R{
  1409. {K: text.Name},
  1410. {K: text.MessageOpen},
  1411. {K: text.MessageClose},
  1412. {K: text.EOF},
  1413. },
  1414. },
  1415. {
  1416. in: space + `m {` + space + "\n# comment\n" + `}` + space,
  1417. want: []R{
  1418. {K: text.Name},
  1419. {K: text.MessageOpen},
  1420. {K: text.MessageClose},
  1421. },
  1422. },
  1423. {
  1424. in: `m { foo: < bar: "hello" > }`,
  1425. want: []R{
  1426. {K: text.Name, RS: "m"},
  1427. {K: text.MessageOpen},
  1428. {K: text.Name, RS: "foo"},
  1429. {K: text.MessageOpen},
  1430. {K: text.Name, RS: "bar"},
  1431. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  1432. {K: text.MessageClose},
  1433. {K: text.MessageClose},
  1434. },
  1435. },
  1436. {
  1437. in: `list [ <s:"hello">, {s:"world"} ]`,
  1438. want: []R{
  1439. {K: text.Name, RS: "list"},
  1440. {K: text.ListOpen},
  1441. {K: text.MessageOpen},
  1442. {K: text.Name, RS: "s"},
  1443. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  1444. {K: text.MessageClose},
  1445. {K: text.MessageOpen},
  1446. {K: text.Name, RS: "s"},
  1447. {K: text.Scalar, T: ST{ok: Str{"world"}}},
  1448. {K: text.MessageClose},
  1449. {K: text.ListClose},
  1450. {K: text.EOF},
  1451. },
  1452. },
  1453. {
  1454. in: `m: { >`,
  1455. want: []R{
  1456. {K: text.Name},
  1457. {K: text.MessageOpen},
  1458. {E: `mismatched close character '>'`},
  1459. },
  1460. },
  1461. {
  1462. in: `m: <s: "hello"}`,
  1463. want: []R{
  1464. {K: text.Name},
  1465. {K: text.MessageOpen},
  1466. {K: text.Name},
  1467. {K: text.Scalar, T: ST{ok: Str{"hello"}}},
  1468. {E: `mismatched close character '}'`},
  1469. },
  1470. },
  1471. {
  1472. in: `{}`,
  1473. want: []R{{E: `invalid field name: {`}},
  1474. },
  1475. {
  1476. in: `
  1477. m: {
  1478. foo: true;
  1479. bar: {
  1480. enum: ENUM
  1481. list: [ < >, { } ] ;
  1482. }
  1483. [qux]: "end"
  1484. }
  1485. `,
  1486. want: []R{
  1487. {K: text.Name},
  1488. {K: text.MessageOpen},
  1489. {K: text.Name, RS: "foo"},
  1490. {K: text.Scalar, T: ST{ok: Bool{true}}},
  1491. {K: text.Name, RS: "bar"},
  1492. {K: text.MessageOpen},
  1493. {K: text.Name, RS: "enum"},
  1494. {K: text.Scalar, T: ST{ok: Enum{"ENUM"}}},
  1495. {K: text.Name, RS: "list"},
  1496. {K: text.ListOpen},
  1497. {K: text.MessageOpen},
  1498. {K: text.MessageClose},
  1499. {K: text.MessageOpen},
  1500. {K: text.MessageClose},
  1501. {K: text.ListClose},
  1502. {K: text.MessageClose},
  1503. {K: text.Name, RS: "[qux]"},
  1504. {K: text.Scalar, T: ST{ok: Str{"end"}}},
  1505. {K: text.MessageClose},
  1506. {K: text.EOF},
  1507. },
  1508. },
  1509. // Other syntax errors.
  1510. {
  1511. in: "x: -",
  1512. want: []R{
  1513. {K: text.Name},
  1514. {E: `syntax error (line 1:4): invalid scalar value: -`},
  1515. },
  1516. },
  1517. {
  1518. in: "x:[\"πŸ’©\"x",
  1519. want: []R{
  1520. {K: text.Name},
  1521. {K: text.ListOpen},
  1522. {K: text.Scalar, T: ST{ok: Str{"πŸ’©"}}, P: 3},
  1523. {E: `syntax error (line 1:7)`},
  1524. },
  1525. },
  1526. {
  1527. in: "x:\n\n[\"πŸ”₯πŸ”₯πŸ”₯\"x",
  1528. want: []R{
  1529. {K: text.Name},
  1530. {K: text.ListOpen},
  1531. {K: text.Scalar, T: ST{ok: Str{"πŸ”₯πŸ”₯πŸ”₯"}}, P: 5},
  1532. {E: `syntax error (line 3:7)`},
  1533. },
  1534. },
  1535. {
  1536. // multi-rune emojis; could be column:8
  1537. in: "x:[\"πŸ‘πŸ»πŸ‘πŸΏ\"x",
  1538. want: []R{
  1539. {K: text.Name},
  1540. {K: text.ListOpen},
  1541. {K: text.Scalar, T: ST{ok: Str{"πŸ‘πŸ»πŸ‘πŸΏ"}}, P: 3},
  1542. {E: `syntax error (line 1:10)`},
  1543. },
  1544. },
  1545. }
  1546. for _, tc := range tests {
  1547. t.Run("", func(t *testing.T) {
  1548. tc := tc
  1549. in := []byte(tc.in)
  1550. dec := text.NewDecoder(in[:len(in):len(in)])
  1551. for i, want := range tc.want {
  1552. peekTok, peekErr := dec.Peek()
  1553. tok, err := dec.Read()
  1554. if err != nil {
  1555. if want.E == "" {
  1556. errorf(t, tc.in, "Read() got unexpected error: %v", err)
  1557. } else if !strings.Contains(err.Error(), want.E) {
  1558. errorf(t, tc.in, "Read() got %q, want %q", err, want.E)
  1559. }
  1560. return
  1561. }
  1562. if want.E != "" {
  1563. errorf(t, tc.in, "Read() got nil error, want %q", want.E)
  1564. return
  1565. }
  1566. gotK := tok.Kind()
  1567. if gotK != want.K {
  1568. errorf(t, tc.in, "Read() got %v, want %v", gotK, want.K)
  1569. return
  1570. }
  1571. checkToken(t, tok, i, want, tc.in)
  1572. if !cmp.Equal(tok, peekTok, cmp.Comparer(text.TokenEquals)) {
  1573. errorf(t, tc.in, "Peek() %+v != Read() token %+v", peekTok, tok)
  1574. }
  1575. if err != peekErr {
  1576. errorf(t, tc.in, "Peek() error %v != Read() error %v", err, peekErr)
  1577. }
  1578. }
  1579. })
  1580. }
  1581. }
  1582. func checkToken(t *testing.T, tok text.Token, idx int, r R, in string) {
  1583. // Validate Token.Pos() if R.P is set.
  1584. if r.P > 0 {
  1585. got := tok.Pos()
  1586. if got != r.P {
  1587. errorf(t, in, "want#%d: Token.Pos() got %v want %v", idx, got, r.P)
  1588. }
  1589. }
  1590. // Validate Token.RawString if R.RS is set.
  1591. if len(r.RS) > 0 {
  1592. got := tok.RawString()
  1593. if got != r.RS {
  1594. errorf(t, in, "want#%d: Token.RawString() got %v want %v", idx, got, r.P)
  1595. }
  1596. }
  1597. // Skip checking for Token details if r.T is not set.
  1598. if r.T == nil {
  1599. return
  1600. }
  1601. switch tok.Kind() {
  1602. case text.Name:
  1603. want := r.T.(NT)
  1604. kind := tok.NameKind()
  1605. if kind != want.K {
  1606. errorf(t, in, "want#%d: Token.NameKind() got %v want %v", idx, kind, want.K)
  1607. return
  1608. }
  1609. switch kind {
  1610. case text.IdentName:
  1611. got := tok.IdentName()
  1612. if got != want.S {
  1613. errorf(t, in, "want#%d: Token.IdentName() got %v want %v", idx, got, want.S)
  1614. }
  1615. case text.TypeName:
  1616. got := tok.TypeName()
  1617. if got != want.S {
  1618. errorf(t, in, "want#%d: Token.TypeName() got %v want %v", idx, got, want.S)
  1619. }
  1620. case text.FieldNumber:
  1621. got := tok.FieldNumber()
  1622. if got != want.N {
  1623. errorf(t, in, "want#%d: Token.FieldNumber() got %v want %v", idx, got, want.N)
  1624. }
  1625. }
  1626. case text.Scalar:
  1627. want := r.T.(ST)
  1628. if ok := want.ok; ok != nil {
  1629. if err := ok.checkOk(tok); err != "" {
  1630. errorf(t, in, "want#%d: %s", idx, err)
  1631. }
  1632. }
  1633. if nok := want.nok; nok != nil {
  1634. if err := nok.checkNok(tok); err != "" {
  1635. errorf(t, in, "want#%d: %s", idx, err)
  1636. }
  1637. }
  1638. }
  1639. }
  1640. func errorf(t *testing.T, in string, fmtStr string, args ...interface{}) {
  1641. t.Helper()
  1642. vargs := []interface{}{in}
  1643. for _, arg := range args {
  1644. vargs = append(vargs, arg)
  1645. }
  1646. t.Errorf("input:\n%s\n~end~\n"+fmtStr, vargs...)
  1647. }
  1648. func TestUnmarshalString(t *testing.T) {
  1649. tests := []struct {
  1650. in string
  1651. // want is expected string result.
  1652. want string
  1653. // err is expected error substring from calling DecodeString if set.
  1654. err string
  1655. }{
  1656. {
  1657. in: func() string {
  1658. var b []byte
  1659. for i := 0; i < utf8.RuneSelf; i++ {
  1660. switch i {
  1661. case 0, '\\', '\n', '\'': // these must be escaped, so ignore them
  1662. default:
  1663. b = append(b, byte(i))
  1664. }
  1665. }
  1666. return "'" + string(b) + "'"
  1667. }(),
  1668. want: "\x01\x02\x03\x04\x05\x06\a\b\t\v\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f",
  1669. },
  1670. {
  1671. in: "'\xde\xad\xbe\xef'",
  1672. err: `invalid UTF-8 detected`,
  1673. },
  1674. {
  1675. // Valid UTF-8 wire encoding, but sub-optimal encoding.
  1676. in: "'\xc0\x80'",
  1677. err: "invalid UTF-8 detected",
  1678. },
  1679. {
  1680. // Valid UTF-8 wire encoding, but invalid rune (surrogate pair).
  1681. in: "'\xed\xa0\x80'",
  1682. err: "invalid UTF-8 detected",
  1683. },
  1684. {
  1685. // Valid UTF-8 wire encoding, but invalid rune (above max rune).
  1686. in: "'\xf7\xbf\xbf\xbf'",
  1687. err: "invalid UTF-8 detected",
  1688. },
  1689. {
  1690. // Valid UTF-8 wire encoding of the RuneError rune.
  1691. in: "'\xef\xbf\xbd'",
  1692. want: string(utf8.RuneError),
  1693. },
  1694. {
  1695. in: "'hello\u1234world'",
  1696. want: "hello\u1234world",
  1697. },
  1698. {
  1699. in: `'\"\'\\\?\a\b\n\r\t\v\f\1\12\123\xA\xaB\x12\uAb8f\U0010FFFF'`,
  1700. want: "\"'\\?\a\b\n\r\t\v\f\x01\nS\n\xab\x12\uab8f\U0010ffff",
  1701. },
  1702. {
  1703. in: `str: '\8'`,
  1704. err: `invalid escape code "\\8" in string`,
  1705. },
  1706. {
  1707. in: `'\1x'`,
  1708. want: "\001x",
  1709. },
  1710. {
  1711. in: `'\12x'`,
  1712. want: "\012x",
  1713. },
  1714. {
  1715. in: `'\123x'`,
  1716. want: "\123x",
  1717. },
  1718. {
  1719. in: `'\1234x'`,
  1720. want: "\1234x",
  1721. },
  1722. {
  1723. in: `'\1'`,
  1724. want: "\001",
  1725. },
  1726. {
  1727. in: `'\12'`,
  1728. want: "\012",
  1729. },
  1730. {
  1731. in: `'\123'`,
  1732. want: "\123",
  1733. },
  1734. {
  1735. in: `'\1234'`,
  1736. want: "\1234",
  1737. },
  1738. {
  1739. in: `'\377'`,
  1740. want: "\377",
  1741. },
  1742. {
  1743. // Overflow octal escape.
  1744. in: `'\400'`,
  1745. err: `invalid octal escape code "\\400" in string`,
  1746. },
  1747. {
  1748. in: `'\xfx'`,
  1749. want: "\x0fx",
  1750. },
  1751. {
  1752. in: `'\xffx'`,
  1753. want: "\xffx",
  1754. },
  1755. {
  1756. in: `'\xfffx'`,
  1757. want: "\xfffx",
  1758. },
  1759. {
  1760. in: `'\xf'`,
  1761. want: "\x0f",
  1762. },
  1763. {
  1764. in: `'\xff'`,
  1765. want: "\xff",
  1766. },
  1767. {
  1768. in: `'\xfff'`,
  1769. want: "\xfff",
  1770. },
  1771. {
  1772. in: `'\xz'`,
  1773. err: `invalid hex escape code "\\x" in string`,
  1774. },
  1775. {
  1776. in: `'\uPo'`,
  1777. err: eofErr,
  1778. },
  1779. {
  1780. in: `'\uPoo'`,
  1781. err: `invalid Unicode escape code "\\uPoo'" in string`,
  1782. },
  1783. {
  1784. in: `str: '\uPoop'`,
  1785. err: `invalid Unicode escape code "\\uPoop" in string`,
  1786. },
  1787. {
  1788. // Unmatched surrogate pair.
  1789. in: `str: '\uDEAD'`,
  1790. err: `unexpected EOF`, // trying to reader other half
  1791. },
  1792. {
  1793. // Surrogate pair with invalid other half.
  1794. in: `str: '\uDEAD\u0000'`,
  1795. err: `invalid Unicode escape code "\\u0000" in string`,
  1796. },
  1797. {
  1798. // Properly matched surrogate pair.
  1799. in: `'\uD800\uDEAD'`,
  1800. want: "𐊭",
  1801. },
  1802. {
  1803. // Overflow on Unicode rune.
  1804. in: `'\U00110000'`,
  1805. err: `invalid Unicode escape code "\\U00110000" in string`,
  1806. },
  1807. {
  1808. in: `'\z'`,
  1809. err: `invalid escape code "\\z" in string`,
  1810. },
  1811. {
  1812. // Strings cannot have NUL literal since C-style strings forbid them.
  1813. in: "'\x00'",
  1814. err: `invalid character '\x00' in string`,
  1815. },
  1816. {
  1817. // Strings cannot have newline literal. The C++ permits them if an
  1818. // option is specified to allow them. In Go, we always forbid them.
  1819. in: "'\n'",
  1820. err: `invalid character '\n' in string`,
  1821. },
  1822. }
  1823. for _, tc := range tests {
  1824. t.Run("", func(t *testing.T) {
  1825. got, err := text.UnmarshalString(tc.in)
  1826. if err != nil {
  1827. if tc.err == "" {
  1828. errorf(t, tc.in, "UnmarshalString() got unexpected error: %q", err)
  1829. } else if !strings.Contains(err.Error(), tc.err) {
  1830. errorf(t, tc.in, "UnmarshalString() error got %q, want %q", err, tc.err)
  1831. }
  1832. return
  1833. }
  1834. if tc.err != "" {
  1835. errorf(t, tc.in, "UnmarshalString() got nil error, want %q", tc.err)
  1836. return
  1837. }
  1838. if got != tc.want {
  1839. errorf(t, tc.in, "UnmarshalString()\n[got]\n%s\n[want]\n%s", got, tc.want)
  1840. }
  1841. })
  1842. }
  1843. }
  1844. // Tests line and column number produced by Decoder.Position.
  1845. func TestPosition(t *testing.T) {
  1846. dec := text.NewDecoder([]byte("0123456789\n12345\n789"))
  1847. tests := []struct {
  1848. pos int
  1849. row int
  1850. col int
  1851. }{
  1852. {
  1853. pos: 0,
  1854. row: 1,
  1855. col: 1,
  1856. },
  1857. {
  1858. pos: 10,
  1859. row: 1,
  1860. col: 11,
  1861. },
  1862. {
  1863. pos: 11,
  1864. row: 2,
  1865. col: 1,
  1866. },
  1867. {
  1868. pos: 18,
  1869. row: 3,
  1870. col: 2,
  1871. },
  1872. }
  1873. for _, tc := range tests {
  1874. t.Run("", func(t *testing.T) {
  1875. row, col := dec.Position(tc.pos)
  1876. if row != tc.row || col != tc.col {
  1877. t.Errorf("Position(%d) got (%d,%d) want (%d,%d)", tc.pos, row, col, tc.row, tc.col)
  1878. }
  1879. })
  1880. }
  1881. }