readerc.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434
  1. //
  2. // Copyright (c) 2011-2019 Canonical Ltd
  3. // Copyright (c) 2006-2010 Kirill Simonov
  4. //
  5. // Permission is hereby granted, free of charge, to any person obtaining a copy of
  6. // this software and associated documentation files (the "Software"), to deal in
  7. // the Software without restriction, including without limitation the rights to
  8. // use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
  9. // of the Software, and to permit persons to whom the Software is furnished to do
  10. // so, subject to the following conditions:
  11. //
  12. // The above copyright notice and this permission notice shall be included in all
  13. // copies or substantial portions of the Software.
  14. //
  15. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. // SOFTWARE.
  22. package yaml
  23. import (
  24. "io"
  25. )
  26. // Set the reader error and return 0.
  27. func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool {
  28. parser.error = yaml_READER_ERROR
  29. parser.problem = problem
  30. parser.problem_offset = offset
  31. parser.problem_value = value
  32. return false
  33. }
  34. // Byte order marks.
  35. const (
  36. bom_UTF8 = "\xef\xbb\xbf"
  37. bom_UTF16LE = "\xff\xfe"
  38. bom_UTF16BE = "\xfe\xff"
  39. )
  40. // Determine the input stream encoding by checking the BOM symbol. If no BOM is
  41. // found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
  42. func yaml_parser_determine_encoding(parser *yaml_parser_t) bool {
  43. // Ensure that we had enough bytes in the raw buffer.
  44. for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 {
  45. if !yaml_parser_update_raw_buffer(parser) {
  46. return false
  47. }
  48. }
  49. // Determine the encoding.
  50. buf := parser.raw_buffer
  51. pos := parser.raw_buffer_pos
  52. avail := len(buf) - pos
  53. if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] {
  54. parser.encoding = yaml_UTF16LE_ENCODING
  55. parser.raw_buffer_pos += 2
  56. parser.offset += 2
  57. } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] {
  58. parser.encoding = yaml_UTF16BE_ENCODING
  59. parser.raw_buffer_pos += 2
  60. parser.offset += 2
  61. } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] {
  62. parser.encoding = yaml_UTF8_ENCODING
  63. parser.raw_buffer_pos += 3
  64. parser.offset += 3
  65. } else {
  66. parser.encoding = yaml_UTF8_ENCODING
  67. }
  68. return true
  69. }
  70. // Update the raw buffer.
  71. func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool {
  72. size_read := 0
  73. // Return if the raw buffer is full.
  74. if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) {
  75. return true
  76. }
  77. // Return on EOF.
  78. if parser.eof {
  79. return true
  80. }
  81. // Move the remaining bytes in the raw buffer to the beginning.
  82. if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) {
  83. copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:])
  84. }
  85. parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos]
  86. parser.raw_buffer_pos = 0
  87. // Call the read handler to fill the buffer.
  88. size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)])
  89. parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read]
  90. if err == io.EOF {
  91. parser.eof = true
  92. } else if err != nil {
  93. return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1)
  94. }
  95. return true
  96. }
  97. // Ensure that the buffer contains at least `length` characters.
  98. // Return true on success, false on failure.
  99. //
  100. // The length is supposed to be significantly less that the buffer size.
  101. func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool {
  102. if parser.read_handler == nil {
  103. panic("read handler must be set")
  104. }
  105. // [Go] This function was changed to guarantee the requested length size at EOF.
  106. // The fact we need to do this is pretty awful, but the description above implies
  107. // for that to be the case, and there are tests
  108. // If the EOF flag is set and the raw buffer is empty, do nothing.
  109. if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) {
  110. // [Go] ACTUALLY! Read the documentation of this function above.
  111. // This is just broken. To return true, we need to have the
  112. // given length in the buffer. Not doing that means every single
  113. // check that calls this function to make sure the buffer has a
  114. // given length is Go) panicking; or C) accessing invalid memory.
  115. //return true
  116. }
  117. // Return if the buffer contains enough characters.
  118. if parser.unread >= length {
  119. return true
  120. }
  121. // Determine the input encoding if it is not known yet.
  122. if parser.encoding == yaml_ANY_ENCODING {
  123. if !yaml_parser_determine_encoding(parser) {
  124. return false
  125. }
  126. }
  127. // Move the unread characters to the beginning of the buffer.
  128. buffer_len := len(parser.buffer)
  129. if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len {
  130. copy(parser.buffer, parser.buffer[parser.buffer_pos:])
  131. buffer_len -= parser.buffer_pos
  132. parser.buffer_pos = 0
  133. } else if parser.buffer_pos == buffer_len {
  134. buffer_len = 0
  135. parser.buffer_pos = 0
  136. }
  137. // Open the whole buffer for writing, and cut it before returning.
  138. parser.buffer = parser.buffer[:cap(parser.buffer)]
  139. // Fill the buffer until it has enough characters.
  140. first := true
  141. for parser.unread < length {
  142. // Fill the raw buffer if necessary.
  143. if !first || parser.raw_buffer_pos == len(parser.raw_buffer) {
  144. if !yaml_parser_update_raw_buffer(parser) {
  145. parser.buffer = parser.buffer[:buffer_len]
  146. return false
  147. }
  148. }
  149. first = false
  150. // Decode the raw buffer.
  151. inner:
  152. for parser.raw_buffer_pos != len(parser.raw_buffer) {
  153. var value rune
  154. var width int
  155. raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos
  156. // Decode the next character.
  157. switch parser.encoding {
  158. case yaml_UTF8_ENCODING:
  159. // Decode a UTF-8 character. Check RFC 3629
  160. // (http://www.ietf.org/rfc/rfc3629.txt) for more details.
  161. //
  162. // The following table (taken from the RFC) is used for
  163. // decoding.
  164. //
  165. // Char. number range | UTF-8 octet sequence
  166. // (hexadecimal) | (binary)
  167. // --------------------+------------------------------------
  168. // 0000 0000-0000 007F | 0xxxxxxx
  169. // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx
  170. // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
  171. // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
  172. //
  173. // Additionally, the characters in the range 0xD800-0xDFFF
  174. // are prohibited as they are reserved for use with UTF-16
  175. // surrogate pairs.
  176. // Determine the length of the UTF-8 sequence.
  177. octet := parser.raw_buffer[parser.raw_buffer_pos]
  178. switch {
  179. case octet&0x80 == 0x00:
  180. width = 1
  181. case octet&0xE0 == 0xC0:
  182. width = 2
  183. case octet&0xF0 == 0xE0:
  184. width = 3
  185. case octet&0xF8 == 0xF0:
  186. width = 4
  187. default:
  188. // The leading octet is invalid.
  189. return yaml_parser_set_reader_error(parser,
  190. "invalid leading UTF-8 octet",
  191. parser.offset, int(octet))
  192. }
  193. // Check if the raw buffer contains an incomplete character.
  194. if width > raw_unread {
  195. if parser.eof {
  196. return yaml_parser_set_reader_error(parser,
  197. "incomplete UTF-8 octet sequence",
  198. parser.offset, -1)
  199. }
  200. break inner
  201. }
  202. // Decode the leading octet.
  203. switch {
  204. case octet&0x80 == 0x00:
  205. value = rune(octet & 0x7F)
  206. case octet&0xE0 == 0xC0:
  207. value = rune(octet & 0x1F)
  208. case octet&0xF0 == 0xE0:
  209. value = rune(octet & 0x0F)
  210. case octet&0xF8 == 0xF0:
  211. value = rune(octet & 0x07)
  212. default:
  213. value = 0
  214. }
  215. // Check and decode the trailing octets.
  216. for k := 1; k < width; k++ {
  217. octet = parser.raw_buffer[parser.raw_buffer_pos+k]
  218. // Check if the octet is valid.
  219. if (octet & 0xC0) != 0x80 {
  220. return yaml_parser_set_reader_error(parser,
  221. "invalid trailing UTF-8 octet",
  222. parser.offset+k, int(octet))
  223. }
  224. // Decode the octet.
  225. value = (value << 6) + rune(octet&0x3F)
  226. }
  227. // Check the length of the sequence against the value.
  228. switch {
  229. case width == 1:
  230. case width == 2 && value >= 0x80:
  231. case width == 3 && value >= 0x800:
  232. case width == 4 && value >= 0x10000:
  233. default:
  234. return yaml_parser_set_reader_error(parser,
  235. "invalid length of a UTF-8 sequence",
  236. parser.offset, -1)
  237. }
  238. // Check the range of the value.
  239. if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF {
  240. return yaml_parser_set_reader_error(parser,
  241. "invalid Unicode character",
  242. parser.offset, int(value))
  243. }
  244. case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING:
  245. var low, high int
  246. if parser.encoding == yaml_UTF16LE_ENCODING {
  247. low, high = 0, 1
  248. } else {
  249. low, high = 1, 0
  250. }
  251. // The UTF-16 encoding is not as simple as one might
  252. // naively think. Check RFC 2781
  253. // (http://www.ietf.org/rfc/rfc2781.txt).
  254. //
  255. // Normally, two subsequent bytes describe a Unicode
  256. // character. However a special technique (called a
  257. // surrogate pair) is used for specifying character
  258. // values larger than 0xFFFF.
  259. //
  260. // A surrogate pair consists of two pseudo-characters:
  261. // high surrogate area (0xD800-0xDBFF)
  262. // low surrogate area (0xDC00-0xDFFF)
  263. //
  264. // The following formulas are used for decoding
  265. // and encoding characters using surrogate pairs:
  266. //
  267. // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF)
  268. // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF)
  269. // W1 = 110110yyyyyyyyyy
  270. // W2 = 110111xxxxxxxxxx
  271. //
  272. // where U is the character value, W1 is the high surrogate
  273. // area, W2 is the low surrogate area.
  274. // Check for incomplete UTF-16 character.
  275. if raw_unread < 2 {
  276. if parser.eof {
  277. return yaml_parser_set_reader_error(parser,
  278. "incomplete UTF-16 character",
  279. parser.offset, -1)
  280. }
  281. break inner
  282. }
  283. // Get the character.
  284. value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) +
  285. (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8)
  286. // Check for unexpected low surrogate area.
  287. if value&0xFC00 == 0xDC00 {
  288. return yaml_parser_set_reader_error(parser,
  289. "unexpected low surrogate area",
  290. parser.offset, int(value))
  291. }
  292. // Check for a high surrogate area.
  293. if value&0xFC00 == 0xD800 {
  294. width = 4
  295. // Check for incomplete surrogate pair.
  296. if raw_unread < 4 {
  297. if parser.eof {
  298. return yaml_parser_set_reader_error(parser,
  299. "incomplete UTF-16 surrogate pair",
  300. parser.offset, -1)
  301. }
  302. break inner
  303. }
  304. // Get the next character.
  305. value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) +
  306. (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8)
  307. // Check for a low surrogate area.
  308. if value2&0xFC00 != 0xDC00 {
  309. return yaml_parser_set_reader_error(parser,
  310. "expected low surrogate area",
  311. parser.offset+2, int(value2))
  312. }
  313. // Generate the value of the surrogate pair.
  314. value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF)
  315. } else {
  316. width = 2
  317. }
  318. default:
  319. panic("impossible")
  320. }
  321. // Check if the character is in the allowed range:
  322. // #x9 | #xA | #xD | [#x20-#x7E] (8 bit)
  323. // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit)
  324. // | [#x10000-#x10FFFF] (32 bit)
  325. switch {
  326. case value == 0x09:
  327. case value == 0x0A:
  328. case value == 0x0D:
  329. case value >= 0x20 && value <= 0x7E:
  330. case value == 0x85:
  331. case value >= 0xA0 && value <= 0xD7FF:
  332. case value >= 0xE000 && value <= 0xFFFD:
  333. case value >= 0x10000 && value <= 0x10FFFF:
  334. default:
  335. return yaml_parser_set_reader_error(parser,
  336. "control characters are not allowed",
  337. parser.offset, int(value))
  338. }
  339. // Move the raw pointers.
  340. parser.raw_buffer_pos += width
  341. parser.offset += width
  342. // Finally put the character into the buffer.
  343. if value <= 0x7F {
  344. // 0000 0000-0000 007F . 0xxxxxxx
  345. parser.buffer[buffer_len+0] = byte(value)
  346. buffer_len += 1
  347. } else if value <= 0x7FF {
  348. // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx
  349. parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6))
  350. parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F))
  351. buffer_len += 2
  352. } else if value <= 0xFFFF {
  353. // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx
  354. parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12))
  355. parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F))
  356. parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F))
  357. buffer_len += 3
  358. } else {
  359. // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
  360. parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18))
  361. parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F))
  362. parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F))
  363. parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F))
  364. buffer_len += 4
  365. }
  366. parser.unread++
  367. }
  368. // On EOF, put NUL into the buffer and return.
  369. if parser.eof {
  370. parser.buffer[buffer_len] = 0
  371. buffer_len++
  372. parser.unread++
  373. break
  374. }
  375. }
  376. // [Go] Read the documentation of this function above. To return true,
  377. // we need to have the given length in the buffer. Not doing that means
  378. // every single check that calls this function to make sure the buffer
  379. // has a given length is Go) panicking; or C) accessing invalid memory.
  380. // This happens here due to the EOF above breaking early.
  381. for buffer_len < length {
  382. parser.buffer[buffer_len] = 0
  383. buffer_len++
  384. }
  385. parser.buffer = parser.buffer[:buffer_len]
  386. return true
  387. }