|
|
@@ -1,10 +1,9 @@
|
|
|
+# -*- coding: utf-8 -*-
|
|
|
from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE
|
|
|
from encodings.aliases import aliases
|
|
|
from re import IGNORECASE, compile as re_compile
|
|
|
from typing import Dict, List, Set, Union
|
|
|
|
|
|
-from .assets import FREQUENCIES
|
|
|
-
|
|
|
# Contain for each eligible encoding a list of/item bytes SIG/BOM
|
|
|
ENCODING_MARKS: Dict[str, Union[bytes, List[bytes]]] = {
|
|
|
"utf_8": BOM_UTF8,
|
|
|
@@ -23,288 +22,338 @@ ENCODING_MARKS: Dict[str, Union[bytes, List[bytes]]] = {
|
|
|
TOO_SMALL_SEQUENCE: int = 32
|
|
|
TOO_BIG_SEQUENCE: int = int(10e6)
|
|
|
|
|
|
-UTF8_MAXIMAL_ALLOCATION: int = 1112064
|
|
|
+UTF8_MAXIMAL_ALLOCATION: int = 1_112_064
|
|
|
|
|
|
+# Up-to-date Unicode ucd/15.0.0
|
|
|
UNICODE_RANGES_COMBINED: Dict[str, range] = {
|
|
|
- "Control character": range(31 + 1),
|
|
|
- "Basic Latin": range(32, 127 + 1),
|
|
|
- "Latin-1 Supplement": range(128, 255 + 1),
|
|
|
- "Latin Extended-A": range(256, 383 + 1),
|
|
|
- "Latin Extended-B": range(384, 591 + 1),
|
|
|
- "IPA Extensions": range(592, 687 + 1),
|
|
|
- "Spacing Modifier Letters": range(688, 767 + 1),
|
|
|
- "Combining Diacritical Marks": range(768, 879 + 1),
|
|
|
- "Greek and Coptic": range(880, 1023 + 1),
|
|
|
- "Cyrillic": range(1024, 1279 + 1),
|
|
|
- "Cyrillic Supplement": range(1280, 1327 + 1),
|
|
|
- "Armenian": range(1328, 1423 + 1),
|
|
|
- "Hebrew": range(1424, 1535 + 1),
|
|
|
- "Arabic": range(1536, 1791 + 1),
|
|
|
- "Syriac": range(1792, 1871 + 1),
|
|
|
- "Arabic Supplement": range(1872, 1919 + 1),
|
|
|
- "Thaana": range(1920, 1983 + 1),
|
|
|
- "NKo": range(1984, 2047 + 1),
|
|
|
- "Samaritan": range(2048, 2111 + 1),
|
|
|
- "Mandaic": range(2112, 2143 + 1),
|
|
|
- "Syriac Supplement": range(2144, 2159 + 1),
|
|
|
- "Arabic Extended-A": range(2208, 2303 + 1),
|
|
|
- "Devanagari": range(2304, 2431 + 1),
|
|
|
- "Bengali": range(2432, 2559 + 1),
|
|
|
- "Gurmukhi": range(2560, 2687 + 1),
|
|
|
- "Gujarati": range(2688, 2815 + 1),
|
|
|
- "Oriya": range(2816, 2943 + 1),
|
|
|
- "Tamil": range(2944, 3071 + 1),
|
|
|
- "Telugu": range(3072, 3199 + 1),
|
|
|
- "Kannada": range(3200, 3327 + 1),
|
|
|
- "Malayalam": range(3328, 3455 + 1),
|
|
|
- "Sinhala": range(3456, 3583 + 1),
|
|
|
- "Thai": range(3584, 3711 + 1),
|
|
|
- "Lao": range(3712, 3839 + 1),
|
|
|
- "Tibetan": range(3840, 4095 + 1),
|
|
|
- "Myanmar": range(4096, 4255 + 1),
|
|
|
- "Georgian": range(4256, 4351 + 1),
|
|
|
- "Hangul Jamo": range(4352, 4607 + 1),
|
|
|
- "Ethiopic": range(4608, 4991 + 1),
|
|
|
- "Ethiopic Supplement": range(4992, 5023 + 1),
|
|
|
- "Cherokee": range(5024, 5119 + 1),
|
|
|
- "Unified Canadian Aboriginal Syllabics": range(5120, 5759 + 1),
|
|
|
- "Ogham": range(5760, 5791 + 1),
|
|
|
- "Runic": range(5792, 5887 + 1),
|
|
|
- "Tagalog": range(5888, 5919 + 1),
|
|
|
- "Hanunoo": range(5920, 5951 + 1),
|
|
|
- "Buhid": range(5952, 5983 + 1),
|
|
|
- "Tagbanwa": range(5984, 6015 + 1),
|
|
|
- "Khmer": range(6016, 6143 + 1),
|
|
|
- "Mongolian": range(6144, 6319 + 1),
|
|
|
- "Unified Canadian Aboriginal Syllabics Extended": range(6320, 6399 + 1),
|
|
|
- "Limbu": range(6400, 6479 + 1),
|
|
|
- "Tai Le": range(6480, 6527 + 1),
|
|
|
- "New Tai Lue": range(6528, 6623 + 1),
|
|
|
- "Khmer Symbols": range(6624, 6655 + 1),
|
|
|
- "Buginese": range(6656, 6687 + 1),
|
|
|
- "Tai Tham": range(6688, 6831 + 1),
|
|
|
- "Combining Diacritical Marks Extended": range(6832, 6911 + 1),
|
|
|
- "Balinese": range(6912, 7039 + 1),
|
|
|
- "Sundanese": range(7040, 7103 + 1),
|
|
|
- "Batak": range(7104, 7167 + 1),
|
|
|
- "Lepcha": range(7168, 7247 + 1),
|
|
|
- "Ol Chiki": range(7248, 7295 + 1),
|
|
|
- "Cyrillic Extended C": range(7296, 7311 + 1),
|
|
|
- "Sundanese Supplement": range(7360, 7375 + 1),
|
|
|
- "Vedic Extensions": range(7376, 7423 + 1),
|
|
|
- "Phonetic Extensions": range(7424, 7551 + 1),
|
|
|
- "Phonetic Extensions Supplement": range(7552, 7615 + 1),
|
|
|
- "Combining Diacritical Marks Supplement": range(7616, 7679 + 1),
|
|
|
- "Latin Extended Additional": range(7680, 7935 + 1),
|
|
|
- "Greek Extended": range(7936, 8191 + 1),
|
|
|
- "General Punctuation": range(8192, 8303 + 1),
|
|
|
- "Superscripts and Subscripts": range(8304, 8351 + 1),
|
|
|
- "Currency Symbols": range(8352, 8399 + 1),
|
|
|
- "Combining Diacritical Marks for Symbols": range(8400, 8447 + 1),
|
|
|
- "Letterlike Symbols": range(8448, 8527 + 1),
|
|
|
- "Number Forms": range(8528, 8591 + 1),
|
|
|
- "Arrows": range(8592, 8703 + 1),
|
|
|
- "Mathematical Operators": range(8704, 8959 + 1),
|
|
|
- "Miscellaneous Technical": range(8960, 9215 + 1),
|
|
|
- "Control Pictures": range(9216, 9279 + 1),
|
|
|
- "Optical Character Recognition": range(9280, 9311 + 1),
|
|
|
- "Enclosed Alphanumerics": range(9312, 9471 + 1),
|
|
|
- "Box Drawing": range(9472, 9599 + 1),
|
|
|
- "Block Elements": range(9600, 9631 + 1),
|
|
|
- "Geometric Shapes": range(9632, 9727 + 1),
|
|
|
- "Miscellaneous Symbols": range(9728, 9983 + 1),
|
|
|
- "Dingbats": range(9984, 10175 + 1),
|
|
|
- "Miscellaneous Mathematical Symbols-A": range(10176, 10223 + 1),
|
|
|
- "Supplemental Arrows-A": range(10224, 10239 + 1),
|
|
|
- "Braille Patterns": range(10240, 10495 + 1),
|
|
|
- "Supplemental Arrows-B": range(10496, 10623 + 1),
|
|
|
- "Miscellaneous Mathematical Symbols-B": range(10624, 10751 + 1),
|
|
|
- "Supplemental Mathematical Operators": range(10752, 11007 + 1),
|
|
|
- "Miscellaneous Symbols and Arrows": range(11008, 11263 + 1),
|
|
|
- "Glagolitic": range(11264, 11359 + 1),
|
|
|
- "Latin Extended-C": range(11360, 11391 + 1),
|
|
|
- "Coptic": range(11392, 11519 + 1),
|
|
|
- "Georgian Supplement": range(11520, 11567 + 1),
|
|
|
- "Tifinagh": range(11568, 11647 + 1),
|
|
|
- "Ethiopic Extended": range(11648, 11743 + 1),
|
|
|
- "Cyrillic Extended-A": range(11744, 11775 + 1),
|
|
|
- "Supplemental Punctuation": range(11776, 11903 + 1),
|
|
|
- "CJK Radicals Supplement": range(11904, 12031 + 1),
|
|
|
- "Kangxi Radicals": range(12032, 12255 + 1),
|
|
|
- "Ideographic Description Characters": range(12272, 12287 + 1),
|
|
|
- "CJK Symbols and Punctuation": range(12288, 12351 + 1),
|
|
|
- "Hiragana": range(12352, 12447 + 1),
|
|
|
- "Katakana": range(12448, 12543 + 1),
|
|
|
- "Bopomofo": range(12544, 12591 + 1),
|
|
|
- "Hangul Compatibility Jamo": range(12592, 12687 + 1),
|
|
|
- "Kanbun": range(12688, 12703 + 1),
|
|
|
- "Bopomofo Extended": range(12704, 12735 + 1),
|
|
|
- "CJK Strokes": range(12736, 12783 + 1),
|
|
|
- "Katakana Phonetic Extensions": range(12784, 12799 + 1),
|
|
|
- "Enclosed CJK Letters and Months": range(12800, 13055 + 1),
|
|
|
- "CJK Compatibility": range(13056, 13311 + 1),
|
|
|
- "CJK Unified Ideographs Extension A": range(13312, 19903 + 1),
|
|
|
- "Yijing Hexagram Symbols": range(19904, 19967 + 1),
|
|
|
- "CJK Unified Ideographs": range(19968, 40959 + 1),
|
|
|
- "Yi Syllables": range(40960, 42127 + 1),
|
|
|
- "Yi Radicals": range(42128, 42191 + 1),
|
|
|
- "Lisu": range(42192, 42239 + 1),
|
|
|
- "Vai": range(42240, 42559 + 1),
|
|
|
- "Cyrillic Extended-B": range(42560, 42655 + 1),
|
|
|
- "Bamum": range(42656, 42751 + 1),
|
|
|
- "Modifier Tone Letters": range(42752, 42783 + 1),
|
|
|
- "Latin Extended-D": range(42784, 43007 + 1),
|
|
|
- "Syloti Nagri": range(43008, 43055 + 1),
|
|
|
- "Common Indic Number Forms": range(43056, 43071 + 1),
|
|
|
- "Phags-pa": range(43072, 43135 + 1),
|
|
|
- "Saurashtra": range(43136, 43231 + 1),
|
|
|
- "Devanagari Extended": range(43232, 43263 + 1),
|
|
|
- "Kayah Li": range(43264, 43311 + 1),
|
|
|
- "Rejang": range(43312, 43359 + 1),
|
|
|
- "Hangul Jamo Extended-A": range(43360, 43391 + 1),
|
|
|
- "Javanese": range(43392, 43487 + 1),
|
|
|
- "Myanmar Extended-B": range(43488, 43519 + 1),
|
|
|
- "Cham": range(43520, 43615 + 1),
|
|
|
- "Myanmar Extended-A": range(43616, 43647 + 1),
|
|
|
- "Tai Viet": range(43648, 43743 + 1),
|
|
|
- "Meetei Mayek Extensions": range(43744, 43775 + 1),
|
|
|
- "Ethiopic Extended-A": range(43776, 43823 + 1),
|
|
|
- "Latin Extended-E": range(43824, 43887 + 1),
|
|
|
- "Cherokee Supplement": range(43888, 43967 + 1),
|
|
|
- "Meetei Mayek": range(43968, 44031 + 1),
|
|
|
- "Hangul Syllables": range(44032, 55215 + 1),
|
|
|
- "Hangul Jamo Extended-B": range(55216, 55295 + 1),
|
|
|
- "High Surrogates": range(55296, 56191 + 1),
|
|
|
- "High Private Use Surrogates": range(56192, 56319 + 1),
|
|
|
- "Low Surrogates": range(56320, 57343 + 1),
|
|
|
- "Private Use Area": range(57344, 63743 + 1),
|
|
|
- "CJK Compatibility Ideographs": range(63744, 64255 + 1),
|
|
|
- "Alphabetic Presentation Forms": range(64256, 64335 + 1),
|
|
|
- "Arabic Presentation Forms-A": range(64336, 65023 + 1),
|
|
|
- "Variation Selectors": range(65024, 65039 + 1),
|
|
|
- "Vertical Forms": range(65040, 65055 + 1),
|
|
|
- "Combining Half Marks": range(65056, 65071 + 1),
|
|
|
- "CJK Compatibility Forms": range(65072, 65103 + 1),
|
|
|
- "Small Form Variants": range(65104, 65135 + 1),
|
|
|
- "Arabic Presentation Forms-B": range(65136, 65279 + 1),
|
|
|
- "Halfwidth and Fullwidth Forms": range(65280, 65519 + 1),
|
|
|
- "Specials": range(65520, 65535 + 1),
|
|
|
- "Linear B Syllabary": range(65536, 65663 + 1),
|
|
|
- "Linear B Ideograms": range(65664, 65791 + 1),
|
|
|
- "Aegean Numbers": range(65792, 65855 + 1),
|
|
|
- "Ancient Greek Numbers": range(65856, 65935 + 1),
|
|
|
- "Ancient Symbols": range(65936, 65999 + 1),
|
|
|
- "Phaistos Disc": range(66000, 66047 + 1),
|
|
|
- "Lycian": range(66176, 66207 + 1),
|
|
|
- "Carian": range(66208, 66271 + 1),
|
|
|
- "Coptic Epact Numbers": range(66272, 66303 + 1),
|
|
|
- "Old Italic": range(66304, 66351 + 1),
|
|
|
- "Gothic": range(66352, 66383 + 1),
|
|
|
- "Old Permic": range(66384, 66431 + 1),
|
|
|
- "Ugaritic": range(66432, 66463 + 1),
|
|
|
- "Old Persian": range(66464, 66527 + 1),
|
|
|
- "Deseret": range(66560, 66639 + 1),
|
|
|
- "Shavian": range(66640, 66687 + 1),
|
|
|
- "Osmanya": range(66688, 66735 + 1),
|
|
|
- "Osage": range(66736, 66815 + 1),
|
|
|
- "Elbasan": range(66816, 66863 + 1),
|
|
|
- "Caucasian Albanian": range(66864, 66927 + 1),
|
|
|
- "Linear A": range(67072, 67455 + 1),
|
|
|
- "Cypriot Syllabary": range(67584, 67647 + 1),
|
|
|
- "Imperial Aramaic": range(67648, 67679 + 1),
|
|
|
- "Palmyrene": range(67680, 67711 + 1),
|
|
|
- "Nabataean": range(67712, 67759 + 1),
|
|
|
- "Hatran": range(67808, 67839 + 1),
|
|
|
- "Phoenician": range(67840, 67871 + 1),
|
|
|
- "Lydian": range(67872, 67903 + 1),
|
|
|
- "Meroitic Hieroglyphs": range(67968, 67999 + 1),
|
|
|
- "Meroitic Cursive": range(68000, 68095 + 1),
|
|
|
- "Kharoshthi": range(68096, 68191 + 1),
|
|
|
- "Old South Arabian": range(68192, 68223 + 1),
|
|
|
- "Old North Arabian": range(68224, 68255 + 1),
|
|
|
- "Manichaean": range(68288, 68351 + 1),
|
|
|
- "Avestan": range(68352, 68415 + 1),
|
|
|
- "Inscriptional Parthian": range(68416, 68447 + 1),
|
|
|
- "Inscriptional Pahlavi": range(68448, 68479 + 1),
|
|
|
- "Psalter Pahlavi": range(68480, 68527 + 1),
|
|
|
- "Old Turkic": range(68608, 68687 + 1),
|
|
|
- "Old Hungarian": range(68736, 68863 + 1),
|
|
|
- "Rumi Numeral Symbols": range(69216, 69247 + 1),
|
|
|
- "Brahmi": range(69632, 69759 + 1),
|
|
|
- "Kaithi": range(69760, 69839 + 1),
|
|
|
- "Sora Sompeng": range(69840, 69887 + 1),
|
|
|
- "Chakma": range(69888, 69967 + 1),
|
|
|
- "Mahajani": range(69968, 70015 + 1),
|
|
|
- "Sharada": range(70016, 70111 + 1),
|
|
|
- "Sinhala Archaic Numbers": range(70112, 70143 + 1),
|
|
|
- "Khojki": range(70144, 70223 + 1),
|
|
|
- "Multani": range(70272, 70319 + 1),
|
|
|
- "Khudawadi": range(70320, 70399 + 1),
|
|
|
- "Grantha": range(70400, 70527 + 1),
|
|
|
- "Newa": range(70656, 70783 + 1),
|
|
|
- "Tirhuta": range(70784, 70879 + 1),
|
|
|
- "Siddham": range(71040, 71167 + 1),
|
|
|
- "Modi": range(71168, 71263 + 1),
|
|
|
- "Mongolian Supplement": range(71264, 71295 + 1),
|
|
|
- "Takri": range(71296, 71375 + 1),
|
|
|
- "Ahom": range(71424, 71487 + 1),
|
|
|
- "Warang Citi": range(71840, 71935 + 1),
|
|
|
- "Zanabazar Square": range(72192, 72271 + 1),
|
|
|
- "Soyombo": range(72272, 72367 + 1),
|
|
|
- "Pau Cin Hau": range(72384, 72447 + 1),
|
|
|
- "Bhaiksuki": range(72704, 72815 + 1),
|
|
|
- "Marchen": range(72816, 72895 + 1),
|
|
|
- "Masaram Gondi": range(72960, 73055 + 1),
|
|
|
- "Cuneiform": range(73728, 74751 + 1),
|
|
|
- "Cuneiform Numbers and Punctuation": range(74752, 74879 + 1),
|
|
|
- "Early Dynastic Cuneiform": range(74880, 75087 + 1),
|
|
|
- "Egyptian Hieroglyphs": range(77824, 78895 + 1),
|
|
|
- "Anatolian Hieroglyphs": range(82944, 83583 + 1),
|
|
|
- "Bamum Supplement": range(92160, 92735 + 1),
|
|
|
- "Mro": range(92736, 92783 + 1),
|
|
|
- "Bassa Vah": range(92880, 92927 + 1),
|
|
|
- "Pahawh Hmong": range(92928, 93071 + 1),
|
|
|
- "Miao": range(93952, 94111 + 1),
|
|
|
- "Ideographic Symbols and Punctuation": range(94176, 94207 + 1),
|
|
|
- "Tangut": range(94208, 100351 + 1),
|
|
|
- "Tangut Components": range(100352, 101119 + 1),
|
|
|
- "Kana Supplement": range(110592, 110847 + 1),
|
|
|
- "Kana Extended-A": range(110848, 110895 + 1),
|
|
|
- "Nushu": range(110960, 111359 + 1),
|
|
|
- "Duployan": range(113664, 113823 + 1),
|
|
|
- "Shorthand Format Controls": range(113824, 113839 + 1),
|
|
|
- "Byzantine Musical Symbols": range(118784, 119039 + 1),
|
|
|
- "Musical Symbols": range(119040, 119295 + 1),
|
|
|
- "Ancient Greek Musical Notation": range(119296, 119375 + 1),
|
|
|
- "Tai Xuan Jing Symbols": range(119552, 119647 + 1),
|
|
|
- "Counting Rod Numerals": range(119648, 119679 + 1),
|
|
|
- "Mathematical Alphanumeric Symbols": range(119808, 120831 + 1),
|
|
|
- "Sutton SignWriting": range(120832, 121519 + 1),
|
|
|
- "Glagolitic Supplement": range(122880, 122927 + 1),
|
|
|
- "Mende Kikakui": range(124928, 125151 + 1),
|
|
|
- "Adlam": range(125184, 125279 + 1),
|
|
|
- "Arabic Mathematical Alphabetic Symbols": range(126464, 126719 + 1),
|
|
|
- "Mahjong Tiles": range(126976, 127023 + 1),
|
|
|
- "Domino Tiles": range(127024, 127135 + 1),
|
|
|
- "Playing Cards": range(127136, 127231 + 1),
|
|
|
- "Enclosed Alphanumeric Supplement": range(127232, 127487 + 1),
|
|
|
- "Enclosed Ideographic Supplement": range(127488, 127743 + 1),
|
|
|
- "Miscellaneous Symbols and Pictographs": range(127744, 128511 + 1),
|
|
|
- "Emoticons range(Emoji)": range(128512, 128591 + 1),
|
|
|
- "Ornamental Dingbats": range(128592, 128639 + 1),
|
|
|
- "Transport and Map Symbols": range(128640, 128767 + 1),
|
|
|
- "Alchemical Symbols": range(128768, 128895 + 1),
|
|
|
- "Geometric Shapes Extended": range(128896, 129023 + 1),
|
|
|
- "Supplemental Arrows-C": range(129024, 129279 + 1),
|
|
|
- "Supplemental Symbols and Pictographs": range(129280, 129535 + 1),
|
|
|
- "CJK Unified Ideographs Extension B": range(131072, 173791 + 1),
|
|
|
- "CJK Unified Ideographs Extension C": range(173824, 177983 + 1),
|
|
|
- "CJK Unified Ideographs Extension D": range(177984, 178207 + 1),
|
|
|
- "CJK Unified Ideographs Extension E": range(178208, 183983 + 1),
|
|
|
- "CJK Unified Ideographs Extension F": range(183984, 191471 + 1),
|
|
|
- "CJK Compatibility Ideographs Supplement": range(194560, 195103 + 1),
|
|
|
- "Tags": range(917504, 917631 + 1),
|
|
|
- "Variation Selectors Supplement": range(917760, 917999 + 1),
|
|
|
+ "Control character": range(32),
|
|
|
+ "Basic Latin": range(32, 128),
|
|
|
+ "Latin-1 Supplement": range(128, 256),
|
|
|
+ "Latin Extended-A": range(256, 384),
|
|
|
+ "Latin Extended-B": range(384, 592),
|
|
|
+ "IPA Extensions": range(592, 688),
|
|
|
+ "Spacing Modifier Letters": range(688, 768),
|
|
|
+ "Combining Diacritical Marks": range(768, 880),
|
|
|
+ "Greek and Coptic": range(880, 1024),
|
|
|
+ "Cyrillic": range(1024, 1280),
|
|
|
+ "Cyrillic Supplement": range(1280, 1328),
|
|
|
+ "Armenian": range(1328, 1424),
|
|
|
+ "Hebrew": range(1424, 1536),
|
|
|
+ "Arabic": range(1536, 1792),
|
|
|
+ "Syriac": range(1792, 1872),
|
|
|
+ "Arabic Supplement": range(1872, 1920),
|
|
|
+ "Thaana": range(1920, 1984),
|
|
|
+ "NKo": range(1984, 2048),
|
|
|
+ "Samaritan": range(2048, 2112),
|
|
|
+ "Mandaic": range(2112, 2144),
|
|
|
+ "Syriac Supplement": range(2144, 2160),
|
|
|
+ "Arabic Extended-B": range(2160, 2208),
|
|
|
+ "Arabic Extended-A": range(2208, 2304),
|
|
|
+ "Devanagari": range(2304, 2432),
|
|
|
+ "Bengali": range(2432, 2560),
|
|
|
+ "Gurmukhi": range(2560, 2688),
|
|
|
+ "Gujarati": range(2688, 2816),
|
|
|
+ "Oriya": range(2816, 2944),
|
|
|
+ "Tamil": range(2944, 3072),
|
|
|
+ "Telugu": range(3072, 3200),
|
|
|
+ "Kannada": range(3200, 3328),
|
|
|
+ "Malayalam": range(3328, 3456),
|
|
|
+ "Sinhala": range(3456, 3584),
|
|
|
+ "Thai": range(3584, 3712),
|
|
|
+ "Lao": range(3712, 3840),
|
|
|
+ "Tibetan": range(3840, 4096),
|
|
|
+ "Myanmar": range(4096, 4256),
|
|
|
+ "Georgian": range(4256, 4352),
|
|
|
+ "Hangul Jamo": range(4352, 4608),
|
|
|
+ "Ethiopic": range(4608, 4992),
|
|
|
+ "Ethiopic Supplement": range(4992, 5024),
|
|
|
+ "Cherokee": range(5024, 5120),
|
|
|
+ "Unified Canadian Aboriginal Syllabics": range(5120, 5760),
|
|
|
+ "Ogham": range(5760, 5792),
|
|
|
+ "Runic": range(5792, 5888),
|
|
|
+ "Tagalog": range(5888, 5920),
|
|
|
+ "Hanunoo": range(5920, 5952),
|
|
|
+ "Buhid": range(5952, 5984),
|
|
|
+ "Tagbanwa": range(5984, 6016),
|
|
|
+ "Khmer": range(6016, 6144),
|
|
|
+ "Mongolian": range(6144, 6320),
|
|
|
+ "Unified Canadian Aboriginal Syllabics Extended": range(6320, 6400),
|
|
|
+ "Limbu": range(6400, 6480),
|
|
|
+ "Tai Le": range(6480, 6528),
|
|
|
+ "New Tai Lue": range(6528, 6624),
|
|
|
+ "Khmer Symbols": range(6624, 6656),
|
|
|
+ "Buginese": range(6656, 6688),
|
|
|
+ "Tai Tham": range(6688, 6832),
|
|
|
+ "Combining Diacritical Marks Extended": range(6832, 6912),
|
|
|
+ "Balinese": range(6912, 7040),
|
|
|
+ "Sundanese": range(7040, 7104),
|
|
|
+ "Batak": range(7104, 7168),
|
|
|
+ "Lepcha": range(7168, 7248),
|
|
|
+ "Ol Chiki": range(7248, 7296),
|
|
|
+ "Cyrillic Extended-C": range(7296, 7312),
|
|
|
+ "Georgian Extended": range(7312, 7360),
|
|
|
+ "Sundanese Supplement": range(7360, 7376),
|
|
|
+ "Vedic Extensions": range(7376, 7424),
|
|
|
+ "Phonetic Extensions": range(7424, 7552),
|
|
|
+ "Phonetic Extensions Supplement": range(7552, 7616),
|
|
|
+ "Combining Diacritical Marks Supplement": range(7616, 7680),
|
|
|
+ "Latin Extended Additional": range(7680, 7936),
|
|
|
+ "Greek Extended": range(7936, 8192),
|
|
|
+ "General Punctuation": range(8192, 8304),
|
|
|
+ "Superscripts and Subscripts": range(8304, 8352),
|
|
|
+ "Currency Symbols": range(8352, 8400),
|
|
|
+ "Combining Diacritical Marks for Symbols": range(8400, 8448),
|
|
|
+ "Letterlike Symbols": range(8448, 8528),
|
|
|
+ "Number Forms": range(8528, 8592),
|
|
|
+ "Arrows": range(8592, 8704),
|
|
|
+ "Mathematical Operators": range(8704, 8960),
|
|
|
+ "Miscellaneous Technical": range(8960, 9216),
|
|
|
+ "Control Pictures": range(9216, 9280),
|
|
|
+ "Optical Character Recognition": range(9280, 9312),
|
|
|
+ "Enclosed Alphanumerics": range(9312, 9472),
|
|
|
+ "Box Drawing": range(9472, 9600),
|
|
|
+ "Block Elements": range(9600, 9632),
|
|
|
+ "Geometric Shapes": range(9632, 9728),
|
|
|
+ "Miscellaneous Symbols": range(9728, 9984),
|
|
|
+ "Dingbats": range(9984, 10176),
|
|
|
+ "Miscellaneous Mathematical Symbols-A": range(10176, 10224),
|
|
|
+ "Supplemental Arrows-A": range(10224, 10240),
|
|
|
+ "Braille Patterns": range(10240, 10496),
|
|
|
+ "Supplemental Arrows-B": range(10496, 10624),
|
|
|
+ "Miscellaneous Mathematical Symbols-B": range(10624, 10752),
|
|
|
+ "Supplemental Mathematical Operators": range(10752, 11008),
|
|
|
+ "Miscellaneous Symbols and Arrows": range(11008, 11264),
|
|
|
+ "Glagolitic": range(11264, 11360),
|
|
|
+ "Latin Extended-C": range(11360, 11392),
|
|
|
+ "Coptic": range(11392, 11520),
|
|
|
+ "Georgian Supplement": range(11520, 11568),
|
|
|
+ "Tifinagh": range(11568, 11648),
|
|
|
+ "Ethiopic Extended": range(11648, 11744),
|
|
|
+ "Cyrillic Extended-A": range(11744, 11776),
|
|
|
+ "Supplemental Punctuation": range(11776, 11904),
|
|
|
+ "CJK Radicals Supplement": range(11904, 12032),
|
|
|
+ "Kangxi Radicals": range(12032, 12256),
|
|
|
+ "Ideographic Description Characters": range(12272, 12288),
|
|
|
+ "CJK Symbols and Punctuation": range(12288, 12352),
|
|
|
+ "Hiragana": range(12352, 12448),
|
|
|
+ "Katakana": range(12448, 12544),
|
|
|
+ "Bopomofo": range(12544, 12592),
|
|
|
+ "Hangul Compatibility Jamo": range(12592, 12688),
|
|
|
+ "Kanbun": range(12688, 12704),
|
|
|
+ "Bopomofo Extended": range(12704, 12736),
|
|
|
+ "CJK Strokes": range(12736, 12784),
|
|
|
+ "Katakana Phonetic Extensions": range(12784, 12800),
|
|
|
+ "Enclosed CJK Letters and Months": range(12800, 13056),
|
|
|
+ "CJK Compatibility": range(13056, 13312),
|
|
|
+ "CJK Unified Ideographs Extension A": range(13312, 19904),
|
|
|
+ "Yijing Hexagram Symbols": range(19904, 19968),
|
|
|
+ "CJK Unified Ideographs": range(19968, 40960),
|
|
|
+ "Yi Syllables": range(40960, 42128),
|
|
|
+ "Yi Radicals": range(42128, 42192),
|
|
|
+ "Lisu": range(42192, 42240),
|
|
|
+ "Vai": range(42240, 42560),
|
|
|
+ "Cyrillic Extended-B": range(42560, 42656),
|
|
|
+ "Bamum": range(42656, 42752),
|
|
|
+ "Modifier Tone Letters": range(42752, 42784),
|
|
|
+ "Latin Extended-D": range(42784, 43008),
|
|
|
+ "Syloti Nagri": range(43008, 43056),
|
|
|
+ "Common Indic Number Forms": range(43056, 43072),
|
|
|
+ "Phags-pa": range(43072, 43136),
|
|
|
+ "Saurashtra": range(43136, 43232),
|
|
|
+ "Devanagari Extended": range(43232, 43264),
|
|
|
+ "Kayah Li": range(43264, 43312),
|
|
|
+ "Rejang": range(43312, 43360),
|
|
|
+ "Hangul Jamo Extended-A": range(43360, 43392),
|
|
|
+ "Javanese": range(43392, 43488),
|
|
|
+ "Myanmar Extended-B": range(43488, 43520),
|
|
|
+ "Cham": range(43520, 43616),
|
|
|
+ "Myanmar Extended-A": range(43616, 43648),
|
|
|
+ "Tai Viet": range(43648, 43744),
|
|
|
+ "Meetei Mayek Extensions": range(43744, 43776),
|
|
|
+ "Ethiopic Extended-A": range(43776, 43824),
|
|
|
+ "Latin Extended-E": range(43824, 43888),
|
|
|
+ "Cherokee Supplement": range(43888, 43968),
|
|
|
+ "Meetei Mayek": range(43968, 44032),
|
|
|
+ "Hangul Syllables": range(44032, 55216),
|
|
|
+ "Hangul Jamo Extended-B": range(55216, 55296),
|
|
|
+ "High Surrogates": range(55296, 56192),
|
|
|
+ "High Private Use Surrogates": range(56192, 56320),
|
|
|
+ "Low Surrogates": range(56320, 57344),
|
|
|
+ "Private Use Area": range(57344, 63744),
|
|
|
+ "CJK Compatibility Ideographs": range(63744, 64256),
|
|
|
+ "Alphabetic Presentation Forms": range(64256, 64336),
|
|
|
+ "Arabic Presentation Forms-A": range(64336, 65024),
|
|
|
+ "Variation Selectors": range(65024, 65040),
|
|
|
+ "Vertical Forms": range(65040, 65056),
|
|
|
+ "Combining Half Marks": range(65056, 65072),
|
|
|
+ "CJK Compatibility Forms": range(65072, 65104),
|
|
|
+ "Small Form Variants": range(65104, 65136),
|
|
|
+ "Arabic Presentation Forms-B": range(65136, 65280),
|
|
|
+ "Halfwidth and Fullwidth Forms": range(65280, 65520),
|
|
|
+ "Specials": range(65520, 65536),
|
|
|
+ "Linear B Syllabary": range(65536, 65664),
|
|
|
+ "Linear B Ideograms": range(65664, 65792),
|
|
|
+ "Aegean Numbers": range(65792, 65856),
|
|
|
+ "Ancient Greek Numbers": range(65856, 65936),
|
|
|
+ "Ancient Symbols": range(65936, 66000),
|
|
|
+ "Phaistos Disc": range(66000, 66048),
|
|
|
+ "Lycian": range(66176, 66208),
|
|
|
+ "Carian": range(66208, 66272),
|
|
|
+ "Coptic Epact Numbers": range(66272, 66304),
|
|
|
+ "Old Italic": range(66304, 66352),
|
|
|
+ "Gothic": range(66352, 66384),
|
|
|
+ "Old Permic": range(66384, 66432),
|
|
|
+ "Ugaritic": range(66432, 66464),
|
|
|
+ "Old Persian": range(66464, 66528),
|
|
|
+ "Deseret": range(66560, 66640),
|
|
|
+ "Shavian": range(66640, 66688),
|
|
|
+ "Osmanya": range(66688, 66736),
|
|
|
+ "Osage": range(66736, 66816),
|
|
|
+ "Elbasan": range(66816, 66864),
|
|
|
+ "Caucasian Albanian": range(66864, 66928),
|
|
|
+ "Vithkuqi": range(66928, 67008),
|
|
|
+ "Linear A": range(67072, 67456),
|
|
|
+ "Latin Extended-F": range(67456, 67520),
|
|
|
+ "Cypriot Syllabary": range(67584, 67648),
|
|
|
+ "Imperial Aramaic": range(67648, 67680),
|
|
|
+ "Palmyrene": range(67680, 67712),
|
|
|
+ "Nabataean": range(67712, 67760),
|
|
|
+ "Hatran": range(67808, 67840),
|
|
|
+ "Phoenician": range(67840, 67872),
|
|
|
+ "Lydian": range(67872, 67904),
|
|
|
+ "Meroitic Hieroglyphs": range(67968, 68000),
|
|
|
+ "Meroitic Cursive": range(68000, 68096),
|
|
|
+ "Kharoshthi": range(68096, 68192),
|
|
|
+ "Old South Arabian": range(68192, 68224),
|
|
|
+ "Old North Arabian": range(68224, 68256),
|
|
|
+ "Manichaean": range(68288, 68352),
|
|
|
+ "Avestan": range(68352, 68416),
|
|
|
+ "Inscriptional Parthian": range(68416, 68448),
|
|
|
+ "Inscriptional Pahlavi": range(68448, 68480),
|
|
|
+ "Psalter Pahlavi": range(68480, 68528),
|
|
|
+ "Old Turkic": range(68608, 68688),
|
|
|
+ "Old Hungarian": range(68736, 68864),
|
|
|
+ "Hanifi Rohingya": range(68864, 68928),
|
|
|
+ "Rumi Numeral Symbols": range(69216, 69248),
|
|
|
+ "Yezidi": range(69248, 69312),
|
|
|
+ "Arabic Extended-C": range(69312, 69376),
|
|
|
+ "Old Sogdian": range(69376, 69424),
|
|
|
+ "Sogdian": range(69424, 69488),
|
|
|
+ "Old Uyghur": range(69488, 69552),
|
|
|
+ "Chorasmian": range(69552, 69600),
|
|
|
+ "Elymaic": range(69600, 69632),
|
|
|
+ "Brahmi": range(69632, 69760),
|
|
|
+ "Kaithi": range(69760, 69840),
|
|
|
+ "Sora Sompeng": range(69840, 69888),
|
|
|
+ "Chakma": range(69888, 69968),
|
|
|
+ "Mahajani": range(69968, 70016),
|
|
|
+ "Sharada": range(70016, 70112),
|
|
|
+ "Sinhala Archaic Numbers": range(70112, 70144),
|
|
|
+ "Khojki": range(70144, 70224),
|
|
|
+ "Multani": range(70272, 70320),
|
|
|
+ "Khudawadi": range(70320, 70400),
|
|
|
+ "Grantha": range(70400, 70528),
|
|
|
+ "Newa": range(70656, 70784),
|
|
|
+ "Tirhuta": range(70784, 70880),
|
|
|
+ "Siddham": range(71040, 71168),
|
|
|
+ "Modi": range(71168, 71264),
|
|
|
+ "Mongolian Supplement": range(71264, 71296),
|
|
|
+ "Takri": range(71296, 71376),
|
|
|
+ "Ahom": range(71424, 71504),
|
|
|
+ "Dogra": range(71680, 71760),
|
|
|
+ "Warang Citi": range(71840, 71936),
|
|
|
+ "Dives Akuru": range(71936, 72032),
|
|
|
+ "Nandinagari": range(72096, 72192),
|
|
|
+ "Zanabazar Square": range(72192, 72272),
|
|
|
+ "Soyombo": range(72272, 72368),
|
|
|
+ "Unified Canadian Aboriginal Syllabics Extended-A": range(72368, 72384),
|
|
|
+ "Pau Cin Hau": range(72384, 72448),
|
|
|
+ "Devanagari Extended-A": range(72448, 72544),
|
|
|
+ "Bhaiksuki": range(72704, 72816),
|
|
|
+ "Marchen": range(72816, 72896),
|
|
|
+ "Masaram Gondi": range(72960, 73056),
|
|
|
+ "Gunjala Gondi": range(73056, 73136),
|
|
|
+ "Makasar": range(73440, 73472),
|
|
|
+ "Kawi": range(73472, 73568),
|
|
|
+ "Lisu Supplement": range(73648, 73664),
|
|
|
+ "Tamil Supplement": range(73664, 73728),
|
|
|
+ "Cuneiform": range(73728, 74752),
|
|
|
+ "Cuneiform Numbers and Punctuation": range(74752, 74880),
|
|
|
+ "Early Dynastic Cuneiform": range(74880, 75088),
|
|
|
+ "Cypro-Minoan": range(77712, 77824),
|
|
|
+ "Egyptian Hieroglyphs": range(77824, 78896),
|
|
|
+ "Egyptian Hieroglyph Format Controls": range(78896, 78944),
|
|
|
+ "Anatolian Hieroglyphs": range(82944, 83584),
|
|
|
+ "Bamum Supplement": range(92160, 92736),
|
|
|
+ "Mro": range(92736, 92784),
|
|
|
+ "Tangsa": range(92784, 92880),
|
|
|
+ "Bassa Vah": range(92880, 92928),
|
|
|
+ "Pahawh Hmong": range(92928, 93072),
|
|
|
+ "Medefaidrin": range(93760, 93856),
|
|
|
+ "Miao": range(93952, 94112),
|
|
|
+ "Ideographic Symbols and Punctuation": range(94176, 94208),
|
|
|
+ "Tangut": range(94208, 100352),
|
|
|
+ "Tangut Components": range(100352, 101120),
|
|
|
+ "Khitan Small Script": range(101120, 101632),
|
|
|
+ "Tangut Supplement": range(101632, 101760),
|
|
|
+ "Kana Extended-B": range(110576, 110592),
|
|
|
+ "Kana Supplement": range(110592, 110848),
|
|
|
+ "Kana Extended-A": range(110848, 110896),
|
|
|
+ "Small Kana Extension": range(110896, 110960),
|
|
|
+ "Nushu": range(110960, 111360),
|
|
|
+ "Duployan": range(113664, 113824),
|
|
|
+ "Shorthand Format Controls": range(113824, 113840),
|
|
|
+ "Znamenny Musical Notation": range(118528, 118736),
|
|
|
+ "Byzantine Musical Symbols": range(118784, 119040),
|
|
|
+ "Musical Symbols": range(119040, 119296),
|
|
|
+ "Ancient Greek Musical Notation": range(119296, 119376),
|
|
|
+ "Kaktovik Numerals": range(119488, 119520),
|
|
|
+ "Mayan Numerals": range(119520, 119552),
|
|
|
+ "Tai Xuan Jing Symbols": range(119552, 119648),
|
|
|
+ "Counting Rod Numerals": range(119648, 119680),
|
|
|
+ "Mathematical Alphanumeric Symbols": range(119808, 120832),
|
|
|
+ "Sutton SignWriting": range(120832, 121520),
|
|
|
+ "Latin Extended-G": range(122624, 122880),
|
|
|
+ "Glagolitic Supplement": range(122880, 122928),
|
|
|
+ "Cyrillic Extended-D": range(122928, 123024),
|
|
|
+ "Nyiakeng Puachue Hmong": range(123136, 123216),
|
|
|
+ "Toto": range(123536, 123584),
|
|
|
+ "Wancho": range(123584, 123648),
|
|
|
+ "Nag Mundari": range(124112, 124160),
|
|
|
+ "Ethiopic Extended-B": range(124896, 124928),
|
|
|
+ "Mende Kikakui": range(124928, 125152),
|
|
|
+ "Adlam": range(125184, 125280),
|
|
|
+ "Indic Siyaq Numbers": range(126064, 126144),
|
|
|
+ "Ottoman Siyaq Numbers": range(126208, 126288),
|
|
|
+ "Arabic Mathematical Alphabetic Symbols": range(126464, 126720),
|
|
|
+ "Mahjong Tiles": range(126976, 127024),
|
|
|
+ "Domino Tiles": range(127024, 127136),
|
|
|
+ "Playing Cards": range(127136, 127232),
|
|
|
+ "Enclosed Alphanumeric Supplement": range(127232, 127488),
|
|
|
+ "Enclosed Ideographic Supplement": range(127488, 127744),
|
|
|
+ "Miscellaneous Symbols and Pictographs": range(127744, 128512),
|
|
|
+ "Emoticons range(Emoji)": range(128512, 128592),
|
|
|
+ "Ornamental Dingbats": range(128592, 128640),
|
|
|
+ "Transport and Map Symbols": range(128640, 128768),
|
|
|
+ "Alchemical Symbols": range(128768, 128896),
|
|
|
+ "Geometric Shapes Extended": range(128896, 129024),
|
|
|
+ "Supplemental Arrows-C": range(129024, 129280),
|
|
|
+ "Supplemental Symbols and Pictographs": range(129280, 129536),
|
|
|
+ "Chess Symbols": range(129536, 129648),
|
|
|
+ "Symbols and Pictographs Extended-A": range(129648, 129792),
|
|
|
+ "Symbols for Legacy Computing": range(129792, 130048),
|
|
|
+ "CJK Unified Ideographs Extension B": range(131072, 173792),
|
|
|
+ "CJK Unified Ideographs Extension C": range(173824, 177984),
|
|
|
+ "CJK Unified Ideographs Extension D": range(177984, 178208),
|
|
|
+ "CJK Unified Ideographs Extension E": range(178208, 183984),
|
|
|
+ "CJK Unified Ideographs Extension F": range(183984, 191472),
|
|
|
+ "CJK Compatibility Ideographs Supplement": range(194560, 195104),
|
|
|
+ "CJK Unified Ideographs Extension G": range(196608, 201552),
|
|
|
+ "CJK Unified Ideographs Extension H": range(201552, 205744),
|
|
|
+ "Tags": range(917504, 917632),
|
|
|
+ "Variation Selectors Supplement": range(917760, 918000),
|
|
|
+ "Supplementary Private Use Area-A": range(983040, 1048576),
|
|
|
+ "Supplementary Private Use Area-B": range(1048576, 1114112),
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -331,11 +380,23 @@ RE_POSSIBLE_ENCODING_INDICATION = re_compile(
|
|
|
IGNORECASE,
|
|
|
)
|
|
|
|
|
|
+IANA_NO_ALIASES = [
|
|
|
+ "cp720",
|
|
|
+ "cp737",
|
|
|
+ "cp856",
|
|
|
+ "cp874",
|
|
|
+ "cp875",
|
|
|
+ "cp1006",
|
|
|
+ "koi8_r",
|
|
|
+ "koi8_t",
|
|
|
+ "koi8_u",
|
|
|
+]
|
|
|
+
|
|
|
IANA_SUPPORTED: List[str] = sorted(
|
|
|
filter(
|
|
|
lambda x: x.endswith("_codec") is False
|
|
|
and x not in {"rot_13", "tactis", "mbcs"},
|
|
|
- list(set(aliases.values())),
|
|
|
+ list(set(aliases.values())) + IANA_NO_ALIASES,
|
|
|
)
|
|
|
)
|
|
|
|
|
|
@@ -489,7 +550,1446 @@ COMMON_SAFE_ASCII_CHARACTERS: Set[str] = {
|
|
|
KO_NAMES: Set[str] = {"johab", "cp949", "euc_kr"}
|
|
|
ZH_NAMES: Set[str] = {"big5", "cp950", "big5hkscs", "hz"}
|
|
|
|
|
|
-LANGUAGE_SUPPORTED_COUNT: int = len(FREQUENCIES)
|
|
|
-
|
|
|
# Logging LEVEL below DEBUG
|
|
|
TRACE: int = 5
|
|
|
+
|
|
|
+
|
|
|
+# Language label that contain the em dash "—"
|
|
|
+# character are to be considered alternative seq to origin
|
|
|
+FREQUENCIES: Dict[str, List[str]] = {
|
|
|
+ "English": [
|
|
|
+ "e",
|
|
|
+ "a",
|
|
|
+ "t",
|
|
|
+ "i",
|
|
|
+ "o",
|
|
|
+ "n",
|
|
|
+ "s",
|
|
|
+ "r",
|
|
|
+ "h",
|
|
|
+ "l",
|
|
|
+ "d",
|
|
|
+ "c",
|
|
|
+ "u",
|
|
|
+ "m",
|
|
|
+ "f",
|
|
|
+ "p",
|
|
|
+ "g",
|
|
|
+ "w",
|
|
|
+ "y",
|
|
|
+ "b",
|
|
|
+ "v",
|
|
|
+ "k",
|
|
|
+ "x",
|
|
|
+ "j",
|
|
|
+ "z",
|
|
|
+ "q",
|
|
|
+ ],
|
|
|
+ "English—": [
|
|
|
+ "e",
|
|
|
+ "a",
|
|
|
+ "t",
|
|
|
+ "i",
|
|
|
+ "o",
|
|
|
+ "n",
|
|
|
+ "s",
|
|
|
+ "r",
|
|
|
+ "h",
|
|
|
+ "l",
|
|
|
+ "d",
|
|
|
+ "c",
|
|
|
+ "m",
|
|
|
+ "u",
|
|
|
+ "f",
|
|
|
+ "p",
|
|
|
+ "g",
|
|
|
+ "w",
|
|
|
+ "b",
|
|
|
+ "y",
|
|
|
+ "v",
|
|
|
+ "k",
|
|
|
+ "j",
|
|
|
+ "x",
|
|
|
+ "z",
|
|
|
+ "q",
|
|
|
+ ],
|
|
|
+ "German": [
|
|
|
+ "e",
|
|
|
+ "n",
|
|
|
+ "i",
|
|
|
+ "r",
|
|
|
+ "s",
|
|
|
+ "t",
|
|
|
+ "a",
|
|
|
+ "d",
|
|
|
+ "h",
|
|
|
+ "u",
|
|
|
+ "l",
|
|
|
+ "g",
|
|
|
+ "o",
|
|
|
+ "c",
|
|
|
+ "m",
|
|
|
+ "b",
|
|
|
+ "f",
|
|
|
+ "k",
|
|
|
+ "w",
|
|
|
+ "z",
|
|
|
+ "p",
|
|
|
+ "v",
|
|
|
+ "ü",
|
|
|
+ "ä",
|
|
|
+ "ö",
|
|
|
+ "j",
|
|
|
+ ],
|
|
|
+ "French": [
|
|
|
+ "e",
|
|
|
+ "a",
|
|
|
+ "s",
|
|
|
+ "n",
|
|
|
+ "i",
|
|
|
+ "t",
|
|
|
+ "r",
|
|
|
+ "l",
|
|
|
+ "u",
|
|
|
+ "o",
|
|
|
+ "d",
|
|
|
+ "c",
|
|
|
+ "p",
|
|
|
+ "m",
|
|
|
+ "é",
|
|
|
+ "v",
|
|
|
+ "g",
|
|
|
+ "f",
|
|
|
+ "b",
|
|
|
+ "h",
|
|
|
+ "q",
|
|
|
+ "à",
|
|
|
+ "x",
|
|
|
+ "è",
|
|
|
+ "y",
|
|
|
+ "j",
|
|
|
+ ],
|
|
|
+ "Dutch": [
|
|
|
+ "e",
|
|
|
+ "n",
|
|
|
+ "a",
|
|
|
+ "i",
|
|
|
+ "r",
|
|
|
+ "t",
|
|
|
+ "o",
|
|
|
+ "d",
|
|
|
+ "s",
|
|
|
+ "l",
|
|
|
+ "g",
|
|
|
+ "h",
|
|
|
+ "v",
|
|
|
+ "m",
|
|
|
+ "u",
|
|
|
+ "k",
|
|
|
+ "c",
|
|
|
+ "p",
|
|
|
+ "b",
|
|
|
+ "w",
|
|
|
+ "j",
|
|
|
+ "z",
|
|
|
+ "f",
|
|
|
+ "y",
|
|
|
+ "x",
|
|
|
+ "ë",
|
|
|
+ ],
|
|
|
+ "Italian": [
|
|
|
+ "e",
|
|
|
+ "i",
|
|
|
+ "a",
|
|
|
+ "o",
|
|
|
+ "n",
|
|
|
+ "l",
|
|
|
+ "t",
|
|
|
+ "r",
|
|
|
+ "s",
|
|
|
+ "c",
|
|
|
+ "d",
|
|
|
+ "u",
|
|
|
+ "p",
|
|
|
+ "m",
|
|
|
+ "g",
|
|
|
+ "v",
|
|
|
+ "f",
|
|
|
+ "b",
|
|
|
+ "z",
|
|
|
+ "h",
|
|
|
+ "q",
|
|
|
+ "è",
|
|
|
+ "à",
|
|
|
+ "k",
|
|
|
+ "y",
|
|
|
+ "ò",
|
|
|
+ ],
|
|
|
+ "Polish": [
|
|
|
+ "a",
|
|
|
+ "i",
|
|
|
+ "o",
|
|
|
+ "e",
|
|
|
+ "n",
|
|
|
+ "r",
|
|
|
+ "z",
|
|
|
+ "w",
|
|
|
+ "s",
|
|
|
+ "c",
|
|
|
+ "t",
|
|
|
+ "k",
|
|
|
+ "y",
|
|
|
+ "d",
|
|
|
+ "p",
|
|
|
+ "m",
|
|
|
+ "u",
|
|
|
+ "l",
|
|
|
+ "j",
|
|
|
+ "ł",
|
|
|
+ "g",
|
|
|
+ "b",
|
|
|
+ "h",
|
|
|
+ "ą",
|
|
|
+ "ę",
|
|
|
+ "ó",
|
|
|
+ ],
|
|
|
+ "Spanish": [
|
|
|
+ "e",
|
|
|
+ "a",
|
|
|
+ "o",
|
|
|
+ "n",
|
|
|
+ "s",
|
|
|
+ "r",
|
|
|
+ "i",
|
|
|
+ "l",
|
|
|
+ "d",
|
|
|
+ "t",
|
|
|
+ "c",
|
|
|
+ "u",
|
|
|
+ "m",
|
|
|
+ "p",
|
|
|
+ "b",
|
|
|
+ "g",
|
|
|
+ "v",
|
|
|
+ "f",
|
|
|
+ "y",
|
|
|
+ "ó",
|
|
|
+ "h",
|
|
|
+ "q",
|
|
|
+ "í",
|
|
|
+ "j",
|
|
|
+ "z",
|
|
|
+ "á",
|
|
|
+ ],
|
|
|
+ "Russian": [
|
|
|
+ "о",
|
|
|
+ "а",
|
|
|
+ "е",
|
|
|
+ "и",
|
|
|
+ "н",
|
|
|
+ "с",
|
|
|
+ "т",
|
|
|
+ "р",
|
|
|
+ "в",
|
|
|
+ "л",
|
|
|
+ "к",
|
|
|
+ "м",
|
|
|
+ "д",
|
|
|
+ "п",
|
|
|
+ "у",
|
|
|
+ "г",
|
|
|
+ "я",
|
|
|
+ "ы",
|
|
|
+ "з",
|
|
|
+ "б",
|
|
|
+ "й",
|
|
|
+ "ь",
|
|
|
+ "ч",
|
|
|
+ "х",
|
|
|
+ "ж",
|
|
|
+ "ц",
|
|
|
+ ],
|
|
|
+ # Jap-Kanji
|
|
|
+ "Japanese": [
|
|
|
+ "人",
|
|
|
+ "一",
|
|
|
+ "大",
|
|
|
+ "亅",
|
|
|
+ "丁",
|
|
|
+ "丨",
|
|
|
+ "竹",
|
|
|
+ "笑",
|
|
|
+ "口",
|
|
|
+ "日",
|
|
|
+ "今",
|
|
|
+ "二",
|
|
|
+ "彳",
|
|
|
+ "行",
|
|
|
+ "十",
|
|
|
+ "土",
|
|
|
+ "丶",
|
|
|
+ "寸",
|
|
|
+ "寺",
|
|
|
+ "時",
|
|
|
+ "乙",
|
|
|
+ "丿",
|
|
|
+ "乂",
|
|
|
+ "气",
|
|
|
+ "気",
|
|
|
+ "冂",
|
|
|
+ "巾",
|
|
|
+ "亠",
|
|
|
+ "市",
|
|
|
+ "目",
|
|
|
+ "儿",
|
|
|
+ "見",
|
|
|
+ "八",
|
|
|
+ "小",
|
|
|
+ "凵",
|
|
|
+ "県",
|
|
|
+ "月",
|
|
|
+ "彐",
|
|
|
+ "門",
|
|
|
+ "間",
|
|
|
+ "木",
|
|
|
+ "東",
|
|
|
+ "山",
|
|
|
+ "出",
|
|
|
+ "本",
|
|
|
+ "中",
|
|
|
+ "刀",
|
|
|
+ "分",
|
|
|
+ "耳",
|
|
|
+ "又",
|
|
|
+ "取",
|
|
|
+ "最",
|
|
|
+ "言",
|
|
|
+ "田",
|
|
|
+ "心",
|
|
|
+ "思",
|
|
|
+ "刂",
|
|
|
+ "前",
|
|
|
+ "京",
|
|
|
+ "尹",
|
|
|
+ "事",
|
|
|
+ "生",
|
|
|
+ "厶",
|
|
|
+ "云",
|
|
|
+ "会",
|
|
|
+ "未",
|
|
|
+ "来",
|
|
|
+ "白",
|
|
|
+ "冫",
|
|
|
+ "楽",
|
|
|
+ "灬",
|
|
|
+ "馬",
|
|
|
+ "尸",
|
|
|
+ "尺",
|
|
|
+ "駅",
|
|
|
+ "明",
|
|
|
+ "耂",
|
|
|
+ "者",
|
|
|
+ "了",
|
|
|
+ "阝",
|
|
|
+ "都",
|
|
|
+ "高",
|
|
|
+ "卜",
|
|
|
+ "占",
|
|
|
+ "厂",
|
|
|
+ "广",
|
|
|
+ "店",
|
|
|
+ "子",
|
|
|
+ "申",
|
|
|
+ "奄",
|
|
|
+ "亻",
|
|
|
+ "俺",
|
|
|
+ "上",
|
|
|
+ "方",
|
|
|
+ "冖",
|
|
|
+ "学",
|
|
|
+ "衣",
|
|
|
+ "艮",
|
|
|
+ "食",
|
|
|
+ "自",
|
|
|
+ ],
|
|
|
+ # Jap-Katakana
|
|
|
+ "Japanese—": [
|
|
|
+ "ー",
|
|
|
+ "ン",
|
|
|
+ "ス",
|
|
|
+ "・",
|
|
|
+ "ル",
|
|
|
+ "ト",
|
|
|
+ "リ",
|
|
|
+ "イ",
|
|
|
+ "ア",
|
|
|
+ "ラ",
|
|
|
+ "ッ",
|
|
|
+ "ク",
|
|
|
+ "ド",
|
|
|
+ "シ",
|
|
|
+ "レ",
|
|
|
+ "ジ",
|
|
|
+ "タ",
|
|
|
+ "フ",
|
|
|
+ "ロ",
|
|
|
+ "カ",
|
|
|
+ "テ",
|
|
|
+ "マ",
|
|
|
+ "ィ",
|
|
|
+ "グ",
|
|
|
+ "バ",
|
|
|
+ "ム",
|
|
|
+ "プ",
|
|
|
+ "オ",
|
|
|
+ "コ",
|
|
|
+ "デ",
|
|
|
+ "ニ",
|
|
|
+ "ウ",
|
|
|
+ "メ",
|
|
|
+ "サ",
|
|
|
+ "ビ",
|
|
|
+ "ナ",
|
|
|
+ "ブ",
|
|
|
+ "ャ",
|
|
|
+ "エ",
|
|
|
+ "ュ",
|
|
|
+ "チ",
|
|
|
+ "キ",
|
|
|
+ "ズ",
|
|
|
+ "ダ",
|
|
|
+ "パ",
|
|
|
+ "ミ",
|
|
|
+ "ェ",
|
|
|
+ "ョ",
|
|
|
+ "ハ",
|
|
|
+ "セ",
|
|
|
+ "ベ",
|
|
|
+ "ガ",
|
|
|
+ "モ",
|
|
|
+ "ツ",
|
|
|
+ "ネ",
|
|
|
+ "ボ",
|
|
|
+ "ソ",
|
|
|
+ "ノ",
|
|
|
+ "ァ",
|
|
|
+ "ヴ",
|
|
|
+ "ワ",
|
|
|
+ "ポ",
|
|
|
+ "ペ",
|
|
|
+ "ピ",
|
|
|
+ "ケ",
|
|
|
+ "ゴ",
|
|
|
+ "ギ",
|
|
|
+ "ザ",
|
|
|
+ "ホ",
|
|
|
+ "ゲ",
|
|
|
+ "ォ",
|
|
|
+ "ヤ",
|
|
|
+ "ヒ",
|
|
|
+ "ユ",
|
|
|
+ "ヨ",
|
|
|
+ "ヘ",
|
|
|
+ "ゼ",
|
|
|
+ "ヌ",
|
|
|
+ "ゥ",
|
|
|
+ "ゾ",
|
|
|
+ "ヶ",
|
|
|
+ "ヂ",
|
|
|
+ "ヲ",
|
|
|
+ "ヅ",
|
|
|
+ "ヵ",
|
|
|
+ "ヱ",
|
|
|
+ "ヰ",
|
|
|
+ "ヮ",
|
|
|
+ "ヽ",
|
|
|
+ "゠",
|
|
|
+ "ヾ",
|
|
|
+ "ヷ",
|
|
|
+ "ヿ",
|
|
|
+ "ヸ",
|
|
|
+ "ヹ",
|
|
|
+ "ヺ",
|
|
|
+ ],
|
|
|
+ # Jap-Hiragana
|
|
|
+ "Japanese——": [
|
|
|
+ "の",
|
|
|
+ "に",
|
|
|
+ "る",
|
|
|
+ "た",
|
|
|
+ "と",
|
|
|
+ "は",
|
|
|
+ "し",
|
|
|
+ "い",
|
|
|
+ "を",
|
|
|
+ "で",
|
|
|
+ "て",
|
|
|
+ "が",
|
|
|
+ "な",
|
|
|
+ "れ",
|
|
|
+ "か",
|
|
|
+ "ら",
|
|
|
+ "さ",
|
|
|
+ "っ",
|
|
|
+ "り",
|
|
|
+ "す",
|
|
|
+ "あ",
|
|
|
+ "も",
|
|
|
+ "こ",
|
|
|
+ "ま",
|
|
|
+ "う",
|
|
|
+ "く",
|
|
|
+ "よ",
|
|
|
+ "き",
|
|
|
+ "ん",
|
|
|
+ "め",
|
|
|
+ "お",
|
|
|
+ "け",
|
|
|
+ "そ",
|
|
|
+ "つ",
|
|
|
+ "だ",
|
|
|
+ "や",
|
|
|
+ "え",
|
|
|
+ "ど",
|
|
|
+ "わ",
|
|
|
+ "ち",
|
|
|
+ "み",
|
|
|
+ "せ",
|
|
|
+ "じ",
|
|
|
+ "ば",
|
|
|
+ "へ",
|
|
|
+ "び",
|
|
|
+ "ず",
|
|
|
+ "ろ",
|
|
|
+ "ほ",
|
|
|
+ "げ",
|
|
|
+ "む",
|
|
|
+ "べ",
|
|
|
+ "ひ",
|
|
|
+ "ょ",
|
|
|
+ "ゆ",
|
|
|
+ "ぶ",
|
|
|
+ "ご",
|
|
|
+ "ゃ",
|
|
|
+ "ね",
|
|
|
+ "ふ",
|
|
|
+ "ぐ",
|
|
|
+ "ぎ",
|
|
|
+ "ぼ",
|
|
|
+ "ゅ",
|
|
|
+ "づ",
|
|
|
+ "ざ",
|
|
|
+ "ぞ",
|
|
|
+ "ぬ",
|
|
|
+ "ぜ",
|
|
|
+ "ぱ",
|
|
|
+ "ぽ",
|
|
|
+ "ぷ",
|
|
|
+ "ぴ",
|
|
|
+ "ぃ",
|
|
|
+ "ぁ",
|
|
|
+ "ぇ",
|
|
|
+ "ぺ",
|
|
|
+ "ゞ",
|
|
|
+ "ぢ",
|
|
|
+ "ぉ",
|
|
|
+ "ぅ",
|
|
|
+ "ゐ",
|
|
|
+ "ゝ",
|
|
|
+ "ゑ",
|
|
|
+ "゛",
|
|
|
+ "゜",
|
|
|
+ "ゎ",
|
|
|
+ "ゔ",
|
|
|
+ "゚",
|
|
|
+ "ゟ",
|
|
|
+ "゙",
|
|
|
+ "ゕ",
|
|
|
+ "ゖ",
|
|
|
+ ],
|
|
|
+ "Portuguese": [
|
|
|
+ "a",
|
|
|
+ "e",
|
|
|
+ "o",
|
|
|
+ "s",
|
|
|
+ "i",
|
|
|
+ "r",
|
|
|
+ "d",
|
|
|
+ "n",
|
|
|
+ "t",
|
|
|
+ "m",
|
|
|
+ "u",
|
|
|
+ "c",
|
|
|
+ "l",
|
|
|
+ "p",
|
|
|
+ "g",
|
|
|
+ "v",
|
|
|
+ "b",
|
|
|
+ "f",
|
|
|
+ "h",
|
|
|
+ "ã",
|
|
|
+ "q",
|
|
|
+ "é",
|
|
|
+ "ç",
|
|
|
+ "á",
|
|
|
+ "z",
|
|
|
+ "í",
|
|
|
+ ],
|
|
|
+ "Swedish": [
|
|
|
+ "e",
|
|
|
+ "a",
|
|
|
+ "n",
|
|
|
+ "r",
|
|
|
+ "t",
|
|
|
+ "s",
|
|
|
+ "i",
|
|
|
+ "l",
|
|
|
+ "d",
|
|
|
+ "o",
|
|
|
+ "m",
|
|
|
+ "k",
|
|
|
+ "g",
|
|
|
+ "v",
|
|
|
+ "h",
|
|
|
+ "f",
|
|
|
+ "u",
|
|
|
+ "p",
|
|
|
+ "ä",
|
|
|
+ "c",
|
|
|
+ "b",
|
|
|
+ "ö",
|
|
|
+ "å",
|
|
|
+ "y",
|
|
|
+ "j",
|
|
|
+ "x",
|
|
|
+ ],
|
|
|
+ "Chinese": [
|
|
|
+ "的",
|
|
|
+ "一",
|
|
|
+ "是",
|
|
|
+ "不",
|
|
|
+ "了",
|
|
|
+ "在",
|
|
|
+ "人",
|
|
|
+ "有",
|
|
|
+ "我",
|
|
|
+ "他",
|
|
|
+ "这",
|
|
|
+ "个",
|
|
|
+ "们",
|
|
|
+ "中",
|
|
|
+ "来",
|
|
|
+ "上",
|
|
|
+ "大",
|
|
|
+ "为",
|
|
|
+ "和",
|
|
|
+ "国",
|
|
|
+ "地",
|
|
|
+ "到",
|
|
|
+ "以",
|
|
|
+ "说",
|
|
|
+ "时",
|
|
|
+ "要",
|
|
|
+ "就",
|
|
|
+ "出",
|
|
|
+ "会",
|
|
|
+ "可",
|
|
|
+ "也",
|
|
|
+ "你",
|
|
|
+ "对",
|
|
|
+ "生",
|
|
|
+ "能",
|
|
|
+ "而",
|
|
|
+ "子",
|
|
|
+ "那",
|
|
|
+ "得",
|
|
|
+ "于",
|
|
|
+ "着",
|
|
|
+ "下",
|
|
|
+ "自",
|
|
|
+ "之",
|
|
|
+ "年",
|
|
|
+ "过",
|
|
|
+ "发",
|
|
|
+ "后",
|
|
|
+ "作",
|
|
|
+ "里",
|
|
|
+ "用",
|
|
|
+ "道",
|
|
|
+ "行",
|
|
|
+ "所",
|
|
|
+ "然",
|
|
|
+ "家",
|
|
|
+ "种",
|
|
|
+ "事",
|
|
|
+ "成",
|
|
|
+ "方",
|
|
|
+ "多",
|
|
|
+ "经",
|
|
|
+ "么",
|
|
|
+ "去",
|
|
|
+ "法",
|
|
|
+ "学",
|
|
|
+ "如",
|
|
|
+ "都",
|
|
|
+ "同",
|
|
|
+ "现",
|
|
|
+ "当",
|
|
|
+ "没",
|
|
|
+ "动",
|
|
|
+ "面",
|
|
|
+ "起",
|
|
|
+ "看",
|
|
|
+ "定",
|
|
|
+ "天",
|
|
|
+ "分",
|
|
|
+ "还",
|
|
|
+ "进",
|
|
|
+ "好",
|
|
|
+ "小",
|
|
|
+ "部",
|
|
|
+ "其",
|
|
|
+ "些",
|
|
|
+ "主",
|
|
|
+ "样",
|
|
|
+ "理",
|
|
|
+ "心",
|
|
|
+ "她",
|
|
|
+ "本",
|
|
|
+ "前",
|
|
|
+ "开",
|
|
|
+ "但",
|
|
|
+ "因",
|
|
|
+ "只",
|
|
|
+ "从",
|
|
|
+ "想",
|
|
|
+ "实",
|
|
|
+ ],
|
|
|
+ "Ukrainian": [
|
|
|
+ "о",
|
|
|
+ "а",
|
|
|
+ "н",
|
|
|
+ "і",
|
|
|
+ "и",
|
|
|
+ "р",
|
|
|
+ "в",
|
|
|
+ "т",
|
|
|
+ "е",
|
|
|
+ "с",
|
|
|
+ "к",
|
|
|
+ "л",
|
|
|
+ "у",
|
|
|
+ "д",
|
|
|
+ "м",
|
|
|
+ "п",
|
|
|
+ "з",
|
|
|
+ "я",
|
|
|
+ "ь",
|
|
|
+ "б",
|
|
|
+ "г",
|
|
|
+ "й",
|
|
|
+ "ч",
|
|
|
+ "х",
|
|
|
+ "ц",
|
|
|
+ "ї",
|
|
|
+ ],
|
|
|
+ "Norwegian": [
|
|
|
+ "e",
|
|
|
+ "r",
|
|
|
+ "n",
|
|
|
+ "t",
|
|
|
+ "a",
|
|
|
+ "s",
|
|
|
+ "i",
|
|
|
+ "o",
|
|
|
+ "l",
|
|
|
+ "d",
|
|
|
+ "g",
|
|
|
+ "k",
|
|
|
+ "m",
|
|
|
+ "v",
|
|
|
+ "f",
|
|
|
+ "p",
|
|
|
+ "u",
|
|
|
+ "b",
|
|
|
+ "h",
|
|
|
+ "å",
|
|
|
+ "y",
|
|
|
+ "j",
|
|
|
+ "ø",
|
|
|
+ "c",
|
|
|
+ "æ",
|
|
|
+ "w",
|
|
|
+ ],
|
|
|
+ "Finnish": [
|
|
|
+ "a",
|
|
|
+ "i",
|
|
|
+ "n",
|
|
|
+ "t",
|
|
|
+ "e",
|
|
|
+ "s",
|
|
|
+ "l",
|
|
|
+ "o",
|
|
|
+ "u",
|
|
|
+ "k",
|
|
|
+ "ä",
|
|
|
+ "m",
|
|
|
+ "r",
|
|
|
+ "v",
|
|
|
+ "j",
|
|
|
+ "h",
|
|
|
+ "p",
|
|
|
+ "y",
|
|
|
+ "d",
|
|
|
+ "ö",
|
|
|
+ "g",
|
|
|
+ "c",
|
|
|
+ "b",
|
|
|
+ "f",
|
|
|
+ "w",
|
|
|
+ "z",
|
|
|
+ ],
|
|
|
+ "Vietnamese": [
|
|
|
+ "n",
|
|
|
+ "h",
|
|
|
+ "t",
|
|
|
+ "i",
|
|
|
+ "c",
|
|
|
+ "g",
|
|
|
+ "a",
|
|
|
+ "o",
|
|
|
+ "u",
|
|
|
+ "m",
|
|
|
+ "l",
|
|
|
+ "r",
|
|
|
+ "à",
|
|
|
+ "đ",
|
|
|
+ "s",
|
|
|
+ "e",
|
|
|
+ "v",
|
|
|
+ "p",
|
|
|
+ "b",
|
|
|
+ "y",
|
|
|
+ "ư",
|
|
|
+ "d",
|
|
|
+ "á",
|
|
|
+ "k",
|
|
|
+ "ộ",
|
|
|
+ "ế",
|
|
|
+ ],
|
|
|
+ "Czech": [
|
|
|
+ "o",
|
|
|
+ "e",
|
|
|
+ "a",
|
|
|
+ "n",
|
|
|
+ "t",
|
|
|
+ "s",
|
|
|
+ "i",
|
|
|
+ "l",
|
|
|
+ "v",
|
|
|
+ "r",
|
|
|
+ "k",
|
|
|
+ "d",
|
|
|
+ "u",
|
|
|
+ "m",
|
|
|
+ "p",
|
|
|
+ "í",
|
|
|
+ "c",
|
|
|
+ "h",
|
|
|
+ "z",
|
|
|
+ "á",
|
|
|
+ "y",
|
|
|
+ "j",
|
|
|
+ "b",
|
|
|
+ "ě",
|
|
|
+ "é",
|
|
|
+ "ř",
|
|
|
+ ],
|
|
|
+ "Hungarian": [
|
|
|
+ "e",
|
|
|
+ "a",
|
|
|
+ "t",
|
|
|
+ "l",
|
|
|
+ "s",
|
|
|
+ "n",
|
|
|
+ "k",
|
|
|
+ "r",
|
|
|
+ "i",
|
|
|
+ "o",
|
|
|
+ "z",
|
|
|
+ "á",
|
|
|
+ "é",
|
|
|
+ "g",
|
|
|
+ "m",
|
|
|
+ "b",
|
|
|
+ "y",
|
|
|
+ "v",
|
|
|
+ "d",
|
|
|
+ "h",
|
|
|
+ "u",
|
|
|
+ "p",
|
|
|
+ "j",
|
|
|
+ "ö",
|
|
|
+ "f",
|
|
|
+ "c",
|
|
|
+ ],
|
|
|
+ "Korean": [
|
|
|
+ "이",
|
|
|
+ "다",
|
|
|
+ "에",
|
|
|
+ "의",
|
|
|
+ "는",
|
|
|
+ "로",
|
|
|
+ "하",
|
|
|
+ "을",
|
|
|
+ "가",
|
|
|
+ "고",
|
|
|
+ "지",
|
|
|
+ "서",
|
|
|
+ "한",
|
|
|
+ "은",
|
|
|
+ "기",
|
|
|
+ "으",
|
|
|
+ "년",
|
|
|
+ "대",
|
|
|
+ "사",
|
|
|
+ "시",
|
|
|
+ "를",
|
|
|
+ "리",
|
|
|
+ "도",
|
|
|
+ "인",
|
|
|
+ "스",
|
|
|
+ "일",
|
|
|
+ ],
|
|
|
+ "Indonesian": [
|
|
|
+ "a",
|
|
|
+ "n",
|
|
|
+ "e",
|
|
|
+ "i",
|
|
|
+ "r",
|
|
|
+ "t",
|
|
|
+ "u",
|
|
|
+ "s",
|
|
|
+ "d",
|
|
|
+ "k",
|
|
|
+ "m",
|
|
|
+ "l",
|
|
|
+ "g",
|
|
|
+ "p",
|
|
|
+ "b",
|
|
|
+ "o",
|
|
|
+ "h",
|
|
|
+ "y",
|
|
|
+ "j",
|
|
|
+ "c",
|
|
|
+ "w",
|
|
|
+ "f",
|
|
|
+ "v",
|
|
|
+ "z",
|
|
|
+ "x",
|
|
|
+ "q",
|
|
|
+ ],
|
|
|
+ "Turkish": [
|
|
|
+ "a",
|
|
|
+ "e",
|
|
|
+ "i",
|
|
|
+ "n",
|
|
|
+ "r",
|
|
|
+ "l",
|
|
|
+ "ı",
|
|
|
+ "k",
|
|
|
+ "d",
|
|
|
+ "t",
|
|
|
+ "s",
|
|
|
+ "m",
|
|
|
+ "y",
|
|
|
+ "u",
|
|
|
+ "o",
|
|
|
+ "b",
|
|
|
+ "ü",
|
|
|
+ "ş",
|
|
|
+ "v",
|
|
|
+ "g",
|
|
|
+ "z",
|
|
|
+ "h",
|
|
|
+ "c",
|
|
|
+ "p",
|
|
|
+ "ç",
|
|
|
+ "ğ",
|
|
|
+ ],
|
|
|
+ "Romanian": [
|
|
|
+ "e",
|
|
|
+ "i",
|
|
|
+ "a",
|
|
|
+ "r",
|
|
|
+ "n",
|
|
|
+ "t",
|
|
|
+ "u",
|
|
|
+ "l",
|
|
|
+ "o",
|
|
|
+ "c",
|
|
|
+ "s",
|
|
|
+ "d",
|
|
|
+ "p",
|
|
|
+ "m",
|
|
|
+ "ă",
|
|
|
+ "f",
|
|
|
+ "v",
|
|
|
+ "î",
|
|
|
+ "g",
|
|
|
+ "b",
|
|
|
+ "ș",
|
|
|
+ "ț",
|
|
|
+ "z",
|
|
|
+ "h",
|
|
|
+ "â",
|
|
|
+ "j",
|
|
|
+ ],
|
|
|
+ "Farsi": [
|
|
|
+ "ا",
|
|
|
+ "ی",
|
|
|
+ "ر",
|
|
|
+ "د",
|
|
|
+ "ن",
|
|
|
+ "ه",
|
|
|
+ "و",
|
|
|
+ "م",
|
|
|
+ "ت",
|
|
|
+ "ب",
|
|
|
+ "س",
|
|
|
+ "ل",
|
|
|
+ "ک",
|
|
|
+ "ش",
|
|
|
+ "ز",
|
|
|
+ "ف",
|
|
|
+ "گ",
|
|
|
+ "ع",
|
|
|
+ "خ",
|
|
|
+ "ق",
|
|
|
+ "ج",
|
|
|
+ "آ",
|
|
|
+ "پ",
|
|
|
+ "ح",
|
|
|
+ "ط",
|
|
|
+ "ص",
|
|
|
+ ],
|
|
|
+ "Arabic": [
|
|
|
+ "ا",
|
|
|
+ "ل",
|
|
|
+ "ي",
|
|
|
+ "م",
|
|
|
+ "و",
|
|
|
+ "ن",
|
|
|
+ "ر",
|
|
|
+ "ت",
|
|
|
+ "ب",
|
|
|
+ "ة",
|
|
|
+ "ع",
|
|
|
+ "د",
|
|
|
+ "س",
|
|
|
+ "ف",
|
|
|
+ "ه",
|
|
|
+ "ك",
|
|
|
+ "ق",
|
|
|
+ "أ",
|
|
|
+ "ح",
|
|
|
+ "ج",
|
|
|
+ "ش",
|
|
|
+ "ط",
|
|
|
+ "ص",
|
|
|
+ "ى",
|
|
|
+ "خ",
|
|
|
+ "إ",
|
|
|
+ ],
|
|
|
+ "Danish": [
|
|
|
+ "e",
|
|
|
+ "r",
|
|
|
+ "n",
|
|
|
+ "t",
|
|
|
+ "a",
|
|
|
+ "i",
|
|
|
+ "s",
|
|
|
+ "d",
|
|
|
+ "l",
|
|
|
+ "o",
|
|
|
+ "g",
|
|
|
+ "m",
|
|
|
+ "k",
|
|
|
+ "f",
|
|
|
+ "v",
|
|
|
+ "u",
|
|
|
+ "b",
|
|
|
+ "h",
|
|
|
+ "p",
|
|
|
+ "å",
|
|
|
+ "y",
|
|
|
+ "ø",
|
|
|
+ "æ",
|
|
|
+ "c",
|
|
|
+ "j",
|
|
|
+ "w",
|
|
|
+ ],
|
|
|
+ "Serbian": [
|
|
|
+ "а",
|
|
|
+ "и",
|
|
|
+ "о",
|
|
|
+ "е",
|
|
|
+ "н",
|
|
|
+ "р",
|
|
|
+ "с",
|
|
|
+ "у",
|
|
|
+ "т",
|
|
|
+ "к",
|
|
|
+ "ј",
|
|
|
+ "в",
|
|
|
+ "д",
|
|
|
+ "м",
|
|
|
+ "п",
|
|
|
+ "л",
|
|
|
+ "г",
|
|
|
+ "з",
|
|
|
+ "б",
|
|
|
+ "a",
|
|
|
+ "i",
|
|
|
+ "e",
|
|
|
+ "o",
|
|
|
+ "n",
|
|
|
+ "ц",
|
|
|
+ "ш",
|
|
|
+ ],
|
|
|
+ "Lithuanian": [
|
|
|
+ "i",
|
|
|
+ "a",
|
|
|
+ "s",
|
|
|
+ "o",
|
|
|
+ "r",
|
|
|
+ "e",
|
|
|
+ "t",
|
|
|
+ "n",
|
|
|
+ "u",
|
|
|
+ "k",
|
|
|
+ "m",
|
|
|
+ "l",
|
|
|
+ "p",
|
|
|
+ "v",
|
|
|
+ "d",
|
|
|
+ "j",
|
|
|
+ "g",
|
|
|
+ "ė",
|
|
|
+ "b",
|
|
|
+ "y",
|
|
|
+ "ų",
|
|
|
+ "š",
|
|
|
+ "ž",
|
|
|
+ "c",
|
|
|
+ "ą",
|
|
|
+ "į",
|
|
|
+ ],
|
|
|
+ "Slovene": [
|
|
|
+ "e",
|
|
|
+ "a",
|
|
|
+ "i",
|
|
|
+ "o",
|
|
|
+ "n",
|
|
|
+ "r",
|
|
|
+ "s",
|
|
|
+ "l",
|
|
|
+ "t",
|
|
|
+ "j",
|
|
|
+ "v",
|
|
|
+ "k",
|
|
|
+ "d",
|
|
|
+ "p",
|
|
|
+ "m",
|
|
|
+ "u",
|
|
|
+ "z",
|
|
|
+ "b",
|
|
|
+ "g",
|
|
|
+ "h",
|
|
|
+ "č",
|
|
|
+ "c",
|
|
|
+ "š",
|
|
|
+ "ž",
|
|
|
+ "f",
|
|
|
+ "y",
|
|
|
+ ],
|
|
|
+ "Slovak": [
|
|
|
+ "o",
|
|
|
+ "a",
|
|
|
+ "e",
|
|
|
+ "n",
|
|
|
+ "i",
|
|
|
+ "r",
|
|
|
+ "v",
|
|
|
+ "t",
|
|
|
+ "s",
|
|
|
+ "l",
|
|
|
+ "k",
|
|
|
+ "d",
|
|
|
+ "m",
|
|
|
+ "p",
|
|
|
+ "u",
|
|
|
+ "c",
|
|
|
+ "h",
|
|
|
+ "j",
|
|
|
+ "b",
|
|
|
+ "z",
|
|
|
+ "á",
|
|
|
+ "y",
|
|
|
+ "ý",
|
|
|
+ "í",
|
|
|
+ "č",
|
|
|
+ "é",
|
|
|
+ ],
|
|
|
+ "Hebrew": [
|
|
|
+ "י",
|
|
|
+ "ו",
|
|
|
+ "ה",
|
|
|
+ "ל",
|
|
|
+ "ר",
|
|
|
+ "ב",
|
|
|
+ "ת",
|
|
|
+ "מ",
|
|
|
+ "א",
|
|
|
+ "ש",
|
|
|
+ "נ",
|
|
|
+ "ע",
|
|
|
+ "ם",
|
|
|
+ "ד",
|
|
|
+ "ק",
|
|
|
+ "ח",
|
|
|
+ "פ",
|
|
|
+ "ס",
|
|
|
+ "כ",
|
|
|
+ "ג",
|
|
|
+ "ט",
|
|
|
+ "צ",
|
|
|
+ "ן",
|
|
|
+ "ז",
|
|
|
+ "ך",
|
|
|
+ ],
|
|
|
+ "Bulgarian": [
|
|
|
+ "а",
|
|
|
+ "и",
|
|
|
+ "о",
|
|
|
+ "е",
|
|
|
+ "н",
|
|
|
+ "т",
|
|
|
+ "р",
|
|
|
+ "с",
|
|
|
+ "в",
|
|
|
+ "л",
|
|
|
+ "к",
|
|
|
+ "д",
|
|
|
+ "п",
|
|
|
+ "м",
|
|
|
+ "з",
|
|
|
+ "г",
|
|
|
+ "я",
|
|
|
+ "ъ",
|
|
|
+ "у",
|
|
|
+ "б",
|
|
|
+ "ч",
|
|
|
+ "ц",
|
|
|
+ "й",
|
|
|
+ "ж",
|
|
|
+ "щ",
|
|
|
+ "х",
|
|
|
+ ],
|
|
|
+ "Croatian": [
|
|
|
+ "a",
|
|
|
+ "i",
|
|
|
+ "o",
|
|
|
+ "e",
|
|
|
+ "n",
|
|
|
+ "r",
|
|
|
+ "j",
|
|
|
+ "s",
|
|
|
+ "t",
|
|
|
+ "u",
|
|
|
+ "k",
|
|
|
+ "l",
|
|
|
+ "v",
|
|
|
+ "d",
|
|
|
+ "m",
|
|
|
+ "p",
|
|
|
+ "g",
|
|
|
+ "z",
|
|
|
+ "b",
|
|
|
+ "c",
|
|
|
+ "č",
|
|
|
+ "h",
|
|
|
+ "š",
|
|
|
+ "ž",
|
|
|
+ "ć",
|
|
|
+ "f",
|
|
|
+ ],
|
|
|
+ "Hindi": [
|
|
|
+ "क",
|
|
|
+ "र",
|
|
|
+ "स",
|
|
|
+ "न",
|
|
|
+ "त",
|
|
|
+ "म",
|
|
|
+ "ह",
|
|
|
+ "प",
|
|
|
+ "य",
|
|
|
+ "ल",
|
|
|
+ "व",
|
|
|
+ "ज",
|
|
|
+ "द",
|
|
|
+ "ग",
|
|
|
+ "ब",
|
|
|
+ "श",
|
|
|
+ "ट",
|
|
|
+ "अ",
|
|
|
+ "ए",
|
|
|
+ "थ",
|
|
|
+ "भ",
|
|
|
+ "ड",
|
|
|
+ "च",
|
|
|
+ "ध",
|
|
|
+ "ष",
|
|
|
+ "इ",
|
|
|
+ ],
|
|
|
+ "Estonian": [
|
|
|
+ "a",
|
|
|
+ "i",
|
|
|
+ "e",
|
|
|
+ "s",
|
|
|
+ "t",
|
|
|
+ "l",
|
|
|
+ "u",
|
|
|
+ "n",
|
|
|
+ "o",
|
|
|
+ "k",
|
|
|
+ "r",
|
|
|
+ "d",
|
|
|
+ "m",
|
|
|
+ "v",
|
|
|
+ "g",
|
|
|
+ "p",
|
|
|
+ "j",
|
|
|
+ "h",
|
|
|
+ "ä",
|
|
|
+ "b",
|
|
|
+ "õ",
|
|
|
+ "ü",
|
|
|
+ "f",
|
|
|
+ "c",
|
|
|
+ "ö",
|
|
|
+ "y",
|
|
|
+ ],
|
|
|
+ "Thai": [
|
|
|
+ "า",
|
|
|
+ "น",
|
|
|
+ "ร",
|
|
|
+ "อ",
|
|
|
+ "ก",
|
|
|
+ "เ",
|
|
|
+ "ง",
|
|
|
+ "ม",
|
|
|
+ "ย",
|
|
|
+ "ล",
|
|
|
+ "ว",
|
|
|
+ "ด",
|
|
|
+ "ท",
|
|
|
+ "ส",
|
|
|
+ "ต",
|
|
|
+ "ะ",
|
|
|
+ "ป",
|
|
|
+ "บ",
|
|
|
+ "ค",
|
|
|
+ "ห",
|
|
|
+ "แ",
|
|
|
+ "จ",
|
|
|
+ "พ",
|
|
|
+ "ช",
|
|
|
+ "ข",
|
|
|
+ "ใ",
|
|
|
+ ],
|
|
|
+ "Greek": [
|
|
|
+ "α",
|
|
|
+ "τ",
|
|
|
+ "ο",
|
|
|
+ "ι",
|
|
|
+ "ε",
|
|
|
+ "ν",
|
|
|
+ "ρ",
|
|
|
+ "σ",
|
|
|
+ "κ",
|
|
|
+ "η",
|
|
|
+ "π",
|
|
|
+ "ς",
|
|
|
+ "υ",
|
|
|
+ "μ",
|
|
|
+ "λ",
|
|
|
+ "ί",
|
|
|
+ "ό",
|
|
|
+ "ά",
|
|
|
+ "γ",
|
|
|
+ "έ",
|
|
|
+ "δ",
|
|
|
+ "ή",
|
|
|
+ "ω",
|
|
|
+ "χ",
|
|
|
+ "θ",
|
|
|
+ "ύ",
|
|
|
+ ],
|
|
|
+ "Tamil": [
|
|
|
+ "க",
|
|
|
+ "த",
|
|
|
+ "ப",
|
|
|
+ "ட",
|
|
|
+ "ர",
|
|
|
+ "ம",
|
|
|
+ "ல",
|
|
|
+ "ன",
|
|
|
+ "வ",
|
|
|
+ "ற",
|
|
|
+ "ய",
|
|
|
+ "ள",
|
|
|
+ "ச",
|
|
|
+ "ந",
|
|
|
+ "இ",
|
|
|
+ "ண",
|
|
|
+ "அ",
|
|
|
+ "ஆ",
|
|
|
+ "ழ",
|
|
|
+ "ங",
|
|
|
+ "எ",
|
|
|
+ "உ",
|
|
|
+ "ஒ",
|
|
|
+ "ஸ",
|
|
|
+ ],
|
|
|
+ "Kazakh": [
|
|
|
+ "а",
|
|
|
+ "ы",
|
|
|
+ "е",
|
|
|
+ "н",
|
|
|
+ "т",
|
|
|
+ "р",
|
|
|
+ "л",
|
|
|
+ "і",
|
|
|
+ "д",
|
|
|
+ "с",
|
|
|
+ "м",
|
|
|
+ "қ",
|
|
|
+ "к",
|
|
|
+ "о",
|
|
|
+ "б",
|
|
|
+ "и",
|
|
|
+ "у",
|
|
|
+ "ғ",
|
|
|
+ "ж",
|
|
|
+ "ң",
|
|
|
+ "з",
|
|
|
+ "ш",
|
|
|
+ "й",
|
|
|
+ "п",
|
|
|
+ "г",
|
|
|
+ "ө",
|
|
|
+ ],
|
|
|
+}
|
|
|
+
|
|
|
+LANGUAGE_SUPPORTED_COUNT: int = len(FREQUENCIES)
|
robot-contrib