charsets.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570
  1. /*
  2. Text conversion from one charset to another.
  3. Copyright (C) 2001-2017
  4. Free Software Foundation, Inc.
  5. Written by:
  6. Walery Studennikov <despair@sama.ru>
  7. This file is part of the Midnight Commander.
  8. The Midnight Commander is free software: you can redistribute it
  9. and/or modify it under the terms of the GNU General Public License as
  10. published by the Free Software Foundation, either version 3 of the License,
  11. or (at your option) any later version.
  12. The Midnight Commander is distributed in the hope that it will be useful,
  13. but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. GNU General Public License for more details.
  16. You should have received a copy of the GNU General Public License
  17. along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. */
  19. /** \file charsets.c
  20. * \brief Source: Text conversion from one charset to another
  21. */
  22. #include <config.h>
  23. #include <stdio.h>
  24. #include <stdlib.h>
  25. #include <string.h>
  26. #include "lib/global.h"
  27. #include "lib/strutil.h" /* utf-8 functions */
  28. #include "lib/fileloc.h"
  29. #include "lib/util.h" /* whitespace() */
  30. #include "lib/charsets.h"
  31. /*** global variables ****************************************************************************/
  32. GPtrArray *codepages = NULL;
  33. unsigned char conv_displ[256];
  34. unsigned char conv_input[256];
  35. const char *cp_display = NULL;
  36. const char *cp_source = NULL;
  37. /*** file scope macro definitions ****************************************************************/
  38. #define UNKNCHAR '\001'
  39. #define OTHER_8BIT "Other_8_bit"
  40. /*** file scope type declarations ****************************************************************/
  41. /*** file scope variables ************************************************************************/
  42. /*** file scope functions ************************************************************************/
  43. /* --------------------------------------------------------------------------------------------- */
  44. static codepage_desc *
  45. new_codepage_desc (const char *id, const char *name)
  46. {
  47. codepage_desc *desc;
  48. desc = g_new (codepage_desc, 1);
  49. desc->id = g_strdup (id);
  50. desc->name = g_strdup (name);
  51. return desc;
  52. }
  53. /* --------------------------------------------------------------------------------------------- */
  54. static void
  55. free_codepage_desc (gpointer data, gpointer user_data)
  56. {
  57. codepage_desc *desc = (codepage_desc *) data;
  58. (void) user_data;
  59. g_free (desc->id);
  60. g_free (desc->name);
  61. g_free (desc);
  62. }
  63. /* --------------------------------------------------------------------------------------------- */
  64. /* returns display codepage */
  65. static void
  66. load_codepages_list_from_file (GPtrArray ** list, const char *fname)
  67. {
  68. FILE *f;
  69. char buf[BUF_MEDIUM];
  70. char *default_codepage = NULL;
  71. f = fopen (fname, "r");
  72. if (f == NULL)
  73. return;
  74. while (fgets (buf, sizeof buf, f) != NULL)
  75. {
  76. /* split string into id and cpname */
  77. char *p = buf;
  78. size_t buflen = strlen (buf);
  79. if (*p == '\n' || *p == '\0' || *p == '#')
  80. continue;
  81. if (buflen > 0 && buf[buflen - 1] == '\n')
  82. buf[buflen - 1] = '\0';
  83. while (*p != '\0' && !whitespace (*p))
  84. ++p;
  85. if (*p == '\0')
  86. goto fail;
  87. *p++ = '\0';
  88. g_strstrip (p);
  89. if (*p == '\0')
  90. goto fail;
  91. if (strcmp (buf, "default") == 0)
  92. default_codepage = g_strdup (p);
  93. else
  94. {
  95. const char *id = buf;
  96. if (*list == NULL)
  97. {
  98. *list = g_ptr_array_sized_new (16);
  99. g_ptr_array_add (*list, new_codepage_desc (id, p));
  100. }
  101. else
  102. {
  103. unsigned int i;
  104. /* whether id is already present in list */
  105. /* if yes, overwrite description */
  106. for (i = 0; i < (*list)->len; i++)
  107. {
  108. codepage_desc *desc;
  109. desc = (codepage_desc *) g_ptr_array_index (*list, i);
  110. if (strcmp (id, desc->id) == 0)
  111. {
  112. /* found */
  113. g_free (desc->name);
  114. desc->name = g_strdup (p);
  115. break;
  116. }
  117. }
  118. /* not found */
  119. if (i == (*list)->len)
  120. g_ptr_array_add (*list, new_codepage_desc (id, p));
  121. }
  122. }
  123. }
  124. if (default_codepage != NULL)
  125. {
  126. mc_global.display_codepage = get_codepage_index (default_codepage);
  127. g_free (default_codepage);
  128. }
  129. fail:
  130. fclose (f);
  131. }
  132. /* --------------------------------------------------------------------------------------------- */
  133. static char
  134. translate_character (GIConv cd, char c)
  135. {
  136. gchar *tmp_buff = NULL;
  137. gsize bytes_read, bytes_written = 0;
  138. const char *ibuf = &c;
  139. char ch = UNKNCHAR;
  140. int ibuflen = 1;
  141. tmp_buff = g_convert_with_iconv (ibuf, ibuflen, cd, &bytes_read, &bytes_written, NULL);
  142. if (tmp_buff)
  143. ch = tmp_buff[0];
  144. g_free (tmp_buff);
  145. return ch;
  146. }
  147. /* --------------------------------------------------------------------------------------------- */
  148. /*** public functions ****************************************************************************/
  149. /* --------------------------------------------------------------------------------------------- */
  150. void
  151. load_codepages_list (void)
  152. {
  153. char *fname;
  154. /* 1: try load /usr/share/mc/mc.charsets */
  155. fname = g_build_filename (mc_global.share_data_dir, CHARSETS_LIST, (char *) NULL);
  156. load_codepages_list_from_file (&codepages, fname);
  157. g_free (fname);
  158. /* 2: try load /etc/mc/mc.charsets */
  159. fname = g_build_filename (mc_global.sysconfig_dir, CHARSETS_LIST, (char *) NULL);
  160. load_codepages_list_from_file (&codepages, fname);
  161. g_free (fname);
  162. if (codepages == NULL)
  163. {
  164. /* files are not found, add defaullt codepage */
  165. fprintf (stderr, "%s\n", _("Warning: cannot load codepages list"));
  166. codepages = g_ptr_array_new ();
  167. g_ptr_array_add (codepages, new_codepage_desc (DEFAULT_CHARSET, _("7-bit ASCII")));
  168. }
  169. }
  170. /* --------------------------------------------------------------------------------------------- */
  171. void
  172. free_codepages_list (void)
  173. {
  174. g_ptr_array_foreach (codepages, free_codepage_desc, NULL);
  175. g_ptr_array_free (codepages, TRUE);
  176. /* NULL-ize pointer to make unit tests happy */
  177. codepages = NULL;
  178. }
  179. /* --------------------------------------------------------------------------------------------- */
  180. const char *
  181. get_codepage_id (const int n)
  182. {
  183. return (n < 0) ? OTHER_8BIT : ((codepage_desc *) g_ptr_array_index (codepages, n))->id;
  184. }
  185. /* --------------------------------------------------------------------------------------------- */
  186. int
  187. get_codepage_index (const char *id)
  188. {
  189. size_t i;
  190. if (strcmp (id, OTHER_8BIT) == 0)
  191. return -1;
  192. if (codepages == NULL)
  193. return -1;
  194. for (i = 0; i < codepages->len; i++)
  195. if (strcmp (id, ((codepage_desc *) g_ptr_array_index (codepages, i))->id) == 0)
  196. return i;
  197. return -1;
  198. }
  199. /* --------------------------------------------------------------------------------------------- */
  200. /** Check if specified encoding can be used in mc.
  201. * @param encoding name of encoding
  202. * @return TRUE if encoding is supported by mc, FALSE otherwise
  203. */
  204. gboolean
  205. is_supported_encoding (const char *encoding)
  206. {
  207. gboolean result = FALSE;
  208. guint t;
  209. for (t = 0; t < codepages->len; t++)
  210. {
  211. const char *id = ((codepage_desc *) g_ptr_array_index (codepages, t))->id;
  212. result |= (g_ascii_strncasecmp (encoding, id, strlen (id)) == 0);
  213. }
  214. return result;
  215. }
  216. /* --------------------------------------------------------------------------------------------- */
  217. char *
  218. init_translation_table (int cpsource, int cpdisplay)
  219. {
  220. int i;
  221. GIConv cd;
  222. /* Fill inpit <-> display tables */
  223. if (cpsource < 0 || cpdisplay < 0 || cpsource == cpdisplay)
  224. {
  225. for (i = 0; i <= 255; ++i)
  226. {
  227. conv_displ[i] = i;
  228. conv_input[i] = i;
  229. cp_source = cp_display;
  230. }
  231. return NULL;
  232. }
  233. for (i = 0; i <= 127; ++i)
  234. {
  235. conv_displ[i] = i;
  236. conv_input[i] = i;
  237. }
  238. cp_source = ((codepage_desc *) g_ptr_array_index (codepages, cpsource))->id;
  239. cp_display = ((codepage_desc *) g_ptr_array_index (codepages, cpdisplay))->id;
  240. /* display <- inpit table */
  241. cd = g_iconv_open (cp_display, cp_source);
  242. if (cd == INVALID_CONV)
  243. return g_strdup_printf (_("Cannot translate from %s to %s"), cp_source, cp_display);
  244. for (i = 128; i <= 255; ++i)
  245. conv_displ[i] = translate_character (cd, i);
  246. g_iconv_close (cd);
  247. /* inpit <- display table */
  248. cd = g_iconv_open (cp_source, cp_display);
  249. if (cd == INVALID_CONV)
  250. return g_strdup_printf (_("Cannot translate from %s to %s"), cp_display, cp_source);
  251. for (i = 128; i <= 255; ++i)
  252. {
  253. unsigned char ch;
  254. ch = translate_character (cd, i);
  255. conv_input[i] = (ch == UNKNCHAR) ? i : ch;
  256. }
  257. g_iconv_close (cd);
  258. return NULL;
  259. }
  260. /* --------------------------------------------------------------------------------------------- */
  261. void
  262. convert_to_display (char *str)
  263. {
  264. if (!str)
  265. return;
  266. while (*str)
  267. {
  268. *str = conv_displ[(unsigned char) *str];
  269. str++;
  270. }
  271. }
  272. /* --------------------------------------------------------------------------------------------- */
  273. GString *
  274. str_convert_to_display (const char *str)
  275. {
  276. return str_nconvert_to_display (str, -1);
  277. }
  278. /* --------------------------------------------------------------------------------------------- */
  279. GString *
  280. str_nconvert_to_display (const char *str, int len)
  281. {
  282. GString *buff;
  283. GIConv conv;
  284. if (!str)
  285. return g_string_new ("");
  286. if (cp_display == cp_source)
  287. return g_string_new (str);
  288. conv = str_crt_conv_from (cp_source);
  289. buff = g_string_new ("");
  290. str_nconvert (conv, str, len, buff);
  291. str_close_conv (conv);
  292. return buff;
  293. }
  294. /* --------------------------------------------------------------------------------------------- */
  295. void
  296. convert_from_input (char *str)
  297. {
  298. if (!str)
  299. return;
  300. while (*str)
  301. {
  302. *str = conv_input[(unsigned char) *str];
  303. str++;
  304. }
  305. }
  306. /* --------------------------------------------------------------------------------------------- */
  307. GString *
  308. str_convert_to_input (const char *str)
  309. {
  310. return str_nconvert_to_input (str, -1);
  311. }
  312. /* --------------------------------------------------------------------------------------------- */
  313. GString *
  314. str_nconvert_to_input (const char *str, int len)
  315. {
  316. GString *buff;
  317. GIConv conv;
  318. if (!str)
  319. return g_string_new ("");
  320. if (cp_display == cp_source)
  321. return g_string_new (str);
  322. conv = str_crt_conv_to (cp_source);
  323. buff = g_string_new ("");
  324. str_nconvert (conv, str, len, buff);
  325. str_close_conv (conv);
  326. return buff;
  327. }
  328. /* --------------------------------------------------------------------------------------------- */
  329. unsigned char
  330. convert_from_utf_to_current (const char *str)
  331. {
  332. unsigned char buf_ch[UTF8_CHAR_LEN + 1];
  333. unsigned char ch = '.';
  334. GIConv conv;
  335. const char *cp_to;
  336. if (str == NULL)
  337. return '.';
  338. cp_to = get_codepage_id (mc_global.source_codepage);
  339. conv = str_crt_conv_to (cp_to);
  340. if (conv != INVALID_CONV)
  341. {
  342. switch (str_translate_char (conv, str, -1, (char *) buf_ch, sizeof (buf_ch)))
  343. {
  344. case ESTR_SUCCESS:
  345. ch = buf_ch[0];
  346. break;
  347. case ESTR_PROBLEM:
  348. case ESTR_FAILURE:
  349. ch = '.';
  350. break;
  351. default:
  352. break;
  353. }
  354. str_close_conv (conv);
  355. }
  356. return ch;
  357. }
  358. /* --------------------------------------------------------------------------------------------- */
  359. unsigned char
  360. convert_from_utf_to_current_c (int input_char, GIConv conv)
  361. {
  362. unsigned char str[UTF8_CHAR_LEN + 1];
  363. unsigned char buf_ch[UTF8_CHAR_LEN + 1];
  364. unsigned char ch = '.';
  365. int res;
  366. res = g_unichar_to_utf8 (input_char, (char *) str);
  367. if (res == 0)
  368. return ch;
  369. str[res] = '\0';
  370. switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
  371. {
  372. case ESTR_SUCCESS:
  373. ch = buf_ch[0];
  374. break;
  375. case ESTR_PROBLEM:
  376. case ESTR_FAILURE:
  377. ch = '.';
  378. break;
  379. default:
  380. break;
  381. }
  382. return ch;
  383. }
  384. /* --------------------------------------------------------------------------------------------- */
  385. int
  386. convert_from_8bit_to_utf_c (char input_char, GIConv conv)
  387. {
  388. unsigned char str[2];
  389. unsigned char buf_ch[UTF8_CHAR_LEN + 1];
  390. int ch;
  391. str[0] = (unsigned char) input_char;
  392. str[1] = '\0';
  393. switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
  394. {
  395. case ESTR_SUCCESS:
  396. {
  397. int res;
  398. res = g_utf8_get_char_validated ((char *) buf_ch, -1);
  399. ch = res >= 0 ? res : buf_ch[0];
  400. break;
  401. }
  402. case ESTR_PROBLEM:
  403. case ESTR_FAILURE:
  404. default:
  405. ch = '.';
  406. break;
  407. }
  408. return ch;
  409. }
  410. /* --------------------------------------------------------------------------------------------- */
  411. int
  412. convert_from_8bit_to_utf_c2 (char input_char)
  413. {
  414. unsigned char str[2];
  415. int ch = '.';
  416. GIConv conv;
  417. const char *cp_from;
  418. str[0] = (unsigned char) input_char;
  419. str[1] = '\0';
  420. cp_from = get_codepage_id (mc_global.source_codepage);
  421. conv = str_crt_conv_to (cp_from);
  422. if (conv != INVALID_CONV)
  423. {
  424. unsigned char buf_ch[UTF8_CHAR_LEN + 1];
  425. switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
  426. {
  427. case ESTR_SUCCESS:
  428. {
  429. int res;
  430. res = g_utf8_get_char_validated ((char *) buf_ch, -1);
  431. ch = res >= 0 ? res : buf_ch[0];
  432. break;
  433. }
  434. case ESTR_PROBLEM:
  435. case ESTR_FAILURE:
  436. default:
  437. ch = '.';
  438. break;
  439. }
  440. str_close_conv (conv);
  441. }
  442. return ch;
  443. }
  444. /* --------------------------------------------------------------------------------------------- */