charsets.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575
  1. /*
  2. Text conversion from one charset to another.
  3. Copyright (C) 2001, 2011
  4. The Free Software Foundation, Inc.
  5. Written by:
  6. Walery Studennikov <despair@sama.ru>
  7. This file is part of the Midnight Commander.
  8. The Midnight Commander is free software: you can redistribute it
  9. and/or modify it under the terms of the GNU General Public License as
  10. published by the Free Software Foundation, either version 3 of the License,
  11. or (at your option) any later version.
  12. The Midnight Commander is distributed in the hope that it will be useful,
  13. but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. GNU General Public License for more details.
  16. You should have received a copy of the GNU General Public License
  17. along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. */
  19. /** \file charsets.c
  20. * \brief Source: Text conversion from one charset to another
  21. */
  22. #include <config.h>
  23. #ifdef HAVE_CHARSET
  24. #include <stdio.h>
  25. #include <stdlib.h>
  26. #include <string.h>
  27. #include "lib/global.h"
  28. #include "lib/strutil.h" /* utf-8 functions */
  29. #include "lib/fileloc.h"
  30. #include "lib/charsets.h"
  31. /*** global variables ****************************************************************************/
  32. GPtrArray *codepages = NULL;
  33. unsigned char conv_displ[256];
  34. unsigned char conv_input[256];
  35. const char *cp_display = NULL;
  36. const char *cp_source = NULL;
  37. /*** file scope macro definitions ****************************************************************/
  38. #define OTHER_8BIT "Other_8_bit"
  39. /*
  40. * FIXME: This assumes that ASCII is always the first encoding
  41. * in mc.charsets
  42. */
  43. #define CP_ASCII 0
  44. /*** file scope type declarations ****************************************************************/
  45. /*** file scope variables ************************************************************************/
  46. /*** file scope functions ************************************************************************/
  47. /* --------------------------------------------------------------------------------------------- */
  48. static codepage_desc *
  49. new_codepage_desc (const char *id, const char *name)
  50. {
  51. codepage_desc *desc;
  52. desc = g_new (codepage_desc, 1);
  53. desc->id = g_strdup (id);
  54. desc->name = g_strdup (name);
  55. return desc;
  56. }
  57. /* --------------------------------------------------------------------------------------------- */
  58. static void
  59. free_codepage_desc (gpointer data, gpointer user_data)
  60. {
  61. codepage_desc *desc = (codepage_desc *) data;
  62. (void) user_data;
  63. g_free (desc->id);
  64. g_free (desc->name);
  65. g_free (desc);
  66. }
  67. /* --------------------------------------------------------------------------------------------- */
  68. /* returns display codepage */
  69. static void
  70. load_codepages_list_from_file (GPtrArray ** list, const char *fname)
  71. {
  72. FILE *f;
  73. guint i;
  74. char buf[BUF_MEDIUM];
  75. char *default_codepage = NULL;
  76. f = fopen (fname, "r");
  77. if (f == NULL)
  78. return;
  79. for (i = 0; fgets (buf, sizeof buf, f) != NULL;)
  80. {
  81. /* split string into id and cpname */
  82. char *p = buf;
  83. size_t buflen = strlen (buf);
  84. if (*p == '\n' || *p == '\0' || *p == '#')
  85. continue;
  86. if (buflen > 0 && buf[buflen - 1] == '\n')
  87. buf[buflen - 1] = '\0';
  88. while (*p != '\t' && *p != ' ' && *p != '\0')
  89. ++p;
  90. if (*p == '\0')
  91. goto fail;
  92. *p++ = '\0';
  93. g_strstrip (p);
  94. if (*p == '\0')
  95. goto fail;
  96. if (strcmp (buf, "default") == 0)
  97. default_codepage = g_strdup (p);
  98. else
  99. {
  100. const char *id = buf;
  101. if (*list == NULL)
  102. {
  103. *list = g_ptr_array_sized_new (16);
  104. g_ptr_array_add (*list, new_codepage_desc (id, p));
  105. }
  106. else
  107. {
  108. /* whether id is already present in list */
  109. /* if yes, overwrite description */
  110. for (i = 0; i < (*list)->len; i++)
  111. {
  112. codepage_desc *desc;
  113. desc = (codepage_desc *) g_ptr_array_index (*list, i);
  114. if (strcmp (id, desc->id) == 0)
  115. {
  116. /* found */
  117. g_free (desc->name);
  118. desc->name = g_strdup (p);
  119. break;
  120. }
  121. }
  122. /* not found */
  123. if (i == (*list)->len)
  124. g_ptr_array_add (*list, new_codepage_desc (id, p));
  125. }
  126. }
  127. }
  128. if (default_codepage != NULL)
  129. {
  130. mc_global.display_codepage = get_codepage_index (default_codepage);
  131. g_free (default_codepage);
  132. }
  133. fail:
  134. fclose (f);
  135. }
  136. /* --------------------------------------------------------------------------------------------- */
  137. static char
  138. translate_character (GIConv cd, char c)
  139. {
  140. gchar *tmp_buff = NULL;
  141. gsize bytes_read, bytes_written = 0;
  142. const char *ibuf = &c;
  143. char ch = UNKNCHAR;
  144. int ibuflen = 1;
  145. tmp_buff = g_convert_with_iconv (ibuf, ibuflen, cd, &bytes_read, &bytes_written, NULL);
  146. if (tmp_buff)
  147. ch = tmp_buff[0];
  148. g_free (tmp_buff);
  149. return ch;
  150. }
  151. /* --------------------------------------------------------------------------------------------- */
  152. /*** public functions ****************************************************************************/
  153. /* --------------------------------------------------------------------------------------------- */
  154. void
  155. load_codepages_list (void)
  156. {
  157. char *fname;
  158. /* 1: try load /usr/share/mc/mc.charsets */
  159. fname = g_build_filename (mc_global.share_data_dir, CHARSETS_LIST, (char *) NULL);
  160. load_codepages_list_from_file (&codepages, fname);
  161. g_free (fname);
  162. /* 2: try load /etc/mc/mc.charsets */
  163. fname = g_build_filename (mc_global.sysconfig_dir, CHARSETS_LIST, (char *) NULL);
  164. load_codepages_list_from_file (&codepages, fname);
  165. g_free (fname);
  166. if (codepages == NULL)
  167. {
  168. /* files are not found, add defaullt codepage */
  169. fprintf (stderr, "%s\n", _("Warning: cannot load codepages list"));
  170. codepages = g_ptr_array_new ();
  171. g_ptr_array_add (codepages, new_codepage_desc ("ASCII", _("7-bit ASCII")));
  172. }
  173. }
  174. /* --------------------------------------------------------------------------------------------- */
  175. void
  176. free_codepages_list (void)
  177. {
  178. g_ptr_array_foreach (codepages, free_codepage_desc, NULL);
  179. g_ptr_array_free (codepages, TRUE);
  180. }
  181. /* --------------------------------------------------------------------------------------------- */
  182. const char *
  183. get_codepage_id (const int n)
  184. {
  185. return (n < 0) ? OTHER_8BIT : ((codepage_desc *) g_ptr_array_index (codepages, n))->id;
  186. }
  187. /* --------------------------------------------------------------------------------------------- */
  188. int
  189. get_codepage_index (const char *id)
  190. {
  191. size_t i;
  192. if (strcmp (id, OTHER_8BIT) == 0)
  193. return -1;
  194. if (codepages == NULL)
  195. return -1;
  196. for (i = 0; i < codepages->len; i++)
  197. if (strcmp (id, ((codepage_desc *) g_ptr_array_index (codepages, i))->id) == 0)
  198. return i;
  199. return -1;
  200. }
  201. /* --------------------------------------------------------------------------------------------- */
  202. /** Check if specified encoding can be used in mc.
  203. * @param encoding name of encoding
  204. * @returns TRUE if encoding has supported by mc, FALSE otherwise
  205. */
  206. gboolean
  207. is_supported_encoding (const char *encoding)
  208. {
  209. gboolean result = FALSE;
  210. guint t;
  211. for (t = 0; t < codepages->len; t++)
  212. {
  213. const char *id = ((codepage_desc *) g_ptr_array_index (codepages, t))->id;
  214. result |= (g_ascii_strncasecmp (encoding, id, strlen (id)) == 0);
  215. }
  216. return result;
  217. }
  218. /* --------------------------------------------------------------------------------------------- */
  219. char *
  220. init_translation_table (int cpsource, int cpdisplay)
  221. {
  222. int i;
  223. GIConv cd;
  224. /* Fill inpit <-> display tables */
  225. if (cpsource < 0 || cpdisplay < 0 || cpsource == cpdisplay)
  226. {
  227. for (i = 0; i <= 255; ++i)
  228. {
  229. conv_displ[i] = i;
  230. conv_input[i] = i;
  231. cp_source = cp_display;
  232. }
  233. return NULL;
  234. }
  235. for (i = 0; i <= 127; ++i)
  236. {
  237. conv_displ[i] = i;
  238. conv_input[i] = i;
  239. }
  240. cp_source = ((codepage_desc *) g_ptr_array_index (codepages, cpsource))->id;
  241. cp_display = ((codepage_desc *) g_ptr_array_index (codepages, cpdisplay))->id;
  242. /* display <- inpit table */
  243. cd = g_iconv_open (cp_display, cp_source);
  244. if (cd == INVALID_CONV)
  245. return g_strdup_printf (_("Cannot translate from %s to %s"), cp_source, cp_display);
  246. for (i = 128; i <= 255; ++i)
  247. conv_displ[i] = translate_character (cd, i);
  248. g_iconv_close (cd);
  249. /* inpit <- display table */
  250. cd = g_iconv_open (cp_source, cp_display);
  251. if (cd == INVALID_CONV)
  252. return g_strdup_printf (_("Cannot translate from %s to %s"), cp_display, cp_source);
  253. for (i = 128; i <= 255; ++i)
  254. {
  255. unsigned char ch;
  256. ch = translate_character (cd, i);
  257. conv_input[i] = (ch == UNKNCHAR) ? i : ch;
  258. }
  259. g_iconv_close (cd);
  260. return NULL;
  261. }
  262. /* --------------------------------------------------------------------------------------------- */
  263. void
  264. convert_to_display (char *str)
  265. {
  266. if (!str)
  267. return;
  268. while (*str)
  269. {
  270. *str = conv_displ[(unsigned char) *str];
  271. str++;
  272. }
  273. }
  274. /* --------------------------------------------------------------------------------------------- */
  275. GString *
  276. str_convert_to_display (char *str)
  277. {
  278. return str_nconvert_to_display (str, -1);
  279. }
  280. /* --------------------------------------------------------------------------------------------- */
  281. GString *
  282. str_nconvert_to_display (char *str, int len)
  283. {
  284. GString *buff;
  285. GIConv conv;
  286. if (!str)
  287. return g_string_new ("");
  288. if (cp_display == cp_source)
  289. return g_string_new (str);
  290. conv = str_crt_conv_from (cp_source);
  291. buff = g_string_new ("");
  292. str_nconvert (conv, str, len, buff);
  293. str_close_conv (conv);
  294. return buff;
  295. }
  296. /* --------------------------------------------------------------------------------------------- */
  297. void
  298. convert_from_input (char *str)
  299. {
  300. if (!str)
  301. return;
  302. while (*str)
  303. {
  304. *str = conv_input[(unsigned char) *str];
  305. str++;
  306. }
  307. }
  308. /* --------------------------------------------------------------------------------------------- */
  309. GString *
  310. str_convert_to_input (char *str)
  311. {
  312. return str_nconvert_to_input (str, -1);
  313. }
  314. /* --------------------------------------------------------------------------------------------- */
  315. GString *
  316. str_nconvert_to_input (char *str, int len)
  317. {
  318. GString *buff;
  319. GIConv conv;
  320. if (!str)
  321. return g_string_new ("");
  322. if (cp_display == cp_source)
  323. return g_string_new (str);
  324. conv = str_crt_conv_to (cp_source);
  325. buff = g_string_new ("");
  326. str_nconvert (conv, str, len, buff);
  327. str_close_conv (conv);
  328. return buff;
  329. }
  330. /* --------------------------------------------------------------------------------------------- */
  331. unsigned char
  332. convert_from_utf_to_current (const char *str)
  333. {
  334. unsigned char buf_ch[6 + 1];
  335. unsigned char ch = '.';
  336. GIConv conv;
  337. const char *cp_to;
  338. if (!str)
  339. return '.';
  340. cp_to = get_codepage_id (mc_global.source_codepage);
  341. conv = str_crt_conv_to (cp_to);
  342. if (conv != INVALID_CONV)
  343. {
  344. switch (str_translate_char (conv, str, -1, (char *) buf_ch, sizeof (buf_ch)))
  345. {
  346. case ESTR_SUCCESS:
  347. ch = buf_ch[0];
  348. break;
  349. case ESTR_PROBLEM:
  350. case ESTR_FAILURE:
  351. ch = '.';
  352. break;
  353. }
  354. str_close_conv (conv);
  355. }
  356. return ch;
  357. }
  358. /* --------------------------------------------------------------------------------------------- */
  359. unsigned char
  360. convert_from_utf_to_current_c (const int input_char, GIConv conv)
  361. {
  362. unsigned char str[6 + 1];
  363. unsigned char buf_ch[6 + 1];
  364. unsigned char ch = '.';
  365. int res = 0;
  366. res = g_unichar_to_utf8 (input_char, (char *) str);
  367. if (res == 0)
  368. {
  369. return ch;
  370. }
  371. str[res] = '\0';
  372. switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
  373. {
  374. case ESTR_SUCCESS:
  375. ch = buf_ch[0];
  376. break;
  377. case ESTR_PROBLEM:
  378. case ESTR_FAILURE:
  379. ch = '.';
  380. break;
  381. }
  382. return ch;
  383. }
  384. /* --------------------------------------------------------------------------------------------- */
  385. int
  386. convert_from_8bit_to_utf_c (const char input_char, GIConv conv)
  387. {
  388. unsigned char str[2];
  389. unsigned char buf_ch[6 + 1];
  390. int ch = '.';
  391. int res = 0;
  392. str[0] = (unsigned char) input_char;
  393. str[1] = '\0';
  394. switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
  395. {
  396. case ESTR_SUCCESS:
  397. res = g_utf8_get_char_validated ((char *) buf_ch, -1);
  398. if (res < 0)
  399. {
  400. ch = buf_ch[0];
  401. }
  402. else
  403. {
  404. ch = res;
  405. }
  406. break;
  407. case ESTR_PROBLEM:
  408. case ESTR_FAILURE:
  409. ch = '.';
  410. break;
  411. }
  412. return ch;
  413. }
  414. /* --------------------------------------------------------------------------------------------- */
  415. int
  416. convert_from_8bit_to_utf_c2 (const char input_char)
  417. {
  418. unsigned char str[2];
  419. unsigned char buf_ch[6 + 1];
  420. int ch = '.';
  421. int res = 0;
  422. GIConv conv;
  423. const char *cp_from;
  424. str[0] = (unsigned char) input_char;
  425. str[1] = '\0';
  426. cp_from = get_codepage_id (mc_global.source_codepage);
  427. conv = str_crt_conv_to (cp_from);
  428. if (conv != INVALID_CONV)
  429. {
  430. switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
  431. {
  432. case ESTR_SUCCESS:
  433. res = g_utf8_get_char_validated ((char *) buf_ch, -1);
  434. if (res < 0)
  435. {
  436. ch = buf_ch[0];
  437. }
  438. else
  439. {
  440. ch = res;
  441. }
  442. break;
  443. case ESTR_PROBLEM:
  444. case ESTR_FAILURE:
  445. ch = '.';
  446. break;
  447. }
  448. str_close_conv (conv);
  449. }
  450. return ch;
  451. }
  452. /* --------------------------------------------------------------------------------------------- */
  453. #endif /* HAVE_CHARSET */