2 * Copyright (C) 1999-2002,2007 Thomas Roessler <roessler@does-not-exist.org>
4 * This program is free software; you can redistribute it
5 * and/or modify it under the terms of the GNU General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later
10 * This program is distributed in the hope that it will be
11 * useful, but WITHOUT ANY WARRANTY; without even the implied
12 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
13 * PURPOSE. See the GNU General Public License for more
16 * You should have received a copy of the GNU General Public
17 * License along with this program; if not, write to the Free
18 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
32 #include <sys/types.h>
41 # define EILSEQ EINVAL
45 * The following list has been created manually from the data under:
46 * http://www.isi.edu/in-notes/iana/assignments/character-sets
47 * Last update: 2000-09-07
49 * Note that it includes only the subset of character sets for which
50 * a preferred MIME name is given.
58 PreferredMIMENames[] =
60 { "ansi_x3.4-1968", "us-ascii" },
61 { "iso-ir-6", "us-ascii" },
62 { "iso_646.irv:1991", "us-ascii" },
63 { "ascii", "us-ascii" },
64 { "iso646-us", "us-ascii" },
66 { "ibm367", "us-ascii" },
67 { "cp367", "us-ascii" },
68 { "csASCII", "us-ascii" },
70 { "csISO2022KR", "iso-2022-kr" },
71 { "csEUCKR", "euc-kr" },
72 { "csISO2022JP", "iso-2022-jp" },
73 { "csISO2022JP2", "iso-2022-jp-2" },
75 { "ISO_8859-1:1987", "iso-8859-1" },
76 { "iso-ir-100", "iso-8859-1" },
77 { "iso_8859-1", "iso-8859-1" },
78 { "latin1", "iso-8859-1" },
79 { "l1", "iso-8859-1" },
80 { "IBM819", "iso-8859-1" },
81 { "CP819", "iso-8859-1" },
82 { "csISOLatin1", "iso-8859-1" },
84 { "ISO_8859-2:1987", "iso-8859-2" },
85 { "iso-ir-101", "iso-8859-2" },
86 { "iso_8859-2", "iso-8859-2" },
87 { "latin2", "iso-8859-2" },
88 { "l2", "iso-8859-2" },
89 { "csISOLatin2", "iso-8859-2" },
91 { "ISO_8859-3:1988", "iso-8859-3" },
92 { "iso-ir-109", "iso-8859-3" },
93 { "ISO_8859-3", "iso-8859-3" },
94 { "latin3", "iso-8859-3" },
95 { "l3", "iso-8859-3" },
96 { "csISOLatin3", "iso-8859-3" },
98 { "ISO_8859-4:1988", "iso-8859-4" },
99 { "iso-ir-110", "iso-8859-4" },
100 { "ISO_8859-4", "iso-8859-4" },
101 { "latin4", "iso-8859-4" },
102 { "l4", "iso-8859-4" },
103 { "csISOLatin4", "iso-8859-4" },
105 { "ISO_8859-6:1987", "iso-8859-6" },
106 { "iso-ir-127", "iso-8859-6" },
107 { "iso_8859-6", "iso-8859-6" },
108 { "ECMA-114", "iso-8859-6" },
109 { "ASMO-708", "iso-8859-6" },
110 { "arabic", "iso-8859-6" },
111 { "csISOLatinArabic", "iso-8859-6" },
113 { "ISO_8859-7:1987", "iso-8859-7" },
114 { "iso-ir-126", "iso-8859-7" },
115 { "ISO_8859-7", "iso-8859-7" },
116 { "ELOT_928", "iso-8859-7" },
117 { "ECMA-118", "iso-8859-7" },
118 { "greek", "iso-8859-7" },
119 { "greek8", "iso-8859-7" },
120 { "csISOLatinGreek", "iso-8859-7" },
122 { "ISO_8859-8:1988", "iso-8859-8" },
123 { "iso-ir-138", "iso-8859-8" },
124 { "ISO_8859-8", "iso-8859-8" },
125 { "hebrew", "iso-8859-8" },
126 { "csISOLatinHebrew", "iso-8859-8" },
128 { "ISO_8859-5:1988", "iso-8859-5" },
129 { "iso-ir-144", "iso-8859-5" },
130 { "ISO_8859-5", "iso-8859-5" },
131 { "cyrillic", "iso-8859-5" },
132 { "csISOLatinCyrillic", "iso-8859-5" },
134 { "ISO_8859-9:1989", "iso-8859-9" },
135 { "iso-ir-148", "iso-8859-9" },
136 { "ISO_8859-9", "iso-8859-9" },
137 { "latin5", "iso-8859-9" }, /* this is not a bug */
138 { "l5", "iso-8859-9" },
139 { "csISOLatin5", "iso-8859-9" },
141 { "ISO_8859-10:1992", "iso-8859-10" },
142 { "iso-ir-157", "iso-8859-10" },
143 { "latin6", "iso-8859-10" }, /* this is not a bug */
144 { "l6", "iso-8859-10" },
145 { "csISOLatin6", "iso-8859-10" },
147 { "csKOI8r", "koi8-r" },
149 { "MS_Kanji", "Shift_JIS" }, /* Note the underscore! */
150 { "csShiftJis", "Shift_JIS" },
152 { "Extended_UNIX_Code_Packed_Format_for_Japanese",
154 { "csEUCPkdFmtJapanese",
157 { "csGB2312", "gb2312" },
158 { "csbig5", "big5" },
161 * End of official brain damage. What follows has been taken
162 * from glibc's localedata files.
165 { "iso_8859-13", "iso-8859-13" },
166 { "iso-ir-179", "iso-8859-13" },
167 { "latin7", "iso-8859-13" }, /* this is not a bug */
168 { "l7", "iso-8859-13" },
170 { "iso_8859-14", "iso-8859-14" },
171 { "latin8", "iso-8859-14" }, /* this is not a bug */
172 { "l8", "iso-8859-14" },
174 { "iso_8859-15", "iso-8859-15" },
175 { "latin9", "iso-8859-15" }, /* this is not a bug */
177 /* Suggested by Ionel Mugurel Ciobica <tgakic@sg10.chem.tue.nl> */
178 { "latin0", "iso-8859-15" }, /* this is not a bug */
180 { "iso_8859-16", "iso-8859-16" },
181 { "latin10", "iso-8859-16" }, /* this is not a bug */
184 * David Champion <dgc@uchicago.edu> has observed this with
185 * nl_langinfo under SunOS 5.8.
188 { "646", "us-ascii" },
191 * http://www.sun.com/software/white-papers/wp-unicode/
194 { "eucJP", "euc-jp" },
195 { "PCK", "Shift_JIS" },
196 { "ko_KR-euc", "euc-kr" },
197 { "zh_TW-big5", "big5" },
199 /* seems to be common on some systems */
201 { "sjis", "Shift_JIS" },
202 { "euc-jp-ms", "eucJP-ms" },
206 * If you happen to encounter system-specific brain-damage with
207 * respect to character set naming, please add it above this
208 * comment, and submit a patch to <mutt-dev@mutt.org>.
211 /* End of aliases. Please keep this line last. */
216 #ifdef HAVE_LANGINFO_CODESET
217 # include <langinfo.h>
220 void mutt_set_langinfo_charset (void)
222 char buff[LONG_STRING];
223 char buff2[LONG_STRING];
225 strfcpy (buff, nl_langinfo (CODESET), sizeof (buff));
226 mutt_canonical_charset (buff2, sizeof (buff2), buff);
228 /* finally, set $charset */
229 if (!(Charset = safe_strdup (buff2)))
230 Charset = safe_strdup ("iso-8859-1");
235 void mutt_set_langinfo_charset (void)
237 Charset = safe_strdup ("iso-8859-1");
242 void mutt_canonical_charset (char *dest, size_t dlen, const char *name)
246 char scratch[LONG_STRING];
248 if (!ascii_strcasecmp (name, "utf-8") || !ascii_strcasecmp (name, "utf8"))
250 strfcpy (dest, "utf-8", dlen);
254 /* catch some common iso-8859-something misspellings */
255 if (!ascii_strncasecmp (name, "8859", 4) && name[4] != '-')
256 snprintf (scratch, sizeof (scratch), "iso-8859-%s", name +4);
257 else if (!ascii_strncasecmp (name, "8859-", 5))
258 snprintf (scratch, sizeof (scratch), "iso-8859-%s", name + 5);
259 else if (!ascii_strncasecmp (name, "iso8859", 7) && name[7] != '-')
260 snprintf (scratch, sizeof (scratch), "iso_8859-%s", name + 7);
261 else if (!ascii_strncasecmp (name, "iso8859-", 8))
262 snprintf (scratch, sizeof (scratch), "iso_8859-%s", name + 8);
264 strfcpy (scratch, NONULL(name), sizeof (scratch));
266 for (i = 0; PreferredMIMENames[i].key; i++)
267 if (!ascii_strcasecmp (scratch, PreferredMIMENames[i].key) ||
268 !mutt_strcasecmp (scratch, PreferredMIMENames[i].key))
270 strfcpy (dest, PreferredMIMENames[i].pref, dlen);
274 strfcpy (dest, scratch, dlen);
276 /* for cosmetics' sake, transform to lowercase. */
277 for (p = dest; *p; p++)
278 *p = ascii_tolower (*p);
281 int mutt_chscmp (const char *s, const char *chs)
287 mutt_canonical_charset (buffer, sizeof (buffer), s);
288 return !ascii_strcasecmp (buffer, chs);
291 char *mutt_get_default_charset ()
293 static char fcharset[SHORT_STRING];
294 const char *c = AssumedCharset;
298 c1 = strchr (c, ':');
299 strfcpy (fcharset, c, c1 ? (c1 - c + 1) : sizeof (fcharset));
302 return strcpy (fcharset, "us-ascii"); /* __STRCPY_CHECKED__ */
307 iconv_t iconv_open (const char *tocode, const char *fromcode)
309 return (iconv_t)(-1);
312 size_t iconv (iconv_t cd, ICONV_CONST char **inbuf, size_t *inbytesleft,
313 char **outbuf, size_t *outbytesleft)
318 int iconv_close (iconv_t cd)
323 #endif /* !HAVE_ICONV */
327 * Like iconv_open, but canonicalises the charsets, applies
328 * charset-hooks, recanonicalises, and finally applies iconv-hooks.
329 * Parameter flags=0 skips charset-hooks, while M_ICONV_HOOK_FROM
330 * applies them to fromcode. Callers should use flags=0 when fromcode
331 * can safely be considered true, either some constant, or some value
332 * provided by the user; M_ICONV_HOOK_FROM should be used only when
333 * fromcode is unsure, taken from a possibly wrong incoming MIME label,
334 * or such. Misusing M_ICONV_HOOK_FROM leads to unwanted interactions
335 * in some setups. Note: By design charset-hooks should never be, and
336 * are never, applied to tocode. Highlight note: The top-well-named
337 * M_ICONV_HOOK_FROM acts on charset-hooks, not at all on iconv-hooks.
340 iconv_t mutt_iconv_open (const char *tocode, const char *fromcode, int flags)
342 char tocode1[SHORT_STRING];
343 char fromcode1[SHORT_STRING];
344 char *tocode2, *fromcode2;
349 /* transform to MIME preferred charset names */
350 mutt_canonical_charset (tocode1, sizeof (tocode1), tocode);
351 mutt_canonical_charset (fromcode1, sizeof (fromcode1), fromcode);
353 /* maybe apply charset-hooks and recanonicalise fromcode,
354 * but only when caller asked us to sanitize a potentialy wrong
355 * charset name incoming from the wild exterior. */
356 if ((flags & M_ICONV_HOOK_FROM) && (tmp = mutt_charset_hook (fromcode1)))
357 mutt_canonical_charset (fromcode1, sizeof (fromcode1), tmp);
359 /* always apply iconv-hooks to suit system's iconv tastes */
360 tocode2 = mutt_iconv_hook (tocode1);
361 tocode2 = (tocode2) ? tocode2 : tocode1;
362 fromcode2 = mutt_iconv_hook (fromcode1);
363 fromcode2 = (fromcode2) ? fromcode2 : fromcode1;
365 /* call system iconv with names it appreciates */
366 if ((cd = iconv_open (tocode2, fromcode2)) != (iconv_t) -1)
374 * Like iconv, but keeps going even when the input is invalid
375 * If you're supplying inrepls, the source charset should be stateless;
376 * if you're supplying an outrepl, the target charset should be.
379 size_t mutt_iconv (iconv_t cd, ICONV_CONST char **inbuf, size_t *inbytesleft,
380 char **outbuf, size_t *outbytesleft,
381 ICONV_CONST char **inrepls, const char *outrepl)
383 size_t ret = 0, ret1;
384 ICONV_CONST char *ib = *inbuf;
385 size_t ibl = *inbytesleft;
387 size_t obl = *outbytesleft;
391 ret1 = iconv (cd, &ib, &ibl, &ob, &obl);
392 if (ret1 != (size_t)-1)
394 if (ibl && obl && errno == EILSEQ)
398 /* Try replacing the input */
399 ICONV_CONST char **t;
400 for (t = inrepls; *t; t++)
402 ICONV_CONST char *ib1 = *t;
403 size_t ibl1 = strlen (*t);
406 iconv (cd, &ib1, &ibl1, &ob1, &obl1);
410 ob = ob1, obl = obl1;
418 /* Replace the output */
421 iconv (cd, 0, 0, &ob, &obl);
424 int n = strlen (outrepl);
430 memcpy (ob, outrepl, n);
434 iconv (cd, 0, 0, 0, 0); /* for good measure */
438 *inbuf = ib, *inbytesleft = ibl;
439 *outbuf = ob, *outbytesleft = obl;
447 * Used in rfc2047.c, rfc2231.c, crypt-gpgme.c, mutt_idna.c, and more.
448 * Parameter flags is given as-is to mutt_iconv_open(). See there
449 * for its meaning and usage policy.
452 int mutt_convert_string (char **ps, const char *from, const char *to, int flags)
455 ICONV_CONST char *repls[] = { "\357\277\275", "?", 0 };
461 if (to && from && (cd = mutt_iconv_open (to, from, flags)) != (iconv_t)-1)
464 ICONV_CONST char *ib;
467 ICONV_CONST char **inrepls = 0;
470 if (mutt_is_utf8 (to))
471 outrepl = "\357\277\275";
472 else if (mutt_is_utf8 (from))
478 ib = s, ibl = len + 1;
479 obl = MB_LEN_MAX * ibl;
480 ob = buf = safe_malloc (obl + 1);
482 mutt_iconv (cd, &ib, &ibl, &ob, &obl, inrepls, outrepl);
487 FREE (ps); /* __FREE_CHECKED__ */
490 mutt_str_adjust (ps);
499 * FGETCONV stuff for converting a file while reading it
500 * Used in sendlib.c for converting from mutt's Charset
513 ICONV_CONST char **inrepls;
523 * Parameter flags is given as-is to mutt_iconv_open(). See there
524 * for its meaning and usage policy.
526 FGETCONV *fgetconv_open (FILE *file, const char *from, const char *to, int flags)
528 struct fgetconv_s *fc;
529 iconv_t cd = (iconv_t)-1;
530 static ICONV_CONST char *repls[] = { "\357\277\275", "?", 0 };
533 cd = mutt_iconv_open (to, from, flags);
535 if (cd != (iconv_t)-1)
537 fc = safe_malloc (sizeof (struct fgetconv_s));
538 fc->p = fc->ob = fc->bufo;
541 fc->inrepls = mutt_is_utf8 (to) ? repls : repls + 1;
544 fc = safe_malloc (sizeof (struct fgetconv_not));
547 return (FGETCONV *)fc;
550 char *fgetconvs (char *buf, size_t l, FGETCONV *_fc)
555 for (r = 0; r + 1 < l;)
557 if ((c = fgetconv (_fc)) == EOF)
571 int fgetconv (FGETCONV *_fc)
573 struct fgetconv_s *fc = (struct fgetconv_s *)_fc;
577 if (fc->cd == (iconv_t)-1)
578 return fgetc (fc->file);
582 return (unsigned char)*(fc->p)++;
584 /* Try to convert some more */
585 fc->p = fc->ob = fc->bufo;
588 size_t obl = sizeof (fc->bufo);
589 iconv (fc->cd, (ICONV_CONST char **)&fc->ib, &fc->ibl, &fc->ob, &obl);
591 return (unsigned char)*(fc->p)++;
594 /* If we trusted iconv a bit more, we would at this point
595 * ask why it had stopped converting ... */
597 /* Try to read some more */
598 if (fc->ibl == sizeof (fc->bufi) ||
599 (fc->ibl && fc->ib + fc->ibl < fc->bufi + sizeof (fc->bufi)))
605 memcpy (fc->bufi, fc->ib, fc->ibl);
607 fc->ibl += fread (fc->ib + fc->ibl, 1, sizeof (fc->bufi) - fc->ibl, fc->file);
609 /* Try harder this time to convert some */
612 size_t obl = sizeof (fc->bufo);
613 mutt_iconv (fc->cd, (ICONV_CONST char **)&fc->ib, &fc->ibl, &fc->ob, &obl,
616 return (unsigned char)*(fc->p)++;
619 /* Either the file has finished or one of the buffers is too small */
624 void fgetconv_close (FGETCONV **_fc)
626 struct fgetconv_s *fc = (struct fgetconv_s *) *_fc;
628 if (fc->cd != (iconv_t)-1)
629 iconv_close (fc->cd);
630 FREE (_fc); /* __FREE_CHECKED__ */
633 int mutt_check_charset (const char *s, int strict)
638 if (mutt_is_utf8 (s))
642 for (i = 0; PreferredMIMENames[i].key; i++)
644 if (ascii_strcasecmp (PreferredMIMENames[i].key, s) == 0 ||
645 ascii_strcasecmp (PreferredMIMENames[i].pref, s) == 0)
649 if ((cd = mutt_iconv_open (s, s, 0)) != (iconv_t)(-1))