Use iconv library for character set conversation

An example from stack overflow

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31

#include <iconv.h> #include <stdio.h> int main (void) { iconv_t cd = iconv_open("ISO_8859-1", "UTF-8"); if (cd == (iconv_t) -1) { perror("iconv_open failed!"); return 1; } char input[] = "Test äöü"; char *in_buf = &input[0]; size_t in_left = sizeof(input) - 1; char output[32]; char *out_buf = &output[0]; size_t out_left = sizeof(output) - 1; do { if (iconv(cd, &in_buf, &in_left, &out_buf, &out_left) == (size_t) -1) { perror("iconv failed!"); return 1; } } while (in_left > 0 && out_left > 0); *out_buf = 0; iconv_close(cd); printf("%s -> %s\n", input, output); return 0; }

• reference
• github

Command line

$ echo -n -e \\xAD\\xA9 > source.bin # output binary data to file
$ incov -f ISO-8859-9 -t UTF-8 source.bin # convert from ISO-8859-9 to UTF-8
$ iconv -f $(file -bi filename.ext | sed -e 's/.*[ ]charset=//') -t utf8 filename.ext > filename.ext # convert any encoding to utf-8
$ iconv -c -f utf-8 -t ascii file.txt # clear all non-ascii chars of file.txt