Convert a mix of latin 1 and UTF-8 to proper UTF-8 The following program takes an arbitrary byte sequence as input and outputs well-formed UTF-8. All UTF-8 sequences from the input are copied unmodified. All other bytes are assumed to be encoded in ISO 8859-1 or some variant, and their byte value is output as the corresponding code point in UTF-8.

#include &lt;stdbool.h&gt;
#include &lt;stdio.h&gt;
#include &lt;stdlib.h&gt;

static int buf[4];
static size_t buflen;

/* Reads from stdin until buf[index] can be accessed. */
static bool read_buf(size_t index) {
    while (!(index &lt; buflen)) {
        buf[buflen] = getchar();
        if (buf[buflen] == EOF) {
            return false;
        }
        buflen++;
    }
    return buf[buflen - 1] != EOF;
}

static void write_buf(size_t n) {
    for (size_t i = 0; i &lt; n; i++) {
        putchar(buf[i]);
    }
    buflen = 0;
}

static bool start_between(int min, int max) {
    return min &lt;= buf[0] &amp;&amp; buf[0] &lt; max;
}

static bool is_cont(int index) {
    return 0x80 &lt;= buf[index] &amp;&amp; buf[index] &lt; 0xC0;
}

static int code_point(int bits18, int bits12, int bits06, int bits00) {
    return   ((bits18 &amp; 0x3F) &lt;&lt; 18)
           | ((bits12 &amp; 0x3F) &lt;&lt; 12)
           | ((bits06 &amp; 0x3F) &lt;&lt;  6)
           |  (bits00 &amp; 0x3F);
}

int main(void) {
    while (read_buf(0) &amp;&amp; !ferror(stdout)) {
        if (start_between(0x00, 0x80)) {
            write_buf(1);
        } else if (start_between(0xC2, 0xE0) &amp;&amp; read_buf(1) &amp;&amp; is_cont(1)
                   &amp;&amp; code_point(0, 0, buf[0] &amp; 0x1F, buf[1]) &gt;= 0x0080) {
            write_buf(2);
        } else if (start_between(0xE0, 0xF0) &amp;&amp; read_buf(2) &amp;&amp; is_cont(1) &amp;&amp; is_cont(2)
                   &amp;&amp; code_point(0, buf[0] &amp; 0x0F, buf[1], buf[2]) &gt;= 0x0800) {
            write_buf(3);
        } else if (start_between(0xF0, 0xF8) &amp;&amp; read_buf(3) &amp;&amp; is_cont(1) &amp;&amp; is_cont(2) &amp;&amp; is_cont(3)
                   &amp;&amp; code_point(buf[0] &amp; 0x07, buf[1], buf[2], buf[3]) &gt;= 0x010000
                   &amp;&amp; code_point(buf[0] &amp; 0x07, buf[1], buf[2], buf[3]) &lt; 0x110000) {
            write_buf(4);
        } else {
            putchar(0xC0 + ((buf[0] &gt;&gt; 6) &amp; 0x1F));
            putchar(0x80 + ((buf[0] &gt;&gt; 0) &amp; 0x3F));
            buflen--;
            buf[0] = buf[1];
            buf[1] = buf[2];
            buf[2] = buf[3];
            buf[3] = 0;
        }
    }

    return ferror(stdin) || ferror(stdout) ? EXIT_FAILURE : EXIT_SUCCESS;
}


To test the code, I manually ran variants of the following command:

perl -e 'print pack("H*", "fcf09f9985e08080fc")' | ./latin1_to_utf8 | hexdump -C
A useful idea, and very well implemented.

I'd re-write this condition:


while (!(index &lt; buflen))



as

while (buflen &lt;= index)


Apart from that, I found the code very readable.

It's a good idea to store the received data as int rather than converting to char immediately (though I do worry somewhat about using a signed type in code_point - perhaps its arguments should be narrowed to unsigned char?).

Another minor nit: it's not necessary to assign to buf[3] in the else case, although I see it might aid debugging.

I was never able to get an error return from the program (I tried closing stdout using &gt;&amp; in Bash, and also tried redirecting to /dev/full), but always got an exit status of 0.  That surprised me, as the code looks correct.