* new version 2.19.9999

[alpine.git] / pith / charconv / utf8.c

#if !defined(lint) && !defined(DOS)
static char rcsid[] = "$Id: utf8.c 1184 2008-12-16 23:52:15Z hubert@u.washington.edu $";
#endif

/*
 * ========================================================================
 * Copyright 2006-2008 University of Washington
 * Copyright 2013-2015 Eduardo Chappa
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * ========================================================================
 */


/* includable WITHOUT dependency on c-client */
#include "../../c-client/mail.h"
#include "../../c-client/utf8.h"

#ifdef _WINDOWS
/* wingdi.h uses ERROR (!) and we aren't using the c-client ERROR so... */
#undef ERROR
#endif

#include <system.h>

#include "../../c-client/fs.h"

/* includable WITHOUT dependency on pico */
#include "../../pico/keydefs.h"

#include "../osdep/collate.h"
#include "../filttype.h"

#include "utf8.h"

#include <stdarg.h>


unsigned single_width_chars_a_to_b(UCS *, int, int);


static char locale_charmap[50];

static int   native_utf8;
static void *display_data;

void
init_utf8_display(int utf8, void *rmap)
{
    native_utf8 = utf8;
    display_data = rmap;
}


/*
 * Argument is a UCS-4 wide character.
 * Returns the environment dependent cell width of the
 * character when printed to the screen.
 * This will be -1 if the character is not printable.
 * It will be >= zero if it is printable.
 *
 * Note that in the case it is not printable but it is still sent to
 * Writechar, Writechar will print a '?' with width 1.
 */
int
wcellwidth(UCS ucs)
{
    char dummy[32];
    long w;

    /*
     * We believe that on modern unix systems wchar_t is a UCS-4 character.
     * That's the assumption here.
     */

    if(native_utf8){                    /* display is UTF-8 capable */
        w = ucs4_width((unsigned long) ucs);
        return((w & U4W_ERROR) ? -1 : w);
    }
    else if(display_data){
        if(wtomb(dummy, ucs) < 0)
          return(-1);
        else{
            w = ucs4_width((unsigned long) ucs);
            return((w & U4W_ERROR) ? -1 : w);
        }
    }
#ifndef _WINDOWS
    else
      return(wcwidth((wchar_t) ucs));
#else
    return(0);
#endif
}


/*
 * Argument is a UCS-4 wide character.
 * It is converted to the multibyte version (for example UTF8 or EUC-JP).
 * Dest is a buffer at least xx chars wide where the multi-byte version
 * of the wide character will be written.
 * The returned value is the number of bytes written to dest or -1
 * if the conversion can't be done.
 */
int
wtomb(char *dest, UCS ucs)
{
    /*
     * We believe that on modern unix systems wchar_t is a UCS-4 character.
     * That's the assumption here.
     */

    if(native_utf8){
        unsigned char *newdptr;

        newdptr = utf8_put((unsigned char *) dest, (unsigned long) ucs);
        return((newdptr == (unsigned char *) dest) ? -1 : newdptr - (unsigned char *) dest);
    }
    else if(display_data){
        unsigned long ucs4;
        int           ret;

        ucs4 = (unsigned long) ucs;
        ret = ucs4_rmaplen(&ucs4, 1, (unsigned short *) display_data, 0);
        if(ret >= 0)
          ucs4_rmapbuf((unsigned char *) dest, &ucs4, 1, (unsigned short *) display_data, 0);
        else
          ret = -1;

        return(ret);
    }
    else
      return(wcrtomb(dest, (wchar_t) ucs, NULL));
}


/*
 * This function does not necessarily update inputp and remaining_octets, so
 * don't rely on that. The c-client version does but the other doesn't.
 */
UCS
mbtow(void *input_cs, unsigned char **inputp, unsigned long *remaining_octets)
{
    UCS ucs;

    if(input_cs){
        CHARSET *cast_input_cs;

        cast_input_cs = (CHARSET *) input_cs;

        switch((ucs = (UCS) ucs4_cs_get(cast_input_cs, inputp, remaining_octets))){
          case U8G_ENDSTRG:
          case U8G_ENDSTRI:
            return(CCONV_NEEDMORE);

          default:
            if(ucs & U8G_ERROR || ucs == UBOGON)
              return(CCONV_BADCHAR);

            return(ucs);
        }
    }
    else{
        size_t ret;
        wchar_t w;

        /*
         * Warning:  input_cs and remaining_octets are unused in this
         * half of the if/else.
         *
         * Unfortunately, we can't tell the difference between a source string
         * that is just not long enough and one that has characters that can't
         * be converted even though it is long enough. We return NEEDMORE in both cases.
         */
        ret = mbstowcs(&w, (char *) (*inputp), 1);
        if(ret == (size_t)(-1))
          return(CCONV_NEEDMORE);
        else{
          ucs = (UCS) w;
          return(ucs);
        }
    }
}


void
set_locale_charmap(char *charmap)
{
    if(charmap){
        strncpy(locale_charmap, charmap, sizeof(locale_charmap));
        locale_charmap[sizeof(locale_charmap)-1] = '\0';
    }
    else
      locale_charmap[0] = '\0';
}


/*
 * This ensures that the string is UTF-8. If str is already a UTF-8 string,
 * NULL is returned. Otherwise, an allocated string which is UTF-8 is returned.
 * The caller is responsible for freeing the returned value.
 *
 * Args  str     -- the string to convert
 */
char *
convert_to_utf8(char *str, char *fromcharset, int flags)
{
    char          *ret = NULL;
    char          *fcharset;
    SIZEDTEXT      src, result;
    const CHARSET *cs;
    int            try;

    src.data = (unsigned char *) str;
    src.size = strlen(str);

    /* already UTF-8, return NULL */
    if(!(flags & CU8_NOINFER)
       && (cs = utf8_infercharset(&src))
       && (cs->type == CT_ASCII || cs->type == CT_UTF8))
      return(ret);

    try = 1;
    while(try < 5){
        switch(try){
          case 1:
            fcharset = fromcharset;
            if(fcharset && strucmp("UTF-8", fcharset) != 0)
              break;    /* give it a try */
            else
              try++;    /* fall through */

          case 2:
            if(!(flags & CU8_NOINFER)){
                fcharset = cs ? cs->name : NULL;
                if(fcharset && strucmp("UTF-8", fcharset) != 0)
                  break;
                else
                  try++;        /* fall through */
            }
            else
              try++;    /* fall through */

          case 3:
            fcharset = locale_charmap;
            if(fcharset && strucmp("UTF-8", fcharset) != 0)
              break;
            else
              try++;    /* fall through */

          default:
            fcharset = "ISO-8859-1";            /* this will "work" */
            break;
        }

        memset(&result, 0, sizeof(result));

        if(fcharset && utf8_text(&src, fcharset, &result, 0L)){
            if(!(result.size == src.size && result.data == src.data)){
                ret = (char *) fs_get((result.size+1) * sizeof(char));
                strncpy(ret, (char *) result.data, result.size);
                ret[result.size] = '\0';
            }
            /* else no conversion necessary */

            return(ret);
        }

        try++;
    }

    /* won't make it to here */
    return(ret);
}


/*
 * Convert from UTF-8 to user's locale charset.
 * This actually uses the wtomb routine to do the conversion, and that
 * relies on setup_for_input_output having been called.
 * If no conversion is necessary, NULL is returned, otherwise an allocated
 * string in the locale charset is returned and the caller is responsible
 * for freeing it.
 */
char *
convert_to_locale(char *utf8str)
{
#define CHNK 500
    char *inp, *retp, *ret = NULL;
    CBUF_S cb;
    int r, alloced;

    if(native_utf8 || !utf8str || !utf8str[0])
      return(NULL);

    cb.cbuf[0] = '\0';
    cb.cbufp = cb.cbufend = cb.cbuf;
    inp = utf8str;

    alloced = CHNK;
    ret = (char *) fs_get(alloced * sizeof(char));
    retp = ret;

    /*
     * There's gotta be a better way to do this but utf8_to_locale was
     * available and everything looks like a nail when all you have
     * is a hammer.
     */
    while(*inp){
        /*
         * We're placing the outgoing stream of characters in ret, a multi-byte
         * array of characters in the user's locale charset. See if there is
         * enough room for the next wide characters worth of output chars
         * and allocate more space if not.
         */
        if((alloced - (retp-ret)) < MAX(MB_LEN_MAX,32)){
            alloced += CHNK;
            fs_resize((void **) &ret, alloced * sizeof(char));
        }

        r = utf8_to_locale((int) *inp++, &cb,
                           (unsigned char *) retp, alloced-(retp-ret));

        retp += r;
    }

    *retp = '\0';

    fs_resize((void **) &ret, strlen(ret)+1);

    return(ret);
}


/*
 * Pass in a stream of UTF-8 characters in 'c' and return obuf
 * filled in with multi-byte characters. The return value is the
 * number of valid characters in obuf to be used.
 */
int
utf8_to_locale(int c, CBUF_S *cb, unsigned char obuf[], size_t obuf_size)
{
    int outchars = 0;
    
    if(!(cb && cb->cbufp))
      return(0);

    if(cb->cbufp < cb->cbuf+sizeof(cb->cbuf)){
        unsigned char *inputp;
        unsigned long remaining_octets;
        UCS ucs;

        *(cb->cbufp)++ = (unsigned char) c;
        inputp = cb->cbuf;
        remaining_octets = (cb->cbufp - cb->cbuf) * sizeof(unsigned char);
        ucs = (UCS) utf8_get(&inputp, &remaining_octets);

        switch(ucs){
          case U8G_ENDSTRG:     /* incomplete character, wait */
          case U8G_ENDSTRI:     /* incomplete character, wait */
            break;

          default:
            if(ucs & U8G_ERROR || ucs == UBOGON){
                /*
                 * None of these cases is supposed to happen. If it
                 * does happen then the input stream isn't UTF-8
                 * so something is wrong. Treat each character in the
                 * input buffer as a separate error character and
                 * print a '?' for each.
                 */
                for(inputp = cb->cbuf; inputp < cb->cbufp; inputp++)
                  obuf[outchars++] = '?';

                cb->cbufp = cb->cbuf;
            }
            else{
                if(ucs >= 0x80 && wcellwidth(ucs) < 0){
                    /*
                     * This happens when we have a UTF-8 character that
                     * we aren't able to print in our locale. For example,
                     * if the locale is setup with the terminal
                     * expecting ISO-8859-1 characters then there are
                     * lots of UTF-8 characters that can't be printed.
                     * Print a '?' instead.
                     */
                    obuf[outchars++] = '?';
                }
                else{
                    /*
                     * Convert the ucs into the multibyte
                     * character that corresponds to the
                     * ucs in the users locale.
                     */
                    outchars = wtomb((char *) obuf, ucs);
                    if(outchars < 0){
                        obuf[0] = '?';
                        outchars = 1;
                    }
                }

                /* update the input buffer */
                if(inputp >= cb->cbufp) /* this should be the case */
                  cb->cbufp = cb->cbuf;
                else{           /* extra chars for some reason? */
                    unsigned char *q, *newcbufp;

                    newcbufp = (cb->cbufp - inputp) + cb->cbuf;
                    q = cb->cbuf;
                    while(inputp < cb->cbufp)
                      *q++ = *inputp++;

                    cb->cbufp = newcbufp;
                }
            }

            break;
        }
    }
    else{                       /* error */
        obuf[0] = '?';
        outchars = 1;
        cb->cbufp = cb->cbuf;   /* start over */
    }

    return(outchars);
}


/*
 * Returns the screen cells width of the UCS-4 string argument.
 * The source string is zero terminated.
 */
unsigned
ucs4_str_width(UCS *ucsstr)
{
    unsigned width = 0;
    int w;

    if(ucsstr)
      while(*ucsstr){
        w = wcellwidth(*ucsstr++);
        if(w != U4W_CTLSRGT)
          width += (w < 0 ? 1 : w);
      }

    return width;
}


/*
 * Returns the screen cells width of the UCS-4 string argument
 * from ucsstr[a] through (inclusive) ucsstr[b].
 * No checking is done to make sure a starts in the middle
 * of a UCS-4 array.
 */
unsigned
ucs4_str_width_a_to_b(UCS *ucsstr, int a, int b)
{
    unsigned width = 0;
    int i, w;

    if(ucsstr)
      for(i = a; i <= b && ucsstr[i]; i++){
        w = wcellwidth(ucsstr[i]);
        if(w != U4W_CTLSRGT)
          width += (w < 0 ? 1 : w);
      }

    return width;
}


/*
 * Returns the screen cells width of the UCS-4 string argument
 * from ustart through (exclusive) uend.
 * No checking is done to make sure it starts in the middle
 * of a UCS-4 array.
 */
unsigned
ucs4_str_width_ptr_to_ptr(UCS *ustart, UCS *uend)
{
    UCS *u;
    unsigned width = 0;
    int w;

    if(!ustart)
      return width;

    if(ustart)
      for(u = ustart; u < uend; u++){
        w = wcellwidth(*u);
        if(w != U4W_CTLSRGT)
          width += (w < 0 ? 1 : w);
      }

    return(width);
}


/*
 * Return the largest possible pointer into ucs4str so that the width
 * of the string from ucs4str to the pointer (exclusive)
 * is maxwidth or less. Also stops at a null character.
 */
UCS *
ucs4_particular_width(UCS *ucs4str, int maxwidth)
{
    UCS *u;
    int w_consumed = 0, w, done = 0;

    u = ucs4str;

    if(u)
      while(!done && *u && w_consumed <= maxwidth){
        w = wcellwidth(*u);
        w = (w >= 0 ? w : 1);
        if(w_consumed + w <= maxwidth){
            w_consumed += w;
            ++u;
        }
        else
          ++done;
      }

    return(u);
}


/*
 * Convert and copy a UTF-8 string into a UCS-4 NULL
 * terminated array. Just like cpystr only it converts
 * from UTF-8 to UCS-4.
 *
 * Returned UCS-4 string needs to be freed by caller.
 */
UCS *
utf8_to_ucs4_cpystr(char *utf8src)
{
    size_t         retsize;
    UCS           *ret = NULL;
    UCS            ucs;
    unsigned long  remaining_octets;
    unsigned char *readptr;
    size_t         arrayindex;

    /*
     * We don't know how big to allocate the return array
     * because variable numbers of octets in the src array
     * will combine to make UCS-4 characters. The number of
     * UCS-4 characters is less than or equal to the number
     * of src characters, though.
     */

    if(!utf8src)
      return NULL;

    retsize = strlen(utf8src) + 1;

    ret = (UCS *) fs_get(retsize * sizeof(*ret));
    memset(ret, 0, retsize * sizeof(*ret));

    readptr = (unsigned char *) utf8src;
    remaining_octets = retsize-1;
    arrayindex = 0;

    while(remaining_octets > 0 && *readptr && arrayindex < retsize-1){
        ucs = (UCS) utf8_get(&readptr, &remaining_octets);

        if(ucs & U8G_ERROR || ucs == UBOGON)
          remaining_octets = 0;
        else
          ret[arrayindex++] = ucs;
    }

    ret[arrayindex] = '\0';

    /* get rid of excess size */
    if(arrayindex+1 < retsize)
      fs_resize((void **) &ret, (arrayindex + 1) * sizeof(*ret));

    return ret;
}


/*
 * Convert and copy a UCS-4 zero-terminated array into a UTF-8 NULL
 * terminated string. Just like cpystr only it converts
 * from UCS-4 to UTF-8.
 *
 * Returned UTF-8 string needs to be freed by caller.
 */
char *
ucs4_to_utf8_cpystr(UCS *ucs4src)
{
    unsigned char *ret = NULL;
    unsigned char *writeptr;
    int            i;

    if(!ucs4src)
      return NULL;

    /*
     * Over-allocate and then resize at the end.
     */

    /* count characters in source */
    for(i = 0; ucs4src[i]; i++)
      ;

    ret = (unsigned char *) fs_get((6*i + 1) * sizeof(*ret));
    memset(ret, 0, (6*i + 1) * sizeof(*ret));

    writeptr = ret;
    for(i = 0; ucs4src[i]; i++)
      writeptr = utf8_put(writeptr, (unsigned long) ucs4src[i]);

    /* get rid of excess size */
    fs_resize((void **) &ret, (writeptr - ret + 1) * sizeof(*ret));

    return ((char *) ret);
}


/*
 * Similar to above but copy a fixed number of source
 * characters instead of going until null terminator.
 */
char *
ucs4_to_utf8_cpystr_n(UCS *ucs4src, int ucs4src_len)
{
    unsigned char *ret = NULL;
    unsigned char *writeptr;
    int            i;

    if(!ucs4src)
      return NULL;

    /*
     * Over-allocate and then resize at the end.
     */

    ret = (unsigned char *) fs_get((6*ucs4src_len + 1) * sizeof(*ret));
    memset(ret, 0, (6*ucs4src_len + 1) * sizeof(*ret));

    writeptr = ret;
    for(i = 0; i < ucs4src_len; i++)
      writeptr = utf8_put(writeptr, (unsigned long) ucs4src[i]);

    /* get rid of excess size */
    fs_resize((void **) &ret, (writeptr - ret + 1) * sizeof(*ret));

    return ((char *) ret);
}


#ifdef _WINDOWS
/*
 * Convert a UTF-8 argument into an LPTSTR version
 * of that argument. The result is allocated here
 * and should be freed by the caller.
 */
LPTSTR
utf8_to_lptstr(LPSTR arg_utf8)
{
     int lptstr_len;
     LPTSTR lptstr_ret = NULL;

     lptstr_len = MultiByteToWideChar( CP_UTF8, 0, arg_utf8, -1, NULL, 0 );
     if(lptstr_len > 0)
     {
         lptstr_ret = (LPTSTR)fs_get(lptstr_len * sizeof(TCHAR));
         lptstr_len = MultiByteToWideChar( CP_UTF8, 0,
             arg_utf8, -1, lptstr_ret, lptstr_len );
     }

     if(!lptstr_len)
     {
         /* check GetLastError()? */
         lptstr_ret = (LPTSTR)fs_get(sizeof(TCHAR));
         lptstr_ret[0] = 0;
     }

     return lptstr_ret;
}


/*
 * Convert an LPTSTR argument into a UTF-8 version
 * of that argument. The result is allocated here
 * and should be freed by the caller.
 */
LPSTR
lptstr_to_utf8(LPTSTR arg_lptstr)
{
     int utf8str_len;
     LPSTR utf8str_ret = NULL;

     utf8str_len = WideCharToMultiByte( CP_UTF8, 0, arg_lptstr, -1, NULL, 0, NULL, NULL );
     if(utf8str_len > 0)
     {
         utf8str_ret = (LPSTR)fs_get(utf8str_len * sizeof(CHAR));
         utf8str_len = WideCharToMultiByte( CP_UTF8, 0,
             arg_lptstr, -1, utf8str_ret, utf8str_len, NULL, NULL );
     }

     if(!utf8str_len)
     {
         /* check GetLastError()? */
         utf8str_ret = (LPSTR)fs_get(sizeof(CHAR));
         utf8str_ret[0] = 0;
     }

     return utf8str_ret;
}


/*
 * Convert a UCS4 argument into an LPTSTR version
 * of that argument. The result is allocated here
 * and should be freed by the caller.
 */
LPTSTR
ucs4_to_lptstr(UCS *arg_ucs4)
{
    LPTSTR ret_lptstr = NULL;
    size_t len;
    size_t i;

    if(arg_ucs4){
        len = ucs4_strlen(arg_ucs4);
        ret_lptstr = (LPTSTR) fs_get((len+1) * sizeof(TCHAR));
        /* bogus conversion ignores UTF-16 */
        for(i = 0; i < len; i++)
          ret_lptstr[i] = arg_ucs4[i];

        ret_lptstr[len] = '\0';
    }

    return(ret_lptstr);
}


/*
 * Convert an LPTSTR argument into a UCS4 version
 * of that argument. The result is MemAlloc'd here
 * and should be freed by the caller.
 */
UCS *
lptstr_to_ucs4(LPTSTR arg_lptstr)
{
    UCS *ret_ucs4 = NULL;
    size_t len;
    size_t i;

    if(arg_lptstr){
        len = _tcslen(arg_lptstr);
        ret_ucs4 = (UCS *) fs_get((len+1)*sizeof(UCS));
        /* bogus conversion ignores UTF-16 */
        for(i = 0; i < len; i++)
          ret_ucs4[i] = arg_lptstr[i];

        ret_ucs4[len] = '\0';
    }

    return(ret_ucs4);
}

#endif /* _WINDOWS */


/*
 * Pass in a stream of UTF-8 characters 1-at-a-time in 'c' and return obuf
 * 1-at-a-time filled in with UCS characters. The return value is the
 * number of valid characters in obuf to be used. It can only
 * be 1 or 0 characters since we're only getting one UTF-8 character
 * at a time.
 */
int
utf8_to_ucs4_oneatatime(int c, CBUF_S *cb, UCS *obuf, int *obufwidth)
{
    int  width = 0, outchars = 0;
    
    if(!(cb && cb->cbufp))
      return(0);

    if(cb->cbufp < cb->cbuf+sizeof(cb->cbuf)){
        unsigned char *inputp;
        unsigned long remaining_octets;
        UCS ucs;

        *cb->cbufp++ = (unsigned char) c;
        inputp = cb->cbuf;
        remaining_octets = (cb->cbufp - cb->cbuf) * sizeof(unsigned char);
        ucs = (UCS) utf8_get(&inputp, &remaining_octets);

        switch(ucs){
          case U8G_ENDSTRG:     /* incomplete character, wait */
          case U8G_ENDSTRI:     /* incomplete character, wait */
            break;

          default:
            if(ucs & U8G_ERROR || ucs == UBOGON){
                /*
                 * None of these cases is supposed to happen. If it
                 * does happen then the input stream isn't UTF-8
                 * so something is wrong.
                 */
                outchars++;
                *obuf = '?';
                cb->cbufp = cb->cbuf;
                width = 1;
            }
            else{
                outchars++;
                if(ucs < 0x80 && ucs >= 0x20)
                  width = 1;

                if(ucs >= 0x80 && (width=wcellwidth(ucs)) < 0){
                    /*
                     * This happens when we have a UTF-8 character that
                     * we aren't able to print in our locale. For example,
                     * if the locale is setup with the terminal
                     * expecting ISO-8859-1 characters then there are
                     * lots of UTF-8 characters that can't be printed.
                     * Print a '?' instead.
                     * Don't think this should happen in Windows.
                     */
                    *obuf = '?';
                }
                else{
                    *obuf = ucs;
                }

                /* update the input buffer */
                if(inputp >= cb->cbufp) /* this should be the case */
                  cb->cbufp = cb->cbuf;
                else{           /* extra chars for some reason? */
                    unsigned char *q, *newcbufp;

                    newcbufp = (cb->cbufp - inputp) + cb->cbuf;
                    q = cb->cbuf;
                    while(inputp < cb->cbufp)
                      *q++ = *inputp++;

                    cb->cbufp = newcbufp;
                }
            }

            break;
        }
    }
    else{                       /* error */
        *obuf = '?';
        outchars = 1;
        width = 1;
        cb->cbufp = cb->cbuf;   /* start over */
    }

    if(obufwidth)
      *obufwidth = width;

    return(outchars);
}


/*
 * Return an allocated copy of a zero-terminated UCS-4 string.
 */
UCS *
ucs4_cpystr(UCS *ucs4src)
{
    size_t         arraysize;
    UCS           *ret = NULL;
    size_t         i;

    if(!ucs4src)
      return NULL;

    arraysize = ucs4_strlen(ucs4src);

    ret = (UCS *) fs_get((arraysize+1) * sizeof(*ret));
    memset(ret, 0, (arraysize+1) * sizeof(*ret));

    for(i = 0; i < arraysize; i++)
      ret[i] = ucs4src[i];

    return ret;
}


UCS *
ucs4_strncpy(UCS *ucs4dst, UCS *ucs4src, size_t n)
{
    size_t i;

    if(ucs4src && ucs4dst){
        for(i = 0; i < n; i++){
            ucs4dst[i] = ucs4src[i];
            if(ucs4dst[i] == '\0')
              break;
        }
    }

    return ucs4dst;
}


UCS *
ucs4_strncat(UCS *ucs4dst, UCS *ucs4src, size_t n)
{
    size_t i;
    UCS *u;

    if(ucs4src && ucs4dst){
        for(u = ucs4dst; *u; u++)
          ;

        for(i = 0; i < n; i++){
            u[i] = ucs4src[i];
            if(u[i] == '\0')
              break;
        }

        if(i == n)
          u[i] = '\0';
    }

    return ucs4dst;
}


/*
 * Like strlen only this returns the number of non-zero characters
 * in a zero-terminated UCS-4 array.
 */
size_t
ucs4_strlen(UCS *ucs4str)
{
    size_t i = 0;

    if(ucs4str)
      while(ucs4str[i])
        i++;

    return(i);
}


int
ucs4_strcmp(UCS *s1, UCS *s2)
{
    for(; *s1 == *s2; s1++, s2++)
      if(*s1 == '\0')
        return 0;

    return((*s1 < *s2) ? -1 : 1);
}


UCS *
ucs4_strchr(UCS *s, UCS c)
{
    if(!s)
      return NULL;

    while(*s && *s != c)
      s++;

    if(*s || !c)
      return s;
    else
      return NULL;
}


UCS *
ucs4_strrchr(UCS *s, UCS c)
{
    UCS *ret = NULL;

    if(!s)
      return ret;

    while(*s){
        if(*s == c)
          ret = s;

        s++;
    }

    return ret;
}


/*
 * Returns the screen cells width of the UTF-8 string argument.
 */
unsigned
utf8_width(char *str)
{
    unsigned width = 0;
    int this_width;
    UCS ucs;
    unsigned long remaining_octets;
    char *readptr;

    if(!(str && *str))
      return(width);

    readptr = str;
    remaining_octets = readptr ? strlen(readptr) : 0;

    while(remaining_octets > 0 && *readptr){

        ucs = (UCS) utf8_get((unsigned char **) &readptr, &remaining_octets);

        if(ucs & U8G_ERROR || ucs == UBOGON){
            /*
             * This should not happen, but do something to handle it anyway.
             * Treat each character as a single width character, which is what should
             * probably happen when we actually go to write it out.
             */
            remaining_octets--;
            readptr++;
            this_width = 1;
        }
        else{
            this_width = wcellwidth(ucs);

            /*
             * If this_width is -1 that means we can't print this character
             * with our current locale. Writechar will print a '?'.
             */
            if(this_width < 0)
              this_width = 1;
        }

        width += (unsigned) this_width;
    }

    return(width);
}


/*
 * Copy UTF-8 characters from src into dst.
 * This is intended to be used if you want to truncate a string at
 * the start instead of the end. For example, you have a long string
 * like
 *       this_is_a_long_string
 * but not enough space to fit it into a particular field. You want to
 * end up with
 *             s_a_long_string
 * where that fits in a particular width. Perhaps you'd use this with ...
 * to get
 *          ...s_a_long_string
 * This right adjusts the end of the string in the width space and
 * cuts it off at the start. If there is enough width for the whole
 * string it will copy the string into dst with no padding.
 *
 * Copy enough characters so that the result will have screen width of
 * want_width screen cells in current locale.
 *
 * Dstlen is the available space in dst. No more than dstlen bytes will be written
 *   to dst. This is just for protection, it shouldn't be relied on to
 *   do anything useful. Dstlen should be large enough. Otherwise you'll get
 *   characters truncated in the middle or something like that.
 *
 * Returned value is the number of bytes written to dst, not including
 *   the possible terminating null.
 *
 * If we can't hit want_width exactly because of double width characters
 *   then we will pad the end of the string with space in order to make
 *   the width exact.
 */
size_t
utf8_to_width_rhs(char *dst,            /* destination buffer */
                  char *src,            /* source string */
                  size_t dstlen,        /* space in dest */
                  unsigned want_width)  /* desired screen width */
{
    int this_width;
    unsigned width_consumed = 0;
    UCS ucs;
    unsigned long remaining_octets;
    char *readptr, *goodreadptr, *savereadptr, *endptr;
    size_t nb = 0;

    if(!src){
        if(dstlen > 0)
          dst[0] = '\0';

        return nb;
    }

    /*
     * Start at the end of the source string and go backwards until we
     * get to the desired width, but not more than the width.
     */
    readptr = src + strlen(src);
    endptr = readptr;
    goodreadptr = readptr;
    width_consumed = 0;
    savereadptr = readptr;

    for(readptr = savereadptr-1; readptr >= src && width_consumed < want_width && (endptr - readptr) < dstlen;
        readptr = savereadptr-1){

        savereadptr = readptr;
        remaining_octets = goodreadptr - readptr;
        ucs = (UCS) utf8_get((unsigned char **) &readptr, &remaining_octets);

        /*
         * Handling the error case is tough because an error will be the normal thing that
         * happens as we back through the string. So we're just going to punt on the
         * error for now.
         */
        if(!(ucs & U8G_ERROR || ucs == UBOGON)){
            if(remaining_octets > 0){
                /*
                 * This means there are some bad octets after this good
                 * character so things are not going to work out well.
                 * Bail out.
                 */
                savereadptr = src;      /* we're done */
            }
            else{
                this_width = wcellwidth(ucs);

                if(this_width < 0)
                  this_width = 1;

                if(width_consumed + (unsigned) this_width <= want_width){  /* ok */
                    width_consumed += (unsigned) this_width;
                    goodreadptr = savereadptr;
                }
                else
                  savereadptr = src;    /* we're done */
            }
        }
    }

    /*
     * Copy characters from goodreadptr to endptr into dst.
     */
    nb = MIN(endptr-goodreadptr, dstlen-1);
    strncpy(dst, goodreadptr, nb);
    dst[nb] = '\0';

    /*
     * Pad out with spaces in order to hit width exactly.
     */
    while(width_consumed < want_width && nb < dstlen-1){
        dst[nb++] = ' ';
        dst[nb] = '\0';
        width_consumed++;
    }

    return nb;
}


/*
 * The arguments being converted are UTF-8 strings.
 * This routine attempts to make it possible to use screen cell
 * widths in a format specifier. In a one-byte per screen cell
 * world we might have used %10.10s to cause a string to occupy
 * 10 screen positions. Since the width and precision are really
 * referring to numbers of bytes instead of screen positions that
 * won't work with UTF-8 input. We emulate that behavior with
 * the format string %w. %m.nw means to use the m and n as
 * screen width indicators instead of bytes indicators.
 *
 * There is no reason to use this routine unless you want to use
 * min field with or precision with the specifier. A plain %w without
 * widths is equivalent exactly to a plain %s in a regular printf.
 *
 * Double-width characters complicate things. It may not be possible
 * to satisfy the request exactly. For example, %3w for an input
 * string that is made up of two double-width characters.
 * This routine will arbitrarily use a trailing space character if
 * needed to make the width come out correctly where a half of a
 * double-width character would have been needed. We'll see how
 * that works for us.
 *
 * %w only works for strings (it's a %s replacement).
 *
 * Buffer overflow is handled by the size argument. %.30s will work
 * to limit a particular string to 30 bytes, but you lose that
 * ability with %w, since it may write more than precision bytes
 * in order to get to the desired width. It is best to choose
 * size large enough so that it doesn't come into play, otherwise
 * it may be possible to get partial UTF-8 characters because of
 * the truncation.
 *
 * The return value isn't quite the same as the return value
 * of snprintf. It is the number of bytes written, not counting
 * the trailing null, just like snprintf. However, if it is
 * truncated due to size then the output is size, not the
 * number of characters that would have been written.
 */
int
utf8_snprintf(char *dest, size_t size, char *fmt, ...)
{
    char    newfmt[100], buf[20], *q, *pdest, *width_str, *end;
    char   *start_of_specifier;
    char   *input_str;
    int     int_arg;
    double  double_arg;
    void   *ptr_arg;
    unsigned got_width;
    int     more_flags, ret, w;
    int     min_field_width, field_precision, modifier;
    int     flags_minus, flags_plus, flags_space, flags_zero, flags_pound;
    va_list args;

    newfmt[0] = '\0';
    q = newfmt;

    pdest = dest;

#define IS_ROOM_IN_DEST(n_more_chars)                   \
    ((pdest - dest + (n_more_chars) <= size) ? 1 : 0)

    /*
     * Strategy: Look through the fmt string for %w's. Replace the
     * %w's in the format string with %s's but with possibly different
     * width and precision arguments which will make it come out right.
     * Then call the regular system vsnprintf with the altered format
     * string but same arguments.
     *
     * That would be nice but it doesn't quite work. Why? Because a
     * %*w will need to have the value in the integer argument the *
     * refers to modified. Can't do it as far as I can tell. Or we could
     * remove the integer argument somehow before calling printf. Can't
     * do it. Or we could somehow add an additional conversion specifier
     * that caused nothing to be printed but ate up the integer arg.
     * Can't figure out how to do that either.
     *
     * Since we can't figure out how to do it, the alternative is to
     * construct the result one piece at a time, pasting together the
     * pieces from the different conversions.
     */
    va_start(args, fmt);

    while(*fmt && IS_ROOM_IN_DEST(1)){
        if(*fmt == '%'){
            start_of_specifier = fmt++;

            min_field_width = field_precision = -1;
            flags_minus = flags_plus = flags_space = flags_zero = flags_pound = 0;

            /* flags */
            more_flags = 1;
            while(more_flags){
                switch(*fmt){
                  case '-':
                    flags_minus++;
                    fmt++;
                    break;

                  case '+':
                    flags_plus++;
                    fmt++;
                    break;

                  case ' ':
                    flags_space++;
                    fmt++;
                    break;

                  case '0':
                    flags_zero++;
                    fmt++;
                    break;

                  case '#':
                    flags_pound++;
                    fmt++;
                    break;
                  
                  default:
                    more_flags = 0;
                    break;
                }
            }

            /* minimum field width */
            if(*fmt == '*'){
                min_field_width = va_arg(args, int);
                fmt++;
            }
            else if(*fmt >= '0' && *fmt <= '9'){
                width_str = fmt;
                while (*fmt >= '0' && *fmt <= '9')
                  fmt++;

                strncpy(buf, width_str, MIN(fmt-width_str,sizeof(buf)));
                if(sizeof(buf) > fmt-width_str)
                  buf[fmt-width_str] = '\0';

                buf[sizeof(buf)-1] = '\0';

                min_field_width = atoi(width_str);
            }

            /* field precision */
            if(*fmt == '.'){
                fmt++;
                if(*fmt == '*'){
                    field_precision = va_arg(args, int);
                    fmt++;
                }
                else if(*fmt >= '0' && *fmt <= '9'){
                    width_str = fmt;
                    while (*fmt >= '0' && *fmt <= '9')
                      fmt++;

                    strncpy(buf, width_str, MIN(fmt-width_str,sizeof(buf)));
                    if(sizeof(buf) > fmt-width_str)
                      buf[fmt-width_str] = '\0';

                    buf[sizeof(buf)-1] = '\0';

                    field_precision = atoi(width_str);
                }
            }

            /* length modifier */
            if(*fmt == 'h' || *fmt == 'l' || *fmt == 'L')
              modifier = *fmt++;

            /* conversion character */
            switch(*fmt){
              case 'w':
                /*
                 * work with va_arg(char *) to figure out width
                 * and precision needed to produce the screen width
                 * and precision asked for in %w using some of the
                 * utf8 width routines we have.
                 */

                input_str = va_arg(args, char *);
                if(field_precision >=0 || min_field_width >= 0)
                  w = utf8_width(input_str);

                if(field_precision >= 0){
                    if(w <= field_precision)
                      field_precision = -1;  /* print it all */
                    else{
                        /*
                         * We need to cut off some of the input_str
                         * in this case.
                         */
                        end = utf8_count_forw_width(input_str, field_precision, &got_width);
                        field_precision = (int) (end - input_str);
                        /* new w with this field_precision */
                        w = got_width;
                    }
                }

                /* need some padding */
                if(min_field_width >= 0)
                  min_field_width = ((field_precision >= 0) ? field_precision : strlen(input_str)) +
                                      MAX(0, min_field_width - w);

                /*
                 * Now we just need to get the new format string
                 * set correctly in newfmt.
                 */
                q = newfmt;
                if(q-newfmt < sizeof(newfmt))
                  *q++ = '%';

                if(flags_minus && q-newfmt < sizeof(newfmt))
                  *q++ = '-';
                if(flags_plus && q-newfmt < sizeof(newfmt))
                  *q++ = '+';
                if(flags_space && q-newfmt < sizeof(newfmt))
                  *q++ = ' ';
                if(flags_zero && q-newfmt < sizeof(newfmt))
                  *q++ = '0';
                if(flags_pound && q-newfmt < sizeof(newfmt))
                  *q++ = '#';

                if(min_field_width >= 0){
                    snprintf(buf, sizeof(buf), "%d", min_field_width);
                    sstrncpy(&q, buf, sizeof(newfmt)-(q-newfmt));
                }

                if(field_precision >= 0){
                    if(q-newfmt < sizeof(newfmt))
                      *q++ = '.';

                    snprintf(buf, sizeof(buf), "%d", field_precision);
                    sstrncpy(&q, buf, sizeof(newfmt)-(q-newfmt));
                }

                if(q-newfmt < sizeof(newfmt))
                  *q++ = 's';

                if(q-newfmt < sizeof(newfmt))
                  *q++ = '\0';

                snprintf(pdest, size - (pdest-dest), newfmt, input_str);
                pdest += strlen(pdest);

                break;

              case '\0':
                fmt--;
                break;

              default:
                /* make a new format which leaves out the dynamic '*' arguments */
                q = newfmt;
                if(q-newfmt < sizeof(newfmt))
                  *q++ = '%';

                if(flags_minus && q-newfmt < sizeof(newfmt))
                  *q++ = '-';
                if(flags_plus && q-newfmt < sizeof(newfmt))
                  *q++ = '+';
                if(flags_space && q-newfmt < sizeof(newfmt))
                  *q++ = ' ';
                if(flags_zero && q-newfmt < sizeof(newfmt))
                  *q++ = '0';
                if(flags_pound && q-newfmt < sizeof(newfmt))
                  *q++ = '#';

                if(min_field_width >= 0){
                    snprintf(buf, sizeof(buf), "%d", min_field_width);
                    sstrncpy(&q, buf, sizeof(newfmt)-(q-newfmt));
                }

                if(field_precision >= 0){
                    if(q-newfmt < sizeof(newfmt))
                      *q++ = '.';

                    snprintf(buf, sizeof(buf), "%d", field_precision);
                    sstrncpy(&q, buf, sizeof(newfmt)-(q-newfmt));
                }

                if(q-newfmt < sizeof(newfmt))
                  *q++ = *fmt;

                if(q-newfmt < sizeof(newfmt))
                  *q++ = '\0';

                switch(*fmt){
                  case 'd': case 'i': case 'o':
                  case 'x': case 'X': case 'u': case 'c':
                    int_arg = va_arg(args, int);
                    snprintf(pdest, size - (pdest-dest), newfmt, int_arg);
                    pdest += strlen(pdest);
                    break;

                  case 's':
                    input_str = va_arg(args, char *);
                    snprintf(pdest, size - (pdest-dest), newfmt, input_str);
                    pdest += strlen(pdest);
                    break;

                  case 'f': case 'e': case 'E':
                  case 'g': case 'G':
                    double_arg = va_arg(args, double);
                    snprintf(pdest, size - (pdest-dest), newfmt, double_arg);
                    pdest += strlen(pdest);
                    break;

                  case 'p':
                    ptr_arg = va_arg(args, void *);
                    snprintf(pdest, size - (pdest-dest), newfmt, ptr_arg);
                    pdest += strlen(pdest);
                    break;

                  case '%':
                    if(IS_ROOM_IN_DEST(1))
                      *pdest++ =  '%';

                    break;
                  
                  default:
                    /* didn't think of this type */
                    assert(0);
                    break;
                }

                break;
            }

            fmt++;
        }
        else{
            if(IS_ROOM_IN_DEST(1))
              *pdest++ = *fmt++;
        }
    }

    ret = pdest - dest;

    if(IS_ROOM_IN_DEST(1))
      *pdest++ = '\0';

    va_end(args);

    return ret;
}


/*
 * Copy UTF-8 characters from src into dst.
 * Copy enough characters so that the result will have (<=) screen width of
 * want_width screen cells in current locale.
 *
 * Dstlen is the available space in dst. No more than dstlen bytes will be written
 *   to dst.
 *
 * Returned value is the number of bytes written to dst, not including
 *   the possible terminating null.
 * Got_width is another returned value. It is the width in screen cells of
 *   the string placed in dst. It will be the same as want_width if there
 *   are enough characters in the src to do that and if the character widths
 *   hit the width exactly. It will be less than want_width if we run out
 *   of src characters or if the next character width would skip over the
 *   width we want, because it is double width.
 *
 * Zero width characters are collected and included at the end of the string.
 *   That is, if we make it to want_width but there is still a zero length
 *   character sitting in src, we add that to dst. This might be an accent
 *   or something like that.
 */
size_t
utf8_to_width(char *dst,                /* destination buffer */
              char *src,                /* source string */
              size_t dstlen,            /* space in dst */
              unsigned want_width,      /* desired screen width */
              unsigned *got_width)      /* returned screen width in dst */
{
    int this_width;
    unsigned width_consumed = 0;
    UCS ucs;
    unsigned long remaining_octets;
    char *writeptr, *readptr, *savereadptr, *endptr;
    int ran_out_of_space = 0;

    readptr = src;

    remaining_octets = readptr ? strlen(readptr) : 0;

    writeptr = dst;
    endptr = writeptr + dstlen;

    if(readptr && writeptr){
      while(width_consumed <= want_width && remaining_octets > 0 && writeptr < dst + dstlen && !ran_out_of_space){
        savereadptr = readptr;
        ucs = (UCS) utf8_get((unsigned char **) &readptr, &remaining_octets);

        if(ucs & U8G_ERROR || ucs == UBOGON)
          remaining_octets = 0;
        else{
          this_width = wcellwidth(ucs);

          /*
           * If this_width is -1 that means we can't print this character
           * with our current locale. Writechar will print a '?'.
           */
          if(this_width < 0)
            this_width = 1;

          if(width_consumed + (unsigned) this_width <= want_width){
            /* append this utf8 character to dst if it will fit */
            if(writeptr + (readptr - savereadptr) < endptr){
              width_consumed += this_width;
              while(savereadptr < readptr)
                *writeptr++ = *savereadptr++;
            }
            else
              ran_out_of_space++;       /* no more utf8 to dst */
          }
          else
            remaining_octets = 0;       /* we're done */
        }
      }

      if(writeptr < endptr)
        *writeptr = '\0';
    }

    if(got_width)
      *got_width = width_consumed;

    return(writeptr ? (writeptr - dst) : 0);
}


/*
 * Str is a UTF-8 string.
 * Count forward width screencell positions and return a pointer to the
 * end of the string that is width wide.
 * The returned pointer points at the next character (where the null would
 * be placed).
 *
 * Got_width is another returned value. It is the width in screen cells of
 *   the string from str to the returned pointer. It will be the same as
 *   want_width if there are enough characters in the str to do that
 *   and if the character widths hit the width exactly. It will be less
 *   than want_width if we run out of characters or if the next character
 *   width would skip over the width we want, because it is double width.
 */
char *
utf8_count_forw_width(char *str, unsigned want_width, unsigned *got_width)
{
    int this_width;
    unsigned width_consumed = 0;
    UCS ucs;
    unsigned long remaining_octets;
    char *readptr;
    char *retptr;

    retptr = readptr = str;

    remaining_octets = readptr ? strlen(readptr) : 0;

    while(width_consumed <= want_width && remaining_octets > 0){

        ucs = (UCS) utf8_get((unsigned char **) &readptr, &remaining_octets);

        if(ucs & U8G_ERROR || ucs == UBOGON){
            /*
             * This should not happen, but do something to handle it anyway.
             * Treat each character as a single width character, which is what should
             * probably happen when we actually go to write it out.
             */
            remaining_octets--;
            readptr++;
            this_width = 1;
        }
        else{
            this_width = wcellwidth(ucs);

            /*
             * If this_width is -1 that means we can't print this character
             * with our current locale. Writechar will print a '?'.
             */
            if(this_width < 0)
              this_width = 1;
        }

        if(width_consumed + (unsigned) this_width <= want_width){
            width_consumed += (unsigned) this_width;
            retptr = readptr;
        }
        else
          remaining_octets = 0; /* we're done */
    }

    if(got_width)
      *got_width = width_consumed;

    return(retptr);
}


/*
 * Copy a null terminator into a UTF-8 string in place so that the string is
 * no more than a certain screen width wide. If the string is already less
 * than or equal in width to the requested width, no change is made.
 *
 * The actual width accomplished is returned. Note that it may be less than
 * max_width due to double width characters as well as due to the fact that
 * it fits wholly in the max_width.
 *
 * Returned value is the actual screen width of str when done.
 *
 * A side effect is that a terminating null may have been written into
 * the passed in string.
 */
unsigned
utf8_truncate(char *str, unsigned max_width)
{
    int this_width;
    unsigned width_consumed = 0;
    UCS ucs;
    unsigned long remaining_octets;
    char *readptr, *savereadptr;

    readptr = str;

    remaining_octets = readptr ? strlen(readptr) : 0;

    if(readptr){
      while(width_consumed <= max_width && remaining_octets > 0){

        savereadptr = readptr;
        ucs = (UCS) utf8_get((unsigned char **) &readptr, &remaining_octets);

        if(ucs & U8G_ERROR || ucs == UBOGON){
            /*
             * This should not happen, but do something to handle it anyway.
             * Treat each character as a single width character, which is what should
             * probably happen when we actually go to write it out.
             */
            remaining_octets--;
            readptr++;
            this_width = 1;
        }
        else{
            this_width = wcellwidth(ucs);

            /*
             * If this_width is -1 that means we can't print this character
             * with our current locale. Writechar will print a '?'.
             */
            if(this_width < 0)
              this_width = 1;
        }

        if(width_consumed + (unsigned) this_width <= max_width){
            width_consumed += (unsigned) this_width;
        }
        else{
            remaining_octets = 0;       /* we're done */
            *savereadptr = '\0';
        }
      }
    }

    return(width_consumed);
}


/*
 * Copy UTF-8 characters from src into dst.
 * Copy enough characters so that the result will have screen width of
 * want_width screen cells in current locale.
 * If there aren't enough characters in src to get to want_width, pad on
 * left or right according to left_adjust argument.
 *
 * Dstlen is the available space in dst. No more than dstlen bytes will be written
 *   to dst. Dst will be null terminated if there is enough room, but not
 *   if that would overflow dst's len.
 *
 * Returned value is the number of bytes written to dst, not including
 *   the possible terminating null.
 */
size_t
utf8_pad_to_width(char *dst,            /* destination buffer */
                  char *src,            /* source string */
                  size_t dstlen,        /* space in dst */
                  unsigned want_width,  /* desired screen width */
                  int left_adjust)      /* adjust left or right in want_width columns */
{
    unsigned got_width = 0;
    int      need_more, howmany;
    size_t   len_left, bytes_used;

    bytes_used = utf8_to_width(dst, src, dstlen, want_width, &got_width);
    len_left = dstlen - bytes_used;

    need_more = want_width - got_width;
    howmany = MIN(need_more, len_left);
    
    if(howmany > 0){
        char *end, *newend, *p, *q;

        end = dst + bytes_used;
        newend = end + howmany;
        if(left_adjust){
            /*
             * Add padding to end of string. Simply append
             * the needed number of spaces, or however many will fit
             * if we don't have enough space.
             */
            for(q = end; q < newend; q++)
              *q = ' ';
        }
        else{
            /*
             * Add padding to start of string.
             */

            /* slide existing string over */
            for(p = end - 1, q = newend - 1; p >= dst; p--, q--)
              *q = *p;

            /* fill rest with spaces */
            for(; q >= dst; q--)
              *q = ' ';
        }

        bytes_used += howmany;
    }

    if(bytes_used < dstlen)
      dst[bytes_used] = '\0';

    return(bytes_used);
}


/*
 * Str is a UTF-8 string.
 * Start_here is a pointer into the string. It points one position past
 * the last byte that should be considered a part of the length string.
 * Count back want_width screencell positions and return a pointer to the
 * start of the string that is want_width wide and ends with start_here.
 *
 * Since characters may be more than one cell width wide we may end up
 * skipping over the exact width. That is, if we need to we'll go back
 * too far (by one cell width). Account for that in the call by looking
 * at got_width.
 *
 * Note that this call gives a possible got_width == want_width+1 as
 * opposed to utf8_count_forw_width which gives got_width == want-1 instead.
 * That was just what was needed at the time, maybe it needs to be
 * optional.
 */
char *
utf8_count_back_width(char *str, char *start_here, unsigned want_width, unsigned *got_width)
{
    unsigned width_consumed = 0;
    int this_width;
    UCS ucs;
    unsigned long remaining_octets;
    char *ptr, *savereadptr, *goodreadptr;

    savereadptr = start_here;
    goodreadptr = start_here;

    for(ptr = savereadptr - 1; width_consumed < want_width && ptr >= str; ptr = savereadptr - 1){

        savereadptr = ptr;
        remaining_octets = goodreadptr - ptr;
        ucs = (UCS) utf8_get((unsigned char **) &ptr, &remaining_octets);

        if(!(ucs & U8G_ERROR || ucs == UBOGON)){
          if(remaining_octets > 0){
              /*
               * This means there are some bad octets after this good
               * character so things are not going to work out well.
               * Bail out.
               */
              savereadptr = str;        /* we're done */
          }
          else{
            this_width = wcellwidth(ucs);

            /*
             * If this_width is -1 that means we can't print this character
             * with our current locale. Writechar will print a '?'.
             */
            if(this_width < 0)
              this_width = 1;

            width_consumed += (unsigned) this_width;
            goodreadptr = savereadptr;
          }
        }
    }

    if(got_width)
      *got_width = width_consumed;

    return(savereadptr);
}


/*----------------------------------------------------------------------
  copy the source string onto the destination string returning with
  the destination string pointer at the end of the destination text

  motivation for this is to avoid twice passing over a string that's
  being appended to twice (i.e., strcpy(t, x); t += strlen(t))

  This doesn't really belong here but it is used here.
 ----*/
void
sstrncpy(char **d, char *s, int n)
{
    while(n-- > 0 && (**d = *s++) != '\0')
      (*d)++;
}


/*
 * If use_system_routines is set then NULL is the return value and it is
 * not an error. Display_charmap and keyboard_charmap should come over as
 * malloced strings and will be filled in with the result.
 *
 * Returns a void pointer to the input_cs CHARSET which is
 * passed to mbtow via kbseq().
 * If !use_system_routines && NULL is returned, that is an error and err should
 * have a message.
 * display_charmap and keyboard_charmap should be malloced data and may be
 * realloced and changed here.
 */
int
setup_for_input_output(int use_system_routines, char **display_charmap,
                       char **keyboard_charmap, void **input_cs_arg, char **err)
{
    const CHARSET *cs;
    const CHARSET *input_cs = NULL;
    int already_tried = 0;
    int supported = 0;
    char buf[1000];

#define cpstr(s) strcpy((char *)fs_get(1+strlen(s)), s)

    if(err)
      *err = NULL;

    if(!display_charmap || !keyboard_charmap || !input_cs_arg){
        *err = cpstr("Bad call to setup_for_input_output");
        return(-1);
    }

    if(use_system_routines){
#if     PREREQ_FOR_SYS_TRANSLATION
        char *dcm;

        dcm = nl_langinfo_codeset_wrapper();
        dcm = dcm ? dcm : "US-ASCII";

        init_utf8_display(0, NULL);
        if(*display_charmap){
            if(dcm && strucmp(*display_charmap, dcm)){
                snprintf(buf, sizeof(buf),
                 _("Display character set \"%s\" is ignored when using system translation"),
                     *display_charmap);

                *err = cpstr(buf);
            }

            fs_give((void **) display_charmap);
        }

        if(*keyboard_charmap){
            if(!*err && dcm && strucmp(*keyboard_charmap, dcm)){
                snprintf(buf, sizeof(buf),
                 _("Keyboard character set \"%s\" is ignored when using system translation"),
                     *keyboard_charmap);

                *err = cpstr(buf);
            }

            fs_give((void **) keyboard_charmap);
        }

        *display_charmap = cpstr(dcm);
        *keyboard_charmap = cpstr(dcm);
#else
        *err = cpstr("Bad call to setup_for_input_output");
#endif

        *input_cs_arg = NULL;
        return(0);
    }


try_again1:
    if(!(*display_charmap))
      *display_charmap = cpstr("US-ASCII");

    if(!(*keyboard_charmap))
      *keyboard_charmap = cpstr(*display_charmap);

    if(*keyboard_charmap){
        supported = input_charset_is_supported(*keyboard_charmap);

        if(supported){
            if(!strucmp(*keyboard_charmap, "utf-8"))
              input_cs = utf8_charset(*keyboard_charmap);
            else if((cs = utf8_charset(*keyboard_charmap)) != NULL)
              input_cs = cs;
        }
        else{
            if(err && !*err){
                int iso2022jp = 0;

                if(!strucmp(*keyboard_charmap, "ISO-2022-JP"))
                  iso2022jp = 1;

                snprintf(buf, sizeof(buf),
                     /* TRANSLATORS: The first argument is the name of the character
                        set the user is trying to use (which is unsupported by alpine).
                        The second argument is " (except for posting)" if they are
                        trying to use ISO-2022-JP for something other than posting. */
                     _("Character set \"%s\" is unsupported%s, using US-ASCII"),
                     *keyboard_charmap,
                     iso2022jp ? _(" (except for posting)") : "");

                *err = cpstr(buf);
            }

            input_cs = NULL;
            fs_give((void **) keyboard_charmap);
            *keyboard_charmap = cpstr("US-ASCII");
            if(!already_tried){
                already_tried++;
                goto try_again1;
            }
        }
    }
   

try_again2:
    if(!(*display_charmap))
      *display_charmap = cpstr("US-ASCII");

    if(*display_charmap){
        supported = output_charset_is_supported(*display_charmap);
        if(supported){
            if(!strucmp(*display_charmap, "utf-8"))
              init_utf8_display(1, NULL);
            else if((cs = utf8_charset(*display_charmap)) != NULL)
              init_utf8_display(0, utf8_rmap_gen(cs, NULL));
        }
        else{
            if(err && !*err){
                int iso2022jp = 0;

                if(!strucmp(*display_charmap, "ISO-2022-JP"))
                  iso2022jp = 1;

                snprintf(buf, sizeof(buf),
                     _("Character set \"%s\" is unsupported%s, using US-ASCII"),
                     *display_charmap,
                     iso2022jp ? _(" (except for posting)") : "");

                *err = cpstr(buf);
            }

            fs_give((void **) display_charmap);
            if(!already_tried){
                already_tried++;
                goto try_again2;
            }
        }
    }
    else{
        if(err && !*err)
          *err = cpstr(_("Help, can't figure out display character set or even use US-ASCII."));
    }

#undef cpstr

    *input_cs_arg = (void *) input_cs;

    return(0);
}


int
input_charset_is_supported(char *input_charset)
{
    const CHARSET *cs;

    if(!(input_charset && *input_charset))
      return 0;

    if(!strucmp(input_charset, "utf-8"))
      return 1;

    if((cs = utf8_charset(input_charset)) != NULL){

        /*
         * This was true 2006-09-25.
         */
        switch(cs->type){
          case CT_ASCII: case CT_1BYTE0: case CT_1BYTE:
          case CT_1BYTE8: case CT_EUC: case CT_DBYTE:
          case CT_DBYTE2: case CT_SJIS: case CT_UCS2:
          case CT_UCS4: case CT_UTF16:
            return 1;
            break;

          default:
            break;
        }
    }

    return 0;
}


int
output_charset_is_supported(char *output_charset)
{
    const CHARSET *cs;

    if(!(output_charset && *output_charset))
      return 0;

    if(!strucmp(output_charset, "utf-8"))
      return 1;

    if((cs = utf8_charset(output_charset)) != NULL && utf8_rmap_gen(cs, NULL))
      return 1;

    return 0;
}


int
posting_charset_is_supported(char *posting_charset)
{
    return(posting_charset && *posting_charset
           && (!strucmp(posting_charset, "ISO-2022-JP")
               || output_charset_is_supported(posting_charset)));
}


/*
 * This function is only defined in this special case and so calls
 * to it should be wrapped in the same macro conditionals.
 *
 * Returns the default display charset for a UNIX terminal emulator,
 * it is what nl_langinfo(CODESET) should return but we need to
 * wrap nl_langinfo because we know of strange behaving implementations.
 */
#if !defined(_WINDOWS) && HAVE_LANGINFO_H && defined(CODESET)
char *
nl_langinfo_codeset_wrapper(void)
{
    char *ret = NULL;

    ret = nl_langinfo(CODESET);
    
    /*
     * If the value returned from nl_langinfo() is not a real charset,
     * see if we can figure out what they meant. If we can't figure it
     * out return NULL and let the caller decide what to do.
     */
    if(ret && *ret && !output_charset_is_supported(ret)){
        if(!strcmp("ANSI_X3.4-1968", ret)
           || !strcmp("646", ret)
           || !strcmp("ASCII", ret)
           || !strcmp("C", ret)
           || !strcmp("POSIX", ret))
          ret = "US-ASCII";
        else if(!strucmp(ret, "UTF8"))
          ret = "UTF-8";
        else if(!strucmp(ret, "EUCJP"))
          ret = "EUC-JP";
        else if(!strucmp(ret, "EUCKP"))
          ret = "EUC-KP";
        else if(!strucmp(ret, "SJIS"))
          ret = "SHIFT-JIS";
        else if(strstr(ret, "8859")){
            char *p;

            /* check for digits after 8859 */
            p = strstr(ret, "8859");
            p += 4;
            if(!isdigit(*p))
              p++;

            if(isdigit(*p)){
                static char buf[12];

                memset(buf, 0, sizeof(buf));
                strncpy(buf, "ISO-8859-", sizeof(buf));
                buf[9] = *p++;
                if(isdigit(*p))
                  buf[10] = *p;

                ret = buf;
            }
        }
    }

    if(ret && !output_charset_is_supported(ret))
      ret = NULL;

    return(ret);
}
#endif


/*
 * Convert the "orig" string from UTF-8 to "charset". If no conversion is
 * needed the return value will point to orig. If a conversion is done,
 * the return string should be freed by the caller.
 * If not possible, returns NULL.
 */
char *
utf8_to_charset(char *orig, char *charset, int report_err)
{
    SIZEDTEXT src, dst;
    char *ret = orig;

    if(!charset || !charset[0] || !orig || !orig[0] || !strucmp(charset, "utf-8"))
      return ret;

    src.size = strlen(orig);
    src.data = (unsigned char *) orig;

    if(!strucmp(charset, "us-ascii")){
        size_t i;

        for(i = 0; i < src.size; i++)
          if(src.data[i] & 0x80)
            return NULL;

        return ret;
    }

    /*
     * This works for ISO-2022-JP because of special code in utf8_cstext
     * but not for other 2022 charsets.
     */
    memset(&dst, 0, sizeof(dst));
    if(utf8_cstext(&src, charset, &dst, report_err ? 0 : '?') && dst.size > 0 && dst.data)
      ret = (char *) dst.data;          /* c-client already null terminates it */
    else
      ret = NULL;

    if((unsigned char *) ret != dst.data && dst.data)
      fs_give((void **) &dst.data);

    return ret;
}


/*
 *      Turn a number into a string with comma's
 *
 * Args: number -- The long to be turned into a string. 
 *
 * Result: pointer to static string representing number with commas
 * Can use up to 3 comatose results at once.
 */
char *
comatose(long int number)
{
    long        i, x, done_one;
    static char buf[3][50];
    static int whichbuf = 0;
    char       *b;

    whichbuf = (whichbuf + 1) % 3;

    if(number == 0){
        strncpy(buf[whichbuf], "0", sizeof(buf[0]));
        buf[whichbuf][sizeof(buf[0])-1] = '\0';
        return(buf[whichbuf]);
    }
    
    done_one = 0;
    b = buf[whichbuf];
    for(i = 1000000000; i >= 1; i /= 1000) {
        x = number / i;
        number = number % i;
        if(x != 0 || done_one) {
            if(b != buf[whichbuf] && (b-buf[whichbuf]) <  sizeof(buf[0]))
              *b++ = ',';

            snprintf(b, sizeof(buf[0])-(b-buf[whichbuf]), done_one ? "%03ld" : "%ld", x);
            b += strlen(b);
            done_one = 1;
        }
    }

    if(b-buf[whichbuf] < sizeof(buf[0]))
      *b = '\0';

    return(buf[whichbuf]);
}


/* leave out the commas */
char *
tose(long int number)
{
    static char buf[3][50];
    static int whichbuf = 0;

    whichbuf = (whichbuf + 1) % 3;

    snprintf(buf[whichbuf], sizeof(buf[0]), "%ld", number);

    return(buf[whichbuf]);
}


/*
 * line_paint - where the real work of managing what is displayed gets done.
 */
void
line_paint(int offset,                  /* current dot offset into vl */
           struct display_line *displ,
           int *passwd)                 /* flag to hide display of chars */
{
    int i, w, w2, already_got_one = 0;
    int vfirst, vlast, dfirst, dlast, vi, di;
    int new_vbase;
    unsigned (*width_a_to_b)(UCS *, int, int);

    /*
     * Set passwd to 10 in caller if you want to conceal the
     * password but not print asterisks for feedback.
     *
     * Set passwd to 1 in caller to conceal by printing asterisks.
     */
    if(passwd && *passwd >= 10){        /* don't show asterisks */
        if(*passwd > 10)
          return;
        else
          *passwd = 11;         /* only blat once */

        i = 0;
        (*displ->movecursor)(displ->row, displ->col);
        while(i++ <= displ->dwid)
          (*displ->writechar)(' ');

        (*displ->movecursor)(displ->row, displ->col);
        return;
    }

    if(passwd && *passwd)
      width_a_to_b = single_width_chars_a_to_b;
    else
      width_a_to_b = ucs4_str_width_a_to_b;

    /*
     * vl is the virtual line (the actual data). We operate on it by typing
     * characters to be added and deleting and so forth. In this routine we
     * copy a subset of those UCS-4 characters in vl into dl, the display
     * array, and show that subset on the screen.
     *
     * Offset is the location of the cursor in vl.
     *
     * We will display the string starting from vbase.
     * We have dwid screen cells to work in.
     * We may have to adjust vbase in order to display the
     * part of the string that contains the cursor.
     *
     * We'll make the display look like
     *   vl    a b c d e f g h i j k l m
     *             xxxxxxxxxxxxx  <- width dwid window
     *             < d e f g h >
     *               |
     *             vbase
     * The < will be there if vbase > 0.
     * The > will be there if the string from vbase to the
     * end can't all fit in the window.
     */

    memset(displ->dl, 0, displ->dlen * sizeof(UCS));

    /*
     * Adjust vbase so offset is not out of the window to the right.
     * (The +2 in w + 2 is for a possible " >" if the string goes past
     *  the right hand edge of the window and if the last visible character
     * is double wide. We don't want the offset to be under that > character.)
     */
    for(w = (*width_a_to_b)(displ->vl, displ->vbase, offset);
        w + 2 + (displ->vbase ? 1 : 0) > displ->dwid;
        w = (*width_a_to_b)(displ->vl, displ->vbase, offset)){
        /*
         * offset is off the window to the right
         * It looks like   a b c d e f g h
         *                   |         |
         *               vbase         offset
         * and offset is either past the right edge,
         * or right at the right edge (and maybe under >),
         * or one before right at the edge (and maybe on space
         * for half a character).
         *
         * Since the characters may be double width it is slightly
         * complicated to figure out how far to increase vbase.
         * We're going to scoot over past width w/2 characters and
         * then see if that's sufficient.
         */
        new_vbase = displ->vbase + 1;
        for(w2 = (*width_a_to_b)(displ->vl, displ->vbase+1, new_vbase);
            w2 < displ->dwid/2;
            w2 = (*width_a_to_b)(displ->vl, displ->vbase+1, new_vbase))
          new_vbase++;

        displ->vbase = new_vbase;
    }

    /* adjust so offset is not out of the window to the left */
    while(displ->vbase > 0 && displ->vbase >= offset){
        /* add about dwid/2 more width */
        new_vbase = displ->vbase - 1;
        for(w2 = (*width_a_to_b)(displ->vl, new_vbase, displ->vbase);
            w2 < (displ->dwid+1)/2 && new_vbase > 0;
            w2 = (*width_a_to_b)(displ->vl, new_vbase, displ->vbase))
          new_vbase--;

        /* but don't let it get too small, recheck off right end */
        for(w = (*width_a_to_b)(displ->vl, new_vbase, offset);
            w + 2 + (new_vbase ? 1 : 0) > displ->dwid;
            w = (*width_a_to_b)(displ->vl, displ->vbase, offset))
          new_vbase++;

        displ->vbase = MAX(new_vbase, 0);
    }

    if(displ->vbase == 1 && ((passwd && *passwd) || wcellwidth(displ->vl[0]) == 1))
      displ->vbase = 0;
         
    vfirst = displ->vbase;
    dfirst = 0;
    if(displ->vbase > 0){                       /* off screen cue left */
        dfirst = 1;                             /* index which matches vfirst */
        displ->dl[0] = '<';
    }

    vlast = displ->vused-1;                     /* end */
    w = (*width_a_to_b)(displ->vl, vfirst, vlast);

    if(w + dfirst > displ->dwid){                       /* off window right */

        /* find last ucs character to be printed */
        while(w + dfirst > displ->dwid - 1)     /* -1 for > */
          w = (*width_a_to_b)(displ->vl, vfirst, --vlast);

        /* worry about double-width characters */
        if(w + dfirst == displ->dwid - 1){      /* no prob, hit it exactly */
            dlast = dfirst + vlast - vfirst + 1;        /* +1 for > */
            displ->dl[dlast] = '>';
        }
        else{
            dlast = dfirst + vlast - vfirst + 1;
            displ->dl[dlast++] = ' ';
            displ->dl[dlast] = '>';
        }
    }
    else
      dlast = dfirst + vlast - vfirst;

    /*
     * Copy the relevant part of the virtual line into the display line.
     */
    for(vi = vfirst, di = dfirst; vi <= vlast; vi++, di++)
      if(passwd && *passwd)
        displ->dl[di] = '*';            /* to conceal password */
      else
        displ->dl[di] = displ->vl[vi];

    /*
     * Add spaces to clear the rest of the line.
     * We have dwid total space to fill.
     */
    w = (*width_a_to_b)(displ->dl, 0, dlast);   /* width through dlast */
    for(di = dlast+1, i = displ->dwid - w; i > 0 ; i--)
      displ->dl[di++] = ' ';

    /*
     * Draw from left to right, skipping until we get to
     * something that is different. Characters may be different
     * widths than they were initially so paint from there the
     * rest of the way.
     */
    for(di = 0; displ->dl[di]; di++){
        if(already_got_one || displ->dl[di] != displ->olddl[di]){
            /* move cursor first time */
            if(!already_got_one++){
                w = (di > 0) ? (*width_a_to_b)(displ->dl, 0, di-1) : 0;
                (*displ->movecursor)(displ->row, displ->col + w);
            }

            (*displ->writechar)(displ->dl[di]);
            displ->olddl[di] = displ->dl[di];
        }
    }

    memset(&displ->olddl[di], 0, (displ->dlen - di) * sizeof(UCS));

    /*
     * Move the cursor to the offset.
     *
     * The offset is relative to the start of the virtual array. We need
     * to find the location on the screen. The offset into the display array
     * will be offset-vbase+dfirst. We want to be at the start of that
     * character, so we need to find the width of all the characters up
     * to that point.
     */
    w = (offset > 0) ? (*width_a_to_b)(displ->dl, 0, offset-displ->vbase+dfirst-1) : 0;

    (*displ->movecursor)(displ->row, displ->col + w);
}


/*
 * This is just like ucs4_str_width_a_to_b() except all of the characters
 * are assumed to be of width 1. This is for printing out *'s when user
 * enters a password, while still managing to use the same code to do the
 * display.
 */
unsigned
single_width_chars_a_to_b(UCS *ucsstr, int a, int b)
{
    unsigned width = 0;
    int i;

    if(ucsstr)
      for(i = a; i <= b && ucsstr[i]; i++)
        width++;

    return width;
}