pack.c

   1 /**********************************************************************
   2
   3   pack.c -
   4
   5   $Author$
   6   created at: Thu Feb 10 15:17:05 JST 1994
   7
   8   Copyright (C) 1993-2007 Yukihiro Matsumoto
   9
  10 **********************************************************************/
  11
  12 #include "ruby/internal/config.h"
  13
  14 #include <ctype.h>
  15 #include <errno.h>
  16 #include <float.h>
  17 #include <sys/types.h>
  18
  19 #include "internal.h"
  20 #include "internal/array.h"
  21 #include "internal/bits.h"
  22 #include "internal/string.h"
  23 #include "internal/symbol.h"
  24 #include "internal/variable.h"
  25 #include "ruby/util.h"
  26
  27 #include "builtin.h"
  28
  29 /*
  30  * It is intentional that the condition for natstr is HAVE_TRUE_LONG_LONG
  31  * instead of HAVE_LONG_LONG or LONG_LONG.
  32  * This means q! and Q! means always the standard long long type and
  33  * causes ArgumentError for platforms which has no long long type,
  34  * even if the platform has an implementation specific 64bit type.
  35  * This behavior is consistent with the document of pack/unpack.
  36  */
  37 #ifdef HAVE_TRUE_LONG_LONG
  38 static const char natstr[] = "sSiIlLqQjJ";
  39 # define endstr natstr
  40 #else
  41 static const char natstr[] = "sSiIlLjJ";
  42 static const char endstr[] = "sSiIlLqQjJ";
  43 #endif
  44
  45 #ifdef HAVE_TRUE_LONG_LONG
  46 /* It is intentional to use long long instead of LONG_LONG. */
  47 # define NATINT_LEN_Q NATINT_LEN(long long, 8)
  48 #else
  49 # define NATINT_LEN_Q 8
  50 #endif
  51
  52 #if SIZEOF_SHORT != 2 || SIZEOF_LONG != 4 || (defined(HAVE_TRUE_LONG_LONG) && SIZEOF_LONG_LONG != 8)
  53 # define NATINT_PACK
  54 #endif
  55
  56 #ifdef DYNAMIC_ENDIAN
  57 /* for universal binary of NEXTSTEP and MacOS X */
  58 /* useless since autoconf 2.63? */
  59 static int
  60 is_bigendian(void)
  61 {
  62     static int init = 0;
  63     static int endian_value;
  64     char *p;
  65
  66     if (init) return endian_value;
  67     init = 1;
  68     p = (char*)&init;
  69     return endian_value = p[0]?0:1;
  70 }
  71 # define BIGENDIAN_P() (is_bigendian())
  72 #elif defined(WORDS_BIGENDIAN)
  73 # define BIGENDIAN_P() 1
  74 #else
  75 # define BIGENDIAN_P() 0
  76 #endif
  77
  78 #ifdef NATINT_PACK
  79 # define NATINT_LEN(type,len) (natint?(int)sizeof(type):(int)(len))
  80 #else
  81 # define NATINT_LEN(type,len) ((int)sizeof(type))
  82 #endif
  83
  84 typedef union {
  85     float f;
  86     uint32_t u;
  87     char buf[4];
  88 } FLOAT_SWAPPER;
  89 typedef union {
  90     double d;
  91     uint64_t u;
  92     char buf[8];
  93 } DOUBLE_SWAPPER;
  94 #define swapf(x) swap32(x)
  95 #define swapd(x) swap64(x)
  96
  97 #define rb_ntohf(x) (BIGENDIAN_P()?(x):swapf(x))
  98 #define rb_ntohd(x) (BIGENDIAN_P()?(x):swapd(x))
  99 #define rb_htonf(x) (BIGENDIAN_P()?(x):swapf(x))
 100 #define rb_htond(x) (BIGENDIAN_P()?(x):swapd(x))
 101 #define rb_htovf(x) (BIGENDIAN_P()?swapf(x):(x))
 102 #define rb_htovd(x) (BIGENDIAN_P()?swapd(x):(x))
 103 #define rb_vtohf(x) (BIGENDIAN_P()?swapf(x):(x))
 104 #define rb_vtohd(x) (BIGENDIAN_P()?swapd(x):(x))
 105
 106 #define FLOAT_CONVWITH(x)       FLOAT_SWAPPER x;
 107 #define HTONF(x)        ((x).u = rb_htonf((x).u))
 108 #define HTOVF(x)        ((x).u = rb_htovf((x).u))
 109 #define NTOHF(x)        ((x).u = rb_ntohf((x).u))
 110 #define VTOHF(x)        ((x).u = rb_vtohf((x).u))
 111
 112 #define DOUBLE_CONVWITH(x)      DOUBLE_SWAPPER x;
 113 #define HTOND(x)        ((x).u = rb_htond((x).u))
 114 #define HTOVD(x)        ((x).u = rb_htovd((x).u))
 115 #define NTOHD(x)        ((x).u = rb_ntohd((x).u))
 116 #define VTOHD(x)        ((x).u = rb_vtohd((x).u))
 117
 118 #define MAX_INTEGER_PACK_SIZE 8
 119
 120 static const char toofew[] = "too few arguments";
 121
 122 static void encodes(VALUE,const char*,long,int,int);
 123 static void qpencode(VALUE,VALUE,long);
 124
 125 static unsigned long utf8_to_uv(const char*,long*);
 126
 127 static ID id_associated;
 128
 129 static void
 130 str_associate(VALUE str, VALUE add)
 131 {
 132     /* assert(NIL_P(rb_attr_get(str, id_associated))); */
 133     rb_ivar_set(str, id_associated, add);
 134 }
 135
 136 static VALUE
 137 str_associated(VALUE str)
 138 {
 139     VALUE associates = rb_ivar_lookup(str, id_associated, Qfalse);
 140     if (!associates)
 141         rb_raise(rb_eArgError, "no associated pointer");
 142     return associates;
 143 }
 144
 145 static VALUE
 146 associated_pointer(VALUE associates, const char *t)
 147 {
 148     const VALUE *p = RARRAY_CONST_PTR(associates);
 149     const VALUE *pend = p + RARRAY_LEN(associates);
 150     for (; p < pend; p++) {
 151         VALUE tmp = *p;
 152         if (RB_TYPE_P(tmp, T_STRING) && RSTRING_PTR(tmp) == t) return tmp;
 153     }
 154     rb_raise(rb_eArgError, "non associated pointer");
 155     UNREACHABLE_RETURN(Qnil);
 156 }
 157
 158 RBIMPL_ATTR_NORETURN()
 159 static void
 160 unknown_directive(const char *mode, char type, VALUE fmt)
 161 {
 162     char unknown[5];
 163
 164     if (ISPRINT(type)) {
 165         unknown[0] = type;
 166         unknown[1] = '\0';
 167     }
 168     else {
 169         snprintf(unknown, sizeof(unknown), "\\x%.2x", type & 0xff);
 170     }
 171     fmt = rb_str_quote_unprintable(fmt);
 172     rb_raise(rb_eArgError, "unknown %s directive '%s' in '%"PRIsVALUE"'",
 173             mode, unknown, fmt);
 174 }
 175
 176 static float
 177 VALUE_to_float(VALUE obj)
 178 {
 179     VALUE v = rb_to_float(obj);
 180     double d = RFLOAT_VALUE(v);
 181
 182     if (isnan(d)) {
 183         return NAN;
 184     }
 185     else if (d < -FLT_MAX) {
 186         return -INFINITY;
 187     }
 188     else if (d <= FLT_MAX) {
 189         return d;
 190     }
 191     else {
 192         return INFINITY;
 193     }
 194 }
 195
 196 static VALUE
 197 pack_pack(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer)
 198 {
 199     static const char nul10[] = "\0\0\0\0\0\0\0\0\0\0";
 200     static const char spc10[] = "          ";
 201     const char *p, *pend;
 202     VALUE res, from, associates = 0;
 203     char type;
 204     long len, idx, plen;
 205     const char *ptr;
 206     int enc_info = 1;           /* 0 - BINARY, 1 - US-ASCII, 2 - UTF-8 */
 207 #ifdef NATINT_PACK
 208     int natint;         /* native integer */
 209 #endif
 210     int integer_size, bigendian_p;
 211
 212     StringValue(fmt);
 213     rb_must_asciicompat(fmt);
 214     p = RSTRING_PTR(fmt);
 215     pend = p + RSTRING_LEN(fmt);
 216
 217     if (NIL_P(buffer)) {
 218         res = rb_str_buf_new(0);
 219     }
 220     else {
 221         if (!RB_TYPE_P(buffer, T_STRING))
 222             rb_raise(rb_eTypeError, "buffer must be String, not %s", rb_obj_classname(buffer));
 223         rb_str_modify(buffer);
 224         res = buffer;
 225     }
 226
 227     idx = 0;
 228
 229 #define TOO_FEW (rb_raise(rb_eArgError, toofew), 0)
 230 #define MORE_ITEM (idx < RARRAY_LEN(ary))
 231 #define THISFROM (MORE_ITEM ? RARRAY_AREF(ary, idx) : TOO_FEW)
 232 #define NEXTFROM (MORE_ITEM ? RARRAY_AREF(ary, idx++) : TOO_FEW)
 233
 234     while (p < pend) {
 235         int explicit_endian = 0;
 236         if (RSTRING_PTR(fmt) + RSTRING_LEN(fmt) != pend) {
 237             rb_raise(rb_eRuntimeError, "format string modified");
 238         }
 239         type = *p++;            /* get data type */
 240 #ifdef NATINT_PACK
 241         natint = 0;
 242 #endif
 243
 244         if (ISSPACE(type)) continue;
 245         if (type == '#') {
 246             while ((p < pend) && (*p != '\n')) {
 247                 p++;
 248             }
 249             continue;
 250         }
 251
 252         {
 253           modifiers:
 254             switch (*p) {
 255               case '_':
 256               case '!':
 257                 if (strchr(natstr, type)) {
 258 #ifdef NATINT_PACK
 259                     natint = 1;
 260 #endif
 261                     p++;
 262                 }
 263                 else {
 264                     rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, natstr);
 265                 }
 266                 goto modifiers;
 267
 268               case '<':
 269               case '>':
 270                 if (!strchr(endstr, type)) {
 271                     rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, endstr);
 272                 }
 273                 if (explicit_endian) {
 274                     rb_raise(rb_eRangeError, "Can't use both '<' and '>'");
 275                 }
 276                 explicit_endian = *p++;
 277                 goto modifiers;
 278             }
 279         }
 280
 281         if (*p == '*') {        /* set data length */
 282             len = strchr("@Xxu", type) ? 0
 283                 : strchr("PMm", type) ? 1
 284                 : RARRAY_LEN(ary) - idx;
 285             p++;
 286         }
 287         else if (ISDIGIT(*p)) {
 288             errno = 0;
 289             len = STRTOUL(p, (char**)&p, 10);
 290             if (errno) {
 291                 rb_raise(rb_eRangeError, "pack length too big");
 292             }
 293         }
 294         else {
 295             len = 1;
 296         }
 297
 298         switch (type) {
 299           case 'U':
 300             /* if encoding is US-ASCII, upgrade to UTF-8 */
 301             if (enc_info == 1) enc_info = 2;
 302             break;
 303           case 'm': case 'M': case 'u':
 304             /* keep US-ASCII (do nothing) */
 305             break;
 306           default:
 307             /* fall back to BINARY */
 308             enc_info = 0;
 309             break;
 310         }
 311         switch (type) {
 312           case 'A': case 'a': case 'Z':
 313           case 'B': case 'b':
 314           case 'H': case 'h':
 315             from = NEXTFROM;
 316             if (NIL_P(from)) {
 317                 ptr = "";
 318                 plen = 0;
 319             }
 320             else {
 321                 StringValue(from);
 322                 ptr = RSTRING_PTR(from);
 323                 plen = RSTRING_LEN(from);
 324             }
 325
 326             if (p[-1] == '*')
 327                 len = plen;
 328
 329             switch (type) {
 330               case 'a':         /* arbitrary binary string (null padded)  */
 331               case 'A':         /* arbitrary binary string (ASCII space padded) */
 332               case 'Z':         /* null terminated string  */
 333                 if (plen >= len) {
 334                     rb_str_buf_cat(res, ptr, len);
 335                     if (p[-1] == '*' && type == 'Z')
 336                         rb_str_buf_cat(res, nul10, 1);
 337                 }
 338                 else {
 339                     rb_str_buf_cat(res, ptr, plen);
 340                     len -= plen;
 341                     while (len >= 10) {
 342                         rb_str_buf_cat(res, (type == 'A')?spc10:nul10, 10);
 343                         len -= 10;
 344                     }
 345                     rb_str_buf_cat(res, (type == 'A')?spc10:nul10, len);
 346                 }
 347                 break;
 348
 349 #define castchar(from) (char)((from) & 0xff)
 350
 351               case 'b':         /* bit string (ascending) */
 352                 {
 353                     int byte = 0;
 354                     long i, j = 0;
 355
 356                     if (len > plen) {
 357                         j = (len - plen + 1)/2;
 358                         len = plen;
 359                     }
 360                     for (i=0; i++ < len; ptr++) {
 361                         if (*ptr & 1)
 362                             byte |= 128;
 363                         if (i & 7)
 364                             byte >>= 1;
 365                         else {
 366                             char c = castchar(byte);
 367                             rb_str_buf_cat(res, &c, 1);
 368                             byte = 0;
 369                         }
 370                     }
 371                     if (len & 7) {
 372                         char c;
 373                         byte >>= 7 - (len & 7);
 374                         c = castchar(byte);
 375                         rb_str_buf_cat(res, &c, 1);
 376                     }
 377                     len = j;
 378                     goto grow;
 379                 }
 380                 break;
 381
 382               case 'B':         /* bit string (descending) */
 383                 {
 384                     int byte = 0;
 385                     long i, j = 0;
 386
 387                     if (len > plen) {
 388                         j = (len - plen + 1)/2;
 389                         len = plen;
 390                     }
 391                     for (i=0; i++ < len; ptr++) {
 392                         byte |= *ptr & 1;
 393                         if (i & 7)
 394                             byte <<= 1;
 395                         else {
 396                             char c = castchar(byte);
 397                             rb_str_buf_cat(res, &c, 1);
 398                             byte = 0;
 399                         }
 400                     }
 401                     if (len & 7) {
 402                         char c;
 403                         byte <<= 7 - (len & 7);
 404                         c = castchar(byte);
 405                         rb_str_buf_cat(res, &c, 1);
 406                     }
 407                     len = j;
 408                     goto grow;
 409                 }
 410                 break;
 411
 412               case 'h':         /* hex string (low nibble first) */
 413                 {
 414                     int byte = 0;
 415                     long i, j = 0;
 416
 417                     if (len > plen) {
 418                         j = (len + 1) / 2 - (plen + 1) / 2;
 419                         len = plen;
 420                     }
 421                     for (i=0; i++ < len; ptr++) {
 422                         if (ISALPHA(*ptr))
 423                             byte |= (((*ptr & 15) + 9) & 15) << 4;
 424                         else
 425                             byte |= (*ptr & 15) << 4;
 426                         if (i & 1)
 427                             byte >>= 4;
 428                         else {
 429                             char c = castchar(byte);
 430                             rb_str_buf_cat(res, &c, 1);
 431                             byte = 0;
 432                         }
 433                     }
 434                     if (len & 1) {
 435                         char c = castchar(byte);
 436                         rb_str_buf_cat(res, &c, 1);
 437                     }
 438                     len = j;
 439                     goto grow;
 440                 }
 441                 break;
 442
 443               case 'H':         /* hex string (high nibble first) */
 444                 {
 445                     int byte = 0;
 446                     long i, j = 0;
 447
 448                     if (len > plen) {
 449                         j = (len + 1) / 2 - (plen + 1) / 2;
 450                         len = plen;
 451                     }
 452                     for (i=0; i++ < len; ptr++) {
 453                         if (ISALPHA(*ptr))
 454                             byte |= ((*ptr & 15) + 9) & 15;
 455                         else
 456                             byte |= *ptr & 15;
 457                         if (i & 1)
 458                             byte <<= 4;
 459                         else {
 460                             char c = castchar(byte);
 461                             rb_str_buf_cat(res, &c, 1);
 462                             byte = 0;
 463                         }
 464                     }
 465                     if (len & 1) {
 466                         char c = castchar(byte);
 467                         rb_str_buf_cat(res, &c, 1);
 468                     }
 469                     len = j;
 470                     goto grow;
 471                 }
 472                 break;
 473             }
 474             break;
 475
 476           case 'c':             /* signed char */
 477           case 'C':             /* unsigned char */
 478             integer_size = 1;
 479             bigendian_p = BIGENDIAN_P(); /* not effective */
 480             goto pack_integer;
 481
 482           case 's':             /* s for int16_t, s! for signed short */
 483           case 'S':             /* S for uint16_t, S! for unsigned short */
 484             integer_size = NATINT_LEN(short, 2);
 485             bigendian_p = BIGENDIAN_P();
 486             goto pack_integer;
 487
 488           case 'i':             /* i and i! for signed int */
 489           case 'I':             /* I and I! for unsigned int */
 490             integer_size = (int)sizeof(int);
 491             bigendian_p = BIGENDIAN_P();
 492             goto pack_integer;
 493
 494           case 'l':             /* l for int32_t, l! for signed long */
 495           case 'L':             /* L for uint32_t, L! for unsigned long */
 496             integer_size = NATINT_LEN(long, 4);
 497             bigendian_p = BIGENDIAN_P();
 498             goto pack_integer;
 499
 500           case 'q':             /* q for int64_t, q! for signed long long */
 501           case 'Q':             /* Q for uint64_t, Q! for unsigned long long */
 502             integer_size = NATINT_LEN_Q;
 503             bigendian_p = BIGENDIAN_P();
 504             goto pack_integer;
 505
 506           case 'j':             /* j for intptr_t */
 507             integer_size = sizeof(intptr_t);
 508             bigendian_p = BIGENDIAN_P();
 509             goto pack_integer;
 510
 511           case 'J':             /* J for uintptr_t */
 512             integer_size = sizeof(uintptr_t);
 513             bigendian_p = BIGENDIAN_P();
 514             goto pack_integer;
 515
 516           case 'n':             /* 16 bit (2 bytes) integer (network byte-order)  */
 517             integer_size = 2;
 518             bigendian_p = 1;
 519             goto pack_integer;
 520
 521           case 'N':             /* 32 bit (4 bytes) integer (network byte-order) */
 522             integer_size = 4;
 523             bigendian_p = 1;
 524             goto pack_integer;
 525
 526           case 'v':             /* 16 bit (2 bytes) integer (VAX byte-order) */
 527             integer_size = 2;
 528             bigendian_p = 0;
 529             goto pack_integer;
 530
 531           case 'V':             /* 32 bit (4 bytes) integer (VAX byte-order) */
 532             integer_size = 4;
 533             bigendian_p = 0;
 534             goto pack_integer;
 535
 536           pack_integer:
 537             if (explicit_endian) {
 538                 bigendian_p = explicit_endian == '>';
 539             }
 540             if (integer_size > MAX_INTEGER_PACK_SIZE)
 541                 rb_bug("unexpected integer size for pack: %d", integer_size);
 542             while (len-- > 0) {
 543                 char intbuf[MAX_INTEGER_PACK_SIZE];
 544
 545                 from = NEXTFROM;
 546                 rb_integer_pack(from, intbuf, integer_size, 1, 0,
 547                     INTEGER_PACK_2COMP |
 548                     (bigendian_p ? INTEGER_PACK_BIG_ENDIAN : INTEGER_PACK_LITTLE_ENDIAN));
 549                 rb_str_buf_cat(res, intbuf, integer_size);
 550             }
 551             break;
 552
 553           case 'f':             /* single precision float in native format */
 554           case 'F':             /* ditto */
 555             while (len-- > 0) {
 556                 float f;
 557
 558                 from = NEXTFROM;
 559                 f = VALUE_to_float(from);
 560                 rb_str_buf_cat(res, (char*)&f, sizeof(float));
 561             }
 562             break;
 563
 564           case 'e':             /* single precision float in VAX byte-order */
 565             while (len-- > 0) {
 566                 FLOAT_CONVWITH(tmp);
 567
 568                 from = NEXTFROM;
 569                 tmp.f = VALUE_to_float(from);
 570                 HTOVF(tmp);
 571                 rb_str_buf_cat(res, tmp.buf, sizeof(float));
 572             }
 573             break;
 574
 575           case 'E':             /* double precision float in VAX byte-order */
 576             while (len-- > 0) {
 577                 DOUBLE_CONVWITH(tmp);
 578                 from = NEXTFROM;
 579                 tmp.d = RFLOAT_VALUE(rb_to_float(from));
 580                 HTOVD(tmp);
 581                 rb_str_buf_cat(res, tmp.buf, sizeof(double));
 582             }
 583             break;
 584
 585           case 'd':             /* double precision float in native format */
 586           case 'D':             /* ditto */
 587             while (len-- > 0) {
 588                 double d;
 589
 590                 from = NEXTFROM;
 591                 d = RFLOAT_VALUE(rb_to_float(from));
 592                 rb_str_buf_cat(res, (char*)&d, sizeof(double));
 593             }
 594             break;
 595
 596           case 'g':             /* single precision float in network byte-order */
 597             while (len-- > 0) {
 598                 FLOAT_CONVWITH(tmp);
 599                 from = NEXTFROM;
 600                 tmp.f = VALUE_to_float(from);
 601                 HTONF(tmp);
 602                 rb_str_buf_cat(res, tmp.buf, sizeof(float));
 603             }
 604             break;
 605
 606           case 'G':             /* double precision float in network byte-order */
 607             while (len-- > 0) {
 608                 DOUBLE_CONVWITH(tmp);
 609
 610                 from = NEXTFROM;
 611                 tmp.d = RFLOAT_VALUE(rb_to_float(from));
 612                 HTOND(tmp);
 613                 rb_str_buf_cat(res, tmp.buf, sizeof(double));
 614             }
 615             break;
 616
 617           case 'x':             /* null byte */
 618           grow:
 619             while (len >= 10) {
 620                 rb_str_buf_cat(res, nul10, 10);
 621                 len -= 10;
 622             }
 623             rb_str_buf_cat(res, nul10, len);
 624             break;
 625
 626           case 'X':             /* back up byte */
 627           shrink:
 628             plen = RSTRING_LEN(res);
 629             if (plen < len)
 630                 rb_raise(rb_eArgError, "X outside of string");
 631             rb_str_set_len(res, plen - len);
 632             break;
 633
 634           case '@':             /* null fill to absolute position */
 635             len -= RSTRING_LEN(res);
 636             if (len > 0) goto grow;
 637             len = -len;
 638             if (len > 0) goto shrink;
 639             break;
 640
 641           case '%':
 642             rb_raise(rb_eArgError, "%% is not supported");
 643             break;
 644
 645           case 'U':             /* Unicode character */
 646             while (len-- > 0) {
 647                 SIGNED_VALUE l;
 648                 char buf[8];
 649                 int le;
 650
 651                 from = NEXTFROM;
 652                 from = rb_to_int(from);
 653                 l = NUM2LONG(from);
 654                 if (l < 0) {
 655                     rb_raise(rb_eRangeError, "pack(U): value out of range");
 656                 }
 657                 le = rb_uv_to_utf8(buf, l);
 658                 rb_str_buf_cat(res, (char*)buf, le);
 659             }
 660             break;
 661
 662           case 'u':             /* uuencoded string */
 663           case 'm':             /* base64 encoded string */
 664             from = NEXTFROM;
 665             StringValue(from);
 666             ptr = RSTRING_PTR(from);
 667             plen = RSTRING_LEN(from);
 668
 669             if (len == 0 && type == 'm') {
 670                 encodes(res, ptr, plen, type, 0);
 671                 ptr += plen;
 672                 break;
 673             }
 674             if (len <= 2)
 675                 len = 45;
 676             else if (len > 63 && type == 'u')
 677                 len = 63;
 678             else
 679                 len = len / 3 * 3;
 680             while (plen > 0) {
 681                 long todo;
 682
 683                 if (plen > len)
 684                     todo = len;
 685                 else
 686                     todo = plen;
 687                 encodes(res, ptr, todo, type, 1);
 688                 plen -= todo;
 689                 ptr += todo;
 690             }
 691             break;
 692
 693           case 'M':             /* quoted-printable encoded string */
 694             from = rb_obj_as_string(NEXTFROM);
 695             if (len <= 1)
 696                 len = 72;
 697             qpencode(res, from, len);
 698             break;
 699
 700           case 'P':             /* pointer to packed byte string */
 701             from = THISFROM;
 702             if (!NIL_P(from)) {
 703                 StringValue(from);
 704                 if (RSTRING_LEN(from) < len) {
 705                     rb_raise(rb_eArgError, "too short buffer for P(%ld for %ld)",
 706                              RSTRING_LEN(from), len);
 707                 }
 708             }
 709             len = 1;
 710             /* FALL THROUGH */
 711           case 'p':             /* pointer to string */
 712             while (len-- > 0) {
 713                 char *t;
 714                 from = NEXTFROM;
 715                 if (NIL_P(from)) {
 716                     t = 0;
 717                 }
 718                 else {
 719                     t = StringValuePtr(from);
 720                 }
 721                 if (!associates) {
 722                     associates = rb_ary_new();
 723                 }
 724                 rb_ary_push(associates, from);
 725                 rb_str_buf_cat(res, (char*)&t, sizeof(char*));
 726             }
 727             break;
 728
 729           case 'w':             /* BER compressed integer  */
 730             while (len-- > 0) {
 731                 VALUE buf = rb_str_new(0, 0);
 732                 size_t numbytes;
 733                 int sign;
 734                 char *cp;
 735
 736                 from = NEXTFROM;
 737                 from = rb_to_int(from);
 738                 numbytes = rb_absint_numwords(from, 7, NULL);
 739                 if (numbytes == 0)
 740                     numbytes = 1;
 741                 buf = rb_str_new(NULL, numbytes);
 742
 743                 sign = rb_integer_pack(from, RSTRING_PTR(buf), RSTRING_LEN(buf), 1, 1, INTEGER_PACK_BIG_ENDIAN);
 744
 745                 if (sign < 0)
 746                     rb_raise(rb_eArgError, "can't compress negative numbers");
 747                 if (sign == 2)
 748                     rb_bug("buffer size problem?");
 749
 750                 cp = RSTRING_PTR(buf);
 751                 while (1 < numbytes) {
 752                     *cp |= 0x80;
 753                     cp++;
 754                     numbytes--;
 755                 }
 756
 757                 rb_str_buf_cat(res, RSTRING_PTR(buf), RSTRING_LEN(buf));
 758             }
 759             break;
 760
 761           default: {
 762             unknown_directive("pack", type, fmt);
 763             break;
 764           }
 765         }
 766     }
 767
 768     if (associates) {
 769         str_associate(res, associates);
 770     }
 771     switch (enc_info) {
 772       case 1:
 773         ENCODING_CODERANGE_SET(res, rb_usascii_encindex(), ENC_CODERANGE_7BIT);
 774         break;
 775       case 2:
 776         rb_enc_set_index(res, rb_utf8_encindex());
 777         break;
 778       default:
 779         /* do nothing, keep ASCII-8BIT */
 780         break;
 781     }
 782     return res;
 783 }
 784
 785 static const char uu_table[] =
 786 "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
 787 static const char b64_table[] =
 788 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 789
 790 static void
 791 encodes(VALUE str, const char *s0, long len, int type, int tail_lf)
 792 {
 793     enum {buff_size = 4096, encoded_unit = 4, input_unit = 3};
 794     char buff[buff_size + 1];   /* +1 for tail_lf */
 795     long i = 0;
 796     const char *const trans = type == 'u' ? uu_table : b64_table;
 797     char padding;
 798     const unsigned char *s = (const unsigned char *)s0;
 799
 800     if (type == 'u') {
 801         buff[i++] = (char)len + ' ';
 802         padding = '`';
 803     }
 804     else {
 805         padding = '=';
 806     }
 807     while (len >= input_unit) {
 808         while (len >= input_unit && buff_size-i >= encoded_unit) {
 809             buff[i++] = trans[077 & (*s >> 2)];
 810             buff[i++] = trans[077 & (((*s << 4) & 060) | ((s[1] >> 4) & 017))];
 811             buff[i++] = trans[077 & (((s[1] << 2) & 074) | ((s[2] >> 6) & 03))];
 812             buff[i++] = trans[077 & s[2]];
 813             s += input_unit;
 814             len -= input_unit;
 815         }
 816         if (buff_size-i < encoded_unit) {
 817             rb_str_buf_cat(str, buff, i);
 818             i = 0;
 819         }
 820     }
 821
 822     if (len == 2) {
 823         buff[i++] = trans[077 & (*s >> 2)];
 824         buff[i++] = trans[077 & (((*s << 4) & 060) | ((s[1] >> 4) & 017))];
 825         buff[i++] = trans[077 & (((s[1] << 2) & 074) | (('\0' >> 6) & 03))];
 826         buff[i++] = padding;
 827     }
 828     else if (len == 1) {
 829         buff[i++] = trans[077 & (*s >> 2)];
 830         buff[i++] = trans[077 & (((*s << 4) & 060) | (('\0' >> 4) & 017))];
 831         buff[i++] = padding;
 832         buff[i++] = padding;
 833     }
 834     if (tail_lf) buff[i++] = '\n';
 835     rb_str_buf_cat(str, buff, i);
 836     if ((size_t)i > sizeof(buff)) rb_bug("encodes() buffer overrun");
 837 }
 838
 839 static const char hex_table[] = "0123456789ABCDEF";
 840
 841 static void
 842 qpencode(VALUE str, VALUE from, long len)
 843 {
 844     char buff[1024];
 845     long i = 0, n = 0, prev = EOF;
 846     unsigned char *s = (unsigned char*)RSTRING_PTR(from);
 847     unsigned char *send = s + RSTRING_LEN(from);
 848
 849     while (s < send) {
 850         if ((*s > 126) ||
 851             (*s < 32 && *s != '\n' && *s != '\t') ||
 852             (*s == '=')) {
 853             buff[i++] = '=';
 854             buff[i++] = hex_table[*s >> 4];
 855             buff[i++] = hex_table[*s & 0x0f];
 856             n += 3;
 857             prev = EOF;
 858         }
 859         else if (*s == '\n') {
 860             if (prev == ' ' || prev == '\t') {
 861                 buff[i++] = '=';
 862                 buff[i++] = *s;
 863             }
 864             buff[i++] = *s;
 865             n = 0;
 866             prev = *s;
 867         }
 868         else {
 869             buff[i++] = *s;
 870             n++;
 871             prev = *s;
 872         }
 873         if (n > len) {
 874             buff[i++] = '=';
 875             buff[i++] = '\n';
 876             n = 0;
 877             prev = '\n';
 878         }
 879         if (i > 1024 - 5) {
 880             rb_str_buf_cat(str, buff, i);
 881             i = 0;
 882         }
 883         s++;
 884     }
 885     if (n > 0) {
 886         buff[i++] = '=';
 887         buff[i++] = '\n';
 888     }
 889     if (i > 0) {
 890         rb_str_buf_cat(str, buff, i);
 891     }
 892 }
 893
 894 static inline int
 895 hex2num(char c)
 896 {
 897     int n;
 898     n = ruby_digit36_to_number_table[(unsigned char)c];
 899     if (16 <= n)
 900         n = -1;
 901     return n;
 902 }
 903
 904 #define PACK_LENGTH_ADJUST_SIZE(sz) do {        \
 905     tmp_len = 0;                                \
 906     if (len > (long)((send-s)/(sz))) {          \
 907         if (!star) {                            \
 908             tmp_len = len-(send-s)/(sz);        \
 909         }                                       \
 910         len = (send-s)/(sz);                    \
 911     }                                           \
 912 } while (0)
 913
 914 #define PACK_ITEM_ADJUST() do { \
 915     if (tmp_len > 0 && mode == UNPACK_ARRAY) \
 916         rb_ary_store(ary, RARRAY_LEN(ary)+tmp_len-1, Qnil); \
 917 } while (0)
 918
 919 /* Workaround for Oracle Developer Studio (Oracle Solaris Studio)
 920  * 12.4/12.5/12.6 C compiler optimization bug
 921  * with "-xO4" optimization option.
 922  */
 923 #if defined(__SUNPRO_C) && 0x5130 <= __SUNPRO_C && __SUNPRO_C <= 0x5150
 924 # define AVOID_CC_BUG volatile
 925 #else
 926 # define AVOID_CC_BUG
 927 #endif
 928
 929 enum unpack_mode {
 930     UNPACK_ARRAY,
 931     UNPACK_BLOCK,
 932     UNPACK_1
 933 };
 934
 935 static VALUE
 936 pack_unpack_internal(VALUE str, VALUE fmt, enum unpack_mode mode, long offset)
 937 {
 938 #define hexdigits ruby_hexdigits
 939     char *s, *send;
 940     char *p, *pend;
 941     VALUE ary, associates = Qfalse;
 942     char type;
 943     long len;
 944     AVOID_CC_BUG long tmp_len;
 945     int star;
 946 #ifdef NATINT_PACK
 947     int natint;                 /* native integer */
 948 #endif
 949     int signed_p, integer_size, bigendian_p;
 950 #define UNPACK_PUSH(item) do {\
 951         VALUE item_val = (item);\
 952         if ((mode) == UNPACK_BLOCK) {\
 953             rb_yield(item_val);\
 954         }\
 955         else if ((mode) == UNPACK_ARRAY) {\
 956             rb_ary_push(ary, item_val);\
 957         }\
 958         else /* if ((mode) == UNPACK_1) { */ {\
 959             return item_val; \
 960         }\
 961     } while (0)
 962
 963     StringValue(str);
 964     StringValue(fmt);
 965     rb_must_asciicompat(fmt);
 966
 967     if (offset < 0) rb_raise(rb_eArgError, "offset can't be negative");
 968     len = RSTRING_LEN(str);
 969     if (offset > len) rb_raise(rb_eArgError, "offset outside of string");
 970
 971     s = RSTRING_PTR(str);
 972     send = s + len;
 973     s += offset;
 974
 975     p = RSTRING_PTR(fmt);
 976     pend = p + RSTRING_LEN(fmt);
 977
 978 #define UNPACK_FETCH(var, type) (memcpy((var), s, sizeof(type)), s += sizeof(type))
 979
 980     ary = mode == UNPACK_ARRAY ? rb_ary_new() : Qnil;
 981     while (p < pend) {
 982         int explicit_endian = 0;
 983         type = *p++;
 984 #ifdef NATINT_PACK
 985         natint = 0;
 986 #endif
 987
 988         if (ISSPACE(type)) continue;
 989         if (type == '#') {
 990             while ((p < pend) && (*p != '\n')) {
 991                 p++;
 992             }
 993             continue;
 994         }
 995
 996         star = 0;
 997         {
 998           modifiers:
 999             switch (*p) {
1000               case '_':
1001               case '!':
1002
1003                 if (strchr(natstr, type)) {
1004 #ifdef NATINT_PACK
1005                     natint = 1;
1006 #endif
1007                     p++;
1008                 }
1009                 else {
1010                     rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, natstr);
1011                 }
1012                 goto modifiers;
1013
1014               case '<':
1015               case '>':
1016                 if (!strchr(endstr, type)) {
1017                     rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, endstr);
1018                 }
1019                 if (explicit_endian) {
1020                     rb_raise(rb_eRangeError, "Can't use both '<' and '>'");
1021                 }
1022                 explicit_endian = *p++;
1023                 goto modifiers;
1024             }
1025         }
1026
1027         if (p >= pend)
1028             len = 1;
1029         else if (*p == '*') {
1030             star = 1;
1031             len = send - s;
1032             p++;
1033         }
1034         else if (ISDIGIT(*p)) {
1035             errno = 0;
1036             len = STRTOUL(p, (char**)&p, 10);
1037             if (len < 0 || errno) {
1038                 rb_raise(rb_eRangeError, "pack length too big");
1039             }
1040         }
1041         else {
1042             len = (type != '@');
1043         }
1044
1045         switch (type) {
1046           case '%':
1047             rb_raise(rb_eArgError, "%% is not supported");
1048             break;
1049
1050           case 'A':
1051             if (len > send - s) len = send - s;
1052             {
1053                 long end = len;
1054                 char *t = s + len - 1;
1055
1056                 while (t >= s) {
1057                     if (*t != ' ' && *t != '\0') break;
1058                     t--; len--;
1059                 }
1060                 UNPACK_PUSH(rb_str_new(s, len));
1061                 s += end;
1062             }
1063             break;
1064
1065           case 'Z':
1066             {
1067                 char *t = s;
1068
1069                 if (len > send-s) len = send-s;
1070                 while (t < s+len && *t) t++;
1071                 UNPACK_PUSH(rb_str_new(s, t-s));
1072                 if (t < send) t++;
1073                 s = star ? t : s+len;
1074             }
1075             break;
1076
1077           case 'a':
1078             if (len > send - s) len = send - s;
1079             UNPACK_PUSH(rb_str_new(s, len));
1080             s += len;
1081             break;
1082
1083           case 'b':
1084             {
1085                 VALUE bitstr;
1086                 char *t;
1087                 int bits;
1088                 long i;
1089
1090                 if (p[-1] == '*' || len > (send - s) * 8)
1091                     len = (send - s) * 8;
1092                 bits = 0;
1093                 bitstr = rb_usascii_str_new(0, len);
1094                 t = RSTRING_PTR(bitstr);
1095                 for (i=0; i<len; i++) {
1096                     if (i & 7) bits >>= 1;
1097                     else bits = (unsigned char)*s++;
1098                     *t++ = (bits & 1) ? '1' : '0';
1099                 }
1100                 UNPACK_PUSH(bitstr);
1101             }
1102             break;
1103
1104           case 'B':
1105             {
1106                 VALUE bitstr;
1107                 char *t;
1108                 int bits;
1109                 long i;
1110
1111                 if (p[-1] == '*' || len > (send - s) * 8)
1112                     len = (send - s) * 8;
1113                 bits = 0;
1114                 bitstr = rb_usascii_str_new(0, len);
1115                 t = RSTRING_PTR(bitstr);
1116                 for (i=0; i<len; i++) {
1117                     if (i & 7) bits <<= 1;
1118                     else bits = (unsigned char)*s++;
1119                     *t++ = (bits & 128) ? '1' : '0';
1120                 }
1121                 UNPACK_PUSH(bitstr);
1122             }
1123             break;
1124
1125           case 'h':
1126             {
1127                 VALUE bitstr;
1128                 char *t;
1129                 int bits;
1130                 long i;
1131
1132                 if (p[-1] == '*' || len > (send - s) * 2)
1133                     len = (send - s) * 2;
1134                 bits = 0;
1135                 bitstr = rb_usascii_str_new(0, len);
1136                 t = RSTRING_PTR(bitstr);
1137                 for (i=0; i<len; i++) {
1138                     if (i & 1)
1139                         bits >>= 4;
1140                     else
1141                         bits = (unsigned char)*s++;
1142                     *t++ = hexdigits[bits & 15];
1143                 }
1144                 UNPACK_PUSH(bitstr);
1145             }
1146             break;
1147
1148           case 'H':
1149             {
1150                 VALUE bitstr;
1151                 char *t;
1152                 int bits;
1153                 long i;
1154
1155                 if (p[-1] == '*' || len > (send - s) * 2)
1156                     len = (send - s) * 2;
1157                 bits = 0;
1158                 bitstr = rb_usascii_str_new(0, len);
1159                 t = RSTRING_PTR(bitstr);
1160                 for (i=0; i<len; i++) {
1161                     if (i & 1)
1162                         bits <<= 4;
1163                     else
1164                         bits = (unsigned char)*s++;
1165                     *t++ = hexdigits[(bits >> 4) & 15];
1166                 }
1167                 UNPACK_PUSH(bitstr);
1168             }
1169             break;
1170
1171           case 'c':
1172             signed_p = 1;
1173             integer_size = 1;
1174             bigendian_p = BIGENDIAN_P(); /* not effective */
1175             goto unpack_integer;
1176
1177           case 'C':
1178             signed_p = 0;
1179             integer_size = 1;
1180             bigendian_p = BIGENDIAN_P(); /* not effective */
1181             goto unpack_integer;
1182
1183           case 's':
1184             signed_p = 1;
1185             integer_size = NATINT_LEN(short, 2);
1186             bigendian_p = BIGENDIAN_P();
1187             goto unpack_integer;
1188
1189           case 'S':
1190             signed_p = 0;
1191             integer_size = NATINT_LEN(short, 2);
1192             bigendian_p = BIGENDIAN_P();
1193             goto unpack_integer;
1194
1195           case 'i':
1196             signed_p = 1;
1197             integer_size = (int)sizeof(int);
1198             bigendian_p = BIGENDIAN_P();
1199             goto unpack_integer;
1200
1201           case 'I':
1202             signed_p = 0;
1203             integer_size = (int)sizeof(int);
1204             bigendian_p = BIGENDIAN_P();
1205             goto unpack_integer;
1206
1207           case 'l':
1208             signed_p = 1;
1209             integer_size = NATINT_LEN(long, 4);
1210             bigendian_p = BIGENDIAN_P();
1211             goto unpack_integer;
1212
1213           case 'L':
1214             signed_p = 0;
1215             integer_size = NATINT_LEN(long, 4);
1216             bigendian_p = BIGENDIAN_P();
1217             goto unpack_integer;
1218
1219           case 'q':
1220             signed_p = 1;
1221             integer_size = NATINT_LEN_Q;
1222             bigendian_p = BIGENDIAN_P();
1223             goto unpack_integer;
1224
1225           case 'Q':
1226             signed_p = 0;
1227             integer_size = NATINT_LEN_Q;
1228             bigendian_p = BIGENDIAN_P();
1229             goto unpack_integer;
1230
1231           case 'j':
1232             signed_p = 1;
1233             integer_size = sizeof(intptr_t);
1234             bigendian_p = BIGENDIAN_P();
1235             goto unpack_integer;
1236
1237           case 'J':
1238             signed_p = 0;
1239             integer_size = sizeof(uintptr_t);
1240             bigendian_p = BIGENDIAN_P();
1241             goto unpack_integer;
1242
1243           case 'n':
1244             signed_p = 0;
1245             integer_size = 2;
1246             bigendian_p = 1;
1247             goto unpack_integer;
1248
1249           case 'N':
1250             signed_p = 0;
1251             integer_size = 4;
1252             bigendian_p = 1;
1253             goto unpack_integer;
1254
1255           case 'v':
1256             signed_p = 0;
1257             integer_size = 2;
1258             bigendian_p = 0;
1259             goto unpack_integer;
1260
1261           case 'V':
1262             signed_p = 0;
1263             integer_size = 4;
1264             bigendian_p = 0;
1265             goto unpack_integer;
1266
1267           unpack_integer:
1268             if (explicit_endian) {
1269                 bigendian_p = explicit_endian == '>';
1270             }
1271             PACK_LENGTH_ADJUST_SIZE(integer_size);
1272             while (len-- > 0) {
1273                 int flags = bigendian_p ? INTEGER_PACK_BIG_ENDIAN : INTEGER_PACK_LITTLE_ENDIAN;
1274                 VALUE val;
1275                 if (signed_p)
1276                     flags |= INTEGER_PACK_2COMP;
1277                 val = rb_integer_unpack(s, integer_size, 1, 0, flags);
1278                 UNPACK_PUSH(val);
1279                 s += integer_size;
1280             }
1281             PACK_ITEM_ADJUST();
1282             break;
1283
1284           case 'f':
1285           case 'F':
1286             PACK_LENGTH_ADJUST_SIZE(sizeof(float));
1287             while (len-- > 0) {
1288                 float tmp;
1289                 UNPACK_FETCH(&tmp, float);
1290                 UNPACK_PUSH(DBL2NUM((double)tmp));
1291             }
1292             PACK_ITEM_ADJUST();
1293             break;
1294
1295           case 'e':
1296             PACK_LENGTH_ADJUST_SIZE(sizeof(float));
1297             while (len-- > 0) {
1298                 FLOAT_CONVWITH(tmp);
1299                 UNPACK_FETCH(tmp.buf, float);
1300                 VTOHF(tmp);
1301                 UNPACK_PUSH(DBL2NUM(tmp.f));
1302             }
1303             PACK_ITEM_ADJUST();
1304             break;
1305
1306           case 'E':
1307             PACK_LENGTH_ADJUST_SIZE(sizeof(double));
1308             while (len-- > 0) {
1309                 DOUBLE_CONVWITH(tmp);
1310                 UNPACK_FETCH(tmp.buf, double);
1311                 VTOHD(tmp);
1312                 UNPACK_PUSH(DBL2NUM(tmp.d));
1313             }
1314             PACK_ITEM_ADJUST();
1315             break;
1316
1317           case 'D':
1318           case 'd':
1319             PACK_LENGTH_ADJUST_SIZE(sizeof(double));
1320             while (len-- > 0) {
1321                 double tmp;
1322                 UNPACK_FETCH(&tmp, double);
1323                 UNPACK_PUSH(DBL2NUM(tmp));
1324             }
1325             PACK_ITEM_ADJUST();
1326             break;
1327
1328           case 'g':
1329             PACK_LENGTH_ADJUST_SIZE(sizeof(float));
1330             while (len-- > 0) {
1331                 FLOAT_CONVWITH(tmp);
1332                 UNPACK_FETCH(tmp.buf, float);
1333                 NTOHF(tmp);
1334                 UNPACK_PUSH(DBL2NUM(tmp.f));
1335             }
1336             PACK_ITEM_ADJUST();
1337             break;
1338
1339           case 'G':
1340             PACK_LENGTH_ADJUST_SIZE(sizeof(double));
1341             while (len-- > 0) {
1342                 DOUBLE_CONVWITH(tmp);
1343                 UNPACK_FETCH(tmp.buf, double);
1344                 NTOHD(tmp);
1345                 UNPACK_PUSH(DBL2NUM(tmp.d));
1346             }
1347             PACK_ITEM_ADJUST();
1348             break;
1349
1350           case 'U':
1351             if (len > send - s) len = send - s;
1352             while (len > 0 && s < send) {
1353                 long alen = send - s;
1354                 unsigned long l;
1355
1356                 l = utf8_to_uv(s, &alen);
1357                 s += alen; len--;
1358                 UNPACK_PUSH(ULONG2NUM(l));
1359             }
1360             break;
1361
1362           case 'u':
1363             {
1364                 VALUE buf = rb_str_new(0, (send - s)*3/4);
1365                 char *ptr = RSTRING_PTR(buf);
1366                 long total = 0;
1367
1368                 while (s < send && (unsigned char)*s > ' ' && (unsigned char)*s < 'a') {
1369                     long a,b,c,d;
1370                     char hunk[3];
1371
1372                     len = ((unsigned char)*s++ - ' ') & 077;
1373
1374                     total += len;
1375                     if (total > RSTRING_LEN(buf)) {
1376                         len -= total - RSTRING_LEN(buf);
1377                         total = RSTRING_LEN(buf);
1378                     }
1379
1380                     while (len > 0) {
1381                         long mlen = len > 3 ? 3 : len;
1382
1383                         if (s < send && (unsigned char)*s >= ' ' && (unsigned char)*s < 'a')
1384                             a = ((unsigned char)*s++ - ' ') & 077;
1385                         else
1386                             a = 0;
1387                         if (s < send && (unsigned char)*s >= ' ' && (unsigned char)*s < 'a')
1388                             b = ((unsigned char)*s++ - ' ') & 077;
1389                         else
1390                             b = 0;
1391                         if (s < send && (unsigned char)*s >= ' ' && (unsigned char)*s < 'a')
1392                             c = ((unsigned char)*s++ - ' ') & 077;
1393                         else
1394                             c = 0;
1395                         if (s < send && (unsigned char)*s >= ' ' && (unsigned char)*s < 'a')
1396                             d = ((unsigned char)*s++ - ' ') & 077;
1397                         else
1398                             d = 0;
1399                         hunk[0] = (char)(a << 2 | b >> 4);
1400                         hunk[1] = (char)(b << 4 | c >> 2);
1401                         hunk[2] = (char)(c << 6 | d);
1402                         memcpy(ptr, hunk, mlen);
1403                         ptr += mlen;
1404                         len -= mlen;
1405                     }
1406                     if (s < send && (unsigned char)*s != '\r' && *s != '\n')
1407                         s++;    /* possible checksum byte */
1408                     if (s < send && *s == '\r') s++;
1409                     if (s < send && *s == '\n') s++;
1410                 }
1411
1412                 rb_str_set_len(buf, total);
1413                 UNPACK_PUSH(buf);
1414             }
1415             break;
1416
1417           case 'm':
1418             {
1419                 VALUE buf = rb_str_new(0, (send - s + 3)*3/4); /* +3 is for skipping paddings */
1420                 char *ptr = RSTRING_PTR(buf);
1421                 int a = -1,b = -1,c = 0,d = 0;
1422                 static signed char b64_xtable[256];
1423
1424                 if (b64_xtable['/'] <= 0) {
1425                     int i;
1426
1427                     for (i = 0; i < 256; i++) {
1428                         b64_xtable[i] = -1;
1429                     }
1430                     for (i = 0; i < 64; i++) {
1431                         b64_xtable[(unsigned char)b64_table[i]] = (char)i;
1432                     }
1433                 }
1434                 if (len == 0) {
1435                     while (s < send) {
1436                         a = b = c = d = -1;
1437                         a = b64_xtable[(unsigned char)*s++];
1438                         if (s >= send || a == -1) rb_raise(rb_eArgError, "invalid base64");
1439                         b = b64_xtable[(unsigned char)*s++];
1440                         if (s >= send || b == -1) rb_raise(rb_eArgError, "invalid base64");
1441                         if (*s == '=') {
1442                             if (s + 2 == send && *(s + 1) == '=') break;
1443                             rb_raise(rb_eArgError, "invalid base64");
1444                         }
1445                         c = b64_xtable[(unsigned char)*s++];
1446                         if (s >= send || c == -1) rb_raise(rb_eArgError, "invalid base64");
1447                         if (s + 1 == send && *s == '=') break;
1448                         d = b64_xtable[(unsigned char)*s++];
1449                         if (d == -1) rb_raise(rb_eArgError, "invalid base64");
1450                         *ptr++ = castchar(a << 2 | b >> 4);
1451                         *ptr++ = castchar(b << 4 | c >> 2);
1452                         *ptr++ = castchar(c << 6 | d);
1453                     }
1454                     if (c == -1) {
1455                         *ptr++ = castchar(a << 2 | b >> 4);
1456                         if (b & 0xf) rb_raise(rb_eArgError, "invalid base64");
1457                     }
1458                     else if (d == -1) {
1459                         *ptr++ = castchar(a << 2 | b >> 4);
1460                         *ptr++ = castchar(b << 4 | c >> 2);
1461                         if (c & 0x3) rb_raise(rb_eArgError, "invalid base64");
1462                     }
1463                 }
1464                 else {
1465                     while (s < send) {
1466                         a = b = c = d = -1;
1467                         while ((a = b64_xtable[(unsigned char)*s]) == -1 && s < send) {s++;}
1468                         if (s >= send) break;
1469                         s++;
1470                         while ((b = b64_xtable[(unsigned char)*s]) == -1 && s < send) {s++;}
1471                         if (s >= send) break;
1472                         s++;
1473                         while ((c = b64_xtable[(unsigned char)*s]) == -1 && s < send) {if (*s == '=') break; s++;}
1474                         if (*s == '=' || s >= send) break;
1475                         s++;
1476                         while ((d = b64_xtable[(unsigned char)*s]) == -1 && s < send) {if (*s == '=') break; s++;}
1477                         if (*s == '=' || s >= send) break;
1478                         s++;
1479                         *ptr++ = castchar(a << 2 | b >> 4);
1480                         *ptr++ = castchar(b << 4 | c >> 2);
1481                         *ptr++ = castchar(c << 6 | d);
1482                         a = -1;
1483                     }
1484                     if (a != -1 && b != -1) {
1485                         if (c == -1)
1486                             *ptr++ = castchar(a << 2 | b >> 4);
1487                         else {
1488                             *ptr++ = castchar(a << 2 | b >> 4);
1489                             *ptr++ = castchar(b << 4 | c >> 2);
1490                         }
1491                     }
1492                 }
1493                 rb_str_set_len(buf, ptr - RSTRING_PTR(buf));
1494                 UNPACK_PUSH(buf);
1495             }
1496             break;
1497
1498           case 'M':
1499             {
1500                 VALUE buf = rb_str_new(0, send - s);
1501                 char *ptr = RSTRING_PTR(buf), *ss = s;
1502                 int csum = 0;
1503                 int c1, c2;
1504
1505                 while (s < send) {
1506                     if (*s == '=') {
1507                         if (++s == send) break;
1508                         if (s+1 < send && *s == '\r' && *(s+1) == '\n')
1509                             s++;
1510                         if (*s != '\n') {
1511                             if ((c1 = hex2num(*s)) == -1) break;
1512                             if (++s == send) break;
1513                             if ((c2 = hex2num(*s)) == -1) break;
1514                             csum |= *ptr++ = castchar(c1 << 4 | c2);
1515                         }
1516                     }
1517                     else {
1518                         csum |= *ptr++ = *s;
1519                     }
1520                     s++;
1521                     ss = s;
1522                 }
1523                 rb_str_set_len(buf, ptr - RSTRING_PTR(buf));
1524                 rb_str_buf_cat(buf, ss, send-ss);
1525                 csum = ISASCII(csum) ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID;
1526                 ENCODING_CODERANGE_SET(buf, rb_ascii8bit_encindex(), csum);
1527                 UNPACK_PUSH(buf);
1528             }
1529             break;
1530
1531           case '@':
1532             if (len > RSTRING_LEN(str))
1533                 rb_raise(rb_eArgError, "@ outside of string");
1534             s = RSTRING_PTR(str) + len;
1535             break;
1536
1537           case 'X':
1538             if (len > s - RSTRING_PTR(str))
1539                 rb_raise(rb_eArgError, "X outside of string");
1540             s -= len;
1541             break;
1542
1543           case 'x':
1544             if (len > send - s)
1545                 rb_raise(rb_eArgError, "x outside of string");
1546             s += len;
1547             break;
1548
1549           case 'P':
1550             if (sizeof(char *) <= (size_t)(send - s)) {
1551                 VALUE tmp = Qnil;
1552                 char *t;
1553
1554                 UNPACK_FETCH(&t, char *);
1555                 if (t) {
1556                     if (!associates) associates = str_associated(str);
1557                     tmp = associated_pointer(associates, t);
1558                     if (len < RSTRING_LEN(tmp)) {
1559                         tmp = rb_str_new(t, len);
1560                         str_associate(tmp, associates);
1561                     }
1562                 }
1563                 UNPACK_PUSH(tmp);
1564             }
1565             break;
1566
1567           case 'p':
1568             if (len > (long)((send - s) / sizeof(char *)))
1569                 len = (send - s) / sizeof(char *);
1570             while (len-- > 0) {
1571                 if ((size_t)(send - s) < sizeof(char *))
1572                     break;
1573                 else {
1574                     VALUE tmp = Qnil;
1575                     char *t;
1576
1577                     UNPACK_FETCH(&t, char *);
1578                     if (t) {
1579                         if (!associates) associates = str_associated(str);
1580                         tmp = associated_pointer(associates, t);
1581                     }
1582                     UNPACK_PUSH(tmp);
1583                 }
1584             }
1585             break;
1586
1587           case 'w':
1588             {
1589                 char *s0 = s;
1590                 while (len > 0 && s < send) {
1591                     if (*s & 0x80) {
1592                         s++;
1593                     }
1594                     else {
1595                         s++;
1596                         UNPACK_PUSH(rb_integer_unpack(s0, s-s0, 1, 1, INTEGER_PACK_BIG_ENDIAN));
1597                         len--;
1598                         s0 = s;
1599                     }
1600                 }
1601             }
1602             break;
1603
1604           default:
1605             unknown_directive("unpack", type, fmt);
1606             break;
1607         }
1608     }
1609
1610     return ary;
1611 }
1612
1613 static VALUE
1614 pack_unpack(rb_execution_context_t *ec, VALUE str, VALUE fmt, VALUE offset)
1615 {
1616     enum unpack_mode mode = rb_block_given_p() ? UNPACK_BLOCK : UNPACK_ARRAY;
1617     return pack_unpack_internal(str, fmt, mode, RB_NUM2LONG(offset));
1618 }
1619
1620 static VALUE
1621 pack_unpack1(rb_execution_context_t *ec, VALUE str, VALUE fmt, VALUE offset)
1622 {
1623     return pack_unpack_internal(str, fmt, UNPACK_1, RB_NUM2LONG(offset));
1624 }
1625
1626 int
1627 rb_uv_to_utf8(char buf[6], unsigned long uv)
1628 {
1629     if (uv <= 0x7f) {
1630         buf[0] = (char)uv;
1631         return 1;
1632     }
1633     if (uv <= 0x7ff) {
1634         buf[0] = castchar(((uv>>6)&0xff)|0xc0);
1635         buf[1] = castchar((uv&0x3f)|0x80);
1636         return 2;
1637     }
1638     if (uv <= 0xffff) {
1639         buf[0] = castchar(((uv>>12)&0xff)|0xe0);
1640         buf[1] = castchar(((uv>>6)&0x3f)|0x80);
1641         buf[2] = castchar((uv&0x3f)|0x80);
1642         return 3;
1643     }
1644     if (uv <= 0x1fffff) {
1645         buf[0] = castchar(((uv>>18)&0xff)|0xf0);
1646         buf[1] = castchar(((uv>>12)&0x3f)|0x80);
1647         buf[2] = castchar(((uv>>6)&0x3f)|0x80);
1648         buf[3] = castchar((uv&0x3f)|0x80);
1649         return 4;
1650     }
1651     if (uv <= 0x3ffffff) {
1652         buf[0] = castchar(((uv>>24)&0xff)|0xf8);
1653         buf[1] = castchar(((uv>>18)&0x3f)|0x80);
1654         buf[2] = castchar(((uv>>12)&0x3f)|0x80);
1655         buf[3] = castchar(((uv>>6)&0x3f)|0x80);
1656         buf[4] = castchar((uv&0x3f)|0x80);
1657         return 5;
1658     }
1659     if (uv <= 0x7fffffff) {
1660         buf[0] = castchar(((uv>>30)&0xff)|0xfc);
1661         buf[1] = castchar(((uv>>24)&0x3f)|0x80);
1662         buf[2] = castchar(((uv>>18)&0x3f)|0x80);
1663         buf[3] = castchar(((uv>>12)&0x3f)|0x80);
1664         buf[4] = castchar(((uv>>6)&0x3f)|0x80);
1665         buf[5] = castchar((uv&0x3f)|0x80);
1666         return 6;
1667     }
1668     rb_raise(rb_eRangeError, "pack(U): value out of range");
1669
1670     UNREACHABLE_RETURN(Qnil);
1671 }
1672
1673 static const unsigned long utf8_limits[] = {
1674     0x0,                        /* 1 */
1675     0x80,                       /* 2 */
1676     0x800,                      /* 3 */
1677     0x10000,                    /* 4 */
1678     0x200000,                   /* 5 */
1679     0x4000000,                  /* 6 */
1680     0x80000000,                 /* 7 */
1681 };
1682
1683 static unsigned long
1684 utf8_to_uv(const char *p, long *lenp)
1685 {
1686     int c = *p++ & 0xff;
1687     unsigned long uv = c;
1688     long n;
1689
1690     if (!(uv & 0x80)) {
1691         *lenp = 1;
1692         return uv;
1693     }
1694     if (!(uv & 0x40)) {
1695         *lenp = 1;
1696         rb_raise(rb_eArgError, "malformed UTF-8 character");
1697     }
1698
1699     if      (!(uv & 0x20)) { n = 2; uv &= 0x1f; }
1700     else if (!(uv & 0x10)) { n = 3; uv &= 0x0f; }
1701     else if (!(uv & 0x08)) { n = 4; uv &= 0x07; }
1702     else if (!(uv & 0x04)) { n = 5; uv &= 0x03; }
1703     else if (!(uv & 0x02)) { n = 6; uv &= 0x01; }
1704     else {
1705         *lenp = 1;
1706         rb_raise(rb_eArgError, "malformed UTF-8 character");
1707     }
1708     if (n > *lenp) {
1709         rb_raise(rb_eArgError, "malformed UTF-8 character (expected %ld bytes, given %ld bytes)",
1710                  n, *lenp);
1711     }
1712     *lenp = n--;
1713     if (n != 0) {
1714         while (n--) {
1715             c = *p++ & 0xff;
1716             if ((c & 0xc0) != 0x80) {
1717                 *lenp -= n + 1;
1718                 rb_raise(rb_eArgError, "malformed UTF-8 character");
1719             }
1720             else {
1721                 c &= 0x3f;
1722                 uv = uv << 6 | c;
1723             }
1724         }
1725     }
1726     n = *lenp - 1;
1727     if (uv < utf8_limits[n]) {
1728         rb_raise(rb_eArgError, "redundant UTF-8 sequence");
1729     }
1730     return uv;
1731 }
1732
1733 #include "pack.rbinc"
1734
1735 void
1736 Init_pack(void)
1737 {
1738     id_associated = rb_make_internal_id();
1739 }