src/include/varatt.h

   1 /*-------------------------------------------------------------------------
   2  *
   3  * varatt.h
   4  *        variable-length datatypes (TOAST support)
   5  *
   6  *
   7  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
   8  * Portions Copyright (c) 1995, Regents of the University of California
   9  *
  10  * src/include/varatt.h
  11  *
  12  *-------------------------------------------------------------------------
  13  */
  14
  15 #ifndef VARATT_H
  16 #define VARATT_H
  17
  18 /*
  19  * struct varatt_external is a traditional "TOAST pointer", that is, the
  20  * information needed to fetch a Datum stored out-of-line in a TOAST table.
  21  * The data is compressed if and only if the external size stored in
  22  * va_extinfo is less than va_rawsize - VARHDRSZ.
  23  *
  24  * This struct must not contain any padding, because we sometimes compare
  25  * these pointers using memcmp.
  26  *
  27  * Note that this information is stored unaligned within actual tuples, so
  28  * you need to memcpy from the tuple into a local struct variable before
  29  * you can look at these fields!  (The reason we use memcmp is to avoid
  30  * having to do that just to detect equality of two TOAST pointers...)
  31  */
  32 typedef struct varatt_external
  33 {
  34         int32           va_rawsize;             /* Original data size (includes header) */
  35         uint32          va_extinfo;             /* External saved size (without header) and
  36                                                                  * compression method */
  37         Oid                     va_valueid;             /* Unique ID of value within TOAST table */
  38         Oid                     va_toastrelid;  /* RelID of TOAST table containing it */
  39 }                       varatt_external;
  40
  41 /*
  42  * These macros define the "saved size" portion of va_extinfo.  Its remaining
  43  * two high-order bits identify the compression method.
  44  */
  45 #define VARLENA_EXTSIZE_BITS    30
  46 #define VARLENA_EXTSIZE_MASK    ((1U << VARLENA_EXTSIZE_BITS) - 1)
  47
  48 /*
  49  * struct varatt_indirect is a "TOAST pointer" representing an out-of-line
  50  * Datum that's stored in memory, not in an external toast relation.
  51  * The creator of such a Datum is entirely responsible that the referenced
  52  * storage survives for as long as referencing pointer Datums can exist.
  53  *
  54  * Note that just as for struct varatt_external, this struct is stored
  55  * unaligned within any containing tuple.
  56  */
  57 typedef struct varatt_indirect
  58 {
  59         struct varlena *pointer;        /* Pointer to in-memory varlena */
  60 }                       varatt_indirect;
  61
  62 /*
  63  * struct varatt_expanded is a "TOAST pointer" representing an out-of-line
  64  * Datum that is stored in memory, in some type-specific, not necessarily
  65  * physically contiguous format that is convenient for computation not
  66  * storage.  APIs for this, in particular the definition of struct
  67  * ExpandedObjectHeader, are in src/include/utils/expandeddatum.h.
  68  *
  69  * Note that just as for struct varatt_external, this struct is stored
  70  * unaligned within any containing tuple.
  71  */
  72 typedef struct ExpandedObjectHeader ExpandedObjectHeader;
  73
  74 typedef struct varatt_expanded
  75 {
  76         ExpandedObjectHeader *eohptr;
  77 } varatt_expanded;
  78
  79 /*
  80  * Type tag for the various sorts of "TOAST pointer" datums.  The peculiar
  81  * value for VARTAG_ONDISK comes from a requirement for on-disk compatibility
  82  * with a previous notion that the tag field was the pointer datum's length.
  83  */
  84 typedef enum vartag_external
  85 {
  86         VARTAG_INDIRECT = 1,
  87         VARTAG_EXPANDED_RO = 2,
  88         VARTAG_EXPANDED_RW = 3,
  89         VARTAG_ONDISK = 18
  90 } vartag_external;
  91
  92 /* this test relies on the specific tag values above */
  93 #define VARTAG_IS_EXPANDED(tag) \
  94         (((tag) & ~1) == VARTAG_EXPANDED_RO)
  95
  96 #define VARTAG_SIZE(tag) \
  97         ((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \
  98          VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \
  99          (tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \
 100          (AssertMacro(false), 0))
 101
 102 /*
 103  * These structs describe the header of a varlena object that may have been
 104  * TOASTed.  Generally, don't reference these structs directly, but use the
 105  * macros below.
 106  *
 107  * We use separate structs for the aligned and unaligned cases because the
 108  * compiler might otherwise think it could generate code that assumes
 109  * alignment while touching fields of a 1-byte-header varlena.
 110  */
 111 typedef union
 112 {
 113         struct                                          /* Normal varlena (4-byte length) */
 114         {
 115                 uint32          va_header;
 116                 char            va_data[FLEXIBLE_ARRAY_MEMBER];
 117         }                       va_4byte;
 118         struct                                          /* Compressed-in-line format */
 119         {
 120                 uint32          va_header;
 121                 uint32          va_tcinfo;      /* Original data size (excludes header) and
 122                                                                  * compression method; see va_extinfo */
 123                 char            va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
 124         }                       va_compressed;
 125 } varattrib_4b;
 126
 127 typedef struct
 128 {
 129         uint8           va_header;
 130         char            va_data[FLEXIBLE_ARRAY_MEMBER]; /* Data begins here */
 131 } varattrib_1b;
 132
 133 /* TOAST pointers are a subset of varattrib_1b with an identifying tag byte */
 134 typedef struct
 135 {
 136         uint8           va_header;              /* Always 0x80 or 0x01 */
 137         uint8           va_tag;                 /* Type of datum */
 138         char            va_data[FLEXIBLE_ARRAY_MEMBER]; /* Type-specific data */
 139 } varattrib_1b_e;
 140
 141 /*
 142  * Bit layouts for varlena headers on big-endian machines:
 143  *
 144  * 00xxxxxx 4-byte length word, aligned, uncompressed data (up to 1G)
 145  * 01xxxxxx 4-byte length word, aligned, *compressed* data (up to 1G)
 146  * 10000000 1-byte length word, unaligned, TOAST pointer
 147  * 1xxxxxxx 1-byte length word, unaligned, uncompressed data (up to 126b)
 148  *
 149  * Bit layouts for varlena headers on little-endian machines:
 150  *
 151  * xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G)
 152  * xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G)
 153  * 00000001 1-byte length word, unaligned, TOAST pointer
 154  * xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to 126b)
 155  *
 156  * The "xxx" bits are the length field (which includes itself in all cases).
 157  * In the big-endian case we mask to extract the length, in the little-endian
 158  * case we shift.  Note that in both cases the flag bits are in the physically
 159  * first byte.  Also, it is not possible for a 1-byte length word to be zero;
 160  * this lets us disambiguate alignment padding bytes from the start of an
 161  * unaligned datum.  (We now *require* pad bytes to be filled with zero!)
 162  *
 163  * In TOAST pointers the va_tag field (see varattrib_1b_e) is used to discern
 164  * the specific type and length of the pointer datum.
 165  */
 166
 167 /*
 168  * Endian-dependent macros.  These are considered internal --- use the
 169  * external macros below instead of using these directly.
 170  *
 171  * Note: IS_1B is true for external toast records but VARSIZE_1B will return 0
 172  * for such records. Hence you should usually check for IS_EXTERNAL before
 173  * checking for IS_1B.
 174  */
 175
 176 #ifdef WORDS_BIGENDIAN
 177
 178 #define VARATT_IS_4B(PTR) \
 179         ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00)
 180 #define VARATT_IS_4B_U(PTR) \
 181         ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x00)
 182 #define VARATT_IS_4B_C(PTR) \
 183         ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x40)
 184 #define VARATT_IS_1B(PTR) \
 185         ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x80)
 186 #define VARATT_IS_1B_E(PTR) \
 187         ((((varattrib_1b *) (PTR))->va_header) == 0x80)
 188 #define VARATT_NOT_PAD_BYTE(PTR) \
 189         (*((uint8 *) (PTR)) != 0)
 190
 191 /* VARSIZE_4B() should only be used on known-aligned data */
 192 #define VARSIZE_4B(PTR) \
 193         (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
 194 #define VARSIZE_1B(PTR) \
 195         (((varattrib_1b *) (PTR))->va_header & 0x7F)
 196 #define VARTAG_1B_E(PTR) \
 197         (((varattrib_1b_e *) (PTR))->va_tag)
 198
 199 #define SET_VARSIZE_4B(PTR,len) \
 200         (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
 201 #define SET_VARSIZE_4B_C(PTR,len) \
 202         (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
 203 #define SET_VARSIZE_1B(PTR,len) \
 204         (((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
 205 #define SET_VARTAG_1B_E(PTR,tag) \
 206         (((varattrib_1b_e *) (PTR))->va_header = 0x80, \
 207          ((varattrib_1b_e *) (PTR))->va_tag = (tag))
 208
 209 #else                                                   /* !WORDS_BIGENDIAN */
 210
 211 #define VARATT_IS_4B(PTR) \
 212         ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00)
 213 #define VARATT_IS_4B_U(PTR) \
 214         ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x00)
 215 #define VARATT_IS_4B_C(PTR) \
 216         ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x02)
 217 #define VARATT_IS_1B(PTR) \
 218         ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x01)
 219 #define VARATT_IS_1B_E(PTR) \
 220         ((((varattrib_1b *) (PTR))->va_header) == 0x01)
 221 #define VARATT_NOT_PAD_BYTE(PTR) \
 222         (*((uint8 *) (PTR)) != 0)
 223
 224 /* VARSIZE_4B() should only be used on known-aligned data */
 225 #define VARSIZE_4B(PTR) \
 226         ((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
 227 #define VARSIZE_1B(PTR) \
 228         ((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
 229 #define VARTAG_1B_E(PTR) \
 230         (((varattrib_1b_e *) (PTR))->va_tag)
 231
 232 #define SET_VARSIZE_4B(PTR,len) \
 233         (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
 234 #define SET_VARSIZE_4B_C(PTR,len) \
 235         (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
 236 #define SET_VARSIZE_1B(PTR,len) \
 237         (((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
 238 #define SET_VARTAG_1B_E(PTR,tag) \
 239         (((varattrib_1b_e *) (PTR))->va_header = 0x01, \
 240          ((varattrib_1b_e *) (PTR))->va_tag = (tag))
 241
 242 #endif                                                  /* WORDS_BIGENDIAN */
 243
 244 #define VARDATA_4B(PTR)         (((varattrib_4b *) (PTR))->va_4byte.va_data)
 245 #define VARDATA_4B_C(PTR)       (((varattrib_4b *) (PTR))->va_compressed.va_data)
 246 #define VARDATA_1B(PTR)         (((varattrib_1b *) (PTR))->va_data)
 247 #define VARDATA_1B_E(PTR)       (((varattrib_1b_e *) (PTR))->va_data)
 248
 249 /*
 250  * Externally visible TOAST macros begin here.
 251  */
 252
 253 #define VARHDRSZ_EXTERNAL               offsetof(varattrib_1b_e, va_data)
 254 #define VARHDRSZ_COMPRESSED             offsetof(varattrib_4b, va_compressed.va_data)
 255 #define VARHDRSZ_SHORT                  offsetof(varattrib_1b, va_data)
 256
 257 #define VARATT_SHORT_MAX                0x7F
 258 #define VARATT_CAN_MAKE_SHORT(PTR) \
 259         (VARATT_IS_4B_U(PTR) && \
 260          (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) <= VARATT_SHORT_MAX)
 261 #define VARATT_CONVERTED_SHORT_SIZE(PTR) \
 262         (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT)
 263
 264 /*
 265  * In consumers oblivious to data alignment, call PG_DETOAST_DATUM_PACKED(),
 266  * VARDATA_ANY(), VARSIZE_ANY() and VARSIZE_ANY_EXHDR().  Elsewhere, call
 267  * PG_DETOAST_DATUM(), VARDATA() and VARSIZE().  Directly fetching an int16,
 268  * int32 or wider field in the struct representing the datum layout requires
 269  * aligned data.  memcpy() is alignment-oblivious, as are most operations on
 270  * datatypes, such as text, whose layout struct contains only char fields.
 271  *
 272  * Code assembling a new datum should call VARDATA() and SET_VARSIZE().
 273  * (Datums begin life untoasted.)
 274  *
 275  * Other macros here should usually be used only by tuple assembly/disassembly
 276  * code and code that specifically wants to work with still-toasted Datums.
 277  */
 278 #define VARDATA(PTR)                                            VARDATA_4B(PTR)
 279 #define VARSIZE(PTR)                                            VARSIZE_4B(PTR)
 280
 281 #define VARSIZE_SHORT(PTR)                                      VARSIZE_1B(PTR)
 282 #define VARDATA_SHORT(PTR)                                      VARDATA_1B(PTR)
 283
 284 #define VARTAG_EXTERNAL(PTR)                            VARTAG_1B_E(PTR)
 285 #define VARSIZE_EXTERNAL(PTR)                           (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR)))
 286 #define VARDATA_EXTERNAL(PTR)                           VARDATA_1B_E(PTR)
 287
 288 #define VARATT_IS_COMPRESSED(PTR)                       VARATT_IS_4B_C(PTR)
 289 #define VARATT_IS_EXTERNAL(PTR)                         VARATT_IS_1B_E(PTR)
 290 #define VARATT_IS_EXTERNAL_ONDISK(PTR) \
 291         (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK)
 292 #define VARATT_IS_EXTERNAL_INDIRECT(PTR) \
 293         (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT)
 294 #define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \
 295         (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RO)
 296 #define VARATT_IS_EXTERNAL_EXPANDED_RW(PTR) \
 297         (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RW)
 298 #define VARATT_IS_EXTERNAL_EXPANDED(PTR) \
 299         (VARATT_IS_EXTERNAL(PTR) && VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
 300 #define VARATT_IS_EXTERNAL_NON_EXPANDED(PTR) \
 301         (VARATT_IS_EXTERNAL(PTR) && !VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
 302 #define VARATT_IS_SHORT(PTR)                            VARATT_IS_1B(PTR)
 303 #define VARATT_IS_EXTENDED(PTR)                         (!VARATT_IS_4B_U(PTR))
 304
 305 #define SET_VARSIZE(PTR, len)                           SET_VARSIZE_4B(PTR, len)
 306 #define SET_VARSIZE_SHORT(PTR, len)                     SET_VARSIZE_1B(PTR, len)
 307 #define SET_VARSIZE_COMPRESSED(PTR, len)        SET_VARSIZE_4B_C(PTR, len)
 308
 309 #define SET_VARTAG_EXTERNAL(PTR, tag)           SET_VARTAG_1B_E(PTR, tag)
 310
 311 #define VARSIZE_ANY(PTR) \
 312         (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR) : \
 313          (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \
 314           VARSIZE_4B(PTR)))
 315
 316 /* Size of a varlena data, excluding header */
 317 #define VARSIZE_ANY_EXHDR(PTR) \
 318         (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR)-VARHDRSZ_EXTERNAL : \
 319          (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-VARHDRSZ_SHORT : \
 320           VARSIZE_4B(PTR)-VARHDRSZ))
 321
 322 /* caution: this will not work on an external or compressed-in-line Datum */
 323 /* caution: this will return a possibly unaligned pointer */
 324 #define VARDATA_ANY(PTR) \
 325          (VARATT_IS_1B(PTR) ? VARDATA_1B(PTR) : VARDATA_4B(PTR))
 326
 327 /* Decompressed size and compression method of a compressed-in-line Datum */
 328 #define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \
 329         (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK)
 330 #define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
 331         (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
 332
 333 /* Same for external Datums; but note argument is a struct varatt_external */
 334 #define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
 335         ((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK)
 336 #define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \
 337         ((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS)
 338
 339 #define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
 340         do { \
 341                 Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
 342                            (cm) == TOAST_LZ4_COMPRESSION_ID); \
 343                 ((toast_pointer).va_extinfo = \
 344                         (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
 345         } while (0)
 346
 347 /*
 348  * Testing whether an externally-stored value is compressed now requires
 349  * comparing size stored in va_extinfo (the actual length of the external data)
 350  * to rawsize (the original uncompressed datum's size).  The latter includes
 351  * VARHDRSZ overhead, the former doesn't.  We never use compression unless it
 352  * actually saves space, so we expect either equality or less-than.
 353  */
 354 #define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
 355         (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \
 356          (toast_pointer).va_rawsize - VARHDRSZ)
 357
 358 #endif