gvdb-builder.c

   1 /*
   2  * Copyright © 2010 Codethink Limited
   3  *
   4  * This library is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the licence, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with this library; if not, write to the
  16  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  17  * Boston, MA 02111-1307, USA.
  18  *
  19  * Author: Ryan Lortie <desrt@desrt.ca>
  20  */
  21
  22 #include "gvdb-builder.h"
  23 #include "gvdb-format.h"
  24
  25 #include <glib.h>
  26 #include <fcntl.h>
  27 #if !defined(G_OS_WIN32) || !defined(_MSC_VER)
  28 #include <unistd.h>
  29 #endif
  30 #include <string.h>
  31
  32
  33 struct _GvdbItem
  34 {
  35   gchar *key;
  36   guint32 hash_value;
  37   guint32_le assigned_index;
  38   GvdbItem *parent;
  39   GvdbItem *sibling;
  40   GvdbItem *next;
  41
  42   /* one of:
  43    * this:
  44    */
  45   GVariant *value;
  46
  47   /* this: */
  48   GHashTable *table;
  49
  50   /* or this: */
  51   GvdbItem *child;
  52 };
  53
  54 static void
  55 gvdb_item_free (gpointer data)
  56 {
  57   GvdbItem *item = data;
  58
  59   g_free (item->key);
  60
  61   if (item->value)
  62     g_variant_unref (item->value);
  63
  64   if (item->table)
  65     g_hash_table_unref (item->table);
  66
  67   g_slice_free (GvdbItem, item);
  68 }
  69
  70 GHashTable *
  71 gvdb_hash_table_new (GHashTable  *parent,
  72                      const gchar *name_in_parent)
  73 {
  74   GHashTable *table;
  75
  76   table = g_hash_table_new_full (g_str_hash, g_str_equal,
  77                                  g_free, gvdb_item_free);
  78
  79   if (parent)
  80     {
  81       GvdbItem *item;
  82
  83       item = gvdb_hash_table_insert (parent, name_in_parent);
  84       gvdb_item_set_hash_table (item, table);
  85     }
  86
  87   return table;
  88 }
  89
  90 static guint32
  91 djb_hash (const gchar *key)
  92 {
  93   guint32 hash_value = 5381;
  94
  95   while (*key)
  96     hash_value = hash_value * 33 + *(signed char *)key++;
  97
  98   return hash_value;
  99 }
 100
 101 GvdbItem *
 102 gvdb_hash_table_insert (GHashTable  *table,
 103                         const gchar *key)
 104 {
 105   GvdbItem *item;
 106
 107   item = g_slice_new0 (GvdbItem);
 108   item->key = g_strdup (key);
 109   item->hash_value = djb_hash (key);
 110
 111   g_hash_table_insert (table, g_strdup (key), item);
 112
 113   return item;
 114 }
 115
 116 void
 117 gvdb_hash_table_insert_string (GHashTable  *table,
 118                                const gchar *key,
 119                                const gchar *value)
 120 {
 121   GvdbItem *item;
 122
 123   item = gvdb_hash_table_insert (table, key);
 124   gvdb_item_set_value (item, g_variant_new_string (value));
 125 }
 126
 127 void
 128 gvdb_item_set_value (GvdbItem *item,
 129                      GVariant *value)
 130 {
 131   g_return_if_fail (!item->value && !item->table && !item->child);
 132
 133   item->value = g_variant_ref_sink (value);
 134 }
 135
 136 void
 137 gvdb_item_set_hash_table (GvdbItem   *item,
 138                           GHashTable *table)
 139 {
 140   g_return_if_fail (!item->value && !item->table && !item->child);
 141
 142   item->table = g_hash_table_ref (table);
 143 }
 144
 145 void
 146 gvdb_item_set_parent (GvdbItem *item,
 147                       GvdbItem *parent)
 148 {
 149   GvdbItem **node;
 150
 151   g_return_if_fail (g_str_has_prefix (item->key, parent->key));
 152   g_return_if_fail (!parent->value && !parent->table);
 153   g_return_if_fail (!item->parent && !item->sibling);
 154
 155   for (node = &parent->child; *node; node = &(*node)->sibling)
 156     if (strcmp ((*node)->key, item->key) > 0)
 157       break;
 158
 159   item->parent = parent;
 160   item->sibling = *node;
 161   *node = item;
 162 }
 163
 164 typedef struct
 165 {
 166   GvdbItem **buckets;
 167   gint n_buckets;
 168 } HashTable;
 169
 170 static HashTable *
 171 hash_table_new (gint n_buckets)
 172 {
 173   HashTable *table;
 174
 175   table = g_slice_new (HashTable);
 176   table->buckets = g_new0 (GvdbItem *, n_buckets);
 177   table->n_buckets = n_buckets;
 178
 179   return table;
 180 }
 181
 182 static void
 183 hash_table_free (HashTable *table)
 184 {
 185   g_free (table->buckets);
 186
 187   g_slice_free (HashTable, table);
 188 }
 189
 190 static void
 191 hash_table_insert (gpointer key,
 192                    gpointer value,
 193                    gpointer data)
 194 {
 195   guint32 hash_value, bucket;
 196   HashTable *table = data;
 197   GvdbItem *item = value;
 198
 199   hash_value = djb_hash (key);
 200   bucket = hash_value % table->n_buckets;
 201   item->next = table->buckets[bucket];
 202   table->buckets[bucket] = item;
 203 }
 204
 205 static guint32_le
 206 item_to_index (GvdbItem *item)
 207 {
 208   if (item != NULL)
 209     return item->assigned_index;
 210
 211   return guint32_to_le (-1u);
 212 }
 213
 214 typedef struct
 215 {
 216   GQueue *chunks;
 217   guint64 offset;
 218   gboolean byteswap;
 219 } FileBuilder;
 220
 221 typedef struct
 222 {
 223   gsize offset;
 224   gsize size;
 225   gpointer data;
 226 } FileChunk;
 227
 228 static gpointer
 229 file_builder_allocate (FileBuilder         *fb,
 230                        guint                alignment,
 231                        gsize                size,
 232                        struct gvdb_pointer *pointer)
 233 {
 234   FileChunk *chunk;
 235
 236   if (size == 0)
 237     return NULL;
 238
 239   fb->offset += (-fb->offset) & (alignment - 1);
 240   chunk = g_slice_new (FileChunk);
 241   chunk->offset = fb->offset;
 242   chunk->size = size;
 243   chunk->data = g_malloc (size);
 244
 245   pointer->start = guint32_to_le (fb->offset);
 246   fb->offset += size;
 247   pointer->end = guint32_to_le (fb->offset);
 248
 249   g_queue_push_tail (fb->chunks, chunk);
 250
 251   return chunk->data;
 252 }
 253
 254 static void
 255 file_builder_add_value (FileBuilder         *fb,
 256                         GVariant            *value,
 257                         struct gvdb_pointer *pointer)
 258 {
 259   GVariant *variant, *normal;
 260   gpointer data;
 261   gsize size;
 262
 263   if (fb->byteswap)
 264     {
 265       value = g_variant_byteswap (value);
 266       variant = g_variant_new_variant (value);
 267       g_variant_unref (value);
 268     }
 269   else
 270     variant = g_variant_new_variant (value);
 271
 272   normal = g_variant_get_normal_form (variant);
 273   g_variant_unref (variant);
 274
 275   size = g_variant_get_size (normal);
 276   data = file_builder_allocate (fb, 8, size, pointer);
 277   g_variant_store (normal, data);
 278   g_variant_unref (normal);
 279 }
 280
 281 static void
 282 file_builder_add_string (FileBuilder *fb,
 283                          const gchar *string,
 284                          guint32_le  *start,
 285                          guint16_le  *size)
 286 {
 287   FileChunk *chunk;
 288   gsize length;
 289
 290   length = strlen (string);
 291
 292   chunk = g_slice_new (FileChunk);
 293   chunk->offset = fb->offset;
 294   chunk->size = length;
 295   chunk->data = g_malloc (length);
 296   if (length != 0)
 297     memcpy (chunk->data, string, length);
 298
 299   *start = guint32_to_le (fb->offset);
 300   *size = guint16_to_le (length);
 301   fb->offset += length;
 302
 303   g_queue_push_tail (fb->chunks, chunk);
 304 }
 305
 306 static void
 307 file_builder_allocate_for_hash (FileBuilder            *fb,
 308                                 gsize                   n_buckets,
 309                                 gsize                   n_items,
 310                                 guint                   bloom_shift,
 311                                 gsize                   n_bloom_words,
 312                                 guint32_le            **bloom_filter,
 313                                 guint32_le            **hash_buckets,
 314                                 struct gvdb_hash_item **hash_items,
 315                                 struct gvdb_pointer    *pointer)
 316 {
 317   guint32_le bloom_hdr, table_hdr;
 318   guchar *data;
 319   gsize size;
 320
 321   g_assert (n_bloom_words < (1u << 27));
 322
 323   bloom_hdr = guint32_to_le (bloom_shift << 27 | n_bloom_words);
 324   table_hdr = guint32_to_le (n_buckets);
 325
 326   size = sizeof bloom_hdr + sizeof table_hdr +
 327          n_bloom_words * sizeof (guint32_le) +
 328          n_buckets     * sizeof (guint32_le) +
 329          n_items       * sizeof (struct gvdb_hash_item);
 330
 331   data = file_builder_allocate (fb, 4, size, pointer);
 332
 333 #define chunk(s) (size -= (s), data += (s), data - (s))
 334   memcpy (chunk (sizeof bloom_hdr), &bloom_hdr, sizeof bloom_hdr);
 335   memcpy (chunk (sizeof table_hdr), &table_hdr, sizeof table_hdr);
 336   *bloom_filter = (guint32_le *) chunk (n_bloom_words * sizeof (guint32_le));
 337   *hash_buckets = (guint32_le *) chunk (n_buckets * sizeof (guint32_le));
 338   *hash_items = (struct gvdb_hash_item *) chunk (n_items *
 339                   sizeof (struct gvdb_hash_item));
 340   g_assert (size == 0);
 341 #undef chunk
 342
 343   memset (*bloom_filter, 0, n_bloom_words * sizeof (guint32_le));
 344
 345   /* NOTE - the code to actually fill in the bloom filter here is missing.
 346    * Patches welcome!
 347    *
 348    * http://en.wikipedia.org/wiki/Bloom_filter
 349    * http://0pointer.de/blog/projects/bloom.html
 350    */
 351 }
 352
 353 static void
 354 file_builder_add_hash (FileBuilder         *fb,
 355                        GHashTable          *table,
 356                        struct gvdb_pointer *pointer)
 357 {
 358   guint32_le *buckets, *bloom_filter;
 359   struct gvdb_hash_item *items;
 360   HashTable *mytable;
 361   GvdbItem *item;
 362   guint32 index;
 363   gint bucket;
 364
 365   mytable = hash_table_new (g_hash_table_size (table));
 366   g_hash_table_foreach (table, hash_table_insert, mytable);
 367   index = 0;
 368
 369   for (bucket = 0; bucket < mytable->n_buckets; bucket++)
 370     for (item = mytable->buckets[bucket]; item; item = item->next)
 371       item->assigned_index = guint32_to_le (index++);
 372
 373   file_builder_allocate_for_hash (fb, mytable->n_buckets, index, 5, 0,
 374                                   &bloom_filter, &buckets, &items, pointer);
 375
 376   index = 0;
 377   for (bucket = 0; bucket < mytable->n_buckets; bucket++)
 378     {
 379       buckets[bucket] = guint32_to_le (index);
 380
 381       for (item = mytable->buckets[bucket]; item; item = item->next)
 382         {
 383           struct gvdb_hash_item *entry = items++;
 384           const gchar *basename;
 385
 386           g_assert (index == guint32_from_le (item->assigned_index));
 387           entry->hash_value = guint32_to_le (item->hash_value);
 388           entry->parent = item_to_index (item->parent);
 389           entry->unused = 0;
 390
 391           if (item->parent != NULL)
 392             basename = item->key + strlen (item->parent->key);
 393           else
 394             basename = item->key;
 395
 396           file_builder_add_string (fb, basename,
 397                                    &entry->key_start,
 398                                    &entry->key_size);
 399
 400           if (item->value != NULL)
 401             {
 402               g_assert (item->child == NULL && item->table == NULL);
 403
 404               file_builder_add_value (fb, item->value, &entry->value.pointer);
 405               entry->type = 'v';
 406             }
 407
 408           if (item->child != NULL)
 409             {
 410               guint32 children = 0, i = 0;
 411               guint32_le *offsets;
 412               GvdbItem *child;
 413
 414               g_assert (item->table == NULL);
 415
 416               for (child = item->child; child; child = child->sibling)
 417                 children++;
 418
 419               offsets = file_builder_allocate (fb, 4, 4 * children,
 420                                                &entry->value.pointer);
 421               entry->type = 'L';
 422
 423               for (child = item->child; child; child = child->sibling)
 424                 offsets[i++] = child->assigned_index;
 425
 426               g_assert (children == i);
 427             }
 428
 429           if (item->table != NULL)
 430             {
 431               entry->type = 'H';
 432               file_builder_add_hash (fb, item->table, &entry->value.pointer);
 433             }
 434
 435           index++;
 436         }
 437     }
 438
 439   hash_table_free (mytable);
 440 }
 441
 442 static FileBuilder *
 443 file_builder_new (gboolean byteswap)
 444 {
 445   FileBuilder *builder;
 446
 447   builder = g_slice_new (FileBuilder);
 448   builder->chunks = g_queue_new ();
 449   builder->offset = sizeof (struct gvdb_header);
 450   builder->byteswap = byteswap;
 451
 452   return builder;
 453 }
 454
 455 static GString *
 456 file_builder_serialise (FileBuilder          *fb,
 457                         struct gvdb_pointer   root)
 458 {
 459   struct gvdb_header header = { { 0, }, };
 460   GString *result;
 461
 462   if (fb->byteswap)
 463     {
 464       header.signature[0] = GVDB_SWAPPED_SIGNATURE0;
 465       header.signature[1] = GVDB_SWAPPED_SIGNATURE1;
 466     }
 467   else
 468     {
 469       header.signature[0] = GVDB_SIGNATURE0;
 470       header.signature[1] = GVDB_SIGNATURE1;
 471     }
 472
 473   result = g_string_new (NULL);
 474
 475   header.root = root;
 476   g_string_append_len (result, (gpointer) &header, sizeof header);
 477
 478   while (!g_queue_is_empty (fb->chunks))
 479     {
 480       FileChunk *chunk = g_queue_pop_head (fb->chunks);
 481
 482       if (result->len != chunk->offset)
 483         {
 484           gchar zero[8] = { 0, };
 485
 486           g_assert (chunk->offset > result->len);
 487           g_assert (chunk->offset - result->len < 8);
 488
 489           g_string_append_len (result, zero, chunk->offset - result->len);
 490           g_assert (result->len == chunk->offset);
 491         }
 492
 493       g_string_append_len (result, chunk->data, chunk->size);
 494       g_free (chunk->data);
 495
 496       g_slice_free (FileChunk, chunk);
 497     }
 498
 499   g_queue_free (fb->chunks);
 500   g_slice_free (FileBuilder, fb);
 501
 502   return result;
 503 }
 504
 505 gboolean
 506 gvdb_table_write_contents (GHashTable   *table,
 507                            const gchar  *filename,
 508                            gboolean      byteswap,
 509                            GError      **error)
 510 {
 511   struct gvdb_pointer root;
 512   gboolean status;
 513   FileBuilder *fb;
 514   GString *str;
 515
 516   fb = file_builder_new (byteswap);
 517   file_builder_add_hash (fb, table, &root);
 518   str = file_builder_serialise (fb, root);
 519
 520   status = g_file_set_contents (filename, str->str, str->len, error);
 521   g_string_free (str, TRUE);
 522
 523   return status;
 524 }