ewah/ewah_io.c

   1 /**
   2  * Copyright 2013, GitHub, Inc
   3  * Copyright 2009-2013, Daniel Lemire, Cliff Moon,
   4  *      David McIntosh, Robert Becho, Google Inc. and Veronika Zenz
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 #include "git-compat-util.h"
  21 #include "ewok.h"
  22 #include "strbuf.h"
  23
  24 int ewah_serialize_native(struct ewah_bitmap *self, int fd)
  25 {
  26         uint32_t write32;
  27         size_t to_write = self->buffer_size * 8;
  28
  29         /* 32 bit -- bit size for the map */
  30         write32 = (uint32_t)self->bit_size;
  31         if (write(fd, &write32, 4) != 4)
  32                 return -1;
  33
  34         /** 32 bit -- number of compressed 64-bit words */
  35         write32 = (uint32_t)self->buffer_size;
  36         if (write(fd, &write32, 4) != 4)
  37                 return -1;
  38
  39         if (write(fd, self->buffer, to_write) != to_write)
  40                 return -1;
  41
  42         /** 32 bit -- position for the RLW */
  43         write32 = self->rlw - self->buffer;
  44         if (write(fd, &write32, 4) != 4)
  45                 return -1;
  46
  47         return (3 * 4) + to_write;
  48 }
  49
  50 int ewah_serialize_to(struct ewah_bitmap *self,
  51                       int (*write_fun)(void *, const void *, size_t),
  52                       void *data)
  53 {
  54         size_t i;
  55         eword_t dump[2048];
  56         const size_t words_per_dump = sizeof(dump) / sizeof(eword_t);
  57         uint32_t bitsize, word_count, rlw_pos;
  58
  59         const eword_t *buffer;
  60         size_t words_left;
  61
  62         /* 32 bit -- bit size for the map */
  63         bitsize =  htonl((uint32_t)self->bit_size);
  64         if (write_fun(data, &bitsize, 4) != 4)
  65                 return -1;
  66
  67         /** 32 bit -- number of compressed 64-bit words */
  68         word_count =  htonl((uint32_t)self->buffer_size);
  69         if (write_fun(data, &word_count, 4) != 4)
  70                 return -1;
  71
  72         /** 64 bit x N -- compressed words */
  73         buffer = self->buffer;
  74         words_left = self->buffer_size;
  75
  76         while (words_left >= words_per_dump) {
  77                 for (i = 0; i < words_per_dump; ++i, ++buffer)
  78                         dump[i] = htonll(*buffer);
  79
  80                 if (write_fun(data, dump, sizeof(dump)) != sizeof(dump))
  81                         return -1;
  82
  83                 words_left -= words_per_dump;
  84         }
  85
  86         if (words_left) {
  87                 for (i = 0; i < words_left; ++i, ++buffer)
  88                         dump[i] = htonll(*buffer);
  89
  90                 if (write_fun(data, dump, words_left * 8) != words_left * 8)
  91                         return -1;
  92         }
  93
  94         /** 32 bit -- position for the RLW */
  95         rlw_pos = (uint8_t*)self->rlw - (uint8_t *)self->buffer;
  96         rlw_pos = htonl(rlw_pos / sizeof(eword_t));
  97
  98         if (write_fun(data, &rlw_pos, 4) != 4)
  99                 return -1;
 100
 101         return (3 * 4) + (self->buffer_size * 8);
 102 }
 103
 104 static int write_helper(void *fd, const void *buf, size_t len)
 105 {
 106         return write((intptr_t)fd, buf, len);
 107 }
 108
 109 int ewah_serialize(struct ewah_bitmap *self, int fd)
 110 {
 111         return ewah_serialize_to(self, write_helper, (void *)(intptr_t)fd);
 112 }
 113
 114 static int write_strbuf(void *user_data, const void *data, size_t len)
 115 {
 116         struct strbuf *sb = user_data;
 117         strbuf_add(sb, data, len);
 118         return len;
 119 }
 120
 121 int ewah_serialize_strbuf(struct ewah_bitmap *self, struct strbuf *sb)
 122 {
 123         return ewah_serialize_to(self, write_strbuf, sb);
 124 }
 125
 126 int ewah_read_mmap(struct ewah_bitmap *self, const void *map, size_t len)
 127 {
 128         const uint8_t *ptr = map;
 129         size_t i;
 130
 131         self->bit_size = get_be32(ptr);
 132         ptr += sizeof(uint32_t);
 133
 134         self->buffer_size = self->alloc_size = get_be32(ptr);
 135         ptr += sizeof(uint32_t);
 136
 137         self->buffer = ewah_realloc(self->buffer,
 138                 self->alloc_size * sizeof(eword_t));
 139
 140         if (!self->buffer)
 141                 return -1;
 142
 143         /*
 144          * Copy the raw data for the bitmap as a whole chunk;
 145          * if we're in a little-endian platform, we'll perform
 146          * the endianness conversion in a separate pass to ensure
 147          * we're loading 8-byte aligned words.
 148          */
 149         memcpy(self->buffer, ptr, self->buffer_size * sizeof(uint64_t));
 150         ptr += self->buffer_size * sizeof(uint64_t);
 151
 152         for (i = 0; i < self->buffer_size; ++i)
 153                 self->buffer[i] = ntohll(self->buffer[i]);
 154
 155         self->rlw = self->buffer + get_be32(ptr);
 156
 157         return (3 * 4) + (self->buffer_size * 8);
 158 }
 159
 160 int ewah_deserialize(struct ewah_bitmap *self, int fd)
 161 {
 162         size_t i;
 163         eword_t dump[2048];
 164         const size_t words_per_dump = sizeof(dump) / sizeof(eword_t);
 165         uint32_t bitsize, word_count, rlw_pos;
 166
 167         eword_t *buffer = NULL;
 168         size_t words_left;
 169
 170         ewah_clear(self);
 171
 172         /* 32 bit -- bit size for the map */
 173         if (read(fd, &bitsize, 4) != 4)
 174                 return -1;
 175
 176         self->bit_size = (size_t)ntohl(bitsize);
 177
 178         /** 32 bit -- number of compressed 64-bit words */
 179         if (read(fd, &word_count, 4) != 4)
 180                 return -1;
 181
 182         self->buffer_size = self->alloc_size = (size_t)ntohl(word_count);
 183         self->buffer = ewah_realloc(self->buffer,
 184                 self->alloc_size * sizeof(eword_t));
 185
 186         if (!self->buffer)
 187                 return -1;
 188
 189         /** 64 bit x N -- compressed words */
 190         buffer = self->buffer;
 191         words_left = self->buffer_size;
 192
 193         while (words_left >= words_per_dump) {
 194                 if (read(fd, dump, sizeof(dump)) != sizeof(dump))
 195                         return -1;
 196
 197                 for (i = 0; i < words_per_dump; ++i, ++buffer)
 198                         *buffer = ntohll(dump[i]);
 199
 200                 words_left -= words_per_dump;
 201         }
 202
 203         if (words_left) {
 204                 if (read(fd, dump, words_left * 8) != words_left * 8)
 205                         return -1;
 206
 207                 for (i = 0; i < words_left; ++i, ++buffer)
 208                         *buffer = ntohll(dump[i]);
 209         }
 210
 211         /** 32 bit -- position for the RLW */
 212         if (read(fd, &rlw_pos, 4) != 4)
 213                 return -1;
 214
 215         self->rlw = self->buffer + ntohl(rlw_pos);
 216         return 0;
 217 }