2 * Copyright 2013, GitHub, Inc
3 * Copyright 2009-2013, Daniel Lemire, Cliff Moon,
4 * David McIntosh, Robert Becho, Google Inc. and Veronika Zenz
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 #include "git-compat-util.h"
24 static inline size_t min_size(size_t a
, size_t b
)
29 static inline size_t max_size(size_t a
, size_t b
)
34 static inline void buffer_grow(struct ewah_bitmap
*self
, size_t new_size
)
36 size_t rlw_offset
= (uint8_t *)self
->rlw
- (uint8_t *)self
->buffer
;
38 if (self
->alloc_size
>= new_size
)
41 self
->alloc_size
= new_size
;
42 self
->buffer
= xrealloc(self
->buffer
,
43 self
->alloc_size
* sizeof(eword_t
));
44 self
->rlw
= self
->buffer
+ (rlw_offset
/ sizeof(eword_t
));
47 static inline void buffer_push(struct ewah_bitmap
*self
, eword_t value
)
49 if (self
->buffer_size
+ 1 >= self
->alloc_size
)
50 buffer_grow(self
, self
->buffer_size
* 3 / 2);
52 self
->buffer
[self
->buffer_size
++] = value
;
55 static void buffer_push_rlw(struct ewah_bitmap
*self
, eword_t value
)
57 buffer_push(self
, value
);
58 self
->rlw
= self
->buffer
+ self
->buffer_size
- 1;
61 static size_t add_empty_words(struct ewah_bitmap
*self
, int v
, size_t number
)
64 eword_t runlen
, can_add
;
66 if (rlw_get_run_bit(self
->rlw
) != v
&& rlw_size(self
->rlw
) == 0) {
67 rlw_set_run_bit(self
->rlw
, v
);
68 } else if (rlw_get_literal_words(self
->rlw
) != 0 ||
69 rlw_get_run_bit(self
->rlw
) != v
) {
70 buffer_push_rlw(self
, 0);
71 if (v
) rlw_set_run_bit(self
->rlw
, v
);
75 runlen
= rlw_get_running_len(self
->rlw
);
76 can_add
= min_size(number
, RLW_LARGEST_RUNNING_COUNT
- runlen
);
78 rlw_set_running_len(self
->rlw
, runlen
+ can_add
);
81 while (number
>= RLW_LARGEST_RUNNING_COUNT
) {
82 buffer_push_rlw(self
, 0);
84 if (v
) rlw_set_run_bit(self
->rlw
, v
);
85 rlw_set_running_len(self
->rlw
, RLW_LARGEST_RUNNING_COUNT
);
86 number
-= RLW_LARGEST_RUNNING_COUNT
;
90 buffer_push_rlw(self
, 0);
93 if (v
) rlw_set_run_bit(self
->rlw
, v
);
94 rlw_set_running_len(self
->rlw
, number
);
100 size_t ewah_add_empty_words(struct ewah_bitmap
*self
, int v
, size_t number
)
105 self
->bit_size
+= number
* BITS_IN_EWORD
;
106 return add_empty_words(self
, v
, number
);
109 static size_t add_literal(struct ewah_bitmap
*self
, eword_t new_data
)
111 eword_t current_num
= rlw_get_literal_words(self
->rlw
);
113 if (current_num
>= RLW_LARGEST_LITERAL_COUNT
) {
114 buffer_push_rlw(self
, 0);
116 rlw_set_literal_words(self
->rlw
, 1);
117 buffer_push(self
, new_data
);
121 rlw_set_literal_words(self
->rlw
, current_num
+ 1);
124 assert(rlw_get_literal_words(self
->rlw
) == current_num
+ 1);
126 buffer_push(self
, new_data
);
130 void ewah_add_dirty_words(
131 struct ewah_bitmap
*self
, const eword_t
*buffer
,
132 size_t number
, int negate
)
134 size_t literals
, can_add
;
137 literals
= rlw_get_literal_words(self
->rlw
);
138 can_add
= min_size(number
, RLW_LARGEST_LITERAL_COUNT
- literals
);
140 rlw_set_literal_words(self
->rlw
, literals
+ can_add
);
142 if (self
->buffer_size
+ can_add
>= self
->alloc_size
)
143 buffer_grow(self
, (self
->buffer_size
+ can_add
) * 3 / 2);
147 for (i
= 0; i
< can_add
; ++i
)
148 self
->buffer
[self
->buffer_size
++] = ~buffer
[i
];
150 memcpy(self
->buffer
+ self
->buffer_size
,
151 buffer
, can_add
* sizeof(eword_t
));
152 self
->buffer_size
+= can_add
;
155 self
->bit_size
+= can_add
* BITS_IN_EWORD
;
157 if (number
- can_add
== 0)
160 buffer_push_rlw(self
, 0);
166 static size_t add_empty_word(struct ewah_bitmap
*self
, int v
)
168 int no_literal
= (rlw_get_literal_words(self
->rlw
) == 0);
169 eword_t run_len
= rlw_get_running_len(self
->rlw
);
171 if (no_literal
&& run_len
== 0) {
172 rlw_set_run_bit(self
->rlw
, v
);
173 assert(rlw_get_run_bit(self
->rlw
) == v
);
176 if (no_literal
&& rlw_get_run_bit(self
->rlw
) == v
&&
177 run_len
< RLW_LARGEST_RUNNING_COUNT
) {
178 rlw_set_running_len(self
->rlw
, run_len
+ 1);
179 assert(rlw_get_running_len(self
->rlw
) == run_len
+ 1);
182 buffer_push_rlw(self
, 0);
184 assert(rlw_get_running_len(self
->rlw
) == 0);
185 assert(rlw_get_run_bit(self
->rlw
) == 0);
186 assert(rlw_get_literal_words(self
->rlw
) == 0);
188 rlw_set_run_bit(self
->rlw
, v
);
189 assert(rlw_get_run_bit(self
->rlw
) == v
);
191 rlw_set_running_len(self
->rlw
, 1);
192 assert(rlw_get_running_len(self
->rlw
) == 1);
193 assert(rlw_get_literal_words(self
->rlw
) == 0);
198 size_t ewah_add(struct ewah_bitmap
*self
, eword_t word
)
200 self
->bit_size
+= BITS_IN_EWORD
;
203 return add_empty_word(self
, 0);
205 if (word
== (eword_t
)(~0))
206 return add_empty_word(self
, 1);
208 return add_literal(self
, word
);
211 void ewah_set(struct ewah_bitmap
*self
, size_t i
)
214 (i
+ BITS_IN_EWORD
) / BITS_IN_EWORD
-
215 (self
->bit_size
+ BITS_IN_EWORD
- 1) / BITS_IN_EWORD
;
217 assert(i
>= self
->bit_size
);
219 self
->bit_size
= i
+ 1;
223 add_empty_words(self
, 0, dist
- 1);
225 add_literal(self
, (eword_t
)1 << (i
% BITS_IN_EWORD
));
229 if (rlw_get_literal_words(self
->rlw
) == 0) {
230 rlw_set_running_len(self
->rlw
,
231 rlw_get_running_len(self
->rlw
) - 1);
232 add_literal(self
, (eword_t
)1 << (i
% BITS_IN_EWORD
));
236 self
->buffer
[self
->buffer_size
- 1] |=
237 ((eword_t
)1 << (i
% BITS_IN_EWORD
));
239 /* check if we just completed a stream of 1s */
240 if (self
->buffer
[self
->buffer_size
- 1] == (eword_t
)(~0)) {
241 self
->buffer
[--self
->buffer_size
] = 0;
242 rlw_set_literal_words(self
->rlw
,
243 rlw_get_literal_words(self
->rlw
) - 1);
244 add_empty_word(self
, 1);
248 void ewah_each_bit(struct ewah_bitmap
*self
, void (*callback
)(size_t, void*), void *payload
)
254 while (pointer
< self
->buffer_size
) {
255 eword_t
*word
= &self
->buffer
[pointer
];
257 if (rlw_get_run_bit(word
)) {
258 size_t len
= rlw_get_running_len(word
) * BITS_IN_EWORD
;
259 for (k
= 0; k
< len
; ++k
, ++pos
)
260 callback(pos
, payload
);
262 pos
+= rlw_get_running_len(word
) * BITS_IN_EWORD
;
267 for (k
= 0; k
< rlw_get_literal_words(word
); ++k
) {
270 /* todo: zero count optimization */
271 for (c
= 0; c
< BITS_IN_EWORD
; ++c
, ++pos
) {
272 if ((self
->buffer
[pointer
] & ((eword_t
)1 << c
)) != 0)
273 callback(pos
, payload
);
281 struct ewah_bitmap
*ewah_new(void)
283 struct ewah_bitmap
*self
;
285 self
= xmalloc(sizeof(struct ewah_bitmap
));
286 self
->buffer
= xmalloc(32 * sizeof(eword_t
));
287 self
->alloc_size
= 32;
293 void ewah_clear(struct ewah_bitmap
*self
)
295 self
->buffer_size
= 1;
298 self
->rlw
= self
->buffer
;
301 void ewah_free(struct ewah_bitmap
*self
)
306 if (self
->alloc_size
)
312 static void read_new_rlw(struct ewah_iterator
*it
)
314 const eword_t
*word
= NULL
;
320 word
= &it
->buffer
[it
->pointer
];
322 it
->rl
= rlw_get_running_len(word
);
323 it
->lw
= rlw_get_literal_words(word
);
324 it
->b
= rlw_get_run_bit(word
);
326 if (it
->rl
|| it
->lw
)
329 if (it
->pointer
< it
->buffer_size
- 1) {
332 it
->pointer
= it
->buffer_size
;
338 int ewah_iterator_next(eword_t
*next
, struct ewah_iterator
*it
)
340 if (it
->pointer
>= it
->buffer_size
)
343 if (it
->compressed
< it
->rl
) {
345 *next
= it
->b
? (eword_t
)(~0) : 0;
347 assert(it
->literals
< it
->lw
);
352 assert(it
->pointer
< it
->buffer_size
);
354 *next
= it
->buffer
[it
->pointer
];
357 if (it
->compressed
== it
->rl
&& it
->literals
== it
->lw
) {
358 if (++it
->pointer
< it
->buffer_size
)
365 void ewah_iterator_init(struct ewah_iterator
*it
, struct ewah_bitmap
*parent
)
367 it
->buffer
= parent
->buffer
;
368 it
->buffer_size
= parent
->buffer_size
;
377 if (it
->pointer
< it
->buffer_size
)
381 void ewah_not(struct ewah_bitmap
*self
)
385 while (pointer
< self
->buffer_size
) {
386 eword_t
*word
= &self
->buffer
[pointer
];
389 rlw_xor_run_bit(word
);
392 literals
= rlw_get_literal_words(word
);
393 for (k
= 0; k
< literals
; ++k
) {
394 self
->buffer
[pointer
] = ~self
->buffer
[pointer
];
401 struct ewah_bitmap
*ewah_i
,
402 struct ewah_bitmap
*ewah_j
,
403 struct ewah_bitmap
*out
)
405 struct rlw_iterator rlw_i
;
406 struct rlw_iterator rlw_j
;
409 rlwit_init(&rlw_i
, ewah_i
);
410 rlwit_init(&rlw_j
, ewah_j
);
412 while (rlwit_word_size(&rlw_i
) > 0 && rlwit_word_size(&rlw_j
) > 0) {
413 while (rlw_i
.rlw
.running_len
> 0 || rlw_j
.rlw
.running_len
> 0) {
414 struct rlw_iterator
*prey
, *predator
;
418 if (rlw_i
.rlw
.running_len
< rlw_j
.rlw
.running_len
) {
426 negate_words
= !!predator
->rlw
.running_bit
;
427 index
= rlwit_discharge(prey
, out
,
428 predator
->rlw
.running_len
, negate_words
);
430 ewah_add_empty_words(out
, negate_words
,
431 predator
->rlw
.running_len
- index
);
433 rlwit_discard_first_words(predator
,
434 predator
->rlw
.running_len
);
438 rlw_i
.rlw
.literal_words
,
439 rlw_j
.rlw
.literal_words
);
444 for (k
= 0; k
< literals
; ++k
) {
446 rlw_i
.buffer
[rlw_i
.literal_word_start
+ k
] ^
447 rlw_j
.buffer
[rlw_j
.literal_word_start
+ k
]
451 rlwit_discard_first_words(&rlw_i
, literals
);
452 rlwit_discard_first_words(&rlw_j
, literals
);
456 if (rlwit_word_size(&rlw_i
) > 0)
457 rlwit_discharge(&rlw_i
, out
, ~0, 0);
459 rlwit_discharge(&rlw_j
, out
, ~0, 0);
461 out
->bit_size
= max_size(ewah_i
->bit_size
, ewah_j
->bit_size
);
465 struct ewah_bitmap
*ewah_i
,
466 struct ewah_bitmap
*ewah_j
,
467 struct ewah_bitmap
*out
)
469 struct rlw_iterator rlw_i
;
470 struct rlw_iterator rlw_j
;
473 rlwit_init(&rlw_i
, ewah_i
);
474 rlwit_init(&rlw_j
, ewah_j
);
476 while (rlwit_word_size(&rlw_i
) > 0 && rlwit_word_size(&rlw_j
) > 0) {
477 while (rlw_i
.rlw
.running_len
> 0 || rlw_j
.rlw
.running_len
> 0) {
478 struct rlw_iterator
*prey
, *predator
;
480 if (rlw_i
.rlw
.running_len
< rlw_j
.rlw
.running_len
) {
488 if (predator
->rlw
.running_bit
== 0) {
489 ewah_add_empty_words(out
, 0,
490 predator
->rlw
.running_len
);
491 rlwit_discard_first_words(prey
,
492 predator
->rlw
.running_len
);
493 rlwit_discard_first_words(predator
,
494 predator
->rlw
.running_len
);
496 size_t index
= rlwit_discharge(prey
, out
,
497 predator
->rlw
.running_len
, 0);
498 ewah_add_empty_words(out
, 0,
499 predator
->rlw
.running_len
- index
);
500 rlwit_discard_first_words(predator
,
501 predator
->rlw
.running_len
);
506 rlw_i
.rlw
.literal_words
,
507 rlw_j
.rlw
.literal_words
);
512 for (k
= 0; k
< literals
; ++k
) {
514 rlw_i
.buffer
[rlw_i
.literal_word_start
+ k
] &
515 rlw_j
.buffer
[rlw_j
.literal_word_start
+ k
]
519 rlwit_discard_first_words(&rlw_i
, literals
);
520 rlwit_discard_first_words(&rlw_j
, literals
);
524 if (rlwit_word_size(&rlw_i
) > 0)
525 rlwit_discharge_empty(&rlw_i
, out
);
527 rlwit_discharge_empty(&rlw_j
, out
);
529 out
->bit_size
= max_size(ewah_i
->bit_size
, ewah_j
->bit_size
);
533 struct ewah_bitmap
*ewah_i
,
534 struct ewah_bitmap
*ewah_j
,
535 struct ewah_bitmap
*out
)
537 struct rlw_iterator rlw_i
;
538 struct rlw_iterator rlw_j
;
541 rlwit_init(&rlw_i
, ewah_i
);
542 rlwit_init(&rlw_j
, ewah_j
);
544 while (rlwit_word_size(&rlw_i
) > 0 && rlwit_word_size(&rlw_j
) > 0) {
545 while (rlw_i
.rlw
.running_len
> 0 || rlw_j
.rlw
.running_len
> 0) {
546 struct rlw_iterator
*prey
, *predator
;
548 if (rlw_i
.rlw
.running_len
< rlw_j
.rlw
.running_len
) {
556 if ((predator
->rlw
.running_bit
&& prey
== &rlw_i
) ||
557 (!predator
->rlw
.running_bit
&& prey
!= &rlw_i
)) {
558 ewah_add_empty_words(out
, 0,
559 predator
->rlw
.running_len
);
560 rlwit_discard_first_words(prey
,
561 predator
->rlw
.running_len
);
562 rlwit_discard_first_words(predator
,
563 predator
->rlw
.running_len
);
568 negate_words
= (&rlw_i
!= prey
);
569 index
= rlwit_discharge(prey
, out
,
570 predator
->rlw
.running_len
, negate_words
);
571 ewah_add_empty_words(out
, negate_words
,
572 predator
->rlw
.running_len
- index
);
573 rlwit_discard_first_words(predator
,
574 predator
->rlw
.running_len
);
579 rlw_i
.rlw
.literal_words
,
580 rlw_j
.rlw
.literal_words
);
585 for (k
= 0; k
< literals
; ++k
) {
587 rlw_i
.buffer
[rlw_i
.literal_word_start
+ k
] &
588 ~(rlw_j
.buffer
[rlw_j
.literal_word_start
+ k
])
592 rlwit_discard_first_words(&rlw_i
, literals
);
593 rlwit_discard_first_words(&rlw_j
, literals
);
597 if (rlwit_word_size(&rlw_i
) > 0)
598 rlwit_discharge(&rlw_i
, out
, ~0, 0);
600 rlwit_discharge_empty(&rlw_j
, out
);
602 out
->bit_size
= max_size(ewah_i
->bit_size
, ewah_j
->bit_size
);
606 struct ewah_bitmap
*ewah_i
,
607 struct ewah_bitmap
*ewah_j
,
608 struct ewah_bitmap
*out
)
610 struct rlw_iterator rlw_i
;
611 struct rlw_iterator rlw_j
;
614 rlwit_init(&rlw_i
, ewah_i
);
615 rlwit_init(&rlw_j
, ewah_j
);
617 while (rlwit_word_size(&rlw_i
) > 0 && rlwit_word_size(&rlw_j
) > 0) {
618 while (rlw_i
.rlw
.running_len
> 0 || rlw_j
.rlw
.running_len
> 0) {
619 struct rlw_iterator
*prey
, *predator
;
621 if (rlw_i
.rlw
.running_len
< rlw_j
.rlw
.running_len
) {
629 if (predator
->rlw
.running_bit
) {
630 ewah_add_empty_words(out
, 0,
631 predator
->rlw
.running_len
);
632 rlwit_discard_first_words(prey
,
633 predator
->rlw
.running_len
);
634 rlwit_discard_first_words(predator
,
635 predator
->rlw
.running_len
);
637 size_t index
= rlwit_discharge(prey
, out
,
638 predator
->rlw
.running_len
, 0);
639 ewah_add_empty_words(out
, 0,
640 predator
->rlw
.running_len
- index
);
641 rlwit_discard_first_words(predator
,
642 predator
->rlw
.running_len
);
647 rlw_i
.rlw
.literal_words
,
648 rlw_j
.rlw
.literal_words
);
653 for (k
= 0; k
< literals
; ++k
) {
655 rlw_i
.buffer
[rlw_i
.literal_word_start
+ k
] |
656 rlw_j
.buffer
[rlw_j
.literal_word_start
+ k
]
660 rlwit_discard_first_words(&rlw_i
, literals
);
661 rlwit_discard_first_words(&rlw_j
, literals
);
665 if (rlwit_word_size(&rlw_i
) > 0)
666 rlwit_discharge(&rlw_i
, out
, ~0, 0);
668 rlwit_discharge(&rlw_j
, out
, ~0, 0);
670 out
->bit_size
= max_size(ewah_i
->bit_size
, ewah_j
->bit_size
);
674 #define BITMAP_POOL_MAX 16
675 static struct ewah_bitmap
*bitmap_pool
[BITMAP_POOL_MAX
];
676 static size_t bitmap_pool_size
;
678 struct ewah_bitmap
*ewah_pool_new(void)
680 if (bitmap_pool_size
)
681 return bitmap_pool
[--bitmap_pool_size
];
686 void ewah_pool_free(struct ewah_bitmap
*self
)
691 if (bitmap_pool_size
== BITMAP_POOL_MAX
||
692 self
->alloc_size
== 0) {
698 bitmap_pool
[bitmap_pool_size
++] = self
;
701 uint32_t ewah_checksum(struct ewah_bitmap
*self
)
703 const uint8_t *p
= (uint8_t *)self
->buffer
;
704 uint32_t crc
= (uint32_t)self
->bit_size
;
705 size_t size
= self
->buffer_size
* sizeof(eword_t
);
708 crc
= (crc
<< 5) - crc
+ (uint32_t)*p
++;