2 * Copyright 2013, GitHub, Inc
3 * Copyright 2009-2013, Daniel Lemire, Cliff Moon,
4 * David McIntosh, Robert Becho, Google Inc. and Veronika Zenz
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, see <http://www.gnu.org/licenses/>.
19 #include "git-compat-util.h"
23 static inline size_t min_size(size_t a
, size_t b
)
28 static inline size_t max_size(size_t a
, size_t b
)
33 static inline void buffer_grow(struct ewah_bitmap
*self
, size_t new_size
)
35 size_t rlw_offset
= (uint8_t *)self
->rlw
- (uint8_t *)self
->buffer
;
37 if (self
->alloc_size
>= new_size
)
40 self
->alloc_size
= new_size
;
41 REALLOC_ARRAY(self
->buffer
, self
->alloc_size
);
42 self
->rlw
= self
->buffer
+ (rlw_offset
/ sizeof(eword_t
));
45 static inline void buffer_push(struct ewah_bitmap
*self
, eword_t value
)
47 if (self
->buffer_size
+ 1 >= self
->alloc_size
)
48 buffer_grow(self
, self
->buffer_size
* 3 / 2);
50 self
->buffer
[self
->buffer_size
++] = value
;
53 static void buffer_push_rlw(struct ewah_bitmap
*self
, eword_t value
)
55 buffer_push(self
, value
);
56 self
->rlw
= self
->buffer
+ self
->buffer_size
- 1;
59 static size_t add_empty_words(struct ewah_bitmap
*self
, int v
, size_t number
)
62 eword_t runlen
, can_add
;
64 if (rlw_get_run_bit(self
->rlw
) != v
&& rlw_size(self
->rlw
) == 0) {
65 rlw_set_run_bit(self
->rlw
, v
);
66 } else if (rlw_get_literal_words(self
->rlw
) != 0 ||
67 rlw_get_run_bit(self
->rlw
) != v
) {
68 buffer_push_rlw(self
, 0);
69 if (v
) rlw_set_run_bit(self
->rlw
, v
);
73 runlen
= rlw_get_running_len(self
->rlw
);
74 can_add
= min_size(number
, RLW_LARGEST_RUNNING_COUNT
- runlen
);
76 rlw_set_running_len(self
->rlw
, runlen
+ can_add
);
79 while (number
>= RLW_LARGEST_RUNNING_COUNT
) {
80 buffer_push_rlw(self
, 0);
82 if (v
) rlw_set_run_bit(self
->rlw
, v
);
83 rlw_set_running_len(self
->rlw
, RLW_LARGEST_RUNNING_COUNT
);
84 number
-= RLW_LARGEST_RUNNING_COUNT
;
88 buffer_push_rlw(self
, 0);
91 if (v
) rlw_set_run_bit(self
->rlw
, v
);
92 rlw_set_running_len(self
->rlw
, number
);
98 size_t ewah_add_empty_words(struct ewah_bitmap
*self
, int v
, size_t number
)
103 self
->bit_size
+= number
* BITS_IN_EWORD
;
104 return add_empty_words(self
, v
, number
);
107 static size_t add_literal(struct ewah_bitmap
*self
, eword_t new_data
)
109 eword_t current_num
= rlw_get_literal_words(self
->rlw
);
111 if (current_num
>= RLW_LARGEST_LITERAL_COUNT
) {
112 buffer_push_rlw(self
, 0);
114 rlw_set_literal_words(self
->rlw
, 1);
115 buffer_push(self
, new_data
);
119 rlw_set_literal_words(self
->rlw
, current_num
+ 1);
122 assert(rlw_get_literal_words(self
->rlw
) == current_num
+ 1);
124 buffer_push(self
, new_data
);
128 void ewah_add_dirty_words(
129 struct ewah_bitmap
*self
, const eword_t
*buffer
,
130 size_t number
, int negate
)
132 size_t literals
, can_add
;
135 literals
= rlw_get_literal_words(self
->rlw
);
136 can_add
= min_size(number
, RLW_LARGEST_LITERAL_COUNT
- literals
);
138 rlw_set_literal_words(self
->rlw
, literals
+ can_add
);
140 if (self
->buffer_size
+ can_add
>= self
->alloc_size
)
141 buffer_grow(self
, (self
->buffer_size
+ can_add
) * 3 / 2);
145 for (i
= 0; i
< can_add
; ++i
)
146 self
->buffer
[self
->buffer_size
++] = ~buffer
[i
];
148 memcpy(self
->buffer
+ self
->buffer_size
,
149 buffer
, can_add
* sizeof(eword_t
));
150 self
->buffer_size
+= can_add
;
153 self
->bit_size
+= can_add
* BITS_IN_EWORD
;
155 if (number
- can_add
== 0)
158 buffer_push_rlw(self
, 0);
164 static size_t add_empty_word(struct ewah_bitmap
*self
, int v
)
166 int no_literal
= (rlw_get_literal_words(self
->rlw
) == 0);
167 eword_t run_len
= rlw_get_running_len(self
->rlw
);
169 if (no_literal
&& run_len
== 0) {
170 rlw_set_run_bit(self
->rlw
, v
);
171 assert(rlw_get_run_bit(self
->rlw
) == v
);
174 if (no_literal
&& rlw_get_run_bit(self
->rlw
) == v
&&
175 run_len
< RLW_LARGEST_RUNNING_COUNT
) {
176 rlw_set_running_len(self
->rlw
, run_len
+ 1);
177 assert(rlw_get_running_len(self
->rlw
) == run_len
+ 1);
180 buffer_push_rlw(self
, 0);
182 assert(rlw_get_running_len(self
->rlw
) == 0);
183 assert(rlw_get_run_bit(self
->rlw
) == 0);
184 assert(rlw_get_literal_words(self
->rlw
) == 0);
186 rlw_set_run_bit(self
->rlw
, v
);
187 assert(rlw_get_run_bit(self
->rlw
) == v
);
189 rlw_set_running_len(self
->rlw
, 1);
190 assert(rlw_get_running_len(self
->rlw
) == 1);
191 assert(rlw_get_literal_words(self
->rlw
) == 0);
196 size_t ewah_add(struct ewah_bitmap
*self
, eword_t word
)
198 self
->bit_size
+= BITS_IN_EWORD
;
201 return add_empty_word(self
, 0);
203 if (word
== (eword_t
)(~0))
204 return add_empty_word(self
, 1);
206 return add_literal(self
, word
);
209 void ewah_set(struct ewah_bitmap
*self
, size_t i
)
212 DIV_ROUND_UP(i
+ 1, BITS_IN_EWORD
) -
213 DIV_ROUND_UP(self
->bit_size
, BITS_IN_EWORD
);
215 assert(i
>= self
->bit_size
);
217 self
->bit_size
= i
+ 1;
221 add_empty_words(self
, 0, dist
- 1);
223 add_literal(self
, (eword_t
)1 << (i
% BITS_IN_EWORD
));
227 if (rlw_get_literal_words(self
->rlw
) == 0) {
228 rlw_set_running_len(self
->rlw
,
229 rlw_get_running_len(self
->rlw
) - 1);
230 add_literal(self
, (eword_t
)1 << (i
% BITS_IN_EWORD
));
234 self
->buffer
[self
->buffer_size
- 1] |=
235 ((eword_t
)1 << (i
% BITS_IN_EWORD
));
237 /* check if we just completed a stream of 1s */
238 if (self
->buffer
[self
->buffer_size
- 1] == (eword_t
)(~0)) {
239 self
->buffer
[--self
->buffer_size
] = 0;
240 rlw_set_literal_words(self
->rlw
,
241 rlw_get_literal_words(self
->rlw
) - 1);
242 add_empty_word(self
, 1);
246 void ewah_each_bit(struct ewah_bitmap
*self
, void (*callback
)(size_t, void*), void *payload
)
252 while (pointer
< self
->buffer_size
) {
253 eword_t
*word
= &self
->buffer
[pointer
];
255 if (rlw_get_run_bit(word
)) {
256 size_t len
= rlw_get_running_len(word
) * BITS_IN_EWORD
;
257 for (k
= 0; k
< len
; ++k
, ++pos
)
258 callback(pos
, payload
);
260 pos
+= rlw_get_running_len(word
) * BITS_IN_EWORD
;
265 for (k
= 0; k
< rlw_get_literal_words(word
); ++k
) {
268 /* todo: zero count optimization */
269 for (c
= 0; c
< BITS_IN_EWORD
; ++c
, ++pos
) {
270 if ((self
->buffer
[pointer
] & ((eword_t
)1 << c
)) != 0)
271 callback(pos
, payload
);
279 struct ewah_bitmap
*ewah_new(void)
281 struct ewah_bitmap
*self
;
283 self
= xmalloc(sizeof(struct ewah_bitmap
));
284 self
->alloc_size
= 32;
285 ALLOC_ARRAY(self
->buffer
, self
->alloc_size
);
291 void ewah_clear(struct ewah_bitmap
*self
)
293 self
->buffer_size
= 1;
296 self
->rlw
= self
->buffer
;
299 void ewah_free(struct ewah_bitmap
*self
)
304 if (self
->alloc_size
)
310 static void read_new_rlw(struct ewah_iterator
*it
)
312 const eword_t
*word
= NULL
;
318 word
= &it
->buffer
[it
->pointer
];
320 it
->rl
= rlw_get_running_len(word
);
321 it
->lw
= rlw_get_literal_words(word
);
322 it
->b
= rlw_get_run_bit(word
);
324 if (it
->rl
|| it
->lw
)
327 if (it
->pointer
< it
->buffer_size
- 1) {
330 it
->pointer
= it
->buffer_size
;
336 int ewah_iterator_next(eword_t
*next
, struct ewah_iterator
*it
)
338 if (it
->pointer
>= it
->buffer_size
)
341 if (it
->compressed
< it
->rl
) {
343 *next
= it
->b
? (eword_t
)(~0) : 0;
345 assert(it
->literals
< it
->lw
);
350 assert(it
->pointer
< it
->buffer_size
);
352 *next
= it
->buffer
[it
->pointer
];
355 if (it
->compressed
== it
->rl
&& it
->literals
== it
->lw
) {
356 if (++it
->pointer
< it
->buffer_size
)
363 void ewah_iterator_init(struct ewah_iterator
*it
, struct ewah_bitmap
*parent
)
365 it
->buffer
= parent
->buffer
;
366 it
->buffer_size
= parent
->buffer_size
;
375 if (it
->pointer
< it
->buffer_size
)
379 void ewah_not(struct ewah_bitmap
*self
)
383 while (pointer
< self
->buffer_size
) {
384 eword_t
*word
= &self
->buffer
[pointer
];
387 rlw_xor_run_bit(word
);
390 literals
= rlw_get_literal_words(word
);
391 for (k
= 0; k
< literals
; ++k
) {
392 self
->buffer
[pointer
] = ~self
->buffer
[pointer
];
399 struct ewah_bitmap
*ewah_i
,
400 struct ewah_bitmap
*ewah_j
,
401 struct ewah_bitmap
*out
)
403 struct rlw_iterator rlw_i
;
404 struct rlw_iterator rlw_j
;
407 rlwit_init(&rlw_i
, ewah_i
);
408 rlwit_init(&rlw_j
, ewah_j
);
410 while (rlwit_word_size(&rlw_i
) > 0 && rlwit_word_size(&rlw_j
) > 0) {
411 while (rlw_i
.rlw
.running_len
> 0 || rlw_j
.rlw
.running_len
> 0) {
412 struct rlw_iterator
*prey
, *predator
;
416 if (rlw_i
.rlw
.running_len
< rlw_j
.rlw
.running_len
) {
424 negate_words
= !!predator
->rlw
.running_bit
;
425 index
= rlwit_discharge(prey
, out
,
426 predator
->rlw
.running_len
, negate_words
);
428 ewah_add_empty_words(out
, negate_words
,
429 predator
->rlw
.running_len
- index
);
431 rlwit_discard_first_words(predator
,
432 predator
->rlw
.running_len
);
436 rlw_i
.rlw
.literal_words
,
437 rlw_j
.rlw
.literal_words
);
442 for (k
= 0; k
< literals
; ++k
) {
444 rlw_i
.buffer
[rlw_i
.literal_word_start
+ k
] ^
445 rlw_j
.buffer
[rlw_j
.literal_word_start
+ k
]
449 rlwit_discard_first_words(&rlw_i
, literals
);
450 rlwit_discard_first_words(&rlw_j
, literals
);
454 if (rlwit_word_size(&rlw_i
) > 0)
455 rlwit_discharge(&rlw_i
, out
, ~0, 0);
457 rlwit_discharge(&rlw_j
, out
, ~0, 0);
459 out
->bit_size
= max_size(ewah_i
->bit_size
, ewah_j
->bit_size
);
463 struct ewah_bitmap
*ewah_i
,
464 struct ewah_bitmap
*ewah_j
,
465 struct ewah_bitmap
*out
)
467 struct rlw_iterator rlw_i
;
468 struct rlw_iterator rlw_j
;
471 rlwit_init(&rlw_i
, ewah_i
);
472 rlwit_init(&rlw_j
, ewah_j
);
474 while (rlwit_word_size(&rlw_i
) > 0 && rlwit_word_size(&rlw_j
) > 0) {
475 while (rlw_i
.rlw
.running_len
> 0 || rlw_j
.rlw
.running_len
> 0) {
476 struct rlw_iterator
*prey
, *predator
;
478 if (rlw_i
.rlw
.running_len
< rlw_j
.rlw
.running_len
) {
486 if (predator
->rlw
.running_bit
== 0) {
487 ewah_add_empty_words(out
, 0,
488 predator
->rlw
.running_len
);
489 rlwit_discard_first_words(prey
,
490 predator
->rlw
.running_len
);
491 rlwit_discard_first_words(predator
,
492 predator
->rlw
.running_len
);
494 size_t index
= rlwit_discharge(prey
, out
,
495 predator
->rlw
.running_len
, 0);
496 ewah_add_empty_words(out
, 0,
497 predator
->rlw
.running_len
- index
);
498 rlwit_discard_first_words(predator
,
499 predator
->rlw
.running_len
);
504 rlw_i
.rlw
.literal_words
,
505 rlw_j
.rlw
.literal_words
);
510 for (k
= 0; k
< literals
; ++k
) {
512 rlw_i
.buffer
[rlw_i
.literal_word_start
+ k
] &
513 rlw_j
.buffer
[rlw_j
.literal_word_start
+ k
]
517 rlwit_discard_first_words(&rlw_i
, literals
);
518 rlwit_discard_first_words(&rlw_j
, literals
);
522 if (rlwit_word_size(&rlw_i
) > 0)
523 rlwit_discharge_empty(&rlw_i
, out
);
525 rlwit_discharge_empty(&rlw_j
, out
);
527 out
->bit_size
= max_size(ewah_i
->bit_size
, ewah_j
->bit_size
);
531 struct ewah_bitmap
*ewah_i
,
532 struct ewah_bitmap
*ewah_j
,
533 struct ewah_bitmap
*out
)
535 struct rlw_iterator rlw_i
;
536 struct rlw_iterator rlw_j
;
539 rlwit_init(&rlw_i
, ewah_i
);
540 rlwit_init(&rlw_j
, ewah_j
);
542 while (rlwit_word_size(&rlw_i
) > 0 && rlwit_word_size(&rlw_j
) > 0) {
543 while (rlw_i
.rlw
.running_len
> 0 || rlw_j
.rlw
.running_len
> 0) {
544 struct rlw_iterator
*prey
, *predator
;
546 if (rlw_i
.rlw
.running_len
< rlw_j
.rlw
.running_len
) {
554 if ((predator
->rlw
.running_bit
&& prey
== &rlw_i
) ||
555 (!predator
->rlw
.running_bit
&& prey
!= &rlw_i
)) {
556 ewah_add_empty_words(out
, 0,
557 predator
->rlw
.running_len
);
558 rlwit_discard_first_words(prey
,
559 predator
->rlw
.running_len
);
560 rlwit_discard_first_words(predator
,
561 predator
->rlw
.running_len
);
566 negate_words
= (&rlw_i
!= prey
);
567 index
= rlwit_discharge(prey
, out
,
568 predator
->rlw
.running_len
, negate_words
);
569 ewah_add_empty_words(out
, negate_words
,
570 predator
->rlw
.running_len
- index
);
571 rlwit_discard_first_words(predator
,
572 predator
->rlw
.running_len
);
577 rlw_i
.rlw
.literal_words
,
578 rlw_j
.rlw
.literal_words
);
583 for (k
= 0; k
< literals
; ++k
) {
585 rlw_i
.buffer
[rlw_i
.literal_word_start
+ k
] &
586 ~(rlw_j
.buffer
[rlw_j
.literal_word_start
+ k
])
590 rlwit_discard_first_words(&rlw_i
, literals
);
591 rlwit_discard_first_words(&rlw_j
, literals
);
595 if (rlwit_word_size(&rlw_i
) > 0)
596 rlwit_discharge(&rlw_i
, out
, ~0, 0);
598 rlwit_discharge_empty(&rlw_j
, out
);
600 out
->bit_size
= max_size(ewah_i
->bit_size
, ewah_j
->bit_size
);
604 struct ewah_bitmap
*ewah_i
,
605 struct ewah_bitmap
*ewah_j
,
606 struct ewah_bitmap
*out
)
608 struct rlw_iterator rlw_i
;
609 struct rlw_iterator rlw_j
;
612 rlwit_init(&rlw_i
, ewah_i
);
613 rlwit_init(&rlw_j
, ewah_j
);
615 while (rlwit_word_size(&rlw_i
) > 0 && rlwit_word_size(&rlw_j
) > 0) {
616 while (rlw_i
.rlw
.running_len
> 0 || rlw_j
.rlw
.running_len
> 0) {
617 struct rlw_iterator
*prey
, *predator
;
619 if (rlw_i
.rlw
.running_len
< rlw_j
.rlw
.running_len
) {
627 if (predator
->rlw
.running_bit
) {
628 ewah_add_empty_words(out
, 0,
629 predator
->rlw
.running_len
);
630 rlwit_discard_first_words(prey
,
631 predator
->rlw
.running_len
);
632 rlwit_discard_first_words(predator
,
633 predator
->rlw
.running_len
);
635 size_t index
= rlwit_discharge(prey
, out
,
636 predator
->rlw
.running_len
, 0);
637 ewah_add_empty_words(out
, 0,
638 predator
->rlw
.running_len
- index
);
639 rlwit_discard_first_words(predator
,
640 predator
->rlw
.running_len
);
645 rlw_i
.rlw
.literal_words
,
646 rlw_j
.rlw
.literal_words
);
651 for (k
= 0; k
< literals
; ++k
) {
653 rlw_i
.buffer
[rlw_i
.literal_word_start
+ k
] |
654 rlw_j
.buffer
[rlw_j
.literal_word_start
+ k
]
658 rlwit_discard_first_words(&rlw_i
, literals
);
659 rlwit_discard_first_words(&rlw_j
, literals
);
663 if (rlwit_word_size(&rlw_i
) > 0)
664 rlwit_discharge(&rlw_i
, out
, ~0, 0);
666 rlwit_discharge(&rlw_j
, out
, ~0, 0);
668 out
->bit_size
= max_size(ewah_i
->bit_size
, ewah_j
->bit_size
);
672 #define BITMAP_POOL_MAX 16
673 static struct ewah_bitmap
*bitmap_pool
[BITMAP_POOL_MAX
];
674 static size_t bitmap_pool_size
;
676 struct ewah_bitmap
*ewah_pool_new(void)
678 if (bitmap_pool_size
)
679 return bitmap_pool
[--bitmap_pool_size
];
684 void ewah_pool_free(struct ewah_bitmap
*self
)
689 if (bitmap_pool_size
== BITMAP_POOL_MAX
||
690 self
->alloc_size
== 0) {
696 bitmap_pool
[bitmap_pool_size
++] = self
;
699 uint32_t ewah_checksum(struct ewah_bitmap
*self
)
701 const uint8_t *p
= (uint8_t *)self
->buffer
;
702 uint32_t crc
= (uint32_t)self
->bit_size
;
703 size_t size
= self
->buffer_size
* sizeof(eword_t
);
706 crc
= (crc
<< 5) - crc
+ (uint32_t)*p
++;