1
2
3
4
5
6
7
8
9
10
11
12#include "qemu/osdep.h"
13#include "qemu/hbitmap.h"
14#include "qemu/host-utils.h"
15#include "trace.h"
16#include "crypto/hash.h"
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55struct HBitmap {
56
57 uint64_t orig_size;
58
59
60 uint64_t size;
61
62
63 uint64_t count;
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83 int granularity;
84
85
86 HBitmap *meta;
87
88
89
90
91
92
93
94
95
96 unsigned long *levels[HBITMAP_LEVELS];
97
98
99 uint64_t sizes[HBITMAP_LEVELS];
100};
101
102
103
104
105unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi)
106{
107 size_t pos = hbi->pos;
108 const HBitmap *hb = hbi->hb;
109 unsigned i = HBITMAP_LEVELS - 1;
110
111 unsigned long cur;
112 do {
113 i--;
114 pos >>= BITS_PER_LEVEL;
115 cur = hbi->cur[i] & hb->levels[i][pos];
116 } while (cur == 0);
117
118
119
120
121
122
123
124 if (i == 0 && cur == (1UL << (BITS_PER_LONG - 1))) {
125 return 0;
126 }
127 for (; i < HBITMAP_LEVELS - 1; i++) {
128
129
130
131
132 assert(cur);
133 pos = (pos << BITS_PER_LEVEL) + ctzl(cur);
134 hbi->cur[i] = cur & (cur - 1);
135
136
137 cur = hb->levels[i + 1][pos];
138 }
139
140 hbi->pos = pos;
141 trace_hbitmap_iter_skip_words(hbi->hb, hbi, pos, cur);
142
143 assert(cur);
144 return cur;
145}
146
147int64_t hbitmap_iter_next(HBitmapIter *hbi)
148{
149 unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1] &
150 hbi->hb->levels[HBITMAP_LEVELS - 1][hbi->pos];
151 int64_t item;
152
153 if (cur == 0) {
154 cur = hbitmap_iter_skip_words(hbi);
155 if (cur == 0) {
156 return -1;
157 }
158 }
159
160
161 hbi->cur[HBITMAP_LEVELS - 1] = cur & (cur - 1);
162 item = ((uint64_t)hbi->pos << BITS_PER_LEVEL) + ctzl(cur);
163
164 return item << hbi->granularity;
165}
166
167void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first)
168{
169 unsigned i, bit;
170 uint64_t pos;
171
172 hbi->hb = hb;
173 pos = first >> hb->granularity;
174 assert(pos < hb->size);
175 hbi->pos = pos >> BITS_PER_LEVEL;
176 hbi->granularity = hb->granularity;
177
178 for (i = HBITMAP_LEVELS; i-- > 0; ) {
179 bit = pos & (BITS_PER_LONG - 1);
180 pos >>= BITS_PER_LEVEL;
181
182
183 hbi->cur[i] = hb->levels[i][pos] & ~((1UL << bit) - 1);
184
185
186
187
188 if (i != HBITMAP_LEVELS - 1) {
189 hbi->cur[i] &= ~(1UL << bit);
190 }
191 }
192}
193
194int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t start, uint64_t count)
195{
196 size_t pos = (start >> hb->granularity) >> BITS_PER_LEVEL;
197 unsigned long *last_lev = hb->levels[HBITMAP_LEVELS - 1];
198 unsigned long cur = last_lev[pos];
199 unsigned start_bit_offset;
200 uint64_t end_bit, sz;
201 int64_t res;
202
203 if (start >= hb->orig_size || count == 0) {
204 return -1;
205 }
206
207 end_bit = count > hb->orig_size - start ?
208 hb->size :
209 ((start + count - 1) >> hb->granularity) + 1;
210 sz = (end_bit + BITS_PER_LONG - 1) >> BITS_PER_LEVEL;
211
212
213
214
215 start_bit_offset = (start >> hb->granularity) & (BITS_PER_LONG - 1);
216 cur |= (1UL << start_bit_offset) - 1;
217 assert((start >> hb->granularity) < hb->size);
218
219 if (cur == (unsigned long)-1) {
220 do {
221 pos++;
222 } while (pos < sz && last_lev[pos] == (unsigned long)-1);
223
224 if (pos >= sz) {
225 return -1;
226 }
227
228 cur = last_lev[pos];
229 }
230
231 res = (pos << BITS_PER_LEVEL) + ctol(cur);
232 if (res >= end_bit) {
233 return -1;
234 }
235
236 res = res << hb->granularity;
237 if (res < start) {
238 assert(((start - res) >> hb->granularity) == 0);
239 return start;
240 }
241
242 return res;
243}
244
245bool hbitmap_next_dirty_area(const HBitmap *hb, uint64_t *start,
246 uint64_t *count)
247{
248 HBitmapIter hbi;
249 int64_t firt_dirty_off, area_end;
250 uint32_t granularity = 1UL << hb->granularity;
251 uint64_t end;
252
253 if (*start >= hb->orig_size || *count == 0) {
254 return false;
255 }
256
257 end = *count > hb->orig_size - *start ? hb->orig_size : *start + *count;
258
259 hbitmap_iter_init(&hbi, hb, *start);
260 firt_dirty_off = hbitmap_iter_next(&hbi);
261
262 if (firt_dirty_off < 0 || firt_dirty_off >= end) {
263 return false;
264 }
265
266 if (firt_dirty_off + granularity >= end) {
267 area_end = end;
268 } else {
269 area_end = hbitmap_next_zero(hb, firt_dirty_off + granularity,
270 end - firt_dirty_off - granularity);
271 if (area_end < 0) {
272 area_end = end;
273 }
274 }
275
276 if (firt_dirty_off > *start) {
277 *start = firt_dirty_off;
278 }
279 *count = area_end - *start;
280
281 return true;
282}
283
284bool hbitmap_empty(const HBitmap *hb)
285{
286 return hb->count == 0;
287}
288
289int hbitmap_granularity(const HBitmap *hb)
290{
291 return hb->granularity;
292}
293
294uint64_t hbitmap_count(const HBitmap *hb)
295{
296 return hb->count << hb->granularity;
297}
298
299
300
301
302static uint64_t hb_count_between(HBitmap *hb, uint64_t start, uint64_t last)
303{
304 HBitmapIter hbi;
305 uint64_t count = 0;
306 uint64_t end = last + 1;
307 unsigned long cur;
308 size_t pos;
309
310 hbitmap_iter_init(&hbi, hb, start << hb->granularity);
311 for (;;) {
312 pos = hbitmap_iter_next_word(&hbi, &cur);
313 if (pos >= (end >> BITS_PER_LEVEL)) {
314 break;
315 }
316 count += ctpopl(cur);
317 }
318
319 if (pos == (end >> BITS_PER_LEVEL)) {
320
321 int bit = end & (BITS_PER_LONG - 1);
322 cur &= (1UL << bit) - 1;
323 count += ctpopl(cur);
324 }
325
326 return count;
327}
328
329
330
331
332static inline bool hb_set_elem(unsigned long *elem, uint64_t start, uint64_t last)
333{
334 unsigned long mask;
335 unsigned long old;
336
337 assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
338 assert(start <= last);
339
340 mask = 2UL << (last & (BITS_PER_LONG - 1));
341 mask -= 1UL << (start & (BITS_PER_LONG - 1));
342 old = *elem;
343 *elem |= mask;
344 return old != *elem;
345}
346
347
348
349static bool hb_set_between(HBitmap *hb, int level, uint64_t start,
350 uint64_t last)
351{
352 size_t pos = start >> BITS_PER_LEVEL;
353 size_t lastpos = last >> BITS_PER_LEVEL;
354 bool changed = false;
355 size_t i;
356
357 i = pos;
358 if (i < lastpos) {
359 uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
360 changed |= hb_set_elem(&hb->levels[level][i], start, next - 1);
361 for (;;) {
362 start = next;
363 next += BITS_PER_LONG;
364 if (++i == lastpos) {
365 break;
366 }
367 changed |= (hb->levels[level][i] == 0);
368 hb->levels[level][i] = ~0UL;
369 }
370 }
371 changed |= hb_set_elem(&hb->levels[level][i], start, last);
372
373
374
375
376 if (level > 0 && changed) {
377 hb_set_between(hb, level - 1, pos, lastpos);
378 }
379 return changed;
380}
381
382void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count)
383{
384
385 uint64_t first, n;
386 uint64_t last = start + count - 1;
387
388 trace_hbitmap_set(hb, start, count,
389 start >> hb->granularity, last >> hb->granularity);
390
391 first = start >> hb->granularity;
392 last >>= hb->granularity;
393 assert(last < hb->size);
394 n = last - first + 1;
395
396 hb->count += n - hb_count_between(hb, first, last);
397 if (hb_set_between(hb, HBITMAP_LEVELS - 1, first, last) &&
398 hb->meta) {
399 hbitmap_set(hb->meta, start, count);
400 }
401}
402
403
404
405
406static inline bool hb_reset_elem(unsigned long *elem, uint64_t start, uint64_t last)
407{
408 unsigned long mask;
409 bool blanked;
410
411 assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
412 assert(start <= last);
413
414 mask = 2UL << (last & (BITS_PER_LONG - 1));
415 mask -= 1UL << (start & (BITS_PER_LONG - 1));
416 blanked = *elem != 0 && ((*elem & ~mask) == 0);
417 *elem &= ~mask;
418 return blanked;
419}
420
421
422
423static bool hb_reset_between(HBitmap *hb, int level, uint64_t start,
424 uint64_t last)
425{
426 size_t pos = start >> BITS_PER_LEVEL;
427 size_t lastpos = last >> BITS_PER_LEVEL;
428 bool changed = false;
429 size_t i;
430
431 i = pos;
432 if (i < lastpos) {
433 uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
434
435
436
437
438
439
440 if (hb_reset_elem(&hb->levels[level][i], start, next - 1)) {
441 changed = true;
442 } else {
443 pos++;
444 }
445
446 for (;;) {
447 start = next;
448 next += BITS_PER_LONG;
449 if (++i == lastpos) {
450 break;
451 }
452 changed |= (hb->levels[level][i] != 0);
453 hb->levels[level][i] = 0UL;
454 }
455 }
456
457
458 if (hb_reset_elem(&hb->levels[level][i], start, last)) {
459 changed = true;
460 } else {
461 lastpos--;
462 }
463
464 if (level > 0 && changed) {
465 hb_reset_between(hb, level - 1, pos, lastpos);
466 }
467
468 return changed;
469
470}
471
472void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count)
473{
474
475 uint64_t first;
476 uint64_t last = start + count - 1;
477
478 trace_hbitmap_reset(hb, start, count,
479 start >> hb->granularity, last >> hb->granularity);
480
481 first = start >> hb->granularity;
482 last >>= hb->granularity;
483 assert(last < hb->size);
484
485 hb->count -= hb_count_between(hb, first, last);
486 if (hb_reset_between(hb, HBITMAP_LEVELS - 1, first, last) &&
487 hb->meta) {
488 hbitmap_set(hb->meta, start, count);
489 }
490}
491
492void hbitmap_reset_all(HBitmap *hb)
493{
494 unsigned int i;
495
496
497 for (i = HBITMAP_LEVELS; --i >= 1; ) {
498 memset(hb->levels[i], 0, hb->sizes[i] * sizeof(unsigned long));
499 }
500
501 hb->levels[0][0] = 1UL << (BITS_PER_LONG - 1);
502 hb->count = 0;
503}
504
505bool hbitmap_is_serializable(const HBitmap *hb)
506{
507
508
509
510
511
512
513
514
515
516
517
518
519
520 return hb->granularity < 58;
521}
522
523bool hbitmap_get(const HBitmap *hb, uint64_t item)
524{
525
526 uint64_t pos = item >> hb->granularity;
527 unsigned long bit = 1UL << (pos & (BITS_PER_LONG - 1));
528 assert(pos < hb->size);
529
530 return (hb->levels[HBITMAP_LEVELS - 1][pos >> BITS_PER_LEVEL] & bit) != 0;
531}
532
533uint64_t hbitmap_serialization_align(const HBitmap *hb)
534{
535 assert(hbitmap_is_serializable(hb));
536
537
538
539 return UINT64_C(64) << hb->granularity;
540}
541
542
543
544
545static void serialization_chunk(const HBitmap *hb,
546 uint64_t start, uint64_t count,
547 unsigned long **first_el, uint64_t *el_count)
548{
549 uint64_t last = start + count - 1;
550 uint64_t gran = hbitmap_serialization_align(hb);
551
552 assert((start & (gran - 1)) == 0);
553 assert((last >> hb->granularity) < hb->size);
554 if ((last >> hb->granularity) != hb->size - 1) {
555 assert((count & (gran - 1)) == 0);
556 }
557
558 start = (start >> hb->granularity) >> BITS_PER_LEVEL;
559 last = (last >> hb->granularity) >> BITS_PER_LEVEL;
560
561 *first_el = &hb->levels[HBITMAP_LEVELS - 1][start];
562 *el_count = last - start + 1;
563}
564
565uint64_t hbitmap_serialization_size(const HBitmap *hb,
566 uint64_t start, uint64_t count)
567{
568 uint64_t el_count;
569 unsigned long *cur;
570
571 if (!count) {
572 return 0;
573 }
574 serialization_chunk(hb, start, count, &cur, &el_count);
575
576 return el_count * sizeof(unsigned long);
577}
578
579void hbitmap_serialize_part(const HBitmap *hb, uint8_t *buf,
580 uint64_t start, uint64_t count)
581{
582 uint64_t el_count;
583 unsigned long *cur, *end;
584
585 if (!count) {
586 return;
587 }
588 serialization_chunk(hb, start, count, &cur, &el_count);
589 end = cur + el_count;
590
591 while (cur != end) {
592 unsigned long el =
593 (BITS_PER_LONG == 32 ? cpu_to_le32(*cur) : cpu_to_le64(*cur));
594
595 memcpy(buf, &el, sizeof(el));
596 buf += sizeof(el);
597 cur++;
598 }
599}
600
601void hbitmap_deserialize_part(HBitmap *hb, uint8_t *buf,
602 uint64_t start, uint64_t count,
603 bool finish)
604{
605 uint64_t el_count;
606 unsigned long *cur, *end;
607
608 if (!count) {
609 return;
610 }
611 serialization_chunk(hb, start, count, &cur, &el_count);
612 end = cur + el_count;
613
614 while (cur != end) {
615 memcpy(cur, buf, sizeof(*cur));
616
617 if (BITS_PER_LONG == 32) {
618 le32_to_cpus((uint32_t *)cur);
619 } else {
620 le64_to_cpus((uint64_t *)cur);
621 }
622
623 buf += sizeof(unsigned long);
624 cur++;
625 }
626 if (finish) {
627 hbitmap_deserialize_finish(hb);
628 }
629}
630
631void hbitmap_deserialize_zeroes(HBitmap *hb, uint64_t start, uint64_t count,
632 bool finish)
633{
634 uint64_t el_count;
635 unsigned long *first;
636
637 if (!count) {
638 return;
639 }
640 serialization_chunk(hb, start, count, &first, &el_count);
641
642 memset(first, 0, el_count * sizeof(unsigned long));
643 if (finish) {
644 hbitmap_deserialize_finish(hb);
645 }
646}
647
648void hbitmap_deserialize_ones(HBitmap *hb, uint64_t start, uint64_t count,
649 bool finish)
650{
651 uint64_t el_count;
652 unsigned long *first;
653
654 if (!count) {
655 return;
656 }
657 serialization_chunk(hb, start, count, &first, &el_count);
658
659 memset(first, 0xff, el_count * sizeof(unsigned long));
660 if (finish) {
661 hbitmap_deserialize_finish(hb);
662 }
663}
664
665void hbitmap_deserialize_finish(HBitmap *bitmap)
666{
667 int64_t i, size, prev_size;
668 int lev;
669
670
671
672 size = MAX((bitmap->size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
673 for (lev = HBITMAP_LEVELS - 1; lev-- > 0; ) {
674 prev_size = size;
675 size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
676 memset(bitmap->levels[lev], 0, size * sizeof(unsigned long));
677
678 for (i = 0; i < prev_size; ++i) {
679 if (bitmap->levels[lev + 1][i]) {
680 bitmap->levels[lev][i >> BITS_PER_LEVEL] |=
681 1UL << (i & (BITS_PER_LONG - 1));
682 }
683 }
684 }
685
686 bitmap->levels[0][0] |= 1UL << (BITS_PER_LONG - 1);
687 bitmap->count = hb_count_between(bitmap, 0, bitmap->size - 1);
688}
689
690void hbitmap_free(HBitmap *hb)
691{
692 unsigned i;
693 assert(!hb->meta);
694 for (i = HBITMAP_LEVELS; i-- > 0; ) {
695 g_free(hb->levels[i]);
696 }
697 g_free(hb);
698}
699
700HBitmap *hbitmap_alloc(uint64_t size, int granularity)
701{
702 HBitmap *hb = g_new0(struct HBitmap, 1);
703 unsigned i;
704
705 hb->orig_size = size;
706
707 assert(granularity >= 0 && granularity < 64);
708 size = (size + (1ULL << granularity) - 1) >> granularity;
709 assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE));
710
711 hb->size = size;
712 hb->granularity = granularity;
713 for (i = HBITMAP_LEVELS; i-- > 0; ) {
714 size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
715 hb->sizes[i] = size;
716 hb->levels[i] = g_new0(unsigned long, size);
717 }
718
719
720
721
722
723 assert(size == 1);
724 hb->levels[0][0] |= 1UL << (BITS_PER_LONG - 1);
725 return hb;
726}
727
728void hbitmap_truncate(HBitmap *hb, uint64_t size)
729{
730 bool shrink;
731 unsigned i;
732 uint64_t num_elements = size;
733 uint64_t old;
734
735
736 size = (size + (1ULL << hb->granularity) - 1) >> hb->granularity;
737 assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE));
738 shrink = size < hb->size;
739
740
741 if (size == hb->size) {
742 return;
743 }
744
745
746
747
748
749 if (shrink) {
750
751
752 uint64_t start = ROUND_UP(num_elements, UINT64_C(1) << hb->granularity);
753 uint64_t fix_count = (hb->size << hb->granularity) - start;
754
755 assert(fix_count);
756 hbitmap_reset(hb, start, fix_count);
757 }
758
759 hb->size = size;
760 for (i = HBITMAP_LEVELS; i-- > 0; ) {
761 size = MAX(BITS_TO_LONGS(size), 1);
762 if (hb->sizes[i] == size) {
763 break;
764 }
765 old = hb->sizes[i];
766 hb->sizes[i] = size;
767 hb->levels[i] = g_realloc(hb->levels[i], size * sizeof(unsigned long));
768 if (!shrink) {
769 memset(&hb->levels[i][old], 0x00,
770 (size - old) * sizeof(*hb->levels[i]));
771 }
772 }
773 if (hb->meta) {
774 hbitmap_truncate(hb->meta, hb->size << hb->granularity);
775 }
776}
777
778bool hbitmap_can_merge(const HBitmap *a, const HBitmap *b)
779{
780 return (a->size == b->size) && (a->granularity == b->granularity);
781}
782
783
784
785
786
787
788
789
790bool hbitmap_merge(const HBitmap *a, const HBitmap *b, HBitmap *result)
791{
792 int i;
793 uint64_t j;
794
795 if (!hbitmap_can_merge(a, b) || !hbitmap_can_merge(a, result)) {
796 return false;
797 }
798 assert(hbitmap_can_merge(b, result));
799
800 if (hbitmap_count(b) == 0) {
801 return true;
802 }
803
804
805
806
807
808 for (i = HBITMAP_LEVELS - 1; i >= 0; i--) {
809 for (j = 0; j < a->sizes[i]; j++) {
810 result->levels[i][j] = a->levels[i][j] | b->levels[i][j];
811 }
812 }
813
814
815 result->count = hb_count_between(result, 0, result->size - 1);
816
817 return true;
818}
819
820HBitmap *hbitmap_create_meta(HBitmap *hb, int chunk_size)
821{
822 assert(!(chunk_size & (chunk_size - 1)));
823 assert(!hb->meta);
824 hb->meta = hbitmap_alloc(hb->size << hb->granularity,
825 hb->granularity + ctz32(chunk_size));
826 return hb->meta;
827}
828
829void hbitmap_free_meta(HBitmap *hb)
830{
831 assert(hb->meta);
832 hbitmap_free(hb->meta);
833 hb->meta = NULL;
834}
835
836char *hbitmap_sha256(const HBitmap *bitmap, Error **errp)
837{
838 size_t size = bitmap->sizes[HBITMAP_LEVELS - 1] * sizeof(unsigned long);
839 char *data = (char *)bitmap->levels[HBITMAP_LEVELS - 1];
840 char *hash = NULL;
841 qcrypto_hash_digest(QCRYPTO_HASH_ALG_SHA256, data, size, &hash, errp);
842
843 return hash;
844}
845