1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/slab.h>
15#include <linux/err.h>
16#include <linux/assoc_array_priv.h>
17
18
19
20
21
22static int assoc_array_subtree_iterate(const struct assoc_array_ptr *root,
23 const struct assoc_array_ptr *stop,
24 int (*iterator)(const void *leaf,
25 void *iterator_data),
26 void *iterator_data)
27{
28 const struct assoc_array_shortcut *shortcut;
29 const struct assoc_array_node *node;
30 const struct assoc_array_ptr *cursor, *ptr, *parent;
31 unsigned long has_meta;
32 int slot, ret;
33
34 cursor = root;
35
36begin_node:
37 if (assoc_array_ptr_is_shortcut(cursor)) {
38
39 shortcut = assoc_array_ptr_to_shortcut(cursor);
40 smp_read_barrier_depends();
41 cursor = ACCESS_ONCE(shortcut->next_node);
42 }
43
44 node = assoc_array_ptr_to_node(cursor);
45 smp_read_barrier_depends();
46 slot = 0;
47
48
49
50
51
52
53
54
55 has_meta = 0;
56 for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
57 ptr = ACCESS_ONCE(node->slots[slot]);
58 has_meta |= (unsigned long)ptr;
59 if (ptr && assoc_array_ptr_is_leaf(ptr)) {
60
61
62
63
64 smp_read_barrier_depends();
65
66
67 ret = iterator(assoc_array_ptr_to_leaf(ptr),
68 iterator_data);
69 if (ret)
70 return ret;
71 }
72 }
73
74
75
76
77
78
79
80
81
82 if (!(has_meta & ASSOC_ARRAY_PTR_META_TYPE))
83 goto finished_node;
84 slot = 0;
85
86continue_node:
87 node = assoc_array_ptr_to_node(cursor);
88 smp_read_barrier_depends();
89
90 for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
91 ptr = ACCESS_ONCE(node->slots[slot]);
92 if (assoc_array_ptr_is_meta(ptr)) {
93 cursor = ptr;
94 goto begin_node;
95 }
96 }
97
98finished_node:
99
100 parent = ACCESS_ONCE(node->back_pointer);
101 slot = node->parent_slot;
102 if (parent == stop)
103 return 0;
104
105 if (assoc_array_ptr_is_shortcut(parent)) {
106 shortcut = assoc_array_ptr_to_shortcut(parent);
107 smp_read_barrier_depends();
108 cursor = parent;
109 parent = ACCESS_ONCE(shortcut->back_pointer);
110 slot = shortcut->parent_slot;
111 if (parent == stop)
112 return 0;
113 }
114
115
116 cursor = parent;
117 slot++;
118 goto continue_node;
119}
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144int assoc_array_iterate(const struct assoc_array *array,
145 int (*iterator)(const void *object,
146 void *iterator_data),
147 void *iterator_data)
148{
149 struct assoc_array_ptr *root = ACCESS_ONCE(array->root);
150
151 if (!root)
152 return 0;
153 return assoc_array_subtree_iterate(root, NULL, iterator, iterator_data);
154}
155
156enum assoc_array_walk_status {
157 assoc_array_walk_tree_empty,
158 assoc_array_walk_found_terminal_node,
159 assoc_array_walk_found_wrong_shortcut,
160};
161
162struct assoc_array_walk_result {
163 struct {
164 struct assoc_array_node *node;
165 int level;
166 int slot;
167 } terminal_node;
168 struct {
169 struct assoc_array_shortcut *shortcut;
170 int level;
171 int sc_level;
172 unsigned long sc_segments;
173 unsigned long dissimilarity;
174 } wrong_shortcut;
175};
176
177
178
179
180static enum assoc_array_walk_status
181assoc_array_walk(const struct assoc_array *array,
182 const struct assoc_array_ops *ops,
183 const void *index_key,
184 struct assoc_array_walk_result *result)
185{
186 struct assoc_array_shortcut *shortcut;
187 struct assoc_array_node *node;
188 struct assoc_array_ptr *cursor, *ptr;
189 unsigned long sc_segments, dissimilarity;
190 unsigned long segments;
191 int level, sc_level, next_sc_level;
192 int slot;
193
194 pr_devel("-->%s()\n", __func__);
195
196 cursor = ACCESS_ONCE(array->root);
197 if (!cursor)
198 return assoc_array_walk_tree_empty;
199
200 level = 0;
201
202
203
204
205
206
207
208
209jumped:
210 segments = ops->get_key_chunk(index_key, level);
211 pr_devel("segments[%d]: %lx\n", level, segments);
212
213 if (assoc_array_ptr_is_shortcut(cursor))
214 goto follow_shortcut;
215
216consider_node:
217 node = assoc_array_ptr_to_node(cursor);
218 smp_read_barrier_depends();
219
220 slot = segments >> (level & ASSOC_ARRAY_KEY_CHUNK_MASK);
221 slot &= ASSOC_ARRAY_FAN_MASK;
222 ptr = ACCESS_ONCE(node->slots[slot]);
223
224 pr_devel("consider slot %x [ix=%d type=%lu]\n",
225 slot, level, (unsigned long)ptr & 3);
226
227 if (!assoc_array_ptr_is_meta(ptr)) {
228
229
230
231 result->terminal_node.node = node;
232 result->terminal_node.level = level;
233 result->terminal_node.slot = slot;
234 pr_devel("<--%s() = terminal_node\n", __func__);
235 return assoc_array_walk_found_terminal_node;
236 }
237
238 if (assoc_array_ptr_is_node(ptr)) {
239
240
241
242 cursor = ptr;
243 level += ASSOC_ARRAY_LEVEL_STEP;
244 if ((level & ASSOC_ARRAY_KEY_CHUNK_MASK) != 0)
245 goto consider_node;
246 goto jumped;
247 }
248
249
250
251
252
253 cursor = ptr;
254follow_shortcut:
255 shortcut = assoc_array_ptr_to_shortcut(cursor);
256 smp_read_barrier_depends();
257 pr_devel("shortcut to %d\n", shortcut->skip_to_level);
258 sc_level = level + ASSOC_ARRAY_LEVEL_STEP;
259 BUG_ON(sc_level > shortcut->skip_to_level);
260
261 do {
262
263
264
265
266 if ((sc_level & ASSOC_ARRAY_KEY_CHUNK_MASK) == 0)
267 segments = ops->get_key_chunk(index_key, sc_level);
268
269 sc_segments = shortcut->index_key[sc_level >> ASSOC_ARRAY_KEY_CHUNK_SHIFT];
270 dissimilarity = segments ^ sc_segments;
271
272 if (round_up(sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE) > shortcut->skip_to_level) {
273
274 int shift = shortcut->skip_to_level & ASSOC_ARRAY_KEY_CHUNK_MASK;
275 dissimilarity &= ~(ULONG_MAX << shift);
276 next_sc_level = shortcut->skip_to_level;
277 } else {
278 next_sc_level = sc_level + ASSOC_ARRAY_KEY_CHUNK_SIZE;
279 next_sc_level = round_down(next_sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
280 }
281
282 if (dissimilarity != 0) {
283
284 result->wrong_shortcut.shortcut = shortcut;
285 result->wrong_shortcut.level = level;
286 result->wrong_shortcut.sc_level = sc_level;
287 result->wrong_shortcut.sc_segments = sc_segments;
288 result->wrong_shortcut.dissimilarity = dissimilarity;
289 return assoc_array_walk_found_wrong_shortcut;
290 }
291
292 sc_level = next_sc_level;
293 } while (sc_level < shortcut->skip_to_level);
294
295
296 cursor = ACCESS_ONCE(shortcut->next_node);
297 if (((level ^ sc_level) & ~ASSOC_ARRAY_KEY_CHUNK_MASK) != 0) {
298 level = sc_level;
299 goto jumped;
300 } else {
301 level = sc_level;
302 goto consider_node;
303 }
304}
305
306
307
308
309
310
311
312
313
314
315
316
317
318void *assoc_array_find(const struct assoc_array *array,
319 const struct assoc_array_ops *ops,
320 const void *index_key)
321{
322 struct assoc_array_walk_result result;
323 const struct assoc_array_node *node;
324 const struct assoc_array_ptr *ptr;
325 const void *leaf;
326 int slot;
327
328 if (assoc_array_walk(array, ops, index_key, &result) !=
329 assoc_array_walk_found_terminal_node)
330 return NULL;
331
332 node = result.terminal_node.node;
333 smp_read_barrier_depends();
334
335
336
337
338 for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
339 ptr = ACCESS_ONCE(node->slots[slot]);
340 if (ptr && assoc_array_ptr_is_leaf(ptr)) {
341
342
343
344
345 leaf = assoc_array_ptr_to_leaf(ptr);
346 smp_read_barrier_depends();
347 if (ops->compare_object(leaf, index_key))
348 return (void *)leaf;
349 }
350 }
351
352 return NULL;
353}
354
355
356
357
358
359static void assoc_array_destroy_subtree(struct assoc_array_ptr *root,
360 const struct assoc_array_ops *ops)
361{
362 struct assoc_array_shortcut *shortcut;
363 struct assoc_array_node *node;
364 struct assoc_array_ptr *cursor, *parent = NULL;
365 int slot = -1;
366
367 pr_devel("-->%s()\n", __func__);
368
369 cursor = root;
370 if (!cursor) {
371 pr_devel("empty\n");
372 return;
373 }
374
375move_to_meta:
376 if (assoc_array_ptr_is_shortcut(cursor)) {
377
378 pr_devel("[%d] shortcut\n", slot);
379 BUG_ON(!assoc_array_ptr_is_shortcut(cursor));
380 shortcut = assoc_array_ptr_to_shortcut(cursor);
381 BUG_ON(shortcut->back_pointer != parent);
382 BUG_ON(slot != -1 && shortcut->parent_slot != slot);
383 parent = cursor;
384 cursor = shortcut->next_node;
385 slot = -1;
386 BUG_ON(!assoc_array_ptr_is_node(cursor));
387 }
388
389 pr_devel("[%d] node\n", slot);
390 node = assoc_array_ptr_to_node(cursor);
391 BUG_ON(node->back_pointer != parent);
392 BUG_ON(slot != -1 && node->parent_slot != slot);
393 slot = 0;
394
395continue_node:
396 pr_devel("Node %p [back=%p]\n", node, node->back_pointer);
397 for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
398 struct assoc_array_ptr *ptr = node->slots[slot];
399 if (!ptr)
400 continue;
401 if (assoc_array_ptr_is_meta(ptr)) {
402 parent = cursor;
403 cursor = ptr;
404 goto move_to_meta;
405 }
406
407 if (ops) {
408 pr_devel("[%d] free leaf\n", slot);
409 ops->free_object(assoc_array_ptr_to_leaf(ptr));
410 }
411 }
412
413 parent = node->back_pointer;
414 slot = node->parent_slot;
415 pr_devel("free node\n");
416 kfree(node);
417 if (!parent)
418 return;
419
420
421
422 if (assoc_array_ptr_is_shortcut(parent)) {
423 shortcut = assoc_array_ptr_to_shortcut(parent);
424 BUG_ON(shortcut->next_node != cursor);
425 cursor = parent;
426 parent = shortcut->back_pointer;
427 slot = shortcut->parent_slot;
428 pr_devel("free shortcut\n");
429 kfree(shortcut);
430 if (!parent)
431 return;
432
433 BUG_ON(!assoc_array_ptr_is_node(parent));
434 }
435
436
437 pr_devel("ascend to %p[%d]\n", parent, slot);
438 cursor = parent;
439 node = assoc_array_ptr_to_node(cursor);
440 slot++;
441 goto continue_node;
442}
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457void assoc_array_destroy(struct assoc_array *array,
458 const struct assoc_array_ops *ops)
459{
460 assoc_array_destroy_subtree(array->root, ops);
461 array->root = NULL;
462}
463
464
465
466
467static bool assoc_array_insert_in_empty_tree(struct assoc_array_edit *edit)
468{
469 struct assoc_array_node *new_n0;
470
471 pr_devel("-->%s()\n", __func__);
472
473 new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
474 if (!new_n0)
475 return false;
476
477 edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
478 edit->leaf_p = &new_n0->slots[0];
479 edit->adjust_count_on = new_n0;
480 edit->set[0].ptr = &edit->array->root;
481 edit->set[0].to = assoc_array_node_to_ptr(new_n0);
482
483 pr_devel("<--%s() = ok [no root]\n", __func__);
484 return true;
485}
486
487
488
489
490static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit,
491 const struct assoc_array_ops *ops,
492 const void *index_key,
493 struct assoc_array_walk_result *result)
494{
495 struct assoc_array_shortcut *shortcut, *new_s0;
496 struct assoc_array_node *node, *new_n0, *new_n1, *side;
497 struct assoc_array_ptr *ptr;
498 unsigned long dissimilarity, base_seg, blank;
499 size_t keylen;
500 bool have_meta;
501 int level, diff;
502 int slot, next_slot, free_slot, i, j;
503
504 node = result->terminal_node.node;
505 level = result->terminal_node.level;
506 edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = result->terminal_node.slot;
507
508 pr_devel("-->%s()\n", __func__);
509
510
511
512
513
514
515 free_slot = -1;
516
517
518
519
520 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
521 ptr = node->slots[i];
522 if (!ptr) {
523 free_slot = i;
524 continue;
525 }
526 if (ops->compare_object(assoc_array_ptr_to_leaf(ptr), index_key)) {
527 pr_devel("replace in slot %d\n", i);
528 edit->leaf_p = &node->slots[i];
529 edit->dead_leaf = node->slots[i];
530 pr_devel("<--%s() = ok [replace]\n", __func__);
531 return true;
532 }
533 }
534
535
536
537
538 if (free_slot >= 0) {
539 pr_devel("insert in free slot %d\n", free_slot);
540 edit->leaf_p = &node->slots[free_slot];
541 edit->adjust_count_on = node;
542 pr_devel("<--%s() = ok [insert]\n", __func__);
543 return true;
544 }
545
546
547
548
549
550
551
552
553 new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
554 if (!new_n0)
555 return false;
556 edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
557 new_n1 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
558 if (!new_n1)
559 return false;
560 edit->new_meta[1] = assoc_array_node_to_ptr(new_n1);
561
562
563 pr_devel("no spare slots\n");
564 have_meta = false;
565 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
566 ptr = node->slots[i];
567 if (assoc_array_ptr_is_meta(ptr)) {
568 edit->segment_cache[i] = 0xff;
569 have_meta = true;
570 continue;
571 }
572 base_seg = ops->get_object_key_chunk(
573 assoc_array_ptr_to_leaf(ptr), level);
574 base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
575 edit->segment_cache[i] = base_seg & ASSOC_ARRAY_FAN_MASK;
576 }
577
578 if (have_meta) {
579 pr_devel("have meta\n");
580 goto split_node;
581 }
582
583
584 dissimilarity = 0;
585 base_seg = edit->segment_cache[0];
586 for (i = 1; i < ASSOC_ARRAY_FAN_OUT; i++)
587 dissimilarity |= edit->segment_cache[i] ^ base_seg;
588
589 pr_devel("only leaves; dissimilarity=%lx\n", dissimilarity);
590
591 if ((dissimilarity & ASSOC_ARRAY_FAN_MASK) == 0) {
592
593
594
595 if ((edit->segment_cache[ASSOC_ARRAY_FAN_OUT] ^ base_seg) == 0)
596 goto all_leaves_cluster_together;
597
598
599
600
601 goto present_leaves_cluster_but_not_new_leaf;
602 }
603
604split_node:
605 pr_devel("split node\n");
606
607
608
609
610
611
612
613
614
615
616
617 edit->set[0].to = assoc_array_node_to_ptr(new_n0);
618 new_n0->back_pointer = node->back_pointer;
619 new_n0->parent_slot = node->parent_slot;
620 new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
621 new_n1->parent_slot = -1;
622
623do_split_node:
624 pr_devel("do_split_node\n");
625
626 new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
627 new_n1->nr_leaves_on_branch = 0;
628
629
630
631
632
633
634
635 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
636 slot = edit->segment_cache[i];
637 if (slot != 0xff)
638 for (j = i + 1; j < ASSOC_ARRAY_FAN_OUT + 1; j++)
639 if (edit->segment_cache[j] == slot)
640 goto found_slot_for_multiple_occupancy;
641 }
642found_slot_for_multiple_occupancy:
643 pr_devel("same slot: %x %x [%02x]\n", i, j, slot);
644 BUG_ON(i >= ASSOC_ARRAY_FAN_OUT);
645 BUG_ON(j >= ASSOC_ARRAY_FAN_OUT + 1);
646 BUG_ON(slot >= ASSOC_ARRAY_FAN_OUT);
647
648 new_n1->parent_slot = slot;
649
650
651 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++)
652 if (assoc_array_ptr_is_meta(node->slots[i]))
653 new_n0->slots[i] = node->slots[i];
654 else
655 new_n0->slots[i] = NULL;
656 BUG_ON(new_n0->slots[slot] != NULL);
657 new_n0->slots[slot] = assoc_array_node_to_ptr(new_n1);
658
659
660 free_slot = -1;
661 next_slot = 0;
662 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
663 if (assoc_array_ptr_is_meta(node->slots[i]))
664 continue;
665 if (edit->segment_cache[i] == slot) {
666 new_n1->slots[next_slot++] = node->slots[i];
667 new_n1->nr_leaves_on_branch++;
668 } else {
669 do {
670 free_slot++;
671 } while (new_n0->slots[free_slot] != NULL);
672 new_n0->slots[free_slot] = node->slots[i];
673 }
674 }
675
676 pr_devel("filtered: f=%x n=%x\n", free_slot, next_slot);
677
678 if (edit->segment_cache[ASSOC_ARRAY_FAN_OUT] != slot) {
679 do {
680 free_slot++;
681 } while (new_n0->slots[free_slot] != NULL);
682 edit->leaf_p = &new_n0->slots[free_slot];
683 edit->adjust_count_on = new_n0;
684 } else {
685 edit->leaf_p = &new_n1->slots[next_slot++];
686 edit->adjust_count_on = new_n1;
687 }
688
689 BUG_ON(next_slot <= 1);
690
691 edit->set_backpointers_to = assoc_array_node_to_ptr(new_n0);
692 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
693 if (edit->segment_cache[i] == 0xff) {
694 ptr = node->slots[i];
695 BUG_ON(assoc_array_ptr_is_leaf(ptr));
696 if (assoc_array_ptr_is_node(ptr)) {
697 side = assoc_array_ptr_to_node(ptr);
698 edit->set_backpointers[i] = &side->back_pointer;
699 } else {
700 shortcut = assoc_array_ptr_to_shortcut(ptr);
701 edit->set_backpointers[i] = &shortcut->back_pointer;
702 }
703 }
704 }
705
706 ptr = node->back_pointer;
707 if (!ptr)
708 edit->set[0].ptr = &edit->array->root;
709 else if (assoc_array_ptr_is_node(ptr))
710 edit->set[0].ptr = &assoc_array_ptr_to_node(ptr)->slots[node->parent_slot];
711 else
712 edit->set[0].ptr = &assoc_array_ptr_to_shortcut(ptr)->next_node;
713 edit->excised_meta[0] = assoc_array_node_to_ptr(node);
714 pr_devel("<--%s() = ok [split node]\n", __func__);
715 return true;
716
717present_leaves_cluster_but_not_new_leaf:
718
719
720
721
722 pr_devel("present leaves cluster but not new leaf\n");
723
724 new_n0->back_pointer = node->back_pointer;
725 new_n0->parent_slot = node->parent_slot;
726 new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
727 new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
728 new_n1->parent_slot = edit->segment_cache[0];
729 new_n1->nr_leaves_on_branch = node->nr_leaves_on_branch;
730 edit->adjust_count_on = new_n0;
731
732 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++)
733 new_n1->slots[i] = node->slots[i];
734
735 new_n0->slots[edit->segment_cache[0]] = assoc_array_node_to_ptr(new_n0);
736 edit->leaf_p = &new_n0->slots[edit->segment_cache[ASSOC_ARRAY_FAN_OUT]];
737
738 edit->set[0].ptr = &assoc_array_ptr_to_node(node->back_pointer)->slots[node->parent_slot];
739 edit->set[0].to = assoc_array_node_to_ptr(new_n0);
740 edit->excised_meta[0] = assoc_array_node_to_ptr(node);
741 pr_devel("<--%s() = ok [insert node before]\n", __func__);
742 return true;
743
744all_leaves_cluster_together:
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759 pr_devel("all leaves cluster together\n");
760 diff = INT_MAX;
761 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
762 int x = ops->diff_objects(assoc_array_ptr_to_leaf(node->slots[i]),
763 index_key);
764 if (x < diff) {
765 BUG_ON(x < 0);
766 diff = x;
767 }
768 }
769 BUG_ON(diff == INT_MAX);
770 BUG_ON(diff < level + ASSOC_ARRAY_LEVEL_STEP);
771
772 keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE);
773 keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
774
775 new_s0 = kzalloc(sizeof(struct assoc_array_shortcut) +
776 keylen * sizeof(unsigned long), GFP_KERNEL);
777 if (!new_s0)
778 return false;
779 edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s0);
780
781 edit->set[0].to = assoc_array_shortcut_to_ptr(new_s0);
782 new_s0->back_pointer = node->back_pointer;
783 new_s0->parent_slot = node->parent_slot;
784 new_s0->next_node = assoc_array_node_to_ptr(new_n0);
785 new_n0->back_pointer = assoc_array_shortcut_to_ptr(new_s0);
786 new_n0->parent_slot = 0;
787 new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
788 new_n1->parent_slot = -1;
789
790 new_s0->skip_to_level = level = diff & ~ASSOC_ARRAY_LEVEL_STEP_MASK;
791 pr_devel("skip_to_level = %d [diff %d]\n", level, diff);
792 BUG_ON(level <= 0);
793
794 for (i = 0; i < keylen; i++)
795 new_s0->index_key[i] =
796 ops->get_key_chunk(index_key, i * ASSOC_ARRAY_KEY_CHUNK_SIZE);
797
798 blank = ULONG_MAX << (level & ASSOC_ARRAY_KEY_CHUNK_MASK);
799 pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, level, blank);
800 new_s0->index_key[keylen - 1] &= ~blank;
801
802
803
804
805 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
806 ptr = node->slots[i];
807 base_seg = ops->get_object_key_chunk(assoc_array_ptr_to_leaf(ptr),
808 level);
809 base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
810 edit->segment_cache[i] = base_seg & ASSOC_ARRAY_FAN_MASK;
811 }
812
813 base_seg = ops->get_key_chunk(index_key, level);
814 base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
815 edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = base_seg & ASSOC_ARRAY_FAN_MASK;
816 goto do_split_node;
817}
818
819
820
821
822static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit,
823 const struct assoc_array_ops *ops,
824 struct assoc_array_walk_result *result)
825{
826 struct assoc_array_shortcut *shortcut, *new_s0, *new_s1;
827 struct assoc_array_node *node, *new_n0, *side;
828 unsigned long sc_segments, dissimilarity, blank;
829 size_t keylen;
830 int level, sc_level, diff;
831 int sc_slot;
832
833 shortcut = result->wrong_shortcut.shortcut;
834 level = result->wrong_shortcut.level;
835 sc_level = result->wrong_shortcut.sc_level;
836 sc_segments = result->wrong_shortcut.sc_segments;
837 dissimilarity = result->wrong_shortcut.dissimilarity;
838
839 pr_devel("-->%s(ix=%d dis=%lx scix=%d)\n",
840 __func__, level, dissimilarity, sc_level);
841
842
843
844
845
846
847
848 diff = __ffs(dissimilarity);
849 diff &= ~ASSOC_ARRAY_LEVEL_STEP_MASK;
850 diff += sc_level & ~ASSOC_ARRAY_KEY_CHUNK_MASK;
851 pr_devel("diff=%d\n", diff);
852
853 if (!shortcut->back_pointer) {
854 edit->set[0].ptr = &edit->array->root;
855 } else if (assoc_array_ptr_is_node(shortcut->back_pointer)) {
856 node = assoc_array_ptr_to_node(shortcut->back_pointer);
857 edit->set[0].ptr = &node->slots[shortcut->parent_slot];
858 } else {
859 BUG();
860 }
861
862 edit->excised_meta[0] = assoc_array_shortcut_to_ptr(shortcut);
863
864
865 new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
866 if (!new_n0)
867 return false;
868 edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
869 edit->adjust_count_on = new_n0;
870
871
872
873
874
875 level += ASSOC_ARRAY_LEVEL_STEP;
876 if (diff > level) {
877 pr_devel("pre-shortcut %d...%d\n", level, diff);
878 keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE);
879 keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
880
881 new_s0 = kzalloc(sizeof(struct assoc_array_shortcut) +
882 keylen * sizeof(unsigned long), GFP_KERNEL);
883 if (!new_s0)
884 return false;
885 edit->new_meta[1] = assoc_array_shortcut_to_ptr(new_s0);
886 edit->set[0].to = assoc_array_shortcut_to_ptr(new_s0);
887 new_s0->back_pointer = shortcut->back_pointer;
888 new_s0->parent_slot = shortcut->parent_slot;
889 new_s0->next_node = assoc_array_node_to_ptr(new_n0);
890 new_s0->skip_to_level = diff;
891
892 new_n0->back_pointer = assoc_array_shortcut_to_ptr(new_s0);
893 new_n0->parent_slot = 0;
894
895 memcpy(new_s0->index_key, shortcut->index_key,
896 keylen * sizeof(unsigned long));
897
898 blank = ULONG_MAX << (diff & ASSOC_ARRAY_KEY_CHUNK_MASK);
899 pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, diff, blank);
900 new_s0->index_key[keylen - 1] &= ~blank;
901 } else {
902 pr_devel("no pre-shortcut\n");
903 edit->set[0].to = assoc_array_node_to_ptr(new_n0);
904 new_n0->back_pointer = shortcut->back_pointer;
905 new_n0->parent_slot = shortcut->parent_slot;
906 }
907
908 side = assoc_array_ptr_to_node(shortcut->next_node);
909 new_n0->nr_leaves_on_branch = side->nr_leaves_on_branch;
910
911
912
913
914 sc_slot = sc_segments >> (diff & ASSOC_ARRAY_KEY_CHUNK_MASK);
915 sc_slot &= ASSOC_ARRAY_FAN_MASK;
916
917 pr_devel("new slot %lx >> %d -> %d\n",
918 sc_segments, diff & ASSOC_ARRAY_KEY_CHUNK_MASK, sc_slot);
919
920
921
922
923
924
925 level = diff + ASSOC_ARRAY_LEVEL_STEP;
926 if (level < shortcut->skip_to_level) {
927 pr_devel("post-shortcut %d...%d\n", level, shortcut->skip_to_level);
928 keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
929 keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
930
931 new_s1 = kzalloc(sizeof(struct assoc_array_shortcut) +
932 keylen * sizeof(unsigned long), GFP_KERNEL);
933 if (!new_s1)
934 return false;
935 edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s1);
936
937 new_s1->back_pointer = assoc_array_node_to_ptr(new_n0);
938 new_s1->parent_slot = sc_slot;
939 new_s1->next_node = shortcut->next_node;
940 new_s1->skip_to_level = shortcut->skip_to_level;
941
942 new_n0->slots[sc_slot] = assoc_array_shortcut_to_ptr(new_s1);
943
944 memcpy(new_s1->index_key, shortcut->index_key,
945 keylen * sizeof(unsigned long));
946
947 edit->set[1].ptr = &side->back_pointer;
948 edit->set[1].to = assoc_array_shortcut_to_ptr(new_s1);
949 } else {
950 pr_devel("no post-shortcut\n");
951
952
953
954
955
956
957 new_n0->slots[sc_slot] = shortcut->next_node;
958 edit->set_parent_slot[0].p = &side->parent_slot;
959 edit->set_parent_slot[0].to = sc_slot;
960 edit->set[1].ptr = &side->back_pointer;
961 edit->set[1].to = assoc_array_node_to_ptr(new_n0);
962 }
963
964
965 if (sc_slot == 0)
966 edit->leaf_p = &new_n0->slots[1];
967 else
968 edit->leaf_p = &new_n0->slots[0];
969
970 pr_devel("<--%s() = ok [split shortcut]\n", __func__);
971 return edit;
972}
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993struct assoc_array_edit *assoc_array_insert(struct assoc_array *array,
994 const struct assoc_array_ops *ops,
995 const void *index_key,
996 void *object)
997{
998 struct assoc_array_walk_result result;
999 struct assoc_array_edit *edit;
1000
1001 pr_devel("-->%s()\n", __func__);
1002
1003
1004
1005
1006
1007
1008 BUG_ON(assoc_array_ptr_is_meta(object));
1009
1010 edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
1011 if (!edit)
1012 return ERR_PTR(-ENOMEM);
1013 edit->array = array;
1014 edit->ops = ops;
1015 edit->leaf = assoc_array_leaf_to_ptr(object);
1016 edit->adjust_count_by = 1;
1017
1018 switch (assoc_array_walk(array, ops, index_key, &result)) {
1019 case assoc_array_walk_tree_empty:
1020
1021 if (!assoc_array_insert_in_empty_tree(edit))
1022 goto enomem;
1023 return edit;
1024
1025 case assoc_array_walk_found_terminal_node:
1026
1027
1028
1029
1030 if (!assoc_array_insert_into_terminal_node(edit, ops, index_key,
1031 &result))
1032 goto enomem;
1033 return edit;
1034
1035 case assoc_array_walk_found_wrong_shortcut:
1036
1037
1038
1039 if (!assoc_array_insert_mid_shortcut(edit, ops, &result))
1040 goto enomem;
1041 return edit;
1042 }
1043
1044enomem:
1045
1046 pr_devel("enomem\n");
1047 assoc_array_cancel_edit(edit);
1048 return ERR_PTR(-ENOMEM);
1049}
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060void assoc_array_insert_set_object(struct assoc_array_edit *edit, void *object)
1061{
1062 BUG_ON(!object);
1063 edit->leaf = assoc_array_leaf_to_ptr(object);
1064}
1065
1066struct assoc_array_delete_collapse_context {
1067 struct assoc_array_node *node;
1068 const void *skip_leaf;
1069 int slot;
1070};
1071
1072
1073
1074
1075static int assoc_array_delete_collapse_iterator(const void *leaf,
1076 void *iterator_data)
1077{
1078 struct assoc_array_delete_collapse_context *collapse = iterator_data;
1079
1080 if (leaf == collapse->skip_leaf)
1081 return 0;
1082
1083 BUG_ON(collapse->slot >= ASSOC_ARRAY_FAN_OUT);
1084
1085 collapse->node->slots[collapse->slot++] = assoc_array_leaf_to_ptr(leaf);
1086 return 0;
1087}
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108struct assoc_array_edit *assoc_array_delete(struct assoc_array *array,
1109 const struct assoc_array_ops *ops,
1110 const void *index_key)
1111{
1112 struct assoc_array_delete_collapse_context collapse;
1113 struct assoc_array_walk_result result;
1114 struct assoc_array_node *node, *new_n0;
1115 struct assoc_array_edit *edit;
1116 struct assoc_array_ptr *ptr;
1117 bool has_meta;
1118 int slot, i;
1119
1120 pr_devel("-->%s()\n", __func__);
1121
1122 edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
1123 if (!edit)
1124 return ERR_PTR(-ENOMEM);
1125 edit->array = array;
1126 edit->ops = ops;
1127 edit->adjust_count_by = -1;
1128
1129 switch (assoc_array_walk(array, ops, index_key, &result)) {
1130 case assoc_array_walk_found_terminal_node:
1131
1132
1133
1134 pr_devel("terminal_node\n");
1135 node = result.terminal_node.node;
1136
1137 for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
1138 ptr = node->slots[slot];
1139 if (ptr &&
1140 assoc_array_ptr_is_leaf(ptr) &&
1141 ops->compare_object(assoc_array_ptr_to_leaf(ptr),
1142 index_key))
1143 goto found_leaf;
1144 }
1145 case assoc_array_walk_tree_empty:
1146 case assoc_array_walk_found_wrong_shortcut:
1147 default:
1148 assoc_array_cancel_edit(edit);
1149 pr_devel("not found\n");
1150 return NULL;
1151 }
1152
1153found_leaf:
1154 BUG_ON(array->nr_leaves_on_tree <= 0);
1155
1156
1157
1158
1159 edit->dead_leaf = node->slots[slot];
1160 edit->set[0].ptr = &node->slots[slot];
1161 edit->set[0].to = NULL;
1162 edit->adjust_count_on = node;
1163
1164
1165
1166
1167 if (array->nr_leaves_on_tree == 1) {
1168 edit->set[1].ptr = &array->root;
1169 edit->set[1].to = NULL;
1170 edit->adjust_count_on = NULL;
1171 edit->excised_subtree = array->root;
1172 pr_devel("all gone\n");
1173 return edit;
1174 }
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186 if (node->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1) {
1187 struct assoc_array_node *parent, *grandparent;
1188 struct assoc_array_ptr *ptr;
1189
1190
1191
1192
1193
1194 has_meta = false;
1195 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
1196 ptr = node->slots[i];
1197 if (assoc_array_ptr_is_meta(ptr)) {
1198 has_meta = true;
1199 break;
1200 }
1201 }
1202
1203 pr_devel("leaves: %ld [m=%d]\n",
1204 node->nr_leaves_on_branch - 1, has_meta);
1205
1206
1207
1208
1209 parent = node;
1210 collapse_up:
1211 pr_devel("collapse subtree: %ld\n", parent->nr_leaves_on_branch);
1212
1213 ptr = parent->back_pointer;
1214 if (!ptr)
1215 goto do_collapse;
1216 if (assoc_array_ptr_is_shortcut(ptr)) {
1217 struct assoc_array_shortcut *s = assoc_array_ptr_to_shortcut(ptr);
1218 ptr = s->back_pointer;
1219 if (!ptr)
1220 goto do_collapse;
1221 }
1222
1223 grandparent = assoc_array_ptr_to_node(ptr);
1224 if (grandparent->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1) {
1225 parent = grandparent;
1226 goto collapse_up;
1227 }
1228
1229 do_collapse:
1230
1231
1232
1233
1234 if (has_meta || parent != node) {
1235 node = parent;
1236
1237
1238 new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
1239 if (!new_n0)
1240 goto enomem;
1241 edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
1242
1243 new_n0->back_pointer = node->back_pointer;
1244 new_n0->parent_slot = node->parent_slot;
1245 new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
1246 edit->adjust_count_on = new_n0;
1247
1248 collapse.node = new_n0;
1249 collapse.skip_leaf = assoc_array_ptr_to_leaf(edit->dead_leaf);
1250 collapse.slot = 0;
1251 assoc_array_subtree_iterate(assoc_array_node_to_ptr(node),
1252 node->back_pointer,
1253 assoc_array_delete_collapse_iterator,
1254 &collapse);
1255 pr_devel("collapsed %d,%lu\n", collapse.slot, new_n0->nr_leaves_on_branch);
1256 BUG_ON(collapse.slot != new_n0->nr_leaves_on_branch - 1);
1257
1258 if (!node->back_pointer) {
1259 edit->set[1].ptr = &array->root;
1260 } else if (assoc_array_ptr_is_leaf(node->back_pointer)) {
1261 BUG();
1262 } else if (assoc_array_ptr_is_node(node->back_pointer)) {
1263 struct assoc_array_node *p =
1264 assoc_array_ptr_to_node(node->back_pointer);
1265 edit->set[1].ptr = &p->slots[node->parent_slot];
1266 } else if (assoc_array_ptr_is_shortcut(node->back_pointer)) {
1267 struct assoc_array_shortcut *s =
1268 assoc_array_ptr_to_shortcut(node->back_pointer);
1269 edit->set[1].ptr = &s->next_node;
1270 }
1271 edit->set[1].to = assoc_array_node_to_ptr(new_n0);
1272 edit->excised_subtree = assoc_array_node_to_ptr(node);
1273 }
1274 }
1275
1276 return edit;
1277
1278enomem:
1279
1280 pr_devel("enomem\n");
1281 assoc_array_cancel_edit(edit);
1282 return ERR_PTR(-ENOMEM);
1283}
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303struct assoc_array_edit *assoc_array_clear(struct assoc_array *array,
1304 const struct assoc_array_ops *ops)
1305{
1306 struct assoc_array_edit *edit;
1307
1308 pr_devel("-->%s()\n", __func__);
1309
1310 if (!array->root)
1311 return NULL;
1312
1313 edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
1314 if (!edit)
1315 return ERR_PTR(-ENOMEM);
1316 edit->array = array;
1317 edit->ops = ops;
1318 edit->set[1].ptr = &array->root;
1319 edit->set[1].to = NULL;
1320 edit->excised_subtree = array->root;
1321 edit->ops_for_excised_subtree = ops;
1322 pr_devel("all gone\n");
1323 return edit;
1324}
1325
1326
1327
1328
1329static void assoc_array_rcu_cleanup(struct rcu_head *head)
1330{
1331 struct assoc_array_edit *edit =
1332 container_of(head, struct assoc_array_edit, rcu);
1333 int i;
1334
1335 pr_devel("-->%s()\n", __func__);
1336
1337 if (edit->dead_leaf)
1338 edit->ops->free_object(assoc_array_ptr_to_leaf(edit->dead_leaf));
1339 for (i = 0; i < ARRAY_SIZE(edit->excised_meta); i++)
1340 if (edit->excised_meta[i])
1341 kfree(assoc_array_ptr_to_node(edit->excised_meta[i]));
1342
1343 if (edit->excised_subtree) {
1344 BUG_ON(assoc_array_ptr_is_leaf(edit->excised_subtree));
1345 if (assoc_array_ptr_is_node(edit->excised_subtree)) {
1346 struct assoc_array_node *n =
1347 assoc_array_ptr_to_node(edit->excised_subtree);
1348 n->back_pointer = NULL;
1349 } else {
1350 struct assoc_array_shortcut *s =
1351 assoc_array_ptr_to_shortcut(edit->excised_subtree);
1352 s->back_pointer = NULL;
1353 }
1354 assoc_array_destroy_subtree(edit->excised_subtree,
1355 edit->ops_for_excised_subtree);
1356 }
1357
1358 kfree(edit);
1359}
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374void assoc_array_apply_edit(struct assoc_array_edit *edit)
1375{
1376 struct assoc_array_shortcut *shortcut;
1377 struct assoc_array_node *node;
1378 struct assoc_array_ptr *ptr;
1379 int i;
1380
1381 pr_devel("-->%s()\n", __func__);
1382
1383 smp_wmb();
1384 if (edit->leaf_p)
1385 *edit->leaf_p = edit->leaf;
1386
1387 smp_wmb();
1388 for (i = 0; i < ARRAY_SIZE(edit->set_parent_slot); i++)
1389 if (edit->set_parent_slot[i].p)
1390 *edit->set_parent_slot[i].p = edit->set_parent_slot[i].to;
1391
1392 smp_wmb();
1393 for (i = 0; i < ARRAY_SIZE(edit->set_backpointers); i++)
1394 if (edit->set_backpointers[i])
1395 *edit->set_backpointers[i] = edit->set_backpointers_to;
1396
1397 smp_wmb();
1398 for (i = 0; i < ARRAY_SIZE(edit->set); i++)
1399 if (edit->set[i].ptr)
1400 *edit->set[i].ptr = edit->set[i].to;
1401
1402 if (edit->array->root == NULL) {
1403 edit->array->nr_leaves_on_tree = 0;
1404 } else if (edit->adjust_count_on) {
1405 node = edit->adjust_count_on;
1406 for (;;) {
1407 node->nr_leaves_on_branch += edit->adjust_count_by;
1408
1409 ptr = node->back_pointer;
1410 if (!ptr)
1411 break;
1412 if (assoc_array_ptr_is_shortcut(ptr)) {
1413 shortcut = assoc_array_ptr_to_shortcut(ptr);
1414 ptr = shortcut->back_pointer;
1415 if (!ptr)
1416 break;
1417 }
1418 BUG_ON(!assoc_array_ptr_is_node(ptr));
1419 node = assoc_array_ptr_to_node(ptr);
1420 }
1421
1422 edit->array->nr_leaves_on_tree += edit->adjust_count_by;
1423 }
1424
1425 call_rcu(&edit->rcu, assoc_array_rcu_cleanup);
1426}
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438void assoc_array_cancel_edit(struct assoc_array_edit *edit)
1439{
1440 struct assoc_array_ptr *ptr;
1441 int i;
1442
1443 pr_devel("-->%s()\n", __func__);
1444
1445
1446 for (i = 0; i < ARRAY_SIZE(edit->new_meta); i++) {
1447 ptr = edit->new_meta[i];
1448 if (ptr) {
1449 if (assoc_array_ptr_is_node(ptr))
1450 kfree(assoc_array_ptr_to_node(ptr));
1451 else
1452 kfree(assoc_array_ptr_to_shortcut(ptr));
1453 }
1454 }
1455 kfree(edit);
1456}
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482int assoc_array_gc(struct assoc_array *array,
1483 const struct assoc_array_ops *ops,
1484 bool (*iterator)(void *object, void *iterator_data),
1485 void *iterator_data)
1486{
1487 struct assoc_array_shortcut *shortcut, *new_s;
1488 struct assoc_array_node *node, *new_n;
1489 struct assoc_array_edit *edit;
1490 struct assoc_array_ptr *cursor, *ptr;
1491 struct assoc_array_ptr *new_root, *new_parent, **new_ptr_pp;
1492 unsigned long nr_leaves_on_tree;
1493 int keylen, slot, nr_free, next_slot, i;
1494
1495 pr_devel("-->%s()\n", __func__);
1496
1497 if (!array->root)
1498 return 0;
1499
1500 edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
1501 if (!edit)
1502 return -ENOMEM;
1503 edit->array = array;
1504 edit->ops = ops;
1505 edit->ops_for_excised_subtree = ops;
1506 edit->set[0].ptr = &array->root;
1507 edit->excised_subtree = array->root;
1508
1509 new_root = new_parent = NULL;
1510 new_ptr_pp = &new_root;
1511 cursor = array->root;
1512
1513descend:
1514
1515
1516
1517 if (assoc_array_ptr_is_shortcut(cursor)) {
1518 shortcut = assoc_array_ptr_to_shortcut(cursor);
1519 keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
1520 keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
1521 new_s = kmalloc(sizeof(struct assoc_array_shortcut) +
1522 keylen * sizeof(unsigned long), GFP_KERNEL);
1523 if (!new_s)
1524 goto enomem;
1525 pr_devel("dup shortcut %p -> %p\n", shortcut, new_s);
1526 memcpy(new_s, shortcut, (sizeof(struct assoc_array_shortcut) +
1527 keylen * sizeof(unsigned long)));
1528 new_s->back_pointer = new_parent;
1529 new_s->parent_slot = shortcut->parent_slot;
1530 *new_ptr_pp = new_parent = assoc_array_shortcut_to_ptr(new_s);
1531 new_ptr_pp = &new_s->next_node;
1532 cursor = shortcut->next_node;
1533 }
1534
1535
1536 node = assoc_array_ptr_to_node(cursor);
1537 new_n = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
1538 if (!new_n)
1539 goto enomem;
1540 pr_devel("dup node %p -> %p\n", node, new_n);
1541 new_n->back_pointer = new_parent;
1542 new_n->parent_slot = node->parent_slot;
1543 *new_ptr_pp = new_parent = assoc_array_node_to_ptr(new_n);
1544 new_ptr_pp = NULL;
1545 slot = 0;
1546
1547continue_node:
1548
1549 for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
1550 ptr = node->slots[slot];
1551 if (!ptr)
1552 continue;
1553
1554 if (assoc_array_ptr_is_leaf(ptr)) {
1555 if (iterator(assoc_array_ptr_to_leaf(ptr),
1556 iterator_data))
1557
1558
1559
1560 new_n->slots[slot] = ptr;
1561 continue;
1562 }
1563
1564 new_ptr_pp = &new_n->slots[slot];
1565 cursor = ptr;
1566 goto descend;
1567 }
1568
1569 pr_devel("-- compress node %p --\n", new_n);
1570
1571
1572
1573
1574 new_n->nr_leaves_on_branch = 0;
1575 nr_free = 0;
1576 for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
1577 ptr = new_n->slots[slot];
1578 if (!ptr)
1579 nr_free++;
1580 else if (assoc_array_ptr_is_leaf(ptr))
1581 new_n->nr_leaves_on_branch++;
1582 }
1583 pr_devel("free=%d, leaves=%lu\n", nr_free, new_n->nr_leaves_on_branch);
1584
1585
1586 next_slot = 0;
1587 for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
1588 struct assoc_array_shortcut *s;
1589 struct assoc_array_node *child;
1590
1591 ptr = new_n->slots[slot];
1592 if (!ptr || assoc_array_ptr_is_leaf(ptr))
1593 continue;
1594
1595 s = NULL;
1596 if (assoc_array_ptr_is_shortcut(ptr)) {
1597 s = assoc_array_ptr_to_shortcut(ptr);
1598 ptr = s->next_node;
1599 }
1600
1601 child = assoc_array_ptr_to_node(ptr);
1602 new_n->nr_leaves_on_branch += child->nr_leaves_on_branch;
1603
1604 if (child->nr_leaves_on_branch <= nr_free + 1) {
1605
1606 pr_devel("[%d] fold node %lu/%d [nx %d]\n",
1607 slot, child->nr_leaves_on_branch, nr_free + 1,
1608 next_slot);
1609
1610
1611
1612
1613 BUG_ON(s);
1614
1615 new_n->slots[slot] = NULL;
1616 nr_free++;
1617 if (slot < next_slot)
1618 next_slot = slot;
1619 for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
1620 struct assoc_array_ptr *p = child->slots[i];
1621 if (!p)
1622 continue;
1623 BUG_ON(assoc_array_ptr_is_meta(p));
1624 while (new_n->slots[next_slot])
1625 next_slot++;
1626 BUG_ON(next_slot >= ASSOC_ARRAY_FAN_OUT);
1627 new_n->slots[next_slot++] = p;
1628 nr_free--;
1629 }
1630 kfree(child);
1631 } else {
1632 pr_devel("[%d] retain node %lu/%d [nx %d]\n",
1633 slot, child->nr_leaves_on_branch, nr_free + 1,
1634 next_slot);
1635 }
1636 }
1637
1638 pr_devel("after: %lu\n", new_n->nr_leaves_on_branch);
1639
1640 nr_leaves_on_tree = new_n->nr_leaves_on_branch;
1641
1642
1643 if (nr_free == ASSOC_ARRAY_FAN_OUT - 1) {
1644 for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++)
1645 if ((ptr = new_n->slots[slot]))
1646 break;
1647
1648 if (assoc_array_ptr_is_meta(ptr) &&
1649 assoc_array_ptr_is_shortcut(ptr)) {
1650 pr_devel("excise node %p with 1 shortcut\n", new_n);
1651 new_s = assoc_array_ptr_to_shortcut(ptr);
1652 new_parent = new_n->back_pointer;
1653 slot = new_n->parent_slot;
1654 kfree(new_n);
1655 if (!new_parent) {
1656 new_s->back_pointer = NULL;
1657 new_s->parent_slot = 0;
1658 new_root = ptr;
1659 goto gc_complete;
1660 }
1661
1662 if (assoc_array_ptr_is_shortcut(new_parent)) {
1663
1664 struct assoc_array_shortcut *s =
1665 assoc_array_ptr_to_shortcut(new_parent);
1666
1667 pr_devel("excise preceding shortcut\n");
1668
1669 new_parent = new_s->back_pointer = s->back_pointer;
1670 slot = new_s->parent_slot = s->parent_slot;
1671 kfree(s);
1672 if (!new_parent) {
1673 new_s->back_pointer = NULL;
1674 new_s->parent_slot = 0;
1675 new_root = ptr;
1676 goto gc_complete;
1677 }
1678 }
1679
1680 new_s->back_pointer = new_parent;
1681 new_s->parent_slot = slot;
1682 new_n = assoc_array_ptr_to_node(new_parent);
1683 new_n->slots[slot] = ptr;
1684 goto ascend_old_tree;
1685 }
1686 }
1687
1688
1689
1690
1691 ptr = new_n->back_pointer;
1692 if (!ptr)
1693 goto gc_complete;
1694
1695 if (assoc_array_ptr_is_shortcut(ptr)) {
1696 new_s = assoc_array_ptr_to_shortcut(ptr);
1697 new_parent = new_s->back_pointer;
1698 slot = new_s->parent_slot;
1699
1700 if (new_n->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT) {
1701 struct assoc_array_node *n;
1702
1703 pr_devel("excise shortcut\n");
1704 new_n->back_pointer = new_parent;
1705 new_n->parent_slot = slot;
1706 kfree(new_s);
1707 if (!new_parent) {
1708 new_root = assoc_array_node_to_ptr(new_n);
1709 goto gc_complete;
1710 }
1711
1712 n = assoc_array_ptr_to_node(new_parent);
1713 n->slots[slot] = assoc_array_node_to_ptr(new_n);
1714 }
1715 } else {
1716 new_parent = ptr;
1717 }
1718 new_n = assoc_array_ptr_to_node(new_parent);
1719
1720ascend_old_tree:
1721 ptr = node->back_pointer;
1722 if (assoc_array_ptr_is_shortcut(ptr)) {
1723 shortcut = assoc_array_ptr_to_shortcut(ptr);
1724 slot = shortcut->parent_slot;
1725 cursor = shortcut->back_pointer;
1726 if (!cursor)
1727 goto gc_complete;
1728 } else {
1729 slot = node->parent_slot;
1730 cursor = ptr;
1731 }
1732 BUG_ON(!cursor);
1733 node = assoc_array_ptr_to_node(cursor);
1734 slot++;
1735 goto continue_node;
1736
1737gc_complete:
1738 edit->set[0].to = new_root;
1739 assoc_array_apply_edit(edit);
1740 array->nr_leaves_on_tree = nr_leaves_on_tree;
1741 return 0;
1742
1743enomem:
1744 pr_devel("enomem\n");
1745 assoc_array_destroy_subtree(new_root, edit->ops);
1746 kfree(edit);
1747 return -ENOMEM;
1748}
1749