1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/rbtree_augmented.h>
25#include <linux/export.h>
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71static inline void rb_set_black(struct rb_node *rb)
72{
73 rb->__rb_parent_color |= RB_BLACK;
74}
75
76static inline struct rb_node *rb_red_parent(struct rb_node *red)
77{
78 return (struct rb_node *)red->__rb_parent_color;
79}
80
81
82
83
84
85
86static inline void
87__rb_rotate_set_parents(struct rb_node *old, struct rb_node *new,
88 struct rb_root *root, int color)
89{
90 struct rb_node *parent = rb_parent(old);
91 new->__rb_parent_color = old->__rb_parent_color;
92 rb_set_parent_color(old, new, color);
93 __rb_change_child(old, new, parent, root);
94}
95
96static __always_inline void
97__rb_insert(struct rb_node *node, struct rb_root *root,
98 bool newleft, struct rb_node **leftmost,
99 void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
100{
101 struct rb_node *parent = rb_red_parent(node), *gparent, *tmp;
102
103 if (newleft)
104 *leftmost = node;
105
106 while (true) {
107
108
109
110 if (unlikely(!parent)) {
111
112
113
114
115
116 rb_set_parent_color(node, NULL, RB_BLACK);
117 break;
118 }
119
120
121
122
123
124
125
126 if(rb_is_black(parent))
127 break;
128
129 gparent = rb_red_parent(parent);
130
131 tmp = gparent->rb_right;
132 if (parent != tmp) {
133 if (tmp && rb_is_red(tmp)) {
134
135
136
137
138
139
140
141
142
143
144
145
146
147 rb_set_parent_color(tmp, gparent, RB_BLACK);
148 rb_set_parent_color(parent, gparent, RB_BLACK);
149 node = gparent;
150 parent = rb_parent(node);
151 rb_set_parent_color(node, parent, RB_RED);
152 continue;
153 }
154
155 tmp = parent->rb_right;
156 if (node == tmp) {
157
158
159
160
161
162
163
164
165
166
167
168
169
170 tmp = node->rb_left;
171 WRITE_ONCE(parent->rb_right, tmp);
172 WRITE_ONCE(node->rb_left, parent);
173 if (tmp)
174 rb_set_parent_color(tmp, parent,
175 RB_BLACK);
176 rb_set_parent_color(parent, node, RB_RED);
177 augment_rotate(parent, node);
178 parent = node;
179 tmp = node->rb_right;
180 }
181
182
183
184
185
186
187
188
189
190
191
192 WRITE_ONCE(gparent->rb_left, tmp);
193 WRITE_ONCE(parent->rb_right, gparent);
194 if (tmp)
195 rb_set_parent_color(tmp, gparent, RB_BLACK);
196 __rb_rotate_set_parents(gparent, parent, root, RB_RED);
197 augment_rotate(gparent, parent);
198 break;
199 } else {
200 tmp = gparent->rb_left;
201 if (tmp && rb_is_red(tmp)) {
202
203 rb_set_parent_color(tmp, gparent, RB_BLACK);
204 rb_set_parent_color(parent, gparent, RB_BLACK);
205 node = gparent;
206 parent = rb_parent(node);
207 rb_set_parent_color(node, parent, RB_RED);
208 continue;
209 }
210
211 tmp = parent->rb_left;
212 if (node == tmp) {
213
214 tmp = node->rb_right;
215 WRITE_ONCE(parent->rb_left, tmp);
216 WRITE_ONCE(node->rb_right, parent);
217 if (tmp)
218 rb_set_parent_color(tmp, parent,
219 RB_BLACK);
220 rb_set_parent_color(parent, node, RB_RED);
221 augment_rotate(parent, node);
222 parent = node;
223 tmp = node->rb_left;
224 }
225
226
227 WRITE_ONCE(gparent->rb_right, tmp);
228 WRITE_ONCE(parent->rb_left, gparent);
229 if (tmp)
230 rb_set_parent_color(tmp, gparent, RB_BLACK);
231 __rb_rotate_set_parents(gparent, parent, root, RB_RED);
232 augment_rotate(gparent, parent);
233 break;
234 }
235 }
236}
237
238
239
240
241
242static __always_inline void
243____rb_erase_color(struct rb_node *parent, struct rb_root *root,
244 void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
245{
246 struct rb_node *node = NULL, *sibling, *tmp1, *tmp2;
247
248 while (true) {
249
250
251
252
253
254
255
256 sibling = parent->rb_right;
257 if (node != sibling) {
258 if (rb_is_red(sibling)) {
259
260
261
262
263
264
265
266
267
268 tmp1 = sibling->rb_left;
269 WRITE_ONCE(parent->rb_right, tmp1);
270 WRITE_ONCE(sibling->rb_left, parent);
271 rb_set_parent_color(tmp1, parent, RB_BLACK);
272 __rb_rotate_set_parents(parent, sibling, root,
273 RB_RED);
274 augment_rotate(parent, sibling);
275 sibling = tmp1;
276 }
277 tmp1 = sibling->rb_right;
278 if (!tmp1 || rb_is_black(tmp1)) {
279 tmp2 = sibling->rb_left;
280 if (!tmp2 || rb_is_black(tmp2)) {
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296 rb_set_parent_color(sibling, parent,
297 RB_RED);
298 if (rb_is_red(parent))
299 rb_set_black(parent);
300 else {
301 node = parent;
302 parent = rb_parent(node);
303 if (parent)
304 continue;
305 }
306 break;
307 }
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335 tmp1 = tmp2->rb_right;
336 WRITE_ONCE(sibling->rb_left, tmp1);
337 WRITE_ONCE(tmp2->rb_right, sibling);
338 WRITE_ONCE(parent->rb_right, tmp2);
339 if (tmp1)
340 rb_set_parent_color(tmp1, sibling,
341 RB_BLACK);
342 augment_rotate(sibling, tmp2);
343 tmp1 = sibling;
344 sibling = tmp2;
345 }
346
347
348
349
350
351
352
353
354
355
356
357
358 tmp2 = sibling->rb_left;
359 WRITE_ONCE(parent->rb_right, tmp2);
360 WRITE_ONCE(sibling->rb_left, parent);
361 rb_set_parent_color(tmp1, sibling, RB_BLACK);
362 if (tmp2)
363 rb_set_parent(tmp2, parent);
364 __rb_rotate_set_parents(parent, sibling, root,
365 RB_BLACK);
366 augment_rotate(parent, sibling);
367 break;
368 } else {
369 sibling = parent->rb_left;
370 if (rb_is_red(sibling)) {
371
372 tmp1 = sibling->rb_right;
373 WRITE_ONCE(parent->rb_left, tmp1);
374 WRITE_ONCE(sibling->rb_right, parent);
375 rb_set_parent_color(tmp1, parent, RB_BLACK);
376 __rb_rotate_set_parents(parent, sibling, root,
377 RB_RED);
378 augment_rotate(parent, sibling);
379 sibling = tmp1;
380 }
381 tmp1 = sibling->rb_left;
382 if (!tmp1 || rb_is_black(tmp1)) {
383 tmp2 = sibling->rb_right;
384 if (!tmp2 || rb_is_black(tmp2)) {
385
386 rb_set_parent_color(sibling, parent,
387 RB_RED);
388 if (rb_is_red(parent))
389 rb_set_black(parent);
390 else {
391 node = parent;
392 parent = rb_parent(node);
393 if (parent)
394 continue;
395 }
396 break;
397 }
398
399 tmp1 = tmp2->rb_left;
400 WRITE_ONCE(sibling->rb_right, tmp1);
401 WRITE_ONCE(tmp2->rb_left, sibling);
402 WRITE_ONCE(parent->rb_left, tmp2);
403 if (tmp1)
404 rb_set_parent_color(tmp1, sibling,
405 RB_BLACK);
406 augment_rotate(sibling, tmp2);
407 tmp1 = sibling;
408 sibling = tmp2;
409 }
410
411 tmp2 = sibling->rb_right;
412 WRITE_ONCE(parent->rb_left, tmp2);
413 WRITE_ONCE(sibling->rb_right, parent);
414 rb_set_parent_color(tmp1, sibling, RB_BLACK);
415 if (tmp2)
416 rb_set_parent(tmp2, parent);
417 __rb_rotate_set_parents(parent, sibling, root,
418 RB_BLACK);
419 augment_rotate(parent, sibling);
420 break;
421 }
422 }
423}
424
425
426void __rb_erase_color(struct rb_node *parent, struct rb_root *root,
427 void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
428{
429 ____rb_erase_color(parent, root, augment_rotate);
430}
431EXPORT_SYMBOL(__rb_erase_color);
432
433
434
435
436
437
438
439
440static inline void dummy_propagate(struct rb_node *node, struct rb_node *stop) {}
441static inline void dummy_copy(struct rb_node *old, struct rb_node *new) {}
442static inline void dummy_rotate(struct rb_node *old, struct rb_node *new) {}
443
444static const struct rb_augment_callbacks dummy_callbacks = {
445 .propagate = dummy_propagate,
446 .copy = dummy_copy,
447 .rotate = dummy_rotate
448};
449
450void rb_insert_color(struct rb_node *node, struct rb_root *root)
451{
452 __rb_insert(node, root, false, NULL, dummy_rotate);
453}
454EXPORT_SYMBOL(rb_insert_color);
455
456void rb_erase(struct rb_node *node, struct rb_root *root)
457{
458 struct rb_node *rebalance;
459 rebalance = __rb_erase_augmented(node, root,
460 NULL, &dummy_callbacks);
461 if (rebalance)
462 ____rb_erase_color(rebalance, root, dummy_rotate);
463}
464EXPORT_SYMBOL(rb_erase);
465
466void rb_insert_color_cached(struct rb_node *node,
467 struct rb_root_cached *root, bool leftmost)
468{
469 __rb_insert(node, &root->rb_root, leftmost,
470 &root->rb_leftmost, dummy_rotate);
471}
472EXPORT_SYMBOL(rb_insert_color_cached);
473
474void rb_erase_cached(struct rb_node *node, struct rb_root_cached *root)
475{
476 struct rb_node *rebalance;
477 rebalance = __rb_erase_augmented(node, &root->rb_root,
478 &root->rb_leftmost, &dummy_callbacks);
479 if (rebalance)
480 ____rb_erase_color(rebalance, &root->rb_root, dummy_rotate);
481}
482EXPORT_SYMBOL(rb_erase_cached);
483
484
485
486
487
488
489
490
491void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
492 bool newleft, struct rb_node **leftmost,
493 void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
494{
495 __rb_insert(node, root, newleft, leftmost, augment_rotate);
496}
497EXPORT_SYMBOL(__rb_insert_augmented);
498
499
500
501
502struct rb_node *rb_first(const struct rb_root *root)
503{
504 struct rb_node *n;
505
506 n = root->rb_node;
507 if (!n)
508 return NULL;
509 while (n->rb_left)
510 n = n->rb_left;
511 return n;
512}
513EXPORT_SYMBOL(rb_first);
514
515struct rb_node *rb_last(const struct rb_root *root)
516{
517 struct rb_node *n;
518
519 n = root->rb_node;
520 if (!n)
521 return NULL;
522 while (n->rb_right)
523 n = n->rb_right;
524 return n;
525}
526EXPORT_SYMBOL(rb_last);
527
528struct rb_node *rb_next(const struct rb_node *node)
529{
530 struct rb_node *parent;
531
532 if (RB_EMPTY_NODE(node))
533 return NULL;
534
535
536
537
538
539 if (node->rb_right) {
540 node = node->rb_right;
541 while (node->rb_left)
542 node=node->rb_left;
543 return (struct rb_node *)node;
544 }
545
546
547
548
549
550
551
552
553 while ((parent = rb_parent(node)) && node == parent->rb_right)
554 node = parent;
555
556 return parent;
557}
558EXPORT_SYMBOL(rb_next);
559
560struct rb_node *rb_prev(const struct rb_node *node)
561{
562 struct rb_node *parent;
563
564 if (RB_EMPTY_NODE(node))
565 return NULL;
566
567
568
569
570
571 if (node->rb_left) {
572 node = node->rb_left;
573 while (node->rb_right)
574 node=node->rb_right;
575 return (struct rb_node *)node;
576 }
577
578
579
580
581
582 while ((parent = rb_parent(node)) && node == parent->rb_left)
583 node = parent;
584
585 return parent;
586}
587EXPORT_SYMBOL(rb_prev);
588
589void rb_replace_node(struct rb_node *victim, struct rb_node *new,
590 struct rb_root *root)
591{
592 struct rb_node *parent = rb_parent(victim);
593
594
595 *new = *victim;
596
597
598 if (victim->rb_left)
599 rb_set_parent(victim->rb_left, new);
600 if (victim->rb_right)
601 rb_set_parent(victim->rb_right, new);
602 __rb_change_child(victim, new, parent, root);
603}
604EXPORT_SYMBOL(rb_replace_node);
605
606void rb_replace_node_cached(struct rb_node *victim, struct rb_node *new,
607 struct rb_root_cached *root)
608{
609 rb_replace_node(victim, new, &root->rb_root);
610
611 if (root->rb_leftmost == victim)
612 root->rb_leftmost = new;
613}
614EXPORT_SYMBOL(rb_replace_node_cached);
615
616void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new,
617 struct rb_root *root)
618{
619 struct rb_node *parent = rb_parent(victim);
620
621
622 *new = *victim;
623
624
625 if (victim->rb_left)
626 rb_set_parent(victim->rb_left, new);
627 if (victim->rb_right)
628 rb_set_parent(victim->rb_right, new);
629
630
631
632
633
634 __rb_change_child_rcu(victim, new, parent, root);
635}
636EXPORT_SYMBOL(rb_replace_node_rcu);
637
638static struct rb_node *rb_left_deepest_node(const struct rb_node *node)
639{
640 for (;;) {
641 if (node->rb_left)
642 node = node->rb_left;
643 else if (node->rb_right)
644 node = node->rb_right;
645 else
646 return (struct rb_node *)node;
647 }
648}
649
650struct rb_node *rb_next_postorder(const struct rb_node *node)
651{
652 const struct rb_node *parent;
653 if (!node)
654 return NULL;
655 parent = rb_parent(node);
656
657
658 if (parent && node == parent->rb_left && parent->rb_right) {
659
660
661 return rb_left_deepest_node(parent->rb_right);
662 } else
663
664
665 return (struct rb_node *)parent;
666}
667EXPORT_SYMBOL(rb_next_postorder);
668
669struct rb_node *rb_first_postorder(const struct rb_root *root)
670{
671 if (!root->rb_node)
672 return NULL;
673
674 return rb_left_deepest_node(root->rb_node);
675}
676EXPORT_SYMBOL(rb_first_postorder);
677