1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42#ifndef __LIBCFS_HASH_H__
43#define __LIBCFS_HASH_H__
44
45#include <linux/hash.h>
46
47
48
49
50
51
52
53
54
55
56
57
58#define CFS_GOLDEN_RATIO_PRIME_32 0x9e370001UL
59
60#define CFS_GOLDEN_RATIO_PRIME_64 0x9e37fffffffc0001ULL
61
62
63#define CFS_HASH_DEBUG_NONE 0
64
65
66#define CFS_HASH_DEBUG_1 1
67
68#define CFS_HASH_DEBUG_2 2
69
70#define CFS_HASH_DEBUG_LEVEL CFS_HASH_DEBUG_NONE
71
72struct cfs_hash_ops;
73struct cfs_hash_lock_ops;
74struct cfs_hash_hlist_ops;
75
76union cfs_hash_lock {
77 rwlock_t rw;
78 spinlock_t spin;
79};
80
81
82
83
84
85
86
87
88
89
90
91
92struct cfs_hash_bucket {
93 union cfs_hash_lock hsb_lock;
94 __u32 hsb_count;
95 __u32 hsb_version;
96 unsigned int hsb_index;
97 int hsb_depmax;
98 long hsb_head[0];
99};
100
101
102
103
104struct cfs_hash_bd {
105
106 struct cfs_hash_bucket *bd_bucket;
107
108 unsigned int bd_offset;
109};
110
111#define CFS_HASH_NAME_LEN 16
112#define CFS_HASH_BIGNAME_LEN 64
113
114#define CFS_HASH_BKT_BITS 3
115#define CFS_HASH_BITS_MAX 30
116#define CFS_HASH_BITS_MIN CFS_HASH_BKT_BITS
117
118
119
120
121enum cfs_hash_tag {
122
123
124
125
126
127
128
129
130 CFS_HASH_NO_LOCK = 1 << 0,
131
132 CFS_HASH_NO_BKTLOCK = 1 << 1,
133
134 CFS_HASH_RW_BKTLOCK = 1 << 2,
135
136 CFS_HASH_SPIN_BKTLOCK = 1 << 3,
137
138 CFS_HASH_ADD_TAIL = 1 << 4,
139
140 CFS_HASH_NO_ITEMREF = 1 << 5,
141
142 CFS_HASH_BIGNAME = 1 << 6,
143
144 CFS_HASH_COUNTER = 1 << 7,
145
146 CFS_HASH_REHASH_KEY = 1 << 8,
147
148 CFS_HASH_REHASH = 1 << 9,
149
150 CFS_HASH_SHRINK = 1 << 10,
151
152 CFS_HASH_ASSERT_EMPTY = 1 << 11,
153
154 CFS_HASH_DEPTH = 1 << 12,
155
156
157
158
159 CFS_HASH_NBLK_CHANGE = 1 << 13,
160
161
162};
163
164
165#define CFS_HASH_DEFAULT (CFS_HASH_RW_BKTLOCK | \
166 CFS_HASH_COUNTER | CFS_HASH_REHASH)
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207struct cfs_hash {
208
209
210 union cfs_hash_lock hs_lock;
211
212 struct cfs_hash_ops *hs_ops;
213
214 struct cfs_hash_lock_ops *hs_lops;
215
216 struct cfs_hash_hlist_ops *hs_hops;
217
218 struct cfs_hash_bucket **hs_buckets;
219
220 atomic_t hs_count;
221
222 __u16 hs_flags;
223
224 __u16 hs_extra_bytes;
225
226 __u8 hs_iterating;
227
228 __u8 hs_exiting;
229
230 __u8 hs_cur_bits;
231
232 __u8 hs_min_bits;
233
234 __u8 hs_max_bits;
235
236 __u8 hs_rehash_bits;
237
238 __u8 hs_bkt_bits;
239
240 __u16 hs_min_theta;
241
242 __u16 hs_max_theta;
243
244 __u32 hs_rehash_count;
245
246 __u32 hs_iterators;
247
248 cfs_workitem_t hs_rehash_wi;
249
250 atomic_t hs_refcount;
251
252 struct cfs_hash_bucket **hs_rehash_buckets;
253#if CFS_HASH_DEBUG_LEVEL >= CFS_HASH_DEBUG_1
254
255 spinlock_t hs_dep_lock;
256
257 unsigned int hs_dep_max;
258
259 unsigned int hs_dep_bkt;
260
261 unsigned int hs_dep_off;
262
263 unsigned int hs_dep_bits;
264
265 cfs_workitem_t hs_dep_wi;
266#endif
267
268 char hs_name[0];
269};
270
271struct cfs_hash_lock_ops {
272
273 void (*hs_lock)(union cfs_hash_lock *lock, int exclusive);
274
275 void (*hs_unlock)(union cfs_hash_lock *lock, int exclusive);
276
277 void (*hs_bkt_lock)(union cfs_hash_lock *lock, int exclusive);
278
279 void (*hs_bkt_unlock)(union cfs_hash_lock *lock, int exclusive);
280};
281
282struct cfs_hash_hlist_ops {
283
284 struct hlist_head *(*hop_hhead)(struct cfs_hash *hs,
285 struct cfs_hash_bd *bd);
286
287 int (*hop_hhead_size)(struct cfs_hash *hs);
288
289 int (*hop_hnode_add)(struct cfs_hash *hs, struct cfs_hash_bd *bd,
290 struct hlist_node *hnode);
291
292 int (*hop_hnode_del)(struct cfs_hash *hs, struct cfs_hash_bd *bd,
293 struct hlist_node *hnode);
294};
295
296struct cfs_hash_ops {
297
298 unsigned (*hs_hash)(struct cfs_hash *hs, const void *key,
299 unsigned mask);
300
301 void * (*hs_key)(struct hlist_node *hnode);
302
303 void (*hs_keycpy)(struct hlist_node *hnode, void *key);
304
305
306
307
308 int (*hs_keycmp)(const void *key, struct hlist_node *hnode);
309
310 void * (*hs_object)(struct hlist_node *hnode);
311
312 void (*hs_get)(struct cfs_hash *hs, struct hlist_node *hnode);
313
314 void (*hs_put)(struct cfs_hash *hs, struct hlist_node *hnode);
315
316 void (*hs_put_locked)(struct cfs_hash *hs,
317 struct hlist_node *hnode);
318
319 void (*hs_exit)(struct cfs_hash *hs, struct hlist_node *hnode);
320};
321
322
323#define CFS_HASH_NBKT(hs) \
324 (1U << ((hs)->hs_cur_bits - (hs)->hs_bkt_bits))
325
326
327#define CFS_HASH_RH_NBKT(hs) \
328 (1U << ((hs)->hs_rehash_bits - (hs)->hs_bkt_bits))
329
330
331#define CFS_HASH_BKT_NHLIST(hs) (1U << (hs)->hs_bkt_bits)
332
333
334#define CFS_HASH_NHLIST(hs) (1U << (hs)->hs_cur_bits)
335
336
337#define CFS_HASH_RH_NHLIST(hs) (1U << (hs)->hs_rehash_bits)
338
339static inline int
340cfs_hash_with_no_lock(struct cfs_hash *hs)
341{
342
343 return (hs->hs_flags & CFS_HASH_NO_LOCK) != 0;
344}
345
346static inline int
347cfs_hash_with_no_bktlock(struct cfs_hash *hs)
348{
349
350 return (hs->hs_flags & CFS_HASH_NO_BKTLOCK) != 0;
351}
352
353static inline int
354cfs_hash_with_rw_bktlock(struct cfs_hash *hs)
355{
356
357 return (hs->hs_flags & CFS_HASH_RW_BKTLOCK) != 0;
358}
359
360static inline int
361cfs_hash_with_spin_bktlock(struct cfs_hash *hs)
362{
363
364 return (hs->hs_flags & CFS_HASH_SPIN_BKTLOCK) != 0;
365}
366
367static inline int
368cfs_hash_with_add_tail(struct cfs_hash *hs)
369{
370 return (hs->hs_flags & CFS_HASH_ADD_TAIL) != 0;
371}
372
373static inline int
374cfs_hash_with_no_itemref(struct cfs_hash *hs)
375{
376
377
378
379 return (hs->hs_flags & CFS_HASH_NO_ITEMREF) != 0;
380}
381
382static inline int
383cfs_hash_with_bigname(struct cfs_hash *hs)
384{
385 return (hs->hs_flags & CFS_HASH_BIGNAME) != 0;
386}
387
388static inline int
389cfs_hash_with_counter(struct cfs_hash *hs)
390{
391 return (hs->hs_flags & CFS_HASH_COUNTER) != 0;
392}
393
394static inline int
395cfs_hash_with_rehash(struct cfs_hash *hs)
396{
397 return (hs->hs_flags & CFS_HASH_REHASH) != 0;
398}
399
400static inline int
401cfs_hash_with_rehash_key(struct cfs_hash *hs)
402{
403 return (hs->hs_flags & CFS_HASH_REHASH_KEY) != 0;
404}
405
406static inline int
407cfs_hash_with_shrink(struct cfs_hash *hs)
408{
409 return (hs->hs_flags & CFS_HASH_SHRINK) != 0;
410}
411
412static inline int
413cfs_hash_with_assert_empty(struct cfs_hash *hs)
414{
415 return (hs->hs_flags & CFS_HASH_ASSERT_EMPTY) != 0;
416}
417
418static inline int
419cfs_hash_with_depth(struct cfs_hash *hs)
420{
421 return (hs->hs_flags & CFS_HASH_DEPTH) != 0;
422}
423
424static inline int
425cfs_hash_with_nblk_change(struct cfs_hash *hs)
426{
427 return (hs->hs_flags & CFS_HASH_NBLK_CHANGE) != 0;
428}
429
430static inline int
431cfs_hash_is_exiting(struct cfs_hash *hs)
432{
433
434 return hs->hs_exiting;
435}
436
437static inline int
438cfs_hash_is_rehashing(struct cfs_hash *hs)
439{
440
441 return hs->hs_rehash_bits != 0;
442}
443
444static inline int
445cfs_hash_is_iterating(struct cfs_hash *hs)
446{
447
448 return hs->hs_iterating || hs->hs_iterators != 0;
449}
450
451static inline int
452cfs_hash_bkt_size(struct cfs_hash *hs)
453{
454 return offsetof(struct cfs_hash_bucket, hsb_head[0]) +
455 hs->hs_hops->hop_hhead_size(hs) * CFS_HASH_BKT_NHLIST(hs) +
456 hs->hs_extra_bytes;
457}
458
459static inline unsigned
460cfs_hash_id(struct cfs_hash *hs, const void *key, unsigned mask)
461{
462 return hs->hs_ops->hs_hash(hs, key, mask);
463}
464
465static inline void *
466cfs_hash_key(struct cfs_hash *hs, struct hlist_node *hnode)
467{
468 return hs->hs_ops->hs_key(hnode);
469}
470
471static inline void
472cfs_hash_keycpy(struct cfs_hash *hs, struct hlist_node *hnode, void *key)
473{
474 if (hs->hs_ops->hs_keycpy)
475 hs->hs_ops->hs_keycpy(hnode, key);
476}
477
478
479
480
481static inline int
482cfs_hash_keycmp(struct cfs_hash *hs, const void *key, struct hlist_node *hnode)
483{
484 return hs->hs_ops->hs_keycmp(key, hnode);
485}
486
487static inline void *
488cfs_hash_object(struct cfs_hash *hs, struct hlist_node *hnode)
489{
490 return hs->hs_ops->hs_object(hnode);
491}
492
493static inline void
494cfs_hash_get(struct cfs_hash *hs, struct hlist_node *hnode)
495{
496 return hs->hs_ops->hs_get(hs, hnode);
497}
498
499static inline void
500cfs_hash_put_locked(struct cfs_hash *hs, struct hlist_node *hnode)
501{
502 return hs->hs_ops->hs_put_locked(hs, hnode);
503}
504
505static inline void
506cfs_hash_put(struct cfs_hash *hs, struct hlist_node *hnode)
507{
508 return hs->hs_ops->hs_put(hs, hnode);
509}
510
511static inline void
512cfs_hash_exit(struct cfs_hash *hs, struct hlist_node *hnode)
513{
514 if (hs->hs_ops->hs_exit)
515 hs->hs_ops->hs_exit(hs, hnode);
516}
517
518static inline void cfs_hash_lock(struct cfs_hash *hs, int excl)
519{
520 hs->hs_lops->hs_lock(&hs->hs_lock, excl);
521}
522
523static inline void cfs_hash_unlock(struct cfs_hash *hs, int excl)
524{
525 hs->hs_lops->hs_unlock(&hs->hs_lock, excl);
526}
527
528static inline int cfs_hash_dec_and_lock(struct cfs_hash *hs,
529 atomic_t *condition)
530{
531 LASSERT(cfs_hash_with_no_bktlock(hs));
532 return atomic_dec_and_lock(condition, &hs->hs_lock.spin);
533}
534
535static inline void cfs_hash_bd_lock(struct cfs_hash *hs,
536 struct cfs_hash_bd *bd, int excl)
537{
538 hs->hs_lops->hs_bkt_lock(&bd->bd_bucket->hsb_lock, excl);
539}
540
541static inline void cfs_hash_bd_unlock(struct cfs_hash *hs,
542 struct cfs_hash_bd *bd, int excl)
543{
544 hs->hs_lops->hs_bkt_unlock(&bd->bd_bucket->hsb_lock, excl);
545}
546
547
548
549
550
551void cfs_hash_bd_get(struct cfs_hash *hs, const void *key,
552 struct cfs_hash_bd *bd);
553
554static inline void
555cfs_hash_bd_get_and_lock(struct cfs_hash *hs, const void *key,
556 struct cfs_hash_bd *bd, int excl)
557{
558 cfs_hash_bd_get(hs, key, bd);
559 cfs_hash_bd_lock(hs, bd, excl);
560}
561
562static inline unsigned
563cfs_hash_bd_index_get(struct cfs_hash *hs, struct cfs_hash_bd *bd)
564{
565 return bd->bd_offset | (bd->bd_bucket->hsb_index << hs->hs_bkt_bits);
566}
567
568static inline void
569cfs_hash_bd_index_set(struct cfs_hash *hs, unsigned index,
570 struct cfs_hash_bd *bd)
571{
572 bd->bd_bucket = hs->hs_buckets[index >> hs->hs_bkt_bits];
573 bd->bd_offset = index & (CFS_HASH_BKT_NHLIST(hs) - 1U);
574}
575
576static inline void *
577cfs_hash_bd_extra_get(struct cfs_hash *hs, struct cfs_hash_bd *bd)
578{
579 return (void *)bd->bd_bucket +
580 cfs_hash_bkt_size(hs) - hs->hs_extra_bytes;
581}
582
583static inline __u32
584cfs_hash_bd_version_get(struct cfs_hash_bd *bd)
585{
586
587 return bd->bd_bucket->hsb_version;
588}
589
590static inline __u32
591cfs_hash_bd_count_get(struct cfs_hash_bd *bd)
592{
593
594 return bd->bd_bucket->hsb_count;
595}
596
597static inline int
598cfs_hash_bd_depmax_get(struct cfs_hash_bd *bd)
599{
600 return bd->bd_bucket->hsb_depmax;
601}
602
603static inline int
604cfs_hash_bd_compare(struct cfs_hash_bd *bd1, struct cfs_hash_bd *bd2)
605{
606 if (bd1->bd_bucket->hsb_index != bd2->bd_bucket->hsb_index)
607 return bd1->bd_bucket->hsb_index - bd2->bd_bucket->hsb_index;
608
609 if (bd1->bd_offset != bd2->bd_offset)
610 return bd1->bd_offset - bd2->bd_offset;
611
612 return 0;
613}
614
615void cfs_hash_bd_add_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd,
616 struct hlist_node *hnode);
617void cfs_hash_bd_del_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd,
618 struct hlist_node *hnode);
619void cfs_hash_bd_move_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd_old,
620 struct cfs_hash_bd *bd_new,
621 struct hlist_node *hnode);
622
623static inline int
624cfs_hash_bd_dec_and_lock(struct cfs_hash *hs, struct cfs_hash_bd *bd,
625 atomic_t *condition)
626{
627 LASSERT(cfs_hash_with_spin_bktlock(hs));
628 return atomic_dec_and_lock(condition, &bd->bd_bucket->hsb_lock.spin);
629}
630
631static inline struct hlist_head *
632cfs_hash_bd_hhead(struct cfs_hash *hs, struct cfs_hash_bd *bd)
633{
634 return hs->hs_hops->hop_hhead(hs, bd);
635}
636
637struct hlist_node *
638cfs_hash_bd_lookup_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd,
639 const void *key);
640struct hlist_node *
641cfs_hash_bd_peek_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd,
642 const void *key);
643
644
645
646
647
648void cfs_hash_dual_bd_get(struct cfs_hash *hs, const void *key,
649 struct cfs_hash_bd *bds);
650void cfs_hash_dual_bd_lock(struct cfs_hash *hs, struct cfs_hash_bd *bds,
651 int excl);
652void cfs_hash_dual_bd_unlock(struct cfs_hash *hs, struct cfs_hash_bd *bds,
653 int excl);
654
655static inline void
656cfs_hash_dual_bd_get_and_lock(struct cfs_hash *hs, const void *key,
657 struct cfs_hash_bd *bds, int excl)
658{
659 cfs_hash_dual_bd_get(hs, key, bds);
660 cfs_hash_dual_bd_lock(hs, bds, excl);
661}
662
663struct hlist_node *
664cfs_hash_dual_bd_lookup_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
665 const void *key);
666struct hlist_node *
667cfs_hash_dual_bd_findadd_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
668 const void *key, struct hlist_node *hnode,
669 int insist_add);
670struct hlist_node *
671cfs_hash_dual_bd_finddel_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
672 const void *key, struct hlist_node *hnode);
673
674
675struct cfs_hash *
676cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits,
677 unsigned bkt_bits, unsigned extra_bytes,
678 unsigned min_theta, unsigned max_theta,
679 struct cfs_hash_ops *ops, unsigned flags);
680
681struct cfs_hash *cfs_hash_getref(struct cfs_hash *hs);
682void cfs_hash_putref(struct cfs_hash *hs);
683
684
685void cfs_hash_add(struct cfs_hash *hs, const void *key,
686 struct hlist_node *hnode);
687int cfs_hash_add_unique(struct cfs_hash *hs, const void *key,
688 struct hlist_node *hnode);
689void *cfs_hash_findadd_unique(struct cfs_hash *hs, const void *key,
690 struct hlist_node *hnode);
691
692
693void *cfs_hash_del(struct cfs_hash *hs, const void *key,
694 struct hlist_node *hnode);
695void *cfs_hash_del_key(struct cfs_hash *hs, const void *key);
696
697
698#define CFS_HASH_LOOP_HOG 1024
699
700typedef int (*cfs_hash_for_each_cb_t)(struct cfs_hash *hs,
701 struct cfs_hash_bd *bd,
702 struct hlist_node *node,
703 void *data);
704void *
705cfs_hash_lookup(struct cfs_hash *hs, const void *key);
706void
707cfs_hash_for_each(struct cfs_hash *hs, cfs_hash_for_each_cb_t, void *data);
708void
709cfs_hash_for_each_safe(struct cfs_hash *hs, cfs_hash_for_each_cb_t, void *data);
710int
711cfs_hash_for_each_nolock(struct cfs_hash *hs, cfs_hash_for_each_cb_t,
712 void *data);
713int
714cfs_hash_for_each_empty(struct cfs_hash *hs, cfs_hash_for_each_cb_t,
715 void *data);
716void
717cfs_hash_for_each_key(struct cfs_hash *hs, const void *key,
718 cfs_hash_for_each_cb_t, void *data);
719typedef int (*cfs_hash_cond_opt_cb_t)(void *obj, void *data);
720void
721cfs_hash_cond_del(struct cfs_hash *hs, cfs_hash_cond_opt_cb_t, void *data);
722
723void
724cfs_hash_hlist_for_each(struct cfs_hash *hs, unsigned hindex,
725 cfs_hash_for_each_cb_t, void *data);
726int cfs_hash_is_empty(struct cfs_hash *hs);
727__u64 cfs_hash_size_get(struct cfs_hash *hs);
728
729
730
731
732
733void cfs_hash_rehash_cancel_locked(struct cfs_hash *hs);
734void cfs_hash_rehash_cancel(struct cfs_hash *hs);
735int cfs_hash_rehash(struct cfs_hash *hs, int do_rehash);
736void cfs_hash_rehash_key(struct cfs_hash *hs, const void *old_key,
737 void *new_key, struct hlist_node *hnode);
738
739#if CFS_HASH_DEBUG_LEVEL > CFS_HASH_DEBUG_1
740
741static inline void
742cfs_hash_key_validate(struct cfs_hash *hs, const void *key,
743 struct hlist_node *hnode)
744{
745 LASSERT(cfs_hash_keycmp(hs, key, hnode));
746}
747
748
749static inline void
750cfs_hash_bucket_validate(struct cfs_hash *hs, struct cfs_hash_bd *bd,
751 struct hlist_node *hnode)
752{
753 struct cfs_hash_bd bds[2];
754
755 cfs_hash_dual_bd_get(hs, cfs_hash_key(hs, hnode), bds);
756 LASSERT(bds[0].bd_bucket == bd->bd_bucket ||
757 bds[1].bd_bucket == bd->bd_bucket);
758}
759
760#else
761
762static inline void
763cfs_hash_key_validate(struct cfs_hash *hs, const void *key,
764 struct hlist_node *hnode) {}
765
766static inline void
767cfs_hash_bucket_validate(struct cfs_hash *hs, struct cfs_hash_bd *bd,
768 struct hlist_node *hnode) {}
769
770#endif
771
772#define CFS_HASH_THETA_BITS 10
773#define CFS_HASH_MIN_THETA (1U << (CFS_HASH_THETA_BITS - 1))
774#define CFS_HASH_MAX_THETA (1U << (CFS_HASH_THETA_BITS + 1))
775
776
777static inline int __cfs_hash_theta_int(int theta)
778{
779 return (theta >> CFS_HASH_THETA_BITS);
780}
781
782
783static inline int __cfs_hash_theta_frac(int theta)
784{
785 return ((theta * 1000) >> CFS_HASH_THETA_BITS) -
786 (__cfs_hash_theta_int(theta) * 1000);
787}
788
789static inline int __cfs_hash_theta(struct cfs_hash *hs)
790{
791 return (atomic_read(&hs->hs_count) <<
792 CFS_HASH_THETA_BITS) >> hs->hs_cur_bits;
793}
794
795static inline void
796__cfs_hash_set_theta(struct cfs_hash *hs, int min, int max)
797{
798 LASSERT(min < max);
799 hs->hs_min_theta = (__u16)min;
800 hs->hs_max_theta = (__u16)max;
801}
802
803
804struct seq_file;
805void cfs_hash_debug_header(struct seq_file *m);
806void cfs_hash_debug_str(struct cfs_hash *hs, struct seq_file *m);
807
808
809
810
811static inline unsigned
812cfs_hash_djb2_hash(const void *key, size_t size, unsigned mask)
813{
814 unsigned i, hash = 5381;
815
816 LASSERT(key != NULL);
817
818 for (i = 0; i < size; i++)
819 hash = hash * 33 + ((char *)key)[i];
820
821 return (hash & mask);
822}
823
824
825
826
827static inline unsigned
828cfs_hash_u32_hash(const __u32 key, unsigned mask)
829{
830 return ((key * CFS_GOLDEN_RATIO_PRIME_32) & mask);
831}
832
833
834
835
836static inline unsigned
837cfs_hash_u64_hash(const __u64 key, unsigned mask)
838{
839 return ((unsigned)(key * CFS_GOLDEN_RATIO_PRIME_64) & mask);
840}
841
842
843#define cfs_hash_for_each_bd(bds, n, i) \
844 for (i = 0; i < n && (bds)[i].bd_bucket != NULL; i++)
845
846
847#define cfs_hash_for_each_bucket(hs, bd, pos) \
848 for (pos = 0; \
849 pos < CFS_HASH_NBKT(hs) && \
850 ((bd)->bd_bucket = (hs)->hs_buckets[pos]) != NULL; pos++)
851
852
853#define cfs_hash_bd_for_each_hlist(hs, bd, hlist) \
854 for ((bd)->bd_offset = 0; \
855 (bd)->bd_offset < CFS_HASH_BKT_NHLIST(hs) && \
856 (hlist = cfs_hash_bd_hhead(hs, bd)) != NULL; \
857 (bd)->bd_offset++)
858
859
860#endif
861