1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45#define DEBUG_SUBSYSTEM S_CLASS
46
47#include "../../include/linux/libcfs/libcfs.h"
48
49# include <linux/module.h>
50
51
52#include "../../include/linux/libcfs/libcfs_hash.h"
53#include "../include/obd_class.h"
54#include "../include/obd_support.h"
55#include "../include/lustre_disk.h"
56#include "../include/lustre_fid.h"
57#include "../include/lu_object.h"
58#include "../include/lu_ref.h"
59#include <linux/list.h>
60
61static void lu_object_free(const struct lu_env *env, struct lu_object *o);
62static __u32 ls_stats_read(struct lprocfs_stats *stats, int idx);
63
64
65
66
67
68
69void lu_object_put(const struct lu_env *env, struct lu_object *o)
70{
71 struct lu_site_bkt_data *bkt;
72 struct lu_object_header *top;
73 struct lu_site *site;
74 struct lu_object *orig;
75 struct cfs_hash_bd bd;
76 const struct lu_fid *fid;
77
78 top = o->lo_header;
79 site = o->lo_dev->ld_site;
80 orig = o;
81
82
83
84
85
86
87 fid = lu_object_fid(o);
88 if (fid_is_zero(fid)) {
89 LASSERT(!top->loh_hash.next && !top->loh_hash.pprev);
90 LASSERT(list_empty(&top->loh_lru));
91 if (!atomic_dec_and_test(&top->loh_ref))
92 return;
93 list_for_each_entry_reverse(o, &top->loh_layers, lo_linkage) {
94 if (o->lo_ops->loo_object_release)
95 o->lo_ops->loo_object_release(env, o);
96 }
97 lu_object_free(env, orig);
98 return;
99 }
100
101 cfs_hash_bd_get(site->ls_obj_hash, &top->loh_fid, &bd);
102 bkt = cfs_hash_bd_extra_get(site->ls_obj_hash, &bd);
103
104 if (!cfs_hash_bd_dec_and_lock(site->ls_obj_hash, &bd, &top->loh_ref)) {
105 if (lu_object_is_dying(top)) {
106
107
108
109
110
111 wake_up_all(&bkt->lsb_marche_funebre);
112 }
113 return;
114 }
115
116
117
118
119
120 list_for_each_entry_reverse(o, &top->loh_layers, lo_linkage) {
121 if (o->lo_ops->loo_object_release)
122 o->lo_ops->loo_object_release(env, o);
123 }
124
125 if (!lu_object_is_dying(top)) {
126 LASSERT(list_empty(&top->loh_lru));
127 list_add_tail(&top->loh_lru, &bkt->lsb_lru);
128 bkt->lsb_lru_len++;
129 lprocfs_counter_incr(site->ls_stats, LU_SS_LRU_LEN);
130 CDEBUG(D_INODE, "Add %p to site lru. hash: %p, bkt: %p, lru_len: %ld\n",
131 o, site->ls_obj_hash, bkt, bkt->lsb_lru_len);
132 cfs_hash_bd_unlock(site->ls_obj_hash, &bd, 1);
133 return;
134 }
135
136
137
138
139
140
141
142
143
144
145
146
147 if (!test_and_set_bit(LU_OBJECT_UNHASHED, &top->loh_flags))
148 cfs_hash_bd_del_locked(site->ls_obj_hash, &bd, &top->loh_hash);
149 cfs_hash_bd_unlock(site->ls_obj_hash, &bd, 1);
150
151
152
153
154 lu_object_free(env, orig);
155}
156EXPORT_SYMBOL(lu_object_put);
157
158
159
160
161
162void lu_object_unhash(const struct lu_env *env, struct lu_object *o)
163{
164 struct lu_object_header *top;
165
166 top = o->lo_header;
167 set_bit(LU_OBJECT_HEARD_BANSHEE, &top->loh_flags);
168 if (!test_and_set_bit(LU_OBJECT_UNHASHED, &top->loh_flags)) {
169 struct lu_site *site = o->lo_dev->ld_site;
170 struct cfs_hash *obj_hash = site->ls_obj_hash;
171 struct cfs_hash_bd bd;
172
173 cfs_hash_bd_get_and_lock(obj_hash, &top->loh_fid, &bd, 1);
174 if (!list_empty(&top->loh_lru)) {
175 struct lu_site_bkt_data *bkt;
176
177 list_del_init(&top->loh_lru);
178 bkt = cfs_hash_bd_extra_get(obj_hash, &bd);
179 bkt->lsb_lru_len--;
180 lprocfs_counter_decr(site->ls_stats, LU_SS_LRU_LEN);
181 }
182 cfs_hash_bd_del_locked(obj_hash, &bd, &top->loh_hash);
183 cfs_hash_bd_unlock(obj_hash, &bd, 1);
184 }
185}
186EXPORT_SYMBOL(lu_object_unhash);
187
188
189
190
191
192
193
194static struct lu_object *lu_object_alloc(const struct lu_env *env,
195 struct lu_device *dev,
196 const struct lu_fid *f,
197 const struct lu_object_conf *conf)
198{
199 struct lu_object *scan;
200 struct lu_object *top;
201 struct list_head *layers;
202 unsigned int init_mask = 0;
203 unsigned int init_flag;
204 int clean;
205 int result;
206
207
208
209
210
211 top = dev->ld_ops->ldo_object_alloc(env, NULL, dev);
212 if (!top)
213 return ERR_PTR(-ENOMEM);
214 if (IS_ERR(top))
215 return top;
216
217
218
219
220 top->lo_header->loh_fid = *f;
221 layers = &top->lo_header->loh_layers;
222
223 do {
224
225
226
227
228 clean = 1;
229 init_flag = 1;
230 list_for_each_entry(scan, layers, lo_linkage) {
231 if (init_mask & init_flag)
232 goto next;
233 clean = 0;
234 scan->lo_header = top->lo_header;
235 result = scan->lo_ops->loo_object_init(env, scan, conf);
236 if (result != 0) {
237 lu_object_free(env, top);
238 return ERR_PTR(result);
239 }
240 init_mask |= init_flag;
241next:
242 init_flag <<= 1;
243 }
244 } while (!clean);
245
246 list_for_each_entry_reverse(scan, layers, lo_linkage) {
247 if (scan->lo_ops->loo_object_start) {
248 result = scan->lo_ops->loo_object_start(env, scan);
249 if (result != 0) {
250 lu_object_free(env, top);
251 return ERR_PTR(result);
252 }
253 }
254 }
255
256 lprocfs_counter_incr(dev->ld_site->ls_stats, LU_SS_CREATED);
257 return top;
258}
259
260
261
262
263static void lu_object_free(const struct lu_env *env, struct lu_object *o)
264{
265 struct lu_site_bkt_data *bkt;
266 struct lu_site *site;
267 struct lu_object *scan;
268 struct list_head *layers;
269 struct list_head splice;
270
271 site = o->lo_dev->ld_site;
272 layers = &o->lo_header->loh_layers;
273 bkt = lu_site_bkt_from_fid(site, &o->lo_header->loh_fid);
274
275
276
277 list_for_each_entry_reverse(scan, layers, lo_linkage) {
278 if (scan->lo_ops->loo_object_delete)
279 scan->lo_ops->loo_object_delete(env, scan);
280 }
281
282
283
284
285
286
287
288 INIT_LIST_HEAD(&splice);
289 list_splice_init(layers, &splice);
290 while (!list_empty(&splice)) {
291
292
293
294
295
296 o = container_of0(splice.prev, struct lu_object, lo_linkage);
297 list_del_init(&o->lo_linkage);
298 o->lo_ops->loo_object_free(env, o);
299 }
300
301 if (waitqueue_active(&bkt->lsb_marche_funebre))
302 wake_up_all(&bkt->lsb_marche_funebre);
303}
304
305
306
307
308int lu_site_purge(const struct lu_env *env, struct lu_site *s, int nr)
309{
310 struct lu_object_header *h;
311 struct lu_object_header *temp;
312 struct lu_site_bkt_data *bkt;
313 struct cfs_hash_bd bd;
314 struct cfs_hash_bd bd2;
315 struct list_head dispose;
316 int did_sth;
317 int start;
318 int count;
319 int bnr;
320 int i;
321
322 if (OBD_FAIL_CHECK(OBD_FAIL_OBD_NO_LRU))
323 return 0;
324
325 INIT_LIST_HEAD(&dispose);
326
327
328
329
330 start = s->ls_purge_start;
331 bnr = (nr == ~0) ? -1 : nr / CFS_HASH_NBKT(s->ls_obj_hash) + 1;
332 again:
333 did_sth = 0;
334 cfs_hash_for_each_bucket(s->ls_obj_hash, &bd, i) {
335 if (i < start)
336 continue;
337 count = bnr;
338 cfs_hash_bd_lock(s->ls_obj_hash, &bd, 1);
339 bkt = cfs_hash_bd_extra_get(s->ls_obj_hash, &bd);
340
341 list_for_each_entry_safe(h, temp, &bkt->lsb_lru, loh_lru) {
342 LASSERT(atomic_read(&h->loh_ref) == 0);
343
344 cfs_hash_bd_get(s->ls_obj_hash, &h->loh_fid, &bd2);
345 LASSERT(bd.bd_bucket == bd2.bd_bucket);
346
347 cfs_hash_bd_del_locked(s->ls_obj_hash,
348 &bd2, &h->loh_hash);
349 list_move(&h->loh_lru, &dispose);
350 bkt->lsb_lru_len--;
351 lprocfs_counter_decr(s->ls_stats, LU_SS_LRU_LEN);
352 if (did_sth == 0)
353 did_sth = 1;
354
355 if (nr != ~0 && --nr == 0)
356 break;
357
358 if (count > 0 && --count == 0)
359 break;
360
361 }
362 cfs_hash_bd_unlock(s->ls_obj_hash, &bd, 1);
363 cond_resched();
364
365
366
367
368 while (!list_empty(&dispose)) {
369 h = container_of0(dispose.next,
370 struct lu_object_header, loh_lru);
371 list_del_init(&h->loh_lru);
372 lu_object_free(env, lu_object_top(h));
373 lprocfs_counter_incr(s->ls_stats, LU_SS_LRU_PURGED);
374 }
375
376 if (nr == 0)
377 break;
378 }
379
380 if (nr != 0 && did_sth && start != 0) {
381 start = 0;
382 goto again;
383 }
384
385 s->ls_purge_start = i % CFS_HASH_NBKT(s->ls_obj_hash);
386
387 return nr;
388}
389EXPORT_SYMBOL(lu_site_purge);
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407enum {
408
409
410
411
412
413 LU_CDEBUG_LINE = 512
414};
415
416struct lu_cdebug_data {
417
418
419
420 char lck_area[LU_CDEBUG_LINE];
421};
422
423
424LU_KEY_INIT_FINI(lu_global, struct lu_cdebug_data);
425
426
427
428
429
430static struct lu_context_key lu_global_key = {
431 .lct_tags = LCT_MD_THREAD | LCT_DT_THREAD |
432 LCT_MG_THREAD | LCT_CL_THREAD | LCT_LOCAL,
433 .lct_init = lu_global_key_init,
434 .lct_fini = lu_global_key_fini
435};
436
437
438
439
440int lu_cdebug_printer(const struct lu_env *env,
441 void *cookie, const char *format, ...)
442{
443 struct libcfs_debug_msg_data *msgdata = cookie;
444 struct lu_cdebug_data *key;
445 int used;
446 int complete;
447 va_list args;
448
449 va_start(args, format);
450
451 key = lu_context_key_get(&env->le_ctx, &lu_global_key);
452
453 used = strlen(key->lck_area);
454 complete = format[strlen(format) - 1] == '\n';
455
456
457
458 vsnprintf(key->lck_area + used,
459 ARRAY_SIZE(key->lck_area) - used, format, args);
460 if (complete) {
461 if (cfs_cdebug_show(msgdata->msg_mask, msgdata->msg_subsys))
462 libcfs_debug_msg(msgdata, "%s\n", key->lck_area);
463 key->lck_area[0] = 0;
464 }
465 va_end(args);
466 return 0;
467}
468EXPORT_SYMBOL(lu_cdebug_printer);
469
470
471
472
473void lu_object_header_print(const struct lu_env *env, void *cookie,
474 lu_printer_t printer,
475 const struct lu_object_header *hdr)
476{
477 (*printer)(env, cookie, "header@%p[%#lx, %d, "DFID"%s%s%s]",
478 hdr, hdr->loh_flags, atomic_read(&hdr->loh_ref),
479 PFID(&hdr->loh_fid),
480 hlist_unhashed(&hdr->loh_hash) ? "" : " hash",
481 list_empty((struct list_head *)&hdr->loh_lru) ? \
482 "" : " lru",
483 hdr->loh_attr & LOHA_EXISTS ? " exist":"");
484}
485EXPORT_SYMBOL(lu_object_header_print);
486
487
488
489
490void lu_object_print(const struct lu_env *env, void *cookie,
491 lu_printer_t printer, const struct lu_object *o)
492{
493 static const char ruler[] = "........................................";
494 struct lu_object_header *top;
495 int depth = 4;
496
497 top = o->lo_header;
498 lu_object_header_print(env, cookie, printer, top);
499 (*printer)(env, cookie, "{\n");
500
501 list_for_each_entry(o, &top->loh_layers, lo_linkage) {
502
503
504
505 (*printer)(env, cookie, "%*.*s%s@%p", depth, depth, ruler,
506 o->lo_dev->ld_type->ldt_name, o);
507
508 if (o->lo_ops->loo_object_print)
509 (*o->lo_ops->loo_object_print)(env, cookie, printer, o);
510
511 (*printer)(env, cookie, "\n");
512 }
513
514 (*printer)(env, cookie, "} header@%p\n", top);
515}
516EXPORT_SYMBOL(lu_object_print);
517
518static struct lu_object *htable_lookup(struct lu_site *s,
519 struct cfs_hash_bd *bd,
520 const struct lu_fid *f,
521 wait_queue_t *waiter,
522 __u64 *version)
523{
524 struct lu_site_bkt_data *bkt;
525 struct lu_object_header *h;
526 struct hlist_node *hnode;
527 __u64 ver = cfs_hash_bd_version_get(bd);
528
529 if (*version == ver)
530 return ERR_PTR(-ENOENT);
531
532 *version = ver;
533 bkt = cfs_hash_bd_extra_get(s->ls_obj_hash, bd);
534
535
536
537 hnode = cfs_hash_bd_peek_locked(s->ls_obj_hash, bd, (void *)f);
538 if (!hnode) {
539 lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_MISS);
540 return ERR_PTR(-ENOENT);
541 }
542
543 h = container_of0(hnode, struct lu_object_header, loh_hash);
544 if (likely(!lu_object_is_dying(h))) {
545 cfs_hash_get(s->ls_obj_hash, hnode);
546 lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_HIT);
547 if (!list_empty(&h->loh_lru)) {
548 list_del_init(&h->loh_lru);
549 bkt->lsb_lru_len--;
550 lprocfs_counter_decr(s->ls_stats, LU_SS_LRU_LEN);
551 }
552 return lu_object_top(h);
553 }
554
555
556
557
558
559
560
561 init_waitqueue_entry(waiter, current);
562 add_wait_queue(&bkt->lsb_marche_funebre, waiter);
563 set_current_state(TASK_UNINTERRUPTIBLE);
564 lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_DEATH_RACE);
565 return ERR_PTR(-EAGAIN);
566}
567
568
569
570
571
572
573static struct lu_object *lu_object_find(const struct lu_env *env,
574 struct lu_device *dev,
575 const struct lu_fid *f,
576 const struct lu_object_conf *conf)
577{
578 return lu_object_find_at(env, dev->ld_site->ls_top_dev, f, conf);
579}
580
581static struct lu_object *lu_object_new(const struct lu_env *env,
582 struct lu_device *dev,
583 const struct lu_fid *f,
584 const struct lu_object_conf *conf)
585{
586 struct lu_object *o;
587 struct cfs_hash *hs;
588 struct cfs_hash_bd bd;
589
590 o = lu_object_alloc(env, dev, f, conf);
591 if (IS_ERR(o))
592 return o;
593
594 hs = dev->ld_site->ls_obj_hash;
595 cfs_hash_bd_get_and_lock(hs, (void *)f, &bd, 1);
596 cfs_hash_bd_add_locked(hs, &bd, &o->lo_header->loh_hash);
597 cfs_hash_bd_unlock(hs, &bd, 1);
598 return o;
599}
600
601
602
603
604static struct lu_object *lu_object_find_try(const struct lu_env *env,
605 struct lu_device *dev,
606 const struct lu_fid *f,
607 const struct lu_object_conf *conf,
608 wait_queue_t *waiter)
609{
610 struct lu_object *o;
611 struct lu_object *shadow;
612 struct lu_site *s;
613 struct cfs_hash *hs;
614 struct cfs_hash_bd bd;
615 __u64 version = 0;
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637 if (conf && conf->loc_flags & LOC_F_NEW)
638 return lu_object_new(env, dev, f, conf);
639
640 s = dev->ld_site;
641 hs = s->ls_obj_hash;
642 cfs_hash_bd_get_and_lock(hs, (void *)f, &bd, 1);
643 o = htable_lookup(s, &bd, f, waiter, &version);
644 cfs_hash_bd_unlock(hs, &bd, 1);
645 if (!IS_ERR(o) || PTR_ERR(o) != -ENOENT)
646 return o;
647
648
649
650
651
652 o = lu_object_alloc(env, dev, f, conf);
653 if (IS_ERR(o))
654 return o;
655
656 LASSERT(lu_fid_eq(lu_object_fid(o), f));
657
658 cfs_hash_bd_lock(hs, &bd, 1);
659
660 shadow = htable_lookup(s, &bd, f, waiter, &version);
661 if (likely(PTR_ERR(shadow) == -ENOENT)) {
662 cfs_hash_bd_add_locked(hs, &bd, &o->lo_header->loh_hash);
663 cfs_hash_bd_unlock(hs, &bd, 1);
664 return o;
665 }
666
667 lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_RACE);
668 cfs_hash_bd_unlock(hs, &bd, 1);
669 lu_object_free(env, o);
670 return shadow;
671}
672
673
674
675
676
677
678struct lu_object *lu_object_find_at(const struct lu_env *env,
679 struct lu_device *dev,
680 const struct lu_fid *f,
681 const struct lu_object_conf *conf)
682{
683 struct lu_site_bkt_data *bkt;
684 struct lu_object *obj;
685 wait_queue_t wait;
686
687 while (1) {
688 obj = lu_object_find_try(env, dev, f, conf, &wait);
689 if (obj != ERR_PTR(-EAGAIN))
690 return obj;
691
692
693
694
695 schedule();
696 bkt = lu_site_bkt_from_fid(dev->ld_site, (void *)f);
697 remove_wait_queue(&bkt->lsb_marche_funebre, &wait);
698 }
699}
700EXPORT_SYMBOL(lu_object_find_at);
701
702
703
704
705struct lu_object *lu_object_find_slice(const struct lu_env *env,
706 struct lu_device *dev,
707 const struct lu_fid *f,
708 const struct lu_object_conf *conf)
709{
710 struct lu_object *top;
711 struct lu_object *obj;
712
713 top = lu_object_find(env, dev, f, conf);
714 if (!IS_ERR(top)) {
715 obj = lu_object_locate(top->lo_header, dev->ld_type);
716 if (!obj)
717 lu_object_put(env, top);
718 } else
719 obj = top;
720 return obj;
721}
722EXPORT_SYMBOL(lu_object_find_slice);
723
724
725
726
727static LIST_HEAD(lu_device_types);
728
729int lu_device_type_init(struct lu_device_type *ldt)
730{
731 int result = 0;
732
733 INIT_LIST_HEAD(&ldt->ldt_linkage);
734 if (ldt->ldt_ops->ldto_init)
735 result = ldt->ldt_ops->ldto_init(ldt);
736 if (result == 0)
737 list_add(&ldt->ldt_linkage, &lu_device_types);
738 return result;
739}
740EXPORT_SYMBOL(lu_device_type_init);
741
742void lu_device_type_fini(struct lu_device_type *ldt)
743{
744 list_del_init(&ldt->ldt_linkage);
745 if (ldt->ldt_ops->ldto_fini)
746 ldt->ldt_ops->ldto_fini(ldt);
747}
748EXPORT_SYMBOL(lu_device_type_fini);
749
750void lu_types_stop(void)
751{
752 struct lu_device_type *ldt;
753
754 list_for_each_entry(ldt, &lu_device_types, ldt_linkage) {
755 if (ldt->ldt_device_nr == 0 && ldt->ldt_ops->ldto_stop)
756 ldt->ldt_ops->ldto_stop(ldt);
757 }
758}
759EXPORT_SYMBOL(lu_types_stop);
760
761
762
763
764static LIST_HEAD(lu_sites);
765static DEFINE_MUTEX(lu_sites_guard);
766
767
768
769
770static struct lu_env lu_shrink_env;
771
772struct lu_site_print_arg {
773 struct lu_env *lsp_env;
774 void *lsp_cookie;
775 lu_printer_t lsp_printer;
776};
777
778static int
779lu_site_obj_print(struct cfs_hash *hs, struct cfs_hash_bd *bd,
780 struct hlist_node *hnode, void *data)
781{
782 struct lu_site_print_arg *arg = (struct lu_site_print_arg *)data;
783 struct lu_object_header *h;
784
785 h = hlist_entry(hnode, struct lu_object_header, loh_hash);
786 if (!list_empty(&h->loh_layers)) {
787 const struct lu_object *o;
788
789 o = lu_object_top(h);
790 lu_object_print(arg->lsp_env, arg->lsp_cookie,
791 arg->lsp_printer, o);
792 } else {
793 lu_object_header_print(arg->lsp_env, arg->lsp_cookie,
794 arg->lsp_printer, h);
795 }
796 return 0;
797}
798
799
800
801
802void lu_site_print(const struct lu_env *env, struct lu_site *s, void *cookie,
803 lu_printer_t printer)
804{
805 struct lu_site_print_arg arg = {
806 .lsp_env = (struct lu_env *)env,
807 .lsp_cookie = cookie,
808 .lsp_printer = printer,
809 };
810
811 cfs_hash_for_each(s->ls_obj_hash, lu_site_obj_print, &arg);
812}
813EXPORT_SYMBOL(lu_site_print);
814
815enum {
816 LU_CACHE_PERCENT_MAX = 50,
817 LU_CACHE_PERCENT_DEFAULT = 20
818};
819
820static unsigned int lu_cache_percent = LU_CACHE_PERCENT_DEFAULT;
821module_param(lu_cache_percent, int, 0644);
822MODULE_PARM_DESC(lu_cache_percent, "Percentage of memory to be used as lu_object cache");
823
824
825
826
827static int lu_htable_order(void)
828{
829 unsigned long cache_size;
830 int bits;
831
832
833
834
835
836
837
838
839 cache_size = totalram_pages;
840
841#if BITS_PER_LONG == 32
842
843 if (cache_size > 1 << (30 - PAGE_SHIFT))
844 cache_size = 1 << (30 - PAGE_SHIFT) * 3 / 4;
845#endif
846
847
848 if (lu_cache_percent == 0 || lu_cache_percent > LU_CACHE_PERCENT_MAX) {
849 CWARN("obdclass: invalid lu_cache_percent: %u, it must be in the range of (0, %u]. Will use default value: %u.\n",
850 lu_cache_percent, LU_CACHE_PERCENT_MAX,
851 LU_CACHE_PERCENT_DEFAULT);
852
853 lu_cache_percent = LU_CACHE_PERCENT_DEFAULT;
854 }
855 cache_size = cache_size / 100 * lu_cache_percent *
856 (PAGE_SIZE / 1024);
857
858 for (bits = 1; (1 << bits) < cache_size; ++bits) {
859 ;
860 }
861 return bits;
862}
863
864static unsigned lu_obj_hop_hash(struct cfs_hash *hs,
865 const void *key, unsigned mask)
866{
867 struct lu_fid *fid = (struct lu_fid *)key;
868 __u32 hash;
869
870 hash = fid_flatten32(fid);
871 hash += (hash >> 4) + (hash << 12);
872 hash = hash_long(hash, hs->hs_bkt_bits);
873
874
875 hash -= hash_long((unsigned long)hs, fid_oid(fid) % 11 + 3);
876
877 hash <<= hs->hs_cur_bits - hs->hs_bkt_bits;
878 hash |= (fid_seq(fid) + fid_oid(fid)) & (CFS_HASH_NBKT(hs) - 1);
879
880 return hash & mask;
881}
882
883static void *lu_obj_hop_object(struct hlist_node *hnode)
884{
885 return hlist_entry(hnode, struct lu_object_header, loh_hash);
886}
887
888static void *lu_obj_hop_key(struct hlist_node *hnode)
889{
890 struct lu_object_header *h;
891
892 h = hlist_entry(hnode, struct lu_object_header, loh_hash);
893 return &h->loh_fid;
894}
895
896static int lu_obj_hop_keycmp(const void *key, struct hlist_node *hnode)
897{
898 struct lu_object_header *h;
899
900 h = hlist_entry(hnode, struct lu_object_header, loh_hash);
901 return lu_fid_eq(&h->loh_fid, (struct lu_fid *)key);
902}
903
904static void lu_obj_hop_get(struct cfs_hash *hs, struct hlist_node *hnode)
905{
906 struct lu_object_header *h;
907
908 h = hlist_entry(hnode, struct lu_object_header, loh_hash);
909 atomic_inc(&h->loh_ref);
910}
911
912static void lu_obj_hop_put_locked(struct cfs_hash *hs, struct hlist_node *hnode)
913{
914 LBUG();
915}
916
917static struct cfs_hash_ops lu_site_hash_ops = {
918 .hs_hash = lu_obj_hop_hash,
919 .hs_key = lu_obj_hop_key,
920 .hs_keycmp = lu_obj_hop_keycmp,
921 .hs_object = lu_obj_hop_object,
922 .hs_get = lu_obj_hop_get,
923 .hs_put_locked = lu_obj_hop_put_locked,
924};
925
926static void lu_dev_add_linkage(struct lu_site *s, struct lu_device *d)
927{
928 spin_lock(&s->ls_ld_lock);
929 if (list_empty(&d->ld_linkage))
930 list_add(&d->ld_linkage, &s->ls_ld_linkage);
931 spin_unlock(&s->ls_ld_lock);
932}
933
934
935
936
937#define LU_SITE_BITS_MIN 12
938#define LU_SITE_BITS_MAX 24
939
940
941
942
943
944#define LU_SITE_BKT_BITS 8
945
946int lu_site_init(struct lu_site *s, struct lu_device *top)
947{
948 struct lu_site_bkt_data *bkt;
949 struct cfs_hash_bd bd;
950 char name[16];
951 int bits;
952 int i;
953
954 memset(s, 0, sizeof(*s));
955 bits = lu_htable_order();
956 snprintf(name, 16, "lu_site_%s", top->ld_type->ldt_name);
957 for (bits = min(max(LU_SITE_BITS_MIN, bits), LU_SITE_BITS_MAX);
958 bits >= LU_SITE_BITS_MIN; bits--) {
959 s->ls_obj_hash = cfs_hash_create(name, bits, bits,
960 bits - LU_SITE_BKT_BITS,
961 sizeof(*bkt), 0, 0,
962 &lu_site_hash_ops,
963 CFS_HASH_SPIN_BKTLOCK |
964 CFS_HASH_NO_ITEMREF |
965 CFS_HASH_DEPTH |
966 CFS_HASH_ASSERT_EMPTY);
967 if (s->ls_obj_hash)
968 break;
969 }
970
971 if (!s->ls_obj_hash) {
972 CERROR("failed to create lu_site hash with bits: %d\n", bits);
973 return -ENOMEM;
974 }
975
976 cfs_hash_for_each_bucket(s->ls_obj_hash, &bd, i) {
977 bkt = cfs_hash_bd_extra_get(s->ls_obj_hash, &bd);
978 INIT_LIST_HEAD(&bkt->lsb_lru);
979 init_waitqueue_head(&bkt->lsb_marche_funebre);
980 }
981
982 s->ls_stats = lprocfs_alloc_stats(LU_SS_LAST_STAT, 0);
983 if (!s->ls_stats) {
984 cfs_hash_putref(s->ls_obj_hash);
985 s->ls_obj_hash = NULL;
986 return -ENOMEM;
987 }
988
989 lprocfs_counter_init(s->ls_stats, LU_SS_CREATED,
990 0, "created", "created");
991 lprocfs_counter_init(s->ls_stats, LU_SS_CACHE_HIT,
992 0, "cache_hit", "cache_hit");
993 lprocfs_counter_init(s->ls_stats, LU_SS_CACHE_MISS,
994 0, "cache_miss", "cache_miss");
995 lprocfs_counter_init(s->ls_stats, LU_SS_CACHE_RACE,
996 0, "cache_race", "cache_race");
997 lprocfs_counter_init(s->ls_stats, LU_SS_CACHE_DEATH_RACE,
998 0, "cache_death_race", "cache_death_race");
999 lprocfs_counter_init(s->ls_stats, LU_SS_LRU_PURGED,
1000 0, "lru_purged", "lru_purged");
1001
1002
1003
1004
1005 lprocfs_counter_init(s->ls_stats, LU_SS_LRU_LEN,
1006 LPROCFS_CNTR_AVGMINMAX, "lru_len", "lru_len");
1007
1008 INIT_LIST_HEAD(&s->ls_linkage);
1009 s->ls_top_dev = top;
1010 top->ld_site = s;
1011 lu_device_get(top);
1012 lu_ref_add(&top->ld_reference, "site-top", s);
1013
1014 INIT_LIST_HEAD(&s->ls_ld_linkage);
1015 spin_lock_init(&s->ls_ld_lock);
1016
1017 lu_dev_add_linkage(s, top);
1018
1019 return 0;
1020}
1021EXPORT_SYMBOL(lu_site_init);
1022
1023
1024
1025
1026void lu_site_fini(struct lu_site *s)
1027{
1028 mutex_lock(&lu_sites_guard);
1029 list_del_init(&s->ls_linkage);
1030 mutex_unlock(&lu_sites_guard);
1031
1032 if (s->ls_obj_hash) {
1033 cfs_hash_putref(s->ls_obj_hash);
1034 s->ls_obj_hash = NULL;
1035 }
1036
1037 if (s->ls_top_dev) {
1038 s->ls_top_dev->ld_site = NULL;
1039 lu_ref_del(&s->ls_top_dev->ld_reference, "site-top", s);
1040 lu_device_put(s->ls_top_dev);
1041 s->ls_top_dev = NULL;
1042 }
1043
1044 if (s->ls_stats)
1045 lprocfs_free_stats(&s->ls_stats);
1046}
1047EXPORT_SYMBOL(lu_site_fini);
1048
1049
1050
1051
1052int lu_site_init_finish(struct lu_site *s)
1053{
1054 int result;
1055
1056 mutex_lock(&lu_sites_guard);
1057 result = lu_context_refill(&lu_shrink_env.le_ctx);
1058 if (result == 0)
1059 list_add(&s->ls_linkage, &lu_sites);
1060 mutex_unlock(&lu_sites_guard);
1061 return result;
1062}
1063EXPORT_SYMBOL(lu_site_init_finish);
1064
1065
1066
1067
1068void lu_device_get(struct lu_device *d)
1069{
1070 atomic_inc(&d->ld_ref);
1071}
1072EXPORT_SYMBOL(lu_device_get);
1073
1074
1075
1076
1077void lu_device_put(struct lu_device *d)
1078{
1079 LASSERT(atomic_read(&d->ld_ref) > 0);
1080 atomic_dec(&d->ld_ref);
1081}
1082EXPORT_SYMBOL(lu_device_put);
1083
1084
1085
1086
1087int lu_device_init(struct lu_device *d, struct lu_device_type *t)
1088{
1089 if (t->ldt_device_nr++ == 0 && t->ldt_ops->ldto_start)
1090 t->ldt_ops->ldto_start(t);
1091 memset(d, 0, sizeof(*d));
1092 atomic_set(&d->ld_ref, 0);
1093 d->ld_type = t;
1094 lu_ref_init(&d->ld_reference);
1095 INIT_LIST_HEAD(&d->ld_linkage);
1096 return 0;
1097}
1098EXPORT_SYMBOL(lu_device_init);
1099
1100
1101
1102
1103void lu_device_fini(struct lu_device *d)
1104{
1105 struct lu_device_type *t;
1106
1107 t = d->ld_type;
1108 if (d->ld_obd) {
1109 d->ld_obd->obd_lu_dev = NULL;
1110 d->ld_obd = NULL;
1111 }
1112
1113 lu_ref_fini(&d->ld_reference);
1114 LASSERTF(atomic_read(&d->ld_ref) == 0,
1115 "Refcount is %u\n", atomic_read(&d->ld_ref));
1116 LASSERT(t->ldt_device_nr > 0);
1117 if (--t->ldt_device_nr == 0 && t->ldt_ops->ldto_stop)
1118 t->ldt_ops->ldto_stop(t);
1119}
1120EXPORT_SYMBOL(lu_device_fini);
1121
1122
1123
1124
1125
1126int lu_object_init(struct lu_object *o, struct lu_object_header *h,
1127 struct lu_device *d)
1128{
1129 memset(o, 0, sizeof(*o));
1130 o->lo_header = h;
1131 o->lo_dev = d;
1132 lu_device_get(d);
1133 lu_ref_add_at(&d->ld_reference, &o->lo_dev_ref, "lu_object", o);
1134 INIT_LIST_HEAD(&o->lo_linkage);
1135
1136 return 0;
1137}
1138EXPORT_SYMBOL(lu_object_init);
1139
1140
1141
1142
1143void lu_object_fini(struct lu_object *o)
1144{
1145 struct lu_device *dev = o->lo_dev;
1146
1147 LASSERT(list_empty(&o->lo_linkage));
1148
1149 if (dev) {
1150 lu_ref_del_at(&dev->ld_reference, &o->lo_dev_ref,
1151 "lu_object", o);
1152 lu_device_put(dev);
1153 o->lo_dev = NULL;
1154 }
1155}
1156EXPORT_SYMBOL(lu_object_fini);
1157
1158
1159
1160
1161
1162
1163
1164void lu_object_add_top(struct lu_object_header *h, struct lu_object *o)
1165{
1166 list_move(&o->lo_linkage, &h->loh_layers);
1167}
1168EXPORT_SYMBOL(lu_object_add_top);
1169
1170
1171
1172
1173
1174
1175
1176void lu_object_add(struct lu_object *before, struct lu_object *o)
1177{
1178 list_move(&o->lo_linkage, &before->lo_linkage);
1179}
1180EXPORT_SYMBOL(lu_object_add);
1181
1182
1183
1184
1185int lu_object_header_init(struct lu_object_header *h)
1186{
1187 memset(h, 0, sizeof(*h));
1188 atomic_set(&h->loh_ref, 1);
1189 INIT_HLIST_NODE(&h->loh_hash);
1190 INIT_LIST_HEAD(&h->loh_lru);
1191 INIT_LIST_HEAD(&h->loh_layers);
1192 lu_ref_init(&h->loh_reference);
1193 return 0;
1194}
1195EXPORT_SYMBOL(lu_object_header_init);
1196
1197
1198
1199
1200void lu_object_header_fini(struct lu_object_header *h)
1201{
1202 LASSERT(list_empty(&h->loh_layers));
1203 LASSERT(list_empty(&h->loh_lru));
1204 LASSERT(hlist_unhashed(&h->loh_hash));
1205 lu_ref_fini(&h->loh_reference);
1206}
1207EXPORT_SYMBOL(lu_object_header_fini);
1208
1209
1210
1211
1212
1213struct lu_object *lu_object_locate(struct lu_object_header *h,
1214 const struct lu_device_type *dtype)
1215{
1216 struct lu_object *o;
1217
1218 list_for_each_entry(o, &h->loh_layers, lo_linkage) {
1219 if (o->lo_dev->ld_type == dtype)
1220 return o;
1221 }
1222 return NULL;
1223}
1224EXPORT_SYMBOL(lu_object_locate);
1225
1226
1227
1228
1229
1230
1231
1232
1233void lu_stack_fini(const struct lu_env *env, struct lu_device *top)
1234{
1235 struct lu_site *site = top->ld_site;
1236 struct lu_device *scan;
1237 struct lu_device *next;
1238
1239 lu_site_purge(env, site, ~0);
1240 for (scan = top; scan; scan = next) {
1241 next = scan->ld_type->ldt_ops->ldto_device_fini(env, scan);
1242 lu_ref_del(&scan->ld_reference, "lu-stack", &lu_site_init);
1243 lu_device_put(scan);
1244 }
1245
1246
1247 lu_site_purge(env, site, ~0);
1248
1249 for (scan = top; scan; scan = next) {
1250 const struct lu_device_type *ldt = scan->ld_type;
1251 struct obd_type *type;
1252
1253 next = ldt->ldt_ops->ldto_device_free(env, scan);
1254 type = ldt->ldt_obd_type;
1255 if (type) {
1256 type->typ_refcnt--;
1257 class_put_type(type);
1258 }
1259 }
1260}
1261EXPORT_SYMBOL(lu_stack_fini);
1262
1263enum {
1264
1265
1266
1267 LU_CONTEXT_KEY_NR = 40
1268};
1269
1270static struct lu_context_key *lu_keys[LU_CONTEXT_KEY_NR] = { NULL, };
1271
1272static DEFINE_SPINLOCK(lu_keys_guard);
1273
1274
1275
1276
1277
1278
1279
1280static unsigned key_set_version;
1281
1282
1283
1284
1285int lu_context_key_register(struct lu_context_key *key)
1286{
1287 int result;
1288 int i;
1289
1290 LASSERT(key->lct_init);
1291 LASSERT(key->lct_fini);
1292 LASSERT(key->lct_tags != 0);
1293
1294 result = -ENFILE;
1295 spin_lock(&lu_keys_guard);
1296 for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
1297 if (!lu_keys[i]) {
1298 key->lct_index = i;
1299 atomic_set(&key->lct_used, 1);
1300 lu_keys[i] = key;
1301 lu_ref_init(&key->lct_reference);
1302 result = 0;
1303 ++key_set_version;
1304 break;
1305 }
1306 }
1307 spin_unlock(&lu_keys_guard);
1308 return result;
1309}
1310EXPORT_SYMBOL(lu_context_key_register);
1311
1312static void key_fini(struct lu_context *ctx, int index)
1313{
1314 if (ctx->lc_value && ctx->lc_value[index]) {
1315 struct lu_context_key *key;
1316
1317 key = lu_keys[index];
1318 LASSERT(atomic_read(&key->lct_used) > 1);
1319
1320 key->lct_fini(ctx, key, ctx->lc_value[index]);
1321 lu_ref_del(&key->lct_reference, "ctx", ctx);
1322 atomic_dec(&key->lct_used);
1323
1324 if ((ctx->lc_tags & LCT_NOREF) == 0) {
1325#ifdef CONFIG_MODULE_UNLOAD
1326 LINVRNT(module_refcount(key->lct_owner) > 0);
1327#endif
1328 module_put(key->lct_owner);
1329 }
1330 ctx->lc_value[index] = NULL;
1331 }
1332}
1333
1334
1335
1336
1337void lu_context_key_degister(struct lu_context_key *key)
1338{
1339 LASSERT(atomic_read(&key->lct_used) >= 1);
1340 LINVRNT(0 <= key->lct_index && key->lct_index < ARRAY_SIZE(lu_keys));
1341
1342 lu_context_key_quiesce(key);
1343
1344 ++key_set_version;
1345 spin_lock(&lu_keys_guard);
1346 key_fini(&lu_shrink_env.le_ctx, key->lct_index);
1347 if (lu_keys[key->lct_index]) {
1348 lu_keys[key->lct_index] = NULL;
1349 lu_ref_fini(&key->lct_reference);
1350 }
1351 spin_unlock(&lu_keys_guard);
1352
1353 LASSERTF(atomic_read(&key->lct_used) == 1,
1354 "key has instances: %d\n",
1355 atomic_read(&key->lct_used));
1356}
1357EXPORT_SYMBOL(lu_context_key_degister);
1358
1359
1360
1361
1362
1363int lu_context_key_register_many(struct lu_context_key *k, ...)
1364{
1365 struct lu_context_key *key = k;
1366 va_list args;
1367 int result;
1368
1369 va_start(args, k);
1370 do {
1371 result = lu_context_key_register(key);
1372 if (result)
1373 break;
1374 key = va_arg(args, struct lu_context_key *);
1375 } while (key);
1376 va_end(args);
1377
1378 if (result != 0) {
1379 va_start(args, k);
1380 while (k != key) {
1381 lu_context_key_degister(k);
1382 k = va_arg(args, struct lu_context_key *);
1383 }
1384 va_end(args);
1385 }
1386
1387 return result;
1388}
1389EXPORT_SYMBOL(lu_context_key_register_many);
1390
1391
1392
1393
1394
1395void lu_context_key_degister_many(struct lu_context_key *k, ...)
1396{
1397 va_list args;
1398
1399 va_start(args, k);
1400 do {
1401 lu_context_key_degister(k);
1402 k = va_arg(args, struct lu_context_key*);
1403 } while (k);
1404 va_end(args);
1405}
1406EXPORT_SYMBOL(lu_context_key_degister_many);
1407
1408
1409
1410
1411void lu_context_key_revive_many(struct lu_context_key *k, ...)
1412{
1413 va_list args;
1414
1415 va_start(args, k);
1416 do {
1417 lu_context_key_revive(k);
1418 k = va_arg(args, struct lu_context_key*);
1419 } while (k);
1420 va_end(args);
1421}
1422EXPORT_SYMBOL(lu_context_key_revive_many);
1423
1424
1425
1426
1427void lu_context_key_quiesce_many(struct lu_context_key *k, ...)
1428{
1429 va_list args;
1430
1431 va_start(args, k);
1432 do {
1433 lu_context_key_quiesce(k);
1434 k = va_arg(args, struct lu_context_key*);
1435 } while (k);
1436 va_end(args);
1437}
1438EXPORT_SYMBOL(lu_context_key_quiesce_many);
1439
1440
1441
1442
1443void *lu_context_key_get(const struct lu_context *ctx,
1444 const struct lu_context_key *key)
1445{
1446 LINVRNT(ctx->lc_state == LCS_ENTERED);
1447 LINVRNT(0 <= key->lct_index && key->lct_index < ARRAY_SIZE(lu_keys));
1448 LASSERT(lu_keys[key->lct_index] == key);
1449 return ctx->lc_value[key->lct_index];
1450}
1451EXPORT_SYMBOL(lu_context_key_get);
1452
1453
1454
1455
1456static LIST_HEAD(lu_context_remembered);
1457
1458
1459
1460
1461
1462
1463void lu_context_key_quiesce(struct lu_context_key *key)
1464{
1465 struct lu_context *ctx;
1466
1467 if (!(key->lct_tags & LCT_QUIESCENT)) {
1468
1469
1470
1471 key->lct_tags |= LCT_QUIESCENT;
1472
1473
1474
1475 spin_lock(&lu_keys_guard);
1476 list_for_each_entry(ctx, &lu_context_remembered, lc_remember)
1477 key_fini(ctx, key->lct_index);
1478 spin_unlock(&lu_keys_guard);
1479 ++key_set_version;
1480 }
1481}
1482EXPORT_SYMBOL(lu_context_key_quiesce);
1483
1484void lu_context_key_revive(struct lu_context_key *key)
1485{
1486 key->lct_tags &= ~LCT_QUIESCENT;
1487 ++key_set_version;
1488}
1489EXPORT_SYMBOL(lu_context_key_revive);
1490
1491static void keys_fini(struct lu_context *ctx)
1492{
1493 int i;
1494
1495 if (!ctx->lc_value)
1496 return;
1497
1498 for (i = 0; i < ARRAY_SIZE(lu_keys); ++i)
1499 key_fini(ctx, i);
1500
1501 kfree(ctx->lc_value);
1502 ctx->lc_value = NULL;
1503}
1504
1505static int keys_fill(struct lu_context *ctx)
1506{
1507 int i;
1508
1509 LINVRNT(ctx->lc_value);
1510 for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
1511 struct lu_context_key *key;
1512
1513 key = lu_keys[i];
1514 if (!ctx->lc_value[i] && key &&
1515 (key->lct_tags & ctx->lc_tags) &&
1516
1517
1518
1519
1520 !(key->lct_tags & LCT_QUIESCENT)) {
1521 void *value;
1522
1523 LINVRNT(key->lct_init);
1524 LINVRNT(key->lct_index == i);
1525
1526 value = key->lct_init(ctx, key);
1527 if (IS_ERR(value))
1528 return PTR_ERR(value);
1529
1530 if (!(ctx->lc_tags & LCT_NOREF))
1531 try_module_get(key->lct_owner);
1532 lu_ref_add_atomic(&key->lct_reference, "ctx", ctx);
1533 atomic_inc(&key->lct_used);
1534
1535
1536
1537
1538
1539 ctx->lc_value[i] = value;
1540 if (key->lct_exit)
1541 ctx->lc_tags |= LCT_HAS_EXIT;
1542 }
1543 ctx->lc_version = key_set_version;
1544 }
1545 return 0;
1546}
1547
1548static int keys_init(struct lu_context *ctx)
1549{
1550 ctx->lc_value = kcalloc(ARRAY_SIZE(lu_keys), sizeof(ctx->lc_value[0]),
1551 GFP_NOFS);
1552 if (likely(ctx->lc_value))
1553 return keys_fill(ctx);
1554
1555 return -ENOMEM;
1556}
1557
1558
1559
1560
1561int lu_context_init(struct lu_context *ctx, __u32 tags)
1562{
1563 int rc;
1564
1565 memset(ctx, 0, sizeof(*ctx));
1566 ctx->lc_state = LCS_INITIALIZED;
1567 ctx->lc_tags = tags;
1568 if (tags & LCT_REMEMBER) {
1569 spin_lock(&lu_keys_guard);
1570 list_add(&ctx->lc_remember, &lu_context_remembered);
1571 spin_unlock(&lu_keys_guard);
1572 } else {
1573 INIT_LIST_HEAD(&ctx->lc_remember);
1574 }
1575
1576 rc = keys_init(ctx);
1577 if (rc != 0)
1578 lu_context_fini(ctx);
1579
1580 return rc;
1581}
1582EXPORT_SYMBOL(lu_context_init);
1583
1584
1585
1586
1587void lu_context_fini(struct lu_context *ctx)
1588{
1589 LINVRNT(ctx->lc_state == LCS_INITIALIZED || ctx->lc_state == LCS_LEFT);
1590 ctx->lc_state = LCS_FINALIZED;
1591
1592 if ((ctx->lc_tags & LCT_REMEMBER) == 0) {
1593 LASSERT(list_empty(&ctx->lc_remember));
1594 keys_fini(ctx);
1595
1596 } else {
1597 spin_lock(&lu_keys_guard);
1598 keys_fini(ctx);
1599 list_del_init(&ctx->lc_remember);
1600 spin_unlock(&lu_keys_guard);
1601 }
1602}
1603EXPORT_SYMBOL(lu_context_fini);
1604
1605
1606
1607
1608void lu_context_enter(struct lu_context *ctx)
1609{
1610 LINVRNT(ctx->lc_state == LCS_INITIALIZED || ctx->lc_state == LCS_LEFT);
1611 ctx->lc_state = LCS_ENTERED;
1612}
1613EXPORT_SYMBOL(lu_context_enter);
1614
1615
1616
1617
1618void lu_context_exit(struct lu_context *ctx)
1619{
1620 int i;
1621
1622 LINVRNT(ctx->lc_state == LCS_ENTERED);
1623 ctx->lc_state = LCS_LEFT;
1624 if (ctx->lc_tags & LCT_HAS_EXIT && ctx->lc_value) {
1625 for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
1626 if (ctx->lc_value[i]) {
1627 struct lu_context_key *key;
1628
1629 key = lu_keys[i];
1630 if (key->lct_exit)
1631 key->lct_exit(ctx,
1632 key, ctx->lc_value[i]);
1633 }
1634 }
1635 }
1636}
1637EXPORT_SYMBOL(lu_context_exit);
1638
1639
1640
1641
1642
1643
1644int lu_context_refill(struct lu_context *ctx)
1645{
1646 return likely(ctx->lc_version == key_set_version) ? 0 : keys_fill(ctx);
1647}
1648EXPORT_SYMBOL(lu_context_refill);
1649
1650
1651
1652
1653
1654
1655
1656
1657__u32 lu_context_tags_default;
1658__u32 lu_session_tags_default;
1659
1660int lu_env_init(struct lu_env *env, __u32 tags)
1661{
1662 int result;
1663
1664 env->le_ses = NULL;
1665 result = lu_context_init(&env->le_ctx, tags);
1666 if (likely(result == 0))
1667 lu_context_enter(&env->le_ctx);
1668 return result;
1669}
1670EXPORT_SYMBOL(lu_env_init);
1671
1672void lu_env_fini(struct lu_env *env)
1673{
1674 lu_context_exit(&env->le_ctx);
1675 lu_context_fini(&env->le_ctx);
1676 env->le_ses = NULL;
1677}
1678EXPORT_SYMBOL(lu_env_fini);
1679
1680int lu_env_refill(struct lu_env *env)
1681{
1682 int result;
1683
1684 result = lu_context_refill(&env->le_ctx);
1685 if (result == 0 && env->le_ses)
1686 result = lu_context_refill(env->le_ses);
1687 return result;
1688}
1689EXPORT_SYMBOL(lu_env_refill);
1690
1691struct lu_site_stats {
1692 unsigned lss_populated;
1693 unsigned lss_max_search;
1694 unsigned lss_total;
1695 unsigned lss_busy;
1696};
1697
1698static void lu_site_stats_get(struct cfs_hash *hs,
1699 struct lu_site_stats *stats, int populated)
1700{
1701 struct cfs_hash_bd bd;
1702 int i;
1703
1704 cfs_hash_for_each_bucket(hs, &bd, i) {
1705 struct lu_site_bkt_data *bkt = cfs_hash_bd_extra_get(hs, &bd);
1706 struct hlist_head *hhead;
1707
1708 cfs_hash_bd_lock(hs, &bd, 1);
1709 stats->lss_busy +=
1710 cfs_hash_bd_count_get(&bd) - bkt->lsb_lru_len;
1711 stats->lss_total += cfs_hash_bd_count_get(&bd);
1712 stats->lss_max_search = max((int)stats->lss_max_search,
1713 cfs_hash_bd_depmax_get(&bd));
1714 if (!populated) {
1715 cfs_hash_bd_unlock(hs, &bd, 1);
1716 continue;
1717 }
1718
1719 cfs_hash_bd_for_each_hlist(hs, &bd, hhead) {
1720 if (!hlist_empty(hhead))
1721 stats->lss_populated++;
1722 }
1723 cfs_hash_bd_unlock(hs, &bd, 1);
1724 }
1725}
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740static unsigned long lu_cache_shrink_count(struct shrinker *sk,
1741 struct shrink_control *sc)
1742{
1743 struct lu_site *s;
1744 struct lu_site *tmp;
1745 unsigned long cached = 0;
1746
1747 if (!(sc->gfp_mask & __GFP_FS))
1748 return 0;
1749
1750 mutex_lock(&lu_sites_guard);
1751 list_for_each_entry_safe(s, tmp, &lu_sites, ls_linkage) {
1752 cached += ls_stats_read(s->ls_stats, LU_SS_LRU_LEN);
1753 }
1754 mutex_unlock(&lu_sites_guard);
1755
1756 cached = (cached / 100) * sysctl_vfs_cache_pressure;
1757 CDEBUG(D_INODE, "%ld objects cached, cache pressure %d\n",
1758 cached, sysctl_vfs_cache_pressure);
1759
1760 return cached;
1761}
1762
1763static unsigned long lu_cache_shrink_scan(struct shrinker *sk,
1764 struct shrink_control *sc)
1765{
1766 struct lu_site *s;
1767 struct lu_site *tmp;
1768 unsigned long remain = sc->nr_to_scan, freed = 0;
1769 LIST_HEAD(splice);
1770
1771 if (!(sc->gfp_mask & __GFP_FS))
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783 return SHRINK_STOP;
1784
1785 mutex_lock(&lu_sites_guard);
1786 list_for_each_entry_safe(s, tmp, &lu_sites, ls_linkage) {
1787 freed = lu_site_purge(&lu_shrink_env, s, remain);
1788 remain -= freed;
1789
1790
1791
1792
1793 list_move_tail(&s->ls_linkage, &splice);
1794 }
1795 list_splice(&splice, lu_sites.prev);
1796 mutex_unlock(&lu_sites_guard);
1797
1798 return sc->nr_to_scan - remain;
1799}
1800
1801
1802
1803
1804static struct shrinker lu_site_shrinker = {
1805 .count_objects = lu_cache_shrink_count,
1806 .scan_objects = lu_cache_shrink_scan,
1807 .seeks = DEFAULT_SEEKS,
1808};
1809
1810
1811
1812
1813int lu_global_init(void)
1814{
1815 int result;
1816
1817 CDEBUG(D_INFO, "Lustre LU module (%p).\n", &lu_keys);
1818
1819 result = lu_ref_global_init();
1820 if (result != 0)
1821 return result;
1822
1823 LU_CONTEXT_KEY_INIT(&lu_global_key);
1824 result = lu_context_key_register(&lu_global_key);
1825 if (result != 0)
1826 return result;
1827
1828
1829
1830
1831
1832
1833 mutex_lock(&lu_sites_guard);
1834 result = lu_env_init(&lu_shrink_env, LCT_SHRINKER);
1835 mutex_unlock(&lu_sites_guard);
1836 if (result != 0)
1837 return result;
1838
1839
1840
1841
1842
1843
1844 register_shrinker(&lu_site_shrinker);
1845
1846 return result;
1847}
1848
1849
1850
1851
1852void lu_global_fini(void)
1853{
1854 unregister_shrinker(&lu_site_shrinker);
1855 lu_context_key_degister(&lu_global_key);
1856
1857
1858
1859
1860
1861 mutex_lock(&lu_sites_guard);
1862 lu_env_fini(&lu_shrink_env);
1863 mutex_unlock(&lu_sites_guard);
1864
1865 lu_ref_global_fini();
1866}
1867
1868static __u32 ls_stats_read(struct lprocfs_stats *stats, int idx)
1869{
1870 struct lprocfs_counter ret;
1871
1872 lprocfs_stats_collect(stats, idx, &ret);
1873 if (idx == LU_SS_LRU_LEN)
1874
1875
1876
1877
1878 return (__u32)((ret.lc_sum > 0) ? ret.lc_sum : 0);
1879
1880 return (__u32)ret.lc_count;
1881}
1882
1883
1884
1885
1886
1887int lu_site_stats_print(const struct lu_site *s, struct seq_file *m)
1888{
1889 struct lu_site_stats stats;
1890
1891 memset(&stats, 0, sizeof(stats));
1892 lu_site_stats_get(s->ls_obj_hash, &stats, 1);
1893
1894 seq_printf(m, "%d/%d %d/%d %d %d %d %d %d %d %d %d\n",
1895 stats.lss_busy,
1896 stats.lss_total,
1897 stats.lss_populated,
1898 CFS_HASH_NHLIST(s->ls_obj_hash),
1899 stats.lss_max_search,
1900 ls_stats_read(s->ls_stats, LU_SS_CREATED),
1901 ls_stats_read(s->ls_stats, LU_SS_CACHE_HIT),
1902 ls_stats_read(s->ls_stats, LU_SS_CACHE_MISS),
1903 ls_stats_read(s->ls_stats, LU_SS_CACHE_RACE),
1904 ls_stats_read(s->ls_stats, LU_SS_CACHE_DEATH_RACE),
1905 ls_stats_read(s->ls_stats, LU_SS_LRU_PURGED),
1906 ls_stats_read(s->ls_stats, LU_SS_LRU_LEN));
1907 return 0;
1908}
1909EXPORT_SYMBOL(lu_site_stats_print);
1910
1911
1912
1913
1914int lu_kmem_init(struct lu_kmem_descr *caches)
1915{
1916 int result;
1917 struct lu_kmem_descr *iter = caches;
1918
1919 for (result = 0; iter->ckd_cache; ++iter) {
1920 *iter->ckd_cache = kmem_cache_create(iter->ckd_name,
1921 iter->ckd_size,
1922 0, 0, NULL);
1923 if (!*iter->ckd_cache) {
1924 result = -ENOMEM;
1925
1926 lu_kmem_fini(caches);
1927 break;
1928 }
1929 }
1930 return result;
1931}
1932EXPORT_SYMBOL(lu_kmem_init);
1933
1934
1935
1936
1937
1938void lu_kmem_fini(struct lu_kmem_descr *caches)
1939{
1940 for (; caches->ckd_cache; ++caches) {
1941 kmem_cache_destroy(*caches->ckd_cache);
1942 *caches->ckd_cache = NULL;
1943 }
1944}
1945EXPORT_SYMBOL(lu_kmem_fini);
1946