1
2
3
4
5
6
7
8
9
10
11#define FSCACHE_DEBUG_LEVEL COOKIE
12#include <linux/module.h>
13#include <linux/slab.h>
14#include "internal.h"
15
16struct kmem_cache *fscache_cookie_jar;
17
18static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
19
20#define fscache_cookie_hash_shift 15
21static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
22static LIST_HEAD(fscache_cookies);
23static DEFINE_RWLOCK(fscache_cookies_lock);
24
25static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
26 loff_t object_size);
27static int fscache_alloc_object(struct fscache_cache *cache,
28 struct fscache_cookie *cookie);
29static int fscache_attach_object(struct fscache_cookie *cookie,
30 struct fscache_object *object);
31
32static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
33{
34 struct fscache_object *object;
35 struct hlist_node *o;
36 const u8 *k;
37 unsigned loop;
38
39 pr_err("%c-cookie c=%08x [p=%08x fl=%lx nc=%u na=%u]\n",
40 prefix,
41 cookie->debug_id,
42 cookie->parent ? cookie->parent->debug_id : 0,
43 cookie->flags,
44 atomic_read(&cookie->n_children),
45 atomic_read(&cookie->n_active));
46 pr_err("%c-cookie d=%p{%s} n=%p\n",
47 prefix,
48 cookie->def,
49 cookie->def ? cookie->def->name : "?",
50 cookie->netfs_data);
51
52 o = READ_ONCE(cookie->backing_objects.first);
53 if (o) {
54 object = hlist_entry(o, struct fscache_object, cookie_link);
55 pr_err("%c-cookie o=%u\n", prefix, object->debug_id);
56 }
57
58 pr_err("%c-key=[%u] '", prefix, cookie->key_len);
59 k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
60 cookie->inline_key : cookie->key;
61 for (loop = 0; loop < cookie->key_len; loop++)
62 pr_cont("%02x", k[loop]);
63 pr_cont("'\n");
64}
65
66void fscache_free_cookie(struct fscache_cookie *cookie)
67{
68 if (cookie) {
69 BUG_ON(!hlist_empty(&cookie->backing_objects));
70 write_lock(&fscache_cookies_lock);
71 list_del(&cookie->proc_link);
72 write_unlock(&fscache_cookies_lock);
73 if (cookie->aux_len > sizeof(cookie->inline_aux))
74 kfree(cookie->aux);
75 if (cookie->key_len > sizeof(cookie->inline_key))
76 kfree(cookie->key);
77 kmem_cache_free(fscache_cookie_jar, cookie);
78 }
79}
80
81
82
83
84
85
86
87static int fscache_set_key(struct fscache_cookie *cookie,
88 const void *index_key, size_t index_key_len)
89{
90 u32 *buf;
91 int bufs;
92
93 bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
94
95 if (index_key_len > sizeof(cookie->inline_key)) {
96 buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
97 if (!buf)
98 return -ENOMEM;
99 cookie->key = buf;
100 } else {
101 buf = (u32 *)cookie->inline_key;
102 }
103
104 memcpy(buf, index_key, index_key_len);
105 cookie->key_hash = fscache_hash(0, buf, bufs);
106 return 0;
107}
108
109static long fscache_compare_cookie(const struct fscache_cookie *a,
110 const struct fscache_cookie *b)
111{
112 const void *ka, *kb;
113
114 if (a->key_hash != b->key_hash)
115 return (long)a->key_hash - (long)b->key_hash;
116 if (a->parent != b->parent)
117 return (long)a->parent - (long)b->parent;
118 if (a->key_len != b->key_len)
119 return (long)a->key_len - (long)b->key_len;
120 if (a->type != b->type)
121 return (long)a->type - (long)b->type;
122
123 if (a->key_len <= sizeof(a->inline_key)) {
124 ka = &a->inline_key;
125 kb = &b->inline_key;
126 } else {
127 ka = a->key;
128 kb = b->key;
129 }
130 return memcmp(ka, kb, a->key_len);
131}
132
133static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1);
134
135
136
137
138struct fscache_cookie *fscache_alloc_cookie(
139 struct fscache_cookie *parent,
140 const struct fscache_cookie_def *def,
141 const void *index_key, size_t index_key_len,
142 const void *aux_data, size_t aux_data_len,
143 void *netfs_data,
144 loff_t object_size)
145{
146 struct fscache_cookie *cookie;
147
148
149 cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
150 if (!cookie)
151 return NULL;
152
153 cookie->key_len = index_key_len;
154 cookie->aux_len = aux_data_len;
155
156 if (fscache_set_key(cookie, index_key, index_key_len) < 0)
157 goto nomem;
158
159 if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
160 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
161 } else {
162 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
163 if (!cookie->aux)
164 goto nomem;
165 }
166
167 refcount_set(&cookie->ref, 1);
168 atomic_set(&cookie->n_children, 0);
169 cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id);
170
171
172
173
174 atomic_set(&cookie->n_active, 1);
175
176 cookie->def = def;
177 cookie->parent = parent;
178 cookie->netfs_data = netfs_data;
179 cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
180 cookie->type = def->type;
181 spin_lock_init(&cookie->lock);
182 spin_lock_init(&cookie->stores_lock);
183 INIT_HLIST_HEAD(&cookie->backing_objects);
184
185
186
187 INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
188
189 write_lock(&fscache_cookies_lock);
190 list_add_tail(&cookie->proc_link, &fscache_cookies);
191 write_unlock(&fscache_cookies_lock);
192 return cookie;
193
194nomem:
195 fscache_free_cookie(cookie);
196 return NULL;
197}
198
199
200
201
202
203struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
204{
205 struct fscache_cookie *cursor;
206 struct hlist_bl_head *h;
207 struct hlist_bl_node *p;
208 unsigned int bucket;
209
210 bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
211 h = &fscache_cookie_hash[bucket];
212
213 hlist_bl_lock(h);
214 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
215 if (fscache_compare_cookie(candidate, cursor) == 0)
216 goto collision;
217 }
218
219 __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
220 fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
221 atomic_inc(&candidate->parent->n_children);
222 hlist_bl_add_head(&candidate->hash_link, h);
223 hlist_bl_unlock(h);
224 return candidate;
225
226collision:
227 if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
228 trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref),
229 fscache_cookie_collision);
230 pr_err("Duplicate cookie detected\n");
231 fscache_print_cookie(cursor, 'O');
232 fscache_print_cookie(candidate, 'N');
233 hlist_bl_unlock(h);
234 return NULL;
235 }
236
237 fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
238 hlist_bl_unlock(h);
239 return cursor;
240}
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257struct fscache_cookie *__fscache_acquire_cookie(
258 struct fscache_cookie *parent,
259 const struct fscache_cookie_def *def,
260 const void *index_key, size_t index_key_len,
261 const void *aux_data, size_t aux_data_len,
262 void *netfs_data,
263 loff_t object_size,
264 bool enable)
265{
266 struct fscache_cookie *candidate, *cookie;
267
268 BUG_ON(!def);
269
270 _enter("{%s},{%s},%p,%u",
271 parent ? (char *) parent->def->name : "<no-parent>",
272 def->name, netfs_data, enable);
273
274 if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
275 return NULL;
276 if (!aux_data || !aux_data_len) {
277 aux_data = NULL;
278 aux_data_len = 0;
279 }
280
281 fscache_stat(&fscache_n_acquires);
282
283
284 if (!parent) {
285 fscache_stat(&fscache_n_acquires_null);
286 _leave(" [no parent]");
287 return NULL;
288 }
289
290
291 BUG_ON(!def->name[0]);
292
293 BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
294 parent->type != FSCACHE_COOKIE_TYPE_INDEX);
295
296 candidate = fscache_alloc_cookie(parent, def,
297 index_key, index_key_len,
298 aux_data, aux_data_len,
299 netfs_data, object_size);
300 if (!candidate) {
301 fscache_stat(&fscache_n_acquires_oom);
302 _leave(" [ENOMEM]");
303 return NULL;
304 }
305
306 cookie = fscache_hash_cookie(candidate);
307 if (!cookie) {
308 trace_fscache_cookie(candidate->debug_id, 1,
309 fscache_cookie_discard);
310 goto out;
311 }
312
313 if (cookie == candidate)
314 candidate = NULL;
315
316 switch (cookie->type) {
317 case FSCACHE_COOKIE_TYPE_INDEX:
318 fscache_stat(&fscache_n_cookie_index);
319 break;
320 case FSCACHE_COOKIE_TYPE_DATAFILE:
321 fscache_stat(&fscache_n_cookie_data);
322 break;
323 default:
324 fscache_stat(&fscache_n_cookie_special);
325 break;
326 }
327
328 trace_fscache_acquire(cookie);
329
330 if (enable) {
331
332
333
334 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
335 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
336 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
337 } else {
338 atomic_dec(&parent->n_children);
339 fscache_cookie_put(cookie,
340 fscache_cookie_put_acquire_nobufs);
341 fscache_stat(&fscache_n_acquires_nobufs);
342 _leave(" = NULL");
343 return NULL;
344 }
345 } else {
346 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
347 }
348 }
349
350 fscache_stat(&fscache_n_acquires_ok);
351
352out:
353 fscache_free_cookie(candidate);
354 return cookie;
355}
356EXPORT_SYMBOL(__fscache_acquire_cookie);
357
358
359
360
361void __fscache_enable_cookie(struct fscache_cookie *cookie,
362 const void *aux_data,
363 loff_t object_size,
364 bool (*can_enable)(void *data),
365 void *data)
366{
367 _enter("%x", cookie->debug_id);
368
369 trace_fscache_enable(cookie);
370
371 wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
372 TASK_UNINTERRUPTIBLE);
373
374 fscache_update_aux(cookie, aux_data);
375
376 if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
377 goto out_unlock;
378
379 if (can_enable && !can_enable(data)) {
380
381 } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
382
383 __fscache_wait_on_invalidate(cookie);
384
385 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
386 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
387 } else {
388 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
389 }
390
391out_unlock:
392 clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
393 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
394}
395EXPORT_SYMBOL(__fscache_enable_cookie);
396
397
398
399
400
401
402static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
403 loff_t object_size)
404{
405 struct fscache_object *object;
406 struct fscache_cache *cache;
407 int ret;
408
409 _enter("");
410
411 set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
412
413
414
415 down_read(&fscache_addremove_sem);
416
417 if (list_empty(&fscache_cache_list)) {
418 up_read(&fscache_addremove_sem);
419 _leave(" = 0 [no caches]");
420 return 0;
421 }
422
423
424 cache = fscache_select_cache_for_object(cookie->parent);
425 if (!cache) {
426 up_read(&fscache_addremove_sem);
427 fscache_stat(&fscache_n_acquires_no_cache);
428 _leave(" = -ENOMEDIUM [no cache]");
429 return -ENOMEDIUM;
430 }
431
432 _debug("cache %s", cache->tag->name);
433
434 set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
435
436
437
438 ret = fscache_alloc_object(cache, cookie);
439 if (ret < 0) {
440 up_read(&fscache_addremove_sem);
441 _leave(" = %d", ret);
442 return ret;
443 }
444
445 spin_lock(&cookie->lock);
446 if (hlist_empty(&cookie->backing_objects)) {
447 spin_unlock(&cookie->lock);
448 goto unavailable;
449 }
450
451 object = hlist_entry(cookie->backing_objects.first,
452 struct fscache_object, cookie_link);
453
454 fscache_set_store_limit(object, object_size);
455
456
457
458 fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
459
460 spin_unlock(&cookie->lock);
461
462
463 if (!fscache_defer_lookup) {
464 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
465 TASK_UNINTERRUPTIBLE);
466 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
467 goto unavailable;
468 }
469
470 up_read(&fscache_addremove_sem);
471 _leave(" = 0 [deferred]");
472 return 0;
473
474unavailable:
475 up_read(&fscache_addremove_sem);
476 _leave(" = -ENOBUFS");
477 return -ENOBUFS;
478}
479
480
481
482
483
484static int fscache_alloc_object(struct fscache_cache *cache,
485 struct fscache_cookie *cookie)
486{
487 struct fscache_object *object;
488 int ret;
489
490 _enter("%s,%x{%s}", cache->tag->name, cookie->debug_id, cookie->def->name);
491
492 spin_lock(&cookie->lock);
493 hlist_for_each_entry(object, &cookie->backing_objects,
494 cookie_link) {
495 if (object->cache == cache)
496 goto object_already_extant;
497 }
498 spin_unlock(&cookie->lock);
499
500
501
502 fscache_stat(&fscache_n_cop_alloc_object);
503 object = cache->ops->alloc_object(cache, cookie);
504 fscache_stat_d(&fscache_n_cop_alloc_object);
505 if (IS_ERR(object)) {
506 fscache_stat(&fscache_n_object_no_alloc);
507 ret = PTR_ERR(object);
508 goto error;
509 }
510
511 ASSERTCMP(object->cookie, ==, cookie);
512 fscache_stat(&fscache_n_object_alloc);
513
514 object->debug_id = atomic_inc_return(&fscache_object_debug_id);
515
516 _debug("ALLOC OBJ%x: %s {%lx}",
517 object->debug_id, cookie->def->name, object->events);
518
519 ret = fscache_alloc_object(cache, cookie->parent);
520 if (ret < 0)
521 goto error_put;
522
523
524
525
526 if (fscache_attach_object(cookie, object) < 0) {
527 fscache_stat(&fscache_n_cop_put_object);
528 cache->ops->put_object(object, fscache_obj_put_attach_fail);
529 fscache_stat_d(&fscache_n_cop_put_object);
530 }
531
532 _leave(" = 0");
533 return 0;
534
535object_already_extant:
536 ret = -ENOBUFS;
537 if (fscache_object_is_dying(object) ||
538 fscache_cache_is_broken(object)) {
539 spin_unlock(&cookie->lock);
540 goto error;
541 }
542 spin_unlock(&cookie->lock);
543 _leave(" = 0 [found]");
544 return 0;
545
546error_put:
547 fscache_stat(&fscache_n_cop_put_object);
548 cache->ops->put_object(object, fscache_obj_put_alloc_fail);
549 fscache_stat_d(&fscache_n_cop_put_object);
550error:
551 _leave(" = %d", ret);
552 return ret;
553}
554
555
556
557
558static int fscache_attach_object(struct fscache_cookie *cookie,
559 struct fscache_object *object)
560{
561 struct fscache_object *p;
562 struct fscache_cache *cache = object->cache;
563 int ret;
564
565 _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
566
567 ASSERTCMP(object->cookie, ==, cookie);
568
569 spin_lock(&cookie->lock);
570
571
572
573 ret = -EEXIST;
574 hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
575 if (p->cache == object->cache) {
576 if (fscache_object_is_dying(p))
577 ret = -ENOBUFS;
578 goto cant_attach_object;
579 }
580 }
581
582
583 spin_lock_nested(&cookie->parent->lock, 1);
584 hlist_for_each_entry(p, &cookie->parent->backing_objects,
585 cookie_link) {
586 if (p->cache == object->cache) {
587 if (fscache_object_is_dying(p)) {
588 ret = -ENOBUFS;
589 spin_unlock(&cookie->parent->lock);
590 goto cant_attach_object;
591 }
592 object->parent = p;
593 spin_lock(&p->lock);
594 p->n_children++;
595 spin_unlock(&p->lock);
596 break;
597 }
598 }
599 spin_unlock(&cookie->parent->lock);
600
601
602 if (list_empty(&object->cache_link)) {
603 spin_lock(&cache->object_list_lock);
604 list_add(&object->cache_link, &cache->object_list);
605 spin_unlock(&cache->object_list_lock);
606 }
607
608
609 hlist_add_head(&object->cookie_link, &cookie->backing_objects);
610 ret = 0;
611
612cant_attach_object:
613 spin_unlock(&cookie->lock);
614 _leave(" = %d", ret);
615 return ret;
616}
617
618
619
620
621void __fscache_invalidate(struct fscache_cookie *cookie)
622{
623 struct fscache_object *object;
624
625 _enter("{%s}", cookie->def->name);
626
627 fscache_stat(&fscache_n_invalidates);
628
629
630
631
632
633
634 ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
635
636
637
638
639 if (!hlist_empty(&cookie->backing_objects)) {
640 spin_lock(&cookie->lock);
641
642 if (fscache_cookie_enabled(cookie) &&
643 !hlist_empty(&cookie->backing_objects) &&
644 !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
645 &cookie->flags)) {
646 object = hlist_entry(cookie->backing_objects.first,
647 struct fscache_object,
648 cookie_link);
649 if (fscache_object_is_live(object))
650 fscache_raise_event(
651 object, FSCACHE_OBJECT_EV_INVALIDATE);
652 }
653
654 spin_unlock(&cookie->lock);
655 }
656
657 _leave("");
658}
659EXPORT_SYMBOL(__fscache_invalidate);
660
661
662
663
664void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
665{
666 _enter("%x", cookie->debug_id);
667
668 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
669 TASK_UNINTERRUPTIBLE);
670
671 _leave("");
672}
673EXPORT_SYMBOL(__fscache_wait_on_invalidate);
674
675
676
677
678void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
679{
680 struct fscache_object *object;
681
682 fscache_stat(&fscache_n_updates);
683
684 if (!cookie) {
685 fscache_stat(&fscache_n_updates_null);
686 _leave(" [no cookie]");
687 return;
688 }
689
690 _enter("{%s}", cookie->def->name);
691
692 spin_lock(&cookie->lock);
693
694 fscache_update_aux(cookie, aux_data);
695
696 if (fscache_cookie_enabled(cookie)) {
697
698
699
700 hlist_for_each_entry(object,
701 &cookie->backing_objects, cookie_link) {
702 fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
703 }
704 }
705
706 spin_unlock(&cookie->lock);
707 _leave("");
708}
709EXPORT_SYMBOL(__fscache_update_cookie);
710
711
712
713
714void __fscache_disable_cookie(struct fscache_cookie *cookie,
715 const void *aux_data,
716 bool invalidate)
717{
718 struct fscache_object *object;
719 bool awaken = false;
720
721 _enter("%x,%u", cookie->debug_id, invalidate);
722
723 trace_fscache_disable(cookie);
724
725 ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
726
727 if (atomic_read(&cookie->n_children) != 0) {
728 pr_err("Cookie '%s' still has children\n",
729 cookie->def->name);
730 BUG();
731 }
732
733 wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
734 TASK_UNINTERRUPTIBLE);
735
736 fscache_update_aux(cookie, aux_data);
737
738 if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
739 goto out_unlock_enable;
740
741
742
743
744 __fscache_wait_on_invalidate(cookie);
745
746
747 set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
748
749 spin_lock(&cookie->lock);
750 if (!hlist_empty(&cookie->backing_objects)) {
751 hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
752 if (invalidate)
753 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
754 clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
755 fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
756 }
757 } else {
758 if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
759 awaken = true;
760 }
761 spin_unlock(&cookie->lock);
762 if (awaken)
763 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
764
765
766
767
768
769 if (!atomic_dec_and_test(&cookie->n_active)) {
770 wait_var_event(&cookie->n_active,
771 !atomic_read(&cookie->n_active));
772 }
773
774
775 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
776 fscache_invalidate_writes(cookie);
777
778
779 if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
780 atomic_inc(&cookie->n_active);
781 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
782 }
783
784out_unlock_enable:
785 clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
786 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
787 _leave("");
788}
789EXPORT_SYMBOL(__fscache_disable_cookie);
790
791
792
793
794
795
796
797void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
798 const void *aux_data,
799 bool retire)
800{
801 fscache_stat(&fscache_n_relinquishes);
802 if (retire)
803 fscache_stat(&fscache_n_relinquishes_retire);
804
805 if (!cookie) {
806 fscache_stat(&fscache_n_relinquishes_null);
807 _leave(" [no cookie]");
808 return;
809 }
810
811 _enter("%x{%s,%d},%d",
812 cookie->debug_id, cookie->def->name,
813 atomic_read(&cookie->n_active), retire);
814
815 trace_fscache_relinquish(cookie, retire);
816
817
818 if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
819 BUG();
820
821 __fscache_disable_cookie(cookie, aux_data, retire);
822
823
824 cookie->netfs_data = NULL;
825 cookie->def = NULL;
826 BUG_ON(!radix_tree_empty(&cookie->stores));
827
828 if (cookie->parent) {
829 ASSERTCMP(refcount_read(&cookie->parent->ref), >, 0);
830 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
831 atomic_dec(&cookie->parent->n_children);
832 }
833
834
835 fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
836
837 _leave("");
838}
839EXPORT_SYMBOL(__fscache_relinquish_cookie);
840
841
842
843
844static void fscache_unhash_cookie(struct fscache_cookie *cookie)
845{
846 struct hlist_bl_head *h;
847 unsigned int bucket;
848
849 bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
850 h = &fscache_cookie_hash[bucket];
851
852 hlist_bl_lock(h);
853 hlist_bl_del(&cookie->hash_link);
854 hlist_bl_unlock(h);
855}
856
857
858
859
860void fscache_cookie_put(struct fscache_cookie *cookie,
861 enum fscache_cookie_trace where)
862{
863 struct fscache_cookie *parent;
864 int ref;
865
866 _enter("%x", cookie->debug_id);
867
868 do {
869 unsigned int cookie_debug_id = cookie->debug_id;
870 bool zero = __refcount_dec_and_test(&cookie->ref, &ref);
871
872 trace_fscache_cookie(cookie_debug_id, ref - 1, where);
873 if (!zero)
874 return;
875
876 parent = cookie->parent;
877 fscache_unhash_cookie(cookie);
878 fscache_free_cookie(cookie);
879
880 cookie = parent;
881 where = fscache_cookie_put_parent;
882 } while (cookie);
883
884 _leave("");
885}
886
887
888
889
890struct fscache_cookie *fscache_cookie_get(struct fscache_cookie *cookie,
891 enum fscache_cookie_trace where)
892{
893 int ref;
894
895 __refcount_inc(&cookie->ref, &ref);
896 trace_fscache_cookie(cookie->debug_id, ref + 1, where);
897 return cookie;
898}
899
900
901
902
903
904
905int __fscache_check_consistency(struct fscache_cookie *cookie,
906 const void *aux_data)
907{
908 struct fscache_operation *op;
909 struct fscache_object *object;
910 bool wake_cookie = false;
911 int ret;
912
913 _enter("%p,", cookie);
914
915 ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
916
917 if (fscache_wait_for_deferred_lookup(cookie) < 0)
918 return -ERESTARTSYS;
919
920 if (hlist_empty(&cookie->backing_objects))
921 return 0;
922
923 op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
924 if (!op)
925 return -ENOMEM;
926
927 fscache_operation_init(cookie, op, NULL, NULL, NULL);
928 op->flags = FSCACHE_OP_MYTHREAD |
929 (1 << FSCACHE_OP_WAITING) |
930 (1 << FSCACHE_OP_UNUSE_COOKIE);
931 trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
932
933 spin_lock(&cookie->lock);
934
935 fscache_update_aux(cookie, aux_data);
936
937 if (!fscache_cookie_enabled(cookie) ||
938 hlist_empty(&cookie->backing_objects))
939 goto inconsistent;
940 object = hlist_entry(cookie->backing_objects.first,
941 struct fscache_object, cookie_link);
942 if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
943 goto inconsistent;
944
945 op->debug_id = atomic_inc_return(&fscache_op_debug_id);
946
947 __fscache_use_cookie(cookie);
948 if (fscache_submit_op(object, op) < 0)
949 goto submit_failed;
950
951
952 spin_unlock(&cookie->lock);
953
954 ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
955 if (ret == 0) {
956
957 ret = object->cache->ops->check_consistency(op);
958 fscache_op_complete(op, false);
959 } else if (ret == -ENOBUFS) {
960 ret = 0;
961 }
962
963 fscache_put_operation(op);
964 _leave(" = %d", ret);
965 return ret;
966
967submit_failed:
968 wake_cookie = __fscache_unuse_cookie(cookie);
969inconsistent:
970 spin_unlock(&cookie->lock);
971 if (wake_cookie)
972 __fscache_wake_unused_cookie(cookie);
973 kfree(op);
974 _leave(" = -ESTALE");
975 return -ESTALE;
976}
977EXPORT_SYMBOL(__fscache_check_consistency);
978
979
980
981
982static int fscache_cookies_seq_show(struct seq_file *m, void *v)
983{
984 struct fscache_cookie *cookie;
985 unsigned int keylen = 0, auxlen = 0;
986 char _type[3], *type;
987 u8 *p;
988
989 if (v == &fscache_cookies) {
990 seq_puts(m,
991 "COOKIE PARENT USAGE CHILD ACT TY FL DEF NETFS_DATA\n"
992 "======== ======== ===== ===== === == === ================ ==========\n"
993 );
994 return 0;
995 }
996
997 cookie = list_entry(v, struct fscache_cookie, proc_link);
998
999 switch (cookie->type) {
1000 case 0:
1001 type = "IX";
1002 break;
1003 case 1:
1004 type = "DT";
1005 break;
1006 default:
1007 snprintf(_type, sizeof(_type), "%02u",
1008 cookie->type);
1009 type = _type;
1010 break;
1011 }
1012
1013 seq_printf(m,
1014 "%08x %08x %5u %5u %3u %s %03lx %-16s %px",
1015 cookie->debug_id,
1016 cookie->parent ? cookie->parent->debug_id : 0,
1017 refcount_read(&cookie->ref),
1018 atomic_read(&cookie->n_children),
1019 atomic_read(&cookie->n_active),
1020 type,
1021 cookie->flags,
1022 cookie->def->name,
1023 cookie->netfs_data);
1024
1025 keylen = cookie->key_len;
1026 auxlen = cookie->aux_len;
1027
1028 if (keylen > 0 || auxlen > 0) {
1029 seq_puts(m, " ");
1030 p = keylen <= sizeof(cookie->inline_key) ?
1031 cookie->inline_key : cookie->key;
1032 for (; keylen > 0; keylen--)
1033 seq_printf(m, "%02x", *p++);
1034 if (auxlen > 0) {
1035 seq_puts(m, ", ");
1036 p = auxlen <= sizeof(cookie->inline_aux) ?
1037 cookie->inline_aux : cookie->aux;
1038 for (; auxlen > 0; auxlen--)
1039 seq_printf(m, "%02x", *p++);
1040 }
1041 }
1042
1043 seq_puts(m, "\n");
1044 return 0;
1045}
1046
1047static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos)
1048 __acquires(fscache_cookies_lock)
1049{
1050 read_lock(&fscache_cookies_lock);
1051 return seq_list_start_head(&fscache_cookies, *_pos);
1052}
1053
1054static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos)
1055{
1056 return seq_list_next(v, &fscache_cookies, _pos);
1057}
1058
1059static void fscache_cookies_seq_stop(struct seq_file *m, void *v)
1060 __releases(rcu)
1061{
1062 read_unlock(&fscache_cookies_lock);
1063}
1064
1065
1066const struct seq_operations fscache_cookies_seq_ops = {
1067 .start = fscache_cookies_seq_start,
1068 .next = fscache_cookies_seq_next,
1069 .stop = fscache_cookies_seq_stop,
1070 .show = fscache_cookies_seq_show,
1071};
1072