1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#define FSCACHE_DEBUG_LEVEL COOKIE
16#include <linux/module.h>
17#include <linux/slab.h>
18#include "internal.h"
19
20struct kmem_cache *fscache_cookie_jar;
21
22static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
23
24#define fscache_cookie_hash_shift 15
25static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
26
27static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
28 loff_t object_size);
29static int fscache_alloc_object(struct fscache_cache *cache,
30 struct fscache_cookie *cookie);
31static int fscache_attach_object(struct fscache_cookie *cookie,
32 struct fscache_object *object);
33
34static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
35{
36 struct hlist_node *object;
37 const u8 *k;
38 unsigned loop;
39
40 pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
41 prefix, cookie, cookie->parent, cookie->flags,
42 atomic_read(&cookie->n_children),
43 atomic_read(&cookie->n_active));
44 pr_err("%c-cookie d=%p n=%p\n",
45 prefix, cookie->def, cookie->netfs_data);
46
47 object = READ_ONCE(cookie->backing_objects.first);
48 if (object)
49 pr_err("%c-cookie o=%p\n",
50 prefix, hlist_entry(object, struct fscache_object, cookie_link));
51
52 pr_err("%c-key=[%u] '", prefix, cookie->key_len);
53 k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
54 cookie->inline_key : cookie->key;
55 for (loop = 0; loop < cookie->key_len; loop++)
56 pr_cont("%02x", k[loop]);
57 pr_cont("'\n");
58}
59
60void fscache_free_cookie(struct fscache_cookie *cookie)
61{
62 if (cookie) {
63 BUG_ON(!hlist_empty(&cookie->backing_objects));
64 if (cookie->aux_len > sizeof(cookie->inline_aux))
65 kfree(cookie->aux);
66 if (cookie->key_len > sizeof(cookie->inline_key))
67 kfree(cookie->key);
68 kmem_cache_free(fscache_cookie_jar, cookie);
69 }
70}
71
72
73
74
75void fscache_cookie_init_once(void *_cookie)
76{
77 struct fscache_cookie *cookie = _cookie;
78
79 memset(cookie, 0, sizeof(*cookie));
80 spin_lock_init(&cookie->lock);
81 spin_lock_init(&cookie->stores_lock);
82 INIT_HLIST_HEAD(&cookie->backing_objects);
83}
84
85
86
87
88
89
90
91static int fscache_set_key(struct fscache_cookie *cookie,
92 const void *index_key, size_t index_key_len)
93{
94 unsigned long long h;
95 u32 *buf;
96 int i;
97
98 cookie->key_len = index_key_len;
99
100 if (index_key_len > sizeof(cookie->inline_key)) {
101 buf = kzalloc(index_key_len, GFP_KERNEL);
102 if (!buf)
103 return -ENOMEM;
104 cookie->key = buf;
105 } else {
106 buf = (u32 *)cookie->inline_key;
107 buf[0] = 0;
108 buf[1] = 0;
109 buf[2] = 0;
110 }
111
112 memcpy(buf, index_key, index_key_len);
113
114
115
116
117 h = (unsigned long)cookie->parent;
118 h += index_key_len + cookie->type;
119 for (i = 0; i < (index_key_len + sizeof(u32) - 1) / sizeof(u32); i++)
120 h += buf[i];
121
122 cookie->key_hash = h ^ (h >> 32);
123 return 0;
124}
125
126static long fscache_compare_cookie(const struct fscache_cookie *a,
127 const struct fscache_cookie *b)
128{
129 const void *ka, *kb;
130
131 if (a->key_hash != b->key_hash)
132 return (long)a->key_hash - (long)b->key_hash;
133 if (a->parent != b->parent)
134 return (long)a->parent - (long)b->parent;
135 if (a->key_len != b->key_len)
136 return (long)a->key_len - (long)b->key_len;
137 if (a->type != b->type)
138 return (long)a->type - (long)b->type;
139
140 if (a->key_len <= sizeof(a->inline_key)) {
141 ka = &a->inline_key;
142 kb = &b->inline_key;
143 } else {
144 ka = a->key;
145 kb = b->key;
146 }
147 return memcmp(ka, kb, a->key_len);
148}
149
150
151
152
153struct fscache_cookie *fscache_alloc_cookie(
154 struct fscache_cookie *parent,
155 const struct fscache_cookie_def *def,
156 const void *index_key, size_t index_key_len,
157 const void *aux_data, size_t aux_data_len,
158 void *netfs_data,
159 loff_t object_size)
160{
161 struct fscache_cookie *cookie;
162
163
164 cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
165 if (!cookie)
166 return NULL;
167
168 cookie->key_len = index_key_len;
169 cookie->aux_len = aux_data_len;
170
171 if (fscache_set_key(cookie, index_key, index_key_len) < 0)
172 goto nomem;
173
174 if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
175 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
176 } else {
177 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
178 if (!cookie->aux)
179 goto nomem;
180 }
181
182 atomic_set(&cookie->usage, 1);
183 atomic_set(&cookie->n_children, 0);
184
185
186
187
188 atomic_set(&cookie->n_active, 1);
189
190 cookie->def = def;
191 cookie->parent = parent;
192 cookie->netfs_data = netfs_data;
193 cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
194 cookie->type = def->type;
195
196
197
198 INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
199 return cookie;
200
201nomem:
202 fscache_free_cookie(cookie);
203 return NULL;
204}
205
206
207
208
209
210struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
211{
212 struct fscache_cookie *cursor;
213 struct hlist_bl_head *h;
214 struct hlist_bl_node *p;
215 unsigned int bucket;
216
217 bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
218 h = &fscache_cookie_hash[bucket];
219
220 hlist_bl_lock(h);
221 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
222 if (fscache_compare_cookie(candidate, cursor) == 0)
223 goto collision;
224 }
225
226 __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
227 fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
228 atomic_inc(&candidate->parent->n_children);
229 hlist_bl_add_head(&candidate->hash_link, h);
230 hlist_bl_unlock(h);
231 return candidate;
232
233collision:
234 if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
235 trace_fscache_cookie(cursor, fscache_cookie_collision,
236 atomic_read(&cursor->usage));
237 pr_err("Duplicate cookie detected\n");
238 fscache_print_cookie(cursor, 'O');
239 fscache_print_cookie(candidate, 'N');
240 hlist_bl_unlock(h);
241 return NULL;
242 }
243
244 fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
245 hlist_bl_unlock(h);
246 return cursor;
247}
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264struct fscache_cookie *__fscache_acquire_cookie(
265 struct fscache_cookie *parent,
266 const struct fscache_cookie_def *def,
267 const void *index_key, size_t index_key_len,
268 const void *aux_data, size_t aux_data_len,
269 void *netfs_data,
270 loff_t object_size,
271 bool enable)
272{
273 struct fscache_cookie *candidate, *cookie;
274
275 BUG_ON(!def);
276
277 _enter("{%s},{%s},%p,%u",
278 parent ? (char *) parent->def->name : "<no-parent>",
279 def->name, netfs_data, enable);
280
281 if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
282 return NULL;
283 if (!aux_data || !aux_data_len) {
284 aux_data = NULL;
285 aux_data_len = 0;
286 }
287
288 fscache_stat(&fscache_n_acquires);
289
290
291 if (!parent) {
292 fscache_stat(&fscache_n_acquires_null);
293 _leave(" [no parent]");
294 return NULL;
295 }
296
297
298 BUG_ON(!def->name[0]);
299
300 BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
301 parent->type != FSCACHE_COOKIE_TYPE_INDEX);
302
303 candidate = fscache_alloc_cookie(parent, def,
304 index_key, index_key_len,
305 aux_data, aux_data_len,
306 netfs_data, object_size);
307 if (!candidate) {
308 fscache_stat(&fscache_n_acquires_oom);
309 _leave(" [ENOMEM]");
310 return NULL;
311 }
312
313 cookie = fscache_hash_cookie(candidate);
314 if (!cookie) {
315 trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
316 goto out;
317 }
318
319 if (cookie == candidate)
320 candidate = NULL;
321
322 switch (cookie->type) {
323 case FSCACHE_COOKIE_TYPE_INDEX:
324 fscache_stat(&fscache_n_cookie_index);
325 break;
326 case FSCACHE_COOKIE_TYPE_DATAFILE:
327 fscache_stat(&fscache_n_cookie_data);
328 break;
329 default:
330 fscache_stat(&fscache_n_cookie_special);
331 break;
332 }
333
334 trace_fscache_acquire(cookie);
335
336 if (enable) {
337
338
339
340 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
341 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
342 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
343 } else {
344 atomic_dec(&parent->n_children);
345 fscache_cookie_put(cookie,
346 fscache_cookie_put_acquire_nobufs);
347 fscache_stat(&fscache_n_acquires_nobufs);
348 _leave(" = NULL");
349 return NULL;
350 }
351 } else {
352 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
353 }
354 }
355
356 fscache_stat(&fscache_n_acquires_ok);
357
358out:
359 fscache_free_cookie(candidate);
360 return cookie;
361}
362EXPORT_SYMBOL(__fscache_acquire_cookie);
363
364
365
366
367void __fscache_enable_cookie(struct fscache_cookie *cookie,
368 const void *aux_data,
369 loff_t object_size,
370 bool (*can_enable)(void *data),
371 void *data)
372{
373 _enter("%p", cookie);
374
375 trace_fscache_enable(cookie);
376
377 wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
378 TASK_UNINTERRUPTIBLE);
379
380 fscache_update_aux(cookie, aux_data);
381
382 if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
383 goto out_unlock;
384
385 if (can_enable && !can_enable(data)) {
386
387 } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
388
389 __fscache_wait_on_invalidate(cookie);
390
391 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
392 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
393 } else {
394 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
395 }
396
397out_unlock:
398 clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
399 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
400}
401EXPORT_SYMBOL(__fscache_enable_cookie);
402
403
404
405
406
407
408static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
409 loff_t object_size)
410{
411 struct fscache_object *object;
412 struct fscache_cache *cache;
413 int ret;
414
415 _enter("");
416
417 set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
418
419
420
421 down_read(&fscache_addremove_sem);
422
423 if (list_empty(&fscache_cache_list)) {
424 up_read(&fscache_addremove_sem);
425 _leave(" = 0 [no caches]");
426 return 0;
427 }
428
429
430 cache = fscache_select_cache_for_object(cookie->parent);
431 if (!cache) {
432 up_read(&fscache_addremove_sem);
433 fscache_stat(&fscache_n_acquires_no_cache);
434 _leave(" = -ENOMEDIUM [no cache]");
435 return -ENOMEDIUM;
436 }
437
438 _debug("cache %s", cache->tag->name);
439
440 set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
441
442
443
444 ret = fscache_alloc_object(cache, cookie);
445 if (ret < 0) {
446 up_read(&fscache_addremove_sem);
447 _leave(" = %d", ret);
448 return ret;
449 }
450
451 spin_lock(&cookie->lock);
452 if (hlist_empty(&cookie->backing_objects)) {
453 spin_unlock(&cookie->lock);
454 goto unavailable;
455 }
456
457 object = hlist_entry(cookie->backing_objects.first,
458 struct fscache_object, cookie_link);
459
460 fscache_set_store_limit(object, object_size);
461
462
463
464 fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
465
466 spin_unlock(&cookie->lock);
467
468
469 if (!fscache_defer_lookup) {
470 _debug("non-deferred lookup %p", &cookie->flags);
471 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
472 TASK_UNINTERRUPTIBLE);
473 _debug("complete");
474 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
475 goto unavailable;
476 }
477
478 up_read(&fscache_addremove_sem);
479 _leave(" = 0 [deferred]");
480 return 0;
481
482unavailable:
483 up_read(&fscache_addremove_sem);
484 _leave(" = -ENOBUFS");
485 return -ENOBUFS;
486}
487
488
489
490
491
492static int fscache_alloc_object(struct fscache_cache *cache,
493 struct fscache_cookie *cookie)
494{
495 struct fscache_object *object;
496 int ret;
497
498 _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
499
500 spin_lock(&cookie->lock);
501 hlist_for_each_entry(object, &cookie->backing_objects,
502 cookie_link) {
503 if (object->cache == cache)
504 goto object_already_extant;
505 }
506 spin_unlock(&cookie->lock);
507
508
509
510 fscache_stat(&fscache_n_cop_alloc_object);
511 object = cache->ops->alloc_object(cache, cookie);
512 fscache_stat_d(&fscache_n_cop_alloc_object);
513 if (IS_ERR(object)) {
514 fscache_stat(&fscache_n_object_no_alloc);
515 ret = PTR_ERR(object);
516 goto error;
517 }
518
519 fscache_stat(&fscache_n_object_alloc);
520
521 object->debug_id = atomic_inc_return(&fscache_object_debug_id);
522
523 _debug("ALLOC OBJ%x: %s {%lx}",
524 object->debug_id, cookie->def->name, object->events);
525
526 ret = fscache_alloc_object(cache, cookie->parent);
527 if (ret < 0)
528 goto error_put;
529
530
531
532
533 if (fscache_attach_object(cookie, object) < 0) {
534 fscache_stat(&fscache_n_cop_put_object);
535 cache->ops->put_object(object, fscache_obj_put_attach_fail);
536 fscache_stat_d(&fscache_n_cop_put_object);
537 }
538
539 _leave(" = 0");
540 return 0;
541
542object_already_extant:
543 ret = -ENOBUFS;
544 if (fscache_object_is_dying(object) ||
545 fscache_cache_is_broken(object)) {
546 spin_unlock(&cookie->lock);
547 goto error;
548 }
549 spin_unlock(&cookie->lock);
550 _leave(" = 0 [found]");
551 return 0;
552
553error_put:
554 fscache_stat(&fscache_n_cop_put_object);
555 cache->ops->put_object(object, fscache_obj_put_alloc_fail);
556 fscache_stat_d(&fscache_n_cop_put_object);
557error:
558 _leave(" = %d", ret);
559 return ret;
560}
561
562
563
564
565static int fscache_attach_object(struct fscache_cookie *cookie,
566 struct fscache_object *object)
567{
568 struct fscache_object *p;
569 struct fscache_cache *cache = object->cache;
570 int ret;
571
572 _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
573
574 spin_lock(&cookie->lock);
575
576
577
578 ret = -EEXIST;
579 hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
580 if (p->cache == object->cache) {
581 if (fscache_object_is_dying(p))
582 ret = -ENOBUFS;
583 goto cant_attach_object;
584 }
585 }
586
587
588 spin_lock_nested(&cookie->parent->lock, 1);
589 hlist_for_each_entry(p, &cookie->parent->backing_objects,
590 cookie_link) {
591 if (p->cache == object->cache) {
592 if (fscache_object_is_dying(p)) {
593 ret = -ENOBUFS;
594 spin_unlock(&cookie->parent->lock);
595 goto cant_attach_object;
596 }
597 object->parent = p;
598 spin_lock(&p->lock);
599 p->n_children++;
600 spin_unlock(&p->lock);
601 break;
602 }
603 }
604 spin_unlock(&cookie->parent->lock);
605
606
607 if (list_empty(&object->cache_link)) {
608 spin_lock(&cache->object_list_lock);
609 list_add(&object->cache_link, &cache->object_list);
610 spin_unlock(&cache->object_list_lock);
611 }
612
613
614 object->cookie = cookie;
615 fscache_cookie_get(cookie, fscache_cookie_get_attach_object);
616 hlist_add_head(&object->cookie_link, &cookie->backing_objects);
617
618 fscache_objlist_add(object);
619 ret = 0;
620
621cant_attach_object:
622 spin_unlock(&cookie->lock);
623 _leave(" = %d", ret);
624 return ret;
625}
626
627
628
629
630void __fscache_invalidate(struct fscache_cookie *cookie)
631{
632 struct fscache_object *object;
633
634 _enter("{%s}", cookie->def->name);
635
636 fscache_stat(&fscache_n_invalidates);
637
638
639
640
641
642
643 ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
644
645
646
647
648 if (!hlist_empty(&cookie->backing_objects)) {
649 spin_lock(&cookie->lock);
650
651 if (fscache_cookie_enabled(cookie) &&
652 !hlist_empty(&cookie->backing_objects) &&
653 !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
654 &cookie->flags)) {
655 object = hlist_entry(cookie->backing_objects.first,
656 struct fscache_object,
657 cookie_link);
658 if (fscache_object_is_live(object))
659 fscache_raise_event(
660 object, FSCACHE_OBJECT_EV_INVALIDATE);
661 }
662
663 spin_unlock(&cookie->lock);
664 }
665
666 _leave("");
667}
668EXPORT_SYMBOL(__fscache_invalidate);
669
670
671
672
673void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
674{
675 _enter("%p", cookie);
676
677 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
678 TASK_UNINTERRUPTIBLE);
679
680 _leave("");
681}
682EXPORT_SYMBOL(__fscache_wait_on_invalidate);
683
684
685
686
687void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
688{
689 struct fscache_object *object;
690
691 fscache_stat(&fscache_n_updates);
692
693 if (!cookie) {
694 fscache_stat(&fscache_n_updates_null);
695 _leave(" [no cookie]");
696 return;
697 }
698
699 _enter("{%s}", cookie->def->name);
700
701 spin_lock(&cookie->lock);
702
703 fscache_update_aux(cookie, aux_data);
704
705 if (fscache_cookie_enabled(cookie)) {
706
707
708
709 hlist_for_each_entry(object,
710 &cookie->backing_objects, cookie_link) {
711 fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
712 }
713 }
714
715 spin_unlock(&cookie->lock);
716 _leave("");
717}
718EXPORT_SYMBOL(__fscache_update_cookie);
719
720
721
722
723void __fscache_disable_cookie(struct fscache_cookie *cookie,
724 const void *aux_data,
725 bool invalidate)
726{
727 struct fscache_object *object;
728 bool awaken = false;
729
730 _enter("%p,%u", cookie, invalidate);
731
732 trace_fscache_disable(cookie);
733
734 ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
735
736 if (atomic_read(&cookie->n_children) != 0) {
737 pr_err("Cookie '%s' still has children\n",
738 cookie->def->name);
739 BUG();
740 }
741
742 wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
743 TASK_UNINTERRUPTIBLE);
744
745 fscache_update_aux(cookie, aux_data);
746
747 if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
748 goto out_unlock_enable;
749
750
751
752
753 __fscache_wait_on_invalidate(cookie);
754
755
756 set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
757
758 spin_lock(&cookie->lock);
759 if (!hlist_empty(&cookie->backing_objects)) {
760 hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
761 if (invalidate)
762 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
763 clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
764 fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
765 }
766 } else {
767 if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
768 awaken = true;
769 }
770 spin_unlock(&cookie->lock);
771 if (awaken)
772 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
773
774
775
776
777
778 if (!atomic_dec_and_test(&cookie->n_active)) {
779 wait_var_event(&cookie->n_active,
780 !atomic_read(&cookie->n_active));
781 }
782
783
784 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
785 fscache_invalidate_writes(cookie);
786
787
788 if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
789 atomic_inc(&cookie->n_active);
790 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
791 }
792
793out_unlock_enable:
794 clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
795 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
796 _leave("");
797}
798EXPORT_SYMBOL(__fscache_disable_cookie);
799
800
801
802
803
804
805
806void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
807 const void *aux_data,
808 bool retire)
809{
810 fscache_stat(&fscache_n_relinquishes);
811 if (retire)
812 fscache_stat(&fscache_n_relinquishes_retire);
813
814 if (!cookie) {
815 fscache_stat(&fscache_n_relinquishes_null);
816 _leave(" [no cookie]");
817 return;
818 }
819
820 _enter("%p{%s,%p,%d},%d",
821 cookie, cookie->def->name, cookie->netfs_data,
822 atomic_read(&cookie->n_active), retire);
823
824 trace_fscache_relinquish(cookie, retire);
825
826
827 if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
828 BUG();
829
830 __fscache_disable_cookie(cookie, aux_data, retire);
831
832
833 cookie->netfs_data = NULL;
834 cookie->def = NULL;
835 BUG_ON(!radix_tree_empty(&cookie->stores));
836
837 if (cookie->parent) {
838 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
839 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
840 atomic_dec(&cookie->parent->n_children);
841 }
842
843
844 ASSERTCMP(atomic_read(&cookie->usage), >, 0);
845 fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
846
847 _leave("");
848}
849EXPORT_SYMBOL(__fscache_relinquish_cookie);
850
851
852
853
854static void fscache_unhash_cookie(struct fscache_cookie *cookie)
855{
856 struct hlist_bl_head *h;
857 unsigned int bucket;
858
859 bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
860 h = &fscache_cookie_hash[bucket];
861
862 hlist_bl_lock(h);
863 hlist_bl_del(&cookie->hash_link);
864 hlist_bl_unlock(h);
865}
866
867
868
869
870void fscache_cookie_put(struct fscache_cookie *cookie,
871 enum fscache_cookie_trace where)
872{
873 struct fscache_cookie *parent;
874 int usage;
875
876 _enter("%p", cookie);
877
878 do {
879 usage = atomic_dec_return(&cookie->usage);
880 trace_fscache_cookie(cookie, where, usage);
881
882 if (usage > 0)
883 return;
884 BUG_ON(usage < 0);
885
886 parent = cookie->parent;
887 fscache_unhash_cookie(cookie);
888 fscache_free_cookie(cookie);
889
890 cookie = parent;
891 where = fscache_cookie_put_parent;
892 } while (cookie);
893
894 _leave("");
895}
896
897
898
899
900
901
902int __fscache_check_consistency(struct fscache_cookie *cookie,
903 const void *aux_data)
904{
905 struct fscache_operation *op;
906 struct fscache_object *object;
907 bool wake_cookie = false;
908 int ret;
909
910 _enter("%p,", cookie);
911
912 ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
913
914 if (fscache_wait_for_deferred_lookup(cookie) < 0)
915 return -ERESTARTSYS;
916
917 if (hlist_empty(&cookie->backing_objects))
918 return 0;
919
920 op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
921 if (!op)
922 return -ENOMEM;
923
924 fscache_operation_init(cookie, op, NULL, NULL, NULL);
925 op->flags = FSCACHE_OP_MYTHREAD |
926 (1 << FSCACHE_OP_WAITING) |
927 (1 << FSCACHE_OP_UNUSE_COOKIE);
928 trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
929
930 spin_lock(&cookie->lock);
931
932 fscache_update_aux(cookie, aux_data);
933
934 if (!fscache_cookie_enabled(cookie) ||
935 hlist_empty(&cookie->backing_objects))
936 goto inconsistent;
937 object = hlist_entry(cookie->backing_objects.first,
938 struct fscache_object, cookie_link);
939 if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
940 goto inconsistent;
941
942 op->debug_id = atomic_inc_return(&fscache_op_debug_id);
943
944 __fscache_use_cookie(cookie);
945 if (fscache_submit_op(object, op) < 0)
946 goto submit_failed;
947
948
949 spin_unlock(&cookie->lock);
950
951 ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
952 if (ret == 0) {
953
954 ret = object->cache->ops->check_consistency(op);
955 fscache_op_complete(op, false);
956 } else if (ret == -ENOBUFS) {
957 ret = 0;
958 }
959
960 fscache_put_operation(op);
961 _leave(" = %d", ret);
962 return ret;
963
964submit_failed:
965 wake_cookie = __fscache_unuse_cookie(cookie);
966inconsistent:
967 spin_unlock(&cookie->lock);
968 if (wake_cookie)
969 __fscache_wake_unused_cookie(cookie);
970 kfree(op);
971 _leave(" = -ESTALE");
972 return -ESTALE;
973}
974EXPORT_SYMBOL(__fscache_check_consistency);
975