linux/fs/fscache/cookie.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/* netfs cookie management
   3 *
   4 * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
   5 * Written by David Howells (dhowells@redhat.com)
   6 *
   7 * See Documentation/filesystems/caching/netfs-api.txt for more information on
   8 * the netfs API.
   9 */
  10
  11#define FSCACHE_DEBUG_LEVEL COOKIE
  12#include <linux/module.h>
  13#include <linux/slab.h>
  14#include "internal.h"
  15
  16struct kmem_cache *fscache_cookie_jar;
  17
  18static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
  19
  20#define fscache_cookie_hash_shift 15
  21static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
  22
  23static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
  24                                            loff_t object_size);
  25static int fscache_alloc_object(struct fscache_cache *cache,
  26                                struct fscache_cookie *cookie);
  27static int fscache_attach_object(struct fscache_cookie *cookie,
  28                                 struct fscache_object *object);
  29
  30static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
  31{
  32        struct hlist_node *object;
  33        const u8 *k;
  34        unsigned loop;
  35
  36        pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
  37               prefix, cookie, cookie->parent, cookie->flags,
  38               atomic_read(&cookie->n_children),
  39               atomic_read(&cookie->n_active));
  40        pr_err("%c-cookie d=%p n=%p\n",
  41               prefix, cookie->def, cookie->netfs_data);
  42
  43        object = READ_ONCE(cookie->backing_objects.first);
  44        if (object)
  45                pr_err("%c-cookie o=%p\n",
  46                       prefix, hlist_entry(object, struct fscache_object, cookie_link));
  47
  48        pr_err("%c-key=[%u] '", prefix, cookie->key_len);
  49        k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
  50                cookie->inline_key : cookie->key;
  51        for (loop = 0; loop < cookie->key_len; loop++)
  52                pr_cont("%02x", k[loop]);
  53        pr_cont("'\n");
  54}
  55
  56void fscache_free_cookie(struct fscache_cookie *cookie)
  57{
  58        if (cookie) {
  59                BUG_ON(!hlist_empty(&cookie->backing_objects));
  60                if (cookie->aux_len > sizeof(cookie->inline_aux))
  61                        kfree(cookie->aux);
  62                if (cookie->key_len > sizeof(cookie->inline_key))
  63                        kfree(cookie->key);
  64                kmem_cache_free(fscache_cookie_jar, cookie);
  65        }
  66}
  67
  68/*
  69 * Set the index key in a cookie.  The cookie struct has space for a 16-byte
  70 * key plus length and hash, but if that's not big enough, it's instead a
  71 * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
  72 * the key data.
  73 */
  74static int fscache_set_key(struct fscache_cookie *cookie,
  75                           const void *index_key, size_t index_key_len)
  76{
  77        unsigned long long h;
  78        u32 *buf;
  79        int bufs;
  80        int i;
  81
  82        bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
  83
  84        if (index_key_len > sizeof(cookie->inline_key)) {
  85                buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
  86                if (!buf)
  87                        return -ENOMEM;
  88                cookie->key = buf;
  89        } else {
  90                buf = (u32 *)cookie->inline_key;
  91        }
  92
  93        memcpy(buf, index_key, index_key_len);
  94
  95        /* Calculate a hash and combine this with the length in the first word
  96         * or first half word
  97         */
  98        h = (unsigned long)cookie->parent;
  99        h += index_key_len + cookie->type;
 100
 101        for (i = 0; i < bufs; i++)
 102                h += buf[i];
 103
 104        cookie->key_hash = h ^ (h >> 32);
 105        return 0;
 106}
 107
 108static long fscache_compare_cookie(const struct fscache_cookie *a,
 109                                   const struct fscache_cookie *b)
 110{
 111        const void *ka, *kb;
 112
 113        if (a->key_hash != b->key_hash)
 114                return (long)a->key_hash - (long)b->key_hash;
 115        if (a->parent != b->parent)
 116                return (long)a->parent - (long)b->parent;
 117        if (a->key_len != b->key_len)
 118                return (long)a->key_len - (long)b->key_len;
 119        if (a->type != b->type)
 120                return (long)a->type - (long)b->type;
 121
 122        if (a->key_len <= sizeof(a->inline_key)) {
 123                ka = &a->inline_key;
 124                kb = &b->inline_key;
 125        } else {
 126                ka = a->key;
 127                kb = b->key;
 128        }
 129        return memcmp(ka, kb, a->key_len);
 130}
 131
 132/*
 133 * Allocate a cookie.
 134 */
 135struct fscache_cookie *fscache_alloc_cookie(
 136        struct fscache_cookie *parent,
 137        const struct fscache_cookie_def *def,
 138        const void *index_key, size_t index_key_len,
 139        const void *aux_data, size_t aux_data_len,
 140        void *netfs_data,
 141        loff_t object_size)
 142{
 143        struct fscache_cookie *cookie;
 144
 145        /* allocate and initialise a cookie */
 146        cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
 147        if (!cookie)
 148                return NULL;
 149
 150        cookie->key_len = index_key_len;
 151        cookie->aux_len = aux_data_len;
 152
 153        if (fscache_set_key(cookie, index_key, index_key_len) < 0)
 154                goto nomem;
 155
 156        if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
 157                memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
 158        } else {
 159                cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
 160                if (!cookie->aux)
 161                        goto nomem;
 162        }
 163
 164        atomic_set(&cookie->usage, 1);
 165        atomic_set(&cookie->n_children, 0);
 166
 167        /* We keep the active count elevated until relinquishment to prevent an
 168         * attempt to wake up every time the object operations queue quiesces.
 169         */
 170        atomic_set(&cookie->n_active, 1);
 171
 172        cookie->def             = def;
 173        cookie->parent          = parent;
 174        cookie->netfs_data      = netfs_data;
 175        cookie->flags           = (1 << FSCACHE_COOKIE_NO_DATA_YET);
 176        cookie->type            = def->type;
 177        spin_lock_init(&cookie->lock);
 178        spin_lock_init(&cookie->stores_lock);
 179        INIT_HLIST_HEAD(&cookie->backing_objects);
 180
 181        /* radix tree insertion won't use the preallocation pool unless it's
 182         * told it may not wait */
 183        INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
 184        return cookie;
 185
 186nomem:
 187        fscache_free_cookie(cookie);
 188        return NULL;
 189}
 190
 191/*
 192 * Attempt to insert the new cookie into the hash.  If there's a collision, we
 193 * return the old cookie if it's not in use and an error otherwise.
 194 */
 195struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
 196{
 197        struct fscache_cookie *cursor;
 198        struct hlist_bl_head *h;
 199        struct hlist_bl_node *p;
 200        unsigned int bucket;
 201
 202        bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 203        h = &fscache_cookie_hash[bucket];
 204
 205        hlist_bl_lock(h);
 206        hlist_bl_for_each_entry(cursor, p, h, hash_link) {
 207                if (fscache_compare_cookie(candidate, cursor) == 0)
 208                        goto collision;
 209        }
 210
 211        __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
 212        fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
 213        atomic_inc(&candidate->parent->n_children);
 214        hlist_bl_add_head(&candidate->hash_link, h);
 215        hlist_bl_unlock(h);
 216        return candidate;
 217
 218collision:
 219        if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
 220                trace_fscache_cookie(cursor, fscache_cookie_collision,
 221                                     atomic_read(&cursor->usage));
 222                pr_err("Duplicate cookie detected\n");
 223                fscache_print_cookie(cursor, 'O');
 224                fscache_print_cookie(candidate, 'N');
 225                hlist_bl_unlock(h);
 226                return NULL;
 227        }
 228
 229        fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
 230        hlist_bl_unlock(h);
 231        return cursor;
 232}
 233
 234/*
 235 * request a cookie to represent an object (index, datafile, xattr, etc)
 236 * - parent specifies the parent object
 237 *   - the top level index cookie for each netfs is stored in the fscache_netfs
 238 *     struct upon registration
 239 * - def points to the definition
 240 * - the netfs_data will be passed to the functions pointed to in *def
 241 * - all attached caches will be searched to see if they contain this object
 242 * - index objects aren't stored on disk until there's a dependent file that
 243 *   needs storing
 244 * - other objects are stored in a selected cache immediately, and all the
 245 *   indices forming the path to it are instantiated if necessary
 246 * - we never let on to the netfs about errors
 247 *   - we may set a negative cookie pointer, but that's okay
 248 */
 249struct fscache_cookie *__fscache_acquire_cookie(
 250        struct fscache_cookie *parent,
 251        const struct fscache_cookie_def *def,
 252        const void *index_key, size_t index_key_len,
 253        const void *aux_data, size_t aux_data_len,
 254        void *netfs_data,
 255        loff_t object_size,
 256        bool enable)
 257{
 258        struct fscache_cookie *candidate, *cookie;
 259
 260        BUG_ON(!def);
 261
 262        _enter("{%s},{%s},%p,%u",
 263               parent ? (char *) parent->def->name : "<no-parent>",
 264               def->name, netfs_data, enable);
 265
 266        if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
 267                return NULL;
 268        if (!aux_data || !aux_data_len) {
 269                aux_data = NULL;
 270                aux_data_len = 0;
 271        }
 272
 273        fscache_stat(&fscache_n_acquires);
 274
 275        /* if there's no parent cookie, then we don't create one here either */
 276        if (!parent) {
 277                fscache_stat(&fscache_n_acquires_null);
 278                _leave(" [no parent]");
 279                return NULL;
 280        }
 281
 282        /* validate the definition */
 283        BUG_ON(!def->name[0]);
 284
 285        BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
 286               parent->type != FSCACHE_COOKIE_TYPE_INDEX);
 287
 288        candidate = fscache_alloc_cookie(parent, def,
 289                                         index_key, index_key_len,
 290                                         aux_data, aux_data_len,
 291                                         netfs_data, object_size);
 292        if (!candidate) {
 293                fscache_stat(&fscache_n_acquires_oom);
 294                _leave(" [ENOMEM]");
 295                return NULL;
 296        }
 297
 298        cookie = fscache_hash_cookie(candidate);
 299        if (!cookie) {
 300                trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
 301                goto out;
 302        }
 303
 304        if (cookie == candidate)
 305                candidate = NULL;
 306
 307        switch (cookie->type) {
 308        case FSCACHE_COOKIE_TYPE_INDEX:
 309                fscache_stat(&fscache_n_cookie_index);
 310                break;
 311        case FSCACHE_COOKIE_TYPE_DATAFILE:
 312                fscache_stat(&fscache_n_cookie_data);
 313                break;
 314        default:
 315                fscache_stat(&fscache_n_cookie_special);
 316                break;
 317        }
 318
 319        trace_fscache_acquire(cookie);
 320
 321        if (enable) {
 322                /* if the object is an index then we need do nothing more here
 323                 * - we create indices on disk when we need them as an index
 324                 * may exist in multiple caches */
 325                if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 326                        if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
 327                                set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 328                        } else {
 329                                atomic_dec(&parent->n_children);
 330                                fscache_cookie_put(cookie,
 331                                                   fscache_cookie_put_acquire_nobufs);
 332                                fscache_stat(&fscache_n_acquires_nobufs);
 333                                _leave(" = NULL");
 334                                return NULL;
 335                        }
 336                } else {
 337                        set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 338                }
 339        }
 340
 341        fscache_stat(&fscache_n_acquires_ok);
 342
 343out:
 344        fscache_free_cookie(candidate);
 345        return cookie;
 346}
 347EXPORT_SYMBOL(__fscache_acquire_cookie);
 348
 349/*
 350 * Enable a cookie to permit it to accept new operations.
 351 */
 352void __fscache_enable_cookie(struct fscache_cookie *cookie,
 353                             const void *aux_data,
 354                             loff_t object_size,
 355                             bool (*can_enable)(void *data),
 356                             void *data)
 357{
 358        _enter("%p", cookie);
 359
 360        trace_fscache_enable(cookie);
 361
 362        wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 363                         TASK_UNINTERRUPTIBLE);
 364
 365        fscache_update_aux(cookie, aux_data);
 366
 367        if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 368                goto out_unlock;
 369
 370        if (can_enable && !can_enable(data)) {
 371                /* The netfs decided it didn't want to enable after all */
 372        } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 373                /* Wait for outstanding disablement to complete */
 374                __fscache_wait_on_invalidate(cookie);
 375
 376                if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
 377                        set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 378        } else {
 379                set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 380        }
 381
 382out_unlock:
 383        clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 384        wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 385}
 386EXPORT_SYMBOL(__fscache_enable_cookie);
 387
 388/*
 389 * acquire a non-index cookie
 390 * - this must make sure the index chain is instantiated and instantiate the
 391 *   object representation too
 392 */
 393static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
 394                                            loff_t object_size)
 395{
 396        struct fscache_object *object;
 397        struct fscache_cache *cache;
 398        int ret;
 399
 400        _enter("");
 401
 402        set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 403
 404        /* now we need to see whether the backing objects for this cookie yet
 405         * exist, if not there'll be nothing to search */
 406        down_read(&fscache_addremove_sem);
 407
 408        if (list_empty(&fscache_cache_list)) {
 409                up_read(&fscache_addremove_sem);
 410                _leave(" = 0 [no caches]");
 411                return 0;
 412        }
 413
 414        /* select a cache in which to store the object */
 415        cache = fscache_select_cache_for_object(cookie->parent);
 416        if (!cache) {
 417                up_read(&fscache_addremove_sem);
 418                fscache_stat(&fscache_n_acquires_no_cache);
 419                _leave(" = -ENOMEDIUM [no cache]");
 420                return -ENOMEDIUM;
 421        }
 422
 423        _debug("cache %s", cache->tag->name);
 424
 425        set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
 426
 427        /* ask the cache to allocate objects for this cookie and its parent
 428         * chain */
 429        ret = fscache_alloc_object(cache, cookie);
 430        if (ret < 0) {
 431                up_read(&fscache_addremove_sem);
 432                _leave(" = %d", ret);
 433                return ret;
 434        }
 435
 436        spin_lock(&cookie->lock);
 437        if (hlist_empty(&cookie->backing_objects)) {
 438                spin_unlock(&cookie->lock);
 439                goto unavailable;
 440        }
 441
 442        object = hlist_entry(cookie->backing_objects.first,
 443                             struct fscache_object, cookie_link);
 444
 445        fscache_set_store_limit(object, object_size);
 446
 447        /* initiate the process of looking up all the objects in the chain
 448         * (done by fscache_initialise_object()) */
 449        fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
 450
 451        spin_unlock(&cookie->lock);
 452
 453        /* we may be required to wait for lookup to complete at this point */
 454        if (!fscache_defer_lookup) {
 455                _debug("non-deferred lookup %p", &cookie->flags);
 456                wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 457                            TASK_UNINTERRUPTIBLE);
 458                _debug("complete");
 459                if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
 460                        goto unavailable;
 461        }
 462
 463        up_read(&fscache_addremove_sem);
 464        _leave(" = 0 [deferred]");
 465        return 0;
 466
 467unavailable:
 468        up_read(&fscache_addremove_sem);
 469        _leave(" = -ENOBUFS");
 470        return -ENOBUFS;
 471}
 472
 473/*
 474 * recursively allocate cache object records for a cookie/cache combination
 475 * - caller must be holding the addremove sem
 476 */
 477static int fscache_alloc_object(struct fscache_cache *cache,
 478                                struct fscache_cookie *cookie)
 479{
 480        struct fscache_object *object;
 481        int ret;
 482
 483        _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
 484
 485        spin_lock(&cookie->lock);
 486        hlist_for_each_entry(object, &cookie->backing_objects,
 487                             cookie_link) {
 488                if (object->cache == cache)
 489                        goto object_already_extant;
 490        }
 491        spin_unlock(&cookie->lock);
 492
 493        /* ask the cache to allocate an object (we may end up with duplicate
 494         * objects at this stage, but we sort that out later) */
 495        fscache_stat(&fscache_n_cop_alloc_object);
 496        object = cache->ops->alloc_object(cache, cookie);
 497        fscache_stat_d(&fscache_n_cop_alloc_object);
 498        if (IS_ERR(object)) {
 499                fscache_stat(&fscache_n_object_no_alloc);
 500                ret = PTR_ERR(object);
 501                goto error;
 502        }
 503
 504        ASSERTCMP(object->cookie, ==, cookie);
 505        fscache_stat(&fscache_n_object_alloc);
 506
 507        object->debug_id = atomic_inc_return(&fscache_object_debug_id);
 508
 509        _debug("ALLOC OBJ%x: %s {%lx}",
 510               object->debug_id, cookie->def->name, object->events);
 511
 512        ret = fscache_alloc_object(cache, cookie->parent);
 513        if (ret < 0)
 514                goto error_put;
 515
 516        /* only attach if we managed to allocate all we needed, otherwise
 517         * discard the object we just allocated and instead use the one
 518         * attached to the cookie */
 519        if (fscache_attach_object(cookie, object) < 0) {
 520                fscache_stat(&fscache_n_cop_put_object);
 521                cache->ops->put_object(object, fscache_obj_put_attach_fail);
 522                fscache_stat_d(&fscache_n_cop_put_object);
 523        }
 524
 525        _leave(" = 0");
 526        return 0;
 527
 528object_already_extant:
 529        ret = -ENOBUFS;
 530        if (fscache_object_is_dying(object) ||
 531            fscache_cache_is_broken(object)) {
 532                spin_unlock(&cookie->lock);
 533                goto error;
 534        }
 535        spin_unlock(&cookie->lock);
 536        _leave(" = 0 [found]");
 537        return 0;
 538
 539error_put:
 540        fscache_stat(&fscache_n_cop_put_object);
 541        cache->ops->put_object(object, fscache_obj_put_alloc_fail);
 542        fscache_stat_d(&fscache_n_cop_put_object);
 543error:
 544        _leave(" = %d", ret);
 545        return ret;
 546}
 547
 548/*
 549 * attach a cache object to a cookie
 550 */
 551static int fscache_attach_object(struct fscache_cookie *cookie,
 552                                 struct fscache_object *object)
 553{
 554        struct fscache_object *p;
 555        struct fscache_cache *cache = object->cache;
 556        int ret;
 557
 558        _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
 559
 560        ASSERTCMP(object->cookie, ==, cookie);
 561
 562        spin_lock(&cookie->lock);
 563
 564        /* there may be multiple initial creations of this object, but we only
 565         * want one */
 566        ret = -EEXIST;
 567        hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
 568                if (p->cache == object->cache) {
 569                        if (fscache_object_is_dying(p))
 570                                ret = -ENOBUFS;
 571                        goto cant_attach_object;
 572                }
 573        }
 574
 575        /* pin the parent object */
 576        spin_lock_nested(&cookie->parent->lock, 1);
 577        hlist_for_each_entry(p, &cookie->parent->backing_objects,
 578                             cookie_link) {
 579                if (p->cache == object->cache) {
 580                        if (fscache_object_is_dying(p)) {
 581                                ret = -ENOBUFS;
 582                                spin_unlock(&cookie->parent->lock);
 583                                goto cant_attach_object;
 584                        }
 585                        object->parent = p;
 586                        spin_lock(&p->lock);
 587                        p->n_children++;
 588                        spin_unlock(&p->lock);
 589                        break;
 590                }
 591        }
 592        spin_unlock(&cookie->parent->lock);
 593
 594        /* attach to the cache's object list */
 595        if (list_empty(&object->cache_link)) {
 596                spin_lock(&cache->object_list_lock);
 597                list_add(&object->cache_link, &cache->object_list);
 598                spin_unlock(&cache->object_list_lock);
 599        }
 600
 601        /* Attach to the cookie.  The object already has a ref on it. */
 602        hlist_add_head(&object->cookie_link, &cookie->backing_objects);
 603
 604        fscache_objlist_add(object);
 605        ret = 0;
 606
 607cant_attach_object:
 608        spin_unlock(&cookie->lock);
 609        _leave(" = %d", ret);
 610        return ret;
 611}
 612
 613/*
 614 * Invalidate an object.  Callable with spinlocks held.
 615 */
 616void __fscache_invalidate(struct fscache_cookie *cookie)
 617{
 618        struct fscache_object *object;
 619
 620        _enter("{%s}", cookie->def->name);
 621
 622        fscache_stat(&fscache_n_invalidates);
 623
 624        /* Only permit invalidation of data files.  Invalidating an index will
 625         * require the caller to release all its attachments to the tree rooted
 626         * there, and if it's doing that, it may as well just retire the
 627         * cookie.
 628         */
 629        ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 630
 631        /* If there's an object, we tell the object state machine to handle the
 632         * invalidation on our behalf, otherwise there's nothing to do.
 633         */
 634        if (!hlist_empty(&cookie->backing_objects)) {
 635                spin_lock(&cookie->lock);
 636
 637                if (fscache_cookie_enabled(cookie) &&
 638                    !hlist_empty(&cookie->backing_objects) &&
 639                    !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
 640                                      &cookie->flags)) {
 641                        object = hlist_entry(cookie->backing_objects.first,
 642                                             struct fscache_object,
 643                                             cookie_link);
 644                        if (fscache_object_is_live(object))
 645                                fscache_raise_event(
 646                                        object, FSCACHE_OBJECT_EV_INVALIDATE);
 647                }
 648
 649                spin_unlock(&cookie->lock);
 650        }
 651
 652        _leave("");
 653}
 654EXPORT_SYMBOL(__fscache_invalidate);
 655
 656/*
 657 * Wait for object invalidation to complete.
 658 */
 659void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
 660{
 661        _enter("%p", cookie);
 662
 663        wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
 664                    TASK_UNINTERRUPTIBLE);
 665
 666        _leave("");
 667}
 668EXPORT_SYMBOL(__fscache_wait_on_invalidate);
 669
 670/*
 671 * update the index entries backing a cookie
 672 */
 673void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
 674{
 675        struct fscache_object *object;
 676
 677        fscache_stat(&fscache_n_updates);
 678
 679        if (!cookie) {
 680                fscache_stat(&fscache_n_updates_null);
 681                _leave(" [no cookie]");
 682                return;
 683        }
 684
 685        _enter("{%s}", cookie->def->name);
 686
 687        spin_lock(&cookie->lock);
 688
 689        fscache_update_aux(cookie, aux_data);
 690
 691        if (fscache_cookie_enabled(cookie)) {
 692                /* update the index entry on disk in each cache backing this
 693                 * cookie.
 694                 */
 695                hlist_for_each_entry(object,
 696                                     &cookie->backing_objects, cookie_link) {
 697                        fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
 698                }
 699        }
 700
 701        spin_unlock(&cookie->lock);
 702        _leave("");
 703}
 704EXPORT_SYMBOL(__fscache_update_cookie);
 705
 706/*
 707 * Disable a cookie to stop it from accepting new requests from the netfs.
 708 */
 709void __fscache_disable_cookie(struct fscache_cookie *cookie,
 710                              const void *aux_data,
 711                              bool invalidate)
 712{
 713        struct fscache_object *object;
 714        bool awaken = false;
 715
 716        _enter("%p,%u", cookie, invalidate);
 717
 718        trace_fscache_disable(cookie);
 719
 720        ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
 721
 722        if (atomic_read(&cookie->n_children) != 0) {
 723                pr_err("Cookie '%s' still has children\n",
 724                       cookie->def->name);
 725                BUG();
 726        }
 727
 728        wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 729                         TASK_UNINTERRUPTIBLE);
 730
 731        fscache_update_aux(cookie, aux_data);
 732
 733        if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 734                goto out_unlock_enable;
 735
 736        /* If the cookie is being invalidated, wait for that to complete first
 737         * so that we can reuse the flag.
 738         */
 739        __fscache_wait_on_invalidate(cookie);
 740
 741        /* Dispose of the backing objects */
 742        set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
 743
 744        spin_lock(&cookie->lock);
 745        if (!hlist_empty(&cookie->backing_objects)) {
 746                hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
 747                        if (invalidate)
 748                                set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
 749                        clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 750                        fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
 751                }
 752        } else {
 753                if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
 754                        awaken = true;
 755        }
 756        spin_unlock(&cookie->lock);
 757        if (awaken)
 758                wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
 759
 760        /* Wait for cessation of activity requiring access to the netfs (when
 761         * n_active reaches 0).  This makes sure outstanding reads and writes
 762         * have completed.
 763         */
 764        if (!atomic_dec_and_test(&cookie->n_active)) {
 765                wait_var_event(&cookie->n_active,
 766                               !atomic_read(&cookie->n_active));
 767        }
 768
 769        /* Make sure any pending writes are cancelled. */
 770        if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
 771                fscache_invalidate_writes(cookie);
 772
 773        /* Reset the cookie state if it wasn't relinquished */
 774        if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
 775                atomic_inc(&cookie->n_active);
 776                set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 777        }
 778
 779out_unlock_enable:
 780        clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 781        wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 782        _leave("");
 783}
 784EXPORT_SYMBOL(__fscache_disable_cookie);
 785
 786/*
 787 * release a cookie back to the cache
 788 * - the object will be marked as recyclable on disk if retire is true
 789 * - all dependents of this cookie must have already been unregistered
 790 *   (indices/files/pages)
 791 */
 792void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
 793                                 const void *aux_data,
 794                                 bool retire)
 795{
 796        fscache_stat(&fscache_n_relinquishes);
 797        if (retire)
 798                fscache_stat(&fscache_n_relinquishes_retire);
 799
 800        if (!cookie) {
 801                fscache_stat(&fscache_n_relinquishes_null);
 802                _leave(" [no cookie]");
 803                return;
 804        }
 805
 806        _enter("%p{%s,%p,%d},%d",
 807               cookie, cookie->def->name, cookie->netfs_data,
 808               atomic_read(&cookie->n_active), retire);
 809
 810        trace_fscache_relinquish(cookie, retire);
 811
 812        /* No further netfs-accessing operations on this cookie permitted */
 813        if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
 814                BUG();
 815
 816        __fscache_disable_cookie(cookie, aux_data, retire);
 817
 818        /* Clear pointers back to the netfs */
 819        cookie->netfs_data      = NULL;
 820        cookie->def             = NULL;
 821        BUG_ON(!radix_tree_empty(&cookie->stores));
 822
 823        if (cookie->parent) {
 824                ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
 825                ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
 826                atomic_dec(&cookie->parent->n_children);
 827        }
 828
 829        /* Dispose of the netfs's link to the cookie */
 830        ASSERTCMP(atomic_read(&cookie->usage), >, 0);
 831        fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
 832
 833        _leave("");
 834}
 835EXPORT_SYMBOL(__fscache_relinquish_cookie);
 836
 837/*
 838 * Remove a cookie from the hash table.
 839 */
 840static void fscache_unhash_cookie(struct fscache_cookie *cookie)
 841{
 842        struct hlist_bl_head *h;
 843        unsigned int bucket;
 844
 845        bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 846        h = &fscache_cookie_hash[bucket];
 847
 848        hlist_bl_lock(h);
 849        hlist_bl_del(&cookie->hash_link);
 850        hlist_bl_unlock(h);
 851}
 852
 853/*
 854 * Drop a reference to a cookie.
 855 */
 856void fscache_cookie_put(struct fscache_cookie *cookie,
 857                        enum fscache_cookie_trace where)
 858{
 859        struct fscache_cookie *parent;
 860        int usage;
 861
 862        _enter("%p", cookie);
 863
 864        do {
 865                usage = atomic_dec_return(&cookie->usage);
 866                trace_fscache_cookie(cookie, where, usage);
 867
 868                if (usage > 0)
 869                        return;
 870                BUG_ON(usage < 0);
 871
 872                parent = cookie->parent;
 873                fscache_unhash_cookie(cookie);
 874                fscache_free_cookie(cookie);
 875
 876                cookie = parent;
 877                where = fscache_cookie_put_parent;
 878        } while (cookie);
 879
 880        _leave("");
 881}
 882
 883/*
 884 * check the consistency between the netfs inode and the backing cache
 885 *
 886 * NOTE: it only serves no-index type
 887 */
 888int __fscache_check_consistency(struct fscache_cookie *cookie,
 889                                const void *aux_data)
 890{
 891        struct fscache_operation *op;
 892        struct fscache_object *object;
 893        bool wake_cookie = false;
 894        int ret;
 895
 896        _enter("%p,", cookie);
 897
 898        ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 899
 900        if (fscache_wait_for_deferred_lookup(cookie) < 0)
 901                return -ERESTARTSYS;
 902
 903        if (hlist_empty(&cookie->backing_objects))
 904                return 0;
 905
 906        op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 907        if (!op)
 908                return -ENOMEM;
 909
 910        fscache_operation_init(cookie, op, NULL, NULL, NULL);
 911        op->flags = FSCACHE_OP_MYTHREAD |
 912                (1 << FSCACHE_OP_WAITING) |
 913                (1 << FSCACHE_OP_UNUSE_COOKIE);
 914        trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
 915
 916        spin_lock(&cookie->lock);
 917
 918        fscache_update_aux(cookie, aux_data);
 919
 920        if (!fscache_cookie_enabled(cookie) ||
 921            hlist_empty(&cookie->backing_objects))
 922                goto inconsistent;
 923        object = hlist_entry(cookie->backing_objects.first,
 924                             struct fscache_object, cookie_link);
 925        if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 926                goto inconsistent;
 927
 928        op->debug_id = atomic_inc_return(&fscache_op_debug_id);
 929
 930        __fscache_use_cookie(cookie);
 931        if (fscache_submit_op(object, op) < 0)
 932                goto submit_failed;
 933
 934        /* the work queue now carries its own ref on the object */
 935        spin_unlock(&cookie->lock);
 936
 937        ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
 938        if (ret == 0) {
 939                /* ask the cache to honour the operation */
 940                ret = object->cache->ops->check_consistency(op);
 941                fscache_op_complete(op, false);
 942        } else if (ret == -ENOBUFS) {
 943                ret = 0;
 944        }
 945
 946        fscache_put_operation(op);
 947        _leave(" = %d", ret);
 948        return ret;
 949
 950submit_failed:
 951        wake_cookie = __fscache_unuse_cookie(cookie);
 952inconsistent:
 953        spin_unlock(&cookie->lock);
 954        if (wake_cookie)
 955                __fscache_wake_unused_cookie(cookie);
 956        kfree(op);
 957        _leave(" = -ESTALE");
 958        return -ESTALE;
 959}
 960EXPORT_SYMBOL(__fscache_check_consistency);
 961