fs/fscache/cookie.c

   1 /* netfs cookie management
   2  *
   3  * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  *
  11  * See Documentation/filesystems/caching/netfs-api.txt for more information on
  12  * the netfs API.
  13  */
  14
  15 #define FSCACHE_DEBUG_LEVEL COOKIE
  16 #include <linux/module.h>
  17 #include <linux/slab.h>
  18 #include "internal.h"
  19
  20 struct kmem_cache *fscache_cookie_jar;
  21
  22 static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
  23
  24 #define fscache_cookie_hash_shift 15
  25 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
  26
  27 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
  28                                             loff_t object_size);
  29 static int fscache_alloc_object(struct fscache_cache *cache,
  30                                 struct fscache_cookie *cookie);
  31 static int fscache_attach_object(struct fscache_cookie *cookie,
  32                                  struct fscache_object *object);
  33
  34 static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
  35 {
  36         struct hlist_node *object;
  37         const u8 *k;
  38         unsigned loop;
  39
  40         pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
  41                prefix, cookie, cookie->parent, cookie->flags,
  42                atomic_read(&cookie->n_children),
  43                atomic_read(&cookie->n_active));
  44         pr_err("%c-cookie d=%p n=%p\n",
  45                prefix, cookie->def, cookie->netfs_data);
  46
  47         object = READ_ONCE(cookie->backing_objects.first);
  48         if (object)
  49                 pr_err("%c-cookie o=%p\n",
  50                        prefix, hlist_entry(object, struct fscache_object, cookie_link));
  51
  52         pr_err("%c-key=[%u] '", prefix, cookie->key_len);
  53         k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
  54                 cookie->inline_key : cookie->key;
  55         for (loop = 0; loop < cookie->key_len; loop++)
  56                 pr_cont("%02x", k[loop]);
  57         pr_cont("'\n");
  58 }
  59
  60 void fscache_free_cookie(struct fscache_cookie *cookie)
  61 {
  62         if (cookie) {
  63                 BUG_ON(!hlist_empty(&cookie->backing_objects));
  64                 if (cookie->aux_len > sizeof(cookie->inline_aux))
  65                         kfree(cookie->aux);
  66                 if (cookie->key_len > sizeof(cookie->inline_key))
  67                         kfree(cookie->key);
  68                 kmem_cache_free(fscache_cookie_jar, cookie);
  69         }
  70 }
  71
  72 /*
  73  * initialise an cookie jar slab element prior to any use
  74  */
  75 void fscache_cookie_init_once(void *_cookie)
  76 {
  77         struct fscache_cookie *cookie = _cookie;
  78
  79         memset(cookie, 0, sizeof(*cookie));
  80         spin_lock_init(&cookie->lock);
  81         spin_lock_init(&cookie->stores_lock);
  82         INIT_HLIST_HEAD(&cookie->backing_objects);
  83 }
  84
  85 /*
  86  * Set the index key in a cookie.  The cookie struct has space for a 12-byte
  87  * key plus length and hash, but if that's not big enough, it's instead a
  88  * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
  89  * the key data.
  90  */
  91 static int fscache_set_key(struct fscache_cookie *cookie,
  92                            const void *index_key, size_t index_key_len)
  93 {
  94         unsigned long long h;
  95         u32 *buf;
  96         int i;
  97
  98         cookie->key_len = index_key_len;
  99
 100         if (index_key_len > sizeof(cookie->inline_key)) {
 101                 buf = kzalloc(index_key_len, GFP_KERNEL);
 102                 if (!buf)
 103                         return -ENOMEM;
 104                 cookie->key = buf;
 105         } else {
 106                 buf = (u32 *)cookie->inline_key;
 107                 buf[0] = 0;
 108                 buf[1] = 0;
 109                 buf[2] = 0;
 110         }
 111
 112         memcpy(buf, index_key, index_key_len);
 113
 114         /* Calculate a hash and combine this with the length in the first word
 115          * or first half word
 116          */
 117         h = (unsigned long)cookie->parent;
 118         h += index_key_len + cookie->type;
 119         for (i = 0; i < (index_key_len + sizeof(u32) - 1) / sizeof(u32); i++)
 120                 h += buf[i];
 121
 122         cookie->key_hash = h ^ (h >> 32);
 123         return 0;
 124 }
 125
 126 static long fscache_compare_cookie(const struct fscache_cookie *a,
 127                                    const struct fscache_cookie *b)
 128 {
 129         const void *ka, *kb;
 130
 131         if (a->key_hash != b->key_hash)
 132                 return (long)a->key_hash - (long)b->key_hash;
 133         if (a->parent != b->parent)
 134                 return (long)a->parent - (long)b->parent;
 135         if (a->key_len != b->key_len)
 136                 return (long)a->key_len - (long)b->key_len;
 137         if (a->type != b->type)
 138                 return (long)a->type - (long)b->type;
 139
 140         if (a->key_len <= sizeof(a->inline_key)) {
 141                 ka = &a->inline_key;
 142                 kb = &b->inline_key;
 143         } else {
 144                 ka = a->key;
 145                 kb = b->key;
 146         }
 147         return memcmp(ka, kb, a->key_len);
 148 }
 149
 150 /*
 151  * Allocate a cookie.
 152  */
 153 struct fscache_cookie *fscache_alloc_cookie(
 154         struct fscache_cookie *parent,
 155         const struct fscache_cookie_def *def,
 156         const void *index_key, size_t index_key_len,
 157         const void *aux_data, size_t aux_data_len,
 158         void *netfs_data,
 159         loff_t object_size)
 160 {
 161         struct fscache_cookie *cookie;
 162
 163         /* allocate and initialise a cookie */
 164         cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
 165         if (!cookie)
 166                 return NULL;
 167
 168         cookie->key_len = index_key_len;
 169         cookie->aux_len = aux_data_len;
 170
 171         if (fscache_set_key(cookie, index_key, index_key_len) < 0)
 172                 goto nomem;
 173
 174         if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
 175                 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
 176         } else {
 177                 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
 178                 if (!cookie->aux)
 179                         goto nomem;
 180         }
 181
 182         atomic_set(&cookie->usage, 1);
 183         atomic_set(&cookie->n_children, 0);
 184
 185         /* We keep the active count elevated until relinquishment to prevent an
 186          * attempt to wake up every time the object operations queue quiesces.
 187          */
 188         atomic_set(&cookie->n_active, 1);
 189
 190         cookie->def             = def;
 191         cookie->parent          = parent;
 192         cookie->netfs_data      = netfs_data;
 193         cookie->flags           = (1 << FSCACHE_COOKIE_NO_DATA_YET);
 194         cookie->type            = def->type;
 195
 196         /* radix tree insertion won't use the preallocation pool unless it's
 197          * told it may not wait */
 198         INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
 199         return cookie;
 200
 201 nomem:
 202         fscache_free_cookie(cookie);
 203         return NULL;
 204 }
 205
 206 /*
 207  * Attempt to insert the new cookie into the hash.  If there's a collision, we
 208  * return the old cookie if it's not in use and an error otherwise.
 209  */
 210 struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
 211 {
 212         struct fscache_cookie *cursor;
 213         struct hlist_bl_head *h;
 214         struct hlist_bl_node *p;
 215         unsigned int bucket;
 216
 217         bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 218         h = &fscache_cookie_hash[bucket];
 219
 220         hlist_bl_lock(h);
 221         hlist_bl_for_each_entry(cursor, p, h, hash_link) {
 222                 if (fscache_compare_cookie(candidate, cursor) == 0)
 223                         goto collision;
 224         }
 225
 226         __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
 227         fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
 228         atomic_inc(&candidate->parent->n_children);
 229         hlist_bl_add_head(&candidate->hash_link, h);
 230         hlist_bl_unlock(h);
 231         return candidate;
 232
 233 collision:
 234         if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
 235                 trace_fscache_cookie(cursor, fscache_cookie_collision,
 236                                      atomic_read(&cursor->usage));
 237                 pr_err("Duplicate cookie detected\n");
 238                 fscache_print_cookie(cursor, 'O');
 239                 fscache_print_cookie(candidate, 'N');
 240                 hlist_bl_unlock(h);
 241                 return NULL;
 242         }
 243
 244         fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
 245         hlist_bl_unlock(h);
 246         return cursor;
 247 }
 248
 249 /*
 250  * request a cookie to represent an object (index, datafile, xattr, etc)
 251  * - parent specifies the parent object
 252  *   - the top level index cookie for each netfs is stored in the fscache_netfs
 253  *     struct upon registration
 254  * - def points to the definition
 255  * - the netfs_data will be passed to the functions pointed to in *def
 256  * - all attached caches will be searched to see if they contain this object
 257  * - index objects aren't stored on disk until there's a dependent file that
 258  *   needs storing
 259  * - other objects are stored in a selected cache immediately, and all the
 260  *   indices forming the path to it are instantiated if necessary
 261  * - we never let on to the netfs about errors
 262  *   - we may set a negative cookie pointer, but that's okay
 263  */
 264 struct fscache_cookie *__fscache_acquire_cookie(
 265         struct fscache_cookie *parent,
 266         const struct fscache_cookie_def *def,
 267         const void *index_key, size_t index_key_len,
 268         const void *aux_data, size_t aux_data_len,
 269         void *netfs_data,
 270         loff_t object_size,
 271         bool enable)
 272 {
 273         struct fscache_cookie *candidate, *cookie;
 274
 275         BUG_ON(!def);
 276
 277         _enter("{%s},{%s},%p,%u",
 278                parent ? (char *) parent->def->name : "<no-parent>",
 279                def->name, netfs_data, enable);
 280
 281         if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
 282                 return NULL;
 283         if (!aux_data || !aux_data_len) {
 284                 aux_data = NULL;
 285                 aux_data_len = 0;
 286         }
 287
 288         fscache_stat(&fscache_n_acquires);
 289
 290         /* if there's no parent cookie, then we don't create one here either */
 291         if (!parent) {
 292                 fscache_stat(&fscache_n_acquires_null);
 293                 _leave(" [no parent]");
 294                 return NULL;
 295         }
 296
 297         /* validate the definition */
 298         BUG_ON(!def->name[0]);
 299
 300         BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
 301                parent->type != FSCACHE_COOKIE_TYPE_INDEX);
 302
 303         candidate = fscache_alloc_cookie(parent, def,
 304                                          index_key, index_key_len,
 305                                          aux_data, aux_data_len,
 306                                          netfs_data, object_size);
 307         if (!candidate) {
 308                 fscache_stat(&fscache_n_acquires_oom);
 309                 _leave(" [ENOMEM]");
 310                 return NULL;
 311         }
 312
 313         cookie = fscache_hash_cookie(candidate);
 314         if (!cookie) {
 315                 trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
 316                 goto out;
 317         }
 318
 319         if (cookie == candidate)
 320                 candidate = NULL;
 321
 322         switch (cookie->type) {
 323         case FSCACHE_COOKIE_TYPE_INDEX:
 324                 fscache_stat(&fscache_n_cookie_index);
 325                 break;
 326         case FSCACHE_COOKIE_TYPE_DATAFILE:
 327                 fscache_stat(&fscache_n_cookie_data);
 328                 break;
 329         default:
 330                 fscache_stat(&fscache_n_cookie_special);
 331                 break;
 332         }
 333
 334         trace_fscache_acquire(cookie);
 335
 336         if (enable) {
 337                 /* if the object is an index then we need do nothing more here
 338                  * - we create indices on disk when we need them as an index
 339                  * may exist in multiple caches */
 340                 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 341                         if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
 342                                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 343                         } else {
 344                                 atomic_dec(&parent->n_children);
 345                                 fscache_cookie_put(cookie,
 346                                                    fscache_cookie_put_acquire_nobufs);
 347                                 fscache_stat(&fscache_n_acquires_nobufs);
 348                                 _leave(" = NULL");
 349                                 return NULL;
 350                         }
 351                 } else {
 352                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 353                 }
 354         }
 355
 356         fscache_stat(&fscache_n_acquires_ok);
 357
 358 out:
 359         fscache_free_cookie(candidate);
 360         return cookie;
 361 }
 362 EXPORT_SYMBOL(__fscache_acquire_cookie);
 363
 364 /*
 365  * Enable a cookie to permit it to accept new operations.
 366  */
 367 void __fscache_enable_cookie(struct fscache_cookie *cookie,
 368                              const void *aux_data,
 369                              loff_t object_size,
 370                              bool (*can_enable)(void *data),
 371                              void *data)
 372 {
 373         _enter("%p", cookie);
 374
 375         trace_fscache_enable(cookie);
 376
 377         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 378                          TASK_UNINTERRUPTIBLE);
 379
 380         fscache_update_aux(cookie, aux_data);
 381
 382         if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 383                 goto out_unlock;
 384
 385         if (can_enable && !can_enable(data)) {
 386                 /* The netfs decided it didn't want to enable after all */
 387         } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 388                 /* Wait for outstanding disablement to complete */
 389                 __fscache_wait_on_invalidate(cookie);
 390
 391                 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
 392                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 393         } else {
 394                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 395         }
 396
 397 out_unlock:
 398         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 399         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 400 }
 401 EXPORT_SYMBOL(__fscache_enable_cookie);
 402
 403 /*
 404  * acquire a non-index cookie
 405  * - this must make sure the index chain is instantiated and instantiate the
 406  *   object representation too
 407  */
 408 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
 409                                             loff_t object_size)
 410 {
 411         struct fscache_object *object;
 412         struct fscache_cache *cache;
 413         int ret;
 414
 415         _enter("");
 416
 417         set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 418
 419         /* now we need to see whether the backing objects for this cookie yet
 420          * exist, if not there'll be nothing to search */
 421         down_read(&fscache_addremove_sem);
 422
 423         if (list_empty(&fscache_cache_list)) {
 424                 up_read(&fscache_addremove_sem);
 425                 _leave(" = 0 [no caches]");
 426                 return 0;
 427         }
 428
 429         /* select a cache in which to store the object */
 430         cache = fscache_select_cache_for_object(cookie->parent);
 431         if (!cache) {
 432                 up_read(&fscache_addremove_sem);
 433                 fscache_stat(&fscache_n_acquires_no_cache);
 434                 _leave(" = -ENOMEDIUM [no cache]");
 435                 return -ENOMEDIUM;
 436         }
 437
 438         _debug("cache %s", cache->tag->name);
 439
 440         set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
 441
 442         /* ask the cache to allocate objects for this cookie and its parent
 443          * chain */
 444         ret = fscache_alloc_object(cache, cookie);
 445         if (ret < 0) {
 446                 up_read(&fscache_addremove_sem);
 447                 _leave(" = %d", ret);
 448                 return ret;
 449         }
 450
 451         spin_lock(&cookie->lock);
 452         if (hlist_empty(&cookie->backing_objects)) {
 453                 spin_unlock(&cookie->lock);
 454                 goto unavailable;
 455         }
 456
 457         object = hlist_entry(cookie->backing_objects.first,
 458                              struct fscache_object, cookie_link);
 459
 460         fscache_set_store_limit(object, object_size);
 461
 462         /* initiate the process of looking up all the objects in the chain
 463          * (done by fscache_initialise_object()) */
 464         fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
 465
 466         spin_unlock(&cookie->lock);
 467
 468         /* we may be required to wait for lookup to complete at this point */
 469         if (!fscache_defer_lookup) {
 470                 _debug("non-deferred lookup %p", &cookie->flags);
 471                 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 472                             TASK_UNINTERRUPTIBLE);
 473                 _debug("complete");
 474                 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
 475                         goto unavailable;
 476         }
 477
 478         up_read(&fscache_addremove_sem);
 479         _leave(" = 0 [deferred]");
 480         return 0;
 481
 482 unavailable:
 483         up_read(&fscache_addremove_sem);
 484         _leave(" = -ENOBUFS");
 485         return -ENOBUFS;
 486 }
 487
 488 /*
 489  * recursively allocate cache object records for a cookie/cache combination
 490  * - caller must be holding the addremove sem
 491  */
 492 static int fscache_alloc_object(struct fscache_cache *cache,
 493                                 struct fscache_cookie *cookie)
 494 {
 495         struct fscache_object *object;
 496         int ret;
 497
 498         _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
 499
 500         spin_lock(&cookie->lock);
 501         hlist_for_each_entry(object, &cookie->backing_objects,
 502                              cookie_link) {
 503                 if (object->cache == cache)
 504                         goto object_already_extant;
 505         }
 506         spin_unlock(&cookie->lock);
 507
 508         /* ask the cache to allocate an object (we may end up with duplicate
 509          * objects at this stage, but we sort that out later) */
 510         fscache_stat(&fscache_n_cop_alloc_object);
 511         object = cache->ops->alloc_object(cache, cookie);
 512         fscache_stat_d(&fscache_n_cop_alloc_object);
 513         if (IS_ERR(object)) {
 514                 fscache_stat(&fscache_n_object_no_alloc);
 515                 ret = PTR_ERR(object);
 516                 goto error;
 517         }
 518
 519         ASSERTCMP(object->cookie, ==, cookie);
 520         fscache_stat(&fscache_n_object_alloc);
 521
 522         object->debug_id = atomic_inc_return(&fscache_object_debug_id);
 523
 524         _debug("ALLOC OBJ%x: %s {%lx}",
 525                object->debug_id, cookie->def->name, object->events);
 526
 527         ret = fscache_alloc_object(cache, cookie->parent);
 528         if (ret < 0)
 529                 goto error_put;
 530
 531         /* only attach if we managed to allocate all we needed, otherwise
 532          * discard the object we just allocated and instead use the one
 533          * attached to the cookie */
 534         if (fscache_attach_object(cookie, object) < 0) {
 535                 fscache_stat(&fscache_n_cop_put_object);
 536                 cache->ops->put_object(object, fscache_obj_put_attach_fail);
 537                 fscache_stat_d(&fscache_n_cop_put_object);
 538         }
 539
 540         _leave(" = 0");
 541         return 0;
 542
 543 object_already_extant:
 544         ret = -ENOBUFS;
 545         if (fscache_object_is_dying(object) ||
 546             fscache_cache_is_broken(object)) {
 547                 spin_unlock(&cookie->lock);
 548                 goto error;
 549         }
 550         spin_unlock(&cookie->lock);
 551         _leave(" = 0 [found]");
 552         return 0;
 553
 554 error_put:
 555         fscache_stat(&fscache_n_cop_put_object);
 556         cache->ops->put_object(object, fscache_obj_put_alloc_fail);
 557         fscache_stat_d(&fscache_n_cop_put_object);
 558 error:
 559         _leave(" = %d", ret);
 560         return ret;
 561 }
 562
 563 /*
 564  * attach a cache object to a cookie
 565  */
 566 static int fscache_attach_object(struct fscache_cookie *cookie,
 567                                  struct fscache_object *object)
 568 {
 569         struct fscache_object *p;
 570         struct fscache_cache *cache = object->cache;
 571         int ret;
 572
 573         _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
 574
 575         ASSERTCMP(object->cookie, ==, cookie);
 576
 577         spin_lock(&cookie->lock);
 578
 579         /* there may be multiple initial creations of this object, but we only
 580          * want one */
 581         ret = -EEXIST;
 582         hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
 583                 if (p->cache == object->cache) {
 584                         if (fscache_object_is_dying(p))
 585                                 ret = -ENOBUFS;
 586                         goto cant_attach_object;
 587                 }
 588         }
 589
 590         /* pin the parent object */
 591         spin_lock_nested(&cookie->parent->lock, 1);
 592         hlist_for_each_entry(p, &cookie->parent->backing_objects,
 593                              cookie_link) {
 594                 if (p->cache == object->cache) {
 595                         if (fscache_object_is_dying(p)) {
 596                                 ret = -ENOBUFS;
 597                                 spin_unlock(&cookie->parent->lock);
 598                                 goto cant_attach_object;
 599                         }
 600                         object->parent = p;
 601                         spin_lock(&p->lock);
 602                         p->n_children++;
 603                         spin_unlock(&p->lock);
 604                         break;
 605                 }
 606         }
 607         spin_unlock(&cookie->parent->lock);
 608
 609         /* attach to the cache's object list */
 610         if (list_empty(&object->cache_link)) {
 611                 spin_lock(&cache->object_list_lock);
 612                 list_add(&object->cache_link, &cache->object_list);
 613                 spin_unlock(&cache->object_list_lock);
 614         }
 615
 616         /* Attach to the cookie.  The object already has a ref on it. */
 617         hlist_add_head(&object->cookie_link, &cookie->backing_objects);
 618
 619         fscache_objlist_add(object);
 620         ret = 0;
 621
 622 cant_attach_object:
 623         spin_unlock(&cookie->lock);
 624         _leave(" = %d", ret);
 625         return ret;
 626 }
 627
 628 /*
 629  * Invalidate an object.  Callable with spinlocks held.
 630  */
 631 void __fscache_invalidate(struct fscache_cookie *cookie)
 632 {
 633         struct fscache_object *object;
 634
 635         _enter("{%s}", cookie->def->name);
 636
 637         fscache_stat(&fscache_n_invalidates);
 638
 639         /* Only permit invalidation of data files.  Invalidating an index will
 640          * require the caller to release all its attachments to the tree rooted
 641          * there, and if it's doing that, it may as well just retire the
 642          * cookie.
 643          */
 644         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 645
 646         /* If there's an object, we tell the object state machine to handle the
 647          * invalidation on our behalf, otherwise there's nothing to do.
 648          */
 649         if (!hlist_empty(&cookie->backing_objects)) {
 650                 spin_lock(&cookie->lock);
 651
 652                 if (fscache_cookie_enabled(cookie) &&
 653                     !hlist_empty(&cookie->backing_objects) &&
 654                     !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
 655                                       &cookie->flags)) {
 656                         object = hlist_entry(cookie->backing_objects.first,
 657                                              struct fscache_object,
 658                                              cookie_link);
 659                         if (fscache_object_is_live(object))
 660                                 fscache_raise_event(
 661                                         object, FSCACHE_OBJECT_EV_INVALIDATE);
 662                 }
 663
 664                 spin_unlock(&cookie->lock);
 665         }
 666
 667         _leave("");
 668 }
 669 EXPORT_SYMBOL(__fscache_invalidate);
 670
 671 /*
 672  * Wait for object invalidation to complete.
 673  */
 674 void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
 675 {
 676         _enter("%p", cookie);
 677
 678         wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
 679                     TASK_UNINTERRUPTIBLE);
 680
 681         _leave("");
 682 }
 683 EXPORT_SYMBOL(__fscache_wait_on_invalidate);
 684
 685 /*
 686  * update the index entries backing a cookie
 687  */
 688 void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
 689 {
 690         struct fscache_object *object;
 691
 692         fscache_stat(&fscache_n_updates);
 693
 694         if (!cookie) {
 695                 fscache_stat(&fscache_n_updates_null);
 696                 _leave(" [no cookie]");
 697                 return;
 698         }
 699
 700         _enter("{%s}", cookie->def->name);
 701
 702         spin_lock(&cookie->lock);
 703
 704         fscache_update_aux(cookie, aux_data);
 705
 706         if (fscache_cookie_enabled(cookie)) {
 707                 /* update the index entry on disk in each cache backing this
 708                  * cookie.
 709                  */
 710                 hlist_for_each_entry(object,
 711                                      &cookie->backing_objects, cookie_link) {
 712                         fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
 713                 }
 714         }
 715
 716         spin_unlock(&cookie->lock);
 717         _leave("");
 718 }
 719 EXPORT_SYMBOL(__fscache_update_cookie);
 720
 721 /*
 722  * Disable a cookie to stop it from accepting new requests from the netfs.
 723  */
 724 void __fscache_disable_cookie(struct fscache_cookie *cookie,
 725                               const void *aux_data,
 726                               bool invalidate)
 727 {
 728         struct fscache_object *object;
 729         bool awaken = false;
 730
 731         _enter("%p,%u", cookie, invalidate);
 732
 733         trace_fscache_disable(cookie);
 734
 735         ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
 736
 737         if (atomic_read(&cookie->n_children) != 0) {
 738                 pr_err("Cookie '%s' still has children\n",
 739                        cookie->def->name);
 740                 BUG();
 741         }
 742
 743         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 744                          TASK_UNINTERRUPTIBLE);
 745
 746         fscache_update_aux(cookie, aux_data);
 747
 748         if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 749                 goto out_unlock_enable;
 750
 751         /* If the cookie is being invalidated, wait for that to complete first
 752          * so that we can reuse the flag.
 753          */
 754         __fscache_wait_on_invalidate(cookie);
 755
 756         /* Dispose of the backing objects */
 757         set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
 758
 759         spin_lock(&cookie->lock);
 760         if (!hlist_empty(&cookie->backing_objects)) {
 761                 hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
 762                         if (invalidate)
 763                                 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
 764                         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 765                         fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
 766                 }
 767         } else {
 768                 if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
 769                         awaken = true;
 770         }
 771         spin_unlock(&cookie->lock);
 772         if (awaken)
 773                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
 774
 775         /* Wait for cessation of activity requiring access to the netfs (when
 776          * n_active reaches 0).  This makes sure outstanding reads and writes
 777          * have completed.
 778          */
 779         if (!atomic_dec_and_test(&cookie->n_active)) {
 780                 wait_var_event(&cookie->n_active,
 781                                !atomic_read(&cookie->n_active));
 782         }
 783
 784         /* Make sure any pending writes are cancelled. */
 785         if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
 786                 fscache_invalidate_writes(cookie);
 787
 788         /* Reset the cookie state if it wasn't relinquished */
 789         if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
 790                 atomic_inc(&cookie->n_active);
 791                 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 792         }
 793
 794 out_unlock_enable:
 795         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 796         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 797         _leave("");
 798 }
 799 EXPORT_SYMBOL(__fscache_disable_cookie);
 800
 801 /*
 802  * release a cookie back to the cache
 803  * - the object will be marked as recyclable on disk if retire is true
 804  * - all dependents of this cookie must have already been unregistered
 805  *   (indices/files/pages)
 806  */
 807 void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
 808                                  const void *aux_data,
 809                                  bool retire)
 810 {
 811         fscache_stat(&fscache_n_relinquishes);
 812         if (retire)
 813                 fscache_stat(&fscache_n_relinquishes_retire);
 814
 815         if (!cookie) {
 816                 fscache_stat(&fscache_n_relinquishes_null);
 817                 _leave(" [no cookie]");
 818                 return;
 819         }
 820
 821         _enter("%p{%s,%p,%d},%d",
 822                cookie, cookie->def->name, cookie->netfs_data,
 823                atomic_read(&cookie->n_active), retire);
 824
 825         trace_fscache_relinquish(cookie, retire);
 826
 827         /* No further netfs-accessing operations on this cookie permitted */
 828         if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
 829                 BUG();
 830
 831         __fscache_disable_cookie(cookie, aux_data, retire);
 832
 833         /* Clear pointers back to the netfs */
 834         cookie->netfs_data      = NULL;
 835         cookie->def             = NULL;
 836         BUG_ON(!radix_tree_empty(&cookie->stores));
 837
 838         if (cookie->parent) {
 839                 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
 840                 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
 841                 atomic_dec(&cookie->parent->n_children);
 842         }
 843
 844         /* Dispose of the netfs's link to the cookie */
 845         ASSERTCMP(atomic_read(&cookie->usage), >, 0);
 846         fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
 847
 848         _leave("");
 849 }
 850 EXPORT_SYMBOL(__fscache_relinquish_cookie);
 851
 852 /*
 853  * Remove a cookie from the hash table.
 854  */
 855 static void fscache_unhash_cookie(struct fscache_cookie *cookie)
 856 {
 857         struct hlist_bl_head *h;
 858         unsigned int bucket;
 859
 860         bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 861         h = &fscache_cookie_hash[bucket];
 862
 863         hlist_bl_lock(h);
 864         hlist_bl_del(&cookie->hash_link);
 865         hlist_bl_unlock(h);
 866 }
 867
 868 /*
 869  * Drop a reference to a cookie.
 870  */
 871 void fscache_cookie_put(struct fscache_cookie *cookie,
 872                         enum fscache_cookie_trace where)
 873 {
 874         struct fscache_cookie *parent;
 875         int usage;
 876
 877         _enter("%p", cookie);
 878
 879         do {
 880                 usage = atomic_dec_return(&cookie->usage);
 881                 trace_fscache_cookie(cookie, where, usage);
 882
 883                 if (usage > 0)
 884                         return;
 885                 BUG_ON(usage < 0);
 886
 887                 parent = cookie->parent;
 888                 fscache_unhash_cookie(cookie);
 889                 fscache_free_cookie(cookie);
 890
 891                 cookie = parent;
 892                 where = fscache_cookie_put_parent;
 893         } while (cookie);
 894
 895         _leave("");
 896 }
 897
 898 /*
 899  * check the consistency between the netfs inode and the backing cache
 900  *
 901  * NOTE: it only serves no-index type
 902  */
 903 int __fscache_check_consistency(struct fscache_cookie *cookie,
 904                                 const void *aux_data)
 905 {
 906         struct fscache_operation *op;
 907         struct fscache_object *object;
 908         bool wake_cookie = false;
 909         int ret;
 910
 911         _enter("%p,", cookie);
 912
 913         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 914
 915         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 916                 return -ERESTARTSYS;
 917
 918         if (hlist_empty(&cookie->backing_objects))
 919                 return 0;
 920
 921         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 922         if (!op)
 923                 return -ENOMEM;
 924
 925         fscache_operation_init(cookie, op, NULL, NULL, NULL);
 926         op->flags = FSCACHE_OP_MYTHREAD |
 927                 (1 << FSCACHE_OP_WAITING) |
 928                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 929         trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
 930
 931         spin_lock(&cookie->lock);
 932
 933         fscache_update_aux(cookie, aux_data);
 934
 935         if (!fscache_cookie_enabled(cookie) ||
 936             hlist_empty(&cookie->backing_objects))
 937                 goto inconsistent;
 938         object = hlist_entry(cookie->backing_objects.first,
 939                              struct fscache_object, cookie_link);
 940         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 941                 goto inconsistent;
 942
 943         op->debug_id = atomic_inc_return(&fscache_op_debug_id);
 944
 945         __fscache_use_cookie(cookie);
 946         if (fscache_submit_op(object, op) < 0)
 947                 goto submit_failed;
 948
 949         /* the work queue now carries its own ref on the object */
 950         spin_unlock(&cookie->lock);
 951
 952         ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
 953         if (ret == 0) {
 954                 /* ask the cache to honour the operation */
 955                 ret = object->cache->ops->check_consistency(op);
 956                 fscache_op_complete(op, false);
 957         } else if (ret == -ENOBUFS) {
 958                 ret = 0;
 959         }
 960
 961         fscache_put_operation(op);
 962         _leave(" = %d", ret);
 963         return ret;
 964
 965 submit_failed:
 966         wake_cookie = __fscache_unuse_cookie(cookie);
 967 inconsistent:
 968         spin_unlock(&cookie->lock);
 969         if (wake_cookie)
 970                 __fscache_wake_unused_cookie(cookie);
 971         kfree(op);
 972         _leave(" = -ESTALE");
 973         return -ESTALE;
 974 }
 975 EXPORT_SYMBOL(__fscache_check_consistency);