linux/fs/inode.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * (C) 1997 Linus Torvalds
   4 * (C) 1999 Andrea Arcangeli <andrea@suse.de> (dynamic inode allocation)
   5 */
   6#include <linux/export.h>
   7#include <linux/fs.h>
   8#include <linux/mm.h>
   9#include <linux/backing-dev.h>
  10#include <linux/hash.h>
  11#include <linux/swap.h>
  12#include <linux/security.h>
  13#include <linux/cdev.h>
  14#include <linux/memblock.h>
  15#include <linux/fsnotify.h>
  16#include <linux/mount.h>
  17#include <linux/posix_acl.h>
  18#include <linux/prefetch.h>
  19#include <linux/buffer_head.h> /* for inode_has_buffers */
  20#include <linux/ratelimit.h>
  21#include <linux/list_lru.h>
  22#include <linux/iversion.h>
  23#include <trace/events/writeback.h>
  24#include "internal.h"
  25
  26/*
  27 * Inode locking rules:
  28 *
  29 * inode->i_lock protects:
  30 *   inode->i_state, inode->i_hash, __iget()
  31 * Inode LRU list locks protect:
  32 *   inode->i_sb->s_inode_lru, inode->i_lru
  33 * inode->i_sb->s_inode_list_lock protects:
  34 *   inode->i_sb->s_inodes, inode->i_sb_list
  35 * bdi->wb.list_lock protects:
  36 *   bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_io_list
  37 * inode_hash_lock protects:
  38 *   inode_hashtable, inode->i_hash
  39 *
  40 * Lock ordering:
  41 *
  42 * inode->i_sb->s_inode_list_lock
  43 *   inode->i_lock
  44 *     Inode LRU list locks
  45 *
  46 * bdi->wb.list_lock
  47 *   inode->i_lock
  48 *
  49 * inode_hash_lock
  50 *   inode->i_sb->s_inode_list_lock
  51 *   inode->i_lock
  52 *
  53 * iunique_lock
  54 *   inode_hash_lock
  55 */
  56
  57static unsigned int i_hash_mask __read_mostly;
  58static unsigned int i_hash_shift __read_mostly;
  59static struct hlist_head *inode_hashtable __read_mostly;
  60static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
  61
  62/*
  63 * Empty aops. Can be used for the cases where the user does not
  64 * define any of the address_space operations.
  65 */
  66const struct address_space_operations empty_aops = {
  67};
  68EXPORT_SYMBOL(empty_aops);
  69
  70/*
  71 * Statistics gathering..
  72 */
  73struct inodes_stat_t inodes_stat;
  74
  75static DEFINE_PER_CPU(unsigned long, nr_inodes);
  76static DEFINE_PER_CPU(unsigned long, nr_unused);
  77
  78static struct kmem_cache *inode_cachep __read_mostly;
  79
  80static long get_nr_inodes(void)
  81{
  82        int i;
  83        long sum = 0;
  84        for_each_possible_cpu(i)
  85                sum += per_cpu(nr_inodes, i);
  86        return sum < 0 ? 0 : sum;
  87}
  88
  89static inline long get_nr_inodes_unused(void)
  90{
  91        int i;
  92        long sum = 0;
  93        for_each_possible_cpu(i)
  94                sum += per_cpu(nr_unused, i);
  95        return sum < 0 ? 0 : sum;
  96}
  97
  98long get_nr_dirty_inodes(void)
  99{
 100        /* not actually dirty inodes, but a wild approximation */
 101        long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
 102        return nr_dirty > 0 ? nr_dirty : 0;
 103}
 104
 105/*
 106 * Handle nr_inode sysctl
 107 */
 108#ifdef CONFIG_SYSCTL
 109int proc_nr_inodes(struct ctl_table *table, int write,
 110                   void __user *buffer, size_t *lenp, loff_t *ppos)
 111{
 112        inodes_stat.nr_inodes = get_nr_inodes();
 113        inodes_stat.nr_unused = get_nr_inodes_unused();
 114        return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
 115}
 116#endif
 117
 118static int no_open(struct inode *inode, struct file *file)
 119{
 120        return -ENXIO;
 121}
 122
 123/**
 124 * inode_init_always - perform inode structure initialisation
 125 * @sb: superblock inode belongs to
 126 * @inode: inode to initialise
 127 *
 128 * These are initializations that need to be done on every inode
 129 * allocation as the fields are not initialised by slab allocation.
 130 */
 131int inode_init_always(struct super_block *sb, struct inode *inode)
 132{
 133        static const struct inode_operations empty_iops;
 134        static const struct file_operations no_open_fops = {.open = no_open};
 135        struct address_space *const mapping = &inode->i_data;
 136
 137        inode->i_sb = sb;
 138        inode->i_blkbits = sb->s_blocksize_bits;
 139        inode->i_flags = 0;
 140        atomic_set(&inode->i_count, 1);
 141        inode->i_op = &empty_iops;
 142        inode->i_fop = &no_open_fops;
 143        inode->__i_nlink = 1;
 144        inode->i_opflags = 0;
 145        if (sb->s_xattr)
 146                inode->i_opflags |= IOP_XATTR;
 147        i_uid_write(inode, 0);
 148        i_gid_write(inode, 0);
 149        atomic_set(&inode->i_writecount, 0);
 150        inode->i_size = 0;
 151        inode->i_write_hint = WRITE_LIFE_NOT_SET;
 152        inode->i_blocks = 0;
 153        inode->i_bytes = 0;
 154        inode->i_generation = 0;
 155        inode->i_pipe = NULL;
 156        inode->i_bdev = NULL;
 157        inode->i_cdev = NULL;
 158        inode->i_link = NULL;
 159        inode->i_dir_seq = 0;
 160        inode->i_rdev = 0;
 161        inode->dirtied_when = 0;
 162
 163#ifdef CONFIG_CGROUP_WRITEBACK
 164        inode->i_wb_frn_winner = 0;
 165        inode->i_wb_frn_avg_time = 0;
 166        inode->i_wb_frn_history = 0;
 167#endif
 168
 169        if (security_inode_alloc(inode))
 170                goto out;
 171        spin_lock_init(&inode->i_lock);
 172        lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
 173
 174        init_rwsem(&inode->i_rwsem);
 175        lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
 176
 177        atomic_set(&inode->i_dio_count, 0);
 178
 179        mapping->a_ops = &empty_aops;
 180        mapping->host = inode;
 181        mapping->flags = 0;
 182        mapping->wb_err = 0;
 183        atomic_set(&mapping->i_mmap_writable, 0);
 184#ifdef CONFIG_READ_ONLY_THP_FOR_FS
 185        atomic_set(&mapping->nr_thps, 0);
 186#endif
 187        mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
 188        mapping->private_data = NULL;
 189        mapping->writeback_index = 0;
 190        inode->i_private = NULL;
 191        inode->i_mapping = mapping;
 192        INIT_HLIST_HEAD(&inode->i_dentry);      /* buggered by rcu freeing */
 193#ifdef CONFIG_FS_POSIX_ACL
 194        inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
 195#endif
 196
 197#ifdef CONFIG_FSNOTIFY
 198        inode->i_fsnotify_mask = 0;
 199#endif
 200        inode->i_flctx = NULL;
 201        this_cpu_inc(nr_inodes);
 202
 203        return 0;
 204out:
 205        return -ENOMEM;
 206}
 207EXPORT_SYMBOL(inode_init_always);
 208
 209void free_inode_nonrcu(struct inode *inode)
 210{
 211        kmem_cache_free(inode_cachep, inode);
 212}
 213EXPORT_SYMBOL(free_inode_nonrcu);
 214
 215static void i_callback(struct rcu_head *head)
 216{
 217        struct inode *inode = container_of(head, struct inode, i_rcu);
 218        if (inode->free_inode)
 219                inode->free_inode(inode);
 220        else
 221                free_inode_nonrcu(inode);
 222}
 223
 224static struct inode *alloc_inode(struct super_block *sb)
 225{
 226        const struct super_operations *ops = sb->s_op;
 227        struct inode *inode;
 228
 229        if (ops->alloc_inode)
 230                inode = ops->alloc_inode(sb);
 231        else
 232                inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
 233
 234        if (!inode)
 235                return NULL;
 236
 237        if (unlikely(inode_init_always(sb, inode))) {
 238                if (ops->destroy_inode) {
 239                        ops->destroy_inode(inode);
 240                        if (!ops->free_inode)
 241                                return NULL;
 242                }
 243                inode->free_inode = ops->free_inode;
 244                i_callback(&inode->i_rcu);
 245                return NULL;
 246        }
 247
 248        return inode;
 249}
 250
 251void __destroy_inode(struct inode *inode)
 252{
 253        BUG_ON(inode_has_buffers(inode));
 254        inode_detach_wb(inode);
 255        security_inode_free(inode);
 256        fsnotify_inode_delete(inode);
 257        locks_free_lock_context(inode);
 258        if (!inode->i_nlink) {
 259                WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
 260                atomic_long_dec(&inode->i_sb->s_remove_count);
 261        }
 262
 263#ifdef CONFIG_FS_POSIX_ACL
 264        if (inode->i_acl && !is_uncached_acl(inode->i_acl))
 265                posix_acl_release(inode->i_acl);
 266        if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
 267                posix_acl_release(inode->i_default_acl);
 268#endif
 269        this_cpu_dec(nr_inodes);
 270}
 271EXPORT_SYMBOL(__destroy_inode);
 272
 273static void destroy_inode(struct inode *inode)
 274{
 275        const struct super_operations *ops = inode->i_sb->s_op;
 276
 277        BUG_ON(!list_empty(&inode->i_lru));
 278        __destroy_inode(inode);
 279        if (ops->destroy_inode) {
 280                ops->destroy_inode(inode);
 281                if (!ops->free_inode)
 282                        return;
 283        }
 284        inode->free_inode = ops->free_inode;
 285        call_rcu(&inode->i_rcu, i_callback);
 286}
 287
 288/**
 289 * drop_nlink - directly drop an inode's link count
 290 * @inode: inode
 291 *
 292 * This is a low-level filesystem helper to replace any
 293 * direct filesystem manipulation of i_nlink.  In cases
 294 * where we are attempting to track writes to the
 295 * filesystem, a decrement to zero means an imminent
 296 * write when the file is truncated and actually unlinked
 297 * on the filesystem.
 298 */
 299void drop_nlink(struct inode *inode)
 300{
 301        WARN_ON(inode->i_nlink == 0);
 302        inode->__i_nlink--;
 303        if (!inode->i_nlink)
 304                atomic_long_inc(&inode->i_sb->s_remove_count);
 305}
 306EXPORT_SYMBOL(drop_nlink);
 307
 308/**
 309 * clear_nlink - directly zero an inode's link count
 310 * @inode: inode
 311 *
 312 * This is a low-level filesystem helper to replace any
 313 * direct filesystem manipulation of i_nlink.  See
 314 * drop_nlink() for why we care about i_nlink hitting zero.
 315 */
 316void clear_nlink(struct inode *inode)
 317{
 318        if (inode->i_nlink) {
 319                inode->__i_nlink = 0;
 320                atomic_long_inc(&inode->i_sb->s_remove_count);
 321        }
 322}
 323EXPORT_SYMBOL(clear_nlink);
 324
 325/**
 326 * set_nlink - directly set an inode's link count
 327 * @inode: inode
 328 * @nlink: new nlink (should be non-zero)
 329 *
 330 * This is a low-level filesystem helper to replace any
 331 * direct filesystem manipulation of i_nlink.
 332 */
 333void set_nlink(struct inode *inode, unsigned int nlink)
 334{
 335        if (!nlink) {
 336                clear_nlink(inode);
 337        } else {
 338                /* Yes, some filesystems do change nlink from zero to one */
 339                if (inode->i_nlink == 0)
 340                        atomic_long_dec(&inode->i_sb->s_remove_count);
 341
 342                inode->__i_nlink = nlink;
 343        }
 344}
 345EXPORT_SYMBOL(set_nlink);
 346
 347/**
 348 * inc_nlink - directly increment an inode's link count
 349 * @inode: inode
 350 *
 351 * This is a low-level filesystem helper to replace any
 352 * direct filesystem manipulation of i_nlink.  Currently,
 353 * it is only here for parity with dec_nlink().
 354 */
 355void inc_nlink(struct inode *inode)
 356{
 357        if (unlikely(inode->i_nlink == 0)) {
 358                WARN_ON(!(inode->i_state & I_LINKABLE));
 359                atomic_long_dec(&inode->i_sb->s_remove_count);
 360        }
 361
 362        inode->__i_nlink++;
 363}
 364EXPORT_SYMBOL(inc_nlink);
 365
 366static void __address_space_init_once(struct address_space *mapping)
 367{
 368        xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT);
 369        init_rwsem(&mapping->i_mmap_rwsem);
 370        INIT_LIST_HEAD(&mapping->private_list);
 371        spin_lock_init(&mapping->private_lock);
 372        mapping->i_mmap = RB_ROOT_CACHED;
 373}
 374
 375void address_space_init_once(struct address_space *mapping)
 376{
 377        memset(mapping, 0, sizeof(*mapping));
 378        __address_space_init_once(mapping);
 379}
 380EXPORT_SYMBOL(address_space_init_once);
 381
 382/*
 383 * These are initializations that only need to be done
 384 * once, because the fields are idempotent across use
 385 * of the inode, so let the slab aware of that.
 386 */
 387void inode_init_once(struct inode *inode)
 388{
 389        memset(inode, 0, sizeof(*inode));
 390        INIT_HLIST_NODE(&inode->i_hash);
 391        INIT_LIST_HEAD(&inode->i_devices);
 392        INIT_LIST_HEAD(&inode->i_io_list);
 393        INIT_LIST_HEAD(&inode->i_wb_list);
 394        INIT_LIST_HEAD(&inode->i_lru);
 395        __address_space_init_once(&inode->i_data);
 396        i_size_ordered_init(inode);
 397}
 398EXPORT_SYMBOL(inode_init_once);
 399
 400static void init_once(void *foo)
 401{
 402        struct inode *inode = (struct inode *) foo;
 403
 404        inode_init_once(inode);
 405}
 406
 407/*
 408 * inode->i_lock must be held
 409 */
 410void __iget(struct inode *inode)
 411{
 412        atomic_inc(&inode->i_count);
 413}
 414
 415/*
 416 * get additional reference to inode; caller must already hold one.
 417 */
 418void ihold(struct inode *inode)
 419{
 420        WARN_ON(atomic_inc_return(&inode->i_count) < 2);
 421}
 422EXPORT_SYMBOL(ihold);
 423
 424static void inode_lru_list_add(struct inode *inode)
 425{
 426        if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
 427                this_cpu_inc(nr_unused);
 428        else
 429                inode->i_state |= I_REFERENCED;
 430}
 431
 432/*
 433 * Add inode to LRU if needed (inode is unused and clean).
 434 *
 435 * Needs inode->i_lock held.
 436 */
 437void inode_add_lru(struct inode *inode)
 438{
 439        if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
 440                                I_FREEING | I_WILL_FREE)) &&
 441            !atomic_read(&inode->i_count) && inode->i_sb->s_flags & SB_ACTIVE)
 442                inode_lru_list_add(inode);
 443}
 444
 445
 446static void inode_lru_list_del(struct inode *inode)
 447{
 448
 449        if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
 450                this_cpu_dec(nr_unused);
 451}
 452
 453/**
 454 * inode_sb_list_add - add inode to the superblock list of inodes
 455 * @inode: inode to add
 456 */
 457void inode_sb_list_add(struct inode *inode)
 458{
 459        spin_lock(&inode->i_sb->s_inode_list_lock);
 460        list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
 461        spin_unlock(&inode->i_sb->s_inode_list_lock);
 462}
 463EXPORT_SYMBOL_GPL(inode_sb_list_add);
 464
 465static inline void inode_sb_list_del(struct inode *inode)
 466{
 467        if (!list_empty(&inode->i_sb_list)) {
 468                spin_lock(&inode->i_sb->s_inode_list_lock);
 469                list_del_init(&inode->i_sb_list);
 470                spin_unlock(&inode->i_sb->s_inode_list_lock);
 471        }
 472}
 473
 474static unsigned long hash(struct super_block *sb, unsigned long hashval)
 475{
 476        unsigned long tmp;
 477
 478        tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
 479                        L1_CACHE_BYTES;
 480        tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
 481        return tmp & i_hash_mask;
 482}
 483
 484/**
 485 *      __insert_inode_hash - hash an inode
 486 *      @inode: unhashed inode
 487 *      @hashval: unsigned long value used to locate this object in the
 488 *              inode_hashtable.
 489 *
 490 *      Add an inode to the inode hash for this superblock.
 491 */
 492void __insert_inode_hash(struct inode *inode, unsigned long hashval)
 493{
 494        struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
 495
 496        spin_lock(&inode_hash_lock);
 497        spin_lock(&inode->i_lock);
 498        hlist_add_head(&inode->i_hash, b);
 499        spin_unlock(&inode->i_lock);
 500        spin_unlock(&inode_hash_lock);
 501}
 502EXPORT_SYMBOL(__insert_inode_hash);
 503
 504/**
 505 *      __remove_inode_hash - remove an inode from the hash
 506 *      @inode: inode to unhash
 507 *
 508 *      Remove an inode from the superblock.
 509 */
 510void __remove_inode_hash(struct inode *inode)
 511{
 512        spin_lock(&inode_hash_lock);
 513        spin_lock(&inode->i_lock);
 514        hlist_del_init(&inode->i_hash);
 515        spin_unlock(&inode->i_lock);
 516        spin_unlock(&inode_hash_lock);
 517}
 518EXPORT_SYMBOL(__remove_inode_hash);
 519
 520void clear_inode(struct inode *inode)
 521{
 522        /*
 523         * We have to cycle the i_pages lock here because reclaim can be in the
 524         * process of removing the last page (in __delete_from_page_cache())
 525         * and we must not free the mapping under it.
 526         */
 527        xa_lock_irq(&inode->i_data.i_pages);
 528        BUG_ON(inode->i_data.nrpages);
 529        BUG_ON(inode->i_data.nrexceptional);
 530        xa_unlock_irq(&inode->i_data.i_pages);
 531        BUG_ON(!list_empty(&inode->i_data.private_list));
 532        BUG_ON(!(inode->i_state & I_FREEING));
 533        BUG_ON(inode->i_state & I_CLEAR);
 534        BUG_ON(!list_empty(&inode->i_wb_list));
 535        /* don't need i_lock here, no concurrent mods to i_state */
 536        inode->i_state = I_FREEING | I_CLEAR;
 537}
 538EXPORT_SYMBOL(clear_inode);
 539
 540/*
 541 * Free the inode passed in, removing it from the lists it is still connected
 542 * to. We remove any pages still attached to the inode and wait for any IO that
 543 * is still in progress before finally destroying the inode.
 544 *
 545 * An inode must already be marked I_FREEING so that we avoid the inode being
 546 * moved back onto lists if we race with other code that manipulates the lists
 547 * (e.g. writeback_single_inode). The caller is responsible for setting this.
 548 *
 549 * An inode must already be removed from the LRU list before being evicted from
 550 * the cache. This should occur atomically with setting the I_FREEING state
 551 * flag, so no inodes here should ever be on the LRU when being evicted.
 552 */
 553static void evict(struct inode *inode)
 554{
 555        const struct super_operations *op = inode->i_sb->s_op;
 556
 557        BUG_ON(!(inode->i_state & I_FREEING));
 558        BUG_ON(!list_empty(&inode->i_lru));
 559
 560        if (!list_empty(&inode->i_io_list))
 561                inode_io_list_del(inode);
 562
 563        inode_sb_list_del(inode);
 564
 565        /*
 566         * Wait for flusher thread to be done with the inode so that filesystem
 567         * does not start destroying it while writeback is still running. Since
 568         * the inode has I_FREEING set, flusher thread won't start new work on
 569         * the inode.  We just have to wait for running writeback to finish.
 570         */
 571        inode_wait_for_writeback(inode);
 572
 573        if (op->evict_inode) {
 574                op->evict_inode(inode);
 575        } else {
 576                truncate_inode_pages_final(&inode->i_data);
 577                clear_inode(inode);
 578        }
 579        if (S_ISBLK(inode->i_mode) && inode->i_bdev)
 580                bd_forget(inode);
 581        if (S_ISCHR(inode->i_mode) && inode->i_cdev)
 582                cd_forget(inode);
 583
 584        remove_inode_hash(inode);
 585
 586        spin_lock(&inode->i_lock);
 587        wake_up_bit(&inode->i_state, __I_NEW);
 588        BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
 589        spin_unlock(&inode->i_lock);
 590
 591        destroy_inode(inode);
 592}
 593
 594/*
 595 * dispose_list - dispose of the contents of a local list
 596 * @head: the head of the list to free
 597 *
 598 * Dispose-list gets a local list with local inodes in it, so it doesn't
 599 * need to worry about list corruption and SMP locks.
 600 */
 601static void dispose_list(struct list_head *head)
 602{
 603        while (!list_empty(head)) {
 604                struct inode *inode;
 605
 606                inode = list_first_entry(head, struct inode, i_lru);
 607                list_del_init(&inode->i_lru);
 608
 609                evict(inode);
 610                cond_resched();
 611        }
 612}
 613
 614/**
 615 * evict_inodes - evict all evictable inodes for a superblock
 616 * @sb:         superblock to operate on
 617 *
 618 * Make sure that no inodes with zero refcount are retained.  This is
 619 * called by superblock shutdown after having SB_ACTIVE flag removed,
 620 * so any inode reaching zero refcount during or after that call will
 621 * be immediately evicted.
 622 */
 623void evict_inodes(struct super_block *sb)
 624{
 625        struct inode *inode, *next;
 626        LIST_HEAD(dispose);
 627
 628again:
 629        spin_lock(&sb->s_inode_list_lock);
 630        list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
 631                if (atomic_read(&inode->i_count))
 632                        continue;
 633
 634                spin_lock(&inode->i_lock);
 635                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
 636                        spin_unlock(&inode->i_lock);
 637                        continue;
 638                }
 639
 640                inode->i_state |= I_FREEING;
 641                inode_lru_list_del(inode);
 642                spin_unlock(&inode->i_lock);
 643                list_add(&inode->i_lru, &dispose);
 644
 645                /*
 646                 * We can have a ton of inodes to evict at unmount time given
 647                 * enough memory, check to see if we need to go to sleep for a
 648                 * bit so we don't livelock.
 649                 */
 650                if (need_resched()) {
 651                        spin_unlock(&sb->s_inode_list_lock);
 652                        cond_resched();
 653                        dispose_list(&dispose);
 654                        goto again;
 655                }
 656        }
 657        spin_unlock(&sb->s_inode_list_lock);
 658
 659        dispose_list(&dispose);
 660}
 661EXPORT_SYMBOL_GPL(evict_inodes);
 662
 663/**
 664 * invalidate_inodes    - attempt to free all inodes on a superblock
 665 * @sb:         superblock to operate on
 666 * @kill_dirty: flag to guide handling of dirty inodes
 667 *
 668 * Attempts to free all inodes for a given superblock.  If there were any
 669 * busy inodes return a non-zero value, else zero.
 670 * If @kill_dirty is set, discard dirty inodes too, otherwise treat
 671 * them as busy.
 672 */
 673int invalidate_inodes(struct super_block *sb, bool kill_dirty)
 674{
 675        int busy = 0;
 676        struct inode *inode, *next;
 677        LIST_HEAD(dispose);
 678
 679        spin_lock(&sb->s_inode_list_lock);
 680        list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
 681                spin_lock(&inode->i_lock);
 682                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
 683                        spin_unlock(&inode->i_lock);
 684                        continue;
 685                }
 686                if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
 687                        spin_unlock(&inode->i_lock);
 688                        busy = 1;
 689                        continue;
 690                }
 691                if (atomic_read(&inode->i_count)) {
 692                        spin_unlock(&inode->i_lock);
 693                        busy = 1;
 694                        continue;
 695                }
 696
 697                inode->i_state |= I_FREEING;
 698                inode_lru_list_del(inode);
 699                spin_unlock(&inode->i_lock);
 700                list_add(&inode->i_lru, &dispose);
 701        }
 702        spin_unlock(&sb->s_inode_list_lock);
 703
 704        dispose_list(&dispose);
 705
 706        return busy;
 707}
 708
 709/*
 710 * Isolate the inode from the LRU in preparation for freeing it.
 711 *
 712 * Any inodes which are pinned purely because of attached pagecache have their
 713 * pagecache removed.  If the inode has metadata buffers attached to
 714 * mapping->private_list then try to remove them.
 715 *
 716 * If the inode has the I_REFERENCED flag set, then it means that it has been
 717 * used recently - the flag is set in iput_final(). When we encounter such an
 718 * inode, clear the flag and move it to the back of the LRU so it gets another
 719 * pass through the LRU before it gets reclaimed. This is necessary because of
 720 * the fact we are doing lazy LRU updates to minimise lock contention so the
 721 * LRU does not have strict ordering. Hence we don't want to reclaim inodes
 722 * with this flag set because they are the inodes that are out of order.
 723 */
 724static enum lru_status inode_lru_isolate(struct list_head *item,
 725                struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
 726{
 727        struct list_head *freeable = arg;
 728        struct inode    *inode = container_of(item, struct inode, i_lru);
 729
 730        /*
 731         * we are inverting the lru lock/inode->i_lock here, so use a trylock.
 732         * If we fail to get the lock, just skip it.
 733         */
 734        if (!spin_trylock(&inode->i_lock))
 735                return LRU_SKIP;
 736
 737        /*
 738         * Referenced or dirty inodes are still in use. Give them another pass
 739         * through the LRU as we canot reclaim them now.
 740         */
 741        if (atomic_read(&inode->i_count) ||
 742            (inode->i_state & ~I_REFERENCED)) {
 743                list_lru_isolate(lru, &inode->i_lru);
 744                spin_unlock(&inode->i_lock);
 745                this_cpu_dec(nr_unused);
 746                return LRU_REMOVED;
 747        }
 748
 749        /* recently referenced inodes get one more pass */
 750        if (inode->i_state & I_REFERENCED) {
 751                inode->i_state &= ~I_REFERENCED;
 752                spin_unlock(&inode->i_lock);
 753                return LRU_ROTATE;
 754        }
 755
 756        if (inode_has_buffers(inode) || inode->i_data.nrpages) {
 757                __iget(inode);
 758                spin_unlock(&inode->i_lock);
 759                spin_unlock(lru_lock);
 760                if (remove_inode_buffers(inode)) {
 761                        unsigned long reap;
 762                        reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
 763                        if (current_is_kswapd())
 764                                __count_vm_events(KSWAPD_INODESTEAL, reap);
 765                        else
 766                                __count_vm_events(PGINODESTEAL, reap);
 767                        if (current->reclaim_state)
 768                                current->reclaim_state->reclaimed_slab += reap;
 769                }
 770                iput(inode);
 771                spin_lock(lru_lock);
 772                return LRU_RETRY;
 773        }
 774
 775        WARN_ON(inode->i_state & I_NEW);
 776        inode->i_state |= I_FREEING;
 777        list_lru_isolate_move(lru, &inode->i_lru, freeable);
 778        spin_unlock(&inode->i_lock);
 779
 780        this_cpu_dec(nr_unused);
 781        return LRU_REMOVED;
 782}
 783
 784/*
 785 * Walk the superblock inode LRU for freeable inodes and attempt to free them.
 786 * This is called from the superblock shrinker function with a number of inodes
 787 * to trim from the LRU. Inodes to be freed are moved to a temporary list and
 788 * then are freed outside inode_lock by dispose_list().
 789 */
 790long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
 791{
 792        LIST_HEAD(freeable);
 793        long freed;
 794
 795        freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
 796                                     inode_lru_isolate, &freeable);
 797        dispose_list(&freeable);
 798        return freed;
 799}
 800
 801static void __wait_on_freeing_inode(struct inode *inode);
 802/*
 803 * Called with the inode lock held.
 804 */
 805static struct inode *find_inode(struct super_block *sb,
 806                                struct hlist_head *head,
 807                                int (*test)(struct inode *, void *),
 808                                void *data)
 809{
 810        struct inode *inode = NULL;
 811
 812repeat:
 813        hlist_for_each_entry(inode, head, i_hash) {
 814                if (inode->i_sb != sb)
 815                        continue;
 816                if (!test(inode, data))
 817                        continue;
 818                spin_lock(&inode->i_lock);
 819                if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
 820                        __wait_on_freeing_inode(inode);
 821                        goto repeat;
 822                }
 823                if (unlikely(inode->i_state & I_CREATING)) {
 824                        spin_unlock(&inode->i_lock);
 825                        return ERR_PTR(-ESTALE);
 826                }
 827                __iget(inode);
 828                spin_unlock(&inode->i_lock);
 829                return inode;
 830        }
 831        return NULL;
 832}
 833
 834/*
 835 * find_inode_fast is the fast path version of find_inode, see the comment at
 836 * iget_locked for details.
 837 */
 838static struct inode *find_inode_fast(struct super_block *sb,
 839                                struct hlist_head *head, unsigned long ino)
 840{
 841        struct inode *inode = NULL;
 842
 843repeat:
 844        hlist_for_each_entry(inode, head, i_hash) {
 845                if (inode->i_ino != ino)
 846                        continue;
 847                if (inode->i_sb != sb)
 848                        continue;
 849                spin_lock(&inode->i_lock);
 850                if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
 851                        __wait_on_freeing_inode(inode);
 852                        goto repeat;
 853                }
 854                if (unlikely(inode->i_state & I_CREATING)) {
 855                        spin_unlock(&inode->i_lock);
 856                        return ERR_PTR(-ESTALE);
 857                }
 858                __iget(inode);
 859                spin_unlock(&inode->i_lock);
 860                return inode;
 861        }
 862        return NULL;
 863}
 864
 865/*
 866 * Each cpu owns a range of LAST_INO_BATCH numbers.
 867 * 'shared_last_ino' is dirtied only once out of LAST_INO_BATCH allocations,
 868 * to renew the exhausted range.
 869 *
 870 * This does not significantly increase overflow rate because every CPU can
 871 * consume at most LAST_INO_BATCH-1 unused inode numbers. So there is
 872 * NR_CPUS*(LAST_INO_BATCH-1) wastage. At 4096 and 1024, this is ~0.1% of the
 873 * 2^32 range, and is a worst-case. Even a 50% wastage would only increase
 874 * overflow rate by 2x, which does not seem too significant.
 875 *
 876 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
 877 * error if st_ino won't fit in target struct field. Use 32bit counter
 878 * here to attempt to avoid that.
 879 */
 880#define LAST_INO_BATCH 1024
 881static DEFINE_PER_CPU(unsigned int, last_ino);
 882
 883unsigned int get_next_ino(void)
 884{
 885        unsigned int *p = &get_cpu_var(last_ino);
 886        unsigned int res = *p;
 887
 888#ifdef CONFIG_SMP
 889        if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
 890                static atomic_t shared_last_ino;
 891                int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
 892
 893                res = next - LAST_INO_BATCH;
 894        }
 895#endif
 896
 897        res++;
 898        /* get_next_ino should not provide a 0 inode number */
 899        if (unlikely(!res))
 900                res++;
 901        *p = res;
 902        put_cpu_var(last_ino);
 903        return res;
 904}
 905EXPORT_SYMBOL(get_next_ino);
 906
 907/**
 908 *      new_inode_pseudo        - obtain an inode
 909 *      @sb: superblock
 910 *
 911 *      Allocates a new inode for given superblock.
 912 *      Inode wont be chained in superblock s_inodes list
 913 *      This means :
 914 *      - fs can't be unmount
 915 *      - quotas, fsnotify, writeback can't work
 916 */
 917struct inode *new_inode_pseudo(struct super_block *sb)
 918{
 919        struct inode *inode = alloc_inode(sb);
 920
 921        if (inode) {
 922                spin_lock(&inode->i_lock);
 923                inode->i_state = 0;
 924                spin_unlock(&inode->i_lock);
 925                INIT_LIST_HEAD(&inode->i_sb_list);
 926        }
 927        return inode;
 928}
 929
 930/**
 931 *      new_inode       - obtain an inode
 932 *      @sb: superblock
 933 *
 934 *      Allocates a new inode for given superblock. The default gfp_mask
 935 *      for allocations related to inode->i_mapping is GFP_HIGHUSER_MOVABLE.
 936 *      If HIGHMEM pages are unsuitable or it is known that pages allocated
 937 *      for the page cache are not reclaimable or migratable,
 938 *      mapping_set_gfp_mask() must be called with suitable flags on the
 939 *      newly created inode's mapping
 940 *
 941 */
 942struct inode *new_inode(struct super_block *sb)
 943{
 944        struct inode *inode;
 945
 946        spin_lock_prefetch(&sb->s_inode_list_lock);
 947
 948        inode = new_inode_pseudo(sb);
 949        if (inode)
 950                inode_sb_list_add(inode);
 951        return inode;
 952}
 953EXPORT_SYMBOL(new_inode);
 954
 955#ifdef CONFIG_DEBUG_LOCK_ALLOC
 956void lockdep_annotate_inode_mutex_key(struct inode *inode)
 957{
 958        if (S_ISDIR(inode->i_mode)) {
 959                struct file_system_type *type = inode->i_sb->s_type;
 960
 961                /* Set new key only if filesystem hasn't already changed it */
 962                if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
 963                        /*
 964                         * ensure nobody is actually holding i_mutex
 965                         */
 966                        // mutex_destroy(&inode->i_mutex);
 967                        init_rwsem(&inode->i_rwsem);
 968                        lockdep_set_class(&inode->i_rwsem,
 969                                          &type->i_mutex_dir_key);
 970                }
 971        }
 972}
 973EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
 974#endif
 975
 976/**
 977 * unlock_new_inode - clear the I_NEW state and wake up any waiters
 978 * @inode:      new inode to unlock
 979 *
 980 * Called when the inode is fully initialised to clear the new state of the
 981 * inode and wake up anyone waiting for the inode to finish initialisation.
 982 */
 983void unlock_new_inode(struct inode *inode)
 984{
 985        lockdep_annotate_inode_mutex_key(inode);
 986        spin_lock(&inode->i_lock);
 987        WARN_ON(!(inode->i_state & I_NEW));
 988        inode->i_state &= ~I_NEW & ~I_CREATING;
 989        smp_mb();
 990        wake_up_bit(&inode->i_state, __I_NEW);
 991        spin_unlock(&inode->i_lock);
 992}
 993EXPORT_SYMBOL(unlock_new_inode);
 994
 995void discard_new_inode(struct inode *inode)
 996{
 997        lockdep_annotate_inode_mutex_key(inode);
 998        spin_lock(&inode->i_lock);
 999        WARN_ON(!(inode->i_state & I_NEW));
1000        inode->i_state &= ~I_NEW;
1001        smp_mb();
1002        wake_up_bit(&inode->i_state, __I_NEW);
1003        spin_unlock(&inode->i_lock);
1004        iput(inode);
1005}
1006EXPORT_SYMBOL(discard_new_inode);
1007
1008/**
1009 * lock_two_nondirectories - take two i_mutexes on non-directory objects
1010 *
1011 * Lock any non-NULL argument that is not a directory.
1012 * Zero, one or two objects may be locked by this function.
1013 *
1014 * @inode1: first inode to lock
1015 * @inode2: second inode to lock
1016 */
1017void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1018{
1019        if (inode1 > inode2)
1020                swap(inode1, inode2);
1021
1022        if (inode1 && !S_ISDIR(inode1->i_mode))
1023                inode_lock(inode1);
1024        if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1025                inode_lock_nested(inode2, I_MUTEX_NONDIR2);
1026}
1027EXPORT_SYMBOL(lock_two_nondirectories);
1028
1029/**
1030 * unlock_two_nondirectories - release locks from lock_two_nondirectories()
1031 * @inode1: first inode to unlock
1032 * @inode2: second inode to unlock
1033 */
1034void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1035{
1036        if (inode1 && !S_ISDIR(inode1->i_mode))
1037                inode_unlock(inode1);
1038        if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1039                inode_unlock(inode2);
1040}
1041EXPORT_SYMBOL(unlock_two_nondirectories);
1042
1043/**
1044 * inode_insert5 - obtain an inode from a mounted file system
1045 * @inode:      pre-allocated inode to use for insert to cache
1046 * @hashval:    hash value (usually inode number) to get
1047 * @test:       callback used for comparisons between inodes
1048 * @set:        callback used to initialize a new struct inode
1049 * @data:       opaque data pointer to pass to @test and @set
1050 *
1051 * Search for the inode specified by @hashval and @data in the inode cache,
1052 * and if present it is return it with an increased reference count. This is
1053 * a variant of iget5_locked() for callers that don't want to fail on memory
1054 * allocation of inode.
1055 *
1056 * If the inode is not in cache, insert the pre-allocated inode to cache and
1057 * return it locked, hashed, and with the I_NEW flag set. The file system gets
1058 * to fill it in before unlocking it via unlock_new_inode().
1059 *
1060 * Note both @test and @set are called with the inode_hash_lock held, so can't
1061 * sleep.
1062 */
1063struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
1064                            int (*test)(struct inode *, void *),
1065                            int (*set)(struct inode *, void *), void *data)
1066{
1067        struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
1068        struct inode *old;
1069        bool creating = inode->i_state & I_CREATING;
1070
1071again:
1072        spin_lock(&inode_hash_lock);
1073        old = find_inode(inode->i_sb, head, test, data);
1074        if (unlikely(old)) {
1075                /*
1076                 * Uhhuh, somebody else created the same inode under us.
1077                 * Use the old inode instead of the preallocated one.
1078                 */
1079                spin_unlock(&inode_hash_lock);
1080                if (IS_ERR(old))
1081                        return NULL;
1082                wait_on_inode(old);
1083                if (unlikely(inode_unhashed(old))) {
1084                        iput(old);
1085                        goto again;
1086                }
1087                return old;
1088        }
1089
1090        if (set && unlikely(set(inode, data))) {
1091                inode = NULL;
1092                goto unlock;
1093        }
1094
1095        /*
1096         * Return the locked inode with I_NEW set, the
1097         * caller is responsible for filling in the contents
1098         */
1099        spin_lock(&inode->i_lock);
1100        inode->i_state |= I_NEW;
1101        hlist_add_head(&inode->i_hash, head);
1102        spin_unlock(&inode->i_lock);
1103        if (!creating)
1104                inode_sb_list_add(inode);
1105unlock:
1106        spin_unlock(&inode_hash_lock);
1107
1108        return inode;
1109}
1110EXPORT_SYMBOL(inode_insert5);
1111
1112/**
1113 * iget5_locked - obtain an inode from a mounted file system
1114 * @sb:         super block of file system
1115 * @hashval:    hash value (usually inode number) to get
1116 * @test:       callback used for comparisons between inodes
1117 * @set:        callback used to initialize a new struct inode
1118 * @data:       opaque data pointer to pass to @test and @set
1119 *
1120 * Search for the inode specified by @hashval and @data in the inode cache,
1121 * and if present it is return it with an increased reference count. This is
1122 * a generalized version of iget_locked() for file systems where the inode
1123 * number is not sufficient for unique identification of an inode.
1124 *
1125 * If the inode is not in cache, allocate a new inode and return it locked,
1126 * hashed, and with the I_NEW flag set. The file system gets to fill it in
1127 * before unlocking it via unlock_new_inode().
1128 *
1129 * Note both @test and @set are called with the inode_hash_lock held, so can't
1130 * sleep.
1131 */
1132struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
1133                int (*test)(struct inode *, void *),
1134                int (*set)(struct inode *, void *), void *data)
1135{
1136        struct inode *inode = ilookup5(sb, hashval, test, data);
1137
1138        if (!inode) {
1139                struct inode *new = alloc_inode(sb);
1140
1141                if (new) {
1142                        new->i_state = 0;
1143                        inode = inode_insert5(new, hashval, test, set, data);
1144                        if (unlikely(inode != new))
1145                                destroy_inode(new);
1146                }
1147        }
1148        return inode;
1149}
1150EXPORT_SYMBOL(iget5_locked);
1151
1152/**
1153 * iget_locked - obtain an inode from a mounted file system
1154 * @sb:         super block of file system
1155 * @ino:        inode number to get
1156 *
1157 * Search for the inode specified by @ino in the inode cache and if present
1158 * return it with an increased reference count. This is for file systems
1159 * where the inode number is sufficient for unique identification of an inode.
1160 *
1161 * If the inode is not in cache, allocate a new inode and return it locked,
1162 * hashed, and with the I_NEW flag set.  The file system gets to fill it in
1163 * before unlocking it via unlock_new_inode().
1164 */
1165struct inode *iget_locked(struct super_block *sb, unsigned long ino)
1166{
1167        struct hlist_head *head = inode_hashtable + hash(sb, ino);
1168        struct inode *inode;
1169again:
1170        spin_lock(&inode_hash_lock);
1171        inode = find_inode_fast(sb, head, ino);
1172        spin_unlock(&inode_hash_lock);
1173        if (inode) {
1174                if (IS_ERR(inode))
1175                        return NULL;
1176                wait_on_inode(inode);
1177                if (unlikely(inode_unhashed(inode))) {
1178                        iput(inode);
1179                        goto again;
1180                }
1181                return inode;
1182        }
1183
1184        inode = alloc_inode(sb);
1185        if (inode) {
1186                struct inode *old;
1187
1188                spin_lock(&inode_hash_lock);
1189                /* We released the lock, so.. */
1190                old = find_inode_fast(sb, head, ino);
1191                if (!old) {
1192                        inode->i_ino = ino;
1193                        spin_lock(&inode->i_lock);
1194                        inode->i_state = I_NEW;
1195                        hlist_add_head(&inode->i_hash, head);
1196                        spin_unlock(&inode->i_lock);
1197                        inode_sb_list_add(inode);
1198                        spin_unlock(&inode_hash_lock);
1199
1200                        /* Return the locked inode with I_NEW set, the
1201                         * caller is responsible for filling in the contents
1202                         */
1203                        return inode;
1204                }
1205
1206                /*
1207                 * Uhhuh, somebody else created the same inode under
1208                 * us. Use the old inode instead of the one we just
1209                 * allocated.
1210                 */
1211                spin_unlock(&inode_hash_lock);
1212                destroy_inode(inode);
1213                if (IS_ERR(old))
1214                        return NULL;
1215                inode = old;
1216                wait_on_inode(inode);
1217                if (unlikely(inode_unhashed(inode))) {
1218                        iput(inode);
1219                        goto again;
1220                }
1221        }
1222        return inode;
1223}
1224EXPORT_SYMBOL(iget_locked);
1225
1226/*
1227 * search the inode cache for a matching inode number.
1228 * If we find one, then the inode number we are trying to
1229 * allocate is not unique and so we should not use it.
1230 *
1231 * Returns 1 if the inode number is unique, 0 if it is not.
1232 */
1233static int test_inode_iunique(struct super_block *sb, unsigned long ino)
1234{
1235        struct hlist_head *b = inode_hashtable + hash(sb, ino);
1236        struct inode *inode;
1237
1238        spin_lock(&inode_hash_lock);
1239        hlist_for_each_entry(inode, b, i_hash) {
1240                if (inode->i_ino == ino && inode->i_sb == sb) {
1241                        spin_unlock(&inode_hash_lock);
1242                        return 0;
1243                }
1244        }
1245        spin_unlock(&inode_hash_lock);
1246
1247        return 1;
1248}
1249
1250/**
1251 *      iunique - get a unique inode number
1252 *      @sb: superblock
1253 *      @max_reserved: highest reserved inode number
1254 *
1255 *      Obtain an inode number that is unique on the system for a given
1256 *      superblock. This is used by file systems that have no natural
1257 *      permanent inode numbering system. An inode number is returned that
1258 *      is higher than the reserved limit but unique.
1259 *
1260 *      BUGS:
1261 *      With a large number of inodes live on the file system this function
1262 *      currently becomes quite slow.
1263 */
1264ino_t iunique(struct super_block *sb, ino_t max_reserved)
1265{
1266        /*
1267         * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
1268         * error if st_ino won't fit in target struct field. Use 32bit counter
1269         * here to attempt to avoid that.
1270         */
1271        static DEFINE_SPINLOCK(iunique_lock);
1272        static unsigned int counter;
1273        ino_t res;
1274
1275        spin_lock(&iunique_lock);
1276        do {
1277                if (counter <= max_reserved)
1278                        counter = max_reserved + 1;
1279                res = counter++;
1280        } while (!test_inode_iunique(sb, res));
1281        spin_unlock(&iunique_lock);
1282
1283        return res;
1284}
1285EXPORT_SYMBOL(iunique);
1286
1287struct inode *igrab(struct inode *inode)
1288{
1289        spin_lock(&inode->i_lock);
1290        if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
1291                __iget(inode);
1292                spin_unlock(&inode->i_lock);
1293        } else {
1294                spin_unlock(&inode->i_lock);
1295                /*
1296                 * Handle the case where s_op->clear_inode is not been
1297                 * called yet, and somebody is calling igrab
1298                 * while the inode is getting freed.
1299                 */
1300                inode = NULL;
1301        }
1302        return inode;
1303}
1304EXPORT_SYMBOL(igrab);
1305
1306/**
1307 * ilookup5_nowait - search for an inode in the inode cache
1308 * @sb:         super block of file system to search
1309 * @hashval:    hash value (usually inode number) to search for
1310 * @test:       callback used for comparisons between inodes
1311 * @data:       opaque data pointer to pass to @test
1312 *
1313 * Search for the inode specified by @hashval and @data in the inode cache.
1314 * If the inode is in the cache, the inode is returned with an incremented
1315 * reference count.
1316 *
1317 * Note: I_NEW is not waited upon so you have to be very careful what you do
1318 * with the returned inode.  You probably should be using ilookup5() instead.
1319 *
1320 * Note2: @test is called with the inode_hash_lock held, so can't sleep.
1321 */
1322struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
1323                int (*test)(struct inode *, void *), void *data)
1324{
1325        struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1326        struct inode *inode;
1327
1328        spin_lock(&inode_hash_lock);
1329        inode = find_inode(sb, head, test, data);
1330        spin_unlock(&inode_hash_lock);
1331
1332        return IS_ERR(inode) ? NULL : inode;
1333}
1334EXPORT_SYMBOL(ilookup5_nowait);
1335
1336/**
1337 * ilookup5 - search for an inode in the inode cache
1338 * @sb:         super block of file system to search
1339 * @hashval:    hash value (usually inode number) to search for
1340 * @test:       callback used for comparisons between inodes
1341 * @data:       opaque data pointer to pass to @test
1342 *
1343 * Search for the inode specified by @hashval and @data in the inode cache,
1344 * and if the inode is in the cache, return the inode with an incremented
1345 * reference count.  Waits on I_NEW before returning the inode.
1346 * returned with an incremented reference count.
1347 *
1348 * This is a generalized version of ilookup() for file systems where the
1349 * inode number is not sufficient for unique identification of an inode.
1350 *
1351 * Note: @test is called with the inode_hash_lock held, so can't sleep.
1352 */
1353struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
1354                int (*test)(struct inode *, void *), void *data)
1355{
1356        struct inode *inode;
1357again:
1358        inode = ilookup5_nowait(sb, hashval, test, data);
1359        if (inode) {
1360                wait_on_inode(inode);
1361                if (unlikely(inode_unhashed(inode))) {
1362                        iput(inode);
1363                        goto again;
1364                }
1365        }
1366        return inode;
1367}
1368EXPORT_SYMBOL(ilookup5);
1369
1370/**
1371 * ilookup - search for an inode in the inode cache
1372 * @sb:         super block of file system to search
1373 * @ino:        inode number to search for
1374 *
1375 * Search for the inode @ino in the inode cache, and if the inode is in the
1376 * cache, the inode is returned with an incremented reference count.
1377 */
1378struct inode *ilookup(struct super_block *sb, unsigned long ino)
1379{
1380        struct hlist_head *head = inode_hashtable + hash(sb, ino);
1381        struct inode *inode;
1382again:
1383        spin_lock(&inode_hash_lock);
1384        inode = find_inode_fast(sb, head, ino);
1385        spin_unlock(&inode_hash_lock);
1386
1387        if (inode) {
1388                if (IS_ERR(inode))
1389                        return NULL;
1390                wait_on_inode(inode);
1391                if (unlikely(inode_unhashed(inode))) {
1392                        iput(inode);
1393                        goto again;
1394                }
1395        }
1396        return inode;
1397}
1398EXPORT_SYMBOL(ilookup);
1399
1400/**
1401 * find_inode_nowait - find an inode in the inode cache
1402 * @sb:         super block of file system to search
1403 * @hashval:    hash value (usually inode number) to search for
1404 * @match:      callback used for comparisons between inodes
1405 * @data:       opaque data pointer to pass to @match
1406 *
1407 * Search for the inode specified by @hashval and @data in the inode
1408 * cache, where the helper function @match will return 0 if the inode
1409 * does not match, 1 if the inode does match, and -1 if the search
1410 * should be stopped.  The @match function must be responsible for
1411 * taking the i_lock spin_lock and checking i_state for an inode being
1412 * freed or being initialized, and incrementing the reference count
1413 * before returning 1.  It also must not sleep, since it is called with
1414 * the inode_hash_lock spinlock held.
1415 *
1416 * This is a even more generalized version of ilookup5() when the
1417 * function must never block --- find_inode() can block in
1418 * __wait_on_freeing_inode() --- or when the caller can not increment
1419 * the reference count because the resulting iput() might cause an
1420 * inode eviction.  The tradeoff is that the @match funtion must be
1421 * very carefully implemented.
1422 */
1423struct inode *find_inode_nowait(struct super_block *sb,
1424                                unsigned long hashval,
1425                                int (*match)(struct inode *, unsigned long,
1426                                             void *),
1427                                void *data)
1428{
1429        struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1430        struct inode *inode, *ret_inode = NULL;
1431        int mval;
1432
1433        spin_lock(&inode_hash_lock);
1434        hlist_for_each_entry(inode, head, i_hash) {
1435                if (inode->i_sb != sb)
1436                        continue;
1437                mval = match(inode, hashval, data);
1438                if (mval == 0)
1439                        continue;
1440                if (mval == 1)
1441                        ret_inode = inode;
1442                goto out;
1443        }
1444out:
1445        spin_unlock(&inode_hash_lock);
1446        return ret_inode;
1447}
1448EXPORT_SYMBOL(find_inode_nowait);
1449
1450int insert_inode_locked(struct inode *inode)
1451{
1452        struct super_block *sb = inode->i_sb;
1453        ino_t ino = inode->i_ino;
1454        struct hlist_head *head = inode_hashtable + hash(sb, ino);
1455
1456        while (1) {
1457                struct inode *old = NULL;
1458                spin_lock(&inode_hash_lock);
1459                hlist_for_each_entry(old, head, i_hash) {
1460                        if (old->i_ino != ino)
1461                                continue;
1462                        if (old->i_sb != sb)
1463                                continue;
1464                        spin_lock(&old->i_lock);
1465                        if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1466                                spin_unlock(&old->i_lock);
1467                                continue;
1468                        }
1469                        break;
1470                }
1471                if (likely(!old)) {
1472                        spin_lock(&inode->i_lock);
1473                        inode->i_state |= I_NEW | I_CREATING;
1474                        hlist_add_head(&inode->i_hash, head);
1475                        spin_unlock(&inode->i_lock);
1476                        spin_unlock(&inode_hash_lock);
1477                        return 0;
1478                }
1479                if (unlikely(old->i_state & I_CREATING)) {
1480                        spin_unlock(&old->i_lock);
1481                        spin_unlock(&inode_hash_lock);
1482                        return -EBUSY;
1483                }
1484                __iget(old);
1485                spin_unlock(&old->i_lock);
1486                spin_unlock(&inode_hash_lock);
1487                wait_on_inode(old);
1488                if (unlikely(!inode_unhashed(old))) {
1489                        iput(old);
1490                        return -EBUSY;
1491                }
1492                iput(old);
1493        }
1494}
1495EXPORT_SYMBOL(insert_inode_locked);
1496
1497int insert_inode_locked4(struct inode *inode, unsigned long hashval,
1498                int (*test)(struct inode *, void *), void *data)
1499{
1500        struct inode *old;
1501
1502        inode->i_state |= I_CREATING;
1503        old = inode_insert5(inode, hashval, test, NULL, data);
1504
1505        if (old != inode) {
1506                iput(old);
1507                return -EBUSY;
1508        }
1509        return 0;
1510}
1511EXPORT_SYMBOL(insert_inode_locked4);
1512
1513
1514int generic_delete_inode(struct inode *inode)
1515{
1516        return 1;
1517}
1518EXPORT_SYMBOL(generic_delete_inode);
1519
1520/*
1521 * Called when we're dropping the last reference
1522 * to an inode.
1523 *
1524 * Call the FS "drop_inode()" function, defaulting to
1525 * the legacy UNIX filesystem behaviour.  If it tells
1526 * us to evict inode, do so.  Otherwise, retain inode
1527 * in cache if fs is alive, sync and evict if fs is
1528 * shutting down.
1529 */
1530static void iput_final(struct inode *inode)
1531{
1532        struct super_block *sb = inode->i_sb;
1533        const struct super_operations *op = inode->i_sb->s_op;
1534        int drop;
1535
1536        WARN_ON(inode->i_state & I_NEW);
1537
1538        if (op->drop_inode)
1539                drop = op->drop_inode(inode);
1540        else
1541                drop = generic_drop_inode(inode);
1542
1543        if (!drop && (sb->s_flags & SB_ACTIVE)) {
1544                inode_add_lru(inode);
1545                spin_unlock(&inode->i_lock);
1546                return;
1547        }
1548
1549        if (!drop) {
1550                inode->i_state |= I_WILL_FREE;
1551                spin_unlock(&inode->i_lock);
1552                write_inode_now(inode, 1);
1553                spin_lock(&inode->i_lock);
1554                WARN_ON(inode->i_state & I_NEW);
1555                inode->i_state &= ~I_WILL_FREE;
1556        }
1557
1558        inode->i_state |= I_FREEING;
1559        if (!list_empty(&inode->i_lru))
1560                inode_lru_list_del(inode);
1561        spin_unlock(&inode->i_lock);
1562
1563        evict(inode);
1564}
1565
1566/**
1567 *      iput    - put an inode
1568 *      @inode: inode to put
1569 *
1570 *      Puts an inode, dropping its usage count. If the inode use count hits
1571 *      zero, the inode is then freed and may also be destroyed.
1572 *
1573 *      Consequently, iput() can sleep.
1574 */
1575void iput(struct inode *inode)
1576{
1577        if (!inode)
1578                return;
1579        BUG_ON(inode->i_state & I_CLEAR);
1580retry:
1581        if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
1582                if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
1583                        atomic_inc(&inode->i_count);
1584                        spin_unlock(&inode->i_lock);
1585                        trace_writeback_lazytime_iput(inode);
1586                        mark_inode_dirty_sync(inode);
1587                        goto retry;
1588                }
1589                iput_final(inode);
1590        }
1591}
1592EXPORT_SYMBOL(iput);
1593
1594/**
1595 *      bmap    - find a block number in a file
1596 *      @inode: inode of file
1597 *      @block: block to find
1598 *
1599 *      Returns the block number on the device holding the inode that
1600 *      is the disk block number for the block of the file requested.
1601 *      That is, asked for block 4 of inode 1 the function will return the
1602 *      disk block relative to the disk start that holds that block of the
1603 *      file.
1604 */
1605sector_t bmap(struct inode *inode, sector_t block)
1606{
1607        sector_t res = 0;
1608        if (inode->i_mapping->a_ops->bmap)
1609                res = inode->i_mapping->a_ops->bmap(inode->i_mapping, block);
1610        return res;
1611}
1612EXPORT_SYMBOL(bmap);
1613
1614/*
1615 * With relative atime, only update atime if the previous atime is
1616 * earlier than either the ctime or mtime or if at least a day has
1617 * passed since the last atime update.
1618 */
1619static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,
1620                             struct timespec64 now)
1621{
1622
1623        if (!(mnt->mnt_flags & MNT_RELATIME))
1624                return 1;
1625        /*
1626         * Is mtime younger than atime? If yes, update atime:
1627         */
1628        if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)
1629                return 1;
1630        /*
1631         * Is ctime younger than atime? If yes, update atime:
1632         */
1633        if (timespec64_compare(&inode->i_ctime, &inode->i_atime) >= 0)
1634                return 1;
1635
1636        /*
1637         * Is the previous atime value older than a day? If yes,
1638         * update atime:
1639         */
1640        if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
1641                return 1;
1642        /*
1643         * Good, we can skip the atime update:
1644         */
1645        return 0;
1646}
1647
1648int generic_update_time(struct inode *inode, struct timespec64 *time, int flags)
1649{
1650        int iflags = I_DIRTY_TIME;
1651        bool dirty = false;
1652
1653        if (flags & S_ATIME)
1654                inode->i_atime = *time;
1655        if (flags & S_VERSION)
1656                dirty = inode_maybe_inc_iversion(inode, false);
1657        if (flags & S_CTIME)
1658                inode->i_ctime = *time;
1659        if (flags & S_MTIME)
1660                inode->i_mtime = *time;
1661        if ((flags & (S_ATIME | S_CTIME | S_MTIME)) &&
1662            !(inode->i_sb->s_flags & SB_LAZYTIME))
1663                dirty = true;
1664
1665        if (dirty)
1666                iflags |= I_DIRTY_SYNC;
1667        __mark_inode_dirty(inode, iflags);
1668        return 0;
1669}
1670EXPORT_SYMBOL(generic_update_time);
1671
1672/*
1673 * This does the actual work of updating an inodes time or version.  Must have
1674 * had called mnt_want_write() before calling this.
1675 */
1676static int update_time(struct inode *inode, struct timespec64 *time, int flags)
1677{
1678        int (*update_time)(struct inode *, struct timespec64 *, int);
1679
1680        update_time = inode->i_op->update_time ? inode->i_op->update_time :
1681                generic_update_time;
1682
1683        return update_time(inode, time, flags);
1684}
1685
1686/**
1687 *      touch_atime     -       update the access time
1688 *      @path: the &struct path to update
1689 *      @inode: inode to update
1690 *
1691 *      Update the accessed time on an inode and mark it for writeback.
1692 *      This function automatically handles read only file systems and media,
1693 *      as well as the "noatime" flag and inode specific "noatime" markers.
1694 */
1695bool atime_needs_update(const struct path *path, struct inode *inode)
1696{
1697        struct vfsmount *mnt = path->mnt;
1698        struct timespec64 now;
1699
1700        if (inode->i_flags & S_NOATIME)
1701                return false;
1702
1703        /* Atime updates will likely cause i_uid and i_gid to be written
1704         * back improprely if their true value is unknown to the vfs.
1705         */
1706        if (HAS_UNMAPPED_ID(inode))
1707                return false;
1708
1709        if (IS_NOATIME(inode))
1710                return false;
1711        if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
1712                return false;
1713
1714        if (mnt->mnt_flags & MNT_NOATIME)
1715                return false;
1716        if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1717                return false;
1718
1719        now = current_time(inode);
1720
1721        if (!relatime_need_update(mnt, inode, now))
1722                return false;
1723
1724        if (timespec64_equal(&inode->i_atime, &now))
1725                return false;
1726
1727        return true;
1728}
1729
1730void touch_atime(const struct path *path)
1731{
1732        struct vfsmount *mnt = path->mnt;
1733        struct inode *inode = d_inode(path->dentry);
1734        struct timespec64 now;
1735
1736        if (!atime_needs_update(path, inode))
1737                return;
1738
1739        if (!sb_start_write_trylock(inode->i_sb))
1740                return;
1741
1742        if (__mnt_want_write(mnt) != 0)
1743                goto skip_update;
1744        /*
1745         * File systems can error out when updating inodes if they need to
1746         * allocate new space to modify an inode (such is the case for
1747         * Btrfs), but since we touch atime while walking down the path we
1748         * really don't care if we failed to update the atime of the file,
1749         * so just ignore the return value.
1750         * We may also fail on filesystems that have the ability to make parts
1751         * of the fs read only, e.g. subvolumes in Btrfs.
1752         */
1753        now = current_time(inode);
1754        update_time(inode, &now, S_ATIME);
1755        __mnt_drop_write(mnt);
1756skip_update:
1757        sb_end_write(inode->i_sb);
1758}
1759EXPORT_SYMBOL(touch_atime);
1760
1761/*
1762 * The logic we want is
1763 *
1764 *      if suid or (sgid and xgrp)
1765 *              remove privs
1766 */
1767int should_remove_suid(struct dentry *dentry)
1768{
1769        umode_t mode = d_inode(dentry)->i_mode;
1770        int kill = 0;
1771
1772        /* suid always must be killed */
1773        if (unlikely(mode & S_ISUID))
1774                kill = ATTR_KILL_SUID;
1775
1776        /*
1777         * sgid without any exec bits is just a mandatory locking mark; leave
1778         * it alone.  If some exec bits are set, it's a real sgid; kill it.
1779         */
1780        if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1781                kill |= ATTR_KILL_SGID;
1782
1783        if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1784                return kill;
1785
1786        return 0;
1787}
1788EXPORT_SYMBOL(should_remove_suid);
1789
1790/*
1791 * Return mask of changes for notify_change() that need to be done as a
1792 * response to write or truncate. Return 0 if nothing has to be changed.
1793 * Negative value on error (change should be denied).
1794 */
1795int dentry_needs_remove_privs(struct dentry *dentry)
1796{
1797        struct inode *inode = d_inode(dentry);
1798        int mask = 0;
1799        int ret;
1800
1801        if (IS_NOSEC(inode))
1802                return 0;
1803
1804        mask = should_remove_suid(dentry);
1805        ret = security_inode_need_killpriv(dentry);
1806        if (ret < 0)
1807                return ret;
1808        if (ret)
1809                mask |= ATTR_KILL_PRIV;
1810        return mask;
1811}
1812
1813static int __remove_privs(struct dentry *dentry, int kill)
1814{
1815        struct iattr newattrs;
1816
1817        newattrs.ia_valid = ATTR_FORCE | kill;
1818        /*
1819         * Note we call this on write, so notify_change will not
1820         * encounter any conflicting delegations:
1821         */
1822        return notify_change(dentry, &newattrs, NULL);
1823}
1824
1825/*
1826 * Remove special file priviledges (suid, capabilities) when file is written
1827 * to or truncated.
1828 */
1829int file_remove_privs(struct file *file)
1830{
1831        struct dentry *dentry = file_dentry(file);
1832        struct inode *inode = file_inode(file);
1833        int kill;
1834        int error = 0;
1835
1836        /*
1837         * Fast path for nothing security related.
1838         * As well for non-regular files, e.g. blkdev inodes.
1839         * For example, blkdev_write_iter() might get here
1840         * trying to remove privs which it is not allowed to.
1841         */
1842        if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))
1843                return 0;
1844
1845        kill = dentry_needs_remove_privs(dentry);
1846        if (kill < 0)
1847                return kill;
1848        if (kill)
1849                error = __remove_privs(dentry, kill);
1850        if (!error)
1851                inode_has_no_xattr(inode);
1852
1853        return error;
1854}
1855EXPORT_SYMBOL(file_remove_privs);
1856
1857/**
1858 *      file_update_time        -       update mtime and ctime time
1859 *      @file: file accessed
1860 *
1861 *      Update the mtime and ctime members of an inode and mark the inode
1862 *      for writeback.  Note that this function is meant exclusively for
1863 *      usage in the file write path of filesystems, and filesystems may
1864 *      choose to explicitly ignore update via this function with the
1865 *      S_NOCMTIME inode flag, e.g. for network filesystem where these
1866 *      timestamps are handled by the server.  This can return an error for
1867 *      file systems who need to allocate space in order to update an inode.
1868 */
1869
1870int file_update_time(struct file *file)
1871{
1872        struct inode *inode = file_inode(file);
1873        struct timespec64 now;
1874        int sync_it = 0;
1875        int ret;
1876
1877        /* First try to exhaust all avenues to not sync */
1878        if (IS_NOCMTIME(inode))
1879                return 0;
1880
1881        now = current_time(inode);
1882        if (!timespec64_equal(&inode->i_mtime, &now))
1883                sync_it = S_MTIME;
1884
1885        if (!timespec64_equal(&inode->i_ctime, &now))
1886                sync_it |= S_CTIME;
1887
1888        if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))
1889                sync_it |= S_VERSION;
1890
1891        if (!sync_it)
1892                return 0;
1893
1894        /* Finally allowed to write? Takes lock. */
1895        if (__mnt_want_write_file(file))
1896                return 0;
1897
1898        ret = update_time(inode, &now, sync_it);
1899        __mnt_drop_write_file(file);
1900
1901        return ret;
1902}
1903EXPORT_SYMBOL(file_update_time);
1904
1905/* Caller must hold the file's inode lock */
1906int file_modified(struct file *file)
1907{
1908        int err;
1909
1910        /*
1911         * Clear the security bits if the process is not being run by root.
1912         * This keeps people from modifying setuid and setgid binaries.
1913         */
1914        err = file_remove_privs(file);
1915        if (err)
1916                return err;
1917
1918        if (unlikely(file->f_mode & FMODE_NOCMTIME))
1919                return 0;
1920
1921        return file_update_time(file);
1922}
1923EXPORT_SYMBOL(file_modified);
1924
1925int inode_needs_sync(struct inode *inode)
1926{
1927        if (IS_SYNC(inode))
1928                return 1;
1929        if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
1930                return 1;
1931        return 0;
1932}
1933EXPORT_SYMBOL(inode_needs_sync);
1934
1935/*
1936 * If we try to find an inode in the inode hash while it is being
1937 * deleted, we have to wait until the filesystem completes its
1938 * deletion before reporting that it isn't found.  This function waits
1939 * until the deletion _might_ have completed.  Callers are responsible
1940 * to recheck inode state.
1941 *
1942 * It doesn't matter if I_NEW is not set initially, a call to
1943 * wake_up_bit(&inode->i_state, __I_NEW) after removing from the hash list
1944 * will DTRT.
1945 */
1946static void __wait_on_freeing_inode(struct inode *inode)
1947{
1948        wait_queue_head_t *wq;
1949        DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
1950        wq = bit_waitqueue(&inode->i_state, __I_NEW);
1951        prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
1952        spin_unlock(&inode->i_lock);
1953        spin_unlock(&inode_hash_lock);
1954        schedule();
1955        finish_wait(wq, &wait.wq_entry);
1956        spin_lock(&inode_hash_lock);
1957}
1958
1959static __initdata unsigned long ihash_entries;
1960static int __init set_ihash_entries(char *str)
1961{
1962        if (!str)
1963                return 0;
1964        ihash_entries = simple_strtoul(str, &str, 0);
1965        return 1;
1966}
1967__setup("ihash_entries=", set_ihash_entries);
1968
1969/*
1970 * Initialize the waitqueues and inode hash table.
1971 */
1972void __init inode_init_early(void)
1973{
1974        /* If hashes are distributed across NUMA nodes, defer
1975         * hash allocation until vmalloc space is available.
1976         */
1977        if (hashdist)
1978                return;
1979
1980        inode_hashtable =
1981                alloc_large_system_hash("Inode-cache",
1982                                        sizeof(struct hlist_head),
1983                                        ihash_entries,
1984                                        14,
1985                                        HASH_EARLY | HASH_ZERO,
1986                                        &i_hash_shift,
1987                                        &i_hash_mask,
1988                                        0,
1989                                        0);
1990}
1991
1992void __init inode_init(void)
1993{
1994        /* inode slab cache */
1995        inode_cachep = kmem_cache_create("inode_cache",
1996                                         sizeof(struct inode),
1997                                         0,
1998                                         (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
1999                                         SLAB_MEM_SPREAD|SLAB_ACCOUNT),
2000                                         init_once);
2001
2002        /* Hash may have been set up in inode_init_early */
2003        if (!hashdist)
2004                return;
2005
2006        inode_hashtable =
2007                alloc_large_system_hash("Inode-cache",
2008                                        sizeof(struct hlist_head),
2009                                        ihash_entries,
2010                                        14,
2011                                        HASH_ZERO,
2012                                        &i_hash_shift,
2013                                        &i_hash_mask,
2014                                        0,
2015                                        0);
2016}
2017
2018void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
2019{
2020        inode->i_mode = mode;
2021        if (S_ISCHR(mode)) {
2022                inode->i_fop = &def_chr_fops;
2023                inode->i_rdev = rdev;
2024        } else if (S_ISBLK(mode)) {
2025                inode->i_fop = &def_blk_fops;
2026                inode->i_rdev = rdev;
2027        } else if (S_ISFIFO(mode))
2028                inode->i_fop = &pipefifo_fops;
2029        else if (S_ISSOCK(mode))
2030                ;       /* leave it no_open_fops */
2031        else
2032                printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
2033                                  " inode %s:%lu\n", mode, inode->i_sb->s_id,
2034                                  inode->i_ino);
2035}
2036EXPORT_SYMBOL(init_special_inode);
2037
2038/**
2039 * inode_init_owner - Init uid,gid,mode for new inode according to posix standards
2040 * @inode: New inode
2041 * @dir: Directory inode
2042 * @mode: mode of the new inode
2043 */
2044void inode_init_owner(struct inode *inode, const struct inode *dir,
2045                        umode_t mode)
2046{
2047        inode->i_uid = current_fsuid();
2048        if (dir && dir->i_mode & S_ISGID) {
2049                inode->i_gid = dir->i_gid;
2050
2051                /* Directories are special, and always inherit S_ISGID */
2052                if (S_ISDIR(mode))
2053                        mode |= S_ISGID;
2054                else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
2055                         !in_group_p(inode->i_gid) &&
2056                         !capable_wrt_inode_uidgid(dir, CAP_FSETID))
2057                        mode &= ~S_ISGID;
2058        } else
2059                inode->i_gid = current_fsgid();
2060        inode->i_mode = mode;
2061}
2062EXPORT_SYMBOL(inode_init_owner);
2063
2064/**
2065 * inode_owner_or_capable - check current task permissions to inode
2066 * @inode: inode being checked
2067 *
2068 * Return true if current either has CAP_FOWNER in a namespace with the
2069 * inode owner uid mapped, or owns the file.
2070 */
2071bool inode_owner_or_capable(const struct inode *inode)
2072{
2073        struct user_namespace *ns;
2074
2075        if (uid_eq(current_fsuid(), inode->i_uid))
2076                return true;
2077
2078        ns = current_user_ns();
2079        if (kuid_has_mapping(ns, inode->i_uid) && ns_capable(ns, CAP_FOWNER))
2080                return true;
2081        return false;
2082}
2083EXPORT_SYMBOL(inode_owner_or_capable);
2084
2085/*
2086 * Direct i/o helper functions
2087 */
2088static void __inode_dio_wait(struct inode *inode)
2089{
2090        wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
2091        DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
2092
2093        do {
2094                prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
2095                if (atomic_read(&inode->i_dio_count))
2096                        schedule();
2097        } while (atomic_read(&inode->i_dio_count));
2098        finish_wait(wq, &q.wq_entry);
2099}
2100
2101/**
2102 * inode_dio_wait - wait for outstanding DIO requests to finish
2103 * @inode: inode to wait for
2104 *
2105 * Waits for all pending direct I/O requests to finish so that we can
2106 * proceed with a truncate or equivalent operation.
2107 *
2108 * Must be called under a lock that serializes taking new references
2109 * to i_dio_count, usually by inode->i_mutex.
2110 */
2111void inode_dio_wait(struct inode *inode)
2112{
2113        if (atomic_read(&inode->i_dio_count))
2114                __inode_dio_wait(inode);
2115}
2116EXPORT_SYMBOL(inode_dio_wait);
2117
2118/*
2119 * inode_set_flags - atomically set some inode flags
2120 *
2121 * Note: the caller should be holding i_mutex, or else be sure that
2122 * they have exclusive access to the inode structure (i.e., while the
2123 * inode is being instantiated).  The reason for the cmpxchg() loop
2124 * --- which wouldn't be necessary if all code paths which modify
2125 * i_flags actually followed this rule, is that there is at least one
2126 * code path which doesn't today so we use cmpxchg() out of an abundance
2127 * of caution.
2128 *
2129 * In the long run, i_mutex is overkill, and we should probably look
2130 * at using the i_lock spinlock to protect i_flags, and then make sure
2131 * it is so documented in include/linux/fs.h and that all code follows
2132 * the locking convention!!
2133 */
2134void inode_set_flags(struct inode *inode, unsigned int flags,
2135                     unsigned int mask)
2136{
2137        WARN_ON_ONCE(flags & ~mask);
2138        set_mask_bits(&inode->i_flags, mask, flags);
2139}
2140EXPORT_SYMBOL(inode_set_flags);
2141
2142void inode_nohighmem(struct inode *inode)
2143{
2144        mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
2145}
2146EXPORT_SYMBOL(inode_nohighmem);
2147
2148/**
2149 * timespec64_trunc - Truncate timespec64 to a granularity
2150 * @t: Timespec64
2151 * @gran: Granularity in ns.
2152 *
2153 * Truncate a timespec64 to a granularity. Always rounds down. gran must
2154 * not be 0 nor greater than a second (NSEC_PER_SEC, or 10^9 ns).
2155 */
2156struct timespec64 timespec64_trunc(struct timespec64 t, unsigned gran)
2157{
2158        /* Avoid division in the common cases 1 ns and 1 s. */
2159        if (gran == 1) {
2160                /* nothing */
2161        } else if (gran == NSEC_PER_SEC) {
2162                t.tv_nsec = 0;
2163        } else if (gran > 1 && gran < NSEC_PER_SEC) {
2164                t.tv_nsec -= t.tv_nsec % gran;
2165        } else {
2166                WARN(1, "illegal file time granularity: %u", gran);
2167        }
2168        return t;
2169}
2170EXPORT_SYMBOL(timespec64_trunc);
2171
2172/**
2173 * timestamp_truncate - Truncate timespec to a granularity
2174 * @t: Timespec
2175 * @inode: inode being updated
2176 *
2177 * Truncate a timespec to the granularity supported by the fs
2178 * containing the inode. Always rounds down. gran must
2179 * not be 0 nor greater than a second (NSEC_PER_SEC, or 10^9 ns).
2180 */
2181struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode)
2182{
2183        struct super_block *sb = inode->i_sb;
2184        unsigned int gran = sb->s_time_gran;
2185
2186        t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max);
2187        if (unlikely(t.tv_sec == sb->s_time_max || t.tv_sec == sb->s_time_min))
2188                t.tv_nsec = 0;
2189
2190        /* Avoid division in the common cases 1 ns and 1 s. */
2191        if (gran == 1)
2192                ; /* nothing */
2193        else if (gran == NSEC_PER_SEC)
2194                t.tv_nsec = 0;
2195        else if (gran > 1 && gran < NSEC_PER_SEC)
2196                t.tv_nsec -= t.tv_nsec % gran;
2197        else
2198                WARN(1, "invalid file time granularity: %u", gran);
2199        return t;
2200}
2201EXPORT_SYMBOL(timestamp_truncate);
2202
2203/**
2204 * current_time - Return FS time
2205 * @inode: inode.
2206 *
2207 * Return the current time truncated to the time granularity supported by
2208 * the fs.
2209 *
2210 * Note that inode and inode->sb cannot be NULL.
2211 * Otherwise, the function warns and returns time without truncation.
2212 */
2213struct timespec64 current_time(struct inode *inode)
2214{
2215        struct timespec64 now;
2216
2217        ktime_get_coarse_real_ts64(&now);
2218
2219        if (unlikely(!inode->i_sb)) {
2220                WARN(1, "current_time() called with uninitialized super_block in the inode");
2221                return now;
2222        }
2223
2224        return timestamp_truncate(now, inode);
2225}
2226EXPORT_SYMBOL(current_time);
2227
2228/*
2229 * Generic function to check FS_IOC_SETFLAGS values and reject any invalid
2230 * configurations.
2231 *
2232 * Note: the caller should be holding i_mutex, or else be sure that they have
2233 * exclusive access to the inode structure.
2234 */
2235int vfs_ioc_setflags_prepare(struct inode *inode, unsigned int oldflags,
2236                             unsigned int flags)
2237{
2238        /*
2239         * The IMMUTABLE and APPEND_ONLY flags can only be changed by
2240         * the relevant capability.
2241         *
2242         * This test looks nicer. Thanks to Pauline Middelink
2243         */
2244        if ((flags ^ oldflags) & (FS_APPEND_FL | FS_IMMUTABLE_FL) &&
2245            !capable(CAP_LINUX_IMMUTABLE))
2246                return -EPERM;
2247
2248        return 0;
2249}
2250EXPORT_SYMBOL(vfs_ioc_setflags_prepare);
2251
2252/*
2253 * Generic function to check FS_IOC_FSSETXATTR values and reject any invalid
2254 * configurations.
2255 *
2256 * Note: the caller should be holding i_mutex, or else be sure that they have
2257 * exclusive access to the inode structure.
2258 */
2259int vfs_ioc_fssetxattr_check(struct inode *inode, const struct fsxattr *old_fa,
2260                             struct fsxattr *fa)
2261{
2262        /*
2263         * Can't modify an immutable/append-only file unless we have
2264         * appropriate permission.
2265         */
2266        if ((old_fa->fsx_xflags ^ fa->fsx_xflags) &
2267                        (FS_XFLAG_IMMUTABLE | FS_XFLAG_APPEND) &&
2268            !capable(CAP_LINUX_IMMUTABLE))
2269                return -EPERM;
2270
2271        /*
2272         * Project Quota ID state is only allowed to change from within the init
2273         * namespace. Enforce that restriction only if we are trying to change
2274         * the quota ID state. Everything else is allowed in user namespaces.
2275         */
2276        if (current_user_ns() != &init_user_ns) {
2277                if (old_fa->fsx_projid != fa->fsx_projid)
2278                        return -EINVAL;
2279                if ((old_fa->fsx_xflags ^ fa->fsx_xflags) &
2280                                FS_XFLAG_PROJINHERIT)
2281                        return -EINVAL;
2282        }
2283
2284        /* Check extent size hints. */
2285        if ((fa->fsx_xflags & FS_XFLAG_EXTSIZE) && !S_ISREG(inode->i_mode))
2286                return -EINVAL;
2287
2288        if ((fa->fsx_xflags & FS_XFLAG_EXTSZINHERIT) &&
2289                        !S_ISDIR(inode->i_mode))
2290                return -EINVAL;
2291
2292        if ((fa->fsx_xflags & FS_XFLAG_COWEXTSIZE) &&
2293            !S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
2294                return -EINVAL;
2295
2296        /*
2297         * It is only valid to set the DAX flag on regular files and
2298         * directories on filesystems.
2299         */
2300        if ((fa->fsx_xflags & FS_XFLAG_DAX) &&
2301            !(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)))
2302                return -EINVAL;
2303
2304        /* Extent size hints of zero turn off the flags. */
2305        if (fa->fsx_extsize == 0)
2306                fa->fsx_xflags &= ~(FS_XFLAG_EXTSIZE | FS_XFLAG_EXTSZINHERIT);
2307        if (fa->fsx_cowextsize == 0)
2308                fa->fsx_xflags &= ~FS_XFLAG_COWEXTSIZE;
2309
2310        return 0;
2311}
2312EXPORT_SYMBOL(vfs_ioc_fssetxattr_check);
2313