LXR linux/fs/nfs/io.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2016 Trond Myklebust
   4 *
   5 * I/O and data path helper functionality.
   6 */
   7
   8#include <linux/types.h>
   9#include <linux/kernel.h>
  10#include <linux/bitops.h>
  11#include <linux/rwsem.h>
  12#include <linux/fs.h>
  13#include <linux/nfs_fs.h>
  14
  15#include "internal.h"
  16
  17/* Call with exclusively locked inode->i_rwsem */
  18static void nfs_block_o_direct(struct nfs_inode *nfsi, struct inode *inode)
  19{
  20        if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) {
  21                clear_bit(NFS_INO_ODIRECT, &nfsi->flags);
  22                inode_dio_wait(inode);
  23        }
  24}
  25
  26/**
  27 * nfs_start_io_read - declare the file is being used for buffered reads
  28 * @inode: file inode
  29 *
  30 * Declare that a buffered read operation is about to start, and ensure
  31 * that we block all direct I/O.
  32 * On exit, the function ensures that the NFS_INO_ODIRECT flag is unset,
  33 * and holds a shared lock on inode->i_rwsem to ensure that the flag
  34 * cannot be changed.
  35 * In practice, this means that buffered read operations are allowed to
  36 * execute in parallel, thanks to the shared lock, whereas direct I/O
  37 * operations need to wait to grab an exclusive lock in order to set
  38 * NFS_INO_ODIRECT.
  39 * Note that buffered writes and truncates both take a write lock on
  40 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
  41 */
  42void
  43nfs_start_io_read(struct inode *inode)
  44{
  45        struct nfs_inode *nfsi = NFS_I(inode);
  46        /* Be an optimist! */
  47        down_read(&inode->i_rwsem);
  48        if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0)
  49                return;
  50        up_read(&inode->i_rwsem);
  51        /* Slow path.... */
  52        down_write(&inode->i_rwsem);
  53        nfs_block_o_direct(nfsi, inode);
  54        downgrade_write(&inode->i_rwsem);
  55}
  56
  57/**
  58 * nfs_end_io_read - declare that the buffered read operation is done
  59 * @inode: file inode
  60 *
  61 * Declare that a buffered read operation is done, and release the shared
  62 * lock on inode->i_rwsem.
  63 */
  64void
  65nfs_end_io_read(struct inode *inode)
  66{
  67        up_read(&inode->i_rwsem);
  68}
  69
  70/**
  71 * nfs_start_io_write - declare the file is being used for buffered writes
  72 * @inode: file inode
  73 *
  74 * Declare that a buffered read operation is about to start, and ensure
  75 * that we block all direct I/O.
  76 */
  77void
  78nfs_start_io_write(struct inode *inode)
  79{
  80        down_write(&inode->i_rwsem);
  81        nfs_block_o_direct(NFS_I(inode), inode);
  82}
  83
  84/**
  85 * nfs_end_io_write - declare that the buffered write operation is done
  86 * @inode: file inode
  87 *
  88 * Declare that a buffered write operation is done, and release the
  89 * lock on inode->i_rwsem.
  90 */
  91void
  92nfs_end_io_write(struct inode *inode)
  93{
  94        up_write(&inode->i_rwsem);
  95}
  96
  97/* Call with exclusively locked inode->i_rwsem */
  98static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode)
  99{
 100        if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) {
 101                set_bit(NFS_INO_ODIRECT, &nfsi->flags);
 102                nfs_sync_mapping(inode->i_mapping);
 103        }
 104}
 105
 106/**
 107 * nfs_start_io_direct - declare the file is being used for direct i/o
 108 * @inode: file inode
 109 *
 110 * Declare that a direct I/O operation is about to start, and ensure
 111 * that we block all buffered I/O.
 112 * On exit, the function ensures that the NFS_INO_ODIRECT flag is set,
 113 * and holds a shared lock on inode->i_rwsem to ensure that the flag
 114 * cannot be changed.
 115 * In practice, this means that direct I/O operations are allowed to
 116 * execute in parallel, thanks to the shared lock, whereas buffered I/O
 117 * operations need to wait to grab an exclusive lock in order to clear
 118 * NFS_INO_ODIRECT.
 119 * Note that buffered writes and truncates both take a write lock on
 120 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
 121 */
 122void
 123nfs_start_io_direct(struct inode *inode)
 124{
 125        struct nfs_inode *nfsi = NFS_I(inode);
 126        /* Be an optimist! */
 127        down_read(&inode->i_rwsem);
 128        if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0)
 129                return;
 130        up_read(&inode->i_rwsem);
 131        /* Slow path.... */
 132        down_write(&inode->i_rwsem);
 133        nfs_block_buffered(nfsi, inode);
 134        downgrade_write(&inode->i_rwsem);
 135}
 136
 137/**
 138 * nfs_end_io_direct - declare that the direct i/o operation is done
 139 * @inode: file inode
 140 *
 141 * Declare that a direct I/O operation is done, and release the shared
 142 * lock on inode->i_rwsem.
 143 */
 144void
 145nfs_end_io_direct(struct inode *inode)
 146{
 147        up_read(&inode->i_rwsem);
 148}
 149