LXR linux/fs/ceph/io.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2016 Trond Myklebust
   4 * Copyright (c) 2019 Jeff Layton
   5 *
   6 * I/O and data path helper functionality.
   7 *
   8 * Heavily borrowed from equivalent code in fs/nfs/io.c
   9 */
  10
  11#include <linux/ceph/ceph_debug.h>
  12
  13#include <linux/types.h>
  14#include <linux/kernel.h>
  15#include <linux/rwsem.h>
  16#include <linux/fs.h>
  17
  18#include "super.h"
  19#include "io.h"
  20
  21/* Call with exclusively locked inode->i_rwsem */
  22static void ceph_block_o_direct(struct ceph_inode_info *ci, struct inode *inode)
  23{
  24        lockdep_assert_held_write(&inode->i_rwsem);
  25
  26        if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT) {
  27                spin_lock(&ci->i_ceph_lock);
  28                ci->i_ceph_flags &= ~CEPH_I_ODIRECT;
  29                spin_unlock(&ci->i_ceph_lock);
  30                inode_dio_wait(inode);
  31        }
  32}
  33
  34/**
  35 * ceph_start_io_read - declare the file is being used for buffered reads
  36 * @inode: file inode
  37 *
  38 * Declare that a buffered read operation is about to start, and ensure
  39 * that we block all direct I/O.
  40 * On exit, the function ensures that the CEPH_I_ODIRECT flag is unset,
  41 * and holds a shared lock on inode->i_rwsem to ensure that the flag
  42 * cannot be changed.
  43 * In practice, this means that buffered read operations are allowed to
  44 * execute in parallel, thanks to the shared lock, whereas direct I/O
  45 * operations need to wait to grab an exclusive lock in order to set
  46 * CEPH_I_ODIRECT.
  47 * Note that buffered writes and truncates both take a write lock on
  48 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
  49 */
  50void
  51ceph_start_io_read(struct inode *inode)
  52{
  53        struct ceph_inode_info *ci = ceph_inode(inode);
  54
  55        /* Be an optimist! */
  56        down_read(&inode->i_rwsem);
  57        if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT))
  58                return;
  59        up_read(&inode->i_rwsem);
  60        /* Slow path.... */
  61        down_write(&inode->i_rwsem);
  62        ceph_block_o_direct(ci, inode);
  63        downgrade_write(&inode->i_rwsem);
  64}
  65
  66/**
  67 * ceph_end_io_read - declare that the buffered read operation is done
  68 * @inode: file inode
  69 *
  70 * Declare that a buffered read operation is done, and release the shared
  71 * lock on inode->i_rwsem.
  72 */
  73void
  74ceph_end_io_read(struct inode *inode)
  75{
  76        up_read(&inode->i_rwsem);
  77}
  78
  79/**
  80 * ceph_start_io_write - declare the file is being used for buffered writes
  81 * @inode: file inode
  82 *
  83 * Declare that a buffered write operation is about to start, and ensure
  84 * that we block all direct I/O.
  85 */
  86void
  87ceph_start_io_write(struct inode *inode)
  88{
  89        down_write(&inode->i_rwsem);
  90        ceph_block_o_direct(ceph_inode(inode), inode);
  91}
  92
  93/**
  94 * ceph_end_io_write - declare that the buffered write operation is done
  95 * @inode: file inode
  96 *
  97 * Declare that a buffered write operation is done, and release the
  98 * lock on inode->i_rwsem.
  99 */
 100void
 101ceph_end_io_write(struct inode *inode)
 102{
 103        up_write(&inode->i_rwsem);
 104}
 105
 106/* Call with exclusively locked inode->i_rwsem */
 107static void ceph_block_buffered(struct ceph_inode_info *ci, struct inode *inode)
 108{
 109        lockdep_assert_held_write(&inode->i_rwsem);
 110
 111        if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT)) {
 112                spin_lock(&ci->i_ceph_lock);
 113                ci->i_ceph_flags |= CEPH_I_ODIRECT;
 114                spin_unlock(&ci->i_ceph_lock);
 115                /* FIXME: unmap_mapping_range? */
 116                filemap_write_and_wait(inode->i_mapping);
 117        }
 118}
 119
 120/**
 121 * ceph_start_io_direct - declare the file is being used for direct i/o
 122 * @inode: file inode
 123 *
 124 * Declare that a direct I/O operation is about to start, and ensure
 125 * that we block all buffered I/O.
 126 * On exit, the function ensures that the CEPH_I_ODIRECT flag is set,
 127 * and holds a shared lock on inode->i_rwsem to ensure that the flag
 128 * cannot be changed.
 129 * In practice, this means that direct I/O operations are allowed to
 130 * execute in parallel, thanks to the shared lock, whereas buffered I/O
 131 * operations need to wait to grab an exclusive lock in order to clear
 132 * CEPH_I_ODIRECT.
 133 * Note that buffered writes and truncates both take a write lock on
 134 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
 135 */
 136void
 137ceph_start_io_direct(struct inode *inode)
 138{
 139        struct ceph_inode_info *ci = ceph_inode(inode);
 140
 141        /* Be an optimist! */
 142        down_read(&inode->i_rwsem);
 143        if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT)
 144                return;
 145        up_read(&inode->i_rwsem);
 146        /* Slow path.... */
 147        down_write(&inode->i_rwsem);
 148        ceph_block_buffered(ci, inode);
 149        downgrade_write(&inode->i_rwsem);
 150}
 151
 152/**
 153 * ceph_end_io_direct - declare that the direct i/o operation is done
 154 * @inode: file inode
 155 *
 156 * Declare that a direct I/O operation is done, and release the shared
 157 * lock on inode->i_rwsem.
 158 */
 159void
 160ceph_end_io_direct(struct inode *inode)
 161{
 162        up_read(&inode->i_rwsem);
 163}
 164