// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2008 Oracle.  All rights reserved.
 */

#include <linux/sched.h>
#include <linux/pagemap.h>
#include <linux/spinlock.h>
#include <linux/page-flags.h>
#include <asm/bug.h>
#include "ctree.h"
#include "extent_io.h"
#include "locking.h"

static void btrfs_assert_tree_read_locked(struct extent_buffer *eb);

/*
 * if we currently have a spinning reader or writer lock
 * (indicated by the rw flag) this will bump the count
 * of blocking holders and drop the spinlock.
 */
void btrfs_set_lock_blocking_rw(struct extent_buffer *eb, int rw)
{
        /*
         * no lock is required.  The lock owner may change if
         * we have a read lock, but it won't change to or away
         * from us.  If we have the write lock, we are the owner
         * and it'll never change.
         */
        if (eb->lock_nested && current->pid == eb->lock_owner)
                return;
        if (rw == BTRFS_WRITE_LOCK) {
                if (atomic_read(&eb->blocking_writers) == 0) {
                        WARN_ON(atomic_read(&eb->spinning_writers) != 1);
                        atomic_dec(&eb->spinning_writers);
                        btrfs_assert_tree_locked(eb);
                        atomic_inc(&eb->blocking_writers);
                        write_unlock(&eb->lock);
                }
        } else if (rw == BTRFS_READ_LOCK) {
                btrfs_assert_tree_read_locked(eb);
                atomic_inc(&eb->blocking_readers);
                WARN_ON(atomic_read(&eb->spinning_readers) == 0);
                atomic_dec(&eb->spinning_readers);
                read_unlock(&eb->lock);
        }
}

/*
 * if we currently have a blocking lock, take the spinlock
 * and drop our blocking count
 */
void btrfs_clear_lock_blocking_rw(struct extent_buffer *eb, int rw)
{
        /*
         * no lock is required.  The lock owner may change if
         * we have a read lock, but it won't change to or away
         * from us.  If we have the write lock, we are the owner
         * and it'll never change.
         */
        if (eb->lock_nested && current->pid == eb->lock_owner)
                return;

        if (rw == BTRFS_WRITE_LOCK_BLOCKING) {
                BUG_ON(atomic_read(&eb->blocking_writers) != 1);
                write_lock(&eb->lock);
                WARN_ON(atomic_read(&eb->spinning_writers));
                atomic_inc(&eb->spinning_writers);
                /* atomic_dec_and_test implies a barrier */
                if (atomic_dec_and_test(&eb->blocking_writers))
                        cond_wake_up_nomb(&eb->write_lock_wq);
        } else if (rw == BTRFS_READ_LOCK_BLOCKING) {
                BUG_ON(atomic_read(&eb->blocking_readers) == 0);
                read_lock(&eb->lock);
                atomic_inc(&eb->spinning_readers);
                /* atomic_dec_and_test implies a barrier */
                if (atomic_dec_and_test(&eb->blocking_readers))
                        cond_wake_up_nomb(&eb->read_lock_wq);
        }
}

/*
 * take a spinning read lock.  This will wait for any blocking
 * writers
 */
void btrfs_tree_read_lock(struct extent_buffer *eb)
{
again:
        BUG_ON(!atomic_read(&eb->blocking_writers) &&
               current->pid == eb->lock_owner);

        read_lock(&eb->lock);
        if (atomic_read(&eb->blocking_writers) &&
            current->pid == eb->lock_owner) {
                /*
                 * This extent is already write-locked by our thread. We allow
                 * an additional read lock to be added because it's for the same
                 * thread. btrfs_find_all_roots() depends on this as it may be
                 * called on a partly (write-)locked tree.
                 */
                BUG_ON(eb->lock_nested);
                eb->lock_nested = 1;
                read_unlock(&eb->lock);
                return;
        }
        if (atomic_read(&eb->blocking_writers)) {
                read_unlock(&eb->lock);
                wait_event(eb->write_lock_wq,
                           atomic_read(&eb->blocking_writers) == 0);
                goto again;
        }
        atomic_inc(&eb->read_locks);
        atomic_inc(&eb->spinning_readers);
}

/*
 * take a spinning read lock.
 * returns 1 if we get the read lock and 0 if we don't
 * this won't wait for blocking writers
 */
int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
{
        if (atomic_read(&eb->blocking_writers))
                return 0;

        read_lock(&eb->lock);
        if (atomic_read(&eb->blocking_writers)) {
                read_unlock(&eb->lock);
                return 0;
        }
        atomic_inc(&eb->read_locks);
        atomic_inc(&eb->spinning_readers);
        return 1;
}

/*
 * returns 1 if we get the read lock and 0 if we don't
 * this won't wait for blocking writers
 */
int btrfs_try_tree_read_lock(struct extent_buffer *eb)
{
        if (atomic_read(&eb->blocking_writers))
                return 0;

        if (!read_trylock(&eb->lock))
                return 0;

        if (atomic_read(&eb->blocking_writers)) {
                read_unlock(&eb->lock);
                return 0;
        }
        atomic_inc(&eb->read_locks);
        atomic_inc(&eb->spinning_readers);
        return 1;
}

/*
 * returns 1 if we get the read lock and 0 if we don't
 * this won't wait for blocking writers or readers
 */
int btrfs_try_tree_write_lock(struct extent_buffer *eb)
{
        if (atomic_read(&eb->blocking_writers) ||
            atomic_read(&eb->blocking_readers))
                return 0;

        write_lock(&eb->lock);
        if (atomic_read(&eb->blocking_writers) ||
            atomic_read(&eb->blocking_readers)) {
                write_unlock(&eb->lock);
                return 0;
        }
        atomic_inc(&eb->write_locks);
        atomic_inc(&eb->spinning_writers);
        eb->lock_owner = current->pid;
        return 1;
}

/*
 * drop a spinning read lock
 */
void btrfs_tree_read_unlock(struct extent_buffer *eb)
{
        /*
         * if we're nested, we have the write lock.  No new locking
         * is needed as long as we are the lock owner.
         * The write unlock will do a barrier for us, and the lock_nested
         * field only matters to the lock owner.
         */
        if (eb->lock_nested && current->pid == eb->lock_owner) {
                eb->lock_nested = 0;
                return;
        }
        btrfs_assert_tree_read_locked(eb);
        WARN_ON(atomic_read(&eb->spinning_readers) == 0);
        atomic_dec(&eb->spinning_readers);
        atomic_dec(&eb->read_locks);
        read_unlock(&eb->lock);
}

/*
 * drop a blocking read lock
 */
void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
{
        /*
         * if we're nested, we have the write lock.  No new locking
         * is needed as long as we are the lock owner.
         * The write unlock will do a barrier for us, and the lock_nested
         * field only matters to the lock owner.
         */
        if (eb->lock_nested && current->pid == eb->lock_owner) {
                eb->lock_nested = 0;
                return;
        }
        btrfs_assert_tree_read_locked(eb);
        WARN_ON(atomic_read(&eb->blocking_readers) == 0);
        /* atomic_dec_and_test implies a barrier */
        if (atomic_dec_and_test(&eb->blocking_readers))
                cond_wake_up_nomb(&eb->read_lock_wq);
        atomic_dec(&eb->read_locks);
}

/*
 * take a spinning write lock.  This will wait for both
 * blocking readers or writers
 */
void btrfs_tree_lock(struct extent_buffer *eb)
{
        WARN_ON(eb->lock_owner == current->pid);
again:
        wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
        wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0);
        write_lock(&eb->lock);
        if (atomic_read(&eb->blocking_readers)) {
                write_unlock(&eb->lock);
                wait_event(eb->read_lock_wq,
                           atomic_read(&eb->blocking_readers) == 0);
                goto again;
        }
        if (atomic_read(&eb->blocking_writers)) {
                write_unlock(&eb->lock);
                wait_event(eb->write_lock_wq,
                           atomic_read(&eb->blocking_writers) == 0);
                goto again;
        }
        WARN_ON(atomic_read(&eb->spinning_writers));
        atomic_inc(&eb->spinning_writers);
        atomic_inc(&eb->write_locks);
        eb->lock_owner = current->pid;
}

/*
 * drop a spinning or a blocking write lock.
 */
void btrfs_tree_unlock(struct extent_buffer *eb)
{
        int blockers = atomic_read(&eb->blocking_writers);

        BUG_ON(blockers > 1);

        btrfs_assert_tree_locked(eb);
        eb->lock_owner = 0;
        atomic_dec(&eb->write_locks);

        if (blockers) {
                WARN_ON(atomic_read(&eb->spinning_writers));
                atomic_dec(&eb->blocking_writers);
                /* Use the lighter barrier after atomic */
                smp_mb__after_atomic();
                cond_wake_up_nomb(&eb->write_lock_wq);
        } else {
                WARN_ON(atomic_read(&eb->spinning_writers) != 1);
                atomic_dec(&eb->spinning_writers);
                write_unlock(&eb->lock);
        }
}

void btrfs_assert_tree_locked(struct extent_buffer *eb)
{
        BUG_ON(!atomic_read(&eb->write_locks));
}

static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
{
        BUG_ON(!atomic_read(&eb->read_locks));
}