// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * shadow.c - Shadow Variables
 *
 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
 */

/**
 * DOC: Shadow variable API concurrency notes:
 *
 * The shadow variable API provides a simple relationship between an
 * <obj, id> pair and a pointer value.  It is the responsibility of the
 * caller to provide any mutual exclusion required of the shadow data.
 *
 * Once a shadow variable is attached to its parent object via the
 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
 * call to klp_shadow_get() may then return the shadow variable's data
 * pointer.  Callers of klp_shadow_*alloc() should prepare shadow data
 * accordingly.
 *
 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
 * variable structures.  Their implementation does not call kmalloc
 * inside any spinlocks, but API callers should pass GFP flags according
 * to their specific needs.
 *
 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
 * concurrent klp_shadow_free() and klp_shadow_get() operations.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/hashtable.h>
#include <linux/slab.h>
#include <linux/livepatch.h>

static DEFINE_HASHTABLE(klp_shadow_hash, 12);

/*
 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
 * the shadow variables it references.
 */
static DEFINE_SPINLOCK(klp_shadow_lock);

/**
 * struct klp_shadow - shadow variable structure
 * @node:       klp_shadow_hash hash table node
 * @rcu_head:   RCU is used to safely free this structure
 * @obj:        pointer to parent object
 * @id:         data identifier
 * @data:       data area
 */
struct klp_shadow {
        struct hlist_node node;
        struct rcu_head rcu_head;
        void *obj;
        unsigned long id;
        char data[];
};

/**
 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
 * @shadow:     shadow variable to match
 * @obj:        pointer to parent object
 * @id:         data identifier
 *
 * Return: true if the shadow variable matches.
 */
static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
                                unsigned long id)
{
        return shadow->obj == obj && shadow->id == id;
}

/**
 * klp_shadow_get() - retrieve a shadow variable data pointer
 * @obj:        pointer to parent object
 * @id:         data identifier
 *
 * Return: the shadow variable data element, NULL on failure.
 */
void *klp_shadow_get(void *obj, unsigned long id)
{
        struct klp_shadow *shadow;

        rcu_read_lock();

        hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
                                   (unsigned long)obj) {

                if (klp_shadow_match(shadow, obj, id)) {
                        rcu_read_unlock();
                        return shadow->data;
                }
        }

        rcu_read_unlock();

        return NULL;
}
EXPORT_SYMBOL_GPL(klp_shadow_get);

static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
                                       size_t size, gfp_t gfp_flags,
                                       klp_shadow_ctor_t ctor, void *ctor_data,
                                       bool warn_on_exist)
{
        struct klp_shadow *new_shadow;
        void *shadow_data;
        unsigned long flags;

        /* Check if the shadow variable already exists */
        shadow_data = klp_shadow_get(obj, id);
        if (shadow_data)
                goto exists;

        /*
         * Allocate a new shadow variable.  Fill it with zeroes by default.
         * More complex setting can be done by @ctor function.  But it is
         * called only when the buffer is really used (under klp_shadow_lock).
         */
        new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
        if (!new_shadow)
                return NULL;

        /* Look for <obj, id> again under the lock */
        spin_lock_irqsave(&klp_shadow_lock, flags);
        shadow_data = klp_shadow_get(obj, id);
        if (unlikely(shadow_data)) {
                /*
                 * Shadow variable was found, throw away speculative
                 * allocation.
                 */
                spin_unlock_irqrestore(&klp_shadow_lock, flags);
                kfree(new_shadow);
                goto exists;
        }

        new_shadow->obj = obj;
        new_shadow->id = id;

        if (ctor) {
                int err;

                err = ctor(obj, new_shadow->data, ctor_data);
                if (err) {
                        spin_unlock_irqrestore(&klp_shadow_lock, flags);
                        kfree(new_shadow);
                        pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
                               obj, id, err);
                        return NULL;
                }
        }

        /* No <obj, id> found, so attach the newly allocated one */
        hash_add_rcu(klp_shadow_hash, &new_shadow->node,
                     (unsigned long)new_shadow->obj);
        spin_unlock_irqrestore(&klp_shadow_lock, flags);

        return new_shadow->data;

exists:
        if (warn_on_exist) {
                WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
                return NULL;
        }

        return shadow_data;
}

/**
 * klp_shadow_alloc() - allocate and add a new shadow variable
 * @obj:        pointer to parent object
 * @id:         data identifier
 * @size:       size of attached data
 * @gfp_flags:  GFP mask for allocation
 * @ctor:       custom constructor to initialize the shadow data (optional)
 * @ctor_data:  pointer to any data needed by @ctor (optional)
 *
 * Allocates @size bytes for new shadow variable data using @gfp_flags.
 * The data are zeroed by default.  They are further initialized by @ctor
 * function if it is not NULL.  The new shadow variable is then added
 * to the global hashtable.
 *
 * If an existing <obj, id> shadow variable can be found, this routine will
 * issue a WARN, exit early and return NULL.
 *
 * This function guarantees that the constructor function is called only when
 * the variable did not exist before.  The cost is that @ctor is called
 * in atomic context under a spin lock.
 *
 * Return: the shadow variable data element, NULL on duplicate or
 * failure.
 */
void *klp_shadow_alloc(void *obj, unsigned long id,
                       size_t size, gfp_t gfp_flags,
                       klp_shadow_ctor_t ctor, void *ctor_data)
{
        return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
                                         ctor, ctor_data, true);
}
EXPORT_SYMBOL_GPL(klp_shadow_alloc);

/**
 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
 * @obj:        pointer to parent object
 * @id:         data identifier
 * @size:       size of attached data
 * @gfp_flags:  GFP mask for allocation
 * @ctor:       custom constructor to initialize the shadow data (optional)
 * @ctor_data:  pointer to any data needed by @ctor (optional)
 *
 * Returns a pointer to existing shadow data if an <obj, id> shadow
 * variable is already present.  Otherwise, it creates a new shadow
 * variable like klp_shadow_alloc().
 *
 * This function guarantees that only one shadow variable exists with the given
 * @id for the given @obj.  It also guarantees that the constructor function
 * will be called only when the variable did not exist before.  The cost is
 * that @ctor is called in atomic context under a spin lock.
 *
 * Return: the shadow variable data element, NULL on failure.
 */
void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
                              size_t size, gfp_t gfp_flags,
                              klp_shadow_ctor_t ctor, void *ctor_data)
{
        return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
                                         ctor, ctor_data, false);
}
EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);

static void klp_shadow_free_struct(struct klp_shadow *shadow,
                                   klp_shadow_dtor_t dtor)
{
        hash_del_rcu(&shadow->node);
        if (dtor)
                dtor(shadow->obj, shadow->data);
        kfree_rcu(shadow, rcu_head);
}

/**
 * klp_shadow_free() - detach and free a <obj, id> shadow variable
 * @obj:        pointer to parent object
 * @id:         data identifier
 * @dtor:       custom callback that can be used to unregister the variable
 *              and/or free data that the shadow variable points to (optional)
 *
 * This function releases the memory for this <obj, id> shadow variable
 * instance, callers should stop referencing it accordingly.
 */
void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
{
        struct klp_shadow *shadow;
        unsigned long flags;

        spin_lock_irqsave(&klp_shadow_lock, flags);

        /* Delete <obj, id> from hash */
        hash_for_each_possible(klp_shadow_hash, shadow, node,
                               (unsigned long)obj) {

                if (klp_shadow_match(shadow, obj, id)) {
                        klp_shadow_free_struct(shadow, dtor);
                        break;
                }
        }

        spin_unlock_irqrestore(&klp_shadow_lock, flags);
}
EXPORT_SYMBOL_GPL(klp_shadow_free);

/**
 * klp_shadow_free_all() - detach and free all <*, id> shadow variables
 * @id:         data identifier
 * @dtor:       custom callback that can be used to unregister the variable
 *              and/or free data that the shadow variable points to (optional)
 *
 * This function releases the memory for all <*, id> shadow variable
 * instances, callers should stop referencing them accordingly.
 */
void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
{
        struct klp_shadow *shadow;
        unsigned long flags;
        int i;

        spin_lock_irqsave(&klp_shadow_lock, flags);

        /* Delete all <*, id> from hash */
        hash_for_each(klp_shadow_hash, i, shadow, node) {
                if (klp_shadow_match(shadow, shadow->obj, id))
                        klp_shadow_free_struct(shadow, dtor);
        }

        spin_unlock_irqrestore(&klp_shadow_lock, flags);
}
EXPORT_SYMBOL_GPL(klp_shadow_free_all);