/*
 * Copyright © 2017 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

#include <linux/slab.h>

#include "i915_syncmap.h"

#include "i915_gem.h" /* GEM_BUG_ON() */
#include "i915_selftest.h"

#define SHIFT ilog2(KSYNCMAP)
#define MASK (KSYNCMAP - 1)

/*
 * struct i915_syncmap is a layer of a radixtree that maps a u64 fence
 * context id to the last u32 fence seqno waited upon from that context.
 * Unlike lib/radixtree it uses a parent pointer that allows traversal back to
 * the root. This allows us to access the whole tree via a single pointer
 * to the most recently used layer. We expect fence contexts to be dense
 * and most reuse to be on the same i915_gem_context but on neighbouring
 * engines (i.e. on adjacent contexts) and reuse the same leaf, a very
 * effective lookup cache. If the new lookup is not on the same leaf, we
 * expect it to be on the neighbouring branch.
 *
 * A leaf holds an array of u32 seqno, and has height 0. The bitmap field
 * allows us to store whether a particular seqno is valid (i.e. allows us
 * to distinguish unset from 0).
 *
 * A branch holds an array of layer pointers, and has height > 0, and always
 * has at least 2 layers (either branches or leaves) below it.
 *
 * For example,
 *      for x in
 *        0 1 2 0x10 0x11 0x200 0x201
 *        0x500000 0x500001 0x503000 0x503001
 *        0xE<<60:
 *              i915_syncmap_set(&sync, x, lower_32_bits(x));
 * will build a tree like:
 *      0xXXXXXXXXXXXXXXXX
 *      0-> 0x0000000000XXXXXX
 *      |   0-> 0x0000000000000XXX
 *      |   |   0-> 0x00000000000000XX
 *      |   |   |   0-> 0x000000000000000X 0:0, 1:1, 2:2
 *      |   |   |   1-> 0x000000000000001X 0:10, 1:11
 *      |   |   2-> 0x000000000000020X 0:200, 1:201
 *      |   5-> 0x000000000050XXXX
 *      |       0-> 0x000000000050000X 0:500000, 1:500001
 *      |       3-> 0x000000000050300X 0:503000, 1:503001
 *      e-> 0xe00000000000000X e:e
 */

struct i915_syncmap {
        u64 prefix;
        unsigned int height;
        unsigned int bitmap;
        struct i915_syncmap *parent;
        /*
         * Following this header is an array of either seqno or child pointers:
         * union {
         *      u32 seqno[KSYNCMAP];
         *      struct i915_syncmap *child[KSYNCMAP];
         * };
         */
};

/**
 * i915_syncmap_init -- initialise the #i915_syncmap
 * @root: pointer to the #i915_syncmap
 */
void i915_syncmap_init(struct i915_syncmap **root)
{
        BUILD_BUG_ON_NOT_POWER_OF_2(KSYNCMAP);
        BUILD_BUG_ON_NOT_POWER_OF_2(SHIFT);
        BUILD_BUG_ON(KSYNCMAP > BITS_PER_TYPE((*root)->bitmap));
        *root = NULL;
}

static inline u32 *__sync_seqno(struct i915_syncmap *p)
{
        GEM_BUG_ON(p->height);
        return (u32 *)(p + 1);
}

static inline struct i915_syncmap **__sync_child(struct i915_syncmap *p)
{
        GEM_BUG_ON(!p->height);
        return (struct i915_syncmap **)(p + 1);
}

static inline unsigned int
__sync_branch_idx(const struct i915_syncmap *p, u64 id)
{
        return (id >> p->height) & MASK;
}

static inline unsigned int
__sync_leaf_idx(const struct i915_syncmap *p, u64 id)
{
        GEM_BUG_ON(p->height);
        return id & MASK;
}

static inline u64 __sync_branch_prefix(const struct i915_syncmap *p, u64 id)
{
        return id >> p->height >> SHIFT;
}

static inline u64 __sync_leaf_prefix(const struct i915_syncmap *p, u64 id)
{
        GEM_BUG_ON(p->height);
        return id >> SHIFT;
}

static inline bool seqno_later(u32 a, u32 b)
{
        return (s32)(a - b) >= 0;
}

/**
 * i915_syncmap_is_later -- compare against the last know sync point
 * @root: pointer to the #i915_syncmap
 * @id: the context id (other timeline) we are synchronising to
 * @seqno: the sequence number along the other timeline
 *
 * If we have already synchronised this @root timeline with another (@id) then
 * we can omit any repeated or earlier synchronisation requests. If the two
 * timelines are already coupled, we can also omit the dependency between the
 * two as that is already known via the timeline.
 *
 * Returns true if the two timelines are already synchronised wrt to @seqno,
 * false if not and the synchronisation must be emitted.
 */
bool i915_syncmap_is_later(struct i915_syncmap **root, u64 id, u32 seqno)
{
        struct i915_syncmap *p;
        unsigned int idx;

        p = *root;
        if (!p)
                return false;

        if (likely(__sync_leaf_prefix(p, id) == p->prefix))
                goto found;

        /* First climb the tree back to a parent branch */
        do {
                p = p->parent;
                if (!p)
                        return false;

                if (__sync_branch_prefix(p, id) == p->prefix)
                        break;
        } while (1);

        /* And then descend again until we find our leaf */
        do {
                if (!p->height)
                        break;

                p = __sync_child(p)[__sync_branch_idx(p, id)];
                if (!p)
                        return false;

                if (__sync_branch_prefix(p, id) != p->prefix)
                        return false;
        } while (1);

        *root = p;
found:
        idx = __sync_leaf_idx(p, id);
        if (!(p->bitmap & BIT(idx)))
                return false;

        return seqno_later(__sync_seqno(p)[idx], seqno);
}

static struct i915_syncmap *
__sync_alloc_leaf(struct i915_syncmap *parent, u64 id)
{
        struct i915_syncmap *p;

        p = kmalloc(sizeof(*p) + KSYNCMAP * sizeof(u32), GFP_KERNEL);
        if (unlikely(!p))
                return NULL;

        p->parent = parent;
        p->height = 0;
        p->bitmap = 0;
        p->prefix = __sync_leaf_prefix(p, id);
        return p;
}

static inline void __sync_set_seqno(struct i915_syncmap *p, u64 id, u32 seqno)
{
        unsigned int idx = __sync_leaf_idx(p, id);

        p->bitmap |= BIT(idx);
        __sync_seqno(p)[idx] = seqno;
}

static inline void __sync_set_child(struct i915_syncmap *p,
                                    unsigned int idx,
                                    struct i915_syncmap *child)
{
        p->bitmap |= BIT(idx);
        __sync_child(p)[idx] = child;
}

static noinline int __sync_set(struct i915_syncmap **root, u64 id, u32 seqno)
{
        struct i915_syncmap *p = *root;
        unsigned int idx;

        if (!p) {
                p = __sync_alloc_leaf(NULL, id);
                if (unlikely(!p))
                        return -ENOMEM;

                goto found;
        }

        /* Caller handled the likely cached case */
        GEM_BUG_ON(__sync_leaf_prefix(p, id) == p->prefix);

        /* Climb back up the tree until we find a common prefix */
        do {
                if (!p->parent)
                        break;

                p = p->parent;

                if (__sync_branch_prefix(p, id) == p->prefix)
                        break;
        } while (1);

        /*
         * No shortcut, we have to descend the tree to find the right layer
         * containing this fence.
         *
         * Each layer in the tree holds 16 (KSYNCMAP) pointers, either fences
         * or lower layers. Leaf nodes (height = 0) contain the fences, all
         * other nodes (height > 0) are internal layers that point to a lower
         * node. Each internal layer has at least 2 descendents.
         *
         * Starting at the top, we check whether the current prefix matches. If
         * it doesn't, we have gone past our target and need to insert a join
         * into the tree, and a new leaf node for the target as a descendent
         * of the join, as well as the original layer.
         *
         * The matching prefix means we are still following the right branch
         * of the tree. If it has height 0, we have found our leaf and just
         * need to replace the fence slot with ourselves. If the height is
         * not zero, our slot contains the next layer in the tree (unless
         * it is empty, in which case we can add ourselves as a new leaf).
         * As descend the tree the prefix grows (and height decreases).
         */
        do {
                struct i915_syncmap *next;

                if (__sync_branch_prefix(p, id) != p->prefix) {
                        unsigned int above;

                        /* Insert a join above the current layer */
                        next = kzalloc(sizeof(*next) + KSYNCMAP * sizeof(next),
                                       GFP_KERNEL);
                        if (unlikely(!next))
                                return -ENOMEM;

                        /* Compute the height at which these two diverge */
                        above = fls64(__sync_branch_prefix(p, id) ^ p->prefix);
                        above = round_up(above, SHIFT);
                        next->height = above + p->height;
                        next->prefix = __sync_branch_prefix(next, id);

                        /* Insert the join into the parent */
                        if (p->parent) {
                                idx = __sync_branch_idx(p->parent, id);
                                __sync_child(p->parent)[idx] = next;
                                GEM_BUG_ON(!(p->parent->bitmap & BIT(idx)));
                        }
                        next->parent = p->parent;

                        /* Compute the idx of the other branch, not our id! */
                        idx = p->prefix >> (above - SHIFT) & MASK;
                        __sync_set_child(next, idx, p);
                        p->parent = next;

                        /* Ascend to the join */
                        p = next;
                } else {
                        if (!p->height)
                                break;
                }

                /* Descend into the next layer */
                GEM_BUG_ON(!p->height);
                idx = __sync_branch_idx(p, id);
                next = __sync_child(p)[idx];
                if (!next) {
                        next = __sync_alloc_leaf(p, id);
                        if (unlikely(!next))
                                return -ENOMEM;

                        __sync_set_child(p, idx, next);
                        p = next;
                        break;
                }

                p = next;
        } while (1);

found:
        GEM_BUG_ON(p->prefix != __sync_leaf_prefix(p, id));
        __sync_set_seqno(p, id, seqno);
        *root = p;
        return 0;
}

/**
 * i915_syncmap_set -- mark the most recent syncpoint between contexts
 * @root: pointer to the #i915_syncmap
 * @id: the context id (other timeline) we have synchronised to
 * @seqno: the sequence number along the other timeline
 *
 * When we synchronise this @root timeline with another (@id), we also know
 * that we have synchronized with all previous seqno along that timeline. If
 * we then have a request to synchronise with the same seqno or older, we can
 * omit it, see i915_syncmap_is_later()
 *
 * Returns 0 on success, or a negative error code.
 */
int i915_syncmap_set(struct i915_syncmap **root, u64 id, u32 seqno)
{
        struct i915_syncmap *p = *root;

        /*
         * We expect to be called in sequence following is_later(id), which
         * should have preloaded the root for us.
         */
        if (likely(p && __sync_leaf_prefix(p, id) == p->prefix)) {
                __sync_set_seqno(p, id, seqno);
                return 0;
        }

        return __sync_set(root, id, seqno);
}

static void __sync_free(struct i915_syncmap *p)
{
        if (p->height) {
                unsigned int i;

                while ((i = ffs(p->bitmap))) {
                        p->bitmap &= ~0u << i;
                        __sync_free(__sync_child(p)[i - 1]);
                }
        }

        kfree(p);
}

/**
 * i915_syncmap_free -- free all memory associated with the syncmap
 * @root: pointer to the #i915_syncmap
 *
 * Either when the timeline is to be freed and we no longer need the sync
 * point tracking, or when the fences are all known to be signaled and the
 * sync point tracking is redundant, we can free the #i915_syncmap to recover
 * its allocations.
 *
 * Will reinitialise the @root pointer so that the #i915_syncmap is ready for
 * reuse.
 */
void i915_syncmap_free(struct i915_syncmap **root)
{
        struct i915_syncmap *p;

        p = *root;
        if (!p)
                return;

        while (p->parent)
                p = p->parent;

        __sync_free(p);
        *root = NULL;
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/i915_syncmap.c"
#endif