// SPDX-License-Identifier: MIT
/*
 * Copyright © 2019 Intel Corporation
 */
#include "i915_gem_client_blt.h"

#include "i915_gem_object_blt.h"
#include "intel_drv.h"

struct i915_sleeve {
        struct i915_vma *vma;
        struct drm_i915_gem_object *obj;
        struct sg_table *pages;
        struct i915_page_sizes page_sizes;
};

static int vma_set_pages(struct i915_vma *vma)
{
        struct i915_sleeve *sleeve = vma->private;

        vma->pages = sleeve->pages;
        vma->page_sizes = sleeve->page_sizes;

        return 0;
}

static void vma_clear_pages(struct i915_vma *vma)
{
        GEM_BUG_ON(!vma->pages);
        vma->pages = NULL;
}

static int vma_bind(struct i915_vma *vma,
                    enum i915_cache_level cache_level,
                    u32 flags)
{
        return vma->vm->vma_ops.bind_vma(vma, cache_level, flags);
}

static void vma_unbind(struct i915_vma *vma)
{
        vma->vm->vma_ops.unbind_vma(vma);
}

static const struct i915_vma_ops proxy_vma_ops = {
        .set_pages = vma_set_pages,
        .clear_pages = vma_clear_pages,
        .bind_vma = vma_bind,
        .unbind_vma = vma_unbind,
};

static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
                                         struct drm_i915_gem_object *obj,
                                         struct sg_table *pages,
                                         struct i915_page_sizes *page_sizes)
{
        struct i915_sleeve *sleeve;
        struct i915_vma *vma;
        int err;

        sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
        if (!sleeve)
                return ERR_PTR(-ENOMEM);

        vma = i915_vma_instance(obj, vm, NULL);
        if (IS_ERR(vma)) {
                err = PTR_ERR(vma);
                goto err_free;
        }

        vma->private = sleeve;
        vma->ops = &proxy_vma_ops;

        sleeve->vma = vma;
        sleeve->obj = i915_gem_object_get(obj);
        sleeve->pages = pages;
        sleeve->page_sizes = *page_sizes;

        return sleeve;

err_free:
        kfree(sleeve);
        return ERR_PTR(err);
}

static void destroy_sleeve(struct i915_sleeve *sleeve)
{
        i915_gem_object_put(sleeve->obj);
        kfree(sleeve);
}

struct clear_pages_work {
        struct dma_fence dma;
        struct dma_fence_cb cb;
        struct i915_sw_fence wait;
        struct work_struct work;
        struct irq_work irq_work;
        struct i915_sleeve *sleeve;
        struct intel_context *ce;
        u32 value;
};

static const char *clear_pages_work_driver_name(struct dma_fence *fence)
{
        return DRIVER_NAME;
}

static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
{
        return "clear";
}

static void clear_pages_work_release(struct dma_fence *fence)
{
        struct clear_pages_work *w = container_of(fence, typeof(*w), dma);

        destroy_sleeve(w->sleeve);

        i915_sw_fence_fini(&w->wait);

        BUILD_BUG_ON(offsetof(typeof(*w), dma));
        dma_fence_free(&w->dma);
}

static const struct dma_fence_ops clear_pages_work_ops = {
        .get_driver_name = clear_pages_work_driver_name,
        .get_timeline_name = clear_pages_work_timeline_name,
        .release = clear_pages_work_release,
};

static void clear_pages_signal_irq_worker(struct irq_work *work)
{
        struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);

        dma_fence_signal(&w->dma);
        dma_fence_put(&w->dma);
}

static void clear_pages_dma_fence_cb(struct dma_fence *fence,
                                     struct dma_fence_cb *cb)
{
        struct clear_pages_work *w = container_of(cb, typeof(*w), cb);

        if (fence->error)
                dma_fence_set_error(&w->dma, fence->error);

        /*
         * Push the signalling of the fence into yet another worker to avoid
         * the nightmare locking around the fence spinlock.
         */
        irq_work_queue(&w->irq_work);
}

static void clear_pages_worker(struct work_struct *work)
{
        struct clear_pages_work *w = container_of(work, typeof(*w), work);
        struct drm_i915_private *i915 = w->ce->gem_context->i915;
        struct drm_i915_gem_object *obj = w->sleeve->obj;
        struct i915_vma *vma = w->sleeve->vma;
        struct i915_request *rq;
        int err = w->dma.error;

        if (unlikely(err))
                goto out_signal;

        if (obj->cache_dirty) {
                obj->write_domain = 0;
                if (i915_gem_object_has_struct_page(obj))
                        drm_clflush_sg(w->sleeve->pages);
                obj->cache_dirty = false;
        }

        /* XXX: we need to kill this */
        mutex_lock(&i915->drm.struct_mutex);
        err = i915_vma_pin(vma, 0, 0, PIN_USER);
        if (unlikely(err))
                goto out_unlock;

        rq = i915_request_create(w->ce);
        if (IS_ERR(rq)) {
                err = PTR_ERR(rq);
                goto out_unpin;
        }

        /* There's no way the fence has signalled */
        if (dma_fence_add_callback(&rq->fence, &w->cb,
                                   clear_pages_dma_fence_cb))
                GEM_BUG_ON(1);

        if (w->ce->engine->emit_init_breadcrumb) {
                err = w->ce->engine->emit_init_breadcrumb(rq);
                if (unlikely(err))
                        goto out_request;
        }

        /* XXX: more feverish nightmares await */
        i915_vma_lock(vma);
        err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
        i915_vma_unlock(vma);
        if (err)
                goto out_request;

        err = intel_emit_vma_fill_blt(rq, vma, w->value);
out_request:
        if (unlikely(err)) {
                i915_request_skip(rq, err);
                err = 0;
        }

        i915_request_add(rq);
out_unpin:
        i915_vma_unpin(vma);
out_unlock:
        mutex_unlock(&i915->drm.struct_mutex);
out_signal:
        if (unlikely(err)) {
                dma_fence_set_error(&w->dma, err);
                dma_fence_signal(&w->dma);
                dma_fence_put(&w->dma);
        }
}

static int __i915_sw_fence_call
clear_pages_work_notify(struct i915_sw_fence *fence,
                        enum i915_sw_fence_notify state)
{
        struct clear_pages_work *w = container_of(fence, typeof(*w), wait);

        switch (state) {
        case FENCE_COMPLETE:
                schedule_work(&w->work);
                break;

        case FENCE_FREE:
                dma_fence_put(&w->dma);
                break;
        }

        return NOTIFY_DONE;
}

static DEFINE_SPINLOCK(fence_lock);

/* XXX: better name please */
int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
                                     struct intel_context *ce,
                                     struct sg_table *pages,
                                     struct i915_page_sizes *page_sizes,
                                     u32 value)
{
        struct drm_i915_private *i915 = to_i915(obj->base.dev);
        struct i915_gem_context *ctx = ce->gem_context;
        struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
        struct clear_pages_work *work;
        struct i915_sleeve *sleeve;
        int err;

        sleeve = create_sleeve(vm, obj, pages, page_sizes);
        if (IS_ERR(sleeve))
                return PTR_ERR(sleeve);

        work = kmalloc(sizeof(*work), GFP_KERNEL);
        if (!work) {
                destroy_sleeve(sleeve);
                return -ENOMEM;
        }

        work->value = value;
        work->sleeve = sleeve;
        work->ce = ce;

        INIT_WORK(&work->work, clear_pages_worker);

        init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);

        dma_fence_init(&work->dma,
                       &clear_pages_work_ops,
                       &fence_lock,
                       i915->mm.unordered_timeline,
                       0);
        i915_sw_fence_init(&work->wait, clear_pages_work_notify);

        i915_gem_object_lock(obj);
        err = i915_sw_fence_await_reservation(&work->wait,
                                              obj->base.resv, NULL,
                                              true, I915_FENCE_TIMEOUT,
                                              I915_FENCE_GFP);
        if (err < 0) {
                dma_fence_set_error(&work->dma, err);
        } else {
                reservation_object_add_excl_fence(obj->base.resv, &work->dma);
                err = 0;
        }
        i915_gem_object_unlock(obj);

        dma_fence_get(&work->dma);
        i915_sw_fence_commit(&work->wait);

        return err;
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/i915_gem_client_blt.c"
#endif