// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/* Copyright (c) 2010-2012 Broadcom. All rights reserved. */

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/dma-mapping.h>
#include <linux/io.h>
#include <linux/platform_device.h>
#include <linux/uaccess.h>
#include <linux/mm.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <soc/bcm2835/raspberrypi-firmware.h>

#define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)

#include "vchiq_arm.h"
#include "vchiq_connected.h"
#include "vchiq_pagelist.h"

#define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)

#define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
#define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX  1

#define BELL0   0x00
#define BELL2   0x08

struct vchiq_2835_state {
        int inited;
        struct vchiq_arm_state arm_state;
};

struct vchiq_pagelist_info {
        struct pagelist *pagelist;
        size_t pagelist_buffer_size;
        dma_addr_t dma_addr;
        enum dma_data_direction dma_dir;
        unsigned int num_pages;
        unsigned int pages_need_release;
        struct page **pages;
        struct scatterlist *scatterlist;
        unsigned int scatterlist_mapped;
};

static void __iomem *g_regs;
/* This value is the size of the L2 cache lines as understood by the
 * VPU firmware, which determines the required alignment of the
 * offsets/sizes in pagelists.
 *
 * Modern VPU firmware looks for a DT "cache-line-size" property in
 * the VCHIQ node and will overwrite it with the actual L2 cache size,
 * which the kernel must then respect.  That property was rejected
 * upstream, so we have to use the VPU firmware's compatibility value
 * of 32.
 */
static unsigned int g_cache_line_size = 32;
static unsigned int g_fragments_size;
static char *g_fragments_base;
static char *g_free_fragments;
static struct semaphore g_free_fragments_sema;
static struct device *g_dev;

static DEFINE_SEMAPHORE(g_free_fragments_mutex);

static irqreturn_t
vchiq_doorbell_irq(int irq, void *dev_id);

static struct vchiq_pagelist_info *
create_pagelist(char __user *buf, size_t count, unsigned short type);

static void
free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
              int actual);

int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state)
{
        struct device *dev = &pdev->dev;
        struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev);
        struct rpi_firmware *fw = drvdata->fw;
        struct vchiq_slot_zero *vchiq_slot_zero;
        void *slot_mem;
        dma_addr_t slot_phys;
        u32 channelbase;
        int slot_mem_size, frag_mem_size;
        int err, irq, i;

        /*
         * VCHI messages between the CPU and firmware use
         * 32-bit bus addresses.
         */
        err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));

        if (err < 0)
                return err;

        g_cache_line_size = drvdata->cache_line_size;
        g_fragments_size = 2 * g_cache_line_size;

        /* Allocate space for the channels in coherent memory */
        slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
        frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);

        slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
                                       &slot_phys, GFP_KERNEL);
        if (!slot_mem) {
                dev_err(dev, "could not allocate DMA memory\n");
                return -ENOMEM;
        }

        WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);

        vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
        if (!vchiq_slot_zero)
                return -EINVAL;

        vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
                (int)slot_phys + slot_mem_size;
        vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
                MAX_FRAGMENTS;

        g_fragments_base = (char *)slot_mem + slot_mem_size;

        g_free_fragments = g_fragments_base;
        for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
                *(char **)&g_fragments_base[i*g_fragments_size] =
                        &g_fragments_base[(i + 1)*g_fragments_size];
        }
        *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
        sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);

        if (vchiq_init_state(state, vchiq_slot_zero) != VCHIQ_SUCCESS)
                return -EINVAL;

        g_regs = devm_platform_ioremap_resource(pdev, 0);
        if (IS_ERR(g_regs))
                return PTR_ERR(g_regs);

        irq = platform_get_irq(pdev, 0);
        if (irq <= 0)
                return irq;

        err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
                               "VCHIQ doorbell", state);
        if (err) {
                dev_err(dev, "failed to register irq=%d\n", irq);
                return err;
        }

        /* Send the base address of the slots to VideoCore */
        channelbase = slot_phys;
        err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
                                    &channelbase, sizeof(channelbase));
        if (err || channelbase) {
                dev_err(dev, "failed to set channelbase\n");
                return err ? : -ENXIO;
        }

        g_dev = dev;
        vchiq_log_info(vchiq_arm_log_level,
                "vchiq_init - done (slots %pK, phys %pad)",
                vchiq_slot_zero, &slot_phys);

        vchiq_call_connected_callbacks();

        return 0;
}

enum vchiq_status
vchiq_platform_init_state(struct vchiq_state *state)
{
        enum vchiq_status status = VCHIQ_SUCCESS;
        struct vchiq_2835_state *platform_state;

        state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL);
        if (!state->platform_state)
                return VCHIQ_ERROR;

        platform_state = (struct vchiq_2835_state *)state->platform_state;

        platform_state->inited = 1;
        status = vchiq_arm_init_state(state, &platform_state->arm_state);

        if (status != VCHIQ_SUCCESS)
                platform_state->inited = 0;

        return status;
}

struct vchiq_arm_state*
vchiq_platform_get_arm_state(struct vchiq_state *state)
{
        struct vchiq_2835_state *platform_state;

        platform_state   = (struct vchiq_2835_state *)state->platform_state;

        WARN_ON_ONCE(!platform_state->inited);

        return &platform_state->arm_state;
}

void
remote_event_signal(struct remote_event *event)
{
        wmb();

        event->fired = 1;

        dsb(sy);         /* data barrier operation */

        if (event->armed)
                writel(0, g_regs + BELL2); /* trigger vc interrupt */
}

enum vchiq_status
vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, int size,
                        int dir)
{
        struct vchiq_pagelist_info *pagelistinfo;

        pagelistinfo = create_pagelist((char __user *)offset, size,
                                       (dir == VCHIQ_BULK_RECEIVE)
                                       ? PAGELIST_READ
                                       : PAGELIST_WRITE);

        if (!pagelistinfo)
                return VCHIQ_ERROR;

        bulk->data = (void *)(unsigned long)pagelistinfo->dma_addr;

        /*
         * Store the pagelistinfo address in remote_data,
         * which isn't used by the slave.
         */
        bulk->remote_data = pagelistinfo;

        return VCHIQ_SUCCESS;
}

void
vchiq_complete_bulk(struct vchiq_bulk *bulk)
{
        if (bulk && bulk->remote_data && bulk->actual)
                free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
                              bulk->actual);
}

int vchiq_dump_platform_state(void *dump_context)
{
        char buf[80];
        int len;

        len = snprintf(buf, sizeof(buf),
                "  Platform: 2835 (VC master)");
        return vchiq_dump(dump_context, buf, len + 1);
}

/*
 * Local functions
 */

static irqreturn_t
vchiq_doorbell_irq(int irq, void *dev_id)
{
        struct vchiq_state *state = dev_id;
        irqreturn_t ret = IRQ_NONE;
        unsigned int status;

        /* Read (and clear) the doorbell */
        status = readl(g_regs + BELL0);

        if (status & 0x4) {  /* Was the doorbell rung? */
                remote_event_pollall(state);
                ret = IRQ_HANDLED;
        }

        return ret;
}

static void
cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
{
        if (pagelistinfo->scatterlist_mapped) {
                dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
                             pagelistinfo->num_pages, pagelistinfo->dma_dir);
        }

        if (pagelistinfo->pages_need_release)
                unpin_user_pages(pagelistinfo->pages, pagelistinfo->num_pages);

        dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
                          pagelistinfo->pagelist, pagelistinfo->dma_addr);
}

/* There is a potential problem with partial cache lines (pages?)
 * at the ends of the block when reading. If the CPU accessed anything in
 * the same line (page?) then it may have pulled old data into the cache,
 * obscuring the new data underneath. We can solve this by transferring the
 * partial cache lines separately, and allowing the ARM to copy into the
 * cached area.
 */

static struct vchiq_pagelist_info *
create_pagelist(char __user *buf, size_t count, unsigned short type)
{
        struct pagelist *pagelist;
        struct vchiq_pagelist_info *pagelistinfo;
        struct page **pages;
        u32 *addrs;
        unsigned int num_pages, offset, i, k;
        int actual_pages;
        size_t pagelist_size;
        struct scatterlist *scatterlist, *sg;
        int dma_buffers;
        dma_addr_t dma_addr;

        if (count >= INT_MAX - PAGE_SIZE)
                return NULL;

        offset = ((unsigned int)(unsigned long)buf & (PAGE_SIZE - 1));
        num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);

        if (num_pages > (SIZE_MAX - sizeof(struct pagelist) -
                         sizeof(struct vchiq_pagelist_info)) /
                        (sizeof(u32) + sizeof(pages[0]) +
                         sizeof(struct scatterlist)))
                return NULL;

        pagelist_size = sizeof(struct pagelist) +
                        (num_pages * sizeof(u32)) +
                        (num_pages * sizeof(pages[0]) +
                        (num_pages * sizeof(struct scatterlist))) +
                        sizeof(struct vchiq_pagelist_info);

        /* Allocate enough storage to hold the page pointers and the page
         * list
         */
        pagelist = dma_alloc_coherent(g_dev, pagelist_size, &dma_addr,
                                      GFP_KERNEL);

        vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist);

        if (!pagelist)
                return NULL;

        addrs           = pagelist->addrs;
        pages           = (struct page **)(addrs + num_pages);
        scatterlist     = (struct scatterlist *)(pages + num_pages);
        pagelistinfo    = (struct vchiq_pagelist_info *)
                          (scatterlist + num_pages);

        pagelist->length = count;
        pagelist->type = type;
        pagelist->offset = offset;

        /* Populate the fields of the pagelistinfo structure */
        pagelistinfo->pagelist = pagelist;
        pagelistinfo->pagelist_buffer_size = pagelist_size;
        pagelistinfo->dma_addr = dma_addr;
        pagelistinfo->dma_dir =  (type == PAGELIST_WRITE) ?
                                  DMA_TO_DEVICE : DMA_FROM_DEVICE;
        pagelistinfo->num_pages = num_pages;
        pagelistinfo->pages_need_release = 0;
        pagelistinfo->pages = pages;
        pagelistinfo->scatterlist = scatterlist;
        pagelistinfo->scatterlist_mapped = 0;

        if (is_vmalloc_addr((void __force *)buf)) {
                unsigned long length = count;
                unsigned int off = offset;

                for (actual_pages = 0; actual_pages < num_pages;
                     actual_pages++) {
                        struct page *pg =
                                vmalloc_to_page((void __force *)(buf +
                                                 (actual_pages * PAGE_SIZE)));
                        size_t bytes = PAGE_SIZE - off;

                        if (!pg) {
                                cleanup_pagelistinfo(pagelistinfo);
                                return NULL;
                        }

                        if (bytes > length)
                                bytes = length;
                        pages[actual_pages] = pg;
                        length -= bytes;
                        off = 0;
                }
                /* do not try and release vmalloc pages */
        } else {
                actual_pages = pin_user_pages_fast(
                                          (unsigned long)buf & PAGE_MASK,
                                          num_pages,
                                          type == PAGELIST_READ,
                                          pages);

                if (actual_pages != num_pages) {
                        vchiq_log_info(vchiq_arm_log_level,
                                       "%s - only %d/%d pages locked",
                                       __func__, actual_pages, num_pages);

                        /* This is probably due to the process being killed */
                        if (actual_pages > 0)
                                unpin_user_pages(pages, actual_pages);
                        cleanup_pagelistinfo(pagelistinfo);
                        return NULL;
                }
                 /* release user pages */
                pagelistinfo->pages_need_release = 1;
        }

        /*
         * Initialize the scatterlist so that the magic cookie
         *  is filled if debugging is enabled
         */
        sg_init_table(scatterlist, num_pages);
        /* Now set the pages for each scatterlist */
        for (i = 0; i < num_pages; i++) {
                unsigned int len = PAGE_SIZE - offset;

                if (len > count)
                        len = count;
                sg_set_page(scatterlist + i, pages[i], len, offset);
                offset = 0;
                count -= len;
        }

        dma_buffers = dma_map_sg(g_dev,
                                 scatterlist,
                                 num_pages,
                                 pagelistinfo->dma_dir);

        if (dma_buffers == 0) {
                cleanup_pagelistinfo(pagelistinfo);
                return NULL;
        }

        pagelistinfo->scatterlist_mapped = 1;

        /* Combine adjacent blocks for performance */
        k = 0;
        for_each_sg(scatterlist, sg, dma_buffers, i) {
                u32 len = sg_dma_len(sg);
                u32 addr = sg_dma_address(sg);

                /* Note: addrs is the address + page_count - 1
                 * The firmware expects blocks after the first to be page-
                 * aligned and a multiple of the page size
                 */
                WARN_ON(len == 0);
                WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
                WARN_ON(i && (addr & ~PAGE_MASK));
                if (k > 0 &&
                    ((addrs[k - 1] & PAGE_MASK) +
                     (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT))
                    == (addr & PAGE_MASK))
                        addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
                else
                        addrs[k++] = (addr & PAGE_MASK) |
                                (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
        }

        /* Partial cache lines (fragments) require special measures */
        if ((type == PAGELIST_READ) &&
                ((pagelist->offset & (g_cache_line_size - 1)) ||
                ((pagelist->offset + pagelist->length) &
                (g_cache_line_size - 1)))) {
                char *fragments;

                if (down_interruptible(&g_free_fragments_sema)) {
                        cleanup_pagelistinfo(pagelistinfo);
                        return NULL;
                }

                WARN_ON(!g_free_fragments);

                down(&g_free_fragments_mutex);
                fragments = g_free_fragments;
                WARN_ON(!fragments);
                g_free_fragments = *(char **) g_free_fragments;
                up(&g_free_fragments_mutex);
                pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
                        (fragments - g_fragments_base) / g_fragments_size;
        }

        return pagelistinfo;
}

static void
free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
              int actual)
{
        struct pagelist *pagelist = pagelistinfo->pagelist;
        struct page **pages = pagelistinfo->pages;
        unsigned int num_pages = pagelistinfo->num_pages;

        vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d",
                        __func__, pagelistinfo->pagelist, actual);

        /*
         * NOTE: dma_unmap_sg must be called before the
         * cpu can touch any of the data/pages.
         */
        dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
                     pagelistinfo->num_pages, pagelistinfo->dma_dir);
        pagelistinfo->scatterlist_mapped = 0;

        /* Deal with any partial cache lines (fragments) */
        if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
                char *fragments = g_fragments_base +
                        (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
                        g_fragments_size;
                int head_bytes, tail_bytes;

                head_bytes = (g_cache_line_size - pagelist->offset) &
                        (g_cache_line_size - 1);
                tail_bytes = (pagelist->offset + actual) &
                        (g_cache_line_size - 1);

                if ((actual >= 0) && (head_bytes != 0)) {
                        if (head_bytes > actual)
                                head_bytes = actual;

                        memcpy((char *)kmap(pages[0]) +
                                pagelist->offset,
                                fragments,
                                head_bytes);
                        kunmap(pages[0]);
                }
                if ((actual >= 0) && (head_bytes < actual) &&
                        (tail_bytes != 0)) {
                        memcpy((char *)kmap(pages[num_pages - 1]) +
                                ((pagelist->offset + actual) &
                                (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
                                fragments + g_cache_line_size,
                                tail_bytes);
                        kunmap(pages[num_pages - 1]);
                }

                down(&g_free_fragments_mutex);
                *(char **)fragments = g_free_fragments;
                g_free_fragments = fragments;
                up(&g_free_fragments_mutex);
                up(&g_free_fragments_sema);
        }

        /* Need to mark all the pages dirty. */
        if (pagelist->type != PAGELIST_WRITE &&
            pagelistinfo->pages_need_release) {
                unsigned int i;

                for (i = 0; i < num_pages; i++)
                        set_page_dirty(pages[i]);
        }

        cleanup_pagelistinfo(pagelistinfo);
}