/*
 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "rxe.h"
#include "rxe_loc.h"

/*
 * lfsr (linear feedback shift register) with period 255
 */
static u8 rxe_get_key(void)
{
        static u32 key = 1;

        key = key << 1;

        key |= (0 != (key & 0x100)) ^ (0 != (key & 0x10))
                ^ (0 != (key & 0x80)) ^ (0 != (key & 0x40));

        key &= 0xff;

        return key;
}

int mem_check_range(struct rxe_mem *mem, u64 iova, size_t length)
{
        switch (mem->type) {
        case RXE_MEM_TYPE_DMA:
                return 0;

        case RXE_MEM_TYPE_MR:
        case RXE_MEM_TYPE_FMR:
                if (iova < mem->iova ||
                    length > mem->length ||
                    iova > mem->iova + mem->length - length)
                        return -EFAULT;
                return 0;

        default:
                return -EFAULT;
        }
}

#define IB_ACCESS_REMOTE        (IB_ACCESS_REMOTE_READ          \
                                | IB_ACCESS_REMOTE_WRITE        \
                                | IB_ACCESS_REMOTE_ATOMIC)

static void rxe_mem_init(int access, struct rxe_mem *mem)
{
        u32 lkey = mem->pelem.index << 8 | rxe_get_key();
        u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;

        if (mem->pelem.pool->type == RXE_TYPE_MR) {
                mem->ibmr.lkey          = lkey;
                mem->ibmr.rkey          = rkey;
        }

        mem->lkey               = lkey;
        mem->rkey               = rkey;
        mem->state              = RXE_MEM_STATE_INVALID;
        mem->type               = RXE_MEM_TYPE_NONE;
        mem->map_shift          = ilog2(RXE_BUF_PER_MAP);
}

void rxe_mem_cleanup(struct rxe_pool_entry *arg)
{
        struct rxe_mem *mem = container_of(arg, typeof(*mem), pelem);
        int i;

        ib_umem_release(mem->umem);

        if (mem->map) {
                for (i = 0; i < mem->num_map; i++)
                        kfree(mem->map[i]);

                kfree(mem->map);
        }
}

static int rxe_mem_alloc(struct rxe_mem *mem, int num_buf)
{
        int i;
        int num_map;
        struct rxe_map **map = mem->map;

        num_map = (num_buf + RXE_BUF_PER_MAP - 1) / RXE_BUF_PER_MAP;

        mem->map = kmalloc_array(num_map, sizeof(*map), GFP_KERNEL);
        if (!mem->map)
                goto err1;

        for (i = 0; i < num_map; i++) {
                mem->map[i] = kmalloc(sizeof(**map), GFP_KERNEL);
                if (!mem->map[i])
                        goto err2;
        }

        BUILD_BUG_ON(!is_power_of_2(RXE_BUF_PER_MAP));

        mem->map_shift  = ilog2(RXE_BUF_PER_MAP);
        mem->map_mask   = RXE_BUF_PER_MAP - 1;

        mem->num_buf = num_buf;
        mem->num_map = num_map;
        mem->max_buf = num_map * RXE_BUF_PER_MAP;

        return 0;

err2:
        for (i--; i >= 0; i--)
                kfree(mem->map[i]);

        kfree(mem->map);
err1:
        return -ENOMEM;
}

int rxe_mem_init_dma(struct rxe_pd *pd,
                     int access, struct rxe_mem *mem)
{
        rxe_mem_init(access, mem);

        mem->pd                 = pd;
        mem->access             = access;
        mem->state              = RXE_MEM_STATE_VALID;
        mem->type               = RXE_MEM_TYPE_DMA;

        return 0;
}

int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
                      u64 length, u64 iova, int access, struct ib_udata *udata,
                      struct rxe_mem *mem)
{
        struct rxe_map          **map;
        struct rxe_phys_buf     *buf = NULL;
        struct ib_umem          *umem;
        struct sg_page_iter     sg_iter;
        int                     num_buf;
        void                    *vaddr;
        int err;

        umem = ib_umem_get(udata, start, length, access, 0);
        if (IS_ERR(umem)) {
                pr_warn("err %d from rxe_umem_get\n",
                        (int)PTR_ERR(umem));
                err = -EINVAL;
                goto err1;
        }

        mem->umem = umem;
        num_buf = ib_umem_num_pages(umem);

        rxe_mem_init(access, mem);

        err = rxe_mem_alloc(mem, num_buf);
        if (err) {
                pr_warn("err %d from rxe_mem_alloc\n", err);
                ib_umem_release(umem);
                goto err1;
        }

        mem->page_shift         = PAGE_SHIFT;
        mem->page_mask = PAGE_SIZE - 1;

        num_buf                 = 0;
        map                     = mem->map;
        if (length > 0) {
                buf = map[0]->buf;

                for_each_sg_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
                        if (num_buf >= RXE_BUF_PER_MAP) {
                                map++;
                                buf = map[0]->buf;
                                num_buf = 0;
                        }

                        vaddr = page_address(sg_page_iter_page(&sg_iter));
                        if (!vaddr) {
                                pr_warn("null vaddr\n");
                                err = -ENOMEM;
                                goto err1;
                        }

                        buf->addr = (uintptr_t)vaddr;
                        buf->size = PAGE_SIZE;
                        num_buf++;
                        buf++;

                }
        }

        mem->pd                 = pd;
        mem->umem               = umem;
        mem->access             = access;
        mem->length             = length;
        mem->iova               = iova;
        mem->va                 = start;
        mem->offset             = ib_umem_offset(umem);
        mem->state              = RXE_MEM_STATE_VALID;
        mem->type               = RXE_MEM_TYPE_MR;

        return 0;

err1:
        return err;
}

int rxe_mem_init_fast(struct rxe_pd *pd,
                      int max_pages, struct rxe_mem *mem)
{
        int err;

        rxe_mem_init(0, mem);

        /* In fastreg, we also set the rkey */
        mem->ibmr.rkey = mem->ibmr.lkey;

        err = rxe_mem_alloc(mem, max_pages);
        if (err)
                goto err1;

        mem->pd                 = pd;
        mem->max_buf            = max_pages;
        mem->state              = RXE_MEM_STATE_FREE;
        mem->type               = RXE_MEM_TYPE_MR;

        return 0;

err1:
        return err;
}

static void lookup_iova(
        struct rxe_mem  *mem,
        u64                     iova,
        int                     *m_out,
        int                     *n_out,
        size_t                  *offset_out)
{
        size_t                  offset = iova - mem->iova + mem->offset;
        int                     map_index;
        int                     buf_index;
        u64                     length;

        if (likely(mem->page_shift)) {
                *offset_out = offset & mem->page_mask;
                offset >>= mem->page_shift;
                *n_out = offset & mem->map_mask;
                *m_out = offset >> mem->map_shift;
        } else {
                map_index = 0;
                buf_index = 0;

                length = mem->map[map_index]->buf[buf_index].size;

                while (offset >= length) {
                        offset -= length;
                        buf_index++;

                        if (buf_index == RXE_BUF_PER_MAP) {
                                map_index++;
                                buf_index = 0;
                        }
                        length = mem->map[map_index]->buf[buf_index].size;
                }

                *m_out = map_index;
                *n_out = buf_index;
                *offset_out = offset;
        }
}

void *iova_to_vaddr(struct rxe_mem *mem, u64 iova, int length)
{
        size_t offset;
        int m, n;
        void *addr;

        if (mem->state != RXE_MEM_STATE_VALID) {
                pr_warn("mem not in valid state\n");
                addr = NULL;
                goto out;
        }

        if (!mem->map) {
                addr = (void *)(uintptr_t)iova;
                goto out;
        }

        if (mem_check_range(mem, iova, length)) {
                pr_warn("range violation\n");
                addr = NULL;
                goto out;
        }

        lookup_iova(mem, iova, &m, &n, &offset);

        if (offset + length > mem->map[m]->buf[n].size) {
                pr_warn("crosses page boundary\n");
                addr = NULL;
                goto out;
        }

        addr = (void *)(uintptr_t)mem->map[m]->buf[n].addr + offset;

out:
        return addr;
}

/* copy data from a range (vaddr, vaddr+length-1) to or from
 * a mem object starting at iova. Compute incremental value of
 * crc32 if crcp is not zero. caller must hold a reference to mem
 */
int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr, int length,
                 enum copy_direction dir, u32 *crcp)
{
        int                     err;
        int                     bytes;
        u8                      *va;
        struct rxe_map          **map;
        struct rxe_phys_buf     *buf;
        int                     m;
        int                     i;
        size_t                  offset;
        u32                     crc = crcp ? (*crcp) : 0;

        if (length == 0)
                return 0;

        if (mem->type == RXE_MEM_TYPE_DMA) {
                u8 *src, *dest;

                src  = (dir == to_mem_obj) ?
                        addr : ((void *)(uintptr_t)iova);

                dest = (dir == to_mem_obj) ?
                        ((void *)(uintptr_t)iova) : addr;

                memcpy(dest, src, length);

                if (crcp)
                        *crcp = rxe_crc32(to_rdev(mem->pd->ibpd.device),
                                        *crcp, dest, length);

                return 0;
        }

        WARN_ON_ONCE(!mem->map);

        err = mem_check_range(mem, iova, length);
        if (err) {
                err = -EFAULT;
                goto err1;
        }

        lookup_iova(mem, iova, &m, &i, &offset);

        map     = mem->map + m;
        buf     = map[0]->buf + i;

        while (length > 0) {
                u8 *src, *dest;

                va      = (u8 *)(uintptr_t)buf->addr + offset;
                src  = (dir == to_mem_obj) ? addr : va;
                dest = (dir == to_mem_obj) ? va : addr;

                bytes   = buf->size - offset;

                if (bytes > length)
                        bytes = length;

                memcpy(dest, src, bytes);

                if (crcp)
                        crc = rxe_crc32(to_rdev(mem->pd->ibpd.device),
                                        crc, dest, bytes);

                length  -= bytes;
                addr    += bytes;

                offset  = 0;
                buf++;
                i++;

                if (i == RXE_BUF_PER_MAP) {
                        i = 0;
                        map++;
                        buf = map[0]->buf;
                }
        }

        if (crcp)
                *crcp = crc;

        return 0;

err1:
        return err;
}

/* copy data in or out of a wqe, i.e. sg list
 * under the control of a dma descriptor
 */
int copy_data(
        struct rxe_pd           *pd,
        int                     access,
        struct rxe_dma_info     *dma,
        void                    *addr,
        int                     length,
        enum copy_direction     dir,
        u32                     *crcp)
{
        int                     bytes;
        struct rxe_sge          *sge    = &dma->sge[dma->cur_sge];
        int                     offset  = dma->sge_offset;
        int                     resid   = dma->resid;
        struct rxe_mem          *mem    = NULL;
        u64                     iova;
        int                     err;

        if (length == 0)
                return 0;

        if (length > resid) {
                err = -EINVAL;
                goto err2;
        }

        if (sge->length && (offset < sge->length)) {
                mem = lookup_mem(pd, access, sge->lkey, lookup_local);
                if (!mem) {
                        err = -EINVAL;
                        goto err1;
                }
        }

        while (length > 0) {
                bytes = length;

                if (offset >= sge->length) {
                        if (mem) {
                                rxe_drop_ref(mem);
                                mem = NULL;
                        }
                        sge++;
                        dma->cur_sge++;
                        offset = 0;

                        if (dma->cur_sge >= dma->num_sge) {
                                err = -ENOSPC;
                                goto err2;
                        }

                        if (sge->length) {
                                mem = lookup_mem(pd, access, sge->lkey,
                                                 lookup_local);
                                if (!mem) {
                                        err = -EINVAL;
                                        goto err1;
                                }
                        } else {
                                continue;
                        }
                }

                if (bytes > sge->length - offset)
                        bytes = sge->length - offset;

                if (bytes > 0) {
                        iova = sge->addr + offset;

                        err = rxe_mem_copy(mem, iova, addr, bytes, dir, crcp);
                        if (err)
                                goto err2;

                        offset  += bytes;
                        resid   -= bytes;
                        length  -= bytes;
                        addr    += bytes;
                }
        }

        dma->sge_offset = offset;
        dma->resid      = resid;

        if (mem)
                rxe_drop_ref(mem);

        return 0;

err2:
        if (mem)
                rxe_drop_ref(mem);
err1:
        return err;
}

int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
{
        struct rxe_sge          *sge    = &dma->sge[dma->cur_sge];
        int                     offset  = dma->sge_offset;
        int                     resid   = dma->resid;

        while (length) {
                unsigned int bytes;

                if (offset >= sge->length) {
                        sge++;
                        dma->cur_sge++;
                        offset = 0;
                        if (dma->cur_sge >= dma->num_sge)
                                return -ENOSPC;
                }

                bytes = length;

                if (bytes > sge->length - offset)
                        bytes = sge->length - offset;

                offset  += bytes;
                resid   -= bytes;
                length  -= bytes;
        }

        dma->sge_offset = offset;
        dma->resid      = resid;

        return 0;
}

/* (1) find the mem (mr or mw) corresponding to lkey/rkey
 *     depending on lookup_type
 * (2) verify that the (qp) pd matches the mem pd
 * (3) verify that the mem can support the requested access
 * (4) verify that mem state is valid
 */
struct rxe_mem *lookup_mem(struct rxe_pd *pd, int access, u32 key,
                           enum lookup_type type)
{
        struct rxe_mem *mem;
        struct rxe_dev *rxe = to_rdev(pd->ibpd.device);
        int index = key >> 8;

        mem = rxe_pool_get_index(&rxe->mr_pool, index);
        if (!mem)
                return NULL;

        if (unlikely((type == lookup_local && mem->lkey != key) ||
                     (type == lookup_remote && mem->rkey != key) ||
                     mem->pd != pd ||
                     (access && !(access & mem->access)) ||
                     mem->state != RXE_MEM_STATE_VALID)) {
                rxe_drop_ref(mem);
                mem = NULL;
        }

        return mem;
}

int rxe_mem_map_pages(struct rxe_dev *rxe, struct rxe_mem *mem,
                      u64 *page, int num_pages, u64 iova)
{
        int i;
        int num_buf;
        int err;
        struct rxe_map **map;
        struct rxe_phys_buf *buf;
        int page_size;

        if (num_pages > mem->max_buf) {
                err = -EINVAL;
                goto err1;
        }

        num_buf         = 0;
        page_size       = 1 << mem->page_shift;
        map             = mem->map;
        buf             = map[0]->buf;

        for (i = 0; i < num_pages; i++) {
                buf->addr = *page++;
                buf->size = page_size;
                buf++;
                num_buf++;

                if (num_buf == RXE_BUF_PER_MAP) {
                        map++;
                        buf = map[0]->buf;
                        num_buf = 0;
                }
        }

        mem->iova       = iova;
        mem->va         = iova;
        mem->length     = num_pages << mem->page_shift;
        mem->state      = RXE_MEM_STATE_VALID;

        return 0;

err1:
        return err;
}