// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * Oracle Data Analytics Accelerator (DAX)
 *
 * DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8
 * (DAX2) processor chips, and has direct access to the CPU's L3
 * caches as well as physical memory. It can perform several
 * operations on data streams with various input and output formats.
 * The driver provides a transport mechanism only and has limited
 * knowledge of the various opcodes and data formats. A user space
 * library provides high level services and translates these into low
 * level commands which are then passed into the driver and
 * subsequently the hypervisor and the coprocessor.  The library is
 * the recommended way for applications to use the coprocessor, and
 * the driver interface is not intended for general use.
 *
 * See Documentation/sparc/oradax/oracle-dax.rst for more details.
 */

#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/cdev.h>
#include <linux/slab.h>
#include <linux/mm.h>

#include <asm/hypervisor.h>
#include <asm/mdesc.h>
#include <asm/oradax.h>

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Driver for Oracle Data Analytics Accelerator");

#define DAX_DBG_FLG_BASIC       0x01
#define DAX_DBG_FLG_STAT        0x02
#define DAX_DBG_FLG_INFO        0x04
#define DAX_DBG_FLG_ALL         0xff

#define dax_err(fmt, ...)      pr_err("%s: " fmt "\n", __func__, ##__VA_ARGS__)
#define dax_info(fmt, ...)     pr_info("%s: " fmt "\n", __func__, ##__VA_ARGS__)

#define dax_dbg(fmt, ...)       do {                                    \
                                        if (dax_debug & DAX_DBG_FLG_BASIC)\
                                                dax_info(fmt, ##__VA_ARGS__); \
                                } while (0)
#define dax_stat_dbg(fmt, ...)  do {                                    \
                                        if (dax_debug & DAX_DBG_FLG_STAT) \
                                                dax_info(fmt, ##__VA_ARGS__); \
                                } while (0)
#define dax_info_dbg(fmt, ...)  do { \
                                        if (dax_debug & DAX_DBG_FLG_INFO) \
                                                dax_info(fmt, ##__VA_ARGS__); \
                                } while (0)

#define DAX1_MINOR              1
#define DAX1_MAJOR              1
#define DAX2_MINOR              0
#define DAX2_MAJOR              2

#define DAX1_STR    "ORCL,sun4v-dax"
#define DAX2_STR    "ORCL,sun4v-dax2"

#define DAX_CA_ELEMS            (DAX_MMAP_LEN / sizeof(struct dax_cca))

#define DAX_CCB_USEC            100
#define DAX_CCB_RETRIES         10000

/* stream types */
enum {
        OUT,
        PRI,
        SEC,
        TBL,
        NUM_STREAM_TYPES
};

/* completion status */
#define CCA_STAT_NOT_COMPLETED  0
#define CCA_STAT_COMPLETED      1
#define CCA_STAT_FAILED         2
#define CCA_STAT_KILLED         3
#define CCA_STAT_NOT_RUN        4
#define CCA_STAT_PIPE_OUT       5
#define CCA_STAT_PIPE_SRC       6
#define CCA_STAT_PIPE_DST       7

/* completion err */
#define CCA_ERR_SUCCESS         0x0     /* no error */
#define CCA_ERR_OVERFLOW        0x1     /* buffer overflow */
#define CCA_ERR_DECODE          0x2     /* CCB decode error */
#define CCA_ERR_PAGE_OVERFLOW   0x3     /* page overflow */
#define CCA_ERR_KILLED          0x7     /* command was killed */
#define CCA_ERR_TIMEOUT         0x8     /* Timeout */
#define CCA_ERR_ADI             0x9     /* ADI error */
#define CCA_ERR_DATA_FMT        0xA     /* data format error */
#define CCA_ERR_OTHER_NO_RETRY  0xE     /* Other error, do not retry */
#define CCA_ERR_OTHER_RETRY     0xF     /* Other error, retry */
#define CCA_ERR_PARTIAL_SYMBOL  0x80    /* QP partial symbol warning */

/* CCB address types */
#define DAX_ADDR_TYPE_NONE      0
#define DAX_ADDR_TYPE_VA_ALT    1       /* secondary context */
#define DAX_ADDR_TYPE_RA        2       /* real address */
#define DAX_ADDR_TYPE_VA        3       /* virtual address */

/* dax_header_t opcode */
#define DAX_OP_SYNC_NOP         0x0
#define DAX_OP_EXTRACT          0x1
#define DAX_OP_SCAN_VALUE       0x2
#define DAX_OP_SCAN_RANGE       0x3
#define DAX_OP_TRANSLATE        0x4
#define DAX_OP_SELECT           0x5
#define DAX_OP_INVERT           0x10    /* OR with translate, scan opcodes */

struct dax_header {
        u32 ccb_version:4;      /* 31:28 CCB Version */
                                /* 27:24 Sync Flags */
        u32 pipe:1;             /* Pipeline */
        u32 longccb:1;          /* Longccb. Set for scan with lu2, lu3, lu4. */
        u32 cond:1;             /* Conditional */
        u32 serial:1;           /* Serial */
        u32 opcode:8;           /* 23:16 Opcode */
                                /* 15:0 Address Type. */
        u32 reserved:3;         /* 15:13 reserved */
        u32 table_addr_type:2;  /* 12:11 Huffman Table Address Type */
        u32 out_addr_type:3;    /* 10:8 Destination Address Type */
        u32 sec_addr_type:3;    /* 7:5 Secondary Source Address Type */
        u32 pri_addr_type:3;    /* 4:2 Primary Source Address Type */
        u32 cca_addr_type:2;    /* 1:0 Completion Address Type */
};

struct dax_control {
        u32 pri_fmt:4;          /* 31:28 Primary Input Format */
        u32 pri_elem_size:5;    /* 27:23 Primary Input Element Size(less1) */
        u32 pri_offset:3;       /* 22:20 Primary Input Starting Offset */
        u32 sec_encoding:1;     /* 19    Secondary Input Encoding */
                                /*       (must be 0 for Select) */
        u32 sec_offset:3;       /* 18:16 Secondary Input Starting Offset */
        u32 sec_elem_size:2;    /* 15:14 Secondary Input Element Size */
                                /*       (must be 0 for Select) */
        u32 out_fmt:2;          /* 13:12 Output Format */
        u32 out_elem_size:2;    /* 11:10 Output Element Size */
        u32 misc:10;            /* 9:0 Opcode specific info */
};

struct dax_data_access {
        u64 flow_ctrl:2;        /* 63:62 Flow Control Type */
        u64 pipe_target:2;      /* 61:60 Pipeline Target */
        u64 out_buf_size:20;    /* 59:40 Output Buffer Size */
                                /*       (cachelines less 1) */
        u64 unused1:8;          /* 39:32 Reserved, Set to 0 */
        u64 out_alloc:5;        /* 31:27 Output Allocation */
        u64 unused2:1;          /* 26    Reserved */
        u64 pri_len_fmt:2;      /* 25:24 Input Length Format */
        u64 pri_len:24;         /* 23:0  Input Element/Byte/Bit Count */
                                /*       (less 1) */
};

struct dax_ccb {
        struct dax_header hdr;  /* CCB Header */
        struct dax_control ctrl;/* Control Word */
        void *ca;               /* Completion Address */
        void *pri;              /* Primary Input Address */
        struct dax_data_access dac; /* Data Access Control */
        void *sec;              /* Secondary Input Address */
        u64 dword5;             /* depends on opcode */
        void *out;              /* Output Address */
        void *tbl;              /* Table Address or bitmap */
};

struct dax_cca {
        u8      status;         /* user may mwait on this address */
        u8      err;            /* user visible error notification */
        u8      rsvd[2];        /* reserved */
        u32     n_remaining;    /* for QP partial symbol warning */
        u32     output_sz;      /* output in bytes */
        u32     rsvd2;          /* reserved */
        u64     run_cycles;     /* run time in OCND2 cycles */
        u64     run_stats;      /* nothing reported in version 1.0 */
        u32     n_processed;    /* number input elements */
        u32     rsvd3[5];       /* reserved */
        u64     retval;         /* command return value */
        u64     rsvd4[8];       /* reserved */
};

/* per thread CCB context */
struct dax_ctx {
        struct dax_ccb          *ccb_buf;
        u64                     ccb_buf_ra;     /* cached RA of ccb_buf  */
        struct dax_cca          *ca_buf;
        u64                     ca_buf_ra;      /* cached RA of ca_buf   */
        struct page             *pages[DAX_CA_ELEMS][NUM_STREAM_TYPES];
                                                /* array of locked pages */
        struct task_struct      *owner;         /* thread that owns ctx  */
        struct task_struct      *client;        /* requesting thread     */
        union ccb_result        result;
        u32                     ccb_count;
        u32                     fail_count;
};

/* driver public entry points */
static int dax_open(struct inode *inode, struct file *file);
static ssize_t dax_read(struct file *filp, char __user *buf,
                        size_t count, loff_t *ppos);
static ssize_t dax_write(struct file *filp, const char __user *buf,
                         size_t count, loff_t *ppos);
static int dax_devmap(struct file *f, struct vm_area_struct *vma);
static int dax_close(struct inode *i, struct file *f);

static const struct file_operations dax_fops = {
        .owner  =       THIS_MODULE,
        .open   =       dax_open,
        .read   =       dax_read,
        .write  =       dax_write,
        .mmap   =       dax_devmap,
        .release =      dax_close,
};

static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf,
                        size_t count, loff_t *ppos);
static int dax_ccb_info(u64 ca, struct ccb_info_result *info);
static int dax_ccb_kill(u64 ca, u16 *kill_res);

static struct cdev c_dev;
static struct class *cl;
static dev_t first;

static int max_ccb_version;
static int dax_debug;
module_param(dax_debug, int, 0644);
MODULE_PARM_DESC(dax_debug, "Debug flags");

static int __init dax_attach(void)
{
        unsigned long dummy, hv_rv, major, minor, minor_requested, max_ccbs;
        struct mdesc_handle *hp = mdesc_grab();
        char *prop, *dax_name;
        bool found = false;
        int len, ret = 0;
        u64 pn;

        if (hp == NULL) {
                dax_err("Unable to grab mdesc");
                return -ENODEV;
        }

        mdesc_for_each_node_by_name(hp, pn, "virtual-device") {
                prop = (char *)mdesc_get_property(hp, pn, "name", &len);
                if (prop == NULL)
                        continue;
                if (strncmp(prop, "dax", strlen("dax")))
                        continue;
                dax_dbg("Found node 0x%llx = %s", pn, prop);

                prop = (char *)mdesc_get_property(hp, pn, "compatible", &len);
                if (prop == NULL)
                        continue;
                dax_dbg("Found node 0x%llx = %s", pn, prop);
                found = true;
                break;
        }

        if (!found) {
                dax_err("No DAX device found");
                ret = -ENODEV;
                goto done;
        }

        if (strncmp(prop, DAX2_STR, strlen(DAX2_STR)) == 0) {
                dax_name = DAX_NAME "2";
                major = DAX2_MAJOR;
                minor_requested = DAX2_MINOR;
                max_ccb_version = 1;
                dax_dbg("MD indicates DAX2 coprocessor");
        } else if (strncmp(prop, DAX1_STR, strlen(DAX1_STR)) == 0) {
                dax_name = DAX_NAME "1";
                major = DAX1_MAJOR;
                minor_requested = DAX1_MINOR;
                max_ccb_version = 0;
                dax_dbg("MD indicates DAX1 coprocessor");
        } else {
                dax_err("Unknown dax type: %s", prop);
                ret = -ENODEV;
                goto done;
        }

        minor = minor_requested;
        dax_dbg("Registering DAX HV api with major %ld minor %ld", major,
                minor);
        if (sun4v_hvapi_register(HV_GRP_DAX, major, &minor)) {
                dax_err("hvapi_register failed");
                ret = -ENODEV;
                goto done;
        } else {
                dax_dbg("Max minor supported by HV = %ld (major %ld)", minor,
                        major);
                minor = min(minor, minor_requested);
                dax_dbg("registered DAX major %ld minor %ld", major, minor);
        }

        /* submit a zero length ccb array to query coprocessor queue size */
        hv_rv = sun4v_ccb_submit(0, 0, HV_CCB_QUERY_CMD, 0, &max_ccbs, &dummy);
        if (hv_rv != 0) {
                dax_err("get_hwqueue_size failed with status=%ld and max_ccbs=%ld",
                        hv_rv, max_ccbs);
                ret = -ENODEV;
                goto done;
        }

        if (max_ccbs != DAX_MAX_CCBS) {
                dax_err("HV reports unsupported max_ccbs=%ld", max_ccbs);
                ret = -ENODEV;
                goto done;
        }

        if (alloc_chrdev_region(&first, 0, 1, DAX_NAME) < 0) {
                dax_err("alloc_chrdev_region failed");
                ret = -ENXIO;
                goto done;
        }

        cl = class_create(THIS_MODULE, DAX_NAME);
        if (IS_ERR(cl)) {
                dax_err("class_create failed");
                ret = PTR_ERR(cl);
                goto class_error;
        }

        if (device_create(cl, NULL, first, NULL, dax_name) == NULL) {
                dax_err("device_create failed");
                ret = -ENXIO;
                goto device_error;
        }

        cdev_init(&c_dev, &dax_fops);
        if (cdev_add(&c_dev, first, 1) == -1) {
                dax_err("cdev_add failed");
                ret = -ENXIO;
                goto cdev_error;
        }

        pr_info("Attached DAX module\n");
        goto done;

cdev_error:
        device_destroy(cl, first);
device_error:
        class_destroy(cl);
class_error:
        unregister_chrdev_region(first, 1);
done:
        mdesc_release(hp);
        return ret;
}
module_init(dax_attach);

static void __exit dax_detach(void)
{
        pr_info("Cleaning up DAX module\n");
        cdev_del(&c_dev);
        device_destroy(cl, first);
        class_destroy(cl);
        unregister_chrdev_region(first, 1);
}
module_exit(dax_detach);

/* map completion area */
static int dax_devmap(struct file *f, struct vm_area_struct *vma)
{
        struct dax_ctx *ctx = (struct dax_ctx *)f->private_data;
        size_t len = vma->vm_end - vma->vm_start;

        dax_dbg("len=0x%lx, flags=0x%lx", len, vma->vm_flags);

        if (ctx->owner != current) {
                dax_dbg("devmap called from wrong thread");
                return -EINVAL;
        }

        if (len != DAX_MMAP_LEN) {
                dax_dbg("len(%lu) != DAX_MMAP_LEN(%d)", len, DAX_MMAP_LEN);
                return -EINVAL;
        }

        /* completion area is mapped read-only for user */
        if (vma->vm_flags & VM_WRITE)
                return -EPERM;
        vma->vm_flags &= ~VM_MAYWRITE;

        if (remap_pfn_range(vma, vma->vm_start, ctx->ca_buf_ra >> PAGE_SHIFT,
                            len, vma->vm_page_prot))
                return -EAGAIN;

        dax_dbg("mmapped completion area at uva 0x%lx", vma->vm_start);
        return 0;
}

/* Unlock user pages. Called during dequeue or device close */
static void dax_unlock_pages(struct dax_ctx *ctx, int ccb_index, int nelem)
{
        int i, j;

        for (i = ccb_index; i < ccb_index + nelem; i++) {
                for (j = 0; j < NUM_STREAM_TYPES; j++) {
                        struct page *p = ctx->pages[i][j];

                        if (p) {
                                dax_dbg("freeing page %p", p);
                                if (j == OUT)
                                        set_page_dirty(p);
                                put_page(p);
                                ctx->pages[i][j] = NULL;
                        }
                }
        }
}

static int dax_lock_page(void *va, struct page **p)
{
        int ret;

        dax_dbg("uva %p", va);

        ret = get_user_pages_fast((unsigned long)va, 1, FOLL_WRITE, p);
        if (ret == 1) {
                dax_dbg("locked page %p, for VA %p", *p, va);
                return 0;
        }

        dax_dbg("get_user_pages failed, va=%p, ret=%d", va, ret);
        return -1;
}

static int dax_lock_pages(struct dax_ctx *ctx, int idx,
                          int nelem, u64 *err_va)
{
        int i;

        for (i = 0; i < nelem; i++) {
                struct dax_ccb *ccbp = &ctx->ccb_buf[i];

                /*
                 * For each address in the CCB whose type is virtual,
                 * lock the page and change the type to virtual alternate
                 * context. On error, return the offending address in
                 * err_va.
                 */
                if (ccbp->hdr.out_addr_type == DAX_ADDR_TYPE_VA) {
                        dax_dbg("output");
                        if (dax_lock_page(ccbp->out,
                                          &ctx->pages[i + idx][OUT]) != 0) {
                                *err_va = (u64)ccbp->out;
                                goto error;
                        }
                        ccbp->hdr.out_addr_type = DAX_ADDR_TYPE_VA_ALT;
                }

                if (ccbp->hdr.pri_addr_type == DAX_ADDR_TYPE_VA) {
                        dax_dbg("input");
                        if (dax_lock_page(ccbp->pri,
                                          &ctx->pages[i + idx][PRI]) != 0) {
                                *err_va = (u64)ccbp->pri;
                                goto error;
                        }
                        ccbp->hdr.pri_addr_type = DAX_ADDR_TYPE_VA_ALT;
                }

                if (ccbp->hdr.sec_addr_type == DAX_ADDR_TYPE_VA) {
                        dax_dbg("sec input");
                        if (dax_lock_page(ccbp->sec,
                                          &ctx->pages[i + idx][SEC]) != 0) {
                                *err_va = (u64)ccbp->sec;
                                goto error;
                        }
                        ccbp->hdr.sec_addr_type = DAX_ADDR_TYPE_VA_ALT;
                }

                if (ccbp->hdr.table_addr_type == DAX_ADDR_TYPE_VA) {
                        dax_dbg("tbl");
                        if (dax_lock_page(ccbp->tbl,
                                          &ctx->pages[i + idx][TBL]) != 0) {
                                *err_va = (u64)ccbp->tbl;
                                goto error;
                        }
                        ccbp->hdr.table_addr_type = DAX_ADDR_TYPE_VA_ALT;
                }

                /* skip over 2nd 64 bytes of long CCB */
                if (ccbp->hdr.longccb)
                        i++;
        }
        return DAX_SUBMIT_OK;

error:
        dax_unlock_pages(ctx, idx, nelem);
        return DAX_SUBMIT_ERR_NOACCESS;
}

static void dax_ccb_wait(struct dax_ctx *ctx, int idx)
{
        int ret, nretries;
        u16 kill_res;

        dax_dbg("idx=%d", idx);

        for (nretries = 0; nretries < DAX_CCB_RETRIES; nretries++) {
                if (ctx->ca_buf[idx].status == CCA_STAT_NOT_COMPLETED)
                        udelay(DAX_CCB_USEC);
                else
                        return;
        }
        dax_dbg("ctx (%p): CCB[%d] timed out, wait usec=%d, retries=%d. Killing ccb",
                (void *)ctx, idx, DAX_CCB_USEC, DAX_CCB_RETRIES);

        ret = dax_ccb_kill(ctx->ca_buf_ra + idx * sizeof(struct dax_cca),
                           &kill_res);
        dax_dbg("Kill CCB[%d] %s", idx, ret ? "failed" : "succeeded");
}

static int dax_close(struct inode *ino, struct file *f)
{
        struct dax_ctx *ctx = (struct dax_ctx *)f->private_data;
        int i;

        f->private_data = NULL;

        for (i = 0; i < DAX_CA_ELEMS; i++) {
                if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) {
                        dax_dbg("CCB[%d] not completed", i);
                        dax_ccb_wait(ctx, i);
                }
                dax_unlock_pages(ctx, i, 1);
        }

        kfree(ctx->ccb_buf);
        kfree(ctx->ca_buf);
        dax_stat_dbg("CCBs: %d good, %d bad", ctx->ccb_count, ctx->fail_count);
        kfree(ctx);

        return 0;
}

static ssize_t dax_read(struct file *f, char __user *buf,
                        size_t count, loff_t *ppos)
{
        struct dax_ctx *ctx = f->private_data;

        if (ctx->client != current)
                return -EUSERS;

        ctx->client = NULL;

        if (count != sizeof(union ccb_result))
                return -EINVAL;
        if (copy_to_user(buf, &ctx->result, sizeof(union ccb_result)))
                return -EFAULT;
        return count;
}

static ssize_t dax_write(struct file *f, const char __user *buf,
                         size_t count, loff_t *ppos)
{
        struct dax_ctx *ctx = f->private_data;
        struct dax_command hdr;
        unsigned long ca;
        int i, idx, ret;

        if (ctx->client != NULL)
                return -EINVAL;

        if (count == 0 || count > DAX_MAX_CCBS * sizeof(struct dax_ccb))
                return -EINVAL;

        if (count % sizeof(struct dax_ccb) == 0)
                return dax_ccb_exec(ctx, buf, count, ppos); /* CCB EXEC */

        if (count != sizeof(struct dax_command))
                return -EINVAL;

        /* immediate command */
        if (ctx->owner != current)
                return -EUSERS;

        if (copy_from_user(&hdr, buf, sizeof(hdr)))
                return -EFAULT;

        ca = ctx->ca_buf_ra + hdr.ca_offset;

        switch (hdr.command) {
        case CCB_KILL:
                if (hdr.ca_offset >= DAX_MMAP_LEN) {
                        dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)",
                                hdr.ca_offset, DAX_MMAP_LEN);
                        return -EINVAL;
                }

                ret = dax_ccb_kill(ca, &ctx->result.kill.action);
                if (ret != 0) {
                        dax_dbg("dax_ccb_kill failed (ret=%d)", ret);
                        return ret;
                }

                dax_info_dbg("killed (ca_offset %d)", hdr.ca_offset);
                idx = hdr.ca_offset / sizeof(struct dax_cca);
                ctx->ca_buf[idx].status = CCA_STAT_KILLED;
                ctx->ca_buf[idx].err = CCA_ERR_KILLED;
                ctx->client = current;
                return count;

        case CCB_INFO:
                if (hdr.ca_offset >= DAX_MMAP_LEN) {
                        dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)",
                                hdr.ca_offset, DAX_MMAP_LEN);
                        return -EINVAL;
                }

                ret = dax_ccb_info(ca, &ctx->result.info);
                if (ret != 0) {
                        dax_dbg("dax_ccb_info failed (ret=%d)", ret);
                        return ret;
                }

                dax_info_dbg("info succeeded on ca_offset %d", hdr.ca_offset);
                ctx->client = current;
                return count;

        case CCB_DEQUEUE:
                for (i = 0; i < DAX_CA_ELEMS; i++) {
                        if (ctx->ca_buf[i].status !=
                            CCA_STAT_NOT_COMPLETED)
                                dax_unlock_pages(ctx, i, 1);
                }
                return count;

        default:
                return -EINVAL;
        }
}

static int dax_open(struct inode *inode, struct file *f)
{
        struct dax_ctx *ctx = NULL;
        int i;

        ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
        if (ctx == NULL)
                goto done;

        ctx->ccb_buf = kcalloc(DAX_MAX_CCBS, sizeof(struct dax_ccb),
                               GFP_KERNEL);
        if (ctx->ccb_buf == NULL)
                goto done;

        ctx->ccb_buf_ra = virt_to_phys(ctx->ccb_buf);
        dax_dbg("ctx->ccb_buf=0x%p, ccb_buf_ra=0x%llx",
                (void *)ctx->ccb_buf, ctx->ccb_buf_ra);

        /* allocate CCB completion area buffer */
        ctx->ca_buf = kzalloc(DAX_MMAP_LEN, GFP_KERNEL);
        if (ctx->ca_buf == NULL)
                goto alloc_error;
        for (i = 0; i < DAX_CA_ELEMS; i++)
                ctx->ca_buf[i].status = CCA_STAT_COMPLETED;

        ctx->ca_buf_ra = virt_to_phys(ctx->ca_buf);
        dax_dbg("ctx=0x%p, ctx->ca_buf=0x%p, ca_buf_ra=0x%llx",
                (void *)ctx, (void *)ctx->ca_buf, ctx->ca_buf_ra);

        ctx->owner = current;
        f->private_data = ctx;
        return 0;

alloc_error:
        kfree(ctx->ccb_buf);
done:
        kfree(ctx);
        return -ENOMEM;
}

static char *dax_hv_errno(unsigned long hv_ret, int *ret)
{
        switch (hv_ret) {
        case HV_EBADALIGN:
                *ret = -EFAULT;
                return "HV_EBADALIGN";
        case HV_ENORADDR:
                *ret = -EFAULT;
                return "HV_ENORADDR";
        case HV_EINVAL:
                *ret = -EINVAL;
                return "HV_EINVAL";
        case HV_EWOULDBLOCK:
                *ret = -EAGAIN;
                return "HV_EWOULDBLOCK";
        case HV_ENOACCESS:
                *ret = -EPERM;
                return "HV_ENOACCESS";
        default:
                break;
        }

        *ret = -EIO;
        return "UNKNOWN";
}

static int dax_ccb_kill(u64 ca, u16 *kill_res)
{
        unsigned long hv_ret;
        int count, ret = 0;
        char *err_str;

        for (count = 0; count < DAX_CCB_RETRIES; count++) {
                dax_dbg("attempting kill on ca_ra 0x%llx", ca);
                hv_ret = sun4v_ccb_kill(ca, kill_res);

                if (hv_ret == HV_EOK) {
                        dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca,
                                     *kill_res);
                } else {
                        err_str = dax_hv_errno(hv_ret, &ret);
                        dax_dbg("%s (ca_ra 0x%llx)", err_str, ca);
                }

                if (ret != -EAGAIN)
                        return ret;
                dax_info_dbg("ccb_kill count = %d", count);
                udelay(DAX_CCB_USEC);
        }

        return -EAGAIN;
}

static int dax_ccb_info(u64 ca, struct ccb_info_result *info)
{
        unsigned long hv_ret;
        char *err_str;
        int ret = 0;

        dax_dbg("attempting info on ca_ra 0x%llx", ca);
        hv_ret = sun4v_ccb_info(ca, info);

        if (hv_ret == HV_EOK) {
                dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca, info->state);
                if (info->state == DAX_CCB_ENQUEUED) {
                        dax_info_dbg("dax_unit %d, queue_num %d, queue_pos %d",
                                     info->inst_num, info->q_num, info->q_pos);
                }
        } else {
                err_str = dax_hv_errno(hv_ret, &ret);
                dax_dbg("%s (ca_ra 0x%llx)", err_str, ca);
        }

        return ret;
}

static void dax_prt_ccbs(struct dax_ccb *ccb, int nelem)
{
        int i, j;
        u64 *ccbp;

        dax_dbg("ccb buffer:");
        for (i = 0; i < nelem; i++) {
                ccbp = (u64 *)&ccb[i];
                dax_dbg(" %sccb[%d]", ccb[i].hdr.longccb ? "long " : "",  i);
                for (j = 0; j < 8; j++)
                        dax_dbg("\tccb[%d].dwords[%d]=0x%llx",
                                i, j, *(ccbp + j));
        }
}

/*
 * Validates user CCB content.  Also sets completion address and address types
 * for all addresses contained in CCB.
 */
static int dax_preprocess_usr_ccbs(struct dax_ctx *ctx, int idx, int nelem)
{
        int i;

        /*
         * The user is not allowed to specify real address types in
         * the CCB header.  This must be enforced by the kernel before
         * submitting the CCBs to HV.  The only allowed values for all
         * address fields are VA or IMM
         */
        for (i = 0; i < nelem; i++) {
                struct dax_ccb *ccbp = &ctx->ccb_buf[i];
                unsigned long ca_offset;

                if (ccbp->hdr.ccb_version > max_ccb_version)
                        return DAX_SUBMIT_ERR_CCB_INVAL;

                switch (ccbp->hdr.opcode) {
                case DAX_OP_SYNC_NOP:
                case DAX_OP_EXTRACT:
                case DAX_OP_SCAN_VALUE:
                case DAX_OP_SCAN_RANGE:
                case DAX_OP_TRANSLATE:
                case DAX_OP_SCAN_VALUE | DAX_OP_INVERT:
                case DAX_OP_SCAN_RANGE | DAX_OP_INVERT:
                case DAX_OP_TRANSLATE | DAX_OP_INVERT:
                case DAX_OP_SELECT:
                        break;
                default:
                        return DAX_SUBMIT_ERR_CCB_INVAL;
                }

                if (ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_VA &&
                    ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_NONE) {
                        dax_dbg("invalid out_addr_type in user CCB[%d]", i);
                        return DAX_SUBMIT_ERR_CCB_INVAL;
                }

                if (ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_VA &&
                    ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_NONE) {
                        dax_dbg("invalid pri_addr_type in user CCB[%d]", i);
                        return DAX_SUBMIT_ERR_CCB_INVAL;
                }

                if (ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_VA &&
                    ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_NONE) {
                        dax_dbg("invalid sec_addr_type in user CCB[%d]", i);
                        return DAX_SUBMIT_ERR_CCB_INVAL;
                }

                if (ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_VA &&
                    ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_NONE) {
                        dax_dbg("invalid table_addr_type in user CCB[%d]", i);
                        return DAX_SUBMIT_ERR_CCB_INVAL;
                }

                /* set completion (real) address and address type */
                ccbp->hdr.cca_addr_type = DAX_ADDR_TYPE_RA;
                ca_offset = (idx + i) * sizeof(struct dax_cca);
                ccbp->ca = (void *)ctx->ca_buf_ra + ca_offset;
                memset(&ctx->ca_buf[idx + i], 0, sizeof(struct dax_cca));

                dax_dbg("ccb[%d]=%p, ca_offset=0x%lx, compl RA=0x%llx",
                        i, ccbp, ca_offset, ctx->ca_buf_ra + ca_offset);

                /* skip over 2nd 64 bytes of long CCB */
                if (ccbp->hdr.longccb)
                        i++;
        }

        return DAX_SUBMIT_OK;
}

static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf,
                        size_t count, loff_t *ppos)
{
        unsigned long accepted_len, hv_rv;
        int i, idx, nccbs, naccepted;

        ctx->client = current;
        idx = *ppos;
        nccbs = count / sizeof(struct dax_ccb);

        if (ctx->owner != current) {
                dax_dbg("wrong thread");
                ctx->result.exec.status = DAX_SUBMIT_ERR_THR_INIT;
                return 0;
        }
        dax_dbg("args: ccb_buf_len=%ld, idx=%d", count, idx);

        /* for given index and length, verify ca_buf range exists */
        if (idx < 0 || idx > (DAX_CA_ELEMS - nccbs)) {
                ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL;
                return 0;
        }

        /*
         * Copy CCBs into kernel buffer to prevent modification by the
         * user in between validation and submission.
         */
        if (copy_from_user(ctx->ccb_buf, buf, count)) {
                dax_dbg("copyin of user CCB buffer failed");
                ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS;
                return 0;
        }

        /* check to see if ca_buf[idx] .. ca_buf[idx + nccbs] are available */
        for (i = idx; i < idx + nccbs; i++) {
                if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) {
                        dax_dbg("CA range not available, dequeue needed");
                        ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL;
                        return 0;
                }
        }
        dax_unlock_pages(ctx, idx, nccbs);

        ctx->result.exec.status = dax_preprocess_usr_ccbs(ctx, idx, nccbs);
        if (ctx->result.exec.status != DAX_SUBMIT_OK)
                return 0;

        ctx->result.exec.status = dax_lock_pages(ctx, idx, nccbs,
                                                 &ctx->result.exec.status_data);
        if (ctx->result.exec.status != DAX_SUBMIT_OK)
                return 0;

        if (dax_debug & DAX_DBG_FLG_BASIC)
                dax_prt_ccbs(ctx->ccb_buf, nccbs);

        hv_rv = sun4v_ccb_submit(ctx->ccb_buf_ra, count,
                                 HV_CCB_QUERY_CMD | HV_CCB_VA_SECONDARY, 0,
                                 &accepted_len, &ctx->result.exec.status_data);

        switch (hv_rv) {
        case HV_EOK:
                /*
                 * Hcall succeeded with no errors but the accepted
                 * length may be less than the requested length.  The
                 * only way the driver can resubmit the remainder is
                 * to wait for completion of the submitted CCBs since
                 * there is no way to guarantee the ordering semantics
                 * required by the client applications.  Therefore we
                 * let the user library deal with resubmissions.
                 */
                ctx->result.exec.status = DAX_SUBMIT_OK;
                break;
        case HV_EWOULDBLOCK:
                /*
                 * This is a transient HV API error. The user library
                 * can retry.
                 */
                dax_dbg("hcall returned HV_EWOULDBLOCK");
                ctx->result.exec.status = DAX_SUBMIT_ERR_WOULDBLOCK;
                break;
        case HV_ENOMAP:
                /*
                 * HV was unable to translate a VA. The VA it could
                 * not translate is returned in the status_data param.
                 */
                dax_dbg("hcall returned HV_ENOMAP");
                ctx->result.exec.status = DAX_SUBMIT_ERR_NOMAP;
                break;
        case HV_EINVAL:
                /*
                 * This is the result of an invalid user CCB as HV is
                 * validating some of the user CCB fields.  Pass this
                 * error back to the user. There is no supporting info
                 * to isolate the invalid field.
                 */
                dax_dbg("hcall returned HV_EINVAL");
                ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_INVAL;
                break;
        case HV_ENOACCESS:
                /*
                 * HV found a VA that did not have the appropriate
                 * permissions (such as the w bit). The VA in question
                 * is returned in status_data param.
                 */
                dax_dbg("hcall returned HV_ENOACCESS");
                ctx->result.exec.status = DAX_SUBMIT_ERR_NOACCESS;
                break;
        case HV_EUNAVAILABLE:
                /*
                 * The requested CCB operation could not be performed
                 * at this time. Return the specific unavailable code
                 * in the status_data field.
                 */
                dax_dbg("hcall returned HV_EUNAVAILABLE");
                ctx->result.exec.status = DAX_SUBMIT_ERR_UNAVAIL;
                break;
        default:
                ctx->result.exec.status = DAX_SUBMIT_ERR_INTERNAL;
                dax_dbg("unknown hcall return value (%ld)", hv_rv);
                break;
        }

        /* unlock pages associated with the unaccepted CCBs */
        naccepted = accepted_len / sizeof(struct dax_ccb);
        dax_unlock_pages(ctx, idx + naccepted, nccbs - naccepted);

        /* mark unaccepted CCBs as not completed */
        for (i = idx + naccepted; i < idx + nccbs; i++)
                ctx->ca_buf[i].status = CCA_STAT_COMPLETED;

        ctx->ccb_count += naccepted;
        ctx->fail_count += nccbs - naccepted;

        dax_dbg("hcall rv=%ld, accepted_len=%ld, status_data=0x%llx, ret status=%d",
                hv_rv, accepted_len, ctx->result.exec.status_data,
                ctx->result.exec.status);

        if (count == accepted_len)
                ctx->client = NULL; /* no read needed to complete protocol */
        return accepted_len;
}