LXR linux/arch/s390/pci/pci

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Access to PCI I/O memory from user space programs.
   4 *
   5 * Copyright IBM Corp. 2014
   6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
   7 */
   8#include <linux/kernel.h>
   9#include <linux/syscalls.h>
  10#include <linux/init.h>
  11#include <linux/mm.h>
  12#include <linux/errno.h>
  13#include <linux/pci.h>
  14#include <asm/pci_io.h>
  15#include <asm/pci_debug.h>
  16
  17static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
  18{
  19        struct {
  20                u64 offset;
  21                u8 cc;
  22                u8 status;
  23        } data = {offset, cc, status};
  24
  25        zpci_err_hex(&data, sizeof(data));
  26}
  27
  28static inline int __pcistb_mio_inuser(
  29                void __iomem *ioaddr, const void __user *src,
  30                u64 len, u8 *status)
  31{
  32        int cc = -ENXIO;
  33
  34        asm volatile (
  35                "       sacf 256\n"
  36                "0:     .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
  37                "1:     ipm     %[cc]\n"
  38                "       srl     %[cc],28\n"
  39                "2:     sacf 768\n"
  40                EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
  41                : [cc] "+d" (cc), [len] "+d" (len)
  42                : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
  43                : "cc", "memory");
  44        *status = len >> 24 & 0xff;
  45        return cc;
  46}
  47
  48static inline int __pcistg_mio_inuser(
  49                void __iomem *ioaddr, const void __user *src,
  50                u64 ulen, u8 *status)
  51{
  52        register u64 addr asm("2") = (u64 __force) ioaddr;
  53        register u64 len asm("3") = ulen;
  54        int cc = -ENXIO;
  55        u64 val = 0;
  56        u64 cnt = ulen;
  57        u8 tmp;
  58
  59        /*
  60         * copy 0 < @len <= 8 bytes from @src into the right most bytes of
  61         * a register, then store it to PCI at @ioaddr while in secondary
  62         * address space. pcistg then uses the user mappings.
  63         */
  64        asm volatile (
  65                "       sacf    256\n"
  66                "0:     llgc    %[tmp],0(%[src])\n"
  67                "       sllg    %[val],%[val],8\n"
  68                "       aghi    %[src],1\n"
  69                "       ogr     %[val],%[tmp]\n"
  70                "       brctg   %[cnt],0b\n"
  71                "1:     .insn   rre,0xb9d40000,%[val],%[ioaddr]\n"
  72                "2:     ipm     %[cc]\n"
  73                "       srl     %[cc],28\n"
  74                "3:     sacf    768\n"
  75                EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
  76                :
  77                [src] "+a" (src), [cnt] "+d" (cnt),
  78                [val] "+d" (val), [tmp] "=d" (tmp),
  79                [len] "+d" (len), [cc] "+d" (cc),
  80                [ioaddr] "+a" (addr)
  81                :: "cc", "memory");
  82        *status = len >> 24 & 0xff;
  83
  84        /* did we read everything from user memory? */
  85        if (!cc && cnt != 0)
  86                cc = -EFAULT;
  87
  88        return cc;
  89}
  90
  91static inline int __memcpy_toio_inuser(void __iomem *dst,
  92                                   const void __user *src, size_t n)
  93{
  94        int size, rc = 0;
  95        u8 status = 0;
  96        mm_segment_t old_fs;
  97
  98        if (!src)
  99                return -EINVAL;
 100
 101        old_fs = enable_sacf_uaccess();
 102        while (n > 0) {
 103                size = zpci_get_max_write_size((u64 __force) dst,
 104                                               (u64 __force) src, n,
 105                                               ZPCI_MAX_WRITE_SIZE);
 106                if (size > 8) /* main path */
 107                        rc = __pcistb_mio_inuser(dst, src, size, &status);
 108                else
 109                        rc = __pcistg_mio_inuser(dst, src, size, &status);
 110                if (rc)
 111                        break;
 112                src += size;
 113                dst += size;
 114                n -= size;
 115        }
 116        disable_sacf_uaccess(old_fs);
 117        if (rc)
 118                zpci_err_mmio(rc, status, (__force u64) dst);
 119        return rc;
 120}
 121
 122static long get_pfn(unsigned long user_addr, unsigned long access,
 123                    unsigned long *pfn)
 124{
 125        struct vm_area_struct *vma;
 126        long ret;
 127
 128        mmap_read_lock(current->mm);
 129        ret = -EINVAL;
 130        vma = find_vma(current->mm, user_addr);
 131        if (!vma)
 132                goto out;
 133        ret = -EACCES;
 134        if (!(vma->vm_flags & access))
 135                goto out;
 136        ret = follow_pfn(vma, user_addr, pfn);
 137out:
 138        mmap_read_unlock(current->mm);
 139        return ret;
 140}
 141
 142SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
 143                const void __user *, user_buffer, size_t, length)
 144{
 145        u8 local_buf[64];
 146        void __iomem *io_addr;
 147        void *buf;
 148        unsigned long pfn;
 149        long ret;
 150
 151        if (!zpci_is_enabled())
 152                return -ENODEV;
 153
 154        if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
 155                return -EINVAL;
 156
 157        /*
 158         * We only support write access to MIO capable devices if we are on
 159         * a MIO enabled system. Otherwise we would have to check for every
 160         * address if it is a special ZPCI_ADDR and would have to do
 161         * a get_pfn() which we don't need for MIO capable devices.  Currently
 162         * ISM devices are the only devices without MIO support and there is no
 163         * known need for accessing these from userspace.
 164         */
 165        if (static_branch_likely(&have_mio)) {
 166                ret = __memcpy_toio_inuser((void  __iomem *) mmio_addr,
 167                                        user_buffer,
 168                                        length);
 169                return ret;
 170        }
 171
 172        if (length > 64) {
 173                buf = kmalloc(length, GFP_KERNEL);
 174                if (!buf)
 175                        return -ENOMEM;
 176        } else
 177                buf = local_buf;
 178
 179        ret = get_pfn(mmio_addr, VM_WRITE, &pfn);
 180        if (ret)
 181                goto out;
 182        io_addr = (void __iomem *)((pfn << PAGE_SHIFT) |
 183                        (mmio_addr & ~PAGE_MASK));
 184
 185        ret = -EFAULT;
 186        if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
 187                goto out;
 188
 189        if (copy_from_user(buf, user_buffer, length))
 190                goto out;
 191
 192        ret = zpci_memcpy_toio(io_addr, buf, length);
 193out:
 194        if (buf != local_buf)
 195                kfree(buf);
 196        return ret;
 197}
 198
 199static inline int __pcilg_mio_inuser(
 200                void __user *dst, const void __iomem *ioaddr,
 201                u64 ulen, u8 *status)
 202{
 203        register u64 addr asm("2") = (u64 __force) ioaddr;
 204        register u64 len asm("3") = ulen;
 205        u64 cnt = ulen;
 206        int shift = ulen * 8;
 207        int cc = -ENXIO;
 208        u64 val, tmp;
 209
 210        /*
 211         * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
 212         * user space) into a register using pcilg then store these bytes at
 213         * user address @dst
 214         */
 215        asm volatile (
 216                "       sacf    256\n"
 217                "0:     .insn   rre,0xb9d60000,%[val],%[ioaddr]\n"
 218                "1:     ipm     %[cc]\n"
 219                "       srl     %[cc],28\n"
 220                "       ltr     %[cc],%[cc]\n"
 221                "       jne     4f\n"
 222                "2:     ahi     %[shift],-8\n"
 223                "       srlg    %[tmp],%[val],0(%[shift])\n"
 224                "3:     stc     %[tmp],0(%[dst])\n"
 225                "       aghi    %[dst],1\n"
 226                "       brctg   %[cnt],2b\n"
 227                "4:     sacf    768\n"
 228                EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b)
 229                :
 230                [cc] "+d" (cc), [val] "=d" (val), [len] "+d" (len),
 231                [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
 232                [shift] "+d" (shift)
 233                :
 234                [ioaddr] "a" (addr)
 235                : "cc", "memory");
 236
 237        /* did we write everything to the user space buffer? */
 238        if (!cc && cnt != 0)
 239                cc = -EFAULT;
 240
 241        *status = len >> 24 & 0xff;
 242        return cc;
 243}
 244
 245static inline int __memcpy_fromio_inuser(void __user *dst,
 246                                     const void __iomem *src,
 247                                     unsigned long n)
 248{
 249        int size, rc = 0;
 250        u8 status;
 251        mm_segment_t old_fs;
 252
 253        old_fs = enable_sacf_uaccess();
 254        while (n > 0) {
 255                size = zpci_get_max_write_size((u64 __force) src,
 256                                               (u64 __force) dst, n,
 257                                               ZPCI_MAX_READ_SIZE);
 258                rc = __pcilg_mio_inuser(dst, src, size, &status);
 259                if (rc)
 260                        break;
 261                src += size;
 262                dst += size;
 263                n -= size;
 264        }
 265        disable_sacf_uaccess(old_fs);
 266        if (rc)
 267                zpci_err_mmio(rc, status, (__force u64) dst);
 268        return rc;
 269}
 270
 271SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
 272                void __user *, user_buffer, size_t, length)
 273{
 274        u8 local_buf[64];
 275        void __iomem *io_addr;
 276        void *buf;
 277        unsigned long pfn;
 278        long ret;
 279
 280        if (!zpci_is_enabled())
 281                return -ENODEV;
 282
 283        if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
 284                return -EINVAL;
 285
 286        /*
 287         * We only support read access to MIO capable devices if we are on
 288         * a MIO enabled system. Otherwise we would have to check for every
 289         * address if it is a special ZPCI_ADDR and would have to do
 290         * a get_pfn() which we don't need for MIO capable devices.  Currently
 291         * ISM devices are the only devices without MIO support and there is no
 292         * known need for accessing these from userspace.
 293         */
 294        if (static_branch_likely(&have_mio)) {
 295                ret = __memcpy_fromio_inuser(
 296                                user_buffer, (const void __iomem *)mmio_addr,
 297                                length);
 298                return ret;
 299        }
 300
 301        if (length > 64) {
 302                buf = kmalloc(length, GFP_KERNEL);
 303                if (!buf)
 304                        return -ENOMEM;
 305        } else {
 306                buf = local_buf;
 307        }
 308
 309        ret = get_pfn(mmio_addr, VM_READ, &pfn);
 310        if (ret)
 311                goto out;
 312        io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK));
 313
 314        if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
 315                ret = -EFAULT;
 316                goto out;
 317        }
 318        ret = zpci_memcpy_fromio(buf, io_addr, length);
 319        if (ret)
 320                goto out;
 321        if (copy_to_user(user_buffer, buf, length))
 322                ret = -EFAULT;
 323
 324out:
 325        if (buf != local_buf)
 326                kfree(buf);
 327        return ret;
 328}
 329