LXR linux/arch/s390/pci/pci

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Access to PCI I/O memory from user space programs.
   4 *
   5 * Copyright IBM Corp. 2014
   6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
   7 */
   8#include <linux/kernel.h>
   9#include <linux/syscalls.h>
  10#include <linux/init.h>
  11#include <linux/mm.h>
  12#include <linux/errno.h>
  13#include <linux/pci.h>
  14#include <asm/pci_io.h>
  15#include <asm/pci_debug.h>
  16
  17static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
  18{
  19        struct {
  20                u64 offset;
  21                u8 cc;
  22                u8 status;
  23        } data = {offset, cc, status};
  24
  25        zpci_err_hex(&data, sizeof(data));
  26}
  27
  28static inline int __pcistb_mio_inuser(
  29                void __iomem *ioaddr, const void __user *src,
  30                u64 len, u8 *status)
  31{
  32        int cc = -ENXIO;
  33
  34        asm volatile (
  35                "       sacf 256\n"
  36                "0:     .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
  37                "1:     ipm     %[cc]\n"
  38                "       srl     %[cc],28\n"
  39                "2:     sacf 768\n"
  40                EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
  41                : [cc] "+d" (cc), [len] "+d" (len)
  42                : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
  43                : "cc", "memory");
  44        *status = len >> 24 & 0xff;
  45        return cc;
  46}
  47
  48static inline int __pcistg_mio_inuser(
  49                void __iomem *ioaddr, const void __user *src,
  50                u64 ulen, u8 *status)
  51{
  52        union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
  53        int cc = -ENXIO;
  54        u64 val = 0;
  55        u64 cnt = ulen;
  56        u8 tmp;
  57
  58        /*
  59         * copy 0 < @len <= 8 bytes from @src into the right most bytes of
  60         * a register, then store it to PCI at @ioaddr while in secondary
  61         * address space. pcistg then uses the user mappings.
  62         */
  63        asm volatile (
  64                "       sacf    256\n"
  65                "0:     llgc    %[tmp],0(%[src])\n"
  66                "       sllg    %[val],%[val],8\n"
  67                "       aghi    %[src],1\n"
  68                "       ogr     %[val],%[tmp]\n"
  69                "       brctg   %[cnt],0b\n"
  70                "1:     .insn   rre,0xb9d40000,%[val],%[ioaddr_len]\n"
  71                "2:     ipm     %[cc]\n"
  72                "       srl     %[cc],28\n"
  73                "3:     sacf    768\n"
  74                EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
  75                :
  76                [src] "+a" (src), [cnt] "+d" (cnt),
  77                [val] "+d" (val), [tmp] "=d" (tmp),
  78                [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair)
  79                :: "cc", "memory");
  80        *status = ioaddr_len.odd >> 24 & 0xff;
  81
  82        /* did we read everything from user memory? */
  83        if (!cc && cnt != 0)
  84                cc = -EFAULT;
  85
  86        return cc;
  87}
  88
  89static inline int __memcpy_toio_inuser(void __iomem *dst,
  90                                   const void __user *src, size_t n)
  91{
  92        int size, rc = 0;
  93        u8 status = 0;
  94
  95        if (!src)
  96                return -EINVAL;
  97
  98        while (n > 0) {
  99                size = zpci_get_max_write_size((u64 __force) dst,
 100                                               (u64 __force) src, n,
 101                                               ZPCI_MAX_WRITE_SIZE);
 102                if (size > 8) /* main path */
 103                        rc = __pcistb_mio_inuser(dst, src, size, &status);
 104                else
 105                        rc = __pcistg_mio_inuser(dst, src, size, &status);
 106                if (rc)
 107                        break;
 108                src += size;
 109                dst += size;
 110                n -= size;
 111        }
 112        if (rc)
 113                zpci_err_mmio(rc, status, (__force u64) dst);
 114        return rc;
 115}
 116
 117SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
 118                const void __user *, user_buffer, size_t, length)
 119{
 120        u8 local_buf[64];
 121        void __iomem *io_addr;
 122        void *buf;
 123        struct vm_area_struct *vma;
 124        pte_t *ptep;
 125        spinlock_t *ptl;
 126        long ret;
 127
 128        if (!zpci_is_enabled())
 129                return -ENODEV;
 130
 131        if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
 132                return -EINVAL;
 133
 134        /*
 135         * We only support write access to MIO capable devices if we are on
 136         * a MIO enabled system. Otherwise we would have to check for every
 137         * address if it is a special ZPCI_ADDR and would have to do
 138         * a pfn lookup which we don't need for MIO capable devices.  Currently
 139         * ISM devices are the only devices without MIO support and there is no
 140         * known need for accessing these from userspace.
 141         */
 142        if (static_branch_likely(&have_mio)) {
 143                ret = __memcpy_toio_inuser((void  __iomem *) mmio_addr,
 144                                        user_buffer,
 145                                        length);
 146                return ret;
 147        }
 148
 149        if (length > 64) {
 150                buf = kmalloc(length, GFP_KERNEL);
 151                if (!buf)
 152                        return -ENOMEM;
 153        } else
 154                buf = local_buf;
 155
 156        ret = -EFAULT;
 157        if (copy_from_user(buf, user_buffer, length))
 158                goto out_free;
 159
 160        mmap_read_lock(current->mm);
 161        ret = -EINVAL;
 162        vma = find_vma(current->mm, mmio_addr);
 163        if (!vma)
 164                goto out_unlock_mmap;
 165        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
 166                goto out_unlock_mmap;
 167        ret = -EACCES;
 168        if (!(vma->vm_flags & VM_WRITE))
 169                goto out_unlock_mmap;
 170
 171        ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
 172        if (ret)
 173                goto out_unlock_mmap;
 174
 175        io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
 176                        (mmio_addr & ~PAGE_MASK));
 177
 178        if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
 179                goto out_unlock_pt;
 180
 181        ret = zpci_memcpy_toio(io_addr, buf, length);
 182out_unlock_pt:
 183        pte_unmap_unlock(ptep, ptl);
 184out_unlock_mmap:
 185        mmap_read_unlock(current->mm);
 186out_free:
 187        if (buf != local_buf)
 188                kfree(buf);
 189        return ret;
 190}
 191
 192static inline int __pcilg_mio_inuser(
 193                void __user *dst, const void __iomem *ioaddr,
 194                u64 ulen, u8 *status)
 195{
 196        union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
 197        u64 cnt = ulen;
 198        int shift = ulen * 8;
 199        int cc = -ENXIO;
 200        u64 val, tmp;
 201
 202        /*
 203         * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
 204         * user space) into a register using pcilg then store these bytes at
 205         * user address @dst
 206         */
 207        asm volatile (
 208                "       sacf    256\n"
 209                "0:     .insn   rre,0xb9d60000,%[val],%[ioaddr_len]\n"
 210                "1:     ipm     %[cc]\n"
 211                "       srl     %[cc],28\n"
 212                "       ltr     %[cc],%[cc]\n"
 213                "       jne     4f\n"
 214                "2:     ahi     %[shift],-8\n"
 215                "       srlg    %[tmp],%[val],0(%[shift])\n"
 216                "3:     stc     %[tmp],0(%[dst])\n"
 217                "       aghi    %[dst],1\n"
 218                "       brctg   %[cnt],2b\n"
 219                "4:     sacf    768\n"
 220                EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b)
 221                :
 222                [ioaddr_len] "+&d" (ioaddr_len.pair),
 223                [cc] "+d" (cc), [val] "=d" (val),
 224                [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
 225                [shift] "+d" (shift)
 226                :: "cc", "memory");
 227
 228        /* did we write everything to the user space buffer? */
 229        if (!cc && cnt != 0)
 230                cc = -EFAULT;
 231
 232        *status = ioaddr_len.odd >> 24 & 0xff;
 233        return cc;
 234}
 235
 236static inline int __memcpy_fromio_inuser(void __user *dst,
 237                                     const void __iomem *src,
 238                                     unsigned long n)
 239{
 240        int size, rc = 0;
 241        u8 status;
 242
 243        while (n > 0) {
 244                size = zpci_get_max_write_size((u64 __force) src,
 245                                               (u64 __force) dst, n,
 246                                               ZPCI_MAX_READ_SIZE);
 247                rc = __pcilg_mio_inuser(dst, src, size, &status);
 248                if (rc)
 249                        break;
 250                src += size;
 251                dst += size;
 252                n -= size;
 253        }
 254        if (rc)
 255                zpci_err_mmio(rc, status, (__force u64) dst);
 256        return rc;
 257}
 258
 259SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
 260                void __user *, user_buffer, size_t, length)
 261{
 262        u8 local_buf[64];
 263        void __iomem *io_addr;
 264        void *buf;
 265        struct vm_area_struct *vma;
 266        pte_t *ptep;
 267        spinlock_t *ptl;
 268        long ret;
 269
 270        if (!zpci_is_enabled())
 271                return -ENODEV;
 272
 273        if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
 274                return -EINVAL;
 275
 276        /*
 277         * We only support read access to MIO capable devices if we are on
 278         * a MIO enabled system. Otherwise we would have to check for every
 279         * address if it is a special ZPCI_ADDR and would have to do
 280         * a pfn lookup which we don't need for MIO capable devices.  Currently
 281         * ISM devices are the only devices without MIO support and there is no
 282         * known need for accessing these from userspace.
 283         */
 284        if (static_branch_likely(&have_mio)) {
 285                ret = __memcpy_fromio_inuser(
 286                                user_buffer, (const void __iomem *)mmio_addr,
 287                                length);
 288                return ret;
 289        }
 290
 291        if (length > 64) {
 292                buf = kmalloc(length, GFP_KERNEL);
 293                if (!buf)
 294                        return -ENOMEM;
 295        } else {
 296                buf = local_buf;
 297        }
 298
 299        mmap_read_lock(current->mm);
 300        ret = -EINVAL;
 301        vma = find_vma(current->mm, mmio_addr);
 302        if (!vma)
 303                goto out_unlock_mmap;
 304        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
 305                goto out_unlock_mmap;
 306        ret = -EACCES;
 307        if (!(vma->vm_flags & VM_WRITE))
 308                goto out_unlock_mmap;
 309
 310        ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
 311        if (ret)
 312                goto out_unlock_mmap;
 313
 314        io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
 315                        (mmio_addr & ~PAGE_MASK));
 316
 317        if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
 318                ret = -EFAULT;
 319                goto out_unlock_pt;
 320        }
 321        ret = zpci_memcpy_fromio(buf, io_addr, length);
 322
 323out_unlock_pt:
 324        pte_unmap_unlock(ptep, ptl);
 325out_unlock_mmap:
 326        mmap_read_unlock(current->mm);
 327
 328        if (!ret && copy_to_user(user_buffer, buf, length))
 329                ret = -EFAULT;
 330
 331        if (buf != local_buf)
 332                kfree(buf);
 333        return ret;
 334}
 335