LXR linux/drivers/misc/cxl/cxllib.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright 2017 IBM Corp.
   4 */
   5
   6#include <linux/hugetlb.h>
   7#include <linux/sched/mm.h>
   8#include <asm/pnv-pci.h>
   9#include <misc/cxllib.h>
  10
  11#include "cxl.h"
  12
  13#define CXL_INVALID_DRA                 ~0ull
  14#define CXL_DUMMY_READ_SIZE             128
  15#define CXL_DUMMY_READ_ALIGN            8
  16#define CXL_CAPI_WINDOW_START           0x2000000000000ull
  17#define CXL_CAPI_WINDOW_LOG_SIZE        48
  18#define CXL_XSL_CONFIG_CURRENT_VERSION  CXL_XSL_CONFIG_VERSION1
  19
  20
  21bool cxllib_slot_is_supported(struct pci_dev *dev, unsigned long flags)
  22{
  23        int rc;
  24        u32 phb_index;
  25        u64 chip_id, capp_unit_id;
  26
  27        /* No flags currently supported */
  28        if (flags)
  29                return false;
  30
  31        if (!cpu_has_feature(CPU_FTR_HVMODE))
  32                return false;
  33
  34        if (!cxl_is_power9())
  35                return false;
  36
  37        if (cxl_slot_is_switched(dev))
  38                return false;
  39
  40        /* on p9, some pci slots are not connected to a CAPP unit */
  41        rc = cxl_calc_capp_routing(dev, &chip_id, &phb_index, &capp_unit_id);
  42        if (rc)
  43                return false;
  44
  45        return true;
  46}
  47EXPORT_SYMBOL_GPL(cxllib_slot_is_supported);
  48
  49static DEFINE_MUTEX(dra_mutex);
  50static u64 dummy_read_addr = CXL_INVALID_DRA;
  51
  52static int allocate_dummy_read_buf(void)
  53{
  54        u64 buf, vaddr;
  55        size_t buf_size;
  56
  57        /*
  58         * Dummy read buffer is 128-byte long, aligned on a
  59         * 256-byte boundary and we need the physical address.
  60         */
  61        buf_size = CXL_DUMMY_READ_SIZE + (1ull << CXL_DUMMY_READ_ALIGN);
  62        buf = (u64) kzalloc(buf_size, GFP_KERNEL);
  63        if (!buf)
  64                return -ENOMEM;
  65
  66        vaddr = (buf + (1ull << CXL_DUMMY_READ_ALIGN) - 1) &
  67                                        (~0ull << CXL_DUMMY_READ_ALIGN);
  68
  69        WARN((vaddr + CXL_DUMMY_READ_SIZE) > (buf + buf_size),
  70                "Dummy read buffer alignment issue");
  71        dummy_read_addr = virt_to_phys((void *) vaddr);
  72        return 0;
  73}
  74
  75int cxllib_get_xsl_config(struct pci_dev *dev, struct cxllib_xsl_config *cfg)
  76{
  77        int rc;
  78        u32 phb_index;
  79        u64 chip_id, capp_unit_id;
  80
  81        if (!cpu_has_feature(CPU_FTR_HVMODE))
  82                return -EINVAL;
  83
  84        mutex_lock(&dra_mutex);
  85        if (dummy_read_addr == CXL_INVALID_DRA) {
  86                rc = allocate_dummy_read_buf();
  87                if (rc) {
  88                        mutex_unlock(&dra_mutex);
  89                        return rc;
  90                }
  91        }
  92        mutex_unlock(&dra_mutex);
  93
  94        rc = cxl_calc_capp_routing(dev, &chip_id, &phb_index, &capp_unit_id);
  95        if (rc)
  96                return rc;
  97
  98        rc = cxl_get_xsl9_dsnctl(dev, capp_unit_id, &cfg->dsnctl);
  99        if (rc)
 100                return rc;
 101
 102        cfg->version  = CXL_XSL_CONFIG_CURRENT_VERSION;
 103        cfg->log_bar_size = CXL_CAPI_WINDOW_LOG_SIZE;
 104        cfg->bar_addr = CXL_CAPI_WINDOW_START;
 105        cfg->dra = dummy_read_addr;
 106        return 0;
 107}
 108EXPORT_SYMBOL_GPL(cxllib_get_xsl_config);
 109
 110int cxllib_switch_phb_mode(struct pci_dev *dev, enum cxllib_mode mode,
 111                        unsigned long flags)
 112{
 113        int rc = 0;
 114
 115        if (!cpu_has_feature(CPU_FTR_HVMODE))
 116                return -EINVAL;
 117
 118        switch (mode) {
 119        case CXL_MODE_PCI:
 120                /*
 121                 * We currently don't support going back to PCI mode
 122                 * However, we'll turn the invalidations off, so that
 123                 * the firmware doesn't have to ack them and can do
 124                 * things like reset, etc.. with no worries.
 125                 * So always return EPERM (can't go back to PCI) or
 126                 * EBUSY if we couldn't even turn off snooping
 127                 */
 128                rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_OFF);
 129                if (rc)
 130                        rc = -EBUSY;
 131                else
 132                        rc = -EPERM;
 133                break;
 134        case CXL_MODE_CXL:
 135                /* DMA only supported on TVT1 for the time being */
 136                if (flags != CXL_MODE_DMA_TVT1)
 137                        return -EINVAL;
 138                rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_DMA_TVT1);
 139                if (rc)
 140                        return rc;
 141                rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_ON);
 142                break;
 143        default:
 144                rc = -EINVAL;
 145        }
 146        return rc;
 147}
 148EXPORT_SYMBOL_GPL(cxllib_switch_phb_mode);
 149
 150/*
 151 * When switching the PHB to capi mode, the TVT#1 entry for
 152 * the Partitionable Endpoint is set in bypass mode, like
 153 * in PCI mode.
 154 * Configure the device dma to use TVT#1, which is done
 155 * by calling dma_set_mask() with a mask large enough.
 156 */
 157int cxllib_set_device_dma(struct pci_dev *dev, unsigned long flags)
 158{
 159        int rc;
 160
 161        if (flags)
 162                return -EINVAL;
 163
 164        rc = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
 165        return rc;
 166}
 167EXPORT_SYMBOL_GPL(cxllib_set_device_dma);
 168
 169int cxllib_get_PE_attributes(struct task_struct *task,
 170                             unsigned long translation_mode,
 171                             struct cxllib_pe_attributes *attr)
 172{
 173        struct mm_struct *mm = NULL;
 174
 175        if (translation_mode != CXL_TRANSLATED_MODE &&
 176                translation_mode != CXL_REAL_MODE)
 177                return -EINVAL;
 178
 179        attr->sr = cxl_calculate_sr(false,
 180                                task == NULL,
 181                                translation_mode == CXL_REAL_MODE,
 182                                true);
 183        attr->lpid = mfspr(SPRN_LPID);
 184        if (task) {
 185                mm = get_task_mm(task);
 186                if (mm == NULL)
 187                        return -EINVAL;
 188                /*
 189                 * Caller is keeping a reference on mm_users for as long
 190                 * as XSL uses the memory context
 191                 */
 192                attr->pid = mm->context.id;
 193                mmput(mm);
 194                attr->tid = task->thread.tidr;
 195        } else {
 196                attr->pid = 0;
 197                attr->tid = 0;
 198        }
 199        return 0;
 200}
 201EXPORT_SYMBOL_GPL(cxllib_get_PE_attributes);
 202
 203static int get_vma_info(struct mm_struct *mm, u64 addr,
 204                        u64 *vma_start, u64 *vma_end,
 205                        unsigned long *page_size)
 206{
 207        struct vm_area_struct *vma = NULL;
 208        int rc = 0;
 209
 210        down_read(&mm->mmap_sem);
 211
 212        vma = find_vma(mm, addr);
 213        if (!vma) {
 214                rc = -EFAULT;
 215                goto out;
 216        }
 217        *page_size = vma_kernel_pagesize(vma);
 218        *vma_start = vma->vm_start;
 219        *vma_end = vma->vm_end;
 220out:
 221        up_read(&mm->mmap_sem);
 222        return rc;
 223}
 224
 225int cxllib_handle_fault(struct mm_struct *mm, u64 addr, u64 size, u64 flags)
 226{
 227        int rc;
 228        u64 dar, vma_start, vma_end;
 229        unsigned long page_size;
 230
 231        if (mm == NULL)
 232                return -EFAULT;
 233
 234        /*
 235         * The buffer we have to process can extend over several pages
 236         * and may also cover several VMAs.
 237         * We iterate over all the pages. The page size could vary
 238         * between VMAs.
 239         */
 240        rc = get_vma_info(mm, addr, &vma_start, &vma_end, &page_size);
 241        if (rc)
 242                return rc;
 243
 244        for (dar = (addr & ~(page_size - 1)); dar < (addr + size);
 245             dar += page_size) {
 246                if (dar < vma_start || dar >= vma_end) {
 247                        /*
 248                         * We don't hold the mm->mmap_sem semaphore
 249                         * while iterating, since the semaphore is
 250                         * required by one of the lower-level page
 251                         * fault processing functions and it could
 252                         * create a deadlock.
 253                         *
 254                         * It means the VMAs can be altered between 2
 255                         * loop iterations and we could theoretically
 256                         * miss a page (however unlikely). But that's
 257                         * not really a problem, as the driver will
 258                         * retry access, get another page fault on the
 259                         * missing page and call us again.
 260                         */
 261                        rc = get_vma_info(mm, dar, &vma_start, &vma_end,
 262                                        &page_size);
 263                        if (rc)
 264                                return rc;
 265                }
 266
 267                rc = cxl_handle_mm_fault(mm, flags, dar);
 268                if (rc)
 269                        return -EFAULT;
 270        }
 271        return 0;
 272}
 273EXPORT_SYMBOL_GPL(cxllib_handle_fault);
 274