linux/drivers/misc/ocxl/link.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0+
   2// Copyright 2017 IBM Corp.
   3#include <linux/sched/mm.h>
   4#include <linux/mutex.h>
   5#include <linux/mm.h>
   6#include <linux/mm_types.h>
   7#include <linux/mmu_context.h>
   8#include <linux/mmu_notifier.h>
   9#include <asm/copro.h>
  10#include <asm/pnv-ocxl.h>
  11#include <misc/ocxl.h>
  12#include "ocxl_internal.h"
  13#include "trace.h"
  14
  15
  16#define SPA_PASID_BITS          15
  17#define SPA_PASID_MAX           ((1 << SPA_PASID_BITS) - 1)
  18#define SPA_PE_MASK             SPA_PASID_MAX
  19#define SPA_SPA_SIZE_LOG        22 /* Each SPA is 4 Mb */
  20
  21#define SPA_CFG_SF              (1ull << (63-0))
  22#define SPA_CFG_TA              (1ull << (63-1))
  23#define SPA_CFG_HV              (1ull << (63-3))
  24#define SPA_CFG_UV              (1ull << (63-4))
  25#define SPA_CFG_XLAT_hpt        (0ull << (63-6)) /* Hashed page table (HPT) mode */
  26#define SPA_CFG_XLAT_roh        (2ull << (63-6)) /* Radix on HPT mode */
  27#define SPA_CFG_XLAT_ror        (3ull << (63-6)) /* Radix on Radix mode */
  28#define SPA_CFG_PR              (1ull << (63-49))
  29#define SPA_CFG_TC              (1ull << (63-54))
  30#define SPA_CFG_DR              (1ull << (63-59))
  31
  32#define SPA_XSL_TF              (1ull << (63-3))  /* Translation fault */
  33#define SPA_XSL_S               (1ull << (63-38)) /* Store operation */
  34
  35#define SPA_PE_VALID            0x80000000
  36
  37struct ocxl_link;
  38
  39struct pe_data {
  40        struct mm_struct *mm;
  41        /* callback to trigger when a translation fault occurs */
  42        void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr);
  43        /* opaque pointer to be passed to the above callback */
  44        void *xsl_err_data;
  45        struct rcu_head rcu;
  46        struct ocxl_link *link;
  47        struct mmu_notifier mmu_notifier;
  48};
  49
  50struct spa {
  51        struct ocxl_process_element *spa_mem;
  52        int spa_order;
  53        struct mutex spa_lock;
  54        struct radix_tree_root pe_tree; /* Maps PE handles to pe_data */
  55        char *irq_name;
  56        int virq;
  57        void __iomem *reg_dsisr;
  58        void __iomem *reg_dar;
  59        void __iomem *reg_tfc;
  60        void __iomem *reg_pe_handle;
  61        /*
  62         * The following field are used by the memory fault
  63         * interrupt handler. We can only have one interrupt at a
  64         * time. The NPU won't raise another interrupt until the
  65         * previous one has been ack'd by writing to the TFC register
  66         */
  67        struct xsl_fault {
  68                struct work_struct fault_work;
  69                u64 pe;
  70                u64 dsisr;
  71                u64 dar;
  72                struct pe_data pe_data;
  73        } xsl_fault;
  74};
  75
  76/*
  77 * A opencapi link can be used be by several PCI functions. We have
  78 * one link per device slot.
  79 *
  80 * A linked list of opencapi links should suffice, as there's a
  81 * limited number of opencapi slots on a system and lookup is only
  82 * done when the device is probed
  83 */
  84struct ocxl_link {
  85        struct list_head list;
  86        struct kref ref;
  87        int domain;
  88        int bus;
  89        int dev;
  90        void __iomem *arva;     /* ATSD register virtual address */
  91        spinlock_t atsd_lock;   /* to serialize shootdowns */
  92        atomic_t irq_available;
  93        struct spa *spa;
  94        void *platform_data;
  95};
  96static struct list_head links_list = LIST_HEAD_INIT(links_list);
  97static DEFINE_MUTEX(links_list_lock);
  98
  99enum xsl_response {
 100        CONTINUE,
 101        ADDRESS_ERROR,
 102        RESTART,
 103};
 104
 105
 106static void read_irq(struct spa *spa, u64 *dsisr, u64 *dar, u64 *pe)
 107{
 108        u64 reg;
 109
 110        *dsisr = in_be64(spa->reg_dsisr);
 111        *dar = in_be64(spa->reg_dar);
 112        reg = in_be64(spa->reg_pe_handle);
 113        *pe = reg & SPA_PE_MASK;
 114}
 115
 116static void ack_irq(struct spa *spa, enum xsl_response r)
 117{
 118        u64 reg = 0;
 119
 120        /* continue is not supported */
 121        if (r == RESTART)
 122                reg = PPC_BIT(31);
 123        else if (r == ADDRESS_ERROR)
 124                reg = PPC_BIT(30);
 125        else
 126                WARN(1, "Invalid irq response %d\n", r);
 127
 128        if (reg) {
 129                trace_ocxl_fault_ack(spa->spa_mem, spa->xsl_fault.pe,
 130                                spa->xsl_fault.dsisr, spa->xsl_fault.dar, reg);
 131                out_be64(spa->reg_tfc, reg);
 132        }
 133}
 134
 135static void xsl_fault_handler_bh(struct work_struct *fault_work)
 136{
 137        vm_fault_t flt = 0;
 138        unsigned long access, flags, inv_flags = 0;
 139        enum xsl_response r;
 140        struct xsl_fault *fault = container_of(fault_work, struct xsl_fault,
 141                                        fault_work);
 142        struct spa *spa = container_of(fault, struct spa, xsl_fault);
 143
 144        int rc;
 145
 146        /*
 147         * We must release a reference on mm_users whenever exiting this
 148         * function (taken in the memory fault interrupt handler)
 149         */
 150        rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr,
 151                                &flt);
 152        if (rc) {
 153                pr_debug("copro_handle_mm_fault failed: %d\n", rc);
 154                if (fault->pe_data.xsl_err_cb) {
 155                        fault->pe_data.xsl_err_cb(
 156                                fault->pe_data.xsl_err_data,
 157                                fault->dar, fault->dsisr);
 158                }
 159                r = ADDRESS_ERROR;
 160                goto ack;
 161        }
 162
 163        if (!radix_enabled()) {
 164                /*
 165                 * update_mmu_cache() will not have loaded the hash
 166                 * since current->trap is not a 0x400 or 0x300, so
 167                 * just call hash_page_mm() here.
 168                 */
 169                access = _PAGE_PRESENT | _PAGE_READ;
 170                if (fault->dsisr & SPA_XSL_S)
 171                        access |= _PAGE_WRITE;
 172
 173                if (REGION_ID(fault->dar) != USER_REGION_ID)
 174                        access |= _PAGE_PRIVILEGED;
 175
 176                local_irq_save(flags);
 177                hash_page_mm(fault->pe_data.mm, fault->dar, access, 0x300,
 178                        inv_flags);
 179                local_irq_restore(flags);
 180        }
 181        r = RESTART;
 182ack:
 183        mmput(fault->pe_data.mm);
 184        ack_irq(spa, r);
 185}
 186
 187static irqreturn_t xsl_fault_handler(int irq, void *data)
 188{
 189        struct ocxl_link *link = (struct ocxl_link *) data;
 190        struct spa *spa = link->spa;
 191        u64 dsisr, dar, pe_handle;
 192        struct pe_data *pe_data;
 193        struct ocxl_process_element *pe;
 194        int pid;
 195        bool schedule = false;
 196
 197        read_irq(spa, &dsisr, &dar, &pe_handle);
 198        trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1);
 199
 200        WARN_ON(pe_handle > SPA_PE_MASK);
 201        pe = spa->spa_mem + pe_handle;
 202        pid = be32_to_cpu(pe->pid);
 203        /* We could be reading all null values here if the PE is being
 204         * removed while an interrupt kicks in. It's not supposed to
 205         * happen if the driver notified the AFU to terminate the
 206         * PASID, and the AFU waited for pending operations before
 207         * acknowledging. But even if it happens, we won't find a
 208         * memory context below and fail silently, so it should be ok.
 209         */
 210        if (!(dsisr & SPA_XSL_TF)) {
 211                WARN(1, "Invalid xsl interrupt fault register %#llx\n", dsisr);
 212                ack_irq(spa, ADDRESS_ERROR);
 213                return IRQ_HANDLED;
 214        }
 215
 216        rcu_read_lock();
 217        pe_data = radix_tree_lookup(&spa->pe_tree, pe_handle);
 218        if (!pe_data) {
 219                /*
 220                 * Could only happen if the driver didn't notify the
 221                 * AFU about PASID termination before removing the PE,
 222                 * or the AFU didn't wait for all memory access to
 223                 * have completed.
 224                 *
 225                 * Either way, we fail early, but we shouldn't log an
 226                 * error message, as it is a valid (if unexpected)
 227                 * scenario
 228                 */
 229                rcu_read_unlock();
 230                pr_debug("Unknown mm context for xsl interrupt\n");
 231                ack_irq(spa, ADDRESS_ERROR);
 232                return IRQ_HANDLED;
 233        }
 234
 235        if (!pe_data->mm) {
 236                /*
 237                 * translation fault from a kernel context - an OpenCAPI
 238                 * device tried to access a bad kernel address
 239                 */
 240                rcu_read_unlock();
 241                pr_warn("Unresolved OpenCAPI xsl fault in kernel context\n");
 242                ack_irq(spa, ADDRESS_ERROR);
 243                return IRQ_HANDLED;
 244        }
 245        WARN_ON(pe_data->mm->context.id != pid);
 246
 247        if (mmget_not_zero(pe_data->mm)) {
 248                        spa->xsl_fault.pe = pe_handle;
 249                        spa->xsl_fault.dar = dar;
 250                        spa->xsl_fault.dsisr = dsisr;
 251                        spa->xsl_fault.pe_data = *pe_data;
 252                        schedule = true;
 253                        /* mm_users count released by bottom half */
 254        }
 255        rcu_read_unlock();
 256        if (schedule)
 257                schedule_work(&spa->xsl_fault.fault_work);
 258        else
 259                ack_irq(spa, ADDRESS_ERROR);
 260        return IRQ_HANDLED;
 261}
 262
 263static void unmap_irq_registers(struct spa *spa)
 264{
 265        pnv_ocxl_unmap_xsl_regs(spa->reg_dsisr, spa->reg_dar, spa->reg_tfc,
 266                                spa->reg_pe_handle);
 267}
 268
 269static int map_irq_registers(struct pci_dev *dev, struct spa *spa)
 270{
 271        return pnv_ocxl_map_xsl_regs(dev, &spa->reg_dsisr, &spa->reg_dar,
 272                                &spa->reg_tfc, &spa->reg_pe_handle);
 273}
 274
 275static int setup_xsl_irq(struct pci_dev *dev, struct ocxl_link *link)
 276{
 277        struct spa *spa = link->spa;
 278        int rc;
 279        int hwirq;
 280
 281        rc = pnv_ocxl_get_xsl_irq(dev, &hwirq);
 282        if (rc)
 283                return rc;
 284
 285        rc = map_irq_registers(dev, spa);
 286        if (rc)
 287                return rc;
 288
 289        spa->irq_name = kasprintf(GFP_KERNEL, "ocxl-xsl-%x-%x-%x",
 290                                link->domain, link->bus, link->dev);
 291        if (!spa->irq_name) {
 292                dev_err(&dev->dev, "Can't allocate name for xsl interrupt\n");
 293                rc = -ENOMEM;
 294                goto err_xsl;
 295        }
 296        /*
 297         * At some point, we'll need to look into allowing a higher
 298         * number of interrupts. Could we have an IRQ domain per link?
 299         */
 300        spa->virq = irq_create_mapping(NULL, hwirq);
 301        if (!spa->virq) {
 302                dev_err(&dev->dev,
 303                        "irq_create_mapping failed for translation interrupt\n");
 304                rc = -EINVAL;
 305                goto err_name;
 306        }
 307
 308        dev_dbg(&dev->dev, "hwirq %d mapped to virq %d\n", hwirq, spa->virq);
 309
 310        rc = request_irq(spa->virq, xsl_fault_handler, 0, spa->irq_name,
 311                        link);
 312        if (rc) {
 313                dev_err(&dev->dev,
 314                        "request_irq failed for translation interrupt: %d\n",
 315                        rc);
 316                rc = -EINVAL;
 317                goto err_mapping;
 318        }
 319        return 0;
 320
 321err_mapping:
 322        irq_dispose_mapping(spa->virq);
 323err_name:
 324        kfree(spa->irq_name);
 325err_xsl:
 326        unmap_irq_registers(spa);
 327        return rc;
 328}
 329
 330static void release_xsl_irq(struct ocxl_link *link)
 331{
 332        struct spa *spa = link->spa;
 333
 334        if (spa->virq) {
 335                free_irq(spa->virq, link);
 336                irq_dispose_mapping(spa->virq);
 337        }
 338        kfree(spa->irq_name);
 339        unmap_irq_registers(spa);
 340}
 341
 342static int alloc_spa(struct pci_dev *dev, struct ocxl_link *link)
 343{
 344        struct spa *spa;
 345
 346        spa = kzalloc(sizeof(struct spa), GFP_KERNEL);
 347        if (!spa)
 348                return -ENOMEM;
 349
 350        mutex_init(&spa->spa_lock);
 351        INIT_RADIX_TREE(&spa->pe_tree, GFP_KERNEL);
 352        INIT_WORK(&spa->xsl_fault.fault_work, xsl_fault_handler_bh);
 353
 354        spa->spa_order = SPA_SPA_SIZE_LOG - PAGE_SHIFT;
 355        spa->spa_mem = (struct ocxl_process_element *)
 356                __get_free_pages(GFP_KERNEL | __GFP_ZERO, spa->spa_order);
 357        if (!spa->spa_mem) {
 358                dev_err(&dev->dev, "Can't allocate Shared Process Area\n");
 359                kfree(spa);
 360                return -ENOMEM;
 361        }
 362        pr_debug("Allocated SPA for %x:%x:%x at %p\n", link->domain, link->bus,
 363                link->dev, spa->spa_mem);
 364
 365        link->spa = spa;
 366        return 0;
 367}
 368
 369static void free_spa(struct ocxl_link *link)
 370{
 371        struct spa *spa = link->spa;
 372
 373        pr_debug("Freeing SPA for %x:%x:%x\n", link->domain, link->bus,
 374                link->dev);
 375
 376        if (spa && spa->spa_mem) {
 377                free_pages((unsigned long) spa->spa_mem, spa->spa_order);
 378                kfree(spa);
 379                link->spa = NULL;
 380        }
 381}
 382
 383static int alloc_link(struct pci_dev *dev, int PE_mask, struct ocxl_link **out_link)
 384{
 385        struct ocxl_link *link;
 386        int rc;
 387
 388        link = kzalloc(sizeof(struct ocxl_link), GFP_KERNEL);
 389        if (!link)
 390                return -ENOMEM;
 391
 392        kref_init(&link->ref);
 393        link->domain = pci_domain_nr(dev->bus);
 394        link->bus = dev->bus->number;
 395        link->dev = PCI_SLOT(dev->devfn);
 396        atomic_set(&link->irq_available, MAX_IRQ_PER_LINK);
 397        spin_lock_init(&link->atsd_lock);
 398
 399        rc = alloc_spa(dev, link);
 400        if (rc)
 401                goto err_free;
 402
 403        rc = setup_xsl_irq(dev, link);
 404        if (rc)
 405                goto err_spa;
 406
 407        /* platform specific hook */
 408        rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
 409                                &link->platform_data);
 410        if (rc)
 411                goto err_xsl_irq;
 412
 413        /* if link->arva is not defeined, MMIO registers are not used to
 414         * generate TLB invalidate. PowerBus snooping is enabled.
 415         * Otherwise, PowerBus snooping is disabled. TLB Invalidates are
 416         * initiated using MMIO registers.
 417         */
 418        pnv_ocxl_map_lpar(dev, mfspr(SPRN_LPID), 0, &link->arva);
 419
 420        *out_link = link;
 421        return 0;
 422
 423err_xsl_irq:
 424        release_xsl_irq(link);
 425err_spa:
 426        free_spa(link);
 427err_free:
 428        kfree(link);
 429        return rc;
 430}
 431
 432static void free_link(struct ocxl_link *link)
 433{
 434        release_xsl_irq(link);
 435        free_spa(link);
 436        kfree(link);
 437}
 438
 439int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle)
 440{
 441        int rc = 0;
 442        struct ocxl_link *link;
 443
 444        mutex_lock(&links_list_lock);
 445        list_for_each_entry(link, &links_list, list) {
 446                /* The functions of a device all share the same link */
 447                if (link->domain == pci_domain_nr(dev->bus) &&
 448                        link->bus == dev->bus->number &&
 449                        link->dev == PCI_SLOT(dev->devfn)) {
 450                        kref_get(&link->ref);
 451                        *link_handle = link;
 452                        goto unlock;
 453                }
 454        }
 455        rc = alloc_link(dev, PE_mask, &link);
 456        if (rc)
 457                goto unlock;
 458
 459        list_add(&link->list, &links_list);
 460        *link_handle = link;
 461unlock:
 462        mutex_unlock(&links_list_lock);
 463        return rc;
 464}
 465EXPORT_SYMBOL_GPL(ocxl_link_setup);
 466
 467static void release_xsl(struct kref *ref)
 468{
 469        struct ocxl_link *link = container_of(ref, struct ocxl_link, ref);
 470
 471        if (link->arva) {
 472                pnv_ocxl_unmap_lpar(link->arva);
 473                link->arva = NULL;
 474        }
 475
 476        list_del(&link->list);
 477        /* call platform code before releasing data */
 478        pnv_ocxl_spa_release(link->platform_data);
 479        free_link(link);
 480}
 481
 482void ocxl_link_release(struct pci_dev *dev, void *link_handle)
 483{
 484        struct ocxl_link *link = (struct ocxl_link *) link_handle;
 485
 486        mutex_lock(&links_list_lock);
 487        kref_put(&link->ref, release_xsl);
 488        mutex_unlock(&links_list_lock);
 489}
 490EXPORT_SYMBOL_GPL(ocxl_link_release);
 491
 492static void invalidate_range(struct mmu_notifier *mn,
 493                             struct mm_struct *mm,
 494                             unsigned long start, unsigned long end)
 495{
 496        struct pe_data *pe_data = container_of(mn, struct pe_data, mmu_notifier);
 497        struct ocxl_link *link = pe_data->link;
 498        unsigned long addr, pid, page_size = PAGE_SIZE;
 499
 500        pid = mm->context.id;
 501        trace_ocxl_mmu_notifier_range(start, end, pid);
 502
 503        spin_lock(&link->atsd_lock);
 504        for (addr = start; addr < end; addr += page_size)
 505                pnv_ocxl_tlb_invalidate(link->arva, pid, addr, page_size);
 506        spin_unlock(&link->atsd_lock);
 507}
 508
 509static const struct mmu_notifier_ops ocxl_mmu_notifier_ops = {
 510        .invalidate_range = invalidate_range,
 511};
 512
 513static u64 calculate_cfg_state(bool kernel)
 514{
 515        u64 state;
 516
 517        state = SPA_CFG_DR;
 518        if (mfspr(SPRN_LPCR) & LPCR_TC)
 519                state |= SPA_CFG_TC;
 520        if (radix_enabled())
 521                state |= SPA_CFG_XLAT_ror;
 522        else
 523                state |= SPA_CFG_XLAT_hpt;
 524        state |= SPA_CFG_HV;
 525        if (kernel) {
 526                if (mfmsr() & MSR_SF)
 527                        state |= SPA_CFG_SF;
 528        } else {
 529                state |= SPA_CFG_PR;
 530                if (!test_tsk_thread_flag(current, TIF_32BIT))
 531                        state |= SPA_CFG_SF;
 532        }
 533        return state;
 534}
 535
 536int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr,
 537                u64 amr, u16 bdf, struct mm_struct *mm,
 538                void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr),
 539                void *xsl_err_data)
 540{
 541        struct ocxl_link *link = (struct ocxl_link *) link_handle;
 542        struct spa *spa = link->spa;
 543        struct ocxl_process_element *pe;
 544        int pe_handle, rc = 0;
 545        struct pe_data *pe_data;
 546
 547        BUILD_BUG_ON(sizeof(struct ocxl_process_element) != 128);
 548        if (pasid > SPA_PASID_MAX)
 549                return -EINVAL;
 550
 551        mutex_lock(&spa->spa_lock);
 552        pe_handle = pasid & SPA_PE_MASK;
 553        pe = spa->spa_mem + pe_handle;
 554
 555        if (pe->software_state) {
 556                rc = -EBUSY;
 557                goto unlock;
 558        }
 559
 560        pe_data = kmalloc(sizeof(*pe_data), GFP_KERNEL);
 561        if (!pe_data) {
 562                rc = -ENOMEM;
 563                goto unlock;
 564        }
 565
 566        pe_data->mm = mm;
 567        pe_data->xsl_err_cb = xsl_err_cb;
 568        pe_data->xsl_err_data = xsl_err_data;
 569        pe_data->link = link;
 570        pe_data->mmu_notifier.ops = &ocxl_mmu_notifier_ops;
 571
 572        memset(pe, 0, sizeof(struct ocxl_process_element));
 573        pe->config_state = cpu_to_be64(calculate_cfg_state(pidr == 0));
 574        pe->pasid = cpu_to_be32(pasid << (31 - 19));
 575        pe->bdf = cpu_to_be16(bdf);
 576        pe->lpid = cpu_to_be32(mfspr(SPRN_LPID));
 577        pe->pid = cpu_to_be32(pidr);
 578        pe->tid = cpu_to_be32(tidr);
 579        pe->amr = cpu_to_be64(amr);
 580        pe->software_state = cpu_to_be32(SPA_PE_VALID);
 581
 582        /*
 583         * For user contexts, register a copro so that TLBIs are seen
 584         * by the nest MMU. If we have a kernel context, TLBIs are
 585         * already global.
 586         */
 587        if (mm) {
 588                mm_context_add_copro(mm);
 589                if (link->arva) {
 590                        /* Use MMIO registers for the TLB Invalidate
 591                         * operations.
 592                         */
 593                        trace_ocxl_init_mmu_notifier(pasid, mm->context.id);
 594                        mmu_notifier_register(&pe_data->mmu_notifier, mm);
 595                }
 596        }
 597
 598        /*
 599         * Barrier is to make sure PE is visible in the SPA before it
 600         * is used by the device. It also helps with the global TLBI
 601         * invalidation
 602         */
 603        mb();
 604        radix_tree_insert(&spa->pe_tree, pe_handle, pe_data);
 605
 606        /*
 607         * The mm must stay valid for as long as the device uses it. We
 608         * lower the count when the context is removed from the SPA.
 609         *
 610         * We grab mm_count (and not mm_users), as we don't want to
 611         * end up in a circular dependency if a process mmaps its
 612         * mmio, therefore incrementing the file ref count when
 613         * calling mmap(), and forgets to unmap before exiting. In
 614         * that scenario, when the kernel handles the death of the
 615         * process, the file is not cleaned because unmap was not
 616         * called, and the mm wouldn't be freed because we would still
 617         * have a reference on mm_users. Incrementing mm_count solves
 618         * the problem.
 619         */
 620        if (mm)
 621                mmgrab(mm);
 622        trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr);
 623unlock:
 624        mutex_unlock(&spa->spa_lock);
 625        return rc;
 626}
 627EXPORT_SYMBOL_GPL(ocxl_link_add_pe);
 628
 629int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid)
 630{
 631        struct ocxl_link *link = (struct ocxl_link *) link_handle;
 632        struct spa *spa = link->spa;
 633        struct ocxl_process_element *pe;
 634        int pe_handle, rc;
 635
 636        if (pasid > SPA_PASID_MAX)
 637                return -EINVAL;
 638
 639        pe_handle = pasid & SPA_PE_MASK;
 640        pe = spa->spa_mem + pe_handle;
 641
 642        mutex_lock(&spa->spa_lock);
 643
 644        pe->tid = cpu_to_be32(tid);
 645
 646        /*
 647         * The barrier makes sure the PE is updated
 648         * before we clear the NPU context cache below, so that the
 649         * old PE cannot be reloaded erroneously.
 650         */
 651        mb();
 652
 653        /*
 654         * hook to platform code
 655         * On powerpc, the entry needs to be cleared from the context
 656         * cache of the NPU.
 657         */
 658        rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
 659        WARN_ON(rc);
 660
 661        mutex_unlock(&spa->spa_lock);
 662        return rc;
 663}
 664
 665int ocxl_link_remove_pe(void *link_handle, int pasid)
 666{
 667        struct ocxl_link *link = (struct ocxl_link *) link_handle;
 668        struct spa *spa = link->spa;
 669        struct ocxl_process_element *pe;
 670        struct pe_data *pe_data;
 671        int pe_handle, rc;
 672
 673        if (pasid > SPA_PASID_MAX)
 674                return -EINVAL;
 675
 676        /*
 677         * About synchronization with our memory fault handler:
 678         *
 679         * Before removing the PE, the driver is supposed to have
 680         * notified the AFU, which should have cleaned up and make
 681         * sure the PASID is no longer in use, including pending
 682         * interrupts. However, there's no way to be sure...
 683         *
 684         * We clear the PE and remove the context from our radix
 685         * tree. From that point on, any new interrupt for that
 686         * context will fail silently, which is ok. As mentioned
 687         * above, that's not expected, but it could happen if the
 688         * driver or AFU didn't do the right thing.
 689         *
 690         * There could still be a bottom half running, but we don't
 691         * need to wait/flush, as it is managing a reference count on
 692         * the mm it reads from the radix tree.
 693         */
 694        pe_handle = pasid & SPA_PE_MASK;
 695        pe = spa->spa_mem + pe_handle;
 696
 697        mutex_lock(&spa->spa_lock);
 698
 699        if (!(be32_to_cpu(pe->software_state) & SPA_PE_VALID)) {
 700                rc = -EINVAL;
 701                goto unlock;
 702        }
 703
 704        trace_ocxl_context_remove(current->pid, spa->spa_mem, pasid,
 705                                be32_to_cpu(pe->pid), be32_to_cpu(pe->tid));
 706
 707        memset(pe, 0, sizeof(struct ocxl_process_element));
 708        /*
 709         * The barrier makes sure the PE is removed from the SPA
 710         * before we clear the NPU context cache below, so that the
 711         * old PE cannot be reloaded erroneously.
 712         */
 713        mb();
 714
 715        /*
 716         * hook to platform code
 717         * On powerpc, the entry needs to be cleared from the context
 718         * cache of the NPU.
 719         */
 720        rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
 721        WARN_ON(rc);
 722
 723        pe_data = radix_tree_delete(&spa->pe_tree, pe_handle);
 724        if (!pe_data) {
 725                WARN(1, "Couldn't find pe data when removing PE\n");
 726        } else {
 727                if (pe_data->mm) {
 728                        if (link->arva) {
 729                                trace_ocxl_release_mmu_notifier(pasid,
 730                                                                pe_data->mm->context.id);
 731                                mmu_notifier_unregister(&pe_data->mmu_notifier,
 732                                                        pe_data->mm);
 733                                spin_lock(&link->atsd_lock);
 734                                pnv_ocxl_tlb_invalidate(link->arva,
 735                                                        pe_data->mm->context.id,
 736                                                        0ull,
 737                                                        PAGE_SIZE);
 738                                spin_unlock(&link->atsd_lock);
 739                        }
 740                        mm_context_remove_copro(pe_data->mm);
 741                        mmdrop(pe_data->mm);
 742                }
 743                kfree_rcu(pe_data, rcu);
 744        }
 745unlock:
 746        mutex_unlock(&spa->spa_lock);
 747        return rc;
 748}
 749EXPORT_SYMBOL_GPL(ocxl_link_remove_pe);
 750
 751int ocxl_link_irq_alloc(void *link_handle, int *hw_irq, u64 *trigger_addr)
 752{
 753        struct ocxl_link *link = (struct ocxl_link *) link_handle;
 754        int rc, irq;
 755        u64 addr;
 756
 757        if (atomic_dec_if_positive(&link->irq_available) < 0)
 758                return -ENOSPC;
 759
 760        rc = pnv_ocxl_alloc_xive_irq(&irq, &addr);
 761        if (rc) {
 762                atomic_inc(&link->irq_available);
 763                return rc;
 764        }
 765
 766        *hw_irq = irq;
 767        *trigger_addr = addr;
 768        return 0;
 769}
 770EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc);
 771
 772void ocxl_link_free_irq(void *link_handle, int hw_irq)
 773{
 774        struct ocxl_link *link = (struct ocxl_link *) link_handle;
 775
 776        pnv_ocxl_free_xive_irq(hw_irq);
 777        atomic_inc(&link->irq_available);
 778}
 779EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
 780