linux/arch/powerpc/platforms/powernv/pci-ioda.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Support PCI/PCIe on PowerNV platforms
   4 *
   5 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   6 */
   7
   8#undef DEBUG
   9
  10#include <linux/kernel.h>
  11#include <linux/pci.h>
  12#include <linux/crash_dump.h>
  13#include <linux/delay.h>
  14#include <linux/string.h>
  15#include <linux/init.h>
  16#include <linux/memblock.h>
  17#include <linux/irq.h>
  18#include <linux/io.h>
  19#include <linux/msi.h>
  20#include <linux/iommu.h>
  21#include <linux/rculist.h>
  22#include <linux/sizes.h>
  23
  24#include <asm/sections.h>
  25#include <asm/io.h>
  26#include <asm/prom.h>
  27#include <asm/pci-bridge.h>
  28#include <asm/machdep.h>
  29#include <asm/msi_bitmap.h>
  30#include <asm/ppc-pci.h>
  31#include <asm/opal.h>
  32#include <asm/iommu.h>
  33#include <asm/tce.h>
  34#include <asm/xics.h>
  35#include <asm/debugfs.h>
  36#include <asm/firmware.h>
  37#include <asm/pnv-pci.h>
  38#include <asm/mmzone.h>
  39
  40#include <misc/cxl-base.h>
  41
  42#include "powernv.h"
  43#include "pci.h"
  44#include "../../../../drivers/pci/pci.h"
  45
  46#define PNV_IODA1_M64_NUM       16      /* Number of M64 BARs   */
  47#define PNV_IODA1_M64_SEGS      8       /* Segments per M64 BAR */
  48#define PNV_IODA1_DMA32_SEGSIZE 0x10000000
  49
  50static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU_NVLINK",
  51                                              "NPU_OCAPI" };
  52
  53static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
  54
  55void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
  56                            const char *fmt, ...)
  57{
  58        struct va_format vaf;
  59        va_list args;
  60        char pfix[32];
  61
  62        va_start(args, fmt);
  63
  64        vaf.fmt = fmt;
  65        vaf.va = &args;
  66
  67        if (pe->flags & PNV_IODA_PE_DEV)
  68                strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
  69        else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  70                sprintf(pfix, "%04x:%02x     ",
  71                        pci_domain_nr(pe->pbus), pe->pbus->number);
  72#ifdef CONFIG_PCI_IOV
  73        else if (pe->flags & PNV_IODA_PE_VF)
  74                sprintf(pfix, "%04x:%02x:%2x.%d",
  75                        pci_domain_nr(pe->parent_dev->bus),
  76                        (pe->rid & 0xff00) >> 8,
  77                        PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
  78#endif /* CONFIG_PCI_IOV*/
  79
  80        printk("%spci %s: [PE# %.2x] %pV",
  81               level, pfix, pe->pe_number, &vaf);
  82
  83        va_end(args);
  84}
  85
  86static bool pnv_iommu_bypass_disabled __read_mostly;
  87static bool pci_reset_phbs __read_mostly;
  88
  89static int __init iommu_setup(char *str)
  90{
  91        if (!str)
  92                return -EINVAL;
  93
  94        while (*str) {
  95                if (!strncmp(str, "nobypass", 8)) {
  96                        pnv_iommu_bypass_disabled = true;
  97                        pr_info("PowerNV: IOMMU bypass window disabled.\n");
  98                        break;
  99                }
 100                str += strcspn(str, ",");
 101                if (*str == ',')
 102                        str++;
 103        }
 104
 105        return 0;
 106}
 107early_param("iommu", iommu_setup);
 108
 109static int __init pci_reset_phbs_setup(char *str)
 110{
 111        pci_reset_phbs = true;
 112        return 0;
 113}
 114
 115early_param("ppc_pci_reset_phbs", pci_reset_phbs_setup);
 116
 117static inline bool pnv_pci_is_m64(struct pnv_phb *phb, struct resource *r)
 118{
 119        /*
 120         * WARNING: We cannot rely on the resource flags. The Linux PCI
 121         * allocation code sometimes decides to put a 64-bit prefetchable
 122         * BAR in the 32-bit window, so we have to compare the addresses.
 123         *
 124         * For simplicity we only test resource start.
 125         */
 126        return (r->start >= phb->ioda.m64_base &&
 127                r->start < (phb->ioda.m64_base + phb->ioda.m64_size));
 128}
 129
 130static inline bool pnv_pci_is_m64_flags(unsigned long resource_flags)
 131{
 132        unsigned long flags = (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
 133
 134        return (resource_flags & flags) == flags;
 135}
 136
 137static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
 138{
 139        s64 rc;
 140
 141        phb->ioda.pe_array[pe_no].phb = phb;
 142        phb->ioda.pe_array[pe_no].pe_number = pe_no;
 143
 144        /*
 145         * Clear the PE frozen state as it might be put into frozen state
 146         * in the last PCI remove path. It's not harmful to do so when the
 147         * PE is already in unfrozen state.
 148         */
 149        rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
 150                                       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 151        if (rc != OPAL_SUCCESS && rc != OPAL_UNSUPPORTED)
 152                pr_warn("%s: Error %lld unfreezing PHB#%x-PE#%x\n",
 153                        __func__, rc, phb->hose->global_number, pe_no);
 154
 155        return &phb->ioda.pe_array[pe_no];
 156}
 157
 158static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
 159{
 160        if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe_num)) {
 161                pr_warn("%s: Invalid PE %x on PHB#%x\n",
 162                        __func__, pe_no, phb->hose->global_number);
 163                return;
 164        }
 165
 166        if (test_and_set_bit(pe_no, phb->ioda.pe_alloc))
 167                pr_debug("%s: PE %x was reserved on PHB#%x\n",
 168                         __func__, pe_no, phb->hose->global_number);
 169
 170        pnv_ioda_init_pe(phb, pe_no);
 171}
 172
 173static struct pnv_ioda_pe *pnv_ioda_alloc_pe(struct pnv_phb *phb)
 174{
 175        long pe;
 176
 177        for (pe = phb->ioda.total_pe_num - 1; pe >= 0; pe--) {
 178                if (!test_and_set_bit(pe, phb->ioda.pe_alloc))
 179                        return pnv_ioda_init_pe(phb, pe);
 180        }
 181
 182        return NULL;
 183}
 184
 185static void pnv_ioda_free_pe(struct pnv_ioda_pe *pe)
 186{
 187        struct pnv_phb *phb = pe->phb;
 188        unsigned int pe_num = pe->pe_number;
 189
 190        WARN_ON(pe->pdev);
 191        WARN_ON(pe->npucomp); /* NPUs are not supposed to be freed */
 192        kfree(pe->npucomp);
 193        memset(pe, 0, sizeof(struct pnv_ioda_pe));
 194        clear_bit(pe_num, phb->ioda.pe_alloc);
 195}
 196
 197/* The default M64 BAR is shared by all PEs */
 198static int pnv_ioda2_init_m64(struct pnv_phb *phb)
 199{
 200        const char *desc;
 201        struct resource *r;
 202        s64 rc;
 203
 204        /* Configure the default M64 BAR */
 205        rc = opal_pci_set_phb_mem_window(phb->opal_id,
 206                                         OPAL_M64_WINDOW_TYPE,
 207                                         phb->ioda.m64_bar_idx,
 208                                         phb->ioda.m64_base,
 209                                         0, /* unused */
 210                                         phb->ioda.m64_size);
 211        if (rc != OPAL_SUCCESS) {
 212                desc = "configuring";
 213                goto fail;
 214        }
 215
 216        /* Enable the default M64 BAR */
 217        rc = opal_pci_phb_mmio_enable(phb->opal_id,
 218                                      OPAL_M64_WINDOW_TYPE,
 219                                      phb->ioda.m64_bar_idx,
 220                                      OPAL_ENABLE_M64_SPLIT);
 221        if (rc != OPAL_SUCCESS) {
 222                desc = "enabling";
 223                goto fail;
 224        }
 225
 226        /*
 227         * Exclude the segments for reserved and root bus PE, which
 228         * are first or last two PEs.
 229         */
 230        r = &phb->hose->mem_resources[1];
 231        if (phb->ioda.reserved_pe_idx == 0)
 232                r->start += (2 * phb->ioda.m64_segsize);
 233        else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
 234                r->end -= (2 * phb->ioda.m64_segsize);
 235        else
 236                pr_warn("  Cannot strip M64 segment for reserved PE#%x\n",
 237                        phb->ioda.reserved_pe_idx);
 238
 239        return 0;
 240
 241fail:
 242        pr_warn("  Failure %lld %s M64 BAR#%d\n",
 243                rc, desc, phb->ioda.m64_bar_idx);
 244        opal_pci_phb_mmio_enable(phb->opal_id,
 245                                 OPAL_M64_WINDOW_TYPE,
 246                                 phb->ioda.m64_bar_idx,
 247                                 OPAL_DISABLE_M64);
 248        return -EIO;
 249}
 250
 251static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev,
 252                                         unsigned long *pe_bitmap)
 253{
 254        struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 255        struct pnv_phb *phb = hose->private_data;
 256        struct resource *r;
 257        resource_size_t base, sgsz, start, end;
 258        int segno, i;
 259
 260        base = phb->ioda.m64_base;
 261        sgsz = phb->ioda.m64_segsize;
 262        for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
 263                r = &pdev->resource[i];
 264                if (!r->parent || !pnv_pci_is_m64(phb, r))
 265                        continue;
 266
 267                start = _ALIGN_DOWN(r->start - base, sgsz);
 268                end = _ALIGN_UP(r->end - base, sgsz);
 269                for (segno = start / sgsz; segno < end / sgsz; segno++) {
 270                        if (pe_bitmap)
 271                                set_bit(segno, pe_bitmap);
 272                        else
 273                                pnv_ioda_reserve_pe(phb, segno);
 274                }
 275        }
 276}
 277
 278static int pnv_ioda1_init_m64(struct pnv_phb *phb)
 279{
 280        struct resource *r;
 281        int index;
 282
 283        /*
 284         * There are 16 M64 BARs, each of which has 8 segments. So
 285         * there are as many M64 segments as the maximum number of
 286         * PEs, which is 128.
 287         */
 288        for (index = 0; index < PNV_IODA1_M64_NUM; index++) {
 289                unsigned long base, segsz = phb->ioda.m64_segsize;
 290                int64_t rc;
 291
 292                base = phb->ioda.m64_base +
 293                       index * PNV_IODA1_M64_SEGS * segsz;
 294                rc = opal_pci_set_phb_mem_window(phb->opal_id,
 295                                OPAL_M64_WINDOW_TYPE, index, base, 0,
 296                                PNV_IODA1_M64_SEGS * segsz);
 297                if (rc != OPAL_SUCCESS) {
 298                        pr_warn("  Error %lld setting M64 PHB#%x-BAR#%d\n",
 299                                rc, phb->hose->global_number, index);
 300                        goto fail;
 301                }
 302
 303                rc = opal_pci_phb_mmio_enable(phb->opal_id,
 304                                OPAL_M64_WINDOW_TYPE, index,
 305                                OPAL_ENABLE_M64_SPLIT);
 306                if (rc != OPAL_SUCCESS) {
 307                        pr_warn("  Error %lld enabling M64 PHB#%x-BAR#%d\n",
 308                                rc, phb->hose->global_number, index);
 309                        goto fail;
 310                }
 311        }
 312
 313        /*
 314         * Exclude the segments for reserved and root bus PE, which
 315         * are first or last two PEs.
 316         */
 317        r = &phb->hose->mem_resources[1];
 318        if (phb->ioda.reserved_pe_idx == 0)
 319                r->start += (2 * phb->ioda.m64_segsize);
 320        else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
 321                r->end -= (2 * phb->ioda.m64_segsize);
 322        else
 323                WARN(1, "Wrong reserved PE#%x on PHB#%x\n",
 324                     phb->ioda.reserved_pe_idx, phb->hose->global_number);
 325
 326        return 0;
 327
 328fail:
 329        for ( ; index >= 0; index--)
 330                opal_pci_phb_mmio_enable(phb->opal_id,
 331                        OPAL_M64_WINDOW_TYPE, index, OPAL_DISABLE_M64);
 332
 333        return -EIO;
 334}
 335
 336static void pnv_ioda_reserve_m64_pe(struct pci_bus *bus,
 337                                    unsigned long *pe_bitmap,
 338                                    bool all)
 339{
 340        struct pci_dev *pdev;
 341
 342        list_for_each_entry(pdev, &bus->devices, bus_list) {
 343                pnv_ioda_reserve_dev_m64_pe(pdev, pe_bitmap);
 344
 345                if (all && pdev->subordinate)
 346                        pnv_ioda_reserve_m64_pe(pdev->subordinate,
 347                                                pe_bitmap, all);
 348        }
 349}
 350
 351static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all)
 352{
 353        struct pci_controller *hose = pci_bus_to_host(bus);
 354        struct pnv_phb *phb = hose->private_data;
 355        struct pnv_ioda_pe *master_pe, *pe;
 356        unsigned long size, *pe_alloc;
 357        int i;
 358
 359        /* Root bus shouldn't use M64 */
 360        if (pci_is_root_bus(bus))
 361                return NULL;
 362
 363        /* Allocate bitmap */
 364        size = _ALIGN_UP(phb->ioda.total_pe_num / 8, sizeof(unsigned long));
 365        pe_alloc = kzalloc(size, GFP_KERNEL);
 366        if (!pe_alloc) {
 367                pr_warn("%s: Out of memory !\n",
 368                        __func__);
 369                return NULL;
 370        }
 371
 372        /* Figure out reserved PE numbers by the PE */
 373        pnv_ioda_reserve_m64_pe(bus, pe_alloc, all);
 374
 375        /*
 376         * the current bus might not own M64 window and that's all
 377         * contributed by its child buses. For the case, we needn't
 378         * pick M64 dependent PE#.
 379         */
 380        if (bitmap_empty(pe_alloc, phb->ioda.total_pe_num)) {
 381                kfree(pe_alloc);
 382                return NULL;
 383        }
 384
 385        /*
 386         * Figure out the master PE and put all slave PEs to master
 387         * PE's list to form compound PE.
 388         */
 389        master_pe = NULL;
 390        i = -1;
 391        while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe_num, i + 1)) <
 392                phb->ioda.total_pe_num) {
 393                pe = &phb->ioda.pe_array[i];
 394
 395                phb->ioda.m64_segmap[pe->pe_number] = pe->pe_number;
 396                if (!master_pe) {
 397                        pe->flags |= PNV_IODA_PE_MASTER;
 398                        INIT_LIST_HEAD(&pe->slaves);
 399                        master_pe = pe;
 400                } else {
 401                        pe->flags |= PNV_IODA_PE_SLAVE;
 402                        pe->master = master_pe;
 403                        list_add_tail(&pe->list, &master_pe->slaves);
 404                }
 405
 406                /*
 407                 * P7IOC supports M64DT, which helps mapping M64 segment
 408                 * to one particular PE#. However, PHB3 has fixed mapping
 409                 * between M64 segment and PE#. In order to have same logic
 410                 * for P7IOC and PHB3, we enforce fixed mapping between M64
 411                 * segment and PE# on P7IOC.
 412                 */
 413                if (phb->type == PNV_PHB_IODA1) {
 414                        int64_t rc;
 415
 416                        rc = opal_pci_map_pe_mmio_window(phb->opal_id,
 417                                        pe->pe_number, OPAL_M64_WINDOW_TYPE,
 418                                        pe->pe_number / PNV_IODA1_M64_SEGS,
 419                                        pe->pe_number % PNV_IODA1_M64_SEGS);
 420                        if (rc != OPAL_SUCCESS)
 421                                pr_warn("%s: Error %lld mapping M64 for PHB#%x-PE#%x\n",
 422                                        __func__, rc, phb->hose->global_number,
 423                                        pe->pe_number);
 424                }
 425        }
 426
 427        kfree(pe_alloc);
 428        return master_pe;
 429}
 430
 431static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
 432{
 433        struct pci_controller *hose = phb->hose;
 434        struct device_node *dn = hose->dn;
 435        struct resource *res;
 436        u32 m64_range[2], i;
 437        const __be32 *r;
 438        u64 pci_addr;
 439
 440        if (phb->type != PNV_PHB_IODA1 && phb->type != PNV_PHB_IODA2) {
 441                pr_info("  Not support M64 window\n");
 442                return;
 443        }
 444
 445        if (!firmware_has_feature(FW_FEATURE_OPAL)) {
 446                pr_info("  Firmware too old to support M64 window\n");
 447                return;
 448        }
 449
 450        r = of_get_property(dn, "ibm,opal-m64-window", NULL);
 451        if (!r) {
 452                pr_info("  No <ibm,opal-m64-window> on %pOF\n",
 453                        dn);
 454                return;
 455        }
 456
 457        /*
 458         * Find the available M64 BAR range and pickup the last one for
 459         * covering the whole 64-bits space. We support only one range.
 460         */
 461        if (of_property_read_u32_array(dn, "ibm,opal-available-m64-ranges",
 462                                       m64_range, 2)) {
 463                /* In absence of the property, assume 0..15 */
 464                m64_range[0] = 0;
 465                m64_range[1] = 16;
 466        }
 467        /* We only support 64 bits in our allocator */
 468        if (m64_range[1] > 63) {
 469                pr_warn("%s: Limiting M64 range to 63 (from %d) on PHB#%x\n",
 470                        __func__, m64_range[1], phb->hose->global_number);
 471                m64_range[1] = 63;
 472        }
 473        /* Empty range, no m64 */
 474        if (m64_range[1] <= m64_range[0]) {
 475                pr_warn("%s: M64 empty, disabling M64 usage on PHB#%x\n",
 476                        __func__, phb->hose->global_number);
 477                return;
 478        }
 479
 480        /* Configure M64 informations */
 481        res = &hose->mem_resources[1];
 482        res->name = dn->full_name;
 483        res->start = of_translate_address(dn, r + 2);
 484        res->end = res->start + of_read_number(r + 4, 2) - 1;
 485        res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
 486        pci_addr = of_read_number(r, 2);
 487        hose->mem_offset[1] = res->start - pci_addr;
 488
 489        phb->ioda.m64_size = resource_size(res);
 490        phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe_num;
 491        phb->ioda.m64_base = pci_addr;
 492
 493        /* This lines up nicely with the display from processing OF ranges */
 494        pr_info(" MEM 0x%016llx..0x%016llx -> 0x%016llx (M64 #%d..%d)\n",
 495                res->start, res->end, pci_addr, m64_range[0],
 496                m64_range[0] + m64_range[1] - 1);
 497
 498        /* Mark all M64 used up by default */
 499        phb->ioda.m64_bar_alloc = (unsigned long)-1;
 500
 501        /* Use last M64 BAR to cover M64 window */
 502        m64_range[1]--;
 503        phb->ioda.m64_bar_idx = m64_range[0] + m64_range[1];
 504
 505        pr_info(" Using M64 #%d as default window\n", phb->ioda.m64_bar_idx);
 506
 507        /* Mark remaining ones free */
 508        for (i = m64_range[0]; i < m64_range[1]; i++)
 509                clear_bit(i, &phb->ioda.m64_bar_alloc);
 510
 511        /*
 512         * Setup init functions for M64 based on IODA version, IODA3 uses
 513         * the IODA2 code.
 514         */
 515        if (phb->type == PNV_PHB_IODA1)
 516                phb->init_m64 = pnv_ioda1_init_m64;
 517        else
 518                phb->init_m64 = pnv_ioda2_init_m64;
 519}
 520
 521static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
 522{
 523        struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
 524        struct pnv_ioda_pe *slave;
 525        s64 rc;
 526
 527        /* Fetch master PE */
 528        if (pe->flags & PNV_IODA_PE_SLAVE) {
 529                pe = pe->master;
 530                if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
 531                        return;
 532
 533                pe_no = pe->pe_number;
 534        }
 535
 536        /* Freeze master PE */
 537        rc = opal_pci_eeh_freeze_set(phb->opal_id,
 538                                     pe_no,
 539                                     OPAL_EEH_ACTION_SET_FREEZE_ALL);
 540        if (rc != OPAL_SUCCESS) {
 541                pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 542                        __func__, rc, phb->hose->global_number, pe_no);
 543                return;
 544        }
 545
 546        /* Freeze slave PEs */
 547        if (!(pe->flags & PNV_IODA_PE_MASTER))
 548                return;
 549
 550        list_for_each_entry(slave, &pe->slaves, list) {
 551                rc = opal_pci_eeh_freeze_set(phb->opal_id,
 552                                             slave->pe_number,
 553                                             OPAL_EEH_ACTION_SET_FREEZE_ALL);
 554                if (rc != OPAL_SUCCESS)
 555                        pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 556                                __func__, rc, phb->hose->global_number,
 557                                slave->pe_number);
 558        }
 559}
 560
 561static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
 562{
 563        struct pnv_ioda_pe *pe, *slave;
 564        s64 rc;
 565
 566        /* Find master PE */
 567        pe = &phb->ioda.pe_array[pe_no];
 568        if (pe->flags & PNV_IODA_PE_SLAVE) {
 569                pe = pe->master;
 570                WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 571                pe_no = pe->pe_number;
 572        }
 573
 574        /* Clear frozen state for master PE */
 575        rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
 576        if (rc != OPAL_SUCCESS) {
 577                pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 578                        __func__, rc, opt, phb->hose->global_number, pe_no);
 579                return -EIO;
 580        }
 581
 582        if (!(pe->flags & PNV_IODA_PE_MASTER))
 583                return 0;
 584
 585        /* Clear frozen state for slave PEs */
 586        list_for_each_entry(slave, &pe->slaves, list) {
 587                rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 588                                             slave->pe_number,
 589                                             opt);
 590                if (rc != OPAL_SUCCESS) {
 591                        pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 592                                __func__, rc, opt, phb->hose->global_number,
 593                                slave->pe_number);
 594                        return -EIO;
 595                }
 596        }
 597
 598        return 0;
 599}
 600
 601static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
 602{
 603        struct pnv_ioda_pe *slave, *pe;
 604        u8 fstate = 0, state;
 605        __be16 pcierr = 0;
 606        s64 rc;
 607
 608        /* Sanity check on PE number */
 609        if (pe_no < 0 || pe_no >= phb->ioda.total_pe_num)
 610                return OPAL_EEH_STOPPED_PERM_UNAVAIL;
 611
 612        /*
 613         * Fetch the master PE and the PE instance might be
 614         * not initialized yet.
 615         */
 616        pe = &phb->ioda.pe_array[pe_no];
 617        if (pe->flags & PNV_IODA_PE_SLAVE) {
 618                pe = pe->master;
 619                WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 620                pe_no = pe->pe_number;
 621        }
 622
 623        /* Check the master PE */
 624        rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
 625                                        &state, &pcierr, NULL);
 626        if (rc != OPAL_SUCCESS) {
 627                pr_warn("%s: Failure %lld getting "
 628                        "PHB#%x-PE#%x state\n",
 629                        __func__, rc,
 630                        phb->hose->global_number, pe_no);
 631                return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 632        }
 633
 634        /* Check the slave PE */
 635        if (!(pe->flags & PNV_IODA_PE_MASTER))
 636                return state;
 637
 638        list_for_each_entry(slave, &pe->slaves, list) {
 639                rc = opal_pci_eeh_freeze_status(phb->opal_id,
 640                                                slave->pe_number,
 641                                                &fstate,
 642                                                &pcierr,
 643                                                NULL);
 644                if (rc != OPAL_SUCCESS) {
 645                        pr_warn("%s: Failure %lld getting "
 646                                "PHB#%x-PE#%x state\n",
 647                                __func__, rc,
 648                                phb->hose->global_number, slave->pe_number);
 649                        return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 650                }
 651
 652                /*
 653                 * Override the result based on the ascending
 654                 * priority.
 655                 */
 656                if (fstate > state)
 657                        state = fstate;
 658        }
 659
 660        return state;
 661}
 662
 663struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
 664{
 665        struct pci_controller *hose = pci_bus_to_host(dev->bus);
 666        struct pnv_phb *phb = hose->private_data;
 667        struct pci_dn *pdn = pci_get_pdn(dev);
 668
 669        if (!pdn)
 670                return NULL;
 671        if (pdn->pe_number == IODA_INVALID_PE)
 672                return NULL;
 673        return &phb->ioda.pe_array[pdn->pe_number];
 674}
 675
 676static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
 677                                  struct pnv_ioda_pe *parent,
 678                                  struct pnv_ioda_pe *child,
 679                                  bool is_add)
 680{
 681        const char *desc = is_add ? "adding" : "removing";
 682        uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
 683                              OPAL_REMOVE_PE_FROM_DOMAIN;
 684        struct pnv_ioda_pe *slave;
 685        long rc;
 686
 687        /* Parent PE affects child PE */
 688        rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 689                                child->pe_number, op);
 690        if (rc != OPAL_SUCCESS) {
 691                pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
 692                        rc, desc);
 693                return -ENXIO;
 694        }
 695
 696        if (!(child->flags & PNV_IODA_PE_MASTER))
 697                return 0;
 698
 699        /* Compound case: parent PE affects slave PEs */
 700        list_for_each_entry(slave, &child->slaves, list) {
 701                rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 702                                        slave->pe_number, op);
 703                if (rc != OPAL_SUCCESS) {
 704                        pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
 705                                rc, desc);
 706                        return -ENXIO;
 707                }
 708        }
 709
 710        return 0;
 711}
 712
 713static int pnv_ioda_set_peltv(struct pnv_phb *phb,
 714                              struct pnv_ioda_pe *pe,
 715                              bool is_add)
 716{
 717        struct pnv_ioda_pe *slave;
 718        struct pci_dev *pdev = NULL;
 719        int ret;
 720
 721        /*
 722         * Clear PE frozen state. If it's master PE, we need
 723         * clear slave PE frozen state as well.
 724         */
 725        if (is_add) {
 726                opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
 727                                          OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 728                if (pe->flags & PNV_IODA_PE_MASTER) {
 729                        list_for_each_entry(slave, &pe->slaves, list)
 730                                opal_pci_eeh_freeze_clear(phb->opal_id,
 731                                                          slave->pe_number,
 732                                                          OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 733                }
 734        }
 735
 736        /*
 737         * Associate PE in PELT. We need add the PE into the
 738         * corresponding PELT-V as well. Otherwise, the error
 739         * originated from the PE might contribute to other
 740         * PEs.
 741         */
 742        ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
 743        if (ret)
 744                return ret;
 745
 746        /* For compound PEs, any one affects all of them */
 747        if (pe->flags & PNV_IODA_PE_MASTER) {
 748                list_for_each_entry(slave, &pe->slaves, list) {
 749                        ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
 750                        if (ret)
 751                                return ret;
 752                }
 753        }
 754
 755        if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
 756                pdev = pe->pbus->self;
 757        else if (pe->flags & PNV_IODA_PE_DEV)
 758                pdev = pe->pdev->bus->self;
 759#ifdef CONFIG_PCI_IOV
 760        else if (pe->flags & PNV_IODA_PE_VF)
 761                pdev = pe->parent_dev;
 762#endif /* CONFIG_PCI_IOV */
 763        while (pdev) {
 764                struct pci_dn *pdn = pci_get_pdn(pdev);
 765                struct pnv_ioda_pe *parent;
 766
 767                if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 768                        parent = &phb->ioda.pe_array[pdn->pe_number];
 769                        ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
 770                        if (ret)
 771                                return ret;
 772                }
 773
 774                pdev = pdev->bus->self;
 775        }
 776
 777        return 0;
 778}
 779
 780static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 781{
 782        struct pci_dev *parent;
 783        uint8_t bcomp, dcomp, fcomp;
 784        int64_t rc;
 785        long rid_end, rid;
 786
 787        /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
 788        if (pe->pbus) {
 789                int count;
 790
 791                dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 792                fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 793                parent = pe->pbus->self;
 794                if (pe->flags & PNV_IODA_PE_BUS_ALL)
 795                        count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 796                else
 797                        count = 1;
 798
 799                switch(count) {
 800                case  1: bcomp = OpalPciBusAll;         break;
 801                case  2: bcomp = OpalPciBus7Bits;       break;
 802                case  4: bcomp = OpalPciBus6Bits;       break;
 803                case  8: bcomp = OpalPciBus5Bits;       break;
 804                case 16: bcomp = OpalPciBus4Bits;       break;
 805                case 32: bcomp = OpalPciBus3Bits;       break;
 806                default:
 807                        dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 808                                count);
 809                        /* Do an exact match only */
 810                        bcomp = OpalPciBusAll;
 811                }
 812                rid_end = pe->rid + (count << 8);
 813        } else {
 814#ifdef CONFIG_PCI_IOV
 815                if (pe->flags & PNV_IODA_PE_VF)
 816                        parent = pe->parent_dev;
 817                else
 818#endif
 819                        parent = pe->pdev->bus->self;
 820                bcomp = OpalPciBusAll;
 821                dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 822                fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 823                rid_end = pe->rid + 1;
 824        }
 825
 826        /* Clear the reverse map */
 827        for (rid = pe->rid; rid < rid_end; rid++)
 828                phb->ioda.pe_rmap[rid] = IODA_INVALID_PE;
 829
 830        /* Release from all parents PELT-V */
 831        while (parent) {
 832                struct pci_dn *pdn = pci_get_pdn(parent);
 833                if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 834                        rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
 835                                                pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 836                        /* XXX What to do in case of error ? */
 837                }
 838                parent = parent->bus->self;
 839        }
 840
 841        opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
 842                                  OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 843
 844        /* Disassociate PE in PELT */
 845        rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
 846                                pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 847        if (rc)
 848                pe_warn(pe, "OPAL error %lld remove self from PELTV\n", rc);
 849        rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 850                             bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
 851        if (rc)
 852                pe_err(pe, "OPAL error %lld trying to setup PELT table\n", rc);
 853
 854        pe->pbus = NULL;
 855        pe->pdev = NULL;
 856#ifdef CONFIG_PCI_IOV
 857        pe->parent_dev = NULL;
 858#endif
 859
 860        return 0;
 861}
 862
 863static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 864{
 865        struct pci_dev *parent;
 866        uint8_t bcomp, dcomp, fcomp;
 867        long rc, rid_end, rid;
 868
 869        /* Bus validation ? */
 870        if (pe->pbus) {
 871                int count;
 872
 873                dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 874                fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 875                parent = pe->pbus->self;
 876                if (pe->flags & PNV_IODA_PE_BUS_ALL)
 877                        count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 878                else
 879                        count = 1;
 880
 881                switch(count) {
 882                case  1: bcomp = OpalPciBusAll;         break;
 883                case  2: bcomp = OpalPciBus7Bits;       break;
 884                case  4: bcomp = OpalPciBus6Bits;       break;
 885                case  8: bcomp = OpalPciBus5Bits;       break;
 886                case 16: bcomp = OpalPciBus4Bits;       break;
 887                case 32: bcomp = OpalPciBus3Bits;       break;
 888                default:
 889                        dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 890                                count);
 891                        /* Do an exact match only */
 892                        bcomp = OpalPciBusAll;
 893                }
 894                rid_end = pe->rid + (count << 8);
 895        } else {
 896#ifdef CONFIG_PCI_IOV
 897                if (pe->flags & PNV_IODA_PE_VF)
 898                        parent = pe->parent_dev;
 899                else
 900#endif /* CONFIG_PCI_IOV */
 901                        parent = pe->pdev->bus->self;
 902                bcomp = OpalPciBusAll;
 903                dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 904                fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 905                rid_end = pe->rid + 1;
 906        }
 907
 908        /*
 909         * Associate PE in PELT. We need add the PE into the
 910         * corresponding PELT-V as well. Otherwise, the error
 911         * originated from the PE might contribute to other
 912         * PEs.
 913         */
 914        rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 915                             bcomp, dcomp, fcomp, OPAL_MAP_PE);
 916        if (rc) {
 917                pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 918                return -ENXIO;
 919        }
 920
 921        /*
 922         * Configure PELTV. NPUs don't have a PELTV table so skip
 923         * configuration on them.
 924         */
 925        if (phb->type != PNV_PHB_NPU_NVLINK && phb->type != PNV_PHB_NPU_OCAPI)
 926                pnv_ioda_set_peltv(phb, pe, true);
 927
 928        /* Setup reverse map */
 929        for (rid = pe->rid; rid < rid_end; rid++)
 930                phb->ioda.pe_rmap[rid] = pe->pe_number;
 931
 932        /* Setup one MVTs on IODA1 */
 933        if (phb->type != PNV_PHB_IODA1) {
 934                pe->mve_number = 0;
 935                goto out;
 936        }
 937
 938        pe->mve_number = pe->pe_number;
 939        rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
 940        if (rc != OPAL_SUCCESS) {
 941                pe_err(pe, "OPAL error %ld setting up MVE %x\n",
 942                       rc, pe->mve_number);
 943                pe->mve_number = -1;
 944        } else {
 945                rc = opal_pci_set_mve_enable(phb->opal_id,
 946                                             pe->mve_number, OPAL_ENABLE_MVE);
 947                if (rc) {
 948                        pe_err(pe, "OPAL error %ld enabling MVE %x\n",
 949                               rc, pe->mve_number);
 950                        pe->mve_number = -1;
 951                }
 952        }
 953
 954out:
 955        return 0;
 956}
 957
 958#ifdef CONFIG_PCI_IOV
 959static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
 960{
 961        struct pci_dn *pdn = pci_get_pdn(dev);
 962        int i;
 963        struct resource *res, res2;
 964        resource_size_t size;
 965        u16 num_vfs;
 966
 967        if (!dev->is_physfn)
 968                return -EINVAL;
 969
 970        /*
 971         * "offset" is in VFs.  The M64 windows are sized so that when they
 972         * are segmented, each segment is the same size as the IOV BAR.
 973         * Each segment is in a separate PE, and the high order bits of the
 974         * address are the PE number.  Therefore, each VF's BAR is in a
 975         * separate PE, and changing the IOV BAR start address changes the
 976         * range of PEs the VFs are in.
 977         */
 978        num_vfs = pdn->num_vfs;
 979        for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 980                res = &dev->resource[i + PCI_IOV_RESOURCES];
 981                if (!res->flags || !res->parent)
 982                        continue;
 983
 984                /*
 985                 * The actual IOV BAR range is determined by the start address
 986                 * and the actual size for num_vfs VFs BAR.  This check is to
 987                 * make sure that after shifting, the range will not overlap
 988                 * with another device.
 989                 */
 990                size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 991                res2.flags = res->flags;
 992                res2.start = res->start + (size * offset);
 993                res2.end = res2.start + (size * num_vfs) - 1;
 994
 995                if (res2.end > res->end) {
 996                        dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
 997                                i, &res2, res, num_vfs, offset);
 998                        return -EBUSY;
 999                }
1000        }
1001
1002        /*
1003         * Since M64 BAR shares segments among all possible 256 PEs,
1004         * we have to shift the beginning of PF IOV BAR to make it start from
1005         * the segment which belongs to the PE number assigned to the first VF.
1006         * This creates a "hole" in the /proc/iomem which could be used for
1007         * allocating other resources so we reserve this area below and
1008         * release when IOV is released.
1009         */
1010        for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1011                res = &dev->resource[i + PCI_IOV_RESOURCES];
1012                if (!res->flags || !res->parent)
1013                        continue;
1014
1015                size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
1016                res2 = *res;
1017                res->start += size * offset;
1018
1019                dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (%sabling %d VFs shifted by %d)\n",
1020                         i, &res2, res, (offset > 0) ? "En" : "Dis",
1021                         num_vfs, offset);
1022
1023                if (offset < 0) {
1024                        devm_release_resource(&dev->dev, &pdn->holes[i]);
1025                        memset(&pdn->holes[i], 0, sizeof(pdn->holes[i]));
1026                }
1027
1028                pci_update_resource(dev, i + PCI_IOV_RESOURCES);
1029
1030                if (offset > 0) {
1031                        pdn->holes[i].start = res2.start;
1032                        pdn->holes[i].end = res2.start + size * offset - 1;
1033                        pdn->holes[i].flags = IORESOURCE_BUS;
1034                        pdn->holes[i].name = "pnv_iov_reserved";
1035                        devm_request_resource(&dev->dev, res->parent,
1036                                        &pdn->holes[i]);
1037                }
1038        }
1039        return 0;
1040}
1041#endif /* CONFIG_PCI_IOV */
1042
1043static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
1044{
1045        struct pci_controller *hose = pci_bus_to_host(dev->bus);
1046        struct pnv_phb *phb = hose->private_data;
1047        struct pci_dn *pdn = pci_get_pdn(dev);
1048        struct pnv_ioda_pe *pe;
1049
1050        if (!pdn) {
1051                pr_err("%s: Device tree node not associated properly\n",
1052                           pci_name(dev));
1053                return NULL;
1054        }
1055        if (pdn->pe_number != IODA_INVALID_PE)
1056                return NULL;
1057
1058        pe = pnv_ioda_alloc_pe(phb);
1059        if (!pe) {
1060                pr_warn("%s: Not enough PE# available, disabling device\n",
1061                        pci_name(dev));
1062                return NULL;
1063        }
1064
1065        /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
1066         * pointer in the PE data structure, both should be destroyed at the
1067         * same time. However, this needs to be looked at more closely again
1068         * once we actually start removing things (Hotplug, SR-IOV, ...)
1069         *
1070         * At some point we want to remove the PDN completely anyways
1071         */
1072        pci_dev_get(dev);
1073        pdn->pe_number = pe->pe_number;
1074        pe->flags = PNV_IODA_PE_DEV;
1075        pe->pdev = dev;
1076        pe->pbus = NULL;
1077        pe->mve_number = -1;
1078        pe->rid = dev->bus->number << 8 | pdn->devfn;
1079
1080        pe_info(pe, "Associated device to PE\n");
1081
1082        if (pnv_ioda_configure_pe(phb, pe)) {
1083                /* XXX What do we do here ? */
1084                pnv_ioda_free_pe(pe);
1085                pdn->pe_number = IODA_INVALID_PE;
1086                pe->pdev = NULL;
1087                pci_dev_put(dev);
1088                return NULL;
1089        }
1090
1091        /* Put PE to the list */
1092        list_add_tail(&pe->list, &phb->ioda.pe_list);
1093
1094        return pe;
1095}
1096
1097static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
1098{
1099        struct pci_dev *dev;
1100
1101        list_for_each_entry(dev, &bus->devices, bus_list) {
1102                struct pci_dn *pdn = pci_get_pdn(dev);
1103
1104                if (pdn == NULL) {
1105                        pr_warn("%s: No device node associated with device !\n",
1106                                pci_name(dev));
1107                        continue;
1108                }
1109
1110                /*
1111                 * In partial hotplug case, the PCI device might be still
1112                 * associated with the PE and needn't attach it to the PE
1113                 * again.
1114                 */
1115                if (pdn->pe_number != IODA_INVALID_PE)
1116                        continue;
1117
1118                pe->device_count++;
1119                pdn->pe_number = pe->pe_number;
1120                if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1121                        pnv_ioda_setup_same_PE(dev->subordinate, pe);
1122        }
1123}
1124
1125/*
1126 * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1127 * single PCI bus. Another one that contains the primary PCI bus and its
1128 * subordinate PCI devices and buses. The second type of PE is normally
1129 * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1130 */
1131static struct pnv_ioda_pe *pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all)
1132{
1133        struct pci_controller *hose = pci_bus_to_host(bus);
1134        struct pnv_phb *phb = hose->private_data;
1135        struct pnv_ioda_pe *pe = NULL;
1136        unsigned int pe_num;
1137
1138        /*
1139         * In partial hotplug case, the PE instance might be still alive.
1140         * We should reuse it instead of allocating a new one.
1141         */
1142        pe_num = phb->ioda.pe_rmap[bus->number << 8];
1143        if (pe_num != IODA_INVALID_PE) {
1144                pe = &phb->ioda.pe_array[pe_num];
1145                pnv_ioda_setup_same_PE(bus, pe);
1146                return NULL;
1147        }
1148
1149        /* PE number for root bus should have been reserved */
1150        if (pci_is_root_bus(bus) &&
1151            phb->ioda.root_pe_idx != IODA_INVALID_PE)
1152                pe = &phb->ioda.pe_array[phb->ioda.root_pe_idx];
1153
1154        /* Check if PE is determined by M64 */
1155        if (!pe)
1156                pe = pnv_ioda_pick_m64_pe(bus, all);
1157
1158        /* The PE number isn't pinned by M64 */
1159        if (!pe)
1160                pe = pnv_ioda_alloc_pe(phb);
1161
1162        if (!pe) {
1163                pr_warn("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1164                        __func__, pci_domain_nr(bus), bus->number);
1165                return NULL;
1166        }
1167
1168        pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
1169        pe->pbus = bus;
1170        pe->pdev = NULL;
1171        pe->mve_number = -1;
1172        pe->rid = bus->busn_res.start << 8;
1173
1174        if (all)
1175                pe_info(pe, "Secondary bus %pad..%pad associated with PE#%x\n",
1176                        &bus->busn_res.start, &bus->busn_res.end,
1177                        pe->pe_number);
1178        else
1179                pe_info(pe, "Secondary bus %pad associated with PE#%x\n",
1180                        &bus->busn_res.start, pe->pe_number);
1181
1182        if (pnv_ioda_configure_pe(phb, pe)) {
1183                /* XXX What do we do here ? */
1184                pnv_ioda_free_pe(pe);
1185                pe->pbus = NULL;
1186                return NULL;
1187        }
1188
1189        /* Associate it with all child devices */
1190        pnv_ioda_setup_same_PE(bus, pe);
1191
1192        /* Put PE to the list */
1193        list_add_tail(&pe->list, &phb->ioda.pe_list);
1194
1195        return pe;
1196}
1197
1198static struct pnv_ioda_pe *pnv_ioda_setup_npu_PE(struct pci_dev *npu_pdev)
1199{
1200        int pe_num, found_pe = false, rc;
1201        long rid;
1202        struct pnv_ioda_pe *pe;
1203        struct pci_dev *gpu_pdev;
1204        struct pci_dn *npu_pdn;
1205        struct pci_controller *hose = pci_bus_to_host(npu_pdev->bus);
1206        struct pnv_phb *phb = hose->private_data;
1207
1208        /*
1209         * Due to a hardware errata PE#0 on the NPU is reserved for
1210         * error handling. This means we only have three PEs remaining
1211         * which need to be assigned to four links, implying some
1212         * links must share PEs.
1213         *
1214         * To achieve this we assign PEs such that NPUs linking the
1215         * same GPU get assigned the same PE.
1216         */
1217        gpu_pdev = pnv_pci_get_gpu_dev(npu_pdev);
1218        for (pe_num = 0; pe_num < phb->ioda.total_pe_num; pe_num++) {
1219                pe = &phb->ioda.pe_array[pe_num];
1220                if (!pe->pdev)
1221                        continue;
1222
1223                if (pnv_pci_get_gpu_dev(pe->pdev) == gpu_pdev) {
1224                        /*
1225                         * This device has the same peer GPU so should
1226                         * be assigned the same PE as the existing
1227                         * peer NPU.
1228                         */
1229                        dev_info(&npu_pdev->dev,
1230                                "Associating to existing PE %x\n", pe_num);
1231                        pci_dev_get(npu_pdev);
1232                        npu_pdn = pci_get_pdn(npu_pdev);
1233                        rid = npu_pdev->bus->number << 8 | npu_pdn->devfn;
1234                        npu_pdn->pe_number = pe_num;
1235                        phb->ioda.pe_rmap[rid] = pe->pe_number;
1236
1237                        /* Map the PE to this link */
1238                        rc = opal_pci_set_pe(phb->opal_id, pe_num, rid,
1239                                        OpalPciBusAll,
1240                                        OPAL_COMPARE_RID_DEVICE_NUMBER,
1241                                        OPAL_COMPARE_RID_FUNCTION_NUMBER,
1242                                        OPAL_MAP_PE);
1243                        WARN_ON(rc != OPAL_SUCCESS);
1244                        found_pe = true;
1245                        break;
1246                }
1247        }
1248
1249        if (!found_pe)
1250                /*
1251                 * Could not find an existing PE so allocate a new
1252                 * one.
1253                 */
1254                return pnv_ioda_setup_dev_PE(npu_pdev);
1255        else
1256                return pe;
1257}
1258
1259static void pnv_ioda_setup_npu_PEs(struct pci_bus *bus)
1260{
1261        struct pci_dev *pdev;
1262
1263        list_for_each_entry(pdev, &bus->devices, bus_list)
1264                pnv_ioda_setup_npu_PE(pdev);
1265}
1266
1267static void pnv_pci_ioda_setup_PEs(void)
1268{
1269        struct pci_controller *hose;
1270        struct pnv_phb *phb;
1271        struct pci_bus *bus;
1272        struct pci_dev *pdev;
1273        struct pnv_ioda_pe *pe;
1274
1275        list_for_each_entry(hose, &hose_list, list_node) {
1276                phb = hose->private_data;
1277                if (phb->type == PNV_PHB_NPU_NVLINK) {
1278                        /* PE#0 is needed for error reporting */
1279                        pnv_ioda_reserve_pe(phb, 0);
1280                        pnv_ioda_setup_npu_PEs(hose->bus);
1281                        if (phb->model == PNV_PHB_MODEL_NPU2)
1282                                WARN_ON_ONCE(pnv_npu2_init(hose));
1283                }
1284                if (phb->type == PNV_PHB_NPU_OCAPI) {
1285                        bus = hose->bus;
1286                        list_for_each_entry(pdev, &bus->devices, bus_list)
1287                                pnv_ioda_setup_dev_PE(pdev);
1288                }
1289        }
1290        list_for_each_entry(hose, &hose_list, list_node) {
1291                phb = hose->private_data;
1292                if (phb->type != PNV_PHB_IODA2)
1293                        continue;
1294
1295                list_for_each_entry(pe, &phb->ioda.pe_list, list)
1296                        pnv_npu2_map_lpar(pe, MSR_DR | MSR_PR | MSR_HV);
1297        }
1298}
1299
1300#ifdef CONFIG_PCI_IOV
1301static int pnv_pci_vf_release_m64(struct pci_dev *pdev, u16 num_vfs)
1302{
1303        struct pci_bus        *bus;
1304        struct pci_controller *hose;
1305        struct pnv_phb        *phb;
1306        struct pci_dn         *pdn;
1307        int                    i, j;
1308        int                    m64_bars;
1309
1310        bus = pdev->bus;
1311        hose = pci_bus_to_host(bus);
1312        phb = hose->private_data;
1313        pdn = pci_get_pdn(pdev);
1314
1315        if (pdn->m64_single_mode)
1316                m64_bars = num_vfs;
1317        else
1318                m64_bars = 1;
1319
1320        for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1321                for (j = 0; j < m64_bars; j++) {
1322                        if (pdn->m64_map[j][i] == IODA_INVALID_M64)
1323                                continue;
1324                        opal_pci_phb_mmio_enable(phb->opal_id,
1325                                OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 0);
1326                        clear_bit(pdn->m64_map[j][i], &phb->ioda.m64_bar_alloc);
1327                        pdn->m64_map[j][i] = IODA_INVALID_M64;
1328                }
1329
1330        kfree(pdn->m64_map);
1331        return 0;
1332}
1333
1334static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
1335{
1336        struct pci_bus        *bus;
1337        struct pci_controller *hose;
1338        struct pnv_phb        *phb;
1339        struct pci_dn         *pdn;
1340        unsigned int           win;
1341        struct resource       *res;
1342        int                    i, j;
1343        int64_t                rc;
1344        int                    total_vfs;
1345        resource_size_t        size, start;
1346        int                    pe_num;
1347        int                    m64_bars;
1348
1349        bus = pdev->bus;
1350        hose = pci_bus_to_host(bus);
1351        phb = hose->private_data;
1352        pdn = pci_get_pdn(pdev);
1353        total_vfs = pci_sriov_get_totalvfs(pdev);
1354
1355        if (pdn->m64_single_mode)
1356                m64_bars = num_vfs;
1357        else
1358                m64_bars = 1;
1359
1360        pdn->m64_map = kmalloc_array(m64_bars,
1361                                     sizeof(*pdn->m64_map),
1362                                     GFP_KERNEL);
1363        if (!pdn->m64_map)
1364                return -ENOMEM;
1365        /* Initialize the m64_map to IODA_INVALID_M64 */
1366        for (i = 0; i < m64_bars ; i++)
1367                for (j = 0; j < PCI_SRIOV_NUM_BARS; j++)
1368                        pdn->m64_map[i][j] = IODA_INVALID_M64;
1369
1370
1371        for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1372                res = &pdev->resource[i + PCI_IOV_RESOURCES];
1373                if (!res->flags || !res->parent)
1374                        continue;
1375
1376                for (j = 0; j < m64_bars; j++) {
1377                        do {
1378                                win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1379                                                phb->ioda.m64_bar_idx + 1, 0);
1380
1381                                if (win >= phb->ioda.m64_bar_idx + 1)
1382                                        goto m64_failed;
1383                        } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
1384
1385                        pdn->m64_map[j][i] = win;
1386
1387                        if (pdn->m64_single_mode) {
1388                                size = pci_iov_resource_size(pdev,
1389                                                        PCI_IOV_RESOURCES + i);
1390                                start = res->start + size * j;
1391                        } else {
1392                                size = resource_size(res);
1393                                start = res->start;
1394                        }
1395
1396                        /* Map the M64 here */
1397                        if (pdn->m64_single_mode) {
1398                                pe_num = pdn->pe_num_map[j];
1399                                rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1400                                                pe_num, OPAL_M64_WINDOW_TYPE,
1401                                                pdn->m64_map[j][i], 0);
1402                        }
1403
1404                        rc = opal_pci_set_phb_mem_window(phb->opal_id,
1405                                                 OPAL_M64_WINDOW_TYPE,
1406                                                 pdn->m64_map[j][i],
1407                                                 start,
1408                                                 0, /* unused */
1409                                                 size);
1410
1411
1412                        if (rc != OPAL_SUCCESS) {
1413                                dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1414                                        win, rc);
1415                                goto m64_failed;
1416                        }
1417
1418                        if (pdn->m64_single_mode)
1419                                rc = opal_pci_phb_mmio_enable(phb->opal_id,
1420                                     OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 2);
1421                        else
1422                                rc = opal_pci_phb_mmio_enable(phb->opal_id,
1423                                     OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 1);
1424
1425                        if (rc != OPAL_SUCCESS) {
1426                                dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1427                                        win, rc);
1428                                goto m64_failed;
1429                        }
1430                }
1431        }
1432        return 0;
1433
1434m64_failed:
1435        pnv_pci_vf_release_m64(pdev, num_vfs);
1436        return -EBUSY;
1437}
1438
1439static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
1440                int num);
1441
1442static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1443{
1444        struct iommu_table    *tbl;
1445        int64_t               rc;
1446
1447        tbl = pe->table_group.tables[0];
1448        rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
1449        if (rc)
1450                pe_warn(pe, "OPAL error %lld release DMA window\n", rc);
1451
1452        pnv_pci_ioda2_set_bypass(pe, false);
1453        if (pe->table_group.group) {
1454                iommu_group_put(pe->table_group.group);
1455                BUG_ON(pe->table_group.group);
1456        }
1457        iommu_tce_table_put(tbl);
1458}
1459
1460static void pnv_ioda_release_vf_PE(struct pci_dev *pdev)
1461{
1462        struct pci_bus        *bus;
1463        struct pci_controller *hose;
1464        struct pnv_phb        *phb;
1465        struct pnv_ioda_pe    *pe, *pe_n;
1466        struct pci_dn         *pdn;
1467
1468        bus = pdev->bus;
1469        hose = pci_bus_to_host(bus);
1470        phb = hose->private_data;
1471        pdn = pci_get_pdn(pdev);
1472
1473        if (!pdev->is_physfn)
1474                return;
1475
1476        list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1477                if (pe->parent_dev != pdev)
1478                        continue;
1479
1480                pnv_pci_ioda2_release_dma_pe(pdev, pe);
1481
1482                /* Remove from list */
1483                mutex_lock(&phb->ioda.pe_list_mutex);
1484                list_del(&pe->list);
1485                mutex_unlock(&phb->ioda.pe_list_mutex);
1486
1487                pnv_ioda_deconfigure_pe(phb, pe);
1488
1489                pnv_ioda_free_pe(pe);
1490        }
1491}
1492
1493void pnv_pci_sriov_disable(struct pci_dev *pdev)
1494{
1495        struct pci_bus        *bus;
1496        struct pci_controller *hose;
1497        struct pnv_phb        *phb;
1498        struct pnv_ioda_pe    *pe;
1499        struct pci_dn         *pdn;
1500        u16                    num_vfs, i;
1501
1502        bus = pdev->bus;
1503        hose = pci_bus_to_host(bus);
1504        phb = hose->private_data;
1505        pdn = pci_get_pdn(pdev);
1506        num_vfs = pdn->num_vfs;
1507
1508        /* Release VF PEs */
1509        pnv_ioda_release_vf_PE(pdev);
1510
1511        if (phb->type == PNV_PHB_IODA2) {
1512                if (!pdn->m64_single_mode)
1513                        pnv_pci_vf_resource_shift(pdev, -*pdn->pe_num_map);
1514
1515                /* Release M64 windows */
1516                pnv_pci_vf_release_m64(pdev, num_vfs);
1517
1518                /* Release PE numbers */
1519                if (pdn->m64_single_mode) {
1520                        for (i = 0; i < num_vfs; i++) {
1521                                if (pdn->pe_num_map[i] == IODA_INVALID_PE)
1522                                        continue;
1523
1524                                pe = &phb->ioda.pe_array[pdn->pe_num_map[i]];
1525                                pnv_ioda_free_pe(pe);
1526                        }
1527                } else
1528                        bitmap_clear(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
1529                /* Releasing pe_num_map */
1530                kfree(pdn->pe_num_map);
1531        }
1532}
1533
1534static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1535                                       struct pnv_ioda_pe *pe);
1536#ifdef CONFIG_IOMMU_API
1537static void pnv_ioda_setup_bus_iommu_group(struct pnv_ioda_pe *pe,
1538                struct iommu_table_group *table_group, struct pci_bus *bus);
1539
1540#endif
1541static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1542{
1543        struct pci_bus        *bus;
1544        struct pci_controller *hose;
1545        struct pnv_phb        *phb;
1546        struct pnv_ioda_pe    *pe;
1547        int                    pe_num;
1548        u16                    vf_index;
1549        struct pci_dn         *pdn;
1550
1551        bus = pdev->bus;
1552        hose = pci_bus_to_host(bus);
1553        phb = hose->private_data;
1554        pdn = pci_get_pdn(pdev);
1555
1556        if (!pdev->is_physfn)
1557                return;
1558
1559        /* Reserve PE for each VF */
1560        for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1561                if (pdn->m64_single_mode)
1562                        pe_num = pdn->pe_num_map[vf_index];
1563                else
1564                        pe_num = *pdn->pe_num_map + vf_index;
1565
1566                pe = &phb->ioda.pe_array[pe_num];
1567                pe->pe_number = pe_num;
1568                pe->phb = phb;
1569                pe->flags = PNV_IODA_PE_VF;
1570                pe->pbus = NULL;
1571                pe->parent_dev = pdev;
1572                pe->mve_number = -1;
1573                pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1574                           pci_iov_virtfn_devfn(pdev, vf_index);
1575
1576                pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%x\n",
1577                        hose->global_number, pdev->bus->number,
1578                        PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1579                        PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1580
1581                if (pnv_ioda_configure_pe(phb, pe)) {
1582                        /* XXX What do we do here ? */
1583                        pnv_ioda_free_pe(pe);
1584                        pe->pdev = NULL;
1585                        continue;
1586                }
1587
1588                /* Put PE to the list */
1589                mutex_lock(&phb->ioda.pe_list_mutex);
1590                list_add_tail(&pe->list, &phb->ioda.pe_list);
1591                mutex_unlock(&phb->ioda.pe_list_mutex);
1592
1593                pnv_pci_ioda2_setup_dma_pe(phb, pe);
1594#ifdef CONFIG_IOMMU_API
1595                iommu_register_group(&pe->table_group,
1596                                pe->phb->hose->global_number, pe->pe_number);
1597                pnv_ioda_setup_bus_iommu_group(pe, &pe->table_group, NULL);
1598#endif
1599        }
1600}
1601
1602int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1603{
1604        struct pci_bus        *bus;
1605        struct pci_controller *hose;
1606        struct pnv_phb        *phb;
1607        struct pnv_ioda_pe    *pe;
1608        struct pci_dn         *pdn;
1609        int                    ret;
1610        u16                    i;
1611
1612        bus = pdev->bus;
1613        hose = pci_bus_to_host(bus);
1614        phb = hose->private_data;
1615        pdn = pci_get_pdn(pdev);
1616
1617        if (phb->type == PNV_PHB_IODA2) {
1618                if (!pdn->vfs_expanded) {
1619                        dev_info(&pdev->dev, "don't support this SRIOV device"
1620                                " with non 64bit-prefetchable IOV BAR\n");
1621                        return -ENOSPC;
1622                }
1623
1624                /*
1625                 * When M64 BARs functions in Single PE mode, the number of VFs
1626                 * could be enabled must be less than the number of M64 BARs.
1627                 */
1628                if (pdn->m64_single_mode && num_vfs > phb->ioda.m64_bar_idx) {
1629                        dev_info(&pdev->dev, "Not enough M64 BAR for VFs\n");
1630                        return -EBUSY;
1631                }
1632
1633                /* Allocating pe_num_map */
1634                if (pdn->m64_single_mode)
1635                        pdn->pe_num_map = kmalloc_array(num_vfs,
1636                                                        sizeof(*pdn->pe_num_map),
1637                                                        GFP_KERNEL);
1638                else
1639                        pdn->pe_num_map = kmalloc(sizeof(*pdn->pe_num_map), GFP_KERNEL);
1640
1641                if (!pdn->pe_num_map)
1642                        return -ENOMEM;
1643
1644                if (pdn->m64_single_mode)
1645                        for (i = 0; i < num_vfs; i++)
1646                                pdn->pe_num_map[i] = IODA_INVALID_PE;
1647
1648                /* Calculate available PE for required VFs */
1649                if (pdn->m64_single_mode) {
1650                        for (i = 0; i < num_vfs; i++) {
1651                                pe = pnv_ioda_alloc_pe(phb);
1652                                if (!pe) {
1653                                        ret = -EBUSY;
1654                                        goto m64_failed;
1655                                }
1656
1657                                pdn->pe_num_map[i] = pe->pe_number;
1658                        }
1659                } else {
1660                        mutex_lock(&phb->ioda.pe_alloc_mutex);
1661                        *pdn->pe_num_map = bitmap_find_next_zero_area(
1662                                phb->ioda.pe_alloc, phb->ioda.total_pe_num,
1663                                0, num_vfs, 0);
1664                        if (*pdn->pe_num_map >= phb->ioda.total_pe_num) {
1665                                mutex_unlock(&phb->ioda.pe_alloc_mutex);
1666                                dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1667                                kfree(pdn->pe_num_map);
1668                                return -EBUSY;
1669                        }
1670                        bitmap_set(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
1671                        mutex_unlock(&phb->ioda.pe_alloc_mutex);
1672                }
1673                pdn->num_vfs = num_vfs;
1674
1675                /* Assign M64 window accordingly */
1676                ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
1677                if (ret) {
1678                        dev_info(&pdev->dev, "Not enough M64 window resources\n");
1679                        goto m64_failed;
1680                }
1681
1682                /*
1683                 * When using one M64 BAR to map one IOV BAR, we need to shift
1684                 * the IOV BAR according to the PE# allocated to the VFs.
1685                 * Otherwise, the PE# for the VF will conflict with others.
1686                 */
1687                if (!pdn->m64_single_mode) {
1688                        ret = pnv_pci_vf_resource_shift(pdev, *pdn->pe_num_map);
1689                        if (ret)
1690                                goto m64_failed;
1691                }
1692        }
1693
1694        /* Setup VF PEs */
1695        pnv_ioda_setup_vf_PE(pdev, num_vfs);
1696
1697        return 0;
1698
1699m64_failed:
1700        if (pdn->m64_single_mode) {
1701                for (i = 0; i < num_vfs; i++) {
1702                        if (pdn->pe_num_map[i] == IODA_INVALID_PE)
1703                                continue;
1704
1705                        pe = &phb->ioda.pe_array[pdn->pe_num_map[i]];
1706                        pnv_ioda_free_pe(pe);
1707                }
1708        } else
1709                bitmap_clear(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
1710
1711        /* Releasing pe_num_map */
1712        kfree(pdn->pe_num_map);
1713
1714        return ret;
1715}
1716
1717int pnv_pcibios_sriov_disable(struct pci_dev *pdev)
1718{
1719        pnv_pci_sriov_disable(pdev);
1720
1721        /* Release PCI data */
1722        remove_dev_pci_data(pdev);
1723        return 0;
1724}
1725
1726int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1727{
1728        /* Allocate PCI data */
1729        add_dev_pci_data(pdev);
1730
1731        return pnv_pci_sriov_enable(pdev, num_vfs);
1732}
1733#endif /* CONFIG_PCI_IOV */
1734
1735static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
1736{
1737        struct pci_dn *pdn = pci_get_pdn(pdev);
1738        struct pnv_ioda_pe *pe;
1739
1740        /*
1741         * The function can be called while the PE#
1742         * hasn't been assigned. Do nothing for the
1743         * case.
1744         */
1745        if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1746                return;
1747
1748        pe = &phb->ioda.pe_array[pdn->pe_number];
1749        WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
1750        pdev->dev.archdata.dma_offset = pe->tce_bypass_base;
1751        set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
1752        /*
1753         * Note: iommu_add_device() will fail here as
1754         * for physical PE: the device is already added by now;
1755         * for virtual PE: sysfs entries are not ready yet and
1756         * tce_iommu_bus_notifier will add the device to a group later.
1757         */
1758}
1759
1760/*
1761 * Reconfigure TVE#0 to be usable as 64-bit DMA space.
1762 *
1763 * The first 4GB of virtual memory for a PE is reserved for 32-bit accesses.
1764 * Devices can only access more than that if bit 59 of the PCI address is set
1765 * by hardware, which indicates TVE#1 should be used instead of TVE#0.
1766 * Many PCI devices are not capable of addressing that many bits, and as a
1767 * result are limited to the 4GB of virtual memory made available to 32-bit
1768 * devices in TVE#0.
1769 *
1770 * In order to work around this, reconfigure TVE#0 to be suitable for 64-bit
1771 * devices by configuring the virtual memory past the first 4GB inaccessible
1772 * by 64-bit DMAs.  This should only be used by devices that want more than
1773 * 4GB, and only on PEs that have no 32-bit devices.
1774 *
1775 * Currently this will only work on PHB3 (POWER8).
1776 */
1777static int pnv_pci_ioda_dma_64bit_bypass(struct pnv_ioda_pe *pe)
1778{
1779        u64 window_size, table_size, tce_count, addr;
1780        struct page *table_pages;
1781        u64 tce_order = 28; /* 256MB TCEs */
1782        __be64 *tces;
1783        s64 rc;
1784
1785        /*
1786         * Window size needs to be a power of two, but needs to account for
1787         * shifting memory by the 4GB offset required to skip 32bit space.
1788         */
1789        window_size = roundup_pow_of_two(memory_hotplug_max() + (1ULL << 32));
1790        tce_count = window_size >> tce_order;
1791        table_size = tce_count << 3;
1792
1793        if (table_size < PAGE_SIZE)
1794                table_size = PAGE_SIZE;
1795
1796        table_pages = alloc_pages_node(pe->phb->hose->node, GFP_KERNEL,
1797                                       get_order(table_size));
1798        if (!table_pages)
1799                goto err;
1800
1801        tces = page_address(table_pages);
1802        if (!tces)
1803                goto err;
1804
1805        memset(tces, 0, table_size);
1806
1807        for (addr = 0; addr < memory_hotplug_max(); addr += (1 << tce_order)) {
1808                tces[(addr + (1ULL << 32)) >> tce_order] =
1809                        cpu_to_be64(addr | TCE_PCI_READ | TCE_PCI_WRITE);
1810        }
1811
1812        rc = opal_pci_map_pe_dma_window(pe->phb->opal_id,
1813                                        pe->pe_number,
1814                                        /* reconfigure window 0 */
1815                                        (pe->pe_number << 1) + 0,
1816                                        1,
1817                                        __pa(tces),
1818                                        table_size,
1819                                        1 << tce_order);
1820        if (rc == OPAL_SUCCESS) {
1821                pe_info(pe, "Using 64-bit DMA iommu bypass (through TVE#0)\n");
1822                return 0;
1823        }
1824err:
1825        pe_err(pe, "Error configuring 64-bit DMA bypass\n");
1826        return -EIO;
1827}
1828
1829static bool pnv_pci_ioda_iommu_bypass_supported(struct pci_dev *pdev,
1830                u64 dma_mask)
1831{
1832        struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1833        struct pnv_phb *phb = hose->private_data;
1834        struct pci_dn *pdn = pci_get_pdn(pdev);
1835        struct pnv_ioda_pe *pe;
1836
1837        if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1838                return false;
1839
1840        pe = &phb->ioda.pe_array[pdn->pe_number];
1841        if (pe->tce_bypass_enabled) {
1842                u64 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1843                if (dma_mask >= top)
1844                        return true;
1845        }
1846
1847        /*
1848         * If the device can't set the TCE bypass bit but still wants
1849         * to access 4GB or more, on PHB3 we can reconfigure TVE#0 to
1850         * bypass the 32-bit region and be usable for 64-bit DMAs.
1851         * The device needs to be able to address all of this space.
1852         */
1853        if (dma_mask >> 32 &&
1854            dma_mask > (memory_hotplug_max() + (1ULL << 32)) &&
1855            /* pe->pdev should be set if it's a single device, pe->pbus if not */
1856            (pe->device_count == 1 || !pe->pbus) &&
1857            phb->model == PNV_PHB_MODEL_PHB3) {
1858                /* Configure the bypass mode */
1859                s64 rc = pnv_pci_ioda_dma_64bit_bypass(pe);
1860                if (rc)
1861                        return false;
1862                /* 4GB offset bypasses 32-bit space */
1863                pdev->dev.archdata.dma_offset = (1ULL << 32);
1864                return true;
1865        }
1866
1867        return false;
1868}
1869
1870static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, struct pci_bus *bus)
1871{
1872        struct pci_dev *dev;
1873
1874        list_for_each_entry(dev, &bus->devices, bus_list) {
1875                set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
1876                dev->dev.archdata.dma_offset = pe->tce_bypass_base;
1877
1878                if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1879                        pnv_ioda_setup_bus_dma(pe, dev->subordinate);
1880        }
1881}
1882
1883static inline __be64 __iomem *pnv_ioda_get_inval_reg(struct pnv_phb *phb,
1884                                                     bool real_mode)
1885{
1886        return real_mode ? (__be64 __iomem *)(phb->regs_phys + 0x210) :
1887                (phb->regs + 0x210);
1888}
1889
1890static void pnv_pci_p7ioc_tce_invalidate(struct iommu_table *tbl,
1891                unsigned long index, unsigned long npages, bool rm)
1892{
1893        struct iommu_table_group_link *tgl = list_first_entry_or_null(
1894                        &tbl->it_group_list, struct iommu_table_group_link,
1895                        next);
1896        struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1897                        struct pnv_ioda_pe, table_group);
1898        __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb, rm);
1899        unsigned long start, end, inc;
1900
1901        start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1902        end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1903                        npages - 1);
1904
1905        /* p7ioc-style invalidation, 2 TCEs per write */
1906        start |= (1ull << 63);
1907        end |= (1ull << 63);
1908        inc = 16;
1909        end |= inc - 1; /* round up end to be different than start */
1910
1911        mb(); /* Ensure above stores are visible */
1912        while (start <= end) {
1913                if (rm)
1914                        __raw_rm_writeq_be(start, invalidate);
1915                else
1916                        __raw_writeq_be(start, invalidate);
1917
1918                start += inc;
1919        }
1920
1921        /*
1922         * The iommu layer will do another mb() for us on build()
1923         * and we don't care on free()
1924         */
1925}
1926
1927static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1928                long npages, unsigned long uaddr,
1929                enum dma_data_direction direction,
1930                unsigned long attrs)
1931{
1932        int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1933                        attrs);
1934
1935        if (!ret)
1936                pnv_pci_p7ioc_tce_invalidate(tbl, index, npages, false);
1937
1938        return ret;
1939}
1940
1941#ifdef CONFIG_IOMMU_API
1942/* Common for IODA1 and IODA2 */
1943static int pnv_ioda_tce_xchg_no_kill(struct iommu_table *tbl, long index,
1944                unsigned long *hpa, enum dma_data_direction *direction,
1945                bool realmode)
1946{
1947        return pnv_tce_xchg(tbl, index, hpa, direction, !realmode);
1948}
1949#endif
1950
1951static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1952                long npages)
1953{
1954        pnv_tce_free(tbl, index, npages);
1955
1956        pnv_pci_p7ioc_tce_invalidate(tbl, index, npages, false);
1957}
1958
1959static struct iommu_table_ops pnv_ioda1_iommu_ops = {
1960        .set = pnv_ioda1_tce_build,
1961#ifdef CONFIG_IOMMU_API
1962        .xchg_no_kill = pnv_ioda_tce_xchg_no_kill,
1963        .tce_kill = pnv_pci_p7ioc_tce_invalidate,
1964        .useraddrptr = pnv_tce_useraddrptr,
1965#endif
1966        .clear = pnv_ioda1_tce_free,
1967        .get = pnv_tce_get,
1968};
1969
1970#define PHB3_TCE_KILL_INVAL_ALL         PPC_BIT(0)
1971#define PHB3_TCE_KILL_INVAL_PE          PPC_BIT(1)
1972#define PHB3_TCE_KILL_INVAL_ONE         PPC_BIT(2)
1973
1974static void pnv_pci_phb3_tce_invalidate_entire(struct pnv_phb *phb, bool rm)
1975{
1976        __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(phb, rm);
1977        const unsigned long val = PHB3_TCE_KILL_INVAL_ALL;
1978
1979        mb(); /* Ensure previous TCE table stores are visible */
1980        if (rm)
1981                __raw_rm_writeq_be(val, invalidate);
1982        else
1983                __raw_writeq_be(val, invalidate);
1984}
1985
1986static inline void pnv_pci_phb3_tce_invalidate_pe(struct pnv_ioda_pe *pe)
1987{
1988        /* 01xb - invalidate TCEs that match the specified PE# */
1989        __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb, false);
1990        unsigned long val = PHB3_TCE_KILL_INVAL_PE | (pe->pe_number & 0xFF);
1991
1992        mb(); /* Ensure above stores are visible */
1993        __raw_writeq_be(val, invalidate);
1994}
1995
1996static void pnv_pci_phb3_tce_invalidate(struct pnv_ioda_pe *pe, bool rm,
1997                                        unsigned shift, unsigned long index,
1998                                        unsigned long npages)
1999{
2000        __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb, rm);
2001        unsigned long start, end, inc;
2002
2003        /* We'll invalidate DMA address in PE scope */
2004        start = PHB3_TCE_KILL_INVAL_ONE;
2005        start |= (pe->pe_number & 0xFF);
2006        end = start;
2007
2008        /* Figure out the start, end and step */
2009        start |= (index << shift);
2010        end |= ((index + npages - 1) << shift);
2011        inc = (0x1ull << shift);
2012        mb();
2013
2014        while (start <= end) {
2015                if (rm)
2016                        __raw_rm_writeq_be(start, invalidate);
2017                else
2018                        __raw_writeq_be(start, invalidate);
2019                start += inc;
2020        }
2021}
2022
2023static inline void pnv_pci_ioda2_tce_invalidate_pe(struct pnv_ioda_pe *pe)
2024{
2025        struct pnv_phb *phb = pe->phb;
2026
2027        if (phb->model == PNV_PHB_MODEL_PHB3 && phb->regs)
2028                pnv_pci_phb3_tce_invalidate_pe(pe);
2029        else
2030                opal_pci_tce_kill(phb->opal_id, OPAL_PCI_TCE_KILL_PE,
2031                                  pe->pe_number, 0, 0, 0);
2032}
2033
2034static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
2035                unsigned long index, unsigned long npages, bool rm)
2036{
2037        struct iommu_table_group_link *tgl;
2038
2039        list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) {
2040                struct pnv_ioda_pe *pe = container_of(tgl->table_group,
2041                                struct pnv_ioda_pe, table_group);
2042                struct pnv_phb *phb = pe->phb;
2043                unsigned int shift = tbl->it_page_shift;
2044
2045                /*
2046                 * NVLink1 can use the TCE kill register directly as
2047                 * it's the same as PHB3. NVLink2 is different and
2048                 * should go via the OPAL call.
2049                 */
2050                if (phb->model == PNV_PHB_MODEL_NPU) {
2051                        /*
2052                         * The NVLink hardware does not support TCE kill
2053                         * per TCE entry so we have to invalidate
2054                         * the entire cache for it.
2055                         */
2056                        pnv_pci_phb3_tce_invalidate_entire(phb, rm);
2057                        continue;
2058                }
2059                if (phb->model == PNV_PHB_MODEL_PHB3 && phb->regs)
2060                        pnv_pci_phb3_tce_invalidate(pe, rm, shift,
2061                                                    index, npages);
2062                else
2063                        opal_pci_tce_kill(phb->opal_id,
2064                                          OPAL_PCI_TCE_KILL_PAGES,
2065                                          pe->pe_number, 1u << shift,
2066                                          index << shift, npages);
2067        }
2068}
2069
2070void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_phb *phb, bool rm)
2071{
2072        if (phb->model == PNV_PHB_MODEL_NPU || phb->model == PNV_PHB_MODEL_PHB3)
2073                pnv_pci_phb3_tce_invalidate_entire(phb, rm);
2074        else
2075                opal_pci_tce_kill(phb->opal_id, OPAL_PCI_TCE_KILL, 0, 0, 0, 0);
2076}
2077
2078static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
2079                long npages, unsigned long uaddr,
2080                enum dma_data_direction direction,
2081                unsigned long attrs)
2082{
2083        int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
2084                        attrs);
2085
2086        if (!ret)
2087                pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
2088
2089        return ret;
2090}
2091
2092static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
2093                long npages)
2094{
2095        pnv_tce_free(tbl, index, npages);
2096
2097        pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
2098}
2099
2100static struct iommu_table_ops pnv_ioda2_iommu_ops = {
2101        .set = pnv_ioda2_tce_build,
2102#ifdef CONFIG_IOMMU_API
2103        .xchg_no_kill = pnv_ioda_tce_xchg_no_kill,
2104        .tce_kill = pnv_pci_ioda2_tce_invalidate,
2105        .useraddrptr = pnv_tce_useraddrptr,
2106#endif
2107        .clear = pnv_ioda2_tce_free,
2108        .get = pnv_tce_get,
2109        .free = pnv_pci_ioda2_table_free_pages,
2110};
2111
2112static int pnv_pci_ioda_dev_dma_weight(struct pci_dev *dev, void *data)
2113{
2114        unsigned int *weight = (unsigned int *)data;
2115
2116        /* This is quite simplistic. The "base" weight of a device
2117         * is 10. 0 means no DMA is to be accounted for it.
2118         */
2119        if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
2120                return 0;
2121
2122        if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
2123            dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
2124            dev->class == PCI_CLASS_SERIAL_USB_EHCI)
2125                *weight += 3;
2126        else if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
2127                *weight += 15;
2128        else
2129                *weight += 10;
2130
2131        return 0;
2132}
2133
2134static unsigned int pnv_pci_ioda_pe_dma_weight(struct pnv_ioda_pe *pe)
2135{
2136        unsigned int weight = 0;
2137
2138        /* SRIOV VF has same DMA32 weight as its PF */
2139#ifdef CONFIG_PCI_IOV
2140        if ((pe->flags & PNV_IODA_PE_VF) && pe->parent_dev) {
2141                pnv_pci_ioda_dev_dma_weight(pe->parent_dev, &weight);
2142                return weight;
2143        }
2144#endif
2145
2146        if ((pe->flags & PNV_IODA_PE_DEV) && pe->pdev) {
2147                pnv_pci_ioda_dev_dma_weight(pe->pdev, &weight);
2148        } else if ((pe->flags & PNV_IODA_PE_BUS) && pe->pbus) {
2149                struct pci_dev *pdev;
2150
2151                list_for_each_entry(pdev, &pe->pbus->devices, bus_list)
2152                        pnv_pci_ioda_dev_dma_weight(pdev, &weight);
2153        } else if ((pe->flags & PNV_IODA_PE_BUS_ALL) && pe->pbus) {
2154                pci_walk_bus(pe->pbus, pnv_pci_ioda_dev_dma_weight, &weight);
2155        }
2156
2157        return weight;
2158}
2159
2160static void pnv_pci_ioda1_setup_dma_pe(struct pnv_phb *phb,
2161                                       struct pnv_ioda_pe *pe)
2162{
2163
2164        struct page *tce_mem = NULL;
2165        struct iommu_table *tbl;
2166        unsigned int weight, total_weight = 0;
2167        unsigned int tce32_segsz, base, segs, avail, i;
2168        int64_t rc;
2169        void *addr;
2170
2171        /* XXX FIXME: Handle 64-bit only DMA devices */
2172        /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
2173        /* XXX FIXME: Allocate multi-level tables on PHB3 */
2174        weight = pnv_pci_ioda_pe_dma_weight(pe);
2175        if (!weight)
2176                return;
2177
2178        pci_walk_bus(phb->hose->bus, pnv_pci_ioda_dev_dma_weight,
2179                     &total_weight);
2180        segs = (weight * phb->ioda.dma32_count) / total_weight;
2181        if (!segs)
2182                segs = 1;
2183
2184        /*
2185         * Allocate contiguous DMA32 segments. We begin with the expected
2186         * number of segments. With one more attempt, the number of DMA32
2187         * segments to be allocated is decreased by one until one segment
2188         * is allocated successfully.
2189         */
2190        do {
2191                for (base = 0; base <= phb->ioda.dma32_count - segs; base++) {
2192                        for (avail = 0, i = base; i < base + segs; i++) {
2193                                if (phb->ioda.dma32_segmap[i] ==
2194                                    IODA_INVALID_PE)
2195                                        avail++;
2196                        }
2197
2198                        if (avail == segs)
2199                                goto found;
2200                }
2201        } while (--segs);
2202
2203        if (!segs) {
2204                pe_warn(pe, "No available DMA32 segments\n");
2205                return;
2206        }
2207
2208found:
2209        tbl = pnv_pci_table_alloc(phb->hose->node);
2210        if (WARN_ON(!tbl))
2211                return;
2212
2213        iommu_register_group(&pe->table_group, phb->hose->global_number,
2214                        pe->pe_number);
2215        pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
2216
2217        /* Grab a 32-bit TCE table */
2218        pe_info(pe, "DMA weight %d (%d), assigned (%d) %d DMA32 segments\n",
2219                weight, total_weight, base, segs);
2220        pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
2221                base * PNV_IODA1_DMA32_SEGSIZE,
2222                (base + segs) * PNV_IODA1_DMA32_SEGSIZE - 1);
2223
2224        /* XXX Currently, we allocate one big contiguous table for the
2225         * TCEs. We only really need one chunk per 256M of TCE space
2226         * (ie per segment) but that's an optimization for later, it
2227         * requires some added smarts with our get/put_tce implementation
2228         *
2229         * Each TCE page is 4KB in size and each TCE entry occupies 8
2230         * bytes
2231         */
2232        tce32_segsz = PNV_IODA1_DMA32_SEGSIZE >> (IOMMU_PAGE_SHIFT_4K - 3);
2233        tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
2234                                   get_order(tce32_segsz * segs));
2235        if (!tce_mem) {
2236                pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
2237                goto fail;
2238        }
2239        addr = page_address(tce_mem);
2240        memset(addr, 0, tce32_segsz * segs);
2241
2242        /* Configure HW */
2243        for (i = 0; i < segs; i++) {
2244                rc = opal_pci_map_pe_dma_window(phb->opal_id,
2245                                              pe->pe_number,
2246                                              base + i, 1,
2247                                              __pa(addr) + tce32_segsz * i,
2248                                              tce32_segsz, IOMMU_PAGE_SIZE_4K);
2249                if (rc) {
2250                        pe_err(pe, " Failed to configure 32-bit TCE table, err %lld\n",
2251                               rc);
2252                        goto fail;
2253                }
2254        }
2255
2256        /* Setup DMA32 segment mapping */
2257        for (i = base; i < base + segs; i++)
2258                phb->ioda.dma32_segmap[i] = pe->pe_number;
2259
2260        /* Setup linux iommu table */
2261        pnv_pci_setup_iommu_table(tbl, addr, tce32_segsz * segs,
2262                                  base * PNV_IODA1_DMA32_SEGSIZE,
2263                                  IOMMU_PAGE_SHIFT_4K);
2264
2265        tbl->it_ops = &pnv_ioda1_iommu_ops;
2266        pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
2267        pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
2268        iommu_init_table(tbl, phb->hose->node, 0, 0);
2269
2270        if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
2271                pnv_ioda_setup_bus_dma(pe, pe->pbus);
2272
2273        return;
2274 fail:
2275        /* XXX Failure: Try to fallback to 64-bit only ? */
2276        if (tce_mem)
2277                __free_pages(tce_mem, get_order(tce32_segsz * segs));
2278        if (tbl) {
2279                pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
2280                iommu_tce_table_put(tbl);
2281        }
2282}
2283
2284static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
2285                int num, struct iommu_table *tbl)
2286{
2287        struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2288                        table_group);
2289        struct pnv_phb *phb = pe->phb;
2290        int64_t rc;
2291        const unsigned long size = tbl->it_indirect_levels ?
2292                        tbl->it_level_size : tbl->it_size;
2293        const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
2294        const __u64 win_size = tbl->it_size << tbl->it_page_shift;
2295
2296        pe_info(pe, "Setting up window#%d %llx..%llx pg=%lx\n",
2297                num, start_addr, start_addr + win_size - 1,
2298                IOMMU_PAGE_SIZE(tbl));
2299
2300        /*
2301         * Map TCE table through TVT. The TVE index is the PE number
2302         * shifted by 1 bit for 32-bits DMA space.
2303         */
2304        rc = opal_pci_map_pe_dma_window(phb->opal_id,
2305                        pe->pe_number,
2306                        (pe->pe_number << 1) + num,
2307                        tbl->it_indirect_levels + 1,
2308                        __pa(tbl->it_base),
2309                        size << 3,
2310                        IOMMU_PAGE_SIZE(tbl));
2311        if (rc) {
2312                pe_err(pe, "Failed to configure TCE table, err %lld\n", rc);
2313                return rc;
2314        }
2315
2316        pnv_pci_link_table_and_group(phb->hose->node, num,
2317                        tbl, &pe->table_group);
2318        pnv_pci_ioda2_tce_invalidate_pe(pe);
2319
2320        return 0;
2321}
2322
2323static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
2324{
2325        uint16_t window_id = (pe->pe_number << 1 ) + 1;
2326        int64_t rc;
2327
2328        pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
2329        if (enable) {
2330                phys_addr_t top = memblock_end_of_DRAM();
2331
2332                top = roundup_pow_of_two(top);
2333                rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2334                                                     pe->pe_number,
2335                                                     window_id,
2336                                                     pe->tce_bypass_base,
2337                                                     top);
2338        } else {
2339                rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2340                                                     pe->pe_number,
2341                                                     window_id,
2342                                                     pe->tce_bypass_base,
2343                                                     0);
2344        }
2345        if (rc)
2346                pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
2347        else
2348                pe->tce_bypass_enabled = enable;
2349}
2350
2351static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
2352                int num, __u32 page_shift, __u64 window_size, __u32 levels,
2353                bool alloc_userspace_copy, struct iommu_table **ptbl)
2354{
2355        struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2356                        table_group);
2357        int nid = pe->phb->hose->node;
2358        __u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;
2359        long ret;
2360        struct iommu_table *tbl;
2361
2362        tbl = pnv_pci_table_alloc(nid);
2363        if (!tbl)
2364                return -ENOMEM;
2365
2366        tbl->it_ops = &pnv_ioda2_iommu_ops;
2367
2368        ret = pnv_pci_ioda2_table_alloc_pages(nid,
2369                        bus_offset, page_shift, window_size,
2370                        levels, alloc_userspace_copy, tbl);
2371        if (ret) {
2372                iommu_tce_table_put(tbl);
2373                return ret;
2374        }
2375
2376        *ptbl = tbl;
2377
2378        return 0;
2379}
2380
2381static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
2382{
2383        struct iommu_table *tbl = NULL;
2384        long rc;
2385        unsigned long res_start, res_end;
2386
2387        /*
2388         * crashkernel= specifies the kdump kernel's maximum memory at
2389         * some offset and there is no guaranteed the result is a power
2390         * of 2, which will cause errors later.
2391         */
2392        const u64 max_memory = __rounddown_pow_of_two(memory_hotplug_max());
2393
2394        /*
2395         * In memory constrained environments, e.g. kdump kernel, the
2396         * DMA window can be larger than available memory, which will
2397         * cause errors later.
2398         */
2399        const u64 maxblock = 1UL << (PAGE_SHIFT + MAX_ORDER - 1);
2400
2401        /*
2402         * We create the default window as big as we can. The constraint is
2403         * the max order of allocation possible. The TCE table is likely to
2404         * end up being multilevel and with on-demand allocation in place,
2405         * the initial use is not going to be huge as the default window aims
2406         * to support crippled devices (i.e. not fully 64bit DMAble) only.
2407         */
2408        /* iommu_table::it_map uses 1 bit per IOMMU page, hence 8 */
2409        const u64 window_size = min((maxblock * 8) << PAGE_SHIFT, max_memory);
2410        /* Each TCE level cannot exceed maxblock so go multilevel if needed */
2411        unsigned long tces_order = ilog2(window_size >> PAGE_SHIFT);
2412        unsigned long tcelevel_order = ilog2(maxblock >> 3);
2413        unsigned int levels = tces_order / tcelevel_order;
2414
2415        if (tces_order % tcelevel_order)
2416                levels += 1;
2417        /*
2418         * We try to stick to default levels (which is >1 at the moment) in
2419         * order to save memory by relying on on-demain TCE level allocation.
2420         */
2421        levels = max_t(unsigned int, levels, POWERNV_IOMMU_DEFAULT_LEVELS);
2422
2423        rc = pnv_pci_ioda2_create_table(&pe->table_group, 0, PAGE_SHIFT,
2424                        window_size, levels, false, &tbl);
2425        if (rc) {
2426                pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
2427                                rc);
2428                return rc;
2429        }
2430
2431        /* We use top part of 32bit space for MMIO so exclude it from DMA */
2432        res_start = 0;
2433        res_end = 0;
2434        if (window_size > pe->phb->ioda.m32_pci_base) {
2435                res_start = pe->phb->ioda.m32_pci_base >> tbl->it_page_shift;
2436                res_end = min(window_size, SZ_4G) >> tbl->it_page_shift;
2437        }
2438        iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end);
2439
2440        rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
2441        if (rc) {
2442                pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
2443                                rc);
2444                iommu_tce_table_put(tbl);
2445                return rc;
2446        }
2447
2448        if (!pnv_iommu_bypass_disabled)
2449                pnv_pci_ioda2_set_bypass(pe, true);
2450
2451        /*
2452         * Set table base for the case of IOMMU DMA use. Usually this is done
2453         * from dma_dev_setup() which is not called when a device is returned
2454         * from VFIO so do it here.
2455         */
2456        if (pe->pdev)
2457                set_iommu_table_base(&pe->pdev->dev, tbl);
2458
2459        return 0;
2460}
2461
2462#if defined(CONFIG_IOMMU_API) || defined(CONFIG_PCI_IOV)
2463static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
2464                int num)
2465{
2466        struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2467                        table_group);
2468        struct pnv_phb *phb = pe->phb;
2469        long ret;
2470
2471        pe_info(pe, "Removing DMA window #%d\n", num);
2472
2473        ret = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
2474                        (pe->pe_number << 1) + num,
2475                        0/* levels */, 0/* table address */,
2476                        0/* table size */, 0/* page size */);
2477        if (ret)
2478                pe_warn(pe, "Unmapping failed, ret = %ld\n", ret);
2479        else
2480                pnv_pci_ioda2_tce_invalidate_pe(pe);
2481
2482        pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
2483
2484        return ret;
2485}
2486#endif
2487
2488#ifdef CONFIG_IOMMU_API
2489unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
2490                __u64 window_size, __u32 levels)
2491{
2492        unsigned long bytes = 0;
2493        const unsigned window_shift = ilog2(window_size);
2494        unsigned entries_shift = window_shift - page_shift;
2495        unsigned table_shift = entries_shift + 3;
2496        unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
2497        unsigned long direct_table_size;
2498
2499        if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
2500                        !is_power_of_2(window_size))
2501                return 0;
2502
2503        /* Calculate a direct table size from window_size and levels */
2504        entries_shift = (entries_shift + levels - 1) / levels;
2505        table_shift = entries_shift + 3;
2506        table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
2507        direct_table_size =  1UL << table_shift;
2508
2509        for ( ; levels; --levels) {
2510                bytes += _ALIGN_UP(tce_table_size, direct_table_size);
2511
2512                tce_table_size /= direct_table_size;
2513                tce_table_size <<= 3;
2514                tce_table_size = max_t(unsigned long,
2515                                tce_table_size, direct_table_size);
2516        }
2517
2518        return bytes + bytes; /* one for HW table, one for userspace copy */
2519}
2520
2521static long pnv_pci_ioda2_create_table_userspace(
2522                struct iommu_table_group *table_group,
2523                int num, __u32 page_shift, __u64 window_size, __u32 levels,
2524                struct iommu_table **ptbl)
2525{
2526        long ret = pnv_pci_ioda2_create_table(table_group,
2527                        num, page_shift, window_size, levels, true, ptbl);
2528
2529        if (!ret)
2530                (*ptbl)->it_allocated_size = pnv_pci_ioda2_get_table_size(
2531                                page_shift, window_size, levels);
2532        return ret;
2533}
2534
2535static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
2536{
2537        struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2538                                                table_group);
2539        /* Store @tbl as pnv_pci_ioda2_unset_window() resets it */
2540        struct iommu_table *tbl = pe->table_group.tables[0];
2541
2542        pnv_pci_ioda2_set_bypass(pe, false);
2543        pnv_pci_ioda2_unset_window(&pe->table_group, 0);
2544        if (pe->pbus)
2545                pnv_ioda_setup_bus_dma(pe, pe->pbus);
2546        else if (pe->pdev)
2547                set_iommu_table_base(&pe->pdev->dev, NULL);
2548        iommu_tce_table_put(tbl);
2549}
2550
2551static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
2552{
2553        struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2554                                                table_group);
2555
2556        pnv_pci_ioda2_setup_default_config(pe);
2557        if (pe->pbus)
2558                pnv_ioda_setup_bus_dma(pe, pe->pbus);
2559}
2560
2561static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
2562        .get_table_size = pnv_pci_ioda2_get_table_size,
2563        .create_table = pnv_pci_ioda2_create_table_userspace,
2564        .set_window = pnv_pci_ioda2_set_window,
2565        .unset_window = pnv_pci_ioda2_unset_window,
2566        .take_ownership = pnv_ioda2_take_ownership,
2567        .release_ownership = pnv_ioda2_release_ownership,
2568};
2569
2570static void pnv_ioda_setup_bus_iommu_group_add_devices(struct pnv_ioda_pe *pe,
2571                struct iommu_table_group *table_group,
2572                struct pci_bus *bus)
2573{
2574        struct pci_dev *dev;
2575
2576        list_for_each_entry(dev, &bus->devices, bus_list) {
2577                iommu_add_device(table_group, &dev->dev);
2578
2579                if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
2580                        pnv_ioda_setup_bus_iommu_group_add_devices(pe,
2581                                        table_group, dev->subordinate);
2582        }
2583}
2584
2585static void pnv_ioda_setup_bus_iommu_group(struct pnv_ioda_pe *pe,
2586                struct iommu_table_group *table_group, struct pci_bus *bus)
2587{
2588
2589        if (pe->flags & PNV_IODA_PE_DEV)
2590                iommu_add_device(table_group, &pe->pdev->dev);
2591
2592        if ((pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)) || bus)
2593                pnv_ioda_setup_bus_iommu_group_add_devices(pe, table_group,
2594                                bus);
2595}
2596
2597static unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb);
2598
2599static void pnv_pci_ioda_setup_iommu_api(void)
2600{
2601        struct pci_controller *hose;
2602        struct pnv_phb *phb;
2603        struct pnv_ioda_pe *pe;
2604
2605        /*
2606         * There are 4 types of PEs:
2607         * - PNV_IODA_PE_BUS: a downstream port with an adapter,
2608         *   created from pnv_pci_setup_bridge();
2609         * - PNV_IODA_PE_BUS_ALL: a PCI-PCIX bridge with devices behind it,
2610         *   created from pnv_pci_setup_bridge();
2611         * - PNV_IODA_PE_VF: a SRIOV virtual function,
2612         *   created from pnv_pcibios_sriov_enable();
2613         * - PNV_IODA_PE_DEV: an NPU or OCAPI device,
2614         *   created from pnv_pci_ioda_fixup().
2615         *
2616         * Normally a PE is represented by an IOMMU group, however for
2617         * devices with side channels the groups need to be more strict.
2618         */
2619        list_for_each_entry(hose, &hose_list, list_node) {
2620                phb = hose->private_data;
2621
2622                if (phb->type == PNV_PHB_NPU_NVLINK ||
2623                    phb->type == PNV_PHB_NPU_OCAPI)
2624                        continue;
2625
2626                list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2627                        struct iommu_table_group *table_group;
2628
2629                        table_group = pnv_try_setup_npu_table_group(pe);
2630                        if (!table_group) {
2631                                if (!pnv_pci_ioda_pe_dma_weight(pe))
2632                                        continue;
2633
2634                                table_group = &pe->table_group;
2635                                iommu_register_group(&pe->table_group,
2636                                                pe->phb->hose->global_number,
2637                                                pe->pe_number);
2638                        }
2639                        pnv_ioda_setup_bus_iommu_group(pe, table_group,
2640                                        pe->pbus);
2641                }
2642        }
2643
2644        /*
2645         * Now we have all PHBs discovered, time to add NPU devices to
2646         * the corresponding IOMMU groups.
2647         */
2648        list_for_each_entry(hose, &hose_list, list_node) {
2649                unsigned long  pgsizes;
2650
2651                phb = hose->private_data;
2652
2653                if (phb->type != PNV_PHB_NPU_NVLINK)
2654                        continue;
2655
2656                pgsizes = pnv_ioda_parse_tce_sizes(phb);
2657                list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2658                        /*
2659                         * IODA2 bridges get this set up from
2660                         * pci_controller_ops::setup_bridge but NPU bridges
2661                         * do not have this hook defined so we do it here.
2662                         */
2663                        pe->table_group.pgsizes = pgsizes;
2664                        pnv_npu_compound_attach(pe);
2665                }
2666        }
2667}
2668#else /* !CONFIG_IOMMU_API */
2669static void pnv_pci_ioda_setup_iommu_api(void) { };
2670#endif
2671
2672static unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb)
2673{
2674        struct pci_controller *hose = phb->hose;
2675        struct device_node *dn = hose->dn;
2676        unsigned long mask = 0;
2677        int i, rc, count;
2678        u32 val;
2679
2680        count = of_property_count_u32_elems(dn, "ibm,supported-tce-sizes");
2681        if (count <= 0) {
2682                mask = SZ_4K | SZ_64K;
2683                /* Add 16M for POWER8 by default */
2684                if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
2685                                !cpu_has_feature(CPU_FTR_ARCH_300))
2686                        mask |= SZ_16M | SZ_256M;
2687                return mask;
2688        }
2689
2690        for (i = 0; i < count; i++) {
2691                rc = of_property_read_u32_index(dn, "ibm,supported-tce-sizes",
2692                                                i, &val);
2693                if (rc == 0)
2694                        mask |= 1ULL << val;
2695        }
2696
2697        return mask;
2698}
2699
2700static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
2701                                       struct pnv_ioda_pe *pe)
2702{
2703        int64_t rc;
2704
2705        if (!pnv_pci_ioda_pe_dma_weight(pe))
2706                return;
2707
2708        /* TVE #1 is selected by PCI address bit 59 */
2709        pe->tce_bypass_base = 1ull << 59;
2710
2711        /* The PE will reserve all possible 32-bits space */
2712        pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
2713                phb->ioda.m32_pci_base);
2714
2715        /* Setup linux iommu table */
2716        pe->table_group.tce32_start = 0;
2717        pe->table_group.tce32_size = phb->ioda.m32_pci_base;
2718        pe->table_group.max_dynamic_windows_supported =
2719                        IOMMU_TABLE_GROUP_MAX_TABLES;
2720        pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
2721        pe->table_group.pgsizes = pnv_ioda_parse_tce_sizes(phb);
2722#ifdef CONFIG_IOMMU_API
2723        pe->table_group.ops = &pnv_pci_ioda2_ops;
2724#endif
2725
2726        rc = pnv_pci_ioda2_setup_default_config(pe);
2727        if (rc)
2728                return;
2729
2730        if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
2731                pnv_ioda_setup_bus_dma(pe, pe->pbus);
2732}
2733
2734int64_t pnv_opal_pci_msi_eoi(struct irq_chip *chip, unsigned int hw_irq)
2735{
2736        struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2737                                           ioda.irq_chip);
2738
2739        return opal_pci_msi_eoi(phb->opal_id, hw_irq);
2740}
2741
2742static void pnv_ioda2_msi_eoi(struct irq_data *d)
2743{
2744        int64_t rc;
2745        unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2746        struct irq_chip *chip = irq_data_get_irq_chip(d);
2747
2748        rc = pnv_opal_pci_msi_eoi(chip, hw_irq);
2749        WARN_ON_ONCE(rc);
2750
2751        icp_native_eoi(d);
2752}
2753
2754
2755void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2756{
2757        struct irq_data *idata;
2758        struct irq_chip *ichip;
2759
2760        /* The MSI EOI OPAL call is only needed on PHB3 */
2761        if (phb->model != PNV_PHB_MODEL_PHB3)
2762                return;
2763
2764        if (!phb->ioda.irq_chip_init) {
2765                /*
2766                 * First time we setup an MSI IRQ, we need to setup the
2767                 * corresponding IRQ chip to route correctly.
2768                 */
2769                idata = irq_get_irq_data(virq);
2770                ichip = irq_data_get_irq_chip(idata);
2771                phb->ioda.irq_chip_init = 1;
2772                phb->ioda.irq_chip = *ichip;
2773                phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2774        }
2775        irq_set_chip(virq, &phb->ioda.irq_chip);
2776}
2777
2778/*
2779 * Returns true iff chip is something that we could call
2780 * pnv_opal_pci_msi_eoi for.
2781 */
2782bool is_pnv_opal_msi(struct irq_chip *chip)
2783{
2784        return chip->irq_eoi == pnv_ioda2_msi_eoi;
2785}
2786EXPORT_SYMBOL_GPL(is_pnv_opal_msi);
2787
2788static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
2789                                  unsigned int hwirq, unsigned int virq,
2790                                  unsigned int is_64, struct msi_msg *msg)
2791{
2792        struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2793        unsigned int xive_num = hwirq - phb->msi_base;
2794        __be32 data;
2795        int rc;
2796
2797        /* No PE assigned ? bail out ... no MSI for you ! */
2798        if (pe == NULL)
2799                return -ENXIO;
2800
2801        /* Check if we have an MVE */
2802        if (pe->mve_number < 0)
2803                return -ENXIO;
2804
2805        /* Force 32-bit MSI on some broken devices */
2806        if (dev->no_64bit_msi)
2807                is_64 = 0;
2808
2809        /* Assign XIVE to PE */
2810        rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2811        if (rc) {
2812                pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2813                        pci_name(dev), rc, xive_num);
2814                return -EIO;
2815        }
2816
2817        if (is_64) {
2818                __be64 addr64;
2819
2820                rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2821                                     &addr64, &data);
2822                if (rc) {
2823                        pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2824                                pci_name(dev), rc);
2825                        return -EIO;
2826                }
2827                msg->address_hi = be64_to_cpu(addr64) >> 32;
2828                msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
2829        } else {
2830                __be32 addr32;
2831
2832                rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2833                                     &addr32, &data);
2834                if (rc) {
2835                        pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2836                                pci_name(dev), rc);
2837                        return -EIO;
2838                }
2839                msg->address_hi = 0;
2840                msg->address_lo = be32_to_cpu(addr32);
2841        }
2842        msg->data = be32_to_cpu(data);
2843
2844        pnv_set_msi_irq_chip(phb, virq);
2845
2846        pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2847                 " address=%x_%08x data=%x PE# %x\n",
2848                 pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2849                 msg->address_hi, msg->address_lo, data, pe->pe_number);
2850
2851        return 0;
2852}
2853
2854static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2855{
2856        unsigned int count;
2857        const __be32 *prop = of_get_property(phb->hose->dn,
2858                                             "ibm,opal-msi-ranges", NULL);
2859        if (!prop) {
2860                /* BML Fallback */
2861                prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2862        }
2863        if (!prop)
2864                return;
2865
2866        phb->msi_base = be32_to_cpup(prop);
2867        count = be32_to_cpup(prop + 1);
2868        if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
2869                pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2870                       phb->hose->global_number);
2871                return;
2872        }
2873
2874        phb->msi_setup = pnv_pci_ioda_msi_setup;
2875        phb->msi32_support = 1;
2876        pr_info("  Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
2877                count, phb->msi_base);
2878}
2879
2880#ifdef CONFIG_PCI_IOV
2881static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2882{
2883        struct pci_controller *hose = pci_bus_to_host(pdev->bus);
2884        struct pnv_phb *phb = hose->private_data;
2885        const resource_size_t gate = phb->ioda.m64_segsize >> 2;
2886        struct resource *res;
2887        int i;
2888        resource_size_t size, total_vf_bar_sz;
2889        struct pci_dn *pdn;
2890        int mul, total_vfs;
2891
2892        if (!pdev->is_physfn || pci_dev_is_added(pdev))
2893                return;
2894
2895        pdn = pci_get_pdn(pdev);
2896        pdn->vfs_expanded = 0;
2897        pdn->m64_single_mode = false;
2898
2899        total_vfs = pci_sriov_get_totalvfs(pdev);
2900        mul = phb->ioda.total_pe_num;
2901        total_vf_bar_sz = 0;
2902
2903        for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2904                res = &pdev->resource[i + PCI_IOV_RESOURCES];
2905                if (!res->flags || res->parent)
2906                        continue;
2907                if (!pnv_pci_is_m64_flags(res->flags)) {
2908                        dev_warn(&pdev->dev, "Don't support SR-IOV with"
2909                                        " non M64 VF BAR%d: %pR. \n",
2910                                 i, res);
2911                        goto truncate_iov;
2912                }
2913
2914                total_vf_bar_sz += pci_iov_resource_size(pdev,
2915                                i + PCI_IOV_RESOURCES);
2916
2917                /*
2918                 * If bigger than quarter of M64 segment size, just round up
2919                 * power of two.
2920                 *
2921                 * Generally, one M64 BAR maps one IOV BAR. To avoid conflict
2922                 * with other devices, IOV BAR size is expanded to be
2923                 * (total_pe * VF_BAR_size).  When VF_BAR_size is half of M64
2924                 * segment size , the expanded size would equal to half of the
2925                 * whole M64 space size, which will exhaust the M64 Space and
2926                 * limit the system flexibility.  This is a design decision to
2927                 * set the boundary to quarter of the M64 segment size.
2928                 */
2929                if (total_vf_bar_sz > gate) {
2930                        mul = roundup_pow_of_two(total_vfs);
2931                        dev_info(&pdev->dev,
2932                                "VF BAR Total IOV size %llx > %llx, roundup to %d VFs\n",
2933                                total_vf_bar_sz, gate, mul);
2934                        pdn->m64_single_mode = true;
2935                        break;
2936                }
2937        }
2938
2939        for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2940                res = &pdev->resource[i + PCI_IOV_RESOURCES];
2941                if (!res->flags || res->parent)
2942                        continue;
2943
2944                size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2945                /*
2946                 * On PHB3, the minimum size alignment of M64 BAR in single
2947                 * mode is 32MB.
2948                 */
2949                if (pdn->m64_single_mode && (size < SZ_32M))
2950                        goto truncate_iov;
2951                dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2952                res->end = res->start + size * mul - 1;
2953                dev_dbg(&pdev->dev, "                       %pR\n", res);
2954                dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
2955                         i, res, mul);
2956        }
2957        pdn->vfs_expanded = mul;
2958
2959        return;
2960
2961truncate_iov:
2962        /* To save MMIO space, IOV BAR is truncated. */
2963        for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2964                res = &pdev->resource[i + PCI_IOV_RESOURCES];
2965                res->flags = 0;
2966                res->end = res->start - 1;
2967        }
2968}
2969#endif /* CONFIG_PCI_IOV */
2970
2971static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
2972                                  struct resource *res)
2973{
2974        struct pnv_phb *phb = pe->phb;
2975        struct pci_bus_region region;
2976        int index;
2977        int64_t rc;
2978
2979        if (!res || !res->flags || res->start > res->end)
2980                return;
2981
2982        if (res->flags & IORESOURCE_IO) {
2983                region.start = res->start - phb->ioda.io_pci_base;
2984                region.end   = res->end - phb->ioda.io_pci_base;
2985                index = region.start / phb->ioda.io_segsize;
2986
2987                while (index < phb->ioda.total_pe_num &&
2988                       region.start <= region.end) {
2989                        phb->ioda.io_segmap[index] = pe->pe_number;
2990                        rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2991                                pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2992                        if (rc != OPAL_SUCCESS) {
2993                                pr_err("%s: Error %lld mapping IO segment#%d to PE#%x\n",
2994                                       __func__, rc, index, pe->pe_number);
2995                                break;
2996                        }
2997
2998                        region.start += phb->ioda.io_segsize;
2999                        index++;
3000                }
3001        } else if ((res->flags & IORESOURCE_MEM) &&
3002                   !pnv_pci_is_m64(phb, res)) {
3003                region.start = res->start -
3004                               phb->hose->mem_offset[0] -
3005                               phb->ioda.m32_pci_base;
3006                region.end   = res->end -
3007                               phb->hose->mem_offset[0] -
3008                               phb->ioda.m32_pci_base;
3009                index = region.start / phb->ioda.m32_segsize;
3010
3011                while (index < phb->ioda.total_pe_num &&
3012                       region.start <= region.end) {
3013                        phb->ioda.m32_segmap[index] = pe->pe_number;
3014                        rc = opal_pci_map_pe_mmio_window(phb->opal_id,
3015                                pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
3016                        if (rc != OPAL_SUCCESS) {
3017                                pr_err("%s: Error %lld mapping M32 segment#%d to PE#%x",
3018                                       __func__, rc, index, pe->pe_number);
3019                                break;
3020                        }
3021
3022                        region.start += phb->ioda.m32_segsize;
3023                        index++;
3024                }
3025        }
3026}
3027
3028/*
3029 * This function is supposed to be called on basis of PE from top
3030 * to bottom style. So the the I/O or MMIO segment assigned to
3031 * parent PE could be overridden by its child PEs if necessary.
3032 */
3033static void pnv_ioda_setup_pe_seg(struct pnv_ioda_pe *pe)
3034{
3035        struct pci_dev *pdev;
3036        int i;
3037
3038        /*
3039         * NOTE: We only care PCI bus based PE for now. For PCI
3040         * device based PE, for example SRIOV sensitive VF should
3041         * be figured out later.
3042         */
3043        BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
3044
3045        list_for_each_entry(pdev, &pe->pbus->devices, bus_list) {
3046                for (i = 0; i <= PCI_ROM_RESOURCE; i++)
3047                        pnv_ioda_setup_pe_res(pe, &pdev->resource[i]);
3048
3049                /*
3050                 * If the PE contains all subordinate PCI buses, the
3051                 * windows of the child bridges should be mapped to
3052                 * the PE as well.
3053                 */
3054                if (!(pe->flags & PNV_IODA_PE_BUS_ALL) || !pci_is_bridge(pdev))
3055                        continue;
3056                for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
3057                        pnv_ioda_setup_pe_res(pe,
3058                                &pdev->resource[PCI_BRIDGE_RESOURCES + i]);
3059        }
3060}
3061
3062#ifdef CONFIG_DEBUG_FS
3063static int pnv_pci_diag_data_set(void *data, u64 val)
3064{
3065        struct pci_controller *hose;
3066        struct pnv_phb *phb;
3067        s64 ret;
3068
3069        if (val != 1ULL)
3070                return -EINVAL;
3071
3072        hose = (struct pci_controller *)data;
3073        if (!hose || !hose->private_data)
3074                return -ENODEV;
3075
3076        phb = hose->private_data;
3077
3078        /* Retrieve the diag data from firmware */
3079        ret = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag_data,
3080                                          phb->diag_data_size);
3081        if (ret != OPAL_SUCCESS)
3082                return -EIO;
3083
3084        /* Print the diag data to the kernel log */
3085        pnv_pci_dump_phb_diag_data(phb->hose, phb->diag_data);
3086        return 0;
3087}
3088
3089DEFINE_DEBUGFS_ATTRIBUTE(pnv_pci_diag_data_fops, NULL, pnv_pci_diag_data_set,
3090                         "%llu\n");
3091
3092#endif /* CONFIG_DEBUG_FS */
3093
3094static void pnv_pci_ioda_create_dbgfs(void)
3095{
3096#ifdef CONFIG_DEBUG_FS
3097        struct pci_controller *hose, *tmp;
3098        struct pnv_phb *phb;
3099        char name[16];
3100
3101        list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
3102                phb = hose->private_data;
3103
3104                /* Notify initialization of PHB done */
3105                phb->initialized = 1;
3106
3107                sprintf(name, "PCI%04x", hose->global_number);
3108                phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
3109                if (!phb->dbgfs) {
3110                        pr_warn("%s: Error on creating debugfs on PHB#%x\n",
3111                                __func__, hose->global_number);
3112                        continue;
3113                }
3114
3115                debugfs_create_file_unsafe("dump_diag_regs", 0200, phb->dbgfs,
3116                                           hose, &pnv_pci_diag_data_fops);
3117        }
3118#endif /* CONFIG_DEBUG_FS */
3119}
3120
3121static void pnv_pci_enable_bridge(struct pci_bus *bus)
3122{
3123        struct pci_dev *dev = bus->self;
3124        struct pci_bus *child;
3125
3126        /* Empty bus ? bail */
3127        if (list_empty(&bus->devices))
3128                return;
3129
3130        /*
3131         * If there's a bridge associated with that bus enable it. This works
3132         * around races in the generic code if the enabling is done during
3133         * parallel probing. This can be removed once those races have been
3134         * fixed.
3135         */
3136        if (dev) {
3137                int rc = pci_enable_device(dev);
3138                if (rc)
3139                        pci_err(dev, "Error enabling bridge (%d)\n", rc);
3140                pci_set_master(dev);
3141        }
3142
3143        /* Perform the same to child busses */
3144        list_for_each_entry(child, &bus->children, node)
3145                pnv_pci_enable_bridge(child);
3146}
3147
3148static void pnv_pci_enable_bridges(void)
3149{
3150        struct pci_controller *hose;
3151
3152        list_for_each_entry(hose, &hose_list, list_node)
3153                pnv_pci_enable_bridge(hose->bus);
3154}
3155
3156static void pnv_pci_ioda_fixup(void)
3157{
3158        pnv_pci_ioda_setup_PEs();
3159        pnv_pci_ioda_setup_iommu_api();
3160        pnv_pci_ioda_create_dbgfs();
3161
3162        pnv_pci_enable_bridges();
3163
3164#ifdef CONFIG_EEH
3165        pnv_eeh_post_init();
3166#endif
3167}
3168
3169/*
3170 * Returns the alignment for I/O or memory windows for P2P
3171 * bridges. That actually depends on how PEs are segmented.
3172 * For now, we return I/O or M32 segment size for PE sensitive
3173 * P2P bridges. Otherwise, the default values (4KiB for I/O,
3174 * 1MiB for memory) will be returned.
3175 *
3176 * The current PCI bus might be put into one PE, which was
3177 * create against the parent PCI bridge. For that case, we
3178 * needn't enlarge the alignment so that we can save some
3179 * resources.
3180 */
3181static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
3182                                                unsigned long type)
3183{
3184        struct pci_dev *bridge;
3185        struct pci_controller *hose = pci_bus_to_host(bus);
3186        struct pnv_phb *phb = hose->private_data;
3187        int num_pci_bridges = 0;
3188
3189        bridge = bus->self;
3190        while (bridge) {
3191                if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
3192                        num_pci_bridges++;
3193                        if (num_pci_bridges >= 2)
3194                                return 1;
3195                }
3196
3197                bridge = bridge->bus->self;
3198        }
3199
3200        /*
3201         * We fall back to M32 if M64 isn't supported. We enforce the M64
3202         * alignment for any 64-bit resource, PCIe doesn't care and
3203         * bridges only do 64-bit prefetchable anyway.
3204         */
3205        if (phb->ioda.m64_segsize && pnv_pci_is_m64_flags(type))
3206                return phb->ioda.m64_segsize;
3207        if (type & IORESOURCE_MEM)
3208                return phb->ioda.m32_segsize;
3209
3210        return phb->ioda.io_segsize;
3211}
3212
3213/*
3214 * We are updating root port or the upstream port of the
3215 * bridge behind the root port with PHB's windows in order
3216 * to accommodate the changes on required resources during
3217 * PCI (slot) hotplug, which is connected to either root
3218 * port or the downstream ports of PCIe switch behind the
3219 * root port.
3220 */
3221static void pnv_pci_fixup_bridge_resources(struct pci_bus *bus,
3222                                           unsigned long type)
3223{
3224        struct pci_controller *hose = pci_bus_to_host(bus);
3225        struct pnv_phb *phb = hose->private_data;
3226        struct pci_dev *bridge = bus->self;
3227        struct resource *r, *w;
3228        bool msi_region = false;
3229        int i;
3230
3231        /* Check if we need apply fixup to the bridge's windows */
3232        if (!pci_is_root_bus(bridge->bus) &&
3233            !pci_is_root_bus(bridge->bus->self->bus))
3234                return;
3235
3236        /* Fixup the resources */
3237        for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
3238                r = &bridge->resource[PCI_BRIDGE_RESOURCES + i];
3239                if (!r->flags || !r->parent)
3240                        continue;
3241
3242                w = NULL;
3243                if (r->flags & type & IORESOURCE_IO)
3244                        w = &hose->io_resource;
3245                else if (pnv_pci_is_m64(phb, r) &&
3246                         (type & IORESOURCE_PREFETCH) &&
3247                         phb->ioda.m64_segsize)
3248                        w = &hose->mem_resources[1];
3249                else if (r->flags & type & IORESOURCE_MEM) {
3250                        w = &hose->mem_resources[0];
3251                        msi_region = true;
3252                }
3253
3254                r->start = w->start;
3255                r->end = w->end;
3256
3257                /* The 64KB 32-bits MSI region shouldn't be included in
3258                 * the 32-bits bridge window. Otherwise, we can see strange
3259                 * issues. One of them is EEH error observed on Garrison.
3260                 *
3261                 * Exclude top 1MB region which is the minimal alignment of
3262                 * 32-bits bridge window.
3263                 */
3264                if (msi_region) {
3265                        r->end += 0x10000;
3266                        r->end -= 0x100000;
3267                }
3268        }
3269}
3270
3271static void pnv_pci_setup_bridge(struct pci_bus *bus, unsigned long type)
3272{
3273        struct pci_controller *hose = pci_bus_to_host(bus);
3274        struct pnv_phb *phb = hose->private_data;
3275        struct pci_dev *bridge = bus->self;
3276        struct pnv_ioda_pe *pe;
3277        bool all = (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE);
3278
3279        /* Extend bridge's windows if necessary */
3280        pnv_pci_fixup_bridge_resources(bus, type);
3281
3282        /* The PE for root bus should be realized before any one else */
3283        if (!phb->ioda.root_pe_populated) {
3284                pe = pnv_ioda_setup_bus_PE(phb->hose->bus, false);
3285                if (pe) {
3286                        phb->ioda.root_pe_idx = pe->pe_number;
3287                        phb->ioda.root_pe_populated = true;
3288                }
3289        }
3290
3291        /* Don't assign PE to PCI bus, which doesn't have subordinate devices */
3292        if (list_empty(&bus->devices))
3293                return;
3294
3295        /* Reserve PEs according to used M64 resources */
3296        pnv_ioda_reserve_m64_pe(bus, NULL, all);
3297
3298        /*
3299         * Assign PE. We might run here because of partial hotplug.
3300         * For the case, we just pick up the existing PE and should
3301         * not allocate resources again.
3302         */
3303        pe = pnv_ioda_setup_bus_PE(bus, all);
3304        if (!pe)
3305                return;
3306
3307        pnv_ioda_setup_pe_seg(pe);
3308        switch (phb->type) {
3309        case PNV_PHB_IODA1:
3310                pnv_pci_ioda1_setup_dma_pe(phb, pe);
3311                break;
3312        case PNV_PHB_IODA2:
3313                pnv_pci_ioda2_setup_dma_pe(phb, pe);
3314                break;
3315        default:
3316                pr_warn("%s: No DMA for PHB#%x (type %d)\n",
3317                        __func__, phb->hose->global_number, phb->type);
3318        }
3319}
3320
3321static resource_size_t pnv_pci_default_alignment(void)
3322{
3323        return PAGE_SIZE;
3324}
3325
3326#ifdef CONFIG_PCI_IOV
3327static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
3328                                                      int resno)
3329{
3330        struct pci_controller *hose = pci_bus_to_host(pdev->bus);
3331        struct pnv_phb *phb = hose->private_data;
3332        struct pci_dn *pdn = pci_get_pdn(pdev);
3333        resource_size_t align;
3334
3335        /*
3336         * On PowerNV platform, IOV BAR is mapped by M64 BAR to enable the
3337         * SR-IOV. While from hardware perspective, the range mapped by M64
3338         * BAR should be size aligned.
3339         *
3340         * When IOV BAR is mapped with M64 BAR in Single PE mode, the extra
3341         * powernv-specific hardware restriction is gone. But if just use the
3342         * VF BAR size as the alignment, PF BAR / VF BAR may be allocated with
3343         * in one segment of M64 #15, which introduces the PE conflict between
3344         * PF and VF. Based on this, the minimum alignment of an IOV BAR is
3345         * m64_segsize.
3346         *
3347         * This function returns the total IOV BAR size if M64 BAR is in
3348         * Shared PE mode or just VF BAR size if not.
3349         * If the M64 BAR is in Single PE mode, return the VF BAR size or
3350         * M64 segment size if IOV BAR size is less.
3351         */
3352        align = pci_iov_resource_size(pdev, resno);
3353        if (!pdn->vfs_expanded)
3354                return align;
3355        if (pdn->m64_single_mode)
3356                return max(align, (resource_size_t)phb->ioda.m64_segsize);
3357
3358        return pdn->vfs_expanded * align;
3359}
3360#endif /* CONFIG_PCI_IOV */
3361
3362/* Prevent enabling devices for which we couldn't properly
3363 * assign a PE
3364 */
3365static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
3366{
3367        struct pci_controller *hose = pci_bus_to_host(dev->bus);
3368        struct pnv_phb *phb = hose->private_data;
3369        struct pci_dn *pdn;
3370
3371        /* The function is probably called while the PEs have
3372         * not be created yet. For example, resource reassignment
3373         * during PCI probe period. We just skip the check if
3374         * PEs isn't ready.
3375         */
3376        if (!phb->initialized)
3377                return true;
3378
3379        pdn = pci_get_pdn(dev);
3380        if (!pdn || pdn->pe_number == IODA_INVALID_PE)
3381                return false;
3382
3383        return true;
3384}
3385
3386static long pnv_pci_ioda1_unset_window(struct iommu_table_group *table_group,
3387                                       int num)
3388{
3389        struct pnv_ioda_pe *pe = container_of(table_group,
3390                                              struct pnv_ioda_pe, table_group);
3391        struct pnv_phb *phb = pe->phb;
3392        unsigned int idx;
3393        long rc;
3394
3395        pe_info(pe, "Removing DMA window #%d\n", num);
3396        for (idx = 0; idx < phb->ioda.dma32_count; idx++) {
3397                if (phb->ioda.dma32_segmap[idx] != pe->pe_number)
3398                        continue;
3399
3400                rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
3401                                                idx, 0, 0ul, 0ul, 0ul);
3402                if (rc != OPAL_SUCCESS) {
3403                        pe_warn(pe, "Failure %ld unmapping DMA32 segment#%d\n",
3404                                rc, idx);
3405                        return rc;
3406                }
3407
3408                phb->ioda.dma32_segmap[idx] = IODA_INVALID_PE;
3409        }
3410
3411        pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
3412        return OPAL_SUCCESS;
3413}
3414
3415static void pnv_pci_ioda1_release_pe_dma(struct pnv_ioda_pe *pe)
3416{
3417        unsigned int weight = pnv_pci_ioda_pe_dma_weight(pe);
3418        struct iommu_table *tbl = pe->table_group.tables[0];
3419        int64_t rc;
3420
3421        if (!weight)
3422                return;
3423
3424        rc = pnv_pci_ioda1_unset_window(&pe->table_group, 0);
3425        if (rc != OPAL_SUCCESS)
3426                return;
3427
3428        pnv_pci_p7ioc_tce_invalidate(tbl, tbl->it_offset, tbl->it_size, false);
3429        if (pe->table_group.group) {
3430                iommu_group_put(pe->table_group.group);
3431                WARN_ON(pe->table_group.group);
3432        }
3433
3434        free_pages(tbl->it_base, get_order(tbl->it_size << 3));
3435        iommu_tce_table_put(tbl);
3436}
3437
3438static void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe)
3439{
3440        struct iommu_table *tbl = pe->table_group.tables[0];
3441        unsigned int weight = pnv_pci_ioda_pe_dma_weight(pe);
3442#ifdef CONFIG_IOMMU_API
3443        int64_t rc;
3444#endif
3445
3446        if (!weight)
3447                return;
3448
3449#ifdef CONFIG_IOMMU_API
3450        rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
3451        if (rc)
3452                pe_warn(pe, "OPAL error %lld release DMA window\n", rc);
3453#endif
3454
3455        pnv_pci_ioda2_set_bypass(pe, false);
3456        if (pe->table_group.group) {
3457                iommu_group_put(pe->table_group.group);
3458                WARN_ON(pe->table_group.group);
3459        }
3460
3461        iommu_tce_table_put(tbl);
3462}
3463
3464static void pnv_ioda_free_pe_seg(struct pnv_ioda_pe *pe,
3465                                 unsigned short win,
3466                                 unsigned int *map)
3467{
3468        struct pnv_phb *phb = pe->phb;
3469        int idx;
3470        int64_t rc;
3471
3472        for (idx = 0; idx < phb->ioda.total_pe_num; idx++) {
3473                if (map[idx] != pe->pe_number)
3474                        continue;
3475
3476                if (win == OPAL_M64_WINDOW_TYPE)
3477                        rc = opal_pci_map_pe_mmio_window(phb->opal_id,
3478                                        phb->ioda.reserved_pe_idx, win,
3479                                        idx / PNV_IODA1_M64_SEGS,
3480                                        idx % PNV_IODA1_M64_SEGS);
3481                else
3482                        rc = opal_pci_map_pe_mmio_window(phb->opal_id,
3483                                        phb->ioda.reserved_pe_idx, win, 0, idx);
3484
3485                if (rc != OPAL_SUCCESS)
3486                        pe_warn(pe, "Error %lld unmapping (%d) segment#%d\n",
3487                                rc, win, idx);
3488
3489                map[idx] = IODA_INVALID_PE;
3490        }
3491}
3492
3493static void pnv_ioda_release_pe_seg(struct pnv_ioda_pe *pe)
3494{
3495        struct pnv_phb *phb = pe->phb;
3496
3497        if (phb->type == PNV_PHB_IODA1) {
3498                pnv_ioda_free_pe_seg(pe, OPAL_IO_WINDOW_TYPE,
3499                                     phb->ioda.io_segmap);
3500                pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
3501                                     phb->ioda.m32_segmap);
3502                pnv_ioda_free_pe_seg(pe, OPAL_M64_WINDOW_TYPE,
3503                                     phb->ioda.m64_segmap);
3504        } else if (phb->type == PNV_PHB_IODA2) {
3505                pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
3506                                     phb->ioda.m32_segmap);
3507        }
3508}
3509
3510static void pnv_ioda_release_pe(struct pnv_ioda_pe *pe)
3511{
3512        struct pnv_phb *phb = pe->phb;
3513        struct pnv_ioda_pe *slave, *tmp;
3514
3515        list_del(&pe->list);
3516        switch (phb->type) {
3517        case PNV_PHB_IODA1:
3518                pnv_pci_ioda1_release_pe_dma(pe);
3519                break;
3520        case PNV_PHB_IODA2:
3521                pnv_pci_ioda2_release_pe_dma(pe);
3522                break;
3523        default:
3524                WARN_ON(1);
3525        }
3526
3527        pnv_ioda_release_pe_seg(pe);
3528        pnv_ioda_deconfigure_pe(pe->phb, pe);
3529
3530        /* Release slave PEs in the compound PE */
3531        if (pe->flags & PNV_IODA_PE_MASTER) {
3532                list_for_each_entry_safe(slave, tmp, &pe->slaves, list) {
3533                        list_del(&slave->list);
3534                        pnv_ioda_free_pe(slave);
3535                }
3536        }
3537
3538        /*
3539         * The PE for root bus can be removed because of hotplug in EEH
3540         * recovery for fenced PHB error. We need to mark the PE dead so
3541         * that it can be populated again in PCI hot add path. The PE
3542         * shouldn't be destroyed as it's the global reserved resource.
3543         */
3544        if (phb->ioda.root_pe_populated &&
3545            phb->ioda.root_pe_idx == pe->pe_number)
3546                phb->ioda.root_pe_populated = false;
3547        else
3548                pnv_ioda_free_pe(pe);
3549}
3550
3551static void pnv_pci_release_device(struct pci_dev *pdev)
3552{
3553        struct pci_controller *hose = pci_bus_to_host(pdev->bus);
3554        struct pnv_phb *phb = hose->private_data;
3555        struct pci_dn *pdn = pci_get_pdn(pdev);
3556        struct pnv_ioda_pe *pe;
3557
3558        if (pdev->is_virtfn)
3559                return;
3560
3561        if (!pdn || pdn->pe_number == IODA_INVALID_PE)
3562                return;
3563
3564        /*
3565         * PCI hotplug can happen as part of EEH error recovery. The @pdn
3566         * isn't removed and added afterwards in this scenario. We should
3567         * set the PE number in @pdn to an invalid one. Otherwise, the PE's
3568         * device count is decreased on removing devices while failing to
3569         * be increased on adding devices. It leads to unbalanced PE's device
3570         * count and eventually make normal PCI hotplug path broken.
3571         */
3572        pe = &phb->ioda.pe_array[pdn->pe_number];
3573        pdn->pe_number = IODA_INVALID_PE;
3574
3575        WARN_ON(--pe->device_count < 0);
3576        if (pe->device_count == 0)
3577                pnv_ioda_release_pe(pe);
3578}
3579
3580static void pnv_npu_disable_device(struct pci_dev *pdev)
3581{
3582        struct eeh_dev *edev = pci_dev_to_eeh_dev(pdev);
3583        struct eeh_pe *eehpe = edev ? edev->pe : NULL;
3584
3585        if (eehpe && eeh_ops && eeh_ops->reset)
3586                eeh_ops->reset(eehpe, EEH_RESET_HOT);
3587}
3588
3589static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
3590{
3591        struct pnv_phb *phb = hose->private_data;
3592
3593        opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
3594                       OPAL_ASSERT_RESET);
3595}
3596
3597static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
3598        .dma_dev_setup          = pnv_pci_dma_dev_setup,
3599        .dma_bus_setup          = pnv_pci_dma_bus_setup,
3600        .iommu_bypass_supported = pnv_pci_ioda_iommu_bypass_supported,
3601        .setup_msi_irqs         = pnv_setup_msi_irqs,
3602        .teardown_msi_irqs      = pnv_teardown_msi_irqs,
3603        .enable_device_hook     = pnv_pci_enable_device_hook,
3604        .release_device         = pnv_pci_release_device,
3605        .window_alignment       = pnv_pci_window_alignment,
3606        .setup_bridge           = pnv_pci_setup_bridge,
3607        .reset_secondary_bus    = pnv_pci_reset_secondary_bus,
3608        .shutdown               = pnv_pci_ioda_shutdown,
3609};
3610
3611static const struct pci_controller_ops pnv_npu_ioda_controller_ops = {
3612        .dma_dev_setup          = pnv_pci_dma_dev_setup,
3613        .setup_msi_irqs         = pnv_setup_msi_irqs,
3614        .teardown_msi_irqs      = pnv_teardown_msi_irqs,
3615        .enable_device_hook     = pnv_pci_enable_device_hook,
3616        .window_alignment       = pnv_pci_window_alignment,
3617        .reset_secondary_bus    = pnv_pci_reset_secondary_bus,
3618        .shutdown               = pnv_pci_ioda_shutdown,
3619        .disable_device         = pnv_npu_disable_device,
3620};
3621
3622static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
3623        .enable_device_hook     = pnv_pci_enable_device_hook,
3624        .window_alignment       = pnv_pci_window_alignment,
3625        .reset_secondary_bus    = pnv_pci_reset_secondary_bus,
3626        .shutdown               = pnv_pci_ioda_shutdown,
3627};
3628
3629static void __init pnv_pci_init_ioda_phb(struct device_node *np,
3630                                         u64 hub_id, int ioda_type)
3631{
3632        struct pci_controller *hose;
3633        struct pnv_phb *phb;
3634        unsigned long size, m64map_off, m32map_off, pemap_off;
3635        unsigned long iomap_off = 0, dma32map_off = 0;
3636        struct resource r;
3637        const __be64 *prop64;
3638        const __be32 *prop32;
3639        int len;
3640        unsigned int segno;
3641        u64 phb_id;
3642        void *aux;
3643        long rc;
3644
3645        if (!of_device_is_available(np))
3646                return;
3647
3648        pr_info("Initializing %s PHB (%pOF)\n", pnv_phb_names[ioda_type], np);
3649
3650        prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
3651        if (!prop64) {
3652                pr_err("  Missing \"ibm,opal-phbid\" property !\n");
3653                return;
3654        }
3655        phb_id = be64_to_cpup(prop64);
3656        pr_debug("  PHB-ID  : 0x%016llx\n", phb_id);
3657
3658        phb = memblock_alloc(sizeof(*phb), SMP_CACHE_BYTES);
3659        if (!phb)
3660                panic("%s: Failed to allocate %zu bytes\n", __func__,
3661                      sizeof(*phb));
3662
3663        /* Allocate PCI controller */
3664        phb->hose = hose = pcibios_alloc_controller(np);
3665        if (!phb->hose) {
3666                pr_err("  Can't allocate PCI controller for %pOF\n",
3667                       np);
3668                memblock_free(__pa(phb), sizeof(struct pnv_phb));
3669                return;
3670        }
3671
3672        spin_lock_init(&phb->lock);
3673        prop32 = of_get_property(np, "bus-range", &len);
3674        if (prop32 && len == 8) {
3675                hose->first_busno = be32_to_cpu(prop32[0]);
3676                hose->last_busno = be32_to_cpu(prop32[1]);
3677        } else {
3678                pr_warn("  Broken <bus-range> on %pOF\n", np);
3679                hose->first_busno = 0;
3680                hose->last_busno = 0xff;
3681        }
3682        hose->private_data = phb;
3683        phb->hub_id = hub_id;
3684        phb->opal_id = phb_id;
3685        phb->type = ioda_type;
3686        mutex_init(&phb->ioda.pe_alloc_mutex);
3687
3688        /* Detect specific models for error handling */
3689        if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
3690                phb->model = PNV_PHB_MODEL_P7IOC;
3691        else if (of_device_is_compatible(np, "ibm,power8-pciex"))
3692                phb->model = PNV_PHB_MODEL_PHB3;
3693        else if (of_device_is_compatible(np, "ibm,power8-npu-pciex"))
3694                phb->model = PNV_PHB_MODEL_NPU;
3695        else if (of_device_is_compatible(np, "ibm,power9-npu-pciex"))
3696                phb->model = PNV_PHB_MODEL_NPU2;
3697        else
3698                phb->model = PNV_PHB_MODEL_UNKNOWN;
3699
3700        /* Initialize diagnostic data buffer */
3701        prop32 = of_get_property(np, "ibm,phb-diag-data-size", NULL);
3702        if (prop32)
3703                phb->diag_data_size = be32_to_cpup(prop32);
3704        else
3705                phb->diag_data_size = PNV_PCI_DIAG_BUF_SIZE;
3706
3707        phb->diag_data = memblock_alloc(phb->diag_data_size, SMP_CACHE_BYTES);
3708        if (!phb->diag_data)
3709                panic("%s: Failed to allocate %u bytes\n", __func__,
3710                      phb->diag_data_size);
3711
3712        /* Parse 32-bit and IO ranges (if any) */
3713        pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
3714
3715        /* Get registers */
3716        if (!of_address_to_resource(np, 0, &r)) {
3717                phb->regs_phys = r.start;
3718                phb->regs = ioremap(r.start, resource_size(&r));
3719                if (phb->regs == NULL)
3720                        pr_err("  Failed to map registers !\n");
3721        }
3722
3723        /* Initialize more IODA stuff */
3724        phb->ioda.total_pe_num = 1;
3725        prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
3726        if (prop32)
3727                phb->ioda.total_pe_num = be32_to_cpup(prop32);
3728        prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
3729        if (prop32)
3730                phb->ioda.reserved_pe_idx = be32_to_cpup(prop32);
3731
3732        /* Invalidate RID to PE# mapping */
3733        for (segno = 0; segno < ARRAY_SIZE(phb->ioda.pe_rmap); segno++)
3734                phb->ioda.pe_rmap[segno] = IODA_INVALID_PE;
3735
3736        /* Parse 64-bit MMIO range */
3737        pnv_ioda_parse_m64_window(phb);
3738
3739        phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
3740        /* FW Has already off top 64k of M32 space (MSI space) */
3741        phb->ioda.m32_size += 0x10000;
3742
3743        phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe_num;
3744        phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
3745        phb->ioda.io_size = hose->pci_io_size;
3746        phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe_num;
3747        phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
3748
3749        /* Calculate how many 32-bit TCE segments we have */
3750        phb->ioda.dma32_count = phb->ioda.m32_pci_base /
3751                                PNV_IODA1_DMA32_SEGSIZE;
3752
3753        /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
3754        size = _ALIGN_UP(max_t(unsigned, phb->ioda.total_pe_num, 8) / 8,
3755                        sizeof(unsigned long));
3756        m64map_off = size;
3757        size += phb->ioda.total_pe_num * sizeof(phb->ioda.m64_segmap[0]);
3758        m32map_off = size;
3759        size += phb->ioda.total_pe_num * sizeof(phb->ioda.m32_segmap[0]);
3760        if (phb->type == PNV_PHB_IODA1) {
3761                iomap_off = size;
3762                size += phb->ioda.total_pe_num * sizeof(phb->ioda.io_segmap[0]);
3763                dma32map_off = size;
3764                size += phb->ioda.dma32_count *
3765                        sizeof(phb->ioda.dma32_segmap[0]);
3766        }
3767        pemap_off = size;
3768        size += phb->ioda.total_pe_num * sizeof(struct pnv_ioda_pe);
3769        aux = memblock_alloc(size, SMP_CACHE_BYTES);
3770        if (!aux)
3771                panic("%s: Failed to allocate %lu bytes\n", __func__, size);
3772        phb->ioda.pe_alloc = aux;
3773        phb->ioda.m64_segmap = aux + m64map_off;
3774        phb->ioda.m32_segmap = aux + m32map_off;
3775        for (segno = 0; segno < phb->ioda.total_pe_num; segno++) {
3776                phb->ioda.m64_segmap[segno] = IODA_INVALID_PE;
3777                phb->ioda.m32_segmap[segno] = IODA_INVALID_PE;
3778        }
3779        if (phb->type == PNV_PHB_IODA1) {
3780                phb->ioda.io_segmap = aux + iomap_off;
3781                for (segno = 0; segno < phb->ioda.total_pe_num; segno++)
3782                        phb->ioda.io_segmap[segno] = IODA_INVALID_PE;
3783
3784                phb->ioda.dma32_segmap = aux + dma32map_off;
3785                for (segno = 0; segno < phb->ioda.dma32_count; segno++)
3786                        phb->ioda.dma32_segmap[segno] = IODA_INVALID_PE;
3787        }
3788        phb->ioda.pe_array = aux + pemap_off;
3789
3790        /*
3791         * Choose PE number for root bus, which shouldn't have
3792         * M64 resources consumed by its child devices. To pick
3793         * the PE number adjacent to the reserved one if possible.
3794         */
3795        pnv_ioda_reserve_pe(phb, phb->ioda.reserved_pe_idx);
3796        if (phb->ioda.reserved_pe_idx == 0) {
3797                phb->ioda.root_pe_idx = 1;
3798                pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
3799        } else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1)) {
3800                phb->ioda.root_pe_idx = phb->ioda.reserved_pe_idx - 1;
3801                pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
3802        } else {
3803                phb->ioda.root_pe_idx = IODA_INVALID_PE;
3804        }
3805
3806        INIT_LIST_HEAD(&phb->ioda.pe_list);
3807        mutex_init(&phb->ioda.pe_list_mutex);
3808
3809        /* Calculate how many 32-bit TCE segments we have */
3810        phb->ioda.dma32_count = phb->ioda.m32_pci_base /
3811                                PNV_IODA1_DMA32_SEGSIZE;
3812
3813#if 0 /* We should really do that ... */
3814        rc = opal_pci_set_phb_mem_window(opal->phb_id,
3815                                         window_type,
3816                                         window_num,
3817                                         starting_real_address,
3818                                         starting_pci_address,
3819                                         segment_size);
3820#endif
3821
3822        pr_info("  %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
3823                phb->ioda.total_pe_num, phb->ioda.reserved_pe_idx,
3824                phb->ioda.m32_size, phb->ioda.m32_segsize);
3825        if (phb->ioda.m64_size)
3826                pr_info("                 M64: 0x%lx [segment=0x%lx]\n",
3827                        phb->ioda.m64_size, phb->ioda.m64_segsize);
3828        if (phb->ioda.io_size)
3829                pr_info("                  IO: 0x%x [segment=0x%x]\n",
3830                        phb->ioda.io_size, phb->ioda.io_segsize);
3831
3832
3833        phb->hose->ops = &pnv_pci_ops;
3834        phb->get_pe_state = pnv_ioda_get_pe_state;
3835        phb->freeze_pe = pnv_ioda_freeze_pe;
3836        phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
3837
3838        /* Setup MSI support */
3839        pnv_pci_init_ioda_msis(phb);
3840
3841        /*
3842         * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
3843         * to let the PCI core do resource assignment. It's supposed
3844         * that the PCI core will do correct I/O and MMIO alignment
3845         * for the P2P bridge bars so that each PCI bus (excluding
3846         * the child P2P bridges) can form individual PE.
3847         */
3848        ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
3849
3850        switch (phb->type) {
3851        case PNV_PHB_NPU_NVLINK:
3852                hose->controller_ops = pnv_npu_ioda_controller_ops;
3853                break;
3854        case PNV_PHB_NPU_OCAPI:
3855                hose->controller_ops = pnv_npu_ocapi_ioda_controller_ops;
3856                break;
3857        default:
3858                phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
3859                hose->controller_ops = pnv_pci_ioda_controller_ops;
3860        }
3861
3862        ppc_md.pcibios_default_alignment = pnv_pci_default_alignment;
3863
3864#ifdef CONFIG_PCI_IOV
3865        ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
3866        ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
3867        ppc_md.pcibios_sriov_enable = pnv_pcibios_sriov_enable;
3868        ppc_md.pcibios_sriov_disable = pnv_pcibios_sriov_disable;
3869#endif
3870
3871        pci_add_flags(PCI_REASSIGN_ALL_RSRC);
3872
3873        /* Reset IODA tables to a clean state */
3874        rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
3875        if (rc)
3876                pr_warn("  OPAL Error %ld performing IODA table reset !\n", rc);
3877
3878        /*
3879         * If we're running in kdump kernel, the previous kernel never
3880         * shutdown PCI devices correctly. We already got IODA table
3881         * cleaned out. So we have to issue PHB reset to stop all PCI
3882         * transactions from previous kernel. The ppc_pci_reset_phbs
3883         * kernel parameter will force this reset too. Additionally,
3884         * if the IODA reset above failed then use a bigger hammer.
3885         * This can happen if we get a PHB fatal error in very early
3886         * boot.
3887         */
3888        if (is_kdump_kernel() || pci_reset_phbs || rc) {
3889                pr_info("  Issue PHB reset ...\n");
3890                pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
3891                pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
3892        }
3893
3894        /* Remove M64 resource if we can't configure it successfully */
3895        if (!phb->init_m64 || phb->init_m64(phb))
3896                hose->mem_resources[1].flags = 0;
3897}
3898
3899void __init pnv_pci_init_ioda2_phb(struct device_node *np)
3900{
3901        pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
3902}
3903
3904void __init pnv_pci_init_npu_phb(struct device_node *np)
3905{
3906        pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU_NVLINK);
3907}
3908
3909void __init pnv_pci_init_npu2_opencapi_phb(struct device_node *np)
3910{
3911        pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU_OCAPI);
3912}
3913
3914static void pnv_npu2_opencapi_cfg_size_fixup(struct pci_dev *dev)
3915{
3916        struct pci_controller *hose = pci_bus_to_host(dev->bus);
3917        struct pnv_phb *phb = hose->private_data;
3918
3919        if (!machine_is(powernv))
3920                return;
3921
3922        if (phb->type == PNV_PHB_NPU_OCAPI)
3923                dev->cfg_size = PCI_CFG_SPACE_EXP_SIZE;
3924}
3925DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, pnv_npu2_opencapi_cfg_size_fixup);
3926
3927void __init pnv_pci_init_ioda_hub(struct device_node *np)
3928{
3929        struct device_node *phbn;
3930        const __be64 *prop64;
3931        u64 hub_id;
3932
3933        pr_info("Probing IODA IO-Hub %pOF\n", np);
3934
3935        prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
3936        if (!prop64) {
3937                pr_err(" Missing \"ibm,opal-hubid\" property !\n");
3938                return;
3939        }
3940        hub_id = be64_to_cpup(prop64);
3941        pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
3942
3943        /* Count child PHBs */
3944        for_each_child_of_node(np, phbn) {
3945                /* Look for IODA1 PHBs */
3946                if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
3947                        pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
3948        }
3949}
3950