LXR linux/drivers/pci/controller/dwc/pcie-designware.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Synopsys DesignWare PCIe host controller driver
   4 *
   5 * Copyright (C) 2013 Samsung Electronics Co., Ltd.
   6 *              https://www.samsung.com
   7 *
   8 * Author: Jingoo Han <jg1.han@samsung.com>
   9 */
  10
  11#include <linux/delay.h>
  12#include <linux/of.h>
  13#include <linux/of_platform.h>
  14#include <linux/types.h>
  15
  16#include "../../pci.h"
  17#include "pcie-designware.h"
  18
  19/*
  20 * These interfaces resemble the pci_find_*capability() interfaces, but these
  21 * are for configuring host controllers, which are bridges *to* PCI devices but
  22 * are not PCI devices themselves.
  23 */
  24static u8 __dw_pcie_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,
  25                                  u8 cap)
  26{
  27        u8 cap_id, next_cap_ptr;
  28        u16 reg;
  29
  30        if (!cap_ptr)
  31                return 0;
  32
  33        reg = dw_pcie_readw_dbi(pci, cap_ptr);
  34        cap_id = (reg & 0x00ff);
  35
  36        if (cap_id > PCI_CAP_ID_MAX)
  37                return 0;
  38
  39        if (cap_id == cap)
  40                return cap_ptr;
  41
  42        next_cap_ptr = (reg & 0xff00) >> 8;
  43        return __dw_pcie_find_next_cap(pci, next_cap_ptr, cap);
  44}
  45
  46u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap)
  47{
  48        u8 next_cap_ptr;
  49        u16 reg;
  50
  51        reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
  52        next_cap_ptr = (reg & 0x00ff);
  53
  54        return __dw_pcie_find_next_cap(pci, next_cap_ptr, cap);
  55}
  56EXPORT_SYMBOL_GPL(dw_pcie_find_capability);
  57
  58static u16 dw_pcie_find_next_ext_capability(struct dw_pcie *pci, u16 start,
  59                                            u8 cap)
  60{
  61        u32 header;
  62        int ttl;
  63        int pos = PCI_CFG_SPACE_SIZE;
  64
  65        /* minimum 8 bytes per capability */
  66        ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
  67
  68        if (start)
  69                pos = start;
  70
  71        header = dw_pcie_readl_dbi(pci, pos);
  72        /*
  73         * If we have no capabilities, this is indicated by cap ID,
  74         * cap version and next pointer all being 0.
  75         */
  76        if (header == 0)
  77                return 0;
  78
  79        while (ttl-- > 0) {
  80                if (PCI_EXT_CAP_ID(header) == cap && pos != start)
  81                        return pos;
  82
  83                pos = PCI_EXT_CAP_NEXT(header);
  84                if (pos < PCI_CFG_SPACE_SIZE)
  85                        break;
  86
  87                header = dw_pcie_readl_dbi(pci, pos);
  88        }
  89
  90        return 0;
  91}
  92
  93u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap)
  94{
  95        return dw_pcie_find_next_ext_capability(pci, 0, cap);
  96}
  97EXPORT_SYMBOL_GPL(dw_pcie_find_ext_capability);
  98
  99int dw_pcie_read(void __iomem *addr, int size, u32 *val)
 100{
 101        if (!IS_ALIGNED((uintptr_t)addr, size)) {
 102                *val = 0;
 103                return PCIBIOS_BAD_REGISTER_NUMBER;
 104        }
 105
 106        if (size == 4) {
 107                *val = readl(addr);
 108        } else if (size == 2) {
 109                *val = readw(addr);
 110        } else if (size == 1) {
 111                *val = readb(addr);
 112        } else {
 113                *val = 0;
 114                return PCIBIOS_BAD_REGISTER_NUMBER;
 115        }
 116
 117        return PCIBIOS_SUCCESSFUL;
 118}
 119EXPORT_SYMBOL_GPL(dw_pcie_read);
 120
 121int dw_pcie_write(void __iomem *addr, int size, u32 val)
 122{
 123        if (!IS_ALIGNED((uintptr_t)addr, size))
 124                return PCIBIOS_BAD_REGISTER_NUMBER;
 125
 126        if (size == 4)
 127                writel(val, addr);
 128        else if (size == 2)
 129                writew(val, addr);
 130        else if (size == 1)
 131                writeb(val, addr);
 132        else
 133                return PCIBIOS_BAD_REGISTER_NUMBER;
 134
 135        return PCIBIOS_SUCCESSFUL;
 136}
 137EXPORT_SYMBOL_GPL(dw_pcie_write);
 138
 139u32 dw_pcie_read_dbi(struct dw_pcie *pci, u32 reg, size_t size)
 140{
 141        int ret;
 142        u32 val;
 143
 144        if (pci->ops && pci->ops->read_dbi)
 145                return pci->ops->read_dbi(pci, pci->dbi_base, reg, size);
 146
 147        ret = dw_pcie_read(pci->dbi_base + reg, size, &val);
 148        if (ret)
 149                dev_err(pci->dev, "Read DBI address failed\n");
 150
 151        return val;
 152}
 153EXPORT_SYMBOL_GPL(dw_pcie_read_dbi);
 154
 155void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val)
 156{
 157        int ret;
 158
 159        if (pci->ops && pci->ops->write_dbi) {
 160                pci->ops->write_dbi(pci, pci->dbi_base, reg, size, val);
 161                return;
 162        }
 163
 164        ret = dw_pcie_write(pci->dbi_base + reg, size, val);
 165        if (ret)
 166                dev_err(pci->dev, "Write DBI address failed\n");
 167}
 168EXPORT_SYMBOL_GPL(dw_pcie_write_dbi);
 169
 170void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val)
 171{
 172        int ret;
 173
 174        if (pci->ops && pci->ops->write_dbi2) {
 175                pci->ops->write_dbi2(pci, pci->dbi_base2, reg, size, val);
 176                return;
 177        }
 178
 179        ret = dw_pcie_write(pci->dbi_base2 + reg, size, val);
 180        if (ret)
 181                dev_err(pci->dev, "write DBI address failed\n");
 182}
 183
 184static u32 dw_pcie_readl_atu(struct dw_pcie *pci, u32 reg)
 185{
 186        int ret;
 187        u32 val;
 188
 189        if (pci->ops && pci->ops->read_dbi)
 190                return pci->ops->read_dbi(pci, pci->atu_base, reg, 4);
 191
 192        ret = dw_pcie_read(pci->atu_base + reg, 4, &val);
 193        if (ret)
 194                dev_err(pci->dev, "Read ATU address failed\n");
 195
 196        return val;
 197}
 198
 199static void dw_pcie_writel_atu(struct dw_pcie *pci, u32 reg, u32 val)
 200{
 201        int ret;
 202
 203        if (pci->ops && pci->ops->write_dbi) {
 204                pci->ops->write_dbi(pci, pci->atu_base, reg, 4, val);
 205                return;
 206        }
 207
 208        ret = dw_pcie_write(pci->atu_base + reg, 4, val);
 209        if (ret)
 210                dev_err(pci->dev, "Write ATU address failed\n");
 211}
 212
 213static u32 dw_pcie_readl_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg)
 214{
 215        u32 offset = PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(index);
 216
 217        return dw_pcie_readl_atu(pci, offset + reg);
 218}
 219
 220static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg,
 221                                     u32 val)
 222{
 223        u32 offset = PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(index);
 224
 225        dw_pcie_writel_atu(pci, offset + reg, val);
 226}
 227
 228static inline u32 dw_pcie_enable_ecrc(u32 val)
 229{
 230        /*
 231         * DesignWare core version 4.90A has a design issue where the 'TD'
 232         * bit in the Control register-1 of the ATU outbound region acts
 233         * like an override for the ECRC setting, i.e., the presence of TLP
 234         * Digest (ECRC) in the outgoing TLPs is solely determined by this
 235         * bit. This is contrary to the PCIe spec which says that the
 236         * enablement of the ECRC is solely determined by the AER
 237         * registers.
 238         *
 239         * Because of this, even when the ECRC is enabled through AER
 240         * registers, the transactions going through ATU won't have TLP
 241         * Digest as there is no way the PCI core AER code could program
 242         * the TD bit which is specific to the DesignWare core.
 243         *
 244         * The best way to handle this scenario is to program the TD bit
 245         * always. It affects only the traffic from root port to downstream
 246         * devices.
 247         *
 248         * At this point,
 249         * When ECRC is enabled in AER registers, everything works normally
 250         * When ECRC is NOT enabled in AER registers, then,
 251         * on Root Port:- TLP Digest (DWord size) gets appended to each packet
 252         *                even through it is not required. Since downstream
 253         *                TLPs are mostly for configuration accesses and BAR
 254         *                accesses, they are not in critical path and won't
 255         *                have much negative effect on the performance.
 256         * on End Point:- TLP Digest is received for some/all the packets coming
 257         *                from the root port. TLP Digest is ignored because,
 258         *                as per the PCIe Spec r5.0 v1.0 section 2.2.3
 259         *                "TLP Digest Rules", when an endpoint receives TLP
 260         *                Digest when its ECRC check functionality is disabled
 261         *                in AER registers, received TLP Digest is just ignored.
 262         * Since there is no issue or error reported either side, best way to
 263         * handle the scenario is to program TD bit by default.
 264         */
 265
 266        return val | PCIE_ATU_TD;
 267}
 268
 269static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no,
 270                                             int index, int type,
 271                                             u64 cpu_addr, u64 pci_addr,
 272                                             u64 size)
 273{
 274        u32 retries, val;
 275        u64 limit_addr = cpu_addr + size - 1;
 276
 277        dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_BASE,
 278                                 lower_32_bits(cpu_addr));
 279        dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_BASE,
 280                                 upper_32_bits(cpu_addr));
 281        dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_LIMIT,
 282                                 lower_32_bits(limit_addr));
 283        dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_LIMIT,
 284                                 upper_32_bits(limit_addr));
 285        dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_TARGET,
 286                                 lower_32_bits(pci_addr));
 287        dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET,
 288                                 upper_32_bits(pci_addr));
 289        val = type | PCIE_ATU_FUNC_NUM(func_no);
 290        val = upper_32_bits(size - 1) ?
 291                val | PCIE_ATU_INCREASE_REGION_SIZE : val;
 292        if (pci->version == 0x490A)
 293                val = dw_pcie_enable_ecrc(val);
 294        dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, val);
 295        dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
 296                                 PCIE_ATU_ENABLE);
 297
 298        /*
 299         * Make sure ATU enable takes effect before any subsequent config
 300         * and I/O accesses.
 301         */
 302        for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
 303                val = dw_pcie_readl_ob_unroll(pci, index,
 304                                              PCIE_ATU_UNR_REGION_CTRL2);
 305                if (val & PCIE_ATU_ENABLE)
 306                        return;
 307
 308                mdelay(LINK_WAIT_IATU);
 309        }
 310        dev_err(pci->dev, "Outbound iATU is not being enabled\n");
 311}
 312
 313static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no,
 314                                        int index, int type, u64 cpu_addr,
 315                                        u64 pci_addr, u64 size)
 316{
 317        u32 retries, val;
 318
 319        if (pci->ops && pci->ops->cpu_addr_fixup)
 320                cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr);
 321
 322        if (pci->iatu_unroll_enabled) {
 323                dw_pcie_prog_outbound_atu_unroll(pci, func_no, index, type,
 324                                                 cpu_addr, pci_addr, size);
 325                return;
 326        }
 327
 328        dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
 329                           PCIE_ATU_REGION_OUTBOUND | index);
 330        dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_BASE,
 331                           lower_32_bits(cpu_addr));
 332        dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_BASE,
 333                           upper_32_bits(cpu_addr));
 334        dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT,
 335                           lower_32_bits(cpu_addr + size - 1));
 336        if (pci->version >= 0x460A)
 337                dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_LIMIT,
 338                                   upper_32_bits(cpu_addr + size - 1));
 339        dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET,
 340                           lower_32_bits(pci_addr));
 341        dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET,
 342                           upper_32_bits(pci_addr));
 343        val = type | PCIE_ATU_FUNC_NUM(func_no);
 344        val = ((upper_32_bits(size - 1)) && (pci->version >= 0x460A)) ?
 345                val | PCIE_ATU_INCREASE_REGION_SIZE : val;
 346        if (pci->version == 0x490A)
 347                val = dw_pcie_enable_ecrc(val);
 348        dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val);
 349        dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE);
 350
 351        /*
 352         * Make sure ATU enable takes effect before any subsequent config
 353         * and I/O accesses.
 354         */
 355        for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
 356                val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
 357                if (val & PCIE_ATU_ENABLE)
 358                        return;
 359
 360                mdelay(LINK_WAIT_IATU);
 361        }
 362        dev_err(pci->dev, "Outbound iATU is not being enabled\n");
 363}
 364
 365void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
 366                               u64 cpu_addr, u64 pci_addr, u64 size)
 367{
 368        __dw_pcie_prog_outbound_atu(pci, 0, index, type,
 369                                    cpu_addr, pci_addr, size);
 370}
 371
 372void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
 373                                  int type, u64 cpu_addr, u64 pci_addr,
 374                                  u64 size)
 375{
 376        __dw_pcie_prog_outbound_atu(pci, func_no, index, type,
 377                                    cpu_addr, pci_addr, size);
 378}
 379
 380static u32 dw_pcie_readl_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg)
 381{
 382        u32 offset = PCIE_GET_ATU_INB_UNR_REG_OFFSET(index);
 383
 384        return dw_pcie_readl_atu(pci, offset + reg);
 385}
 386
 387static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg,
 388                                     u32 val)
 389{
 390        u32 offset = PCIE_GET_ATU_INB_UNR_REG_OFFSET(index);
 391
 392        dw_pcie_writel_atu(pci, offset + reg, val);
 393}
 394
 395static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, u8 func_no,
 396                                           int index, int bar, u64 cpu_addr,
 397                                           enum dw_pcie_as_type as_type)
 398{
 399        int type;
 400        u32 retries, val;
 401
 402        dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_LOWER_TARGET,
 403                                 lower_32_bits(cpu_addr));
 404        dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET,
 405                                 upper_32_bits(cpu_addr));
 406
 407        switch (as_type) {
 408        case DW_PCIE_AS_MEM:
 409                type = PCIE_ATU_TYPE_MEM;
 410                break;
 411        case DW_PCIE_AS_IO:
 412                type = PCIE_ATU_TYPE_IO;
 413                break;
 414        default:
 415                return -EINVAL;
 416        }
 417
 418        dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type |
 419                                 PCIE_ATU_FUNC_NUM(func_no));
 420        dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
 421                                 PCIE_ATU_FUNC_NUM_MATCH_EN |
 422                                 PCIE_ATU_ENABLE |
 423                                 PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
 424
 425        /*
 426         * Make sure ATU enable takes effect before any subsequent config
 427         * and I/O accesses.
 428         */
 429        for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
 430                val = dw_pcie_readl_ib_unroll(pci, index,
 431                                              PCIE_ATU_UNR_REGION_CTRL2);
 432                if (val & PCIE_ATU_ENABLE)
 433                        return 0;
 434
 435                mdelay(LINK_WAIT_IATU);
 436        }
 437        dev_err(pci->dev, "Inbound iATU is not being enabled\n");
 438
 439        return -EBUSY;
 440}
 441
 442int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
 443                             int bar, u64 cpu_addr,
 444                             enum dw_pcie_as_type as_type)
 445{
 446        int type;
 447        u32 retries, val;
 448
 449        if (pci->iatu_unroll_enabled)
 450                return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index, bar,
 451                                                       cpu_addr, as_type);
 452
 453        dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, PCIE_ATU_REGION_INBOUND |
 454                           index);
 455        dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET, lower_32_bits(cpu_addr));
 456        dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET, upper_32_bits(cpu_addr));
 457
 458        switch (as_type) {
 459        case DW_PCIE_AS_MEM:
 460                type = PCIE_ATU_TYPE_MEM;
 461                break;
 462        case DW_PCIE_AS_IO:
 463                type = PCIE_ATU_TYPE_IO;
 464                break;
 465        default:
 466                return -EINVAL;
 467        }
 468
 469        dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
 470                           PCIE_ATU_FUNC_NUM(func_no));
 471        dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE |
 472                           PCIE_ATU_FUNC_NUM_MATCH_EN |
 473                           PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
 474
 475        /*
 476         * Make sure ATU enable takes effect before any subsequent config
 477         * and I/O accesses.
 478         */
 479        for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
 480                val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
 481                if (val & PCIE_ATU_ENABLE)
 482                        return 0;
 483
 484                mdelay(LINK_WAIT_IATU);
 485        }
 486        dev_err(pci->dev, "Inbound iATU is not being enabled\n");
 487
 488        return -EBUSY;
 489}
 490
 491void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
 492                         enum dw_pcie_region_type type)
 493{
 494        int region;
 495
 496        switch (type) {
 497        case DW_PCIE_REGION_INBOUND:
 498                region = PCIE_ATU_REGION_INBOUND;
 499                break;
 500        case DW_PCIE_REGION_OUTBOUND:
 501                region = PCIE_ATU_REGION_OUTBOUND;
 502                break;
 503        default:
 504                return;
 505        }
 506
 507        dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, region | index);
 508        dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, ~(u32)PCIE_ATU_ENABLE);
 509}
 510
 511int dw_pcie_wait_for_link(struct dw_pcie *pci)
 512{
 513        int retries;
 514
 515        /* Check if the link is up or not */
 516        for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
 517                if (dw_pcie_link_up(pci)) {
 518                        dev_info(pci->dev, "Link up\n");
 519                        return 0;
 520                }
 521                usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
 522        }
 523
 524        dev_info(pci->dev, "Phy link never came up\n");
 525
 526        return -ETIMEDOUT;
 527}
 528EXPORT_SYMBOL_GPL(dw_pcie_wait_for_link);
 529
 530int dw_pcie_link_up(struct dw_pcie *pci)
 531{
 532        u32 val;
 533
 534        if (pci->ops && pci->ops->link_up)
 535                return pci->ops->link_up(pci);
 536
 537        val = readl(pci->dbi_base + PCIE_PORT_DEBUG1);
 538        return ((val & PCIE_PORT_DEBUG1_LINK_UP) &&
 539                (!(val & PCIE_PORT_DEBUG1_LINK_IN_TRAINING)));
 540}
 541
 542void dw_pcie_upconfig_setup(struct dw_pcie *pci)
 543{
 544        u32 val;
 545
 546        val = dw_pcie_readl_dbi(pci, PCIE_PORT_MULTI_LANE_CTRL);
 547        val |= PORT_MLTI_UPCFG_SUPPORT;
 548        dw_pcie_writel_dbi(pci, PCIE_PORT_MULTI_LANE_CTRL, val);
 549}
 550EXPORT_SYMBOL_GPL(dw_pcie_upconfig_setup);
 551
 552static void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 link_gen)
 553{
 554        u32 cap, ctrl2, link_speed;
 555        u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
 556
 557        cap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
 558        ctrl2 = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCTL2);
 559        ctrl2 &= ~PCI_EXP_LNKCTL2_TLS;
 560
 561        switch (pcie_link_speed[link_gen]) {
 562        case PCIE_SPEED_2_5GT:
 563                link_speed = PCI_EXP_LNKCTL2_TLS_2_5GT;
 564                break;
 565        case PCIE_SPEED_5_0GT:
 566                link_speed = PCI_EXP_LNKCTL2_TLS_5_0GT;
 567                break;
 568        case PCIE_SPEED_8_0GT:
 569                link_speed = PCI_EXP_LNKCTL2_TLS_8_0GT;
 570                break;
 571        case PCIE_SPEED_16_0GT:
 572                link_speed = PCI_EXP_LNKCTL2_TLS_16_0GT;
 573                break;
 574        default:
 575                /* Use hardware capability */
 576                link_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, cap);
 577                ctrl2 &= ~PCI_EXP_LNKCTL2_HASD;
 578                break;
 579        }
 580
 581        dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCTL2, ctrl2 | link_speed);
 582
 583        cap &= ~((u32)PCI_EXP_LNKCAP_SLS);
 584        dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, cap | link_speed);
 585
 586}
 587
 588static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci)
 589{
 590        u32 val;
 591
 592        val = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT);
 593        if (val == 0xffffffff)
 594                return 1;
 595
 596        return 0;
 597}
 598
 599static void dw_pcie_iatu_detect_regions_unroll(struct dw_pcie *pci)
 600{
 601        int max_region, i, ob = 0, ib = 0;
 602        u32 val;
 603
 604        max_region = min((int)pci->atu_size / 512, 256);
 605
 606        for (i = 0; i < max_region; i++) {
 607                dw_pcie_writel_ob_unroll(pci, i, PCIE_ATU_UNR_LOWER_TARGET,
 608                                        0x11110000);
 609
 610                val = dw_pcie_readl_ob_unroll(pci, i, PCIE_ATU_UNR_LOWER_TARGET);
 611                if (val == 0x11110000)
 612                        ob++;
 613                else
 614                        break;
 615        }
 616
 617        for (i = 0; i < max_region; i++) {
 618                dw_pcie_writel_ib_unroll(pci, i, PCIE_ATU_UNR_LOWER_TARGET,
 619                                        0x11110000);
 620
 621                val = dw_pcie_readl_ib_unroll(pci, i, PCIE_ATU_UNR_LOWER_TARGET);
 622                if (val == 0x11110000)
 623                        ib++;
 624                else
 625                        break;
 626        }
 627        pci->num_ib_windows = ib;
 628        pci->num_ob_windows = ob;
 629}
 630
 631static void dw_pcie_iatu_detect_regions(struct dw_pcie *pci)
 632{
 633        int max_region, i, ob = 0, ib = 0;
 634        u32 val;
 635
 636        dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, 0xFF);
 637        max_region = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT) + 1;
 638
 639        for (i = 0; i < max_region; i++) {
 640                dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, PCIE_ATU_REGION_OUTBOUND | i);
 641                dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET, 0x11110000);
 642                val = dw_pcie_readl_dbi(pci, PCIE_ATU_LOWER_TARGET);
 643                if (val == 0x11110000)
 644                        ob++;
 645                else
 646                        break;
 647        }
 648
 649        for (i = 0; i < max_region; i++) {
 650                dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, PCIE_ATU_REGION_INBOUND | i);
 651                dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET, 0x11110000);
 652                val = dw_pcie_readl_dbi(pci, PCIE_ATU_LOWER_TARGET);
 653                if (val == 0x11110000)
 654                        ib++;
 655                else
 656                        break;
 657        }
 658
 659        pci->num_ib_windows = ib;
 660        pci->num_ob_windows = ob;
 661}
 662
 663void dw_pcie_iatu_detect(struct dw_pcie *pci)
 664{
 665        struct device *dev = pci->dev;
 666        struct platform_device *pdev = to_platform_device(dev);
 667
 668        if (pci->version >= 0x480A || (!pci->version &&
 669                                       dw_pcie_iatu_unroll_enabled(pci))) {
 670                pci->iatu_unroll_enabled = true;
 671                if (!pci->atu_base) {
 672                        struct resource *res =
 673                                platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu");
 674                        if (res)
 675                                pci->atu_size = resource_size(res);
 676                        pci->atu_base = devm_ioremap_resource(dev, res);
 677                        if (IS_ERR(pci->atu_base))
 678                                pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET;
 679                }
 680
 681                if (!pci->atu_size)
 682                        /* Pick a minimal default, enough for 8 in and 8 out windows */
 683                        pci->atu_size = SZ_4K;
 684
 685                dw_pcie_iatu_detect_regions_unroll(pci);
 686        } else
 687                dw_pcie_iatu_detect_regions(pci);
 688
 689        dev_info(pci->dev, "iATU unroll: %s\n", pci->iatu_unroll_enabled ?
 690                "enabled" : "disabled");
 691
 692        dev_info(pci->dev, "Detected iATU regions: %u outbound, %u inbound",
 693                 pci->num_ob_windows, pci->num_ib_windows);
 694}
 695
 696void dw_pcie_setup(struct dw_pcie *pci)
 697{
 698        u32 val;
 699        struct device *dev = pci->dev;
 700        struct device_node *np = dev->of_node;
 701
 702        if (pci->link_gen > 0)
 703                dw_pcie_link_set_max_speed(pci, pci->link_gen);
 704
 705        /* Configure Gen1 N_FTS */
 706        if (pci->n_fts[0]) {
 707                val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
 708                val &= ~(PORT_AFR_N_FTS_MASK | PORT_AFR_CC_N_FTS_MASK);
 709                val |= PORT_AFR_N_FTS(pci->n_fts[0]);
 710                val |= PORT_AFR_CC_N_FTS(pci->n_fts[0]);
 711                dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
 712        }
 713
 714        /* Configure Gen2+ N_FTS */
 715        if (pci->n_fts[1]) {
 716                val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
 717                val &= ~PORT_LOGIC_N_FTS_MASK;
 718                val |= pci->n_fts[pci->link_gen - 1];
 719                dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
 720        }
 721
 722        val = dw_pcie_readl_dbi(pci, PCIE_PORT_LINK_CONTROL);
 723        val &= ~PORT_LINK_FAST_LINK_MODE;
 724        val |= PORT_LINK_DLL_LINK_EN;
 725        dw_pcie_writel_dbi(pci, PCIE_PORT_LINK_CONTROL, val);
 726
 727        of_property_read_u32(np, "num-lanes", &pci->num_lanes);
 728        if (!pci->num_lanes) {
 729                dev_dbg(pci->dev, "Using h/w default number of lanes\n");
 730                return;
 731        }
 732
 733        /* Set the number of lanes */
 734        val &= ~PORT_LINK_FAST_LINK_MODE;
 735        val &= ~PORT_LINK_MODE_MASK;
 736        switch (pci->num_lanes) {
 737        case 1:
 738                val |= PORT_LINK_MODE_1_LANES;
 739                break;
 740        case 2:
 741                val |= PORT_LINK_MODE_2_LANES;
 742                break;
 743        case 4:
 744                val |= PORT_LINK_MODE_4_LANES;
 745                break;
 746        case 8:
 747                val |= PORT_LINK_MODE_8_LANES;
 748                break;
 749        default:
 750                dev_err(pci->dev, "num-lanes %u: invalid value\n", pci->num_lanes);
 751                return;
 752        }
 753        dw_pcie_writel_dbi(pci, PCIE_PORT_LINK_CONTROL, val);
 754
 755        /* Set link width speed control register */
 756        val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
 757        val &= ~PORT_LOGIC_LINK_WIDTH_MASK;
 758        switch (pci->num_lanes) {
 759        case 1:
 760                val |= PORT_LOGIC_LINK_WIDTH_1_LANES;
 761                break;
 762        case 2:
 763                val |= PORT_LOGIC_LINK_WIDTH_2_LANES;
 764                break;
 765        case 4:
 766                val |= PORT_LOGIC_LINK_WIDTH_4_LANES;
 767                break;
 768        case 8:
 769                val |= PORT_LOGIC_LINK_WIDTH_8_LANES;
 770                break;
 771        }
 772        dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
 773
 774        if (of_property_read_bool(np, "snps,enable-cdm-check")) {
 775                val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
 776                val |= PCIE_PL_CHK_REG_CHK_REG_CONTINUOUS |
 777                       PCIE_PL_CHK_REG_CHK_REG_START;
 778                dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);
 779        }
 780}
 781