LXR linux/drivers/pci/access.c

   1// SPDX-License-Identifier: GPL-2.0
   2#include <linux/pci.h>
   3#include <linux/module.h>
   4#include <linux/slab.h>
   5#include <linux/ioport.h>
   6#include <linux/wait.h>
   7
   8#include "pci.h"
   9
  10/*
  11 * This interrupt-safe spinlock protects all accesses to PCI
  12 * configuration space.
  13 */
  14
  15DEFINE_RAW_SPINLOCK(pci_lock);
  16
  17/*
  18 * Wrappers for all PCI configuration access functions.  They just check
  19 * alignment, do locking and call the low-level functions pointed to
  20 * by pci_dev->ops.
  21 */
  22
  23#define PCI_byte_BAD 0
  24#define PCI_word_BAD (pos & 1)
  25#define PCI_dword_BAD (pos & 3)
  26
  27#ifdef CONFIG_PCI_LOCKLESS_CONFIG
  28# define pci_lock_config(f)     do { (void)(f); } while (0)
  29# define pci_unlock_config(f)   do { (void)(f); } while (0)
  30#else
  31# define pci_lock_config(f)     raw_spin_lock_irqsave(&pci_lock, f)
  32# define pci_unlock_config(f)   raw_spin_unlock_irqrestore(&pci_lock, f)
  33#endif
  34
  35#define PCI_OP_READ(size, type, len) \
  36int pci_bus_read_config_##size \
  37        (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \
  38{                                                                       \
  39        int res;                                                        \
  40        unsigned long flags;                                            \
  41        u32 data = 0;                                                   \
  42        if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER;       \
  43        pci_lock_config(flags);                                         \
  44        res = bus->ops->read(bus, devfn, pos, len, &data);              \
  45        *value = (type)data;                                            \
  46        pci_unlock_config(flags);                                       \
  47        return res;                                                     \
  48}
  49
  50#define PCI_OP_WRITE(size, type, len) \
  51int pci_bus_write_config_##size \
  52        (struct pci_bus *bus, unsigned int devfn, int pos, type value)  \
  53{                                                                       \
  54        int res;                                                        \
  55        unsigned long flags;                                            \
  56        if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER;       \
  57        pci_lock_config(flags);                                         \
  58        res = bus->ops->write(bus, devfn, pos, len, value);             \
  59        pci_unlock_config(flags);                                       \
  60        return res;                                                     \
  61}
  62
  63PCI_OP_READ(byte, u8, 1)
  64PCI_OP_READ(word, u16, 2)
  65PCI_OP_READ(dword, u32, 4)
  66PCI_OP_WRITE(byte, u8, 1)
  67PCI_OP_WRITE(word, u16, 2)
  68PCI_OP_WRITE(dword, u32, 4)
  69
  70EXPORT_SYMBOL(pci_bus_read_config_byte);
  71EXPORT_SYMBOL(pci_bus_read_config_word);
  72EXPORT_SYMBOL(pci_bus_read_config_dword);
  73EXPORT_SYMBOL(pci_bus_write_config_byte);
  74EXPORT_SYMBOL(pci_bus_write_config_word);
  75EXPORT_SYMBOL(pci_bus_write_config_dword);
  76
  77int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn,
  78                            int where, int size, u32 *val)
  79{
  80        void __iomem *addr;
  81
  82        addr = bus->ops->map_bus(bus, devfn, where);
  83        if (!addr) {
  84                *val = ~0;
  85                return PCIBIOS_DEVICE_NOT_FOUND;
  86        }
  87
  88        if (size == 1)
  89                *val = readb(addr);
  90        else if (size == 2)
  91                *val = readw(addr);
  92        else
  93                *val = readl(addr);
  94
  95        return PCIBIOS_SUCCESSFUL;
  96}
  97EXPORT_SYMBOL_GPL(pci_generic_config_read);
  98
  99int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn,
 100                             int where, int size, u32 val)
 101{
 102        void __iomem *addr;
 103
 104        addr = bus->ops->map_bus(bus, devfn, where);
 105        if (!addr)
 106                return PCIBIOS_DEVICE_NOT_FOUND;
 107
 108        if (size == 1)
 109                writeb(val, addr);
 110        else if (size == 2)
 111                writew(val, addr);
 112        else
 113                writel(val, addr);
 114
 115        return PCIBIOS_SUCCESSFUL;
 116}
 117EXPORT_SYMBOL_GPL(pci_generic_config_write);
 118
 119int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn,
 120                              int where, int size, u32 *val)
 121{
 122        void __iomem *addr;
 123
 124        addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
 125        if (!addr) {
 126                *val = ~0;
 127                return PCIBIOS_DEVICE_NOT_FOUND;
 128        }
 129
 130        *val = readl(addr);
 131
 132        if (size <= 2)
 133                *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
 134
 135        return PCIBIOS_SUCCESSFUL;
 136}
 137EXPORT_SYMBOL_GPL(pci_generic_config_read32);
 138
 139int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn,
 140                               int where, int size, u32 val)
 141{
 142        void __iomem *addr;
 143        u32 mask, tmp;
 144
 145        addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
 146        if (!addr)
 147                return PCIBIOS_DEVICE_NOT_FOUND;
 148
 149        if (size == 4) {
 150                writel(val, addr);
 151                return PCIBIOS_SUCCESSFUL;
 152        }
 153
 154        /*
 155         * In general, hardware that supports only 32-bit writes on PCI is
 156         * not spec-compliant.  For example, software may perform a 16-bit
 157         * write.  If the hardware only supports 32-bit accesses, we must
 158         * do a 32-bit read, merge in the 16 bits we intend to write,
 159         * followed by a 32-bit write.  If the 16 bits we *don't* intend to
 160         * write happen to have any RW1C (write-one-to-clear) bits set, we
 161         * just inadvertently cleared something we shouldn't have.
 162         */
 163        dev_warn_ratelimited(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n",
 164                             size, pci_domain_nr(bus), bus->number,
 165                             PCI_SLOT(devfn), PCI_FUNC(devfn), where);
 166
 167        mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
 168        tmp = readl(addr) & mask;
 169        tmp |= val << ((where & 0x3) * 8);
 170        writel(tmp, addr);
 171
 172        return PCIBIOS_SUCCESSFUL;
 173}
 174EXPORT_SYMBOL_GPL(pci_generic_config_write32);
 175
 176/**
 177 * pci_bus_set_ops - Set raw operations of pci bus
 178 * @bus:        pci bus struct
 179 * @ops:        new raw operations
 180 *
 181 * Return previous raw operations
 182 */
 183struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops)
 184{
 185        struct pci_ops *old_ops;
 186        unsigned long flags;
 187
 188        raw_spin_lock_irqsave(&pci_lock, flags);
 189        old_ops = bus->ops;
 190        bus->ops = ops;
 191        raw_spin_unlock_irqrestore(&pci_lock, flags);
 192        return old_ops;
 193}
 194EXPORT_SYMBOL(pci_bus_set_ops);
 195
 196/*
 197 * The following routines are to prevent the user from accessing PCI config
 198 * space when it's unsafe to do so.  Some devices require this during BIST and
 199 * we're required to prevent it during D-state transitions.
 200 *
 201 * We have a bit per device to indicate it's blocked and a global wait queue
 202 * for callers to sleep on until devices are unblocked.
 203 */
 204static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait);
 205
 206static noinline void pci_wait_cfg(struct pci_dev *dev)
 207{
 208        DECLARE_WAITQUEUE(wait, current);
 209
 210        __add_wait_queue(&pci_cfg_wait, &wait);
 211        do {
 212                set_current_state(TASK_UNINTERRUPTIBLE);
 213                raw_spin_unlock_irq(&pci_lock);
 214                schedule();
 215                raw_spin_lock_irq(&pci_lock);
 216        } while (dev->block_cfg_access);
 217        __remove_wait_queue(&pci_cfg_wait, &wait);
 218}
 219
 220/* Returns 0 on success, negative values indicate error. */
 221#define PCI_USER_READ_CONFIG(size, type)                                        \
 222int pci_user_read_config_##size                                         \
 223        (struct pci_dev *dev, int pos, type *val)                       \
 224{                                                                       \
 225        int ret = PCIBIOS_SUCCESSFUL;                                   \
 226        u32 data = -1;                                                  \
 227        if (PCI_##size##_BAD)                                           \
 228                return -EINVAL;                                         \
 229        raw_spin_lock_irq(&pci_lock);                           \
 230        if (unlikely(dev->block_cfg_access))                            \
 231                pci_wait_cfg(dev);                                      \
 232        ret = dev->bus->ops->read(dev->bus, dev->devfn,                 \
 233                                        pos, sizeof(type), &data);      \
 234        raw_spin_unlock_irq(&pci_lock);                         \
 235        *val = (type)data;                                              \
 236        return pcibios_err_to_errno(ret);                               \
 237}                                                                       \
 238EXPORT_SYMBOL_GPL(pci_user_read_config_##size);
 239
 240/* Returns 0 on success, negative values indicate error. */
 241#define PCI_USER_WRITE_CONFIG(size, type)                               \
 242int pci_user_write_config_##size                                        \
 243        (struct pci_dev *dev, int pos, type val)                        \
 244{                                                                       \
 245        int ret = PCIBIOS_SUCCESSFUL;                                   \
 246        if (PCI_##size##_BAD)                                           \
 247                return -EINVAL;                                         \
 248        raw_spin_lock_irq(&pci_lock);                           \
 249        if (unlikely(dev->block_cfg_access))                            \
 250                pci_wait_cfg(dev);                                      \
 251        ret = dev->bus->ops->write(dev->bus, dev->devfn,                \
 252                                        pos, sizeof(type), val);        \
 253        raw_spin_unlock_irq(&pci_lock);                         \
 254        return pcibios_err_to_errno(ret);                               \
 255}                                                                       \
 256EXPORT_SYMBOL_GPL(pci_user_write_config_##size);
 257
 258PCI_USER_READ_CONFIG(byte, u8)
 259PCI_USER_READ_CONFIG(word, u16)
 260PCI_USER_READ_CONFIG(dword, u32)
 261PCI_USER_WRITE_CONFIG(byte, u8)
 262PCI_USER_WRITE_CONFIG(word, u16)
 263PCI_USER_WRITE_CONFIG(dword, u32)
 264
 265/**
 266 * pci_cfg_access_lock - Lock PCI config reads/writes
 267 * @dev:        pci device struct
 268 *
 269 * When access is locked, any userspace reads or writes to config
 270 * space and concurrent lock requests will sleep until access is
 271 * allowed via pci_cfg_access_unlock() again.
 272 */
 273void pci_cfg_access_lock(struct pci_dev *dev)
 274{
 275        might_sleep();
 276
 277        raw_spin_lock_irq(&pci_lock);
 278        if (dev->block_cfg_access)
 279                pci_wait_cfg(dev);
 280        dev->block_cfg_access = 1;
 281        raw_spin_unlock_irq(&pci_lock);
 282}
 283EXPORT_SYMBOL_GPL(pci_cfg_access_lock);
 284
 285/**
 286 * pci_cfg_access_trylock - try to lock PCI config reads/writes
 287 * @dev:        pci device struct
 288 *
 289 * Same as pci_cfg_access_lock, but will return 0 if access is
 290 * already locked, 1 otherwise. This function can be used from
 291 * atomic contexts.
 292 */
 293bool pci_cfg_access_trylock(struct pci_dev *dev)
 294{
 295        unsigned long flags;
 296        bool locked = true;
 297
 298        raw_spin_lock_irqsave(&pci_lock, flags);
 299        if (dev->block_cfg_access)
 300                locked = false;
 301        else
 302                dev->block_cfg_access = 1;
 303        raw_spin_unlock_irqrestore(&pci_lock, flags);
 304
 305        return locked;
 306}
 307EXPORT_SYMBOL_GPL(pci_cfg_access_trylock);
 308
 309/**
 310 * pci_cfg_access_unlock - Unlock PCI config reads/writes
 311 * @dev:        pci device struct
 312 *
 313 * This function allows PCI config accesses to resume.
 314 */
 315void pci_cfg_access_unlock(struct pci_dev *dev)
 316{
 317        unsigned long flags;
 318
 319        raw_spin_lock_irqsave(&pci_lock, flags);
 320
 321        /*
 322         * This indicates a problem in the caller, but we don't need
 323         * to kill them, unlike a double-block above.
 324         */
 325        WARN_ON(!dev->block_cfg_access);
 326
 327        dev->block_cfg_access = 0;
 328        raw_spin_unlock_irqrestore(&pci_lock, flags);
 329
 330        wake_up_all(&pci_cfg_wait);
 331}
 332EXPORT_SYMBOL_GPL(pci_cfg_access_unlock);
 333
 334static inline int pcie_cap_version(const struct pci_dev *dev)
 335{
 336        return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS;
 337}
 338
 339static bool pcie_downstream_port(const struct pci_dev *dev)
 340{
 341        int type = pci_pcie_type(dev);
 342
 343        return type == PCI_EXP_TYPE_ROOT_PORT ||
 344               type == PCI_EXP_TYPE_DOWNSTREAM ||
 345               type == PCI_EXP_TYPE_PCIE_BRIDGE;
 346}
 347
 348bool pcie_cap_has_lnkctl(const struct pci_dev *dev)
 349{
 350        int type = pci_pcie_type(dev);
 351
 352        return type == PCI_EXP_TYPE_ENDPOINT ||
 353               type == PCI_EXP_TYPE_LEG_END ||
 354               type == PCI_EXP_TYPE_ROOT_PORT ||
 355               type == PCI_EXP_TYPE_UPSTREAM ||
 356               type == PCI_EXP_TYPE_DOWNSTREAM ||
 357               type == PCI_EXP_TYPE_PCI_BRIDGE ||
 358               type == PCI_EXP_TYPE_PCIE_BRIDGE;
 359}
 360
 361static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev)
 362{
 363        return pcie_downstream_port(dev) &&
 364               pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT;
 365}
 366
 367static inline bool pcie_cap_has_rtctl(const struct pci_dev *dev)
 368{
 369        int type = pci_pcie_type(dev);
 370
 371        return type == PCI_EXP_TYPE_ROOT_PORT ||
 372               type == PCI_EXP_TYPE_RC_EC;
 373}
 374
 375static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos)
 376{
 377        if (!pci_is_pcie(dev))
 378                return false;
 379
 380        switch (pos) {
 381        case PCI_EXP_FLAGS:
 382                return true;
 383        case PCI_EXP_DEVCAP:
 384        case PCI_EXP_DEVCTL:
 385        case PCI_EXP_DEVSTA:
 386                return true;
 387        case PCI_EXP_LNKCAP:
 388        case PCI_EXP_LNKCTL:
 389        case PCI_EXP_LNKSTA:
 390                return pcie_cap_has_lnkctl(dev);
 391        case PCI_EXP_SLTCAP:
 392        case PCI_EXP_SLTCTL:
 393        case PCI_EXP_SLTSTA:
 394                return pcie_cap_has_sltctl(dev);
 395        case PCI_EXP_RTCTL:
 396        case PCI_EXP_RTCAP:
 397        case PCI_EXP_RTSTA:
 398                return pcie_cap_has_rtctl(dev);
 399        case PCI_EXP_DEVCAP2:
 400        case PCI_EXP_DEVCTL2:
 401        case PCI_EXP_LNKCAP2:
 402        case PCI_EXP_LNKCTL2:
 403        case PCI_EXP_LNKSTA2:
 404                return pcie_cap_version(dev) > 1;
 405        default:
 406                return false;
 407        }
 408}
 409
 410/*
 411 * Note that these accessor functions are only for the "PCI Express
 412 * Capability" (see PCIe spec r3.0, sec 7.8).  They do not apply to the
 413 * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.)
 414 */
 415int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val)
 416{
 417        int ret;
 418
 419        *val = 0;
 420        if (pos & 1)
 421                return -EINVAL;
 422
 423        if (pcie_capability_reg_implemented(dev, pos)) {
 424                ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val);
 425                /*
 426                 * Reset *val to 0 if pci_read_config_word() fails, it may
 427                 * have been written as 0xFFFF if hardware error happens
 428                 * during pci_read_config_word().
 429                 */
 430                if (ret)
 431                        *val = 0;
 432                return ret;
 433        }
 434
 435        /*
 436         * For Functions that do not implement the Slot Capabilities,
 437         * Slot Status, and Slot Control registers, these spaces must
 438         * be hardwired to 0b, with the exception of the Presence Detect
 439         * State bit in the Slot Status register of Downstream Ports,
 440         * which must be hardwired to 1b.  (PCIe Base Spec 3.0, sec 7.8)
 441         */
 442        if (pci_is_pcie(dev) && pcie_downstream_port(dev) &&
 443            pos == PCI_EXP_SLTSTA)
 444                *val = PCI_EXP_SLTSTA_PDS;
 445
 446        return 0;
 447}
 448EXPORT_SYMBOL(pcie_capability_read_word);
 449
 450int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val)
 451{
 452        int ret;
 453
 454        *val = 0;
 455        if (pos & 3)
 456                return -EINVAL;
 457
 458        if (pcie_capability_reg_implemented(dev, pos)) {
 459                ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val);
 460                /*
 461                 * Reset *val to 0 if pci_read_config_dword() fails, it may
 462                 * have been written as 0xFFFFFFFF if hardware error happens
 463                 * during pci_read_config_dword().
 464                 */
 465                if (ret)
 466                        *val = 0;
 467                return ret;
 468        }
 469
 470        if (pci_is_pcie(dev) && pcie_downstream_port(dev) &&
 471            pos == PCI_EXP_SLTSTA)
 472                *val = PCI_EXP_SLTSTA_PDS;
 473
 474        return 0;
 475}
 476EXPORT_SYMBOL(pcie_capability_read_dword);
 477
 478int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val)
 479{
 480        if (pos & 1)
 481                return -EINVAL;
 482
 483        if (!pcie_capability_reg_implemented(dev, pos))
 484                return 0;
 485
 486        return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val);
 487}
 488EXPORT_SYMBOL(pcie_capability_write_word);
 489
 490int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val)
 491{
 492        if (pos & 3)
 493                return -EINVAL;
 494
 495        if (!pcie_capability_reg_implemented(dev, pos))
 496                return 0;
 497
 498        return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val);
 499}
 500EXPORT_SYMBOL(pcie_capability_write_dword);
 501
 502int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos,
 503                                       u16 clear, u16 set)
 504{
 505        int ret;
 506        u16 val;
 507
 508        ret = pcie_capability_read_word(dev, pos, &val);
 509        if (!ret) {
 510                val &= ~clear;
 511                val |= set;
 512                ret = pcie_capability_write_word(dev, pos, val);
 513        }
 514
 515        return ret;
 516}
 517EXPORT_SYMBOL(pcie_capability_clear_and_set_word);
 518
 519int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos,
 520                                        u32 clear, u32 set)
 521{
 522        int ret;
 523        u32 val;
 524
 525        ret = pcie_capability_read_dword(dev, pos, &val);
 526        if (!ret) {
 527                val &= ~clear;
 528                val |= set;
 529                ret = pcie_capability_write_dword(dev, pos, val);
 530        }
 531
 532        return ret;
 533}
 534EXPORT_SYMBOL(pcie_capability_clear_and_set_dword);
 535
 536int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val)
 537{
 538        if (pci_dev_is_disconnected(dev)) {
 539                *val = ~0;
 540                return PCIBIOS_DEVICE_NOT_FOUND;
 541        }
 542        return pci_bus_read_config_byte(dev->bus, dev->devfn, where, val);
 543}
 544EXPORT_SYMBOL(pci_read_config_byte);
 545
 546int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val)
 547{
 548        if (pci_dev_is_disconnected(dev)) {
 549                *val = ~0;
 550                return PCIBIOS_DEVICE_NOT_FOUND;
 551        }
 552        return pci_bus_read_config_word(dev->bus, dev->devfn, where, val);
 553}
 554EXPORT_SYMBOL(pci_read_config_word);
 555
 556int pci_read_config_dword(const struct pci_dev *dev, int where,
 557                                        u32 *val)
 558{
 559        if (pci_dev_is_disconnected(dev)) {
 560                *val = ~0;
 561                return PCIBIOS_DEVICE_NOT_FOUND;
 562        }
 563        return pci_bus_read_config_dword(dev->bus, dev->devfn, where, val);
 564}
 565EXPORT_SYMBOL(pci_read_config_dword);
 566
 567int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val)
 568{
 569        if (pci_dev_is_disconnected(dev))
 570                return PCIBIOS_DEVICE_NOT_FOUND;
 571        return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val);
 572}
 573EXPORT_SYMBOL(pci_write_config_byte);
 574
 575int pci_write_config_word(const struct pci_dev *dev, int where, u16 val)
 576{
 577        if (pci_dev_is_disconnected(dev))
 578                return PCIBIOS_DEVICE_NOT_FOUND;
 579        return pci_bus_write_config_word(dev->bus, dev->devfn, where, val);
 580}
 581EXPORT_SYMBOL(pci_write_config_word);
 582
 583int pci_write_config_dword(const struct pci_dev *dev, int where,
 584                                         u32 val)
 585{
 586        if (pci_dev_is_disconnected(dev))
 587                return PCIBIOS_DEVICE_NOT_FOUND;
 588        return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val);
 589}
 590EXPORT_SYMBOL(pci_write_config_dword);
 591