LXR linux/drivers/pci/vc.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * PCI Virtual Channel support
   4 *
   5 * Copyright (C) 2013 Red Hat, Inc.  All rights reserved.
   6 *     Author: Alex Williamson <alex.williamson@redhat.com>
   7 */
   8
   9#include <linux/device.h>
  10#include <linux/kernel.h>
  11#include <linux/module.h>
  12#include <linux/pci.h>
  13#include <linux/pci_regs.h>
  14#include <linux/types.h>
  15
  16#include "pci.h"
  17
  18/**
  19 * pci_vc_save_restore_dwords - Save or restore a series of dwords
  20 * @dev: device
  21 * @pos: starting config space position
  22 * @buf: buffer to save to or restore from
  23 * @dwords: number of dwords to save/restore
  24 * @save: whether to save or restore
  25 */
  26static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos,
  27                                       u32 *buf, int dwords, bool save)
  28{
  29        int i;
  30
  31        for (i = 0; i < dwords; i++, buf++) {
  32                if (save)
  33                        pci_read_config_dword(dev, pos + (i * 4), buf);
  34                else
  35                        pci_write_config_dword(dev, pos + (i * 4), *buf);
  36        }
  37}
  38
  39/**
  40 * pci_vc_load_arb_table - load and wait for VC arbitration table
  41 * @dev: device
  42 * @pos: starting position of VC capability (VC/VC9/MFVC)
  43 *
  44 * Set Load VC Arbitration Table bit requesting hardware to apply the VC
  45 * Arbitration Table (previously loaded).  When the VC Arbitration Table
  46 * Status clears, hardware has latched the table into VC arbitration logic.
  47 */
  48static void pci_vc_load_arb_table(struct pci_dev *dev, int pos)
  49{
  50        u16 ctrl;
  51
  52        pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl);
  53        pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL,
  54                              ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE);
  55        if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS,
  56                                 PCI_VC_PORT_STATUS_TABLE))
  57                return;
  58
  59        pci_err(dev, "VC arbitration table failed to load\n");
  60}
  61
  62/**
  63 * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table
  64 * @dev: device
  65 * @pos: starting position of VC capability (VC/VC9/MFVC)
  66 * @res: VC resource number, ie. VCn (0-7)
  67 *
  68 * Set Load Port Arbitration Table bit requesting hardware to apply the Port
  69 * Arbitration Table (previously loaded).  When the Port Arbitration Table
  70 * Status clears, hardware has latched the table into port arbitration logic.
  71 */
  72static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res)
  73{
  74        int ctrl_pos, status_pos;
  75        u32 ctrl;
  76
  77        ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
  78        status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
  79
  80        pci_read_config_dword(dev, ctrl_pos, &ctrl);
  81        pci_write_config_dword(dev, ctrl_pos,
  82                               ctrl | PCI_VC_RES_CTRL_LOAD_TABLE);
  83
  84        if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE))
  85                return;
  86
  87        pci_err(dev, "VC%d port arbitration table failed to load\n", res);
  88}
  89
  90/**
  91 * pci_vc_enable - Enable virtual channel
  92 * @dev: device
  93 * @pos: starting position of VC capability (VC/VC9/MFVC)
  94 * @res: VC res number, ie. VCn (0-7)
  95 *
  96 * A VC is enabled by setting the enable bit in matching resource control
  97 * registers on both sides of a link.  We therefore need to find the opposite
  98 * end of the link.  To keep this simple we enable from the downstream device.
  99 * RC devices do not have an upstream device, nor does it seem that VC9 do
 100 * (spec is unclear).  Once we find the upstream device, match the VC ID to
 101 * get the correct resource, disable and enable on both ends.
 102 */
 103static void pci_vc_enable(struct pci_dev *dev, int pos, int res)
 104{
 105        int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2;
 106        u32 ctrl, header, cap1, ctrl2;
 107        struct pci_dev *link = NULL;
 108
 109        /* Enable VCs from the downstream device */
 110        if (!pci_is_pcie(dev) || !pcie_downstream_port(dev))
 111                return;
 112
 113        ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
 114        status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
 115
 116        pci_read_config_dword(dev, ctrl_pos, &ctrl);
 117        id = ctrl & PCI_VC_RES_CTRL_ID;
 118
 119        pci_read_config_dword(dev, pos, &header);
 120
 121        /* If there is no opposite end of the link, skip to enable */
 122        if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 ||
 123            pci_is_root_bus(dev->bus))
 124                goto enable;
 125
 126        pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC);
 127        if (!pos2)
 128                goto enable;
 129
 130        pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1);
 131        evcc = cap1 & PCI_VC_CAP1_EVCC;
 132
 133        /* VC0 is hardwired enabled, so we can start with 1 */
 134        for (i = 1; i < evcc + 1; i++) {
 135                ctrl_pos2 = pos2 + PCI_VC_RES_CTRL +
 136                                (i * PCI_CAP_VC_PER_VC_SIZEOF);
 137                status_pos2 = pos2 + PCI_VC_RES_STATUS +
 138                                (i * PCI_CAP_VC_PER_VC_SIZEOF);
 139                pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2);
 140                if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) {
 141                        link = dev->bus->self;
 142                        break;
 143                }
 144        }
 145
 146        if (!link)
 147                goto enable;
 148
 149        /* Disable if enabled */
 150        if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) {
 151                ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE;
 152                pci_write_config_dword(link, ctrl_pos2, ctrl2);
 153        }
 154
 155        /* Enable on both ends */
 156        ctrl2 |= PCI_VC_RES_CTRL_ENABLE;
 157        pci_write_config_dword(link, ctrl_pos2, ctrl2);
 158enable:
 159        ctrl |= PCI_VC_RES_CTRL_ENABLE;
 160        pci_write_config_dword(dev, ctrl_pos, ctrl);
 161
 162        if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO))
 163                pci_err(dev, "VC%d negotiation stuck pending\n", id);
 164
 165        if (link && !pci_wait_for_pending(link, status_pos2,
 166                                          PCI_VC_RES_STATUS_NEGO))
 167                pci_err(link, "VC%d negotiation stuck pending\n", id);
 168}
 169
 170/**
 171 * pci_vc_do_save_buffer - Size, save, or restore VC state
 172 * @dev: device
 173 * @pos: starting position of VC capability (VC/VC9/MFVC)
 174 * @save_state: buffer for save/restore
 175 * @name: for error message
 176 * @save: if provided a buffer, this indicates what to do with it
 177 *
 178 * Walking Virtual Channel config space to size, save, or restore it
 179 * is complicated, so we do it all from one function to reduce code and
 180 * guarantee ordering matches in the buffer.  When called with NULL
 181 * @save_state, return the size of the necessary save buffer.  When called
 182 * with a non-NULL @save_state, @save determines whether we save to the
 183 * buffer or restore from it.
 184 */
 185static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
 186                                 struct pci_cap_saved_state *save_state,
 187                                 bool save)
 188{
 189        u32 cap1;
 190        char evcc, lpevcc, parb_size;
 191        int i, len = 0;
 192        u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL;
 193
 194        /* Sanity check buffer size for save/restore */
 195        if (buf && save_state->cap.size !=
 196            pci_vc_do_save_buffer(dev, pos, NULL, save)) {
 197                pci_err(dev, "VC save buffer size does not match @0x%x\n", pos);
 198                return -ENOMEM;
 199        }
 200
 201        pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1);
 202        /* Extended VC Count (not counting VC0) */
 203        evcc = cap1 & PCI_VC_CAP1_EVCC;
 204        /* Low Priority Extended VC Count (not counting VC0) */
 205        lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4;
 206        /* Port Arbitration Table Entry Size (bits) */
 207        parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10);
 208
 209        /*
 210         * Port VC Control Register contains VC Arbitration Select, which
 211         * cannot be modified when more than one LPVC is in operation.  We
 212         * therefore save/restore it first, as only VC0 should be enabled
 213         * after device reset.
 214         */
 215        if (buf) {
 216                if (save)
 217                        pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL,
 218                                             (u16 *)buf);
 219                else
 220                        pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL,
 221                                              *(u16 *)buf);
 222                buf += 4;
 223        }
 224        len += 4;
 225
 226        /*
 227         * If we have any Low Priority VCs and a VC Arbitration Table Offset
 228         * in Port VC Capability Register 2 then save/restore it next.
 229         */
 230        if (lpevcc) {
 231                u32 cap2;
 232                int vcarb_offset;
 233
 234                pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2);
 235                vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16;
 236
 237                if (vcarb_offset) {
 238                        int size, vcarb_phases = 0;
 239
 240                        if (cap2 & PCI_VC_CAP2_128_PHASE)
 241                                vcarb_phases = 128;
 242                        else if (cap2 & PCI_VC_CAP2_64_PHASE)
 243                                vcarb_phases = 64;
 244                        else if (cap2 & PCI_VC_CAP2_32_PHASE)
 245                                vcarb_phases = 32;
 246
 247                        /* Fixed 4 bits per phase per lpevcc (plus VC0) */
 248                        size = ((lpevcc + 1) * vcarb_phases * 4) / 8;
 249
 250                        if (size && buf) {
 251                                pci_vc_save_restore_dwords(dev,
 252                                                           pos + vcarb_offset,
 253                                                           (u32 *)buf,
 254                                                           size / 4, save);
 255                                /*
 256                                 * On restore, we need to signal hardware to
 257                                 * re-load the VC Arbitration Table.
 258                                 */
 259                                if (!save)
 260                                        pci_vc_load_arb_table(dev, pos);
 261
 262                                buf += size;
 263                        }
 264                        len += size;
 265                }
 266        }
 267
 268        /*
 269         * In addition to each VC Resource Control Register, we may have a
 270         * Port Arbitration Table attached to each VC.  The Port Arbitration
 271         * Table Offset in each VC Resource Capability Register tells us if
 272         * it exists.  The entry size is global from the Port VC Capability
 273         * Register1 above.  The number of phases is determined per VC.
 274         */
 275        for (i = 0; i < evcc + 1; i++) {
 276                u32 cap;
 277                int parb_offset;
 278
 279                pci_read_config_dword(dev, pos + PCI_VC_RES_CAP +
 280                                      (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap);
 281                parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16;
 282                if (parb_offset) {
 283                        int size, parb_phases = 0;
 284
 285                        if (cap & PCI_VC_RES_CAP_256_PHASE)
 286                                parb_phases = 256;
 287                        else if (cap & (PCI_VC_RES_CAP_128_PHASE |
 288                                        PCI_VC_RES_CAP_128_PHASE_TB))
 289                                parb_phases = 128;
 290                        else if (cap & PCI_VC_RES_CAP_64_PHASE)
 291                                parb_phases = 64;
 292                        else if (cap & PCI_VC_RES_CAP_32_PHASE)
 293                                parb_phases = 32;
 294
 295                        size = (parb_size * parb_phases) / 8;
 296
 297                        if (size && buf) {
 298                                pci_vc_save_restore_dwords(dev,
 299                                                           pos + parb_offset,
 300                                                           (u32 *)buf,
 301                                                           size / 4, save);
 302                                buf += size;
 303                        }
 304                        len += size;
 305                }
 306
 307                /* VC Resource Control Register */
 308                if (buf) {
 309                        int ctrl_pos = pos + PCI_VC_RES_CTRL +
 310                                                (i * PCI_CAP_VC_PER_VC_SIZEOF);
 311                        if (save)
 312                                pci_read_config_dword(dev, ctrl_pos,
 313                                                      (u32 *)buf);
 314                        else {
 315                                u32 tmp, ctrl = *(u32 *)buf;
 316                                /*
 317                                 * For an FLR case, the VC config may remain.
 318                                 * Preserve enable bit, restore the rest.
 319                                 */
 320                                pci_read_config_dword(dev, ctrl_pos, &tmp);
 321                                tmp &= PCI_VC_RES_CTRL_ENABLE;
 322                                tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE;
 323                                pci_write_config_dword(dev, ctrl_pos, tmp);
 324                                /* Load port arbitration table if used */
 325                                if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT)
 326                                        pci_vc_load_port_arb_table(dev, pos, i);
 327                                /* Re-enable if needed */
 328                                if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE)
 329                                        pci_vc_enable(dev, pos, i);
 330                        }
 331                        buf += 4;
 332                }
 333                len += 4;
 334        }
 335
 336        return buf ? 0 : len;
 337}
 338
 339static struct {
 340        u16 id;
 341        const char *name;
 342} vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" },
 343                { PCI_EXT_CAP_ID_VC, "VC" },
 344                { PCI_EXT_CAP_ID_VC9, "VC9" } };
 345
 346/**
 347 * pci_save_vc_state - Save VC state to pre-allocate save buffer
 348 * @dev: device
 349 *
 350 * For each type of VC capability, VC/VC9/MFVC, find the capability and
 351 * save it to the pre-allocated save buffer.
 352 */
 353int pci_save_vc_state(struct pci_dev *dev)
 354{
 355        int i;
 356
 357        for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
 358                int pos, ret;
 359                struct pci_cap_saved_state *save_state;
 360
 361                pos = pci_find_ext_capability(dev, vc_caps[i].id);
 362                if (!pos)
 363                        continue;
 364
 365                save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id);
 366                if (!save_state) {
 367                        pci_err(dev, "%s buffer not found in %s\n",
 368                                vc_caps[i].name, __func__);
 369                        return -ENOMEM;
 370                }
 371
 372                ret = pci_vc_do_save_buffer(dev, pos, save_state, true);
 373                if (ret) {
 374                        pci_err(dev, "%s save unsuccessful %s\n",
 375                                vc_caps[i].name, __func__);
 376                        return ret;
 377                }
 378        }
 379
 380        return 0;
 381}
 382
 383/**
 384 * pci_restore_vc_state - Restore VC state from save buffer
 385 * @dev: device
 386 *
 387 * For each type of VC capability, VC/VC9/MFVC, find the capability and
 388 * restore it from the previously saved buffer.
 389 */
 390void pci_restore_vc_state(struct pci_dev *dev)
 391{
 392        int i;
 393
 394        for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
 395                int pos;
 396                struct pci_cap_saved_state *save_state;
 397
 398                pos = pci_find_ext_capability(dev, vc_caps[i].id);
 399                save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id);
 400                if (!save_state || !pos)
 401                        continue;
 402
 403                pci_vc_do_save_buffer(dev, pos, save_state, false);
 404        }
 405}
 406
 407/**
 408 * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps
 409 * @dev: device
 410 *
 411 * For each type of VC capability, VC/VC9/MFVC, find the capability, size
 412 * it, and allocate a buffer for save/restore.
 413 */
 414void pci_allocate_vc_save_buffers(struct pci_dev *dev)
 415{
 416        int i;
 417
 418        for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
 419                int len, pos = pci_find_ext_capability(dev, vc_caps[i].id);
 420
 421                if (!pos)
 422                        continue;
 423
 424                len = pci_vc_do_save_buffer(dev, pos, NULL, false);
 425                if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len))
 426                        pci_err(dev, "unable to preallocate %s save buffer\n",
 427                                vc_caps[i].name);
 428        }
 429}
 430