LXR linux/drivers/pci/vc.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * PCI Virtual Channel support
   4 *
   5 * Copyright (C) 2013 Red Hat, Inc.  All rights reserved.
   6 *     Author: Alex Williamson <alex.williamson@redhat.com>
   7 */
   8
   9#include <linux/device.h>
  10#include <linux/kernel.h>
  11#include <linux/module.h>
  12#include <linux/pci.h>
  13#include <linux/pci_regs.h>
  14#include <linux/types.h>
  15
  16#include "pci.h"
  17
  18/**
  19 * pci_vc_save_restore_dwords - Save or restore a series of dwords
  20 * @dev: device
  21 * @pos: starting config space position
  22 * @buf: buffer to save to or restore from
  23 * @dwords: number of dwords to save/restore
  24 * @save: whether to save or restore
  25 */
  26static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos,
  27                                       u32 *buf, int dwords, bool save)
  28{
  29        int i;
  30
  31        for (i = 0; i < dwords; i++, buf++) {
  32                if (save)
  33                        pci_read_config_dword(dev, pos + (i * 4), buf);
  34                else
  35                        pci_write_config_dword(dev, pos + (i * 4), *buf);
  36        }
  37}
  38
  39/**
  40 * pci_vc_load_arb_table - load and wait for VC arbitration table
  41 * @dev: device
  42 * @pos: starting position of VC capability (VC/VC9/MFVC)
  43 *
  44 * Set Load VC Arbitration Table bit requesting hardware to apply the VC
  45 * Arbitration Table (previously loaded).  When the VC Arbitration Table
  46 * Status clears, hardware has latched the table into VC arbitration logic.
  47 */
  48static void pci_vc_load_arb_table(struct pci_dev *dev, int pos)
  49{
  50        u16 ctrl;
  51
  52        pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl);
  53        pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL,
  54                              ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE);
  55        if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS,
  56                                 PCI_VC_PORT_STATUS_TABLE))
  57                return;
  58
  59        pci_err(dev, "VC arbitration table failed to load\n");
  60}
  61
  62/**
  63 * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table
  64 * @dev: device
  65 * @pos: starting position of VC capability (VC/VC9/MFVC)
  66 * @res: VC resource number, ie. VCn (0-7)
  67 *
  68 * Set Load Port Arbitration Table bit requesting hardware to apply the Port
  69 * Arbitration Table (previously loaded).  When the Port Arbitration Table
  70 * Status clears, hardware has latched the table into port arbitration logic.
  71 */
  72static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res)
  73{
  74        int ctrl_pos, status_pos;
  75        u32 ctrl;
  76
  77        ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
  78        status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
  79
  80        pci_read_config_dword(dev, ctrl_pos, &ctrl);
  81        pci_write_config_dword(dev, ctrl_pos,
  82                               ctrl | PCI_VC_RES_CTRL_LOAD_TABLE);
  83
  84        if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE))
  85                return;
  86
  87        pci_err(dev, "VC%d port arbitration table failed to load\n", res);
  88}
  89
  90/**
  91 * pci_vc_enable - Enable virtual channel
  92 * @dev: device
  93 * @pos: starting position of VC capability (VC/VC9/MFVC)
  94 * @res: VC res number, ie. VCn (0-7)
  95 *
  96 * A VC is enabled by setting the enable bit in matching resource control
  97 * registers on both sides of a link.  We therefore need to find the opposite
  98 * end of the link.  To keep this simple we enable from the downstream device.
  99 * RC devices do not have an upstream device, nor does it seem that VC9 do
 100 * (spec is unclear).  Once we find the upstream device, match the VC ID to
 101 * get the correct resource, disable and enable on both ends.
 102 */
 103static void pci_vc_enable(struct pci_dev *dev, int pos, int res)
 104{
 105        int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2;
 106        u32 ctrl, header, cap1, ctrl2;
 107        struct pci_dev *link = NULL;
 108
 109        /* Enable VCs from the downstream device */
 110        if (!pci_is_pcie(dev) || !pcie_downstream_port(dev))
 111                return;
 112
 113        ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
 114        status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
 115
 116        pci_read_config_dword(dev, ctrl_pos, &ctrl);
 117        id = ctrl & PCI_VC_RES_CTRL_ID;
 118
 119        pci_read_config_dword(dev, pos, &header);
 120
 121        /* If there is no opposite end of the link, skip to enable */
 122        if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 ||
 123            pci_is_root_bus(dev->bus))
 124                goto enable;
 125
 126        pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC);
 127        if (!pos2)
 128                goto enable;
 129
 130        pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1);
 131        evcc = cap1 & PCI_VC_CAP1_EVCC;
 132
 133        /* VC0 is hardwired enabled, so we can start with 1 */
 134        for (i = 1; i < evcc + 1; i++) {
 135                ctrl_pos2 = pos2 + PCI_VC_RES_CTRL +
 136                                (i * PCI_CAP_VC_PER_VC_SIZEOF);
 137                status_pos2 = pos2 + PCI_VC_RES_STATUS +
 138                                (i * PCI_CAP_VC_PER_VC_SIZEOF);
 139                pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2);
 140                if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) {
 141                        link = dev->bus->self;
 142                        break;
 143                }
 144        }
 145
 146        if (!link)
 147                goto enable;
 148
 149        /* Disable if enabled */
 150        if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) {
 151                ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE;
 152                pci_write_config_dword(link, ctrl_pos2, ctrl2);
 153        }
 154
 155        /* Enable on both ends */
 156        ctrl2 |= PCI_VC_RES_CTRL_ENABLE;
 157        pci_write_config_dword(link, ctrl_pos2, ctrl2);
 158enable:
 159        ctrl |= PCI_VC_RES_CTRL_ENABLE;
 160        pci_write_config_dword(dev, ctrl_pos, ctrl);
 161
 162        if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO))
 163                pci_err(dev, "VC%d negotiation stuck pending\n", id);
 164
 165        if (link && !pci_wait_for_pending(link, status_pos2,
 166                                          PCI_VC_RES_STATUS_NEGO))
 167                pci_err(link, "VC%d negotiation stuck pending\n", id);
 168}
 169
 170/**
 171 * pci_vc_do_save_buffer - Size, save, or restore VC state
 172 * @dev: device
 173 * @pos: starting position of VC capability (VC/VC9/MFVC)
 174 * @save_state: buffer for save/restore
 175 * @save: if provided a buffer, this indicates what to do with it
 176 *
 177 * Walking Virtual Channel config space to size, save, or restore it
 178 * is complicated, so we do it all from one function to reduce code and
 179 * guarantee ordering matches in the buffer.  When called with NULL
 180 * @save_state, return the size of the necessary save buffer.  When called
 181 * with a non-NULL @save_state, @save determines whether we save to the
 182 * buffer or restore from it.
 183 */
 184static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
 185                                 struct pci_cap_saved_state *save_state,
 186                                 bool save)
 187{
 188        u32 cap1;
 189        char evcc, lpevcc, parb_size;
 190        int i, len = 0;
 191        u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL;
 192
 193        /* Sanity check buffer size for save/restore */
 194        if (buf && save_state->cap.size !=
 195            pci_vc_do_save_buffer(dev, pos, NULL, save)) {
 196                pci_err(dev, "VC save buffer size does not match @0x%x\n", pos);
 197                return -ENOMEM;
 198        }
 199
 200        pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1);
 201        /* Extended VC Count (not counting VC0) */
 202        evcc = cap1 & PCI_VC_CAP1_EVCC;
 203        /* Low Priority Extended VC Count (not counting VC0) */
 204        lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4;
 205        /* Port Arbitration Table Entry Size (bits) */
 206        parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10);
 207
 208        /*
 209         * Port VC Control Register contains VC Arbitration Select, which
 210         * cannot be modified when more than one LPVC is in operation.  We
 211         * therefore save/restore it first, as only VC0 should be enabled
 212         * after device reset.
 213         */
 214        if (buf) {
 215                if (save)
 216                        pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL,
 217                                             (u16 *)buf);
 218                else
 219                        pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL,
 220                                              *(u16 *)buf);
 221                buf += 4;
 222        }
 223        len += 4;
 224
 225        /*
 226         * If we have any Low Priority VCs and a VC Arbitration Table Offset
 227         * in Port VC Capability Register 2 then save/restore it next.
 228         */
 229        if (lpevcc) {
 230                u32 cap2;
 231                int vcarb_offset;
 232
 233                pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2);
 234                vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16;
 235
 236                if (vcarb_offset) {
 237                        int size, vcarb_phases = 0;
 238
 239                        if (cap2 & PCI_VC_CAP2_128_PHASE)
 240                                vcarb_phases = 128;
 241                        else if (cap2 & PCI_VC_CAP2_64_PHASE)
 242                                vcarb_phases = 64;
 243                        else if (cap2 & PCI_VC_CAP2_32_PHASE)
 244                                vcarb_phases = 32;
 245
 246                        /* Fixed 4 bits per phase per lpevcc (plus VC0) */
 247                        size = ((lpevcc + 1) * vcarb_phases * 4) / 8;
 248
 249                        if (size && buf) {
 250                                pci_vc_save_restore_dwords(dev,
 251                                                           pos + vcarb_offset,
 252                                                           (u32 *)buf,
 253                                                           size / 4, save);
 254                                /*
 255                                 * On restore, we need to signal hardware to
 256                                 * re-load the VC Arbitration Table.
 257                                 */
 258                                if (!save)
 259                                        pci_vc_load_arb_table(dev, pos);
 260
 261                                buf += size;
 262                        }
 263                        len += size;
 264                }
 265        }
 266
 267        /*
 268         * In addition to each VC Resource Control Register, we may have a
 269         * Port Arbitration Table attached to each VC.  The Port Arbitration
 270         * Table Offset in each VC Resource Capability Register tells us if
 271         * it exists.  The entry size is global from the Port VC Capability
 272         * Register1 above.  The number of phases is determined per VC.
 273         */
 274        for (i = 0; i < evcc + 1; i++) {
 275                u32 cap;
 276                int parb_offset;
 277
 278                pci_read_config_dword(dev, pos + PCI_VC_RES_CAP +
 279                                      (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap);
 280                parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16;
 281                if (parb_offset) {
 282                        int size, parb_phases = 0;
 283
 284                        if (cap & PCI_VC_RES_CAP_256_PHASE)
 285                                parb_phases = 256;
 286                        else if (cap & (PCI_VC_RES_CAP_128_PHASE |
 287                                        PCI_VC_RES_CAP_128_PHASE_TB))
 288                                parb_phases = 128;
 289                        else if (cap & PCI_VC_RES_CAP_64_PHASE)
 290                                parb_phases = 64;
 291                        else if (cap & PCI_VC_RES_CAP_32_PHASE)
 292                                parb_phases = 32;
 293
 294                        size = (parb_size * parb_phases) / 8;
 295
 296                        if (size && buf) {
 297                                pci_vc_save_restore_dwords(dev,
 298                                                           pos + parb_offset,
 299                                                           (u32 *)buf,
 300                                                           size / 4, save);
 301                                buf += size;
 302                        }
 303                        len += size;
 304                }
 305
 306                /* VC Resource Control Register */
 307                if (buf) {
 308                        int ctrl_pos = pos + PCI_VC_RES_CTRL +
 309                                                (i * PCI_CAP_VC_PER_VC_SIZEOF);
 310                        if (save)
 311                                pci_read_config_dword(dev, ctrl_pos,
 312                                                      (u32 *)buf);
 313                        else {
 314                                u32 tmp, ctrl = *(u32 *)buf;
 315                                /*
 316                                 * For an FLR case, the VC config may remain.
 317                                 * Preserve enable bit, restore the rest.
 318                                 */
 319                                pci_read_config_dword(dev, ctrl_pos, &tmp);
 320                                tmp &= PCI_VC_RES_CTRL_ENABLE;
 321                                tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE;
 322                                pci_write_config_dword(dev, ctrl_pos, tmp);
 323                                /* Load port arbitration table if used */
 324                                if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT)
 325                                        pci_vc_load_port_arb_table(dev, pos, i);
 326                                /* Re-enable if needed */
 327                                if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE)
 328                                        pci_vc_enable(dev, pos, i);
 329                        }
 330                        buf += 4;
 331                }
 332                len += 4;
 333        }
 334
 335        return buf ? 0 : len;
 336}
 337
 338static struct {
 339        u16 id;
 340        const char *name;
 341} vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" },
 342                { PCI_EXT_CAP_ID_VC, "VC" },
 343                { PCI_EXT_CAP_ID_VC9, "VC9" } };
 344
 345/**
 346 * pci_save_vc_state - Save VC state to pre-allocate save buffer
 347 * @dev: device
 348 *
 349 * For each type of VC capability, VC/VC9/MFVC, find the capability and
 350 * save it to the pre-allocated save buffer.
 351 */
 352int pci_save_vc_state(struct pci_dev *dev)
 353{
 354        int i;
 355
 356        for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
 357                int pos, ret;
 358                struct pci_cap_saved_state *save_state;
 359
 360                pos = pci_find_ext_capability(dev, vc_caps[i].id);
 361                if (!pos)
 362                        continue;
 363
 364                save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id);
 365                if (!save_state) {
 366                        pci_err(dev, "%s buffer not found in %s\n",
 367                                vc_caps[i].name, __func__);
 368                        return -ENOMEM;
 369                }
 370
 371                ret = pci_vc_do_save_buffer(dev, pos, save_state, true);
 372                if (ret) {
 373                        pci_err(dev, "%s save unsuccessful %s\n",
 374                                vc_caps[i].name, __func__);
 375                        return ret;
 376                }
 377        }
 378
 379        return 0;
 380}
 381
 382/**
 383 * pci_restore_vc_state - Restore VC state from save buffer
 384 * @dev: device
 385 *
 386 * For each type of VC capability, VC/VC9/MFVC, find the capability and
 387 * restore it from the previously saved buffer.
 388 */
 389void pci_restore_vc_state(struct pci_dev *dev)
 390{
 391        int i;
 392
 393        for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
 394                int pos;
 395                struct pci_cap_saved_state *save_state;
 396
 397                pos = pci_find_ext_capability(dev, vc_caps[i].id);
 398                save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id);
 399                if (!save_state || !pos)
 400                        continue;
 401
 402                pci_vc_do_save_buffer(dev, pos, save_state, false);
 403        }
 404}
 405
 406/**
 407 * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps
 408 * @dev: device
 409 *
 410 * For each type of VC capability, VC/VC9/MFVC, find the capability, size
 411 * it, and allocate a buffer for save/restore.
 412 */
 413void pci_allocate_vc_save_buffers(struct pci_dev *dev)
 414{
 415        int i;
 416
 417        for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
 418                int len, pos = pci_find_ext_capability(dev, vc_caps[i].id);
 419
 420                if (!pos)
 421                        continue;
 422
 423                len = pci_vc_do_save_buffer(dev, pos, NULL, false);
 424                if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len))
 425                        pci_err(dev, "unable to preallocate %s save buffer\n",
 426                                vc_caps[i].name);
 427        }
 428}
 429