LXR linux/drivers/net/ethernet/sfc/falcon/nic.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/****************************************************************************
   3 * Driver for Solarflare network controllers and boards
   4 * Copyright 2005-2006 Fen Systems Ltd.
   5 * Copyright 2006-2013 Solarflare Communications Inc.
   6 */
   7
   8#include <linux/bitops.h>
   9#include <linux/delay.h>
  10#include <linux/interrupt.h>
  11#include <linux/pci.h>
  12#include <linux/module.h>
  13#include <linux/seq_file.h>
  14#include <linux/cpu_rmap.h>
  15#include "net_driver.h"
  16#include "bitfield.h"
  17#include "efx.h"
  18#include "nic.h"
  19#include "farch_regs.h"
  20#include "io.h"
  21#include "workarounds.h"
  22
  23/**************************************************************************
  24 *
  25 * Generic buffer handling
  26 * These buffers are used for interrupt status, MAC stats, etc.
  27 *
  28 **************************************************************************/
  29
  30int ef4_nic_alloc_buffer(struct ef4_nic *efx, struct ef4_buffer *buffer,
  31                         unsigned int len, gfp_t gfp_flags)
  32{
  33        buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,
  34                                          &buffer->dma_addr, gfp_flags);
  35        if (!buffer->addr)
  36                return -ENOMEM;
  37        buffer->len = len;
  38        return 0;
  39}
  40
  41void ef4_nic_free_buffer(struct ef4_nic *efx, struct ef4_buffer *buffer)
  42{
  43        if (buffer->addr) {
  44                dma_free_coherent(&efx->pci_dev->dev, buffer->len,
  45                                  buffer->addr, buffer->dma_addr);
  46                buffer->addr = NULL;
  47        }
  48}
  49
  50/* Check whether an event is present in the eventq at the current
  51 * read pointer.  Only useful for self-test.
  52 */
  53bool ef4_nic_event_present(struct ef4_channel *channel)
  54{
  55        return ef4_event_present(ef4_event(channel, channel->eventq_read_ptr));
  56}
  57
  58void ef4_nic_event_test_start(struct ef4_channel *channel)
  59{
  60        channel->event_test_cpu = -1;
  61        smp_wmb();
  62        channel->efx->type->ev_test_generate(channel);
  63}
  64
  65int ef4_nic_irq_test_start(struct ef4_nic *efx)
  66{
  67        efx->last_irq_cpu = -1;
  68        smp_wmb();
  69        return efx->type->irq_test_generate(efx);
  70}
  71
  72/* Hook interrupt handler(s)
  73 * Try MSI and then legacy interrupts.
  74 */
  75int ef4_nic_init_interrupt(struct ef4_nic *efx)
  76{
  77        struct ef4_channel *channel;
  78        unsigned int n_irqs;
  79        int rc;
  80
  81        if (!EF4_INT_MODE_USE_MSI(efx)) {
  82                rc = request_irq(efx->legacy_irq,
  83                                 efx->type->irq_handle_legacy, IRQF_SHARED,
  84                                 efx->name, efx);
  85                if (rc) {
  86                        netif_err(efx, drv, efx->net_dev,
  87                                  "failed to hook legacy IRQ %d\n",
  88                                  efx->pci_dev->irq);
  89                        goto fail1;
  90                }
  91                return 0;
  92        }
  93
  94#ifdef CONFIG_RFS_ACCEL
  95        if (efx->interrupt_mode == EF4_INT_MODE_MSIX) {
  96                efx->net_dev->rx_cpu_rmap =
  97                        alloc_irq_cpu_rmap(efx->n_rx_channels);
  98                if (!efx->net_dev->rx_cpu_rmap) {
  99                        rc = -ENOMEM;
 100                        goto fail1;
 101                }
 102        }
 103#endif
 104
 105        /* Hook MSI or MSI-X interrupt */
 106        n_irqs = 0;
 107        ef4_for_each_channel(channel, efx) {
 108                rc = request_irq(channel->irq, efx->type->irq_handle_msi,
 109                                 IRQF_PROBE_SHARED, /* Not shared */
 110                                 efx->msi_context[channel->channel].name,
 111                                 &efx->msi_context[channel->channel]);
 112                if (rc) {
 113                        netif_err(efx, drv, efx->net_dev,
 114                                  "failed to hook IRQ %d\n", channel->irq);
 115                        goto fail2;
 116                }
 117                ++n_irqs;
 118
 119#ifdef CONFIG_RFS_ACCEL
 120                if (efx->interrupt_mode == EF4_INT_MODE_MSIX &&
 121                    channel->channel < efx->n_rx_channels) {
 122                        rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,
 123                                              channel->irq);
 124                        if (rc)
 125                                goto fail2;
 126                }
 127#endif
 128        }
 129
 130        return 0;
 131
 132 fail2:
 133#ifdef CONFIG_RFS_ACCEL
 134        free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
 135        efx->net_dev->rx_cpu_rmap = NULL;
 136#endif
 137        ef4_for_each_channel(channel, efx) {
 138                if (n_irqs-- == 0)
 139                        break;
 140                free_irq(channel->irq, &efx->msi_context[channel->channel]);
 141        }
 142 fail1:
 143        return rc;
 144}
 145
 146void ef4_nic_fini_interrupt(struct ef4_nic *efx)
 147{
 148        struct ef4_channel *channel;
 149
 150#ifdef CONFIG_RFS_ACCEL
 151        free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
 152        efx->net_dev->rx_cpu_rmap = NULL;
 153#endif
 154
 155        if (EF4_INT_MODE_USE_MSI(efx)) {
 156                /* Disable MSI/MSI-X interrupts */
 157                ef4_for_each_channel(channel, efx)
 158                        free_irq(channel->irq,
 159                                 &efx->msi_context[channel->channel]);
 160        } else {
 161                /* Disable legacy interrupt */
 162                free_irq(efx->legacy_irq, efx);
 163        }
 164}
 165
 166/* Register dump */
 167
 168#define REGISTER_REVISION_FA    1
 169#define REGISTER_REVISION_FB    2
 170#define REGISTER_REVISION_FC    3
 171#define REGISTER_REVISION_FZ    3       /* last Falcon arch revision */
 172#define REGISTER_REVISION_ED    4
 173#define REGISTER_REVISION_EZ    4       /* latest EF10 revision */
 174
 175struct ef4_nic_reg {
 176        u32 offset:24;
 177        u32 min_revision:3, max_revision:3;
 178};
 179
 180#define REGISTER(name, arch, min_rev, max_rev) {                        \
 181        arch ## R_ ## min_rev ## max_rev ## _ ## name,                  \
 182        REGISTER_REVISION_ ## arch ## min_rev,                          \
 183        REGISTER_REVISION_ ## arch ## max_rev                           \
 184}
 185#define REGISTER_AA(name) REGISTER(name, F, A, A)
 186#define REGISTER_AB(name) REGISTER(name, F, A, B)
 187#define REGISTER_AZ(name) REGISTER(name, F, A, Z)
 188#define REGISTER_BB(name) REGISTER(name, F, B, B)
 189#define REGISTER_BZ(name) REGISTER(name, F, B, Z)
 190#define REGISTER_CZ(name) REGISTER(name, F, C, Z)
 191
 192static const struct ef4_nic_reg ef4_nic_regs[] = {
 193        REGISTER_AZ(ADR_REGION),
 194        REGISTER_AZ(INT_EN_KER),
 195        REGISTER_BZ(INT_EN_CHAR),
 196        REGISTER_AZ(INT_ADR_KER),
 197        REGISTER_BZ(INT_ADR_CHAR),
 198        /* INT_ACK_KER is WO */
 199        /* INT_ISR0 is RC */
 200        REGISTER_AZ(HW_INIT),
 201        REGISTER_CZ(USR_EV_CFG),
 202        REGISTER_AB(EE_SPI_HCMD),
 203        REGISTER_AB(EE_SPI_HADR),
 204        REGISTER_AB(EE_SPI_HDATA),
 205        REGISTER_AB(EE_BASE_PAGE),
 206        REGISTER_AB(EE_VPD_CFG0),
 207        /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */
 208        /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */
 209        /* PCIE_CORE_INDIRECT is indirect */
 210        REGISTER_AB(NIC_STAT),
 211        REGISTER_AB(GPIO_CTL),
 212        REGISTER_AB(GLB_CTL),
 213        /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */
 214        REGISTER_BZ(DP_CTRL),
 215        REGISTER_AZ(MEM_STAT),
 216        REGISTER_AZ(CS_DEBUG),
 217        REGISTER_AZ(ALTERA_BUILD),
 218        REGISTER_AZ(CSR_SPARE),
 219        REGISTER_AB(PCIE_SD_CTL0123),
 220        REGISTER_AB(PCIE_SD_CTL45),
 221        REGISTER_AB(PCIE_PCS_CTL_STAT),
 222        /* DEBUG_DATA_OUT is not used */
 223        /* DRV_EV is WO */
 224        REGISTER_AZ(EVQ_CTL),
 225        REGISTER_AZ(EVQ_CNT1),
 226        REGISTER_AZ(EVQ_CNT2),
 227        REGISTER_AZ(BUF_TBL_CFG),
 228        REGISTER_AZ(SRM_RX_DC_CFG),
 229        REGISTER_AZ(SRM_TX_DC_CFG),
 230        REGISTER_AZ(SRM_CFG),
 231        /* BUF_TBL_UPD is WO */
 232        REGISTER_AZ(SRM_UPD_EVQ),
 233        REGISTER_AZ(SRAM_PARITY),
 234        REGISTER_AZ(RX_CFG),
 235        REGISTER_BZ(RX_FILTER_CTL),
 236        /* RX_FLUSH_DESCQ is WO */
 237        REGISTER_AZ(RX_DC_CFG),
 238        REGISTER_AZ(RX_DC_PF_WM),
 239        REGISTER_BZ(RX_RSS_TKEY),
 240        /* RX_NODESC_DROP is RC */
 241        REGISTER_AA(RX_SELF_RST),
 242        /* RX_DEBUG, RX_PUSH_DROP are not used */
 243        REGISTER_CZ(RX_RSS_IPV6_REG1),
 244        REGISTER_CZ(RX_RSS_IPV6_REG2),
 245        REGISTER_CZ(RX_RSS_IPV6_REG3),
 246        /* TX_FLUSH_DESCQ is WO */
 247        REGISTER_AZ(TX_DC_CFG),
 248        REGISTER_AA(TX_CHKSM_CFG),
 249        REGISTER_AZ(TX_CFG),
 250        /* TX_PUSH_DROP is not used */
 251        REGISTER_AZ(TX_RESERVED),
 252        REGISTER_BZ(TX_PACE),
 253        /* TX_PACE_DROP_QID is RC */
 254        REGISTER_BB(TX_VLAN),
 255        REGISTER_BZ(TX_IPFIL_PORTEN),
 256        REGISTER_AB(MD_TXD),
 257        REGISTER_AB(MD_RXD),
 258        REGISTER_AB(MD_CS),
 259        REGISTER_AB(MD_PHY_ADR),
 260        REGISTER_AB(MD_ID),
 261        /* MD_STAT is RC */
 262        REGISTER_AB(MAC_STAT_DMA),
 263        REGISTER_AB(MAC_CTRL),
 264        REGISTER_BB(GEN_MODE),
 265        REGISTER_AB(MAC_MC_HASH_REG0),
 266        REGISTER_AB(MAC_MC_HASH_REG1),
 267        REGISTER_AB(GM_CFG1),
 268        REGISTER_AB(GM_CFG2),
 269        /* GM_IPG and GM_HD are not used */
 270        REGISTER_AB(GM_MAX_FLEN),
 271        /* GM_TEST is not used */
 272        REGISTER_AB(GM_ADR1),
 273        REGISTER_AB(GM_ADR2),
 274        REGISTER_AB(GMF_CFG0),
 275        REGISTER_AB(GMF_CFG1),
 276        REGISTER_AB(GMF_CFG2),
 277        REGISTER_AB(GMF_CFG3),
 278        REGISTER_AB(GMF_CFG4),
 279        REGISTER_AB(GMF_CFG5),
 280        REGISTER_BB(TX_SRC_MAC_CTL),
 281        REGISTER_AB(XM_ADR_LO),
 282        REGISTER_AB(XM_ADR_HI),
 283        REGISTER_AB(XM_GLB_CFG),
 284        REGISTER_AB(XM_TX_CFG),
 285        REGISTER_AB(XM_RX_CFG),
 286        REGISTER_AB(XM_MGT_INT_MASK),
 287        REGISTER_AB(XM_FC),
 288        REGISTER_AB(XM_PAUSE_TIME),
 289        REGISTER_AB(XM_TX_PARAM),
 290        REGISTER_AB(XM_RX_PARAM),
 291        /* XM_MGT_INT_MSK (note no 'A') is RC */
 292        REGISTER_AB(XX_PWR_RST),
 293        REGISTER_AB(XX_SD_CTL),
 294        REGISTER_AB(XX_TXDRV_CTL),
 295        /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */
 296        /* XX_CORE_STAT is partly RC */
 297};
 298
 299struct ef4_nic_reg_table {
 300        u32 offset:24;
 301        u32 min_revision:3, max_revision:3;
 302        u32 step:6, rows:21;
 303};
 304
 305#define REGISTER_TABLE_DIMENSIONS(_, offset, arch, min_rev, max_rev, step, rows) { \
 306        offset,                                                         \
 307        REGISTER_REVISION_ ## arch ## min_rev,                          \
 308        REGISTER_REVISION_ ## arch ## max_rev,                          \
 309        step, rows                                                      \
 310}
 311#define REGISTER_TABLE(name, arch, min_rev, max_rev)                    \
 312        REGISTER_TABLE_DIMENSIONS(                                      \
 313                name, arch ## R_ ## min_rev ## max_rev ## _ ## name,    \
 314                arch, min_rev, max_rev,                                 \
 315                arch ## R_ ## min_rev ## max_rev ## _ ## name ## _STEP, \
 316                arch ## R_ ## min_rev ## max_rev ## _ ## name ## _ROWS)
 317#define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, F, A, A)
 318#define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, F, A, Z)
 319#define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, F, B, B)
 320#define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, F, B, Z)
 321#define REGISTER_TABLE_BB_CZ(name)                                      \
 322        REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, B, B,        \
 323                                  FR_BZ_ ## name ## _STEP,              \
 324                                  FR_BB_ ## name ## _ROWS),             \
 325        REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, C, Z,        \
 326                                  FR_BZ_ ## name ## _STEP,              \
 327                                  FR_CZ_ ## name ## _ROWS)
 328#define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, F, C, Z)
 329
 330static const struct ef4_nic_reg_table ef4_nic_reg_tables[] = {
 331        /* DRIVER is not used */
 332        /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */
 333        REGISTER_TABLE_BB(TX_IPFIL_TBL),
 334        REGISTER_TABLE_BB(TX_SRC_MAC_TBL),
 335        REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),
 336        REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),
 337        REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),
 338        REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),
 339        REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),
 340        REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),
 341        /* We can't reasonably read all of the buffer table (up to 8MB!).
 342         * However this driver will only use a few entries.  Reading
 343         * 1K entries allows for some expansion of queue count and
 344         * size before we need to change the version. */
 345        REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,
 346                                  F, A, A, 8, 1024),
 347        REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,
 348                                  F, B, Z, 8, 1024),
 349        REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),
 350        REGISTER_TABLE_BB_CZ(TIMER_TBL),
 351        REGISTER_TABLE_BB_CZ(TX_PACE_TBL),
 352        REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),
 353        /* TX_FILTER_TBL0 is huge and not used by this driver */
 354        REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),
 355        REGISTER_TABLE_CZ(MC_TREG_SMEM),
 356        /* MSIX_PBA_TABLE is not mapped */
 357        /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */
 358        REGISTER_TABLE_BZ(RX_FILTER_TBL0),
 359};
 360
 361size_t ef4_nic_get_regs_len(struct ef4_nic *efx)
 362{
 363        const struct ef4_nic_reg *reg;
 364        const struct ef4_nic_reg_table *table;
 365        size_t len = 0;
 366
 367        for (reg = ef4_nic_regs;
 368             reg < ef4_nic_regs + ARRAY_SIZE(ef4_nic_regs);
 369             reg++)
 370                if (efx->type->revision >= reg->min_revision &&
 371                    efx->type->revision <= reg->max_revision)
 372                        len += sizeof(ef4_oword_t);
 373
 374        for (table = ef4_nic_reg_tables;
 375             table < ef4_nic_reg_tables + ARRAY_SIZE(ef4_nic_reg_tables);
 376             table++)
 377                if (efx->type->revision >= table->min_revision &&
 378                    efx->type->revision <= table->max_revision)
 379                        len += table->rows * min_t(size_t, table->step, 16);
 380
 381        return len;
 382}
 383
 384void ef4_nic_get_regs(struct ef4_nic *efx, void *buf)
 385{
 386        const struct ef4_nic_reg *reg;
 387        const struct ef4_nic_reg_table *table;
 388
 389        for (reg = ef4_nic_regs;
 390             reg < ef4_nic_regs + ARRAY_SIZE(ef4_nic_regs);
 391             reg++) {
 392                if (efx->type->revision >= reg->min_revision &&
 393                    efx->type->revision <= reg->max_revision) {
 394                        ef4_reado(efx, (ef4_oword_t *)buf, reg->offset);
 395                        buf += sizeof(ef4_oword_t);
 396                }
 397        }
 398
 399        for (table = ef4_nic_reg_tables;
 400             table < ef4_nic_reg_tables + ARRAY_SIZE(ef4_nic_reg_tables);
 401             table++) {
 402                size_t size, i;
 403
 404                if (!(efx->type->revision >= table->min_revision &&
 405                      efx->type->revision <= table->max_revision))
 406                        continue;
 407
 408                size = min_t(size_t, table->step, 16);
 409
 410                for (i = 0; i < table->rows; i++) {
 411                        switch (table->step) {
 412                        case 4: /* 32-bit SRAM */
 413                                ef4_readd(efx, buf, table->offset + 4 * i);
 414                                break;
 415                        case 8: /* 64-bit SRAM */
 416                                ef4_sram_readq(efx,
 417                                               efx->membase + table->offset,
 418                                               buf, i);
 419                                break;
 420                        case 16: /* 128-bit-readable register */
 421                                ef4_reado_table(efx, buf, table->offset, i);
 422                                break;
 423                        case 32: /* 128-bit register, interleaved */
 424                                ef4_reado_table(efx, buf, table->offset, 2 * i);
 425                                break;
 426                        default:
 427                                WARN_ON(1);
 428                                return;
 429                        }
 430                        buf += size;
 431                }
 432        }
 433}
 434
 435/**
 436 * ef4_nic_describe_stats - Describe supported statistics for ethtool
 437 * @desc: Array of &struct ef4_hw_stat_desc describing the statistics
 438 * @count: Length of the @desc array
 439 * @mask: Bitmask of which elements of @desc are enabled
 440 * @names: Buffer to copy names to, or %NULL.  The names are copied
 441 *      starting at intervals of %ETH_GSTRING_LEN bytes.
 442 *
 443 * Returns the number of visible statistics, i.e. the number of set
 444 * bits in the first @count bits of @mask for which a name is defined.
 445 */
 446size_t ef4_nic_describe_stats(const struct ef4_hw_stat_desc *desc, size_t count,
 447                              const unsigned long *mask, u8 *names)
 448{
 449        size_t visible = 0;
 450        size_t index;
 451
 452        for_each_set_bit(index, mask, count) {
 453                if (desc[index].name) {
 454                        if (names) {
 455                                strlcpy(names, desc[index].name,
 456                                        ETH_GSTRING_LEN);
 457                                names += ETH_GSTRING_LEN;
 458                        }
 459                        ++visible;
 460                }
 461        }
 462
 463        return visible;
 464}
 465
 466/**
 467 * ef4_nic_update_stats - Convert statistics DMA buffer to array of u64
 468 * @desc: Array of &struct ef4_hw_stat_desc describing the DMA buffer
 469 *      layout.  DMA widths of 0, 16, 32 and 64 are supported; where
 470 *      the width is specified as 0 the corresponding element of
 471 *      @stats is not updated.
 472 * @count: Length of the @desc array
 473 * @mask: Bitmask of which elements of @desc are enabled
 474 * @stats: Buffer to update with the converted statistics.  The length
 475 *      of this array must be at least @count.
 476 * @dma_buf: DMA buffer containing hardware statistics
 477 * @accumulate: If set, the converted values will be added rather than
 478 *      directly stored to the corresponding elements of @stats
 479 */
 480void ef4_nic_update_stats(const struct ef4_hw_stat_desc *desc, size_t count,
 481                          const unsigned long *mask,
 482                          u64 *stats, const void *dma_buf, bool accumulate)
 483{
 484        size_t index;
 485
 486        for_each_set_bit(index, mask, count) {
 487                if (desc[index].dma_width) {
 488                        const void *addr = dma_buf + desc[index].offset;
 489                        u64 val;
 490
 491                        switch (desc[index].dma_width) {
 492                        case 16:
 493                                val = le16_to_cpup((__le16 *)addr);
 494                                break;
 495                        case 32:
 496                                val = le32_to_cpup((__le32 *)addr);
 497                                break;
 498                        case 64:
 499                                val = le64_to_cpup((__le64 *)addr);
 500                                break;
 501                        default:
 502                                WARN_ON(1);
 503                                val = 0;
 504                                break;
 505                        }
 506
 507                        if (accumulate)
 508                                stats[index] += val;
 509                        else
 510                                stats[index] = val;
 511                }
 512        }
 513}
 514
 515void ef4_nic_fix_nodesc_drop_stat(struct ef4_nic *efx, u64 *rx_nodesc_drops)
 516{
 517        /* if down, or this is the first update after coming up */
 518        if (!(efx->net_dev->flags & IFF_UP) || !efx->rx_nodesc_drops_prev_state)
 519                efx->rx_nodesc_drops_while_down +=
 520                        *rx_nodesc_drops - efx->rx_nodesc_drops_total;
 521        efx->rx_nodesc_drops_total = *rx_nodesc_drops;
 522        efx->rx_nodesc_drops_prev_state = !!(efx->net_dev->flags & IFF_UP);
 523        *rx_nodesc_drops -= efx->rx_nodesc_drops_while_down;
 524}
 525