LXR linux/drivers/net/ethernet/chelsio/cxgb4/t4

   1/*
   2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3 *
   4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
   5 *
   6 * This software is available to you under a choice of one of two
   7 * licenses.  You may choose to be licensed under the terms of the GNU
   8 * General Public License (GPL) Version 2, available from the file
   9 * COPYING in the main directory of this source tree, or the
  10 * OpenIB.org BSD license below:
  11 *
  12 *     Redistribution and use in source and binary forms, with or
  13 *     without modification, are permitted provided that the following
  14 *     conditions are met:
  15 *
  16 *      - Redistributions of source code must retain the above
  17 *        copyright notice, this list of conditions and the following
  18 *        disclaimer.
  19 *
  20 *      - Redistributions in binary form must reproduce the above
  21 *        copyright notice, this list of conditions and the following
  22 *        disclaimer in the documentation and/or other materials
  23 *        provided with the distribution.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32 * SOFTWARE.
  33 */
  34
  35#include <linux/init.h>
  36#include <linux/delay.h>
  37#include "cxgb4.h"
  38#include "t4_regs.h"
  39#include "t4_values.h"
  40#include "t4fw_api.h"
  41#include "t4fw_version.h"
  42
  43/**
  44 *      t4_wait_op_done_val - wait until an operation is completed
  45 *      @adapter: the adapter performing the operation
  46 *      @reg: the register to check for completion
  47 *      @mask: a single-bit field within @reg that indicates completion
  48 *      @polarity: the value of the field when the operation is completed
  49 *      @attempts: number of check iterations
  50 *      @delay: delay in usecs between iterations
  51 *      @valp: where to store the value of the register at completion time
  52 *
  53 *      Wait until an operation is completed by checking a bit in a register
  54 *      up to @attempts times.  If @valp is not NULL the value of the register
  55 *      at the time it indicated completion is stored there.  Returns 0 if the
  56 *      operation completes and -EAGAIN otherwise.
  57 */
  58static int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask,
  59                               int polarity, int attempts, int delay, u32 *valp)
  60{
  61        while (1) {
  62                u32 val = t4_read_reg(adapter, reg);
  63
  64                if (!!(val & mask) == polarity) {
  65                        if (valp)
  66                                *valp = val;
  67                        return 0;
  68                }
  69                if (--attempts == 0)
  70                        return -EAGAIN;
  71                if (delay)
  72                        udelay(delay);
  73        }
  74}
  75
  76static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask,
  77                                  int polarity, int attempts, int delay)
  78{
  79        return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts,
  80                                   delay, NULL);
  81}
  82
  83/**
  84 *      t4_set_reg_field - set a register field to a value
  85 *      @adapter: the adapter to program
  86 *      @addr: the register address
  87 *      @mask: specifies the portion of the register to modify
  88 *      @val: the new value for the register field
  89 *
  90 *      Sets a register field specified by the supplied mask to the
  91 *      given value.
  92 */
  93void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask,
  94                      u32 val)
  95{
  96        u32 v = t4_read_reg(adapter, addr) & ~mask;
  97
  98        t4_write_reg(adapter, addr, v | val);
  99        (void) t4_read_reg(adapter, addr);      /* flush */
 100}
 101
 102/**
 103 *      t4_read_indirect - read indirectly addressed registers
 104 *      @adap: the adapter
 105 *      @addr_reg: register holding the indirect address
 106 *      @data_reg: register holding the value of the indirect register
 107 *      @vals: where the read register values are stored
 108 *      @nregs: how many indirect registers to read
 109 *      @start_idx: index of first indirect register to read
 110 *
 111 *      Reads registers that are accessed indirectly through an address/data
 112 *      register pair.
 113 */
 114void t4_read_indirect(struct adapter *adap, unsigned int addr_reg,
 115                             unsigned int data_reg, u32 *vals,
 116                             unsigned int nregs, unsigned int start_idx)
 117{
 118        while (nregs--) {
 119                t4_write_reg(adap, addr_reg, start_idx);
 120                *vals++ = t4_read_reg(adap, data_reg);
 121                start_idx++;
 122        }
 123}
 124
 125/**
 126 *      t4_write_indirect - write indirectly addressed registers
 127 *      @adap: the adapter
 128 *      @addr_reg: register holding the indirect addresses
 129 *      @data_reg: register holding the value for the indirect registers
 130 *      @vals: values to write
 131 *      @nregs: how many indirect registers to write
 132 *      @start_idx: address of first indirect register to write
 133 *
 134 *      Writes a sequential block of registers that are accessed indirectly
 135 *      through an address/data register pair.
 136 */
 137void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
 138                       unsigned int data_reg, const u32 *vals,
 139                       unsigned int nregs, unsigned int start_idx)
 140{
 141        while (nregs--) {
 142                t4_write_reg(adap, addr_reg, start_idx++);
 143                t4_write_reg(adap, data_reg, *vals++);
 144        }
 145}
 146
 147/*
 148 * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor
 149 * mechanism.  This guarantees that we get the real value even if we're
 150 * operating within a Virtual Machine and the Hypervisor is trapping our
 151 * Configuration Space accesses.
 152 */
 153void t4_hw_pci_read_cfg4(struct adapter *adap, int reg, u32 *val)
 154{
 155        u32 req = FUNCTION_V(adap->pf) | REGISTER_V(reg);
 156
 157        if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
 158                req |= ENABLE_F;
 159        else
 160                req |= T6_ENABLE_F;
 161
 162        if (is_t4(adap->params.chip))
 163                req |= LOCALCFG_F;
 164
 165        t4_write_reg(adap, PCIE_CFG_SPACE_REQ_A, req);
 166        *val = t4_read_reg(adap, PCIE_CFG_SPACE_DATA_A);
 167
 168        /* Reset ENABLE to 0 so reads of PCIE_CFG_SPACE_DATA won't cause a
 169         * Configuration Space read.  (None of the other fields matter when
 170         * ENABLE is 0 so a simple register write is easier than a
 171         * read-modify-write via t4_set_reg_field().)
 172         */
 173        t4_write_reg(adap, PCIE_CFG_SPACE_REQ_A, 0);
 174}
 175
 176/*
 177 * t4_report_fw_error - report firmware error
 178 * @adap: the adapter
 179 *
 180 * The adapter firmware can indicate error conditions to the host.
 181 * If the firmware has indicated an error, print out the reason for
 182 * the firmware error.
 183 */
 184static void t4_report_fw_error(struct adapter *adap)
 185{
 186        static const char *const reason[] = {
 187                "Crash",                        /* PCIE_FW_EVAL_CRASH */
 188                "During Device Preparation",    /* PCIE_FW_EVAL_PREP */
 189                "During Device Configuration",  /* PCIE_FW_EVAL_CONF */
 190                "During Device Initialization", /* PCIE_FW_EVAL_INIT */
 191                "Unexpected Event",             /* PCIE_FW_EVAL_UNEXPECTEDEVENT */
 192                "Insufficient Airflow",         /* PCIE_FW_EVAL_OVERHEAT */
 193                "Device Shutdown",              /* PCIE_FW_EVAL_DEVICESHUTDOWN */
 194                "Reserved",                     /* reserved */
 195        };
 196        u32 pcie_fw;
 197
 198        pcie_fw = t4_read_reg(adap, PCIE_FW_A);
 199        if (pcie_fw & PCIE_FW_ERR_F)
 200                dev_err(adap->pdev_dev, "Firmware reports adapter error: %s\n",
 201                        reason[PCIE_FW_EVAL_G(pcie_fw)]);
 202}
 203
 204/*
 205 * Get the reply to a mailbox command and store it in @rpl in big-endian order.
 206 */
 207static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit,
 208                         u32 mbox_addr)
 209{
 210        for ( ; nflit; nflit--, mbox_addr += 8)
 211                *rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr));
 212}
 213
 214/*
 215 * Handle a FW assertion reported in a mailbox.
 216 */
 217static void fw_asrt(struct adapter *adap, u32 mbox_addr)
 218{
 219        struct fw_debug_cmd asrt;
 220
 221        get_mbox_rpl(adap, (__be64 *)&asrt, sizeof(asrt) / 8, mbox_addr);
 222        dev_alert(adap->pdev_dev,
 223                  "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n",
 224                  asrt.u.assert.filename_0_7, be32_to_cpu(asrt.u.assert.line),
 225                  be32_to_cpu(asrt.u.assert.x), be32_to_cpu(asrt.u.assert.y));
 226}
 227
 228/**
 229 *      t4_record_mbox - record a Firmware Mailbox Command/Reply in the log
 230 *      @adapter: the adapter
 231 *      @cmd: the Firmware Mailbox Command or Reply
 232 *      @size: command length in bytes
 233 *      @access: the time (ms) needed to access the Firmware Mailbox
 234 *      @execute: the time (ms) the command spent being executed
 235 */
 236static void t4_record_mbox(struct adapter *adapter,
 237                           const __be64 *cmd, unsigned int size,
 238                           int access, int execute)
 239{
 240        struct mbox_cmd_log *log = adapter->mbox_log;
 241        struct mbox_cmd *entry;
 242        int i;
 243
 244        entry = mbox_cmd_log_entry(log, log->cursor++);
 245        if (log->cursor == log->size)
 246                log->cursor = 0;
 247
 248        for (i = 0; i < size / 8; i++)
 249                entry->cmd[i] = be64_to_cpu(cmd[i]);
 250        while (i < MBOX_LEN / 8)
 251                entry->cmd[i++] = 0;
 252        entry->timestamp = jiffies;
 253        entry->seqno = log->seqno++;
 254        entry->access = access;
 255        entry->execute = execute;
 256}
 257
 258/**
 259 *      t4_wr_mbox_meat_timeout - send a command to FW through the given mailbox
 260 *      @adap: the adapter
 261 *      @mbox: index of the mailbox to use
 262 *      @cmd: the command to write
 263 *      @size: command length in bytes
 264 *      @rpl: where to optionally store the reply
 265 *      @sleep_ok: if true we may sleep while awaiting command completion
 266 *      @timeout: time to wait for command to finish before timing out
 267 *
 268 *      Sends the given command to FW through the selected mailbox and waits
 269 *      for the FW to execute the command.  If @rpl is not %NULL it is used to
 270 *      store the FW's reply to the command.  The command and its optional
 271 *      reply are of the same length.  FW can take up to %FW_CMD_MAX_TIMEOUT ms
 272 *      to respond.  @sleep_ok determines whether we may sleep while awaiting
 273 *      the response.  If sleeping is allowed we use progressive backoff
 274 *      otherwise we spin.
 275 *
 276 *      The return value is 0 on success or a negative errno on failure.  A
 277 *      failure can happen either because we are not able to execute the
 278 *      command or FW executes it but signals an error.  In the latter case
 279 *      the return value is the error code indicated by FW (negated).
 280 */
 281int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 282                            int size, void *rpl, bool sleep_ok, int timeout)
 283{
 284        static const int delay[] = {
 285                1, 1, 3, 5, 10, 10, 20, 50, 100, 200
 286        };
 287
 288        struct mbox_list entry;
 289        u16 access = 0;
 290        u16 execute = 0;
 291        u32 v;
 292        u64 res;
 293        int i, ms, delay_idx, ret;
 294        const __be64 *p = cmd;
 295        u32 data_reg = PF_REG(mbox, CIM_PF_MAILBOX_DATA_A);
 296        u32 ctl_reg = PF_REG(mbox, CIM_PF_MAILBOX_CTRL_A);
 297        __be64 cmd_rpl[MBOX_LEN / 8];
 298        u32 pcie_fw;
 299
 300        if ((size & 15) || size > MBOX_LEN)
 301                return -EINVAL;
 302
 303        /*
 304         * If the device is off-line, as in EEH, commands will time out.
 305         * Fail them early so we don't waste time waiting.
 306         */
 307        if (adap->pdev->error_state != pci_channel_io_normal)
 308                return -EIO;
 309
 310        /* If we have a negative timeout, that implies that we can't sleep. */
 311        if (timeout < 0) {
 312                sleep_ok = false;
 313                timeout = -timeout;
 314        }
 315
 316        /* Queue ourselves onto the mailbox access list.  When our entry is at
 317         * the front of the list, we have rights to access the mailbox.  So we
 318         * wait [for a while] till we're at the front [or bail out with an
 319         * EBUSY] ...
 320         */
 321        spin_lock_bh(&adap->mbox_lock);
 322        list_add_tail(&entry.list, &adap->mlist.list);
 323        spin_unlock_bh(&adap->mbox_lock);
 324
 325        delay_idx = 0;
 326        ms = delay[0];
 327
 328        for (i = 0; ; i += ms) {
 329                /* If we've waited too long, return a busy indication.  This
 330                 * really ought to be based on our initial position in the
 331                 * mailbox access list but this is a start.  We very rearely
 332                 * contend on access to the mailbox ...
 333                 */
 334                pcie_fw = t4_read_reg(adap, PCIE_FW_A);
 335                if (i > FW_CMD_MAX_TIMEOUT || (pcie_fw & PCIE_FW_ERR_F)) {
 336                        spin_lock_bh(&adap->mbox_lock);
 337                        list_del(&entry.list);
 338                        spin_unlock_bh(&adap->mbox_lock);
 339                        ret = (pcie_fw & PCIE_FW_ERR_F) ? -ENXIO : -EBUSY;
 340                        t4_record_mbox(adap, cmd, size, access, ret);
 341                        return ret;
 342                }
 343
 344                /* If we're at the head, break out and start the mailbox
 345                 * protocol.
 346                 */
 347                if (list_first_entry(&adap->mlist.list, struct mbox_list,
 348                                     list) == &entry)
 349                        break;
 350
 351                /* Delay for a bit before checking again ... */
 352                if (sleep_ok) {
 353                        ms = delay[delay_idx];  /* last element may repeat */
 354                        if (delay_idx < ARRAY_SIZE(delay) - 1)
 355                                delay_idx++;
 356                        msleep(ms);
 357                } else {
 358                        mdelay(ms);
 359                }
 360        }
 361
 362        /* Loop trying to get ownership of the mailbox.  Return an error
 363         * if we can't gain ownership.
 364         */
 365        v = MBOWNER_G(t4_read_reg(adap, ctl_reg));
 366        for (i = 0; v == MBOX_OWNER_NONE && i < 3; i++)
 367                v = MBOWNER_G(t4_read_reg(adap, ctl_reg));
 368        if (v != MBOX_OWNER_DRV) {
 369                spin_lock_bh(&adap->mbox_lock);
 370                list_del(&entry.list);
 371                spin_unlock_bh(&adap->mbox_lock);
 372                ret = (v == MBOX_OWNER_FW) ? -EBUSY : -ETIMEDOUT;
 373                t4_record_mbox(adap, cmd, size, access, ret);
 374                return ret;
 375        }
 376
 377        /* Copy in the new mailbox command and send it on its way ... */
 378        t4_record_mbox(adap, cmd, size, access, 0);
 379        for (i = 0; i < size; i += 8)
 380                t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p++));
 381
 382        t4_write_reg(adap, ctl_reg, MBMSGVALID_F | MBOWNER_V(MBOX_OWNER_FW));
 383        t4_read_reg(adap, ctl_reg);          /* flush write */
 384
 385        delay_idx = 0;
 386        ms = delay[0];
 387
 388        for (i = 0;
 389             !((pcie_fw = t4_read_reg(adap, PCIE_FW_A)) & PCIE_FW_ERR_F) &&
 390             i < timeout;
 391             i += ms) {
 392                if (sleep_ok) {
 393                        ms = delay[delay_idx];  /* last element may repeat */
 394                        if (delay_idx < ARRAY_SIZE(delay) - 1)
 395                                delay_idx++;
 396                        msleep(ms);
 397                } else
 398                        mdelay(ms);
 399
 400                v = t4_read_reg(adap, ctl_reg);
 401                if (MBOWNER_G(v) == MBOX_OWNER_DRV) {
 402                        if (!(v & MBMSGVALID_F)) {
 403                                t4_write_reg(adap, ctl_reg, 0);
 404                                continue;
 405                        }
 406
 407                        get_mbox_rpl(adap, cmd_rpl, MBOX_LEN / 8, data_reg);
 408                        res = be64_to_cpu(cmd_rpl[0]);
 409
 410                        if (FW_CMD_OP_G(res >> 32) == FW_DEBUG_CMD) {
 411                                fw_asrt(adap, data_reg);
 412                                res = FW_CMD_RETVAL_V(EIO);
 413                        } else if (rpl) {
 414                                memcpy(rpl, cmd_rpl, size);
 415                        }
 416
 417                        t4_write_reg(adap, ctl_reg, 0);
 418
 419                        execute = i + ms;
 420                        t4_record_mbox(adap, cmd_rpl,
 421                                       MBOX_LEN, access, execute);
 422                        spin_lock_bh(&adap->mbox_lock);
 423                        list_del(&entry.list);
 424                        spin_unlock_bh(&adap->mbox_lock);
 425                        return -FW_CMD_RETVAL_G((int)res);
 426                }
 427        }
 428
 429        ret = (pcie_fw & PCIE_FW_ERR_F) ? -ENXIO : -ETIMEDOUT;
 430        t4_record_mbox(adap, cmd, size, access, ret);
 431        dev_err(adap->pdev_dev, "command %#x in mailbox %d timed out\n",
 432                *(const u8 *)cmd, mbox);
 433        t4_report_fw_error(adap);
 434        spin_lock_bh(&adap->mbox_lock);
 435        list_del(&entry.list);
 436        spin_unlock_bh(&adap->mbox_lock);
 437        t4_fatal_err(adap);
 438        return ret;
 439}
 440
 441int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
 442                    void *rpl, bool sleep_ok)
 443{
 444        return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl, sleep_ok,
 445                                       FW_CMD_MAX_TIMEOUT);
 446}
 447
 448static int t4_edc_err_read(struct adapter *adap, int idx)
 449{
 450        u32 edc_ecc_err_addr_reg;
 451        u32 rdata_reg;
 452
 453        if (is_t4(adap->params.chip)) {
 454                CH_WARN(adap, "%s: T4 NOT supported.\n", __func__);
 455                return 0;
 456        }
 457        if (idx != 0 && idx != 1) {
 458                CH_WARN(adap, "%s: idx %d NOT supported.\n", __func__, idx);
 459                return 0;
 460        }
 461
 462        edc_ecc_err_addr_reg = EDC_T5_REG(EDC_H_ECC_ERR_ADDR_A, idx);
 463        rdata_reg = EDC_T5_REG(EDC_H_BIST_STATUS_RDATA_A, idx);
 464
 465        CH_WARN(adap,
 466                "edc%d err addr 0x%x: 0x%x.\n",
 467                idx, edc_ecc_err_addr_reg,
 468                t4_read_reg(adap, edc_ecc_err_addr_reg));
 469        CH_WARN(adap,
 470                "bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n",
 471                rdata_reg,
 472                (unsigned long long)t4_read_reg64(adap, rdata_reg),
 473                (unsigned long long)t4_read_reg64(adap, rdata_reg + 8),
 474                (unsigned long long)t4_read_reg64(adap, rdata_reg + 16),
 475                (unsigned long long)t4_read_reg64(adap, rdata_reg + 24),
 476                (unsigned long long)t4_read_reg64(adap, rdata_reg + 32),
 477                (unsigned long long)t4_read_reg64(adap, rdata_reg + 40),
 478                (unsigned long long)t4_read_reg64(adap, rdata_reg + 48),
 479                (unsigned long long)t4_read_reg64(adap, rdata_reg + 56),
 480                (unsigned long long)t4_read_reg64(adap, rdata_reg + 64));
 481
 482        return 0;
 483}
 484
 485/**
 486 *      t4_memory_rw - read/write EDC 0, EDC 1 or MC via PCIE memory window
 487 *      @adap: the adapter
 488 *      @win: PCI-E Memory Window to use
 489 *      @mtype: memory type: MEM_EDC0, MEM_EDC1 or MEM_MC
 490 *      @addr: address within indicated memory type
 491 *      @len: amount of memory to transfer
 492 *      @hbuf: host memory buffer
 493 *      @dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
 494 *
 495 *      Reads/writes an [almost] arbitrary memory region in the firmware: the
 496 *      firmware memory address and host buffer must be aligned on 32-bit
 497 *      boudaries; the length may be arbitrary.  The memory is transferred as
 498 *      a raw byte sequence from/to the firmware's memory.  If this memory
 499 *      contains data structures which contain multi-byte integers, it's the
 500 *      caller's responsibility to perform appropriate byte order conversions.
 501 */
 502int t4_memory_rw(struct adapter *adap, int win, int mtype, u32 addr,
 503                 u32 len, void *hbuf, int dir)
 504{
 505        u32 pos, offset, resid, memoffset;
 506        u32 edc_size, mc_size, win_pf, mem_reg, mem_aperture, mem_base;
 507        u32 *buf;
 508
 509        /* Argument sanity checks ...
 510         */
 511        if (addr & 0x3 || (uintptr_t)hbuf & 0x3)
 512                return -EINVAL;
 513        buf = (u32 *)hbuf;
 514
 515        /* It's convenient to be able to handle lengths which aren't a
 516         * multiple of 32-bits because we often end up transferring files to
 517         * the firmware.  So we'll handle that by normalizing the length here
 518         * and then handling any residual transfer at the end.
 519         */
 520        resid = len & 0x3;
 521        len -= resid;
 522
 523        /* Offset into the region of memory which is being accessed
 524         * MEM_EDC0 = 0
 525         * MEM_EDC1 = 1
 526         * MEM_MC   = 2 -- MEM_MC for chips with only 1 memory controller
 527         * MEM_MC1  = 3 -- for chips with 2 memory controllers (e.g. T5)
 528         */
 529        edc_size  = EDRAM0_SIZE_G(t4_read_reg(adap, MA_EDRAM0_BAR_A));
 530        if (mtype != MEM_MC1)
 531                memoffset = (mtype * (edc_size * 1024 * 1024));
 532        else {
 533                mc_size = EXT_MEM0_SIZE_G(t4_read_reg(adap,
 534                                                      MA_EXT_MEMORY0_BAR_A));
 535                memoffset = (MEM_MC0 * edc_size + mc_size) * 1024 * 1024;
 536        }
 537
 538        /* Determine the PCIE_MEM_ACCESS_OFFSET */
 539        addr = addr + memoffset;
 540
 541        /* Each PCI-E Memory Window is programmed with a window size -- or
 542         * "aperture" -- which controls the granularity of its mapping onto
 543         * adapter memory.  We need to grab that aperture in order to know
 544         * how to use the specified window.  The window is also programmed
 545         * with the base address of the Memory Window in BAR0's address
 546         * space.  For T4 this is an absolute PCI-E Bus Address.  For T5
 547         * the address is relative to BAR0.
 548         */
 549        mem_reg = t4_read_reg(adap,
 550                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A,
 551                                                  win));
 552        mem_aperture = 1 << (WINDOW_G(mem_reg) + WINDOW_SHIFT_X);
 553        mem_base = PCIEOFST_G(mem_reg) << PCIEOFST_SHIFT_X;
 554        if (is_t4(adap->params.chip))
 555                mem_base -= adap->t4_bar0;
 556        win_pf = is_t4(adap->params.chip) ? 0 : PFNUM_V(adap->pf);
 557
 558        /* Calculate our initial PCI-E Memory Window Position and Offset into
 559         * that Window.
 560         */
 561        pos = addr & ~(mem_aperture-1);
 562        offset = addr - pos;
 563
 564        /* Set up initial PCI-E Memory Window to cover the start of our
 565         * transfer.  (Read it back to ensure that changes propagate before we
 566         * attempt to use the new value.)
 567         */
 568        t4_write_reg(adap,
 569                     PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, win),
 570                     pos | win_pf);
 571        t4_read_reg(adap,
 572                    PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, win));
 573
 574        /* Transfer data to/from the adapter as long as there's an integral
 575         * number of 32-bit transfers to complete.
 576         *
 577         * A note on Endianness issues:
 578         *
 579         * The "register" reads and writes below from/to the PCI-E Memory
 580         * Window invoke the standard adapter Big-Endian to PCI-E Link
 581         * Little-Endian "swizzel."  As a result, if we have the following
 582         * data in adapter memory:
 583         *
 584         *     Memory:  ... | b0 | b1 | b2 | b3 | ...
 585         *     Address:      i+0  i+1  i+2  i+3
 586         *
 587         * Then a read of the adapter memory via the PCI-E Memory Window
 588         * will yield:
 589         *
 590         *     x = readl(i)
 591         *         31                  0
 592         *         [ b3 | b2 | b1 | b0 ]
 593         *
 594         * If this value is stored into local memory on a Little-Endian system
 595         * it will show up correctly in local memory as:
 596         *
 597         *     ( ..., b0, b1, b2, b3, ... )
 598         *
 599         * But on a Big-Endian system, the store will show up in memory
 600         * incorrectly swizzled as:
 601         *
 602         *     ( ..., b3, b2, b1, b0, ... )
 603         *
 604         * So we need to account for this in the reads and writes to the
 605         * PCI-E Memory Window below by undoing the register read/write
 606         * swizzels.
 607         */
 608        while (len > 0) {
 609                if (dir == T4_MEMORY_READ)
 610                        *buf++ = le32_to_cpu((__force __le32)t4_read_reg(adap,
 611                                                mem_base + offset));
 612                else
 613                        t4_write_reg(adap, mem_base + offset,
 614                                     (__force u32)cpu_to_le32(*buf++));
 615                offset += sizeof(__be32);
 616                len -= sizeof(__be32);
 617
 618                /* If we've reached the end of our current window aperture,
 619                 * move the PCI-E Memory Window on to the next.  Note that
 620                 * doing this here after "len" may be 0 allows us to set up
 621                 * the PCI-E Memory Window for a possible final residual
 622                 * transfer below ...
 623                 */
 624                if (offset == mem_aperture) {
 625                        pos += mem_aperture;
 626                        offset = 0;
 627                        t4_write_reg(adap,
 628                                PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A,
 629                                                    win), pos | win_pf);
 630                        t4_read_reg(adap,
 631                                PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A,
 632                                                    win));
 633                }
 634        }
 635
 636        /* If the original transfer had a length which wasn't a multiple of
 637         * 32-bits, now's where we need to finish off the transfer of the
 638         * residual amount.  The PCI-E Memory Window has already been moved
 639         * above (if necessary) to cover this final transfer.
 640         */
 641        if (resid) {
 642                union {
 643                        u32 word;
 644                        char byte[4];
 645                } last;
 646                unsigned char *bp;
 647                int i;
 648
 649                if (dir == T4_MEMORY_READ) {
 650                        last.word = le32_to_cpu(
 651                                        (__force __le32)t4_read_reg(adap,
 652                                                mem_base + offset));
 653                        for (bp = (unsigned char *)buf, i = resid; i < 4; i++)
 654                                bp[i] = last.byte[i];
 655                } else {
 656                        last.word = *buf;
 657                        for (i = resid; i < 4; i++)
 658                                last.byte[i] = 0;
 659                        t4_write_reg(adap, mem_base + offset,
 660                                     (__force u32)cpu_to_le32(last.word));
 661                }
 662        }
 663
 664        return 0;
 665}
 666
 667/* Return the specified PCI-E Configuration Space register from our Physical
 668 * Function.  We try first via a Firmware LDST Command since we prefer to let
 669 * the firmware own all of these registers, but if that fails we go for it
 670 * directly ourselves.
 671 */
 672u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
 673{
 674        u32 val, ldst_addrspace;
 675
 676        /* If fw_attach != 0, construct and send the Firmware LDST Command to
 677         * retrieve the specified PCI-E Configuration Space register.
 678         */
 679        struct fw_ldst_cmd ldst_cmd;
 680        int ret;
 681
 682        memset(&ldst_cmd, 0, sizeof(ldst_cmd));
 683        ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE);
 684        ldst_cmd.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
 685                                               FW_CMD_REQUEST_F |
 686                                               FW_CMD_READ_F |
 687                                               ldst_addrspace);
 688        ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd));
 689        ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
 690        ldst_cmd.u.pcie.ctrl_to_fn =
 691                (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->pf));
 692        ldst_cmd.u.pcie.r = reg;
 693
 694        /* If the LDST Command succeeds, return the result, otherwise
 695         * fall through to reading it directly ourselves ...
 696         */
 697        ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
 698                         &ldst_cmd);
 699        if (ret == 0)
 700                val = be32_to_cpu(ldst_cmd.u.pcie.data[0]);
 701        else
 702                /* Read the desired Configuration Space register via the PCI-E
 703                 * Backdoor mechanism.
 704                 */
 705                t4_hw_pci_read_cfg4(adap, reg, &val);
 706        return val;
 707}
 708
 709/* Get the window based on base passed to it.
 710 * Window aperture is currently unhandled, but there is no use case for it
 711 * right now
 712 */
 713static u32 t4_get_window(struct adapter *adap, u32 pci_base, u64 pci_mask,
 714                         u32 memwin_base)
 715{
 716        u32 ret;
 717
 718        if (is_t4(adap->params.chip)) {
 719                u32 bar0;
 720
 721                /* Truncation intentional: we only read the bottom 32-bits of
 722                 * the 64-bit BAR0/BAR1 ...  We use the hardware backdoor
 723                 * mechanism to read BAR0 instead of using
 724                 * pci_resource_start() because we could be operating from
 725                 * within a Virtual Machine which is trapping our accesses to
 726                 * our Configuration Space and we need to set up the PCI-E
 727                 * Memory Window decoders with the actual addresses which will
 728                 * be coming across the PCI-E link.
 729                 */
 730                bar0 = t4_read_pcie_cfg4(adap, pci_base);
 731                bar0 &= pci_mask;
 732                adap->t4_bar0 = bar0;
 733
 734                ret = bar0 + memwin_base;
 735        } else {
 736                /* For T5, only relative offset inside the PCIe BAR is passed */
 737                ret = memwin_base;
 738        }
 739        return ret;
 740}
 741
 742/* Get the default utility window (win0) used by everyone */
 743u32 t4_get_util_window(struct adapter *adap)
 744{
 745        return t4_get_window(adap, PCI_BASE_ADDRESS_0,
 746                             PCI_BASE_ADDRESS_MEM_MASK, MEMWIN0_BASE);
 747}
 748
 749/* Set up memory window for accessing adapter memory ranges.  (Read
 750 * back MA register to ensure that changes propagate before we attempt
 751 * to use the new values.)
 752 */
 753void t4_setup_memwin(struct adapter *adap, u32 memwin_base, u32 window)
 754{
 755        t4_write_reg(adap,
 756                     PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, window),
 757                     memwin_base | BIR_V(0) |
 758                     WINDOW_V(ilog2(MEMWIN0_APERTURE) - WINDOW_SHIFT_X));
 759        t4_read_reg(adap,
 760                    PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, window));
 761}
 762
 763/**
 764 *      t4_get_regs_len - return the size of the chips register set
 765 *      @adapter: the adapter
 766 *
 767 *      Returns the size of the chip's BAR0 register space.
 768 */
 769unsigned int t4_get_regs_len(struct adapter *adapter)
 770{
 771        unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);
 772
 773        switch (chip_version) {
 774        case CHELSIO_T4:
 775                return T4_REGMAP_SIZE;
 776
 777        case CHELSIO_T5:
 778        case CHELSIO_T6:
 779                return T5_REGMAP_SIZE;
 780        }
 781
 782        dev_err(adapter->pdev_dev,
 783                "Unsupported chip version %d\n", chip_version);
 784        return 0;
 785}
 786
 787/**
 788 *      t4_get_regs - read chip registers into provided buffer
 789 *      @adap: the adapter
 790 *      @buf: register buffer
 791 *      @buf_size: size (in bytes) of register buffer
 792 *
 793 *      If the provided register buffer isn't large enough for the chip's
 794 *      full register range, the register dump will be truncated to the
 795 *      register buffer's size.
 796 */
 797void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 798{
 799        static const unsigned int t4_reg_ranges[] = {
 800                0x1008, 0x1108,
 801                0x1180, 0x1184,
 802                0x1190, 0x1194,
 803                0x11a0, 0x11a4,
 804                0x11b0, 0x11b4,
 805                0x11fc, 0x123c,
 806                0x1300, 0x173c,
 807                0x1800, 0x18fc,
 808                0x3000, 0x30d8,
 809                0x30e0, 0x30e4,
 810                0x30ec, 0x5910,
 811                0x5920, 0x5924,
 812                0x5960, 0x5960,
 813                0x5968, 0x5968,
 814                0x5970, 0x5970,
 815                0x5978, 0x5978,
 816                0x5980, 0x5980,
 817                0x5988, 0x5988,
 818                0x5990, 0x5990,
 819                0x5998, 0x5998,
 820                0x59a0, 0x59d4,
 821                0x5a00, 0x5ae0,
 822                0x5ae8, 0x5ae8,
 823                0x5af0, 0x5af0,
 824                0x5af8, 0x5af8,
 825                0x6000, 0x6098,
 826                0x6100, 0x6150,
 827                0x6200, 0x6208,
 828                0x6240, 0x6248,
 829                0x6280, 0x62b0,
 830                0x62c0, 0x6338,
 831                0x6370, 0x638c,
 832                0x6400, 0x643c,
 833                0x6500, 0x6524,
 834                0x6a00, 0x6a04,
 835                0x6a14, 0x6a38,
 836                0x6a60, 0x6a70,
 837                0x6a78, 0x6a78,
 838                0x6b00, 0x6b0c,
 839                0x6b1c, 0x6b84,
 840                0x6bf0, 0x6bf8,
 841                0x6c00, 0x6c0c,
 842                0x6c1c, 0x6c84,
 843                0x6cf0, 0x6cf8,
 844                0x6d00, 0x6d0c,
 845                0x6d1c, 0x6d84,
 846                0x6df0, 0x6df8,
 847                0x6e00, 0x6e0c,
 848                0x6e1c, 0x6e84,
 849                0x6ef0, 0x6ef8,
 850                0x6f00, 0x6f0c,
 851                0x6f1c, 0x6f84,
 852                0x6ff0, 0x6ff8,
 853                0x7000, 0x700c,
 854                0x701c, 0x7084,
 855                0x70f0, 0x70f8,
 856                0x7100, 0x710c,
 857                0x711c, 0x7184,
 858                0x71f0, 0x71f8,
 859                0x7200, 0x720c,
 860                0x721c, 0x7284,
 861                0x72f0, 0x72f8,
 862                0x7300, 0x730c,
 863                0x731c, 0x7384,
 864                0x73f0, 0x73f8,
 865                0x7400, 0x7450,
 866                0x7500, 0x7530,
 867                0x7600, 0x760c,
 868                0x7614, 0x761c,
 869                0x7680, 0x76cc,
 870                0x7700, 0x7798,
 871                0x77c0, 0x77fc,
 872                0x7900, 0x79fc,
 873                0x7b00, 0x7b58,
 874                0x7b60, 0x7b84,
 875                0x7b8c, 0x7c38,
 876                0x7d00, 0x7d38,
 877                0x7d40, 0x7d80,
 878                0x7d8c, 0x7ddc,
 879                0x7de4, 0x7e04,
 880                0x7e10, 0x7e1c,
 881                0x7e24, 0x7e38,
 882                0x7e40, 0x7e44,
 883                0x7e4c, 0x7e78,
 884                0x7e80, 0x7ea4,
 885                0x7eac, 0x7edc,
 886                0x7ee8, 0x7efc,
 887                0x8dc0, 0x8e04,
 888                0x8e10, 0x8e1c,
 889                0x8e30, 0x8e78,
 890                0x8ea0, 0x8eb8,
 891                0x8ec0, 0x8f6c,
 892                0x8fc0, 0x9008,
 893                0x9010, 0x9058,
 894                0x9060, 0x9060,
 895                0x9068, 0x9074,
 896                0x90fc, 0x90fc,
 897                0x9400, 0x9408,
 898                0x9410, 0x9458,
 899                0x9600, 0x9600,
 900                0x9608, 0x9638,
 901                0x9640, 0x96bc,
 902                0x9800, 0x9808,
 903                0x9820, 0x983c,
 904                0x9850, 0x9864,
 905                0x9c00, 0x9c6c,
 906                0x9c80, 0x9cec,
 907                0x9d00, 0x9d6c,
 908                0x9d80, 0x9dec,
 909                0x9e00, 0x9e6c,
 910                0x9e80, 0x9eec,
 911                0x9f00, 0x9f6c,
 912                0x9f80, 0x9fec,
 913                0xd004, 0xd004,
 914                0xd010, 0xd03c,
 915                0xdfc0, 0xdfe0,
 916                0xe000, 0xea7c,
 917                0xf000, 0x11110,
 918                0x11118, 0x11190,
 919                0x19040, 0x1906c,
 920                0x19078, 0x19080,
 921                0x1908c, 0x190e4,
 922                0x190f0, 0x190f8,
 923                0x19100, 0x19110,
 924                0x19120, 0x19124,
 925                0x19150, 0x19194,
 926                0x1919c, 0x191b0,
 927                0x191d0, 0x191e8,
 928                0x19238, 0x1924c,
 929                0x193f8, 0x1943c,
 930                0x1944c, 0x19474,
 931                0x19490, 0x194e0,
 932                0x194f0, 0x194f8,
 933                0x19800, 0x19c08,
 934                0x19c10, 0x19c90,
 935                0x19ca0, 0x19ce4,
 936                0x19cf0, 0x19d40,
 937                0x19d50, 0x19d94,
 938                0x19da0, 0x19de8,
 939                0x19df0, 0x19e40,
 940                0x19e50, 0x19e90,
 941                0x19ea0, 0x19f4c,
 942                0x1a000, 0x1a004,
 943                0x1a010, 0x1a06c,
 944                0x1a0b0, 0x1a0e4,
 945                0x1a0ec, 0x1a0f4,
 946                0x1a100, 0x1a108,
 947                0x1a114, 0x1a120,
 948                0x1a128, 0x1a130,
 949                0x1a138, 0x1a138,
 950                0x1a190, 0x1a1c4,
 951                0x1a1fc, 0x1a1fc,
 952                0x1e040, 0x1e04c,
 953                0x1e284, 0x1e28c,
 954                0x1e2c0, 0x1e2c0,
 955                0x1e2e0, 0x1e2e0,
 956                0x1e300, 0x1e384,
 957                0x1e3c0, 0x1e3c8,
 958                0x1e440, 0x1e44c,
 959                0x1e684, 0x1e68c,
 960                0x1e6c0, 0x1e6c0,
 961                0x1e6e0, 0x1e6e0,
 962                0x1e700, 0x1e784,
 963                0x1e7c0, 0x1e7c8,
 964                0x1e840, 0x1e84c,
 965                0x1ea84, 0x1ea8c,
 966                0x1eac0, 0x1eac0,
 967                0x1eae0, 0x1eae0,
 968                0x1eb00, 0x1eb84,
 969                0x1ebc0, 0x1ebc8,
 970                0x1ec40, 0x1ec4c,
 971                0x1ee84, 0x1ee8c,
 972                0x1eec0, 0x1eec0,
 973                0x1eee0, 0x1eee0,
 974                0x1ef00, 0x1ef84,
 975                0x1efc0, 0x1efc8,
 976                0x1f040, 0x1f04c,
 977                0x1f284, 0x1f28c,
 978                0x1f2c0, 0x1f2c0,
 979                0x1f2e0, 0x1f2e0,
 980                0x1f300, 0x1f384,
 981                0x1f3c0, 0x1f3c8,
 982                0x1f440, 0x1f44c,
 983                0x1f684, 0x1f68c,
 984                0x1f6c0, 0x1f6c0,
 985                0x1f6e0, 0x1f6e0,
 986                0x1f700, 0x1f784,
 987                0x1f7c0, 0x1f7c8,
 988                0x1f840, 0x1f84c,
 989                0x1fa84, 0x1fa8c,
 990                0x1fac0, 0x1fac0,
 991                0x1fae0, 0x1fae0,
 992                0x1fb00, 0x1fb84,
 993                0x1fbc0, 0x1fbc8,
 994                0x1fc40, 0x1fc4c,
 995                0x1fe84, 0x1fe8c,
 996                0x1fec0, 0x1fec0,
 997                0x1fee0, 0x1fee0,
 998                0x1ff00, 0x1ff84,
 999                0x1ffc0, 0x1ffc8,
1000                0x20000, 0x2002c,

1001                0x20100, 0x2013c,
1002                0x20190, 0x201a0,
1003                0x201a8, 0x201b8,
1004                0x201c4, 0x201c8,
1005                0x20200, 0x20318,
1006                0x20400, 0x204b4,
1007                0x204c0, 0x20528,
1008                0x20540, 0x20614,
1009                0x21000, 0x21040,
1010                0x2104c, 0x21060,
1011                0x210c0, 0x210ec,
1012                0x21200, 0x21268,
1013                0x21270, 0x21284,
1014                0x212fc, 0x21388,
1015                0x21400, 0x21404,
1016                0x21500, 0x21500,
1017                0x21510, 0x21518,
1018                0x2152c, 0x21530,
1019                0x2153c, 0x2153c,
1020                0x21550, 0x21554,
1021                0x21600, 0x21600,
1022                0x21608, 0x2161c,
1023                0x21624, 0x21628,
1024                0x21630, 0x21634,
1025                0x2163c, 0x2163c,
1026                0x21700, 0x2171c,
1027                0x21780, 0x2178c,
1028                0x21800, 0x21818,
1029                0x21820, 0x21828,
1030                0x21830, 0x21848,
1031                0x21850, 0x21854,
1032                0x21860, 0x21868,
1033                0x21870, 0x21870,
1034                0x21878, 0x21898,
1035                0x218a0, 0x218a8,
1036                0x218b0, 0x218c8,
1037                0x218d0, 0x218d4,
1038                0x218e0, 0x218e8,
1039                0x218f0, 0x218f0,
1040                0x218f8, 0x21a18,
1041                0x21a20, 0x21a28,
1042                0x21a30, 0x21a48,
1043                0x21a50, 0x21a54,
1044                0x21a60, 0x21a68,
1045                0x21a70, 0x21a70,
1046                0x21a78, 0x21a98,
1047                0x21aa0, 0x21aa8,
1048                0x21ab0, 0x21ac8,
1049                0x21ad0, 0x21ad4,
1050                0x21ae0, 0x21ae8,
1051                0x21af0, 0x21af0,
1052                0x21af8, 0x21c18,
1053                0x21c20, 0x21c20,
1054                0x21c28, 0x21c30,
1055                0x21c38, 0x21c38,
1056                0x21c80, 0x21c98,
1057                0x21ca0, 0x21ca8,
1058                0x21cb0, 0x21cc8,
1059                0x21cd0, 0x21cd4,
1060                0x21ce0, 0x21ce8,
1061                0x21cf0, 0x21cf0,
1062                0x21cf8, 0x21d7c,
1063                0x21e00, 0x21e04,
1064                0x22000, 0x2202c,
1065                0x22100, 0x2213c,
1066                0x22190, 0x221a0,
1067                0x221a8, 0x221b8,
1068                0x221c4, 0x221c8,
1069                0x22200, 0x22318,
1070                0x22400, 0x224b4,
1071                0x224c0, 0x22528,
1072                0x22540, 0x22614,
1073                0x23000, 0x23040,
1074                0x2304c, 0x23060,
1075                0x230c0, 0x230ec,
1076                0x23200, 0x23268,
1077                0x23270, 0x23284,
1078                0x232fc, 0x23388,
1079                0x23400, 0x23404,
1080                0x23500, 0x23500,
1081                0x23510, 0x23518,
1082                0x2352c, 0x23530,
1083                0x2353c, 0x2353c,
1084                0x23550, 0x23554,
1085                0x23600, 0x23600,
1086                0x23608, 0x2361c,
1087                0x23624, 0x23628,
1088                0x23630, 0x23634,
1089                0x2363c, 0x2363c,
1090                0x23700, 0x2371c,
1091                0x23780, 0x2378c,
1092                0x23800, 0x23818,
1093                0x23820, 0x23828,
1094                0x23830, 0x23848,
1095                0x23850, 0x23854,
1096                0x23860, 0x23868,
1097                0x23870, 0x23870,
1098                0x23878, 0x23898,
1099                0x238a0, 0x238a8,
1100                0x238b0, 0x238c8,
1101                0x238d0, 0x238d4,
1102                0x238e0, 0x238e8,
1103                0x238f0, 0x238f0,
1104                0x238f8, 0x23a18,
1105                0x23a20, 0x23a28,
1106                0x23a30, 0x23a48,
1107                0x23a50, 0x23a54,
1108                0x23a60, 0x23a68,
1109                0x23a70, 0x23a70,
1110                0x23a78, 0x23a98,
1111                0x23aa0, 0x23aa8,
1112                0x23ab0, 0x23ac8,
1113                0x23ad0, 0x23ad4,
1114                0x23ae0, 0x23ae8,
1115                0x23af0, 0x23af0,
1116                0x23af8, 0x23c18,
1117                0x23c20, 0x23c20,
1118                0x23c28, 0x23c30,
1119                0x23c38, 0x23c38,
1120                0x23c80, 0x23c98,
1121                0x23ca0, 0x23ca8,
1122                0x23cb0, 0x23cc8,
1123                0x23cd0, 0x23cd4,
1124                0x23ce0, 0x23ce8,
1125                0x23cf0, 0x23cf0,
1126                0x23cf8, 0x23d7c,
1127                0x23e00, 0x23e04,
1128                0x24000, 0x2402c,
1129                0x24100, 0x2413c,
1130                0x24190, 0x241a0,
1131                0x241a8, 0x241b8,
1132                0x241c4, 0x241c8,
1133                0x24200, 0x24318,
1134                0x24400, 0x244b4,
1135                0x244c0, 0x24528,
1136                0x24540, 0x24614,
1137                0x25000, 0x25040,
1138                0x2504c, 0x25060,
1139                0x250c0, 0x250ec,
1140                0x25200, 0x25268,
1141                0x25270, 0x25284,
1142                0x252fc, 0x25388,
1143                0x25400, 0x25404,
1144                0x25500, 0x25500,
1145                0x25510, 0x25518,
1146                0x2552c, 0x25530,
1147                0x2553c, 0x2553c,
1148                0x25550, 0x25554,
1149                0x25600, 0x25600,
1150                0x25608, 0x2561c,
1151                0x25624, 0x25628,
1152                0x25630, 0x25634,
1153                0x2563c, 0x2563c,
1154                0x25700, 0x2571c,
1155                0x25780, 0x2578c,
1156                0x25800, 0x25818,
1157                0x25820, 0x25828,
1158                0x25830, 0x25848,
1159                0x25850, 0x25854,
1160                0x25860, 0x25868,
1161                0x25870, 0x25870,
1162                0x25878, 0x25898,
1163                0x258a0, 0x258a8,
1164                0x258b0, 0x258c8,
1165                0x258d0, 0x258d4,
1166                0x258e0, 0x258e8,
1167                0x258f0, 0x258f0,
1168                0x258f8, 0x25a18,
1169                0x25a20, 0x25a28,
1170                0x25a30, 0x25a48,
1171                0x25a50, 0x25a54,
1172                0x25a60, 0x25a68,
1173                0x25a70, 0x25a70,
1174                0x25a78, 0x25a98,
1175                0x25aa0, 0x25aa8,
1176                0x25ab0, 0x25ac8,
1177                0x25ad0, 0x25ad4,
1178                0x25ae0, 0x25ae8,
1179                0x25af0, 0x25af0,
1180                0x25af8, 0x25c18,
1181                0x25c20, 0x25c20,
1182                0x25c28, 0x25c30,
1183                0x25c38, 0x25c38,
1184                0x25c80, 0x25c98,
1185                0x25ca0, 0x25ca8,
1186                0x25cb0, 0x25cc8,
1187                0x25cd0, 0x25cd4,
1188                0x25ce0, 0x25ce8,
1189                0x25cf0, 0x25cf0,
1190                0x25cf8, 0x25d7c,
1191                0x25e00, 0x25e04,
1192                0x26000, 0x2602c,
1193                0x26100, 0x2613c,
1194                0x26190, 0x261a0,
1195                0x261a8, 0x261b8,
1196                0x261c4, 0x261c8,
1197                0x26200, 0x26318,
1198                0x26400, 0x264b4,
1199                0x264c0, 0x26528,
1200                0x26540, 0x26614,
1201                0x27000, 0x27040,
1202                0x2704c, 0x27060,
1203                0x270c0, 0x270ec,
1204                0x27200, 0x27268,
1205                0x27270, 0x27284,
1206                0x272fc, 0x27388,
1207                0x27400, 0x27404,
1208                0x27500, 0x27500,
1209                0x27510, 0x27518,
1210                0x2752c, 0x27530,
1211                0x2753c, 0x2753c,
1212                0x27550, 0x27554,
1213                0x27600, 0x27600,
1214                0x27608, 0x2761c,
1215                0x27624, 0x27628,
1216                0x27630, 0x27634,
1217                0x2763c, 0x2763c,
1218                0x27700, 0x2771c,
1219                0x27780, 0x2778c,
1220                0x27800, 0x27818,
1221                0x27820, 0x27828,
1222                0x27830, 0x27848,
1223                0x27850, 0x27854,
1224                0x27860, 0x27868,
1225                0x27870, 0x27870,
1226                0x27878, 0x27898,
1227                0x278a0, 0x278a8,
1228                0x278b0, 0x278c8,
1229                0x278d0, 0x278d4,
1230                0x278e0, 0x278e8,
1231                0x278f0, 0x278f0,
1232                0x278f8, 0x27a18,
1233                0x27a20, 0x27a28,
1234                0x27a30, 0x27a48,
1235                0x27a50, 0x27a54,
1236                0x27a60, 0x27a68,
1237                0x27a70, 0x27a70,
1238                0x27a78, 0x27a98,
1239                0x27aa0, 0x27aa8,
1240                0x27ab0, 0x27ac8,
1241                0x27ad0, 0x27ad4,
1242                0x27ae0, 0x27ae8,
1243                0x27af0, 0x27af0,
1244                0x27af8, 0x27c18,
1245                0x27c20, 0x27c20,
1246                0x27c28, 0x27c30,
1247                0x27c38, 0x27c38,
1248                0x27c80, 0x27c98,
1249                0x27ca0, 0x27ca8,
1250                0x27cb0, 0x27cc8,
1251                0x27cd0, 0x27cd4,
1252                0x27ce0, 0x27ce8,
1253                0x27cf0, 0x27cf0,
1254                0x27cf8, 0x27d7c,
1255                0x27e00, 0x27e04,
1256        };
1257
1258        static const unsigned int t5_reg_ranges[] = {
1259                0x1008, 0x10c0,
1260                0x10cc, 0x10f8,
1261                0x1100, 0x1100,
1262                0x110c, 0x1148,
1263                0x1180, 0x1184,
1264                0x1190, 0x1194,
1265                0x11a0, 0x11a4,
1266                0x11b0, 0x11b4,
1267                0x11fc, 0x123c,
1268                0x1280, 0x173c,
1269                0x1800, 0x18fc,
1270                0x3000, 0x3028,
1271                0x3060, 0x30b0,
1272                0x30b8, 0x30d8,
1273                0x30e0, 0x30fc,
1274                0x3140, 0x357c,
1275                0x35a8, 0x35cc,
1276                0x35ec, 0x35ec,
1277                0x3600, 0x5624,
1278                0x56cc, 0x56ec,
1279                0x56f4, 0x5720,
1280                0x5728, 0x575c,
1281                0x580c, 0x5814,
1282                0x5890, 0x589c,
1283                0x58a4, 0x58ac,
1284                0x58b8, 0x58bc,
1285                0x5940, 0x59c8,
1286                0x59d0, 0x59dc,
1287                0x59fc, 0x5a18,
1288                0x5a60, 0x5a70,
1289                0x5a80, 0x5a9c,
1290                0x5b94, 0x5bfc,
1291                0x6000, 0x6020,
1292                0x6028, 0x6040,
1293                0x6058, 0x609c,
1294                0x60a8, 0x614c,
1295                0x7700, 0x7798,
1296                0x77c0, 0x78fc,
1297                0x7b00, 0x7b58,
1298                0x7b60, 0x7b84,
1299                0x7b8c, 0x7c54,
1300                0x7d00, 0x7d38,
1301                0x7d40, 0x7d80,
1302                0x7d8c, 0x7ddc,
1303                0x7de4, 0x7e04,
1304                0x7e10, 0x7e1c,
1305                0x7e24, 0x7e38,
1306                0x7e40, 0x7e44,
1307                0x7e4c, 0x7e78,
1308                0x7e80, 0x7edc,
1309                0x7ee8, 0x7efc,
1310                0x8dc0, 0x8de0,
1311                0x8df8, 0x8e04,
1312                0x8e10, 0x8e84,
1313                0x8ea0, 0x8f84,
1314                0x8fc0, 0x9058,
1315                0x9060, 0x9060,
1316                0x9068, 0x90f8,
1317                0x9400, 0x9408,
1318                0x9410, 0x9470,
1319                0x9600, 0x9600,
1320                0x9608, 0x9638,
1321                0x9640, 0x96f4,
1322                0x9800, 0x9808,
1323                0x9820, 0x983c,
1324                0x9850, 0x9864,
1325                0x9c00, 0x9c6c,
1326                0x9c80, 0x9cec,
1327                0x9d00, 0x9d6c,
1328                0x9d80, 0x9dec,
1329                0x9e00, 0x9e6c,
1330                0x9e80, 0x9eec,
1331                0x9f00, 0x9f6c,
1332                0x9f80, 0xa020,
1333                0xd004, 0xd004,
1334                0xd010, 0xd03c,
1335                0xdfc0, 0xdfe0,
1336                0xe000, 0x1106c,
1337                0x11074, 0x11088,
1338                0x1109c, 0x1117c,
1339                0x11190, 0x11204,
1340                0x19040, 0x1906c,
1341                0x19078, 0x19080,
1342                0x1908c, 0x190e8,
1343                0x190f0, 0x190f8,
1344                0x19100, 0x19110,
1345                0x19120, 0x19124,
1346                0x19150, 0x19194,
1347                0x1919c, 0x191b0,
1348                0x191d0, 0x191e8,
1349                0x19238, 0x19290,
1350                0x193f8, 0x19428,
1351                0x19430, 0x19444,
1352                0x1944c, 0x1946c,
1353                0x19474, 0x19474,
1354                0x19490, 0x194cc,
1355                0x194f0, 0x194f8,
1356                0x19c00, 0x19c08,
1357                0x19c10, 0x19c60,
1358                0x19c94, 0x19ce4,
1359                0x19cf0, 0x19d40,
1360                0x19d50, 0x19d94,
1361                0x19da0, 0x19de8,
1362                0x19df0, 0x19e10,
1363                0x19e50, 0x19e90,
1364                0x19ea0, 0x19f24,
1365                0x19f34, 0x19f34,
1366                0x19f40, 0x19f50,
1367                0x19f90, 0x19fb4,
1368                0x19fc4, 0x19fe4,
1369                0x1a000, 0x1a004,
1370                0x1a010, 0x1a06c,
1371                0x1a0b0, 0x1a0e4,
1372                0x1a0ec, 0x1a0f8,
1373                0x1a100, 0x1a108,
1374                0x1a114, 0x1a120,
1375                0x1a128, 0x1a130,
1376                0x1a138, 0x1a138,
1377                0x1a190, 0x1a1c4,
1378                0x1a1fc, 0x1a1fc,
1379                0x1e008, 0x1e00c,
1380                0x1e040, 0x1e044,
1381                0x1e04c, 0x1e04c,
1382                0x1e284, 0x1e290,
1383                0x1e2c0, 0x1e2c0,
1384                0x1e2e0, 0x1e2e0,
1385                0x1e300, 0x1e384,
1386                0x1e3c0, 0x1e3c8,
1387                0x1e408, 0x1e40c,
1388                0x1e440, 0x1e444,
1389                0x1e44c, 0x1e44c,
1390                0x1e684, 0x1e690,
1391                0x1e6c0, 0x1e6c0,
1392                0x1e6e0, 0x1e6e0,
1393                0x1e700, 0x1e784,
1394                0x1e7c0, 0x1e7c8,
1395                0x1e808, 0x1e80c,
1396                0x1e840, 0x1e844,
1397                0x1e84c, 0x1e84c,
1398                0x1ea84, 0x1ea90,
1399                0x1eac0, 0x1eac0,
1400                0x1eae0, 0x1eae0,
1401                0x1eb00, 0x1eb84,
1402                0x1ebc0, 0x1ebc8,
1403                0x1ec08, 0x1ec0c,
1404                0x1ec40, 0x1ec44,
1405                0x1ec4c, 0x1ec4c,
1406                0x1ee84, 0x1ee90,
1407                0x1eec0, 0x1eec0,
1408                0x1eee0, 0x1eee0,
1409                0x1ef00, 0x1ef84,
1410                0x1efc0, 0x1efc8,
1411                0x1f008, 0x1f00c,
1412                0x1f040, 0x1f044,
1413                0x1f04c, 0x1f04c,
1414                0x1f284, 0x1f290,
1415                0x1f2c0, 0x1f2c0,
1416                0x1f2e0, 0x1f2e0,
1417                0x1f300, 0x1f384,
1418                0x1f3c0, 0x1f3c8,
1419                0x1f408, 0x1f40c,
1420                0x1f440, 0x1f444,
1421                0x1f44c, 0x1f44c,
1422                0x1f684, 0x1f690,
1423                0x1f6c0, 0x1f6c0,
1424                0x1f6e0, 0x1f6e0,
1425                0x1f700, 0x1f784,
1426                0x1f7c0, 0x1f7c8,
1427                0x1f808, 0x1f80c,
1428                0x1f840, 0x1f844,
1429                0x1f84c, 0x1f84c,
1430                0x1fa84, 0x1fa90,
1431                0x1fac0, 0x1fac0,
1432                0x1fae0, 0x1fae0,
1433                0x1fb00, 0x1fb84,
1434                0x1fbc0, 0x1fbc8,
1435                0x1fc08, 0x1fc0c,
1436                0x1fc40, 0x1fc44,
1437                0x1fc4c, 0x1fc4c,
1438                0x1fe84, 0x1fe90,
1439                0x1fec0, 0x1fec0,
1440                0x1fee0, 0x1fee0,
1441                0x1ff00, 0x1ff84,
1442                0x1ffc0, 0x1ffc8,
1443                0x30000, 0x30030,
1444                0x30100, 0x30144,
1445                0x30190, 0x301a0,
1446                0x301a8, 0x301b8,
1447                0x301c4, 0x301c8,
1448                0x301d0, 0x301d0,
1449                0x30200, 0x30318,
1450                0x30400, 0x304b4,
1451                0x304c0, 0x3052c,
1452                0x30540, 0x3061c,
1453                0x30800, 0x30828,
1454                0x30834, 0x30834,
1455                0x308c0, 0x30908,
1456                0x30910, 0x309ac,
1457                0x30a00, 0x30a14,
1458                0x30a1c, 0x30a2c,
1459                0x30a44, 0x30a50,
1460                0x30a74, 0x30a74,
1461                0x30a7c, 0x30afc,
1462                0x30b08, 0x30c24,
1463                0x30d00, 0x30d00,
1464                0x30d08, 0x30d14,
1465                0x30d1c, 0x30d20,
1466                0x30d3c, 0x30d3c,
1467                0x30d48, 0x30d50,
1468                0x31200, 0x3120c,
1469                0x31220, 0x31220,
1470                0x31240, 0x31240,
1471                0x31600, 0x3160c,
1472                0x31a00, 0x31a1c,
1473                0x31e00, 0x31e20,
1474                0x31e38, 0x31e3c,
1475                0x31e80, 0x31e80,
1476                0x31e88, 0x31ea8,
1477                0x31eb0, 0x31eb4,
1478                0x31ec8, 0x31ed4,
1479                0x31fb8, 0x32004,
1480                0x32200, 0x32200,
1481                0x32208, 0x32240,
1482                0x32248, 0x32280,
1483                0x32288, 0x322c0,
1484                0x322c8, 0x322fc,
1485                0x32600, 0x32630,
1486                0x32a00, 0x32abc,
1487                0x32b00, 0x32b10,
1488                0x32b20, 0x32b30,
1489                0x32b40, 0x32b50,
1490                0x32b60, 0x32b70,
1491                0x33000, 0x33028,
1492                0x33030, 0x33048,
1493                0x33060, 0x33068,
1494                0x33070, 0x3309c,
1495                0x330f0, 0x33128,
1496                0x33130, 0x33148,
1497                0x33160, 0x33168,
1498                0x33170, 0x3319c,
1499                0x331f0, 0x33238,
1500                0x33240, 0x33240,
1501                0x33248, 0x33250,
1502                0x3325c, 0x33264,
1503                0x33270, 0x332b8,
1504                0x332c0, 0x332e4,
1505                0x332f8, 0x33338,
1506                0x33340, 0x33340,
1507                0x33348, 0x33350,
1508                0x3335c, 0x33364,
1509                0x33370, 0x333b8,
1510                0x333c0, 0x333e4,
1511                0x333f8, 0x33428,
1512                0x33430, 0x33448,
1513                0x33460, 0x33468,
1514                0x33470, 0x3349c,
1515                0x334f0, 0x33528,
1516                0x33530, 0x33548,
1517                0x33560, 0x33568,
1518                0x33570, 0x3359c,
1519                0x335f0, 0x33638,
1520                0x33640, 0x33640,
1521                0x33648, 0x33650,
1522                0x3365c, 0x33664,
1523                0x33670, 0x336b8,
1524                0x336c0, 0x336e4,
1525                0x336f8, 0x33738,
1526                0x33740, 0x33740,
1527                0x33748, 0x33750,
1528                0x3375c, 0x33764,
1529                0x33770, 0x337b8,
1530                0x337c0, 0x337e4,
1531                0x337f8, 0x337fc,
1532                0x33814, 0x33814,
1533                0x3382c, 0x3382c,
1534                0x33880, 0x3388c,
1535                0x338e8, 0x338ec,
1536                0x33900, 0x33928,
1537                0x33930, 0x33948,
1538                0x33960, 0x33968,
1539                0x33970, 0x3399c,
1540                0x339f0, 0x33a38,
1541                0x33a40, 0x33a40,
1542                0x33a48, 0x33a50,
1543                0x33a5c, 0x33a64,
1544                0x33a70, 0x33ab8,
1545                0x33ac0, 0x33ae4,
1546                0x33af8, 0x33b10,
1547                0x33b28, 0x33b28,
1548                0x33b3c, 0x33b50,
1549                0x33bf0, 0x33c10,
1550                0x33c28, 0x33c28,
1551                0x33c3c, 0x33c50,
1552                0x33cf0, 0x33cfc,
1553                0x34000, 0x34030,
1554                0x34100, 0x34144,
1555                0x34190, 0x341a0,
1556                0x341a8, 0x341b8,
1557                0x341c4, 0x341c8,
1558                0x341d0, 0x341d0,
1559                0x34200, 0x34318,
1560                0x34400, 0x344b4,
1561                0x344c0, 0x3452c,
1562                0x34540, 0x3461c,
1563                0x34800, 0x34828,
1564                0x34834, 0x34834,
1565                0x348c0, 0x34908,
1566                0x34910, 0x349ac,
1567                0x34a00, 0x34a14,
1568                0x34a1c, 0x34a2c,
1569                0x34a44, 0x34a50,
1570                0x34a74, 0x34a74,
1571                0x34a7c, 0x34afc,
1572                0x34b08, 0x34c24,
1573                0x34d00, 0x34d00,
1574                0x34d08, 0x34d14,
1575                0x34d1c, 0x34d20,
1576                0x34d3c, 0x34d3c,
1577                0x34d48, 0x34d50,
1578                0x35200, 0x3520c,
1579                0x35220, 0x35220,
1580                0x35240, 0x35240,
1581                0x35600, 0x3560c,
1582                0x35a00, 0x35a1c,
1583                0x35e00, 0x35e20,
1584                0x35e38, 0x35e3c,
1585                0x35e80, 0x35e80,
1586                0x35e88, 0x35ea8,
1587                0x35eb0, 0x35eb4,
1588                0x35ec8, 0x35ed4,
1589                0x35fb8, 0x36004,
1590                0x36200, 0x36200,
1591                0x36208, 0x36240,
1592                0x36248, 0x36280,
1593                0x36288, 0x362c0,
1594                0x362c8, 0x362fc,
1595                0x36600, 0x36630,
1596                0x36a00, 0x36abc,
1597                0x36b00, 0x36b10,
1598                0x36b20, 0x36b30,
1599                0x36b40, 0x36b50,
1600                0x36b60, 0x36b70,
1601                0x37000, 0x37028,
1602                0x37030, 0x37048,
1603                0x37060, 0x37068,
1604                0x37070, 0x3709c,
1605                0x370f0, 0x37128,
1606                0x37130, 0x37148,
1607                0x37160, 0x37168,
1608                0x37170, 0x3719c,
1609                0x371f0, 0x37238,
1610                0x37240, 0x37240,
1611                0x37248, 0x37250,
1612                0x3725c, 0x37264,
1613                0x37270, 0x372b8,
1614                0x372c0, 0x372e4,
1615                0x372f8, 0x37338,
1616                0x37340, 0x37340,
1617                0x37348, 0x37350,
1618                0x3735c, 0x37364,
1619                0x37370, 0x373b8,
1620                0x373c0, 0x373e4,
1621                0x373f8, 0x37428,
1622                0x37430, 0x37448,
1623                0x37460, 0x37468,
1624                0x37470, 0x3749c,
1625                0x374f0, 0x37528,
1626                0x37530, 0x37548,
1627                0x37560, 0x37568,
1628                0x37570, 0x3759c,
1629                0x375f0, 0x37638,
1630                0x37640, 0x37640,
1631                0x37648, 0x37650,
1632                0x3765c, 0x37664,
1633                0x37670, 0x376b8,
1634                0x376c0, 0x376e4,
1635                0x376f8, 0x37738,
1636                0x37740, 0x37740,
1637                0x37748, 0x37750,
1638                0x3775c, 0x37764,
1639                0x37770, 0x377b8,
1640                0x377c0, 0x377e4,
1641                0x377f8, 0x377fc,
1642                0x37814, 0x37814,
1643                0x3782c, 0x3782c,
1644                0x37880, 0x3788c,
1645                0x378e8, 0x378ec,
1646                0x37900, 0x37928,
1647                0x37930, 0x37948,
1648                0x37960, 0x37968,
1649                0x37970, 0x3799c,
1650                0x379f0, 0x37a38,
1651                0x37a40, 0x37a40,
1652                0x37a48, 0x37a50,
1653                0x37a5c, 0x37a64,
1654                0x37a70, 0x37ab8,
1655                0x37ac0, 0x37ae4,
1656                0x37af8, 0x37b10,
1657                0x37b28, 0x37b28,
1658                0x37b3c, 0x37b50,
1659                0x37bf0, 0x37c10,
1660                0x37c28, 0x37c28,
1661                0x37c3c, 0x37c50,
1662                0x37cf0, 0x37cfc,
1663                0x38000, 0x38030,
1664                0x38100, 0x38144,
1665                0x38190, 0x381a0,
1666                0x381a8, 0x381b8,
1667                0x381c4, 0x381c8,
1668                0x381d0, 0x381d0,
1669                0x38200, 0x38318,
1670                0x38400, 0x384b4,
1671                0x384c0, 0x3852c,
1672                0x38540, 0x3861c,
1673                0x38800, 0x38828,
1674                0x38834, 0x38834,
1675                0x388c0, 0x38908,
1676                0x38910, 0x389ac,
1677                0x38a00, 0x38a14,
1678                0x38a1c, 0x38a2c,
1679                0x38a44, 0x38a50,
1680                0x38a74, 0x38a74,
1681                0x38a7c, 0x38afc,
1682                0x38b08, 0x38c24,
1683                0x38d00, 0x38d00,
1684                0x38d08, 0x38d14,
1685                0x38d1c, 0x38d20,
1686                0x38d3c, 0x38d3c,
1687                0x38d48, 0x38d50,
1688                0x39200, 0x3920c,
1689                0x39220, 0x39220,
1690                0x39240, 0x39240,
1691                0x39600, 0x3960c,
1692                0x39a00, 0x39a1c,
1693                0x39e00, 0x39e20,
1694                0x39e38, 0x39e3c,
1695                0x39e80, 0x39e80,
1696                0x39e88, 0x39ea8,
1697                0x39eb0, 0x39eb4,
1698                0x39ec8, 0x39ed4,
1699                0x39fb8, 0x3a004,
1700                0x3a200, 0x3a200,
1701                0x3a208, 0x3a240,
1702                0x3a248, 0x3a280,
1703                0x3a288, 0x3a2c0,
1704                0x3a2c8, 0x3a2fc,
1705                0x3a600, 0x3a630,
1706                0x3aa00, 0x3aabc,
1707                0x3ab00, 0x3ab10,
1708                0x3ab20, 0x3ab30,
1709                0x3ab40, 0x3ab50,
1710                0x3ab60, 0x3ab70,
1711                0x3b000, 0x3b028,
1712                0x3b030, 0x3b048,
1713                0x3b060, 0x3b068,
1714                0x3b070, 0x3b09c,
1715                0x3b0f0, 0x3b128,
1716                0x3b130, 0x3b148,
1717                0x3b160, 0x3b168,
1718                0x3b170, 0x3b19c,
1719                0x3b1f0, 0x3b238,
1720                0x3b240, 0x3b240,
1721                0x3b248, 0x3b250,
1722                0x3b25c, 0x3b264,
1723                0x3b270, 0x3b2b8,
1724                0x3b2c0, 0x3b2e4,
1725                0x3b2f8, 0x3b338,
1726                0x3b340, 0x3b340,
1727                0x3b348, 0x3b350,
1728                0x3b35c, 0x3b364,
1729                0x3b370, 0x3b3b8,
1730                0x3b3c0, 0x3b3e4,
1731                0x3b3f8, 0x3b428,
1732                0x3b430, 0x3b448,
1733                0x3b460, 0x3b468,
1734                0x3b470, 0x3b49c,
1735                0x3b4f0, 0x3b528,
1736                0x3b530, 0x3b548,
1737                0x3b560, 0x3b568,
1738                0x3b570, 0x3b59c,
1739                0x3b5f0, 0x3b638,
1740                0x3b640, 0x3b640,
1741                0x3b648, 0x3b650,
1742                0x3b65c, 0x3b664,
1743                0x3b670, 0x3b6b8,
1744                0x3b6c0, 0x3b6e4,
1745                0x3b6f8, 0x3b738,
1746                0x3b740, 0x3b740,
1747                0x3b748, 0x3b750,
1748                0x3b75c, 0x3b764,
1749                0x3b770, 0x3b7b8,
1750                0x3b7c0, 0x3b7e4,
1751                0x3b7f8, 0x3b7fc,
1752                0x3b814, 0x3b814,
1753                0x3b82c, 0x3b82c,
1754                0x3b880, 0x3b88c,
1755                0x3b8e8, 0x3b8ec,
1756                0x3b900, 0x3b928,
1757                0x3b930, 0x3b948,
1758                0x3b960, 0x3b968,
1759                0x3b970, 0x3b99c,
1760                0x3b9f0, 0x3ba38,
1761                0x3ba40, 0x3ba40,
1762                0x3ba48, 0x3ba50,
1763                0x3ba5c, 0x3ba64,
1764                0x3ba70, 0x3bab8,
1765                0x3bac0, 0x3bae4,
1766                0x3baf8, 0x3bb10,
1767                0x3bb28, 0x3bb28,
1768                0x3bb3c, 0x3bb50,
1769                0x3bbf0, 0x3bc10,
1770                0x3bc28, 0x3bc28,
1771                0x3bc3c, 0x3bc50,
1772                0x3bcf0, 0x3bcfc,
1773                0x3c000, 0x3c030,
1774                0x3c100, 0x3c144,
1775                0x3c190, 0x3c1a0,
1776                0x3c1a8, 0x3c1b8,
1777                0x3c1c4, 0x3c1c8,
1778                0x3c1d0, 0x3c1d0,
1779                0x3c200, 0x3c318,
1780                0x3c400, 0x3c4b4,
1781                0x3c4c0, 0x3c52c,
1782                0x3c540, 0x3c61c,
1783                0x3c800, 0x3c828,
1784                0x3c834, 0x3c834,
1785                0x3c8c0, 0x3c908,
1786                0x3c910, 0x3c9ac,
1787                0x3ca00, 0x3ca14,
1788                0x3ca1c, 0x3ca2c,
1789                0x3ca44, 0x3ca50,
1790                0x3ca74, 0x3ca74,
1791                0x3ca7c, 0x3cafc,
1792                0x3cb08, 0x3cc24,
1793                0x3cd00, 0x3cd00,
1794                0x3cd08, 0x3cd14,
1795                0x3cd1c, 0x3cd20,
1796                0x3cd3c, 0x3cd3c,
1797                0x3cd48, 0x3cd50,
1798                0x3d200, 0x3d20c,
1799                0x3d220, 0x3d220,
1800                0x3d240, 0x3d240,
1801                0x3d600, 0x3d60c,
1802                0x3da00, 0x3da1c,
1803                0x3de00, 0x3de20,
1804                0x3de38, 0x3de3c,
1805                0x3de80, 0x3de80,
1806                0x3de88, 0x3dea8,
1807                0x3deb0, 0x3deb4,
1808                0x3dec8, 0x3ded4,
1809                0x3dfb8, 0x3e004,
1810                0x3e200, 0x3e200,
1811                0x3e208, 0x3e240,
1812                0x3e248, 0x3e280,
1813                0x3e288, 0x3e2c0,
1814                0x3e2c8, 0x3e2fc,
1815                0x3e600, 0x3e630,
1816                0x3ea00, 0x3eabc,
1817                0x3eb00, 0x3eb10,
1818                0x3eb20, 0x3eb30,
1819                0x3eb40, 0x3eb50,
1820                0x3eb60, 0x3eb70,
1821                0x3f000, 0x3f028,
1822                0x3f030, 0x3f048,
1823                0x3f060, 0x3f068,
1824                0x3f070, 0x3f09c,
1825                0x3f0f0, 0x3f128,
1826                0x3f130, 0x3f148,
1827                0x3f160, 0x3f168,
1828                0x3f170, 0x3f19c,
1829                0x3f1f0, 0x3f238,
1830                0x3f240, 0x3f240,
1831                0x3f248, 0x3f250,
1832                0x3f25c, 0x3f264,
1833                0x3f270, 0x3f2b8,
1834                0x3f2c0, 0x3f2e4,
1835                0x3f2f8, 0x3f338,
1836                0x3f340, 0x3f340,
1837                0x3f348, 0x3f350,
1838                0x3f35c, 0x3f364,
1839                0x3f370, 0x3f3b8,
1840                0x3f3c0, 0x3f3e4,
1841                0x3f3f8, 0x3f428,
1842                0x3f430, 0x3f448,
1843                0x3f460, 0x3f468,
1844                0x3f470, 0x3f49c,
1845                0x3f4f0, 0x3f528,
1846                0x3f530, 0x3f548,
1847                0x3f560, 0x3f568,
1848                0x3f570, 0x3f59c,
1849                0x3f5f0, 0x3f638,
1850                0x3f640, 0x3f640,
1851                0x3f648, 0x3f650,
1852                0x3f65c, 0x3f664,
1853                0x3f670, 0x3f6b8,
1854                0x3f6c0, 0x3f6e4,
1855                0x3f6f8, 0x3f738,
1856                0x3f740, 0x3f740,
1857                0x3f748, 0x3f750,
1858                0x3f75c, 0x3f764,
1859                0x3f770, 0x3f7b8,
1860                0x3f7c0, 0x3f7e4,
1861                0x3f7f8, 0x3f7fc,
1862                0x3f814, 0x3f814,
1863                0x3f82c, 0x3f82c,
1864                0x3f880, 0x3f88c,
1865                0x3f8e8, 0x3f8ec,
1866                0x3f900, 0x3f928,
1867                0x3f930, 0x3f948,
1868                0x3f960, 0x3f968,
1869                0x3f970, 0x3f99c,
1870                0x3f9f0, 0x3fa38,
1871                0x3fa40, 0x3fa40,
1872                0x3fa48, 0x3fa50,
1873                0x3fa5c, 0x3fa64,
1874                0x3fa70, 0x3fab8,
1875                0x3fac0, 0x3fae4,
1876                0x3faf8, 0x3fb10,
1877                0x3fb28, 0x3fb28,
1878                0x3fb3c, 0x3fb50,
1879                0x3fbf0, 0x3fc10,
1880                0x3fc28, 0x3fc28,
1881                0x3fc3c, 0x3fc50,
1882                0x3fcf0, 0x3fcfc,
1883                0x40000, 0x4000c,
1884                0x40040, 0x40050,
1885                0x40060, 0x40068,
1886                0x4007c, 0x4008c,
1887                0x40094, 0x400b0,
1888                0x400c0, 0x40144,
1889                0x40180, 0x4018c,
1890                0x40200, 0x40254,
1891                0x40260, 0x40264,
1892                0x40270, 0x40288,
1893                0x40290, 0x40298,
1894                0x402ac, 0x402c8,
1895                0x402d0, 0x402e0,
1896                0x402f0, 0x402f0,
1897                0x40300, 0x4033c,
1898                0x403f8, 0x403fc,
1899                0x41304, 0x413c4,
1900                0x41400, 0x4140c,
1901                0x41414, 0x4141c,
1902                0x41480, 0x414d0,
1903                0x44000, 0x44054,
1904                0x4405c, 0x44078,
1905                0x440c0, 0x44174,
1906                0x44180, 0x441ac,
1907                0x441b4, 0x441b8,
1908                0x441c0, 0x44254,
1909                0x4425c, 0x44278,
1910                0x442c0, 0x44374,
1911                0x44380, 0x443ac,
1912                0x443b4, 0x443b8,
1913                0x443c0, 0x44454,
1914                0x4445c, 0x44478,
1915                0x444c0, 0x44574,
1916                0x44580, 0x445ac,
1917                0x445b4, 0x445b8,
1918                0x445c0, 0x44654,
1919                0x4465c, 0x44678,
1920                0x446c0, 0x44774,
1921                0x44780, 0x447ac,
1922                0x447b4, 0x447b8,
1923                0x447c0, 0x44854,
1924                0x4485c, 0x44878,
1925                0x448c0, 0x44974,
1926                0x44980, 0x449ac,
1927                0x449b4, 0x449b8,
1928                0x449c0, 0x449fc,
1929                0x45000, 0x45004,
1930                0x45010, 0x45030,
1931                0x45040, 0x45060,
1932                0x45068, 0x45068,
1933                0x45080, 0x45084,
1934                0x450a0, 0x450b0,
1935                0x45200, 0x45204,
1936                0x45210, 0x45230,
1937                0x45240, 0x45260,
1938                0x45268, 0x45268,
1939                0x45280, 0x45284,
1940                0x452a0, 0x452b0,
1941                0x460c0, 0x460e4,
1942                0x47000, 0x4703c,
1943                0x47044, 0x4708c,
1944                0x47200, 0x47250,
1945                0x47400, 0x47408,
1946                0x47414, 0x47420,
1947                0x47600, 0x47618,
1948                0x47800, 0x47814,
1949                0x48000, 0x4800c,
1950                0x48040, 0x48050,
1951                0x48060, 0x48068,
1952                0x4807c, 0x4808c,
1953                0x48094, 0x480b0,
1954                0x480c0, 0x48144,
1955                0x48180, 0x4818c,
1956                0x48200, 0x48254,
1957                0x48260, 0x48264,
1958                0x48270, 0x48288,
1959                0x48290, 0x48298,
1960                0x482ac, 0x482c8,
1961                0x482d0, 0x482e0,
1962                0x482f0, 0x482f0,
1963                0x48300, 0x4833c,
1964                0x483f8, 0x483fc,
1965                0x49304, 0x493c4,
1966                0x49400, 0x4940c,
1967                0x49414, 0x4941c,
1968                0x49480, 0x494d0,
1969                0x4c000, 0x4c054,
1970                0x4c05c, 0x4c078,
1971                0x4c0c0, 0x4c174,
1972                0x4c180, 0x4c1ac,
1973                0x4c1b4, 0x4c1b8,
1974                0x4c1c0, 0x4c254,
1975                0x4c25c, 0x4c278,
1976                0x4c2c0, 0x4c374,
1977                0x4c380, 0x4c3ac,
1978                0x4c3b4, 0x4c3b8,
1979                0x4c3c0, 0x4c454,
1980                0x4c45c, 0x4c478,
1981                0x4c4c0, 0x4c574,
1982                0x4c580, 0x4c5ac,
1983                0x4c5b4, 0x4c5b8,
1984                0x4c5c0, 0x4c654,
1985                0x4c65c, 0x4c678,
1986                0x4c6c0, 0x4c774,
1987                0x4c780, 0x4c7ac,
1988                0x4c7b4, 0x4c7b8,
1989                0x4c7c0, 0x4c854,
1990                0x4c85c, 0x4c878,
1991                0x4c8c0, 0x4c974,
1992                0x4c980, 0x4c9ac,
1993                0x4c9b4, 0x4c9b8,
1994                0x4c9c0, 0x4c9fc,
1995                0x4d000, 0x4d004,
1996                0x4d010, 0x4d030,
1997                0x4d040, 0x4d060,
1998                0x4d068, 0x4d068,
1999                0x4d080, 0x4d084,
2000                0x4d0a0, 0x4d0b0,

2001                0x4d200, 0x4d204,
2002                0x4d210, 0x4d230,
2003                0x4d240, 0x4d260,
2004                0x4d268, 0x4d268,
2005                0x4d280, 0x4d284,
2006                0x4d2a0, 0x4d2b0,
2007                0x4e0c0, 0x4e0e4,
2008                0x4f000, 0x4f03c,
2009                0x4f044, 0x4f08c,
2010                0x4f200, 0x4f250,
2011                0x4f400, 0x4f408,
2012                0x4f414, 0x4f420,
2013                0x4f600, 0x4f618,
2014                0x4f800, 0x4f814,
2015                0x50000, 0x50084,
2016                0x50090, 0x500cc,
2017                0x50400, 0x50400,
2018                0x50800, 0x50884,
2019                0x50890, 0x508cc,
2020                0x50c00, 0x50c00,
2021                0x51000, 0x5101c,
2022                0x51300, 0x51308,
2023        };
2024
2025        static const unsigned int t6_reg_ranges[] = {
2026                0x1008, 0x101c,
2027                0x1024, 0x10a8,
2028                0x10b4, 0x10f8,
2029                0x1100, 0x1114,
2030                0x111c, 0x112c,
2031                0x1138, 0x113c,
2032                0x1144, 0x114c,
2033                0x1180, 0x1184,
2034                0x1190, 0x1194,
2035                0x11a0, 0x11a4,
2036                0x11b0, 0x11b4,
2037                0x11fc, 0x1274,
2038                0x1280, 0x133c,
2039                0x1800, 0x18fc,
2040                0x3000, 0x302c,
2041                0x3060, 0x30b0,
2042                0x30b8, 0x30d8,
2043                0x30e0, 0x30fc,
2044                0x3140, 0x357c,
2045                0x35a8, 0x35cc,
2046                0x35ec, 0x35ec,
2047                0x3600, 0x5624,
2048                0x56cc, 0x56ec,
2049                0x56f4, 0x5720,
2050                0x5728, 0x575c,
2051                0x580c, 0x5814,
2052                0x5890, 0x589c,
2053                0x58a4, 0x58ac,
2054                0x58b8, 0x58bc,
2055                0x5940, 0x595c,
2056                0x5980, 0x598c,
2057                0x59b0, 0x59c8,
2058                0x59d0, 0x59dc,
2059                0x59fc, 0x5a18,
2060                0x5a60, 0x5a6c,
2061                0x5a80, 0x5a8c,
2062                0x5a94, 0x5a9c,
2063                0x5b94, 0x5bfc,
2064                0x5c10, 0x5e48,
2065                0x5e50, 0x5e94,
2066                0x5ea0, 0x5eb0,
2067                0x5ec0, 0x5ec0,
2068                0x5ec8, 0x5ed0,
2069                0x5ee0, 0x5ee0,
2070                0x5ef0, 0x5ef0,
2071                0x5f00, 0x5f00,
2072                0x6000, 0x6020,
2073                0x6028, 0x6040,
2074                0x6058, 0x609c,
2075                0x60a8, 0x619c,
2076                0x7700, 0x7798,
2077                0x77c0, 0x7880,
2078                0x78cc, 0x78fc,
2079                0x7b00, 0x7b58,
2080                0x7b60, 0x7b84,
2081                0x7b8c, 0x7c54,
2082                0x7d00, 0x7d38,
2083                0x7d40, 0x7d84,
2084                0x7d8c, 0x7ddc,
2085                0x7de4, 0x7e04,
2086                0x7e10, 0x7e1c,
2087                0x7e24, 0x7e38,
2088                0x7e40, 0x7e44,
2089                0x7e4c, 0x7e78,
2090                0x7e80, 0x7edc,
2091                0x7ee8, 0x7efc,
2092                0x8dc0, 0x8de4,
2093                0x8df8, 0x8e04,
2094                0x8e10, 0x8e84,
2095                0x8ea0, 0x8f88,
2096                0x8fb8, 0x9058,
2097                0x9060, 0x9060,
2098                0x9068, 0x90f8,
2099                0x9100, 0x9124,
2100                0x9400, 0x9470,
2101                0x9600, 0x9600,
2102                0x9608, 0x9638,
2103                0x9640, 0x9704,
2104                0x9710, 0x971c,
2105                0x9800, 0x9808,
2106                0x9820, 0x983c,
2107                0x9850, 0x9864,
2108                0x9c00, 0x9c6c,
2109                0x9c80, 0x9cec,
2110                0x9d00, 0x9d6c,
2111                0x9d80, 0x9dec,
2112                0x9e00, 0x9e6c,
2113                0x9e80, 0x9eec,
2114                0x9f00, 0x9f6c,
2115                0x9f80, 0xa020,
2116                0xd004, 0xd03c,
2117                0xd100, 0xd118,
2118                0xd200, 0xd214,
2119                0xd220, 0xd234,
2120                0xd240, 0xd254,
2121                0xd260, 0xd274,
2122                0xd280, 0xd294,
2123                0xd2a0, 0xd2b4,
2124                0xd2c0, 0xd2d4,
2125                0xd2e0, 0xd2f4,
2126                0xd300, 0xd31c,
2127                0xdfc0, 0xdfe0,
2128                0xe000, 0xf008,
2129                0xf010, 0xf018,
2130                0xf020, 0xf028,
2131                0x11000, 0x11014,
2132                0x11048, 0x1106c,
2133                0x11074, 0x11088,
2134                0x11098, 0x11120,
2135                0x1112c, 0x1117c,
2136                0x11190, 0x112e0,
2137                0x11300, 0x1130c,
2138                0x12000, 0x1206c,
2139                0x19040, 0x1906c,
2140                0x19078, 0x19080,
2141                0x1908c, 0x190e8,
2142                0x190f0, 0x190f8,
2143                0x19100, 0x19110,
2144                0x19120, 0x19124,
2145                0x19150, 0x19194,
2146                0x1919c, 0x191b0,
2147                0x191d0, 0x191e8,
2148                0x19238, 0x19290,
2149                0x192a4, 0x192b0,
2150                0x192bc, 0x192bc,
2151                0x19348, 0x1934c,
2152                0x193f8, 0x19418,
2153                0x19420, 0x19428,
2154                0x19430, 0x19444,
2155                0x1944c, 0x1946c,
2156                0x19474, 0x19474,
2157                0x19490, 0x194cc,
2158                0x194f0, 0x194f8,
2159                0x19c00, 0x19c48,
2160                0x19c50, 0x19c80,
2161                0x19c94, 0x19c98,
2162                0x19ca0, 0x19cbc,
2163                0x19ce4, 0x19ce4,
2164                0x19cf0, 0x19cf8,
2165                0x19d00, 0x19d28,
2166                0x19d50, 0x19d78,
2167                0x19d94, 0x19d98,
2168                0x19da0, 0x19dc8,
2169                0x19df0, 0x19e10,
2170                0x19e50, 0x19e6c,
2171                0x19ea0, 0x19ebc,
2172                0x19ec4, 0x19ef4,
2173                0x19f04, 0x19f2c,
2174                0x19f34, 0x19f34,
2175                0x19f40, 0x19f50,
2176                0x19f90, 0x19fac,
2177                0x19fc4, 0x19fc8,
2178                0x19fd0, 0x19fe4,
2179                0x1a000, 0x1a004,
2180                0x1a010, 0x1a06c,
2181                0x1a0b0, 0x1a0e4,
2182                0x1a0ec, 0x1a0f8,
2183                0x1a100, 0x1a108,
2184                0x1a114, 0x1a120,
2185                0x1a128, 0x1a130,
2186                0x1a138, 0x1a138,
2187                0x1a190, 0x1a1c4,
2188                0x1a1fc, 0x1a1fc,
2189                0x1e008, 0x1e00c,
2190                0x1e040, 0x1e044,
2191                0x1e04c, 0x1e04c,
2192                0x1e284, 0x1e290,
2193                0x1e2c0, 0x1e2c0,
2194                0x1e2e0, 0x1e2e0,
2195                0x1e300, 0x1e384,
2196                0x1e3c0, 0x1e3c8,
2197                0x1e408, 0x1e40c,
2198                0x1e440, 0x1e444,
2199                0x1e44c, 0x1e44c,
2200                0x1e684, 0x1e690,
2201                0x1e6c0, 0x1e6c0,
2202                0x1e6e0, 0x1e6e0,
2203                0x1e700, 0x1e784,
2204                0x1e7c0, 0x1e7c8,
2205                0x1e808, 0x1e80c,
2206                0x1e840, 0x1e844,
2207                0x1e84c, 0x1e84c,
2208                0x1ea84, 0x1ea90,
2209                0x1eac0, 0x1eac0,
2210                0x1eae0, 0x1eae0,
2211                0x1eb00, 0x1eb84,
2212                0x1ebc0, 0x1ebc8,
2213                0x1ec08, 0x1ec0c,
2214                0x1ec40, 0x1ec44,
2215                0x1ec4c, 0x1ec4c,
2216                0x1ee84, 0x1ee90,
2217                0x1eec0, 0x1eec0,
2218                0x1eee0, 0x1eee0,
2219                0x1ef00, 0x1ef84,
2220                0x1efc0, 0x1efc8,
2221                0x1f008, 0x1f00c,
2222                0x1f040, 0x1f044,
2223                0x1f04c, 0x1f04c,
2224                0x1f284, 0x1f290,
2225                0x1f2c0, 0x1f2c0,
2226                0x1f2e0, 0x1f2e0,
2227                0x1f300, 0x1f384,
2228                0x1f3c0, 0x1f3c8,
2229                0x1f408, 0x1f40c,
2230                0x1f440, 0x1f444,
2231                0x1f44c, 0x1f44c,
2232                0x1f684, 0x1f690,
2233                0x1f6c0, 0x1f6c0,
2234                0x1f6e0, 0x1f6e0,
2235                0x1f700, 0x1f784,
2236                0x1f7c0, 0x1f7c8,
2237                0x1f808, 0x1f80c,
2238                0x1f840, 0x1f844,
2239                0x1f84c, 0x1f84c,
2240                0x1fa84, 0x1fa90,
2241                0x1fac0, 0x1fac0,
2242                0x1fae0, 0x1fae0,
2243                0x1fb00, 0x1fb84,
2244                0x1fbc0, 0x1fbc8,
2245                0x1fc08, 0x1fc0c,
2246                0x1fc40, 0x1fc44,
2247                0x1fc4c, 0x1fc4c,
2248                0x1fe84, 0x1fe90,
2249                0x1fec0, 0x1fec0,
2250                0x1fee0, 0x1fee0,
2251                0x1ff00, 0x1ff84,
2252                0x1ffc0, 0x1ffc8,
2253                0x30000, 0x30030,
2254                0x30100, 0x30168,
2255                0x30190, 0x301a0,
2256                0x301a8, 0x301b8,
2257                0x301c4, 0x301c8,
2258                0x301d0, 0x301d0,
2259                0x30200, 0x30320,
2260                0x30400, 0x304b4,
2261                0x304c0, 0x3052c,
2262                0x30540, 0x3061c,
2263                0x30800, 0x308a0,
2264                0x308c0, 0x30908,
2265                0x30910, 0x309b8,
2266                0x30a00, 0x30a04,
2267                0x30a0c, 0x30a14,
2268                0x30a1c, 0x30a2c,
2269                0x30a44, 0x30a50,
2270                0x30a74, 0x30a74,
2271                0x30a7c, 0x30afc,
2272                0x30b08, 0x30c24,
2273                0x30d00, 0x30d14,
2274                0x30d1c, 0x30d3c,
2275                0x30d44, 0x30d4c,
2276                0x30d54, 0x30d74,
2277                0x30d7c, 0x30d7c,
2278                0x30de0, 0x30de0,
2279                0x30e00, 0x30ed4,
2280                0x30f00, 0x30fa4,
2281                0x30fc0, 0x30fc4,
2282                0x31000, 0x31004,
2283                0x31080, 0x310fc,
2284                0x31208, 0x31220,
2285                0x3123c, 0x31254,
2286                0x31300, 0x31300,
2287                0x31308, 0x3131c,
2288                0x31338, 0x3133c,
2289                0x31380, 0x31380,
2290                0x31388, 0x313a8,
2291                0x313b4, 0x313b4,
2292                0x31400, 0x31420,
2293                0x31438, 0x3143c,
2294                0x31480, 0x31480,
2295                0x314a8, 0x314a8,
2296                0x314b0, 0x314b4,
2297                0x314c8, 0x314d4,
2298                0x31a40, 0x31a4c,
2299                0x31af0, 0x31b20,
2300                0x31b38, 0x31b3c,
2301                0x31b80, 0x31b80,
2302                0x31ba8, 0x31ba8,
2303                0x31bb0, 0x31bb4,
2304                0x31bc8, 0x31bd4,
2305                0x32140, 0x3218c,
2306                0x321f0, 0x321f4,
2307                0x32200, 0x32200,
2308                0x32218, 0x32218,
2309                0x32400, 0x32400,
2310                0x32408, 0x3241c,
2311                0x32618, 0x32620,
2312                0x32664, 0x32664,
2313                0x326a8, 0x326a8,
2314                0x326ec, 0x326ec,
2315                0x32a00, 0x32abc,
2316                0x32b00, 0x32b18,
2317                0x32b20, 0x32b38,
2318                0x32b40, 0x32b58,
2319                0x32b60, 0x32b78,
2320                0x32c00, 0x32c00,
2321                0x32c08, 0x32c3c,
2322                0x33000, 0x3302c,
2323                0x33034, 0x33050,
2324                0x33058, 0x33058,
2325                0x33060, 0x3308c,
2326                0x3309c, 0x330ac,
2327                0x330c0, 0x330c0,
2328                0x330c8, 0x330d0,
2329                0x330d8, 0x330e0,
2330                0x330ec, 0x3312c,
2331                0x33134, 0x33150,
2332                0x33158, 0x33158,
2333                0x33160, 0x3318c,
2334                0x3319c, 0x331ac,
2335                0x331c0, 0x331c0,
2336                0x331c8, 0x331d0,
2337                0x331d8, 0x331e0,
2338                0x331ec, 0x33290,
2339                0x33298, 0x332c4,
2340                0x332e4, 0x33390,
2341                0x33398, 0x333c4,
2342                0x333e4, 0x3342c,
2343                0x33434, 0x33450,
2344                0x33458, 0x33458,
2345                0x33460, 0x3348c,
2346                0x3349c, 0x334ac,
2347                0x334c0, 0x334c0,
2348                0x334c8, 0x334d0,
2349                0x334d8, 0x334e0,
2350                0x334ec, 0x3352c,
2351                0x33534, 0x33550,
2352                0x33558, 0x33558,
2353                0x33560, 0x3358c,
2354                0x3359c, 0x335ac,
2355                0x335c0, 0x335c0,
2356                0x335c8, 0x335d0,
2357                0x335d8, 0x335e0,
2358                0x335ec, 0x33690,
2359                0x33698, 0x336c4,
2360                0x336e4, 0x33790,
2361                0x33798, 0x337c4,
2362                0x337e4, 0x337fc,
2363                0x33814, 0x33814,
2364                0x33854, 0x33868,
2365                0x33880, 0x3388c,
2366                0x338c0, 0x338d0,
2367                0x338e8, 0x338ec,
2368                0x33900, 0x3392c,
2369                0x33934, 0x33950,
2370                0x33958, 0x33958,
2371                0x33960, 0x3398c,
2372                0x3399c, 0x339ac,
2373                0x339c0, 0x339c0,
2374                0x339c8, 0x339d0,
2375                0x339d8, 0x339e0,
2376                0x339ec, 0x33a90,
2377                0x33a98, 0x33ac4,
2378                0x33ae4, 0x33b10,
2379                0x33b24, 0x33b28,
2380                0x33b38, 0x33b50,
2381                0x33bf0, 0x33c10,
2382                0x33c24, 0x33c28,
2383                0x33c38, 0x33c50,
2384                0x33cf0, 0x33cfc,
2385                0x34000, 0x34030,
2386                0x34100, 0x34168,
2387                0x34190, 0x341a0,
2388                0x341a8, 0x341b8,
2389                0x341c4, 0x341c8,
2390                0x341d0, 0x341d0,
2391                0x34200, 0x34320,
2392                0x34400, 0x344b4,
2393                0x344c0, 0x3452c,
2394                0x34540, 0x3461c,
2395                0x34800, 0x348a0,
2396                0x348c0, 0x34908,
2397                0x34910, 0x349b8,
2398                0x34a00, 0x34a04,
2399                0x34a0c, 0x34a14,
2400                0x34a1c, 0x34a2c,
2401                0x34a44, 0x34a50,
2402                0x34a74, 0x34a74,
2403                0x34a7c, 0x34afc,
2404                0x34b08, 0x34c24,
2405                0x34d00, 0x34d14,
2406                0x34d1c, 0x34d3c,
2407                0x34d44, 0x34d4c,
2408                0x34d54, 0x34d74,
2409                0x34d7c, 0x34d7c,
2410                0x34de0, 0x34de0,
2411                0x34e00, 0x34ed4,
2412                0x34f00, 0x34fa4,
2413                0x34fc0, 0x34fc4,
2414                0x35000, 0x35004,
2415                0x35080, 0x350fc,
2416                0x35208, 0x35220,
2417                0x3523c, 0x35254,
2418                0x35300, 0x35300,
2419                0x35308, 0x3531c,
2420                0x35338, 0x3533c,
2421                0x35380, 0x35380,
2422                0x35388, 0x353a8,
2423                0x353b4, 0x353b4,
2424                0x35400, 0x35420,
2425                0x35438, 0x3543c,
2426                0x35480, 0x35480,
2427                0x354a8, 0x354a8,
2428                0x354b0, 0x354b4,
2429                0x354c8, 0x354d4,
2430                0x35a40, 0x35a4c,
2431                0x35af0, 0x35b20,
2432                0x35b38, 0x35b3c,
2433                0x35b80, 0x35b80,
2434                0x35ba8, 0x35ba8,
2435                0x35bb0, 0x35bb4,
2436                0x35bc8, 0x35bd4,
2437                0x36140, 0x3618c,
2438                0x361f0, 0x361f4,
2439                0x36200, 0x36200,
2440                0x36218, 0x36218,
2441                0x36400, 0x36400,
2442                0x36408, 0x3641c,
2443                0x36618, 0x36620,
2444                0x36664, 0x36664,
2445                0x366a8, 0x366a8,
2446                0x366ec, 0x366ec,
2447                0x36a00, 0x36abc,
2448                0x36b00, 0x36b18,
2449                0x36b20, 0x36b38,
2450                0x36b40, 0x36b58,
2451                0x36b60, 0x36b78,
2452                0x36c00, 0x36c00,
2453                0x36c08, 0x36c3c,
2454                0x37000, 0x3702c,
2455                0x37034, 0x37050,
2456                0x37058, 0x37058,
2457                0x37060, 0x3708c,
2458                0x3709c, 0x370ac,
2459                0x370c0, 0x370c0,
2460                0x370c8, 0x370d0,
2461                0x370d8, 0x370e0,
2462                0x370ec, 0x3712c,
2463                0x37134, 0x37150,
2464                0x37158, 0x37158,
2465                0x37160, 0x3718c,
2466                0x3719c, 0x371ac,
2467                0x371c0, 0x371c0,
2468                0x371c8, 0x371d0,
2469                0x371d8, 0x371e0,
2470                0x371ec, 0x37290,
2471                0x37298, 0x372c4,
2472                0x372e4, 0x37390,
2473                0x37398, 0x373c4,
2474                0x373e4, 0x3742c,
2475                0x37434, 0x37450,
2476                0x37458, 0x37458,
2477                0x37460, 0x3748c,
2478                0x3749c, 0x374ac,
2479                0x374c0, 0x374c0,
2480                0x374c8, 0x374d0,
2481                0x374d8, 0x374e0,
2482                0x374ec, 0x3752c,
2483                0x37534, 0x37550,
2484                0x37558, 0x37558,
2485                0x37560, 0x3758c,
2486                0x3759c, 0x375ac,
2487                0x375c0, 0x375c0,
2488                0x375c8, 0x375d0,
2489                0x375d8, 0x375e0,
2490                0x375ec, 0x37690,
2491                0x37698, 0x376c4,
2492                0x376e4, 0x37790,
2493                0x37798, 0x377c4,
2494                0x377e4, 0x377fc,
2495                0x37814, 0x37814,
2496                0x37854, 0x37868,
2497                0x37880, 0x3788c,
2498                0x378c0, 0x378d0,
2499                0x378e8, 0x378ec,
2500                0x37900, 0x3792c,
2501                0x37934, 0x37950,
2502                0x37958, 0x37958,
2503                0x37960, 0x3798c,
2504                0x3799c, 0x379ac,
2505                0x379c0, 0x379c0,
2506                0x379c8, 0x379d0,
2507                0x379d8, 0x379e0,
2508                0x379ec, 0x37a90,
2509                0x37a98, 0x37ac4,
2510                0x37ae4, 0x37b10,
2511                0x37b24, 0x37b28,
2512                0x37b38, 0x37b50,
2513                0x37bf0, 0x37c10,
2514                0x37c24, 0x37c28,
2515                0x37c38, 0x37c50,
2516                0x37cf0, 0x37cfc,
2517                0x40040, 0x40040,
2518                0x40080, 0x40084,
2519                0x40100, 0x40100,
2520                0x40140, 0x401bc,
2521                0x40200, 0x40214,
2522                0x40228, 0x40228,
2523                0x40240, 0x40258,
2524                0x40280, 0x40280,
2525                0x40304, 0x40304,
2526                0x40330, 0x4033c,
2527                0x41304, 0x413c8,
2528                0x413d0, 0x413dc,
2529                0x413f0, 0x413f0,
2530                0x41400, 0x4140c,
2531                0x41414, 0x4141c,
2532                0x41480, 0x414d0,
2533                0x44000, 0x4407c,
2534                0x440c0, 0x441ac,
2535                0x441b4, 0x4427c,
2536                0x442c0, 0x443ac,
2537                0x443b4, 0x4447c,
2538                0x444c0, 0x445ac,
2539                0x445b4, 0x4467c,
2540                0x446c0, 0x447ac,
2541                0x447b4, 0x4487c,
2542                0x448c0, 0x449ac,
2543                0x449b4, 0x44a7c,
2544                0x44ac0, 0x44bac,
2545                0x44bb4, 0x44c7c,
2546                0x44cc0, 0x44dac,
2547                0x44db4, 0x44e7c,
2548                0x44ec0, 0x44fac,
2549                0x44fb4, 0x4507c,
2550                0x450c0, 0x451ac,
2551                0x451b4, 0x451fc,
2552                0x45800, 0x45804,
2553                0x45810, 0x45830,
2554                0x45840, 0x45860,
2555                0x45868, 0x45868,
2556                0x45880, 0x45884,
2557                0x458a0, 0x458b0,
2558                0x45a00, 0x45a04,
2559                0x45a10, 0x45a30,
2560                0x45a40, 0x45a60,
2561                0x45a68, 0x45a68,
2562                0x45a80, 0x45a84,
2563                0x45aa0, 0x45ab0,
2564                0x460c0, 0x460e4,
2565                0x47000, 0x4703c,
2566                0x47044, 0x4708c,
2567                0x47200, 0x47250,
2568                0x47400, 0x47408,
2569                0x47414, 0x47420,
2570                0x47600, 0x47618,
2571                0x47800, 0x47814,
2572                0x47820, 0x4782c,
2573                0x50000, 0x50084,
2574                0x50090, 0x500cc,
2575                0x50300, 0x50384,
2576                0x50400, 0x50400,
2577                0x50800, 0x50884,
2578                0x50890, 0x508cc,
2579                0x50b00, 0x50b84,
2580                0x50c00, 0x50c00,
2581                0x51000, 0x51020,
2582                0x51028, 0x510b0,
2583                0x51300, 0x51324,
2584        };
2585
2586        u32 *buf_end = (u32 *)((char *)buf + buf_size);
2587        const unsigned int *reg_ranges;
2588        int reg_ranges_size, range;
2589        unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip);
2590
2591        /* Select the right set of register ranges to dump depending on the
2592         * adapter chip type.
2593         */
2594        switch (chip_version) {
2595        case CHELSIO_T4:
2596                reg_ranges = t4_reg_ranges;
2597                reg_ranges_size = ARRAY_SIZE(t4_reg_ranges);
2598                break;
2599
2600        case CHELSIO_T5:
2601                reg_ranges = t5_reg_ranges;
2602                reg_ranges_size = ARRAY_SIZE(t5_reg_ranges);
2603                break;
2604
2605        case CHELSIO_T6:
2606                reg_ranges = t6_reg_ranges;
2607                reg_ranges_size = ARRAY_SIZE(t6_reg_ranges);
2608                break;
2609
2610        default:
2611                dev_err(adap->pdev_dev,
2612                        "Unsupported chip version %d\n", chip_version);
2613                return;
2614        }
2615
2616        /* Clear the register buffer and insert the appropriate register
2617         * values selected by the above register ranges.
2618         */
2619        memset(buf, 0, buf_size);
2620        for (range = 0; range < reg_ranges_size; range += 2) {
2621                unsigned int reg = reg_ranges[range];
2622                unsigned int last_reg = reg_ranges[range + 1];
2623                u32 *bufp = (u32 *)((char *)buf + reg);
2624
2625                /* Iterate across the register range filling in the register
2626                 * buffer but don't write past the end of the register buffer.
2627                 */
2628                while (reg <= last_reg && bufp < buf_end) {
2629                        *bufp++ = t4_read_reg(adap, reg);
2630                        reg += sizeof(u32);
2631                }
2632        }
2633}
2634
2635#define EEPROM_STAT_ADDR   0x7bfc
2636#define VPD_SIZE           0x800
2637#define VPD_BASE           0x400
2638#define VPD_BASE_OLD       0
2639#define VPD_LEN            1024
2640#define CHELSIO_VPD_UNIQUE_ID 0x82
2641
2642/**
2643 *      t4_seeprom_wp - enable/disable EEPROM write protection
2644 *      @adapter: the adapter
2645 *      @enable: whether to enable or disable write protection
2646 *
2647 *      Enables or disables write protection on the serial EEPROM.
2648 */
2649int t4_seeprom_wp(struct adapter *adapter, bool enable)
2650{
2651        unsigned int v = enable ? 0xc : 0;
2652        int ret = pci_write_vpd(adapter->pdev, EEPROM_STAT_ADDR, 4, &v);
2653        return ret < 0 ? ret : 0;
2654}
2655
2656/**
2657 *      t4_get_raw_vpd_params - read VPD parameters from VPD EEPROM
2658 *      @adapter: adapter to read
2659 *      @p: where to store the parameters
2660 *
2661 *      Reads card parameters stored in VPD EEPROM.
2662 */
2663int t4_get_raw_vpd_params(struct adapter *adapter, struct vpd_params *p)
2664{
2665        int i, ret = 0, addr;
2666        int ec, sn, pn, na;
2667        u8 *vpd, csum;
2668        unsigned int vpdr_len, kw_offset, id_len;
2669
2670        vpd = vmalloc(VPD_LEN);
2671        if (!vpd)
2672                return -ENOMEM;
2673
2674        /* We have two VPD data structures stored in the adapter VPD area.
2675         * By default, Linux calculates the size of the VPD area by traversing
2676         * the first VPD area at offset 0x0, so we need to tell the OS what
2677         * our real VPD size is.
2678         */
2679        ret = pci_set_vpd_size(adapter->pdev, VPD_SIZE);
2680        if (ret < 0)
2681                goto out;
2682
2683        /* Card information normally starts at VPD_BASE but early cards had
2684         * it at 0.
2685         */
2686        ret = pci_read_vpd(adapter->pdev, VPD_BASE, sizeof(u32), vpd);
2687        if (ret < 0)
2688                goto out;
2689
2690        /* The VPD shall have a unique identifier specified by the PCI SIG.
2691         * For chelsio adapters, the identifier is 0x82. The first byte of a VPD
2692         * shall be CHELSIO_VPD_UNIQUE_ID (0x82). The VPD programming software
2693         * is expected to automatically put this entry at the
2694         * beginning of the VPD.
2695         */
2696        addr = *vpd == CHELSIO_VPD_UNIQUE_ID ? VPD_BASE : VPD_BASE_OLD;
2697
2698        ret = pci_read_vpd(adapter->pdev, addr, VPD_LEN, vpd);
2699        if (ret < 0)
2700                goto out;
2701
2702        if (vpd[0] != PCI_VPD_LRDT_ID_STRING) {
2703                dev_err(adapter->pdev_dev, "missing VPD ID string\n");
2704                ret = -EINVAL;
2705                goto out;
2706        }
2707
2708        id_len = pci_vpd_lrdt_size(vpd);
2709        if (id_len > ID_LEN)
2710                id_len = ID_LEN;
2711
2712        i = pci_vpd_find_tag(vpd, 0, VPD_LEN, PCI_VPD_LRDT_RO_DATA);
2713        if (i < 0) {
2714                dev_err(adapter->pdev_dev, "missing VPD-R section\n");
2715                ret = -EINVAL;
2716                goto out;
2717        }
2718
2719        vpdr_len = pci_vpd_lrdt_size(&vpd[i]);
2720        kw_offset = i + PCI_VPD_LRDT_TAG_SIZE;
2721        if (vpdr_len + kw_offset > VPD_LEN) {
2722                dev_err(adapter->pdev_dev, "bad VPD-R length %u\n", vpdr_len);
2723                ret = -EINVAL;
2724                goto out;
2725        }
2726
2727#define FIND_VPD_KW(var, name) do { \
2728        var = pci_vpd_find_info_keyword(vpd, kw_offset, vpdr_len, name); \
2729        if (var < 0) { \
2730                dev_err(adapter->pdev_dev, "missing VPD keyword " name "\n"); \
2731                ret = -EINVAL; \
2732                goto out; \
2733        } \
2734        var += PCI_VPD_INFO_FLD_HDR_SIZE; \
2735} while (0)
2736
2737        FIND_VPD_KW(i, "RV");
2738        for (csum = 0; i >= 0; i--)
2739                csum += vpd[i];
2740
2741        if (csum) {
2742                dev_err(adapter->pdev_dev,
2743                        "corrupted VPD EEPROM, actual csum %u\n", csum);
2744                ret = -EINVAL;
2745                goto out;
2746        }
2747
2748        FIND_VPD_KW(ec, "EC");
2749        FIND_VPD_KW(sn, "SN");
2750        FIND_VPD_KW(pn, "PN");
2751        FIND_VPD_KW(na, "NA");
2752#undef FIND_VPD_KW
2753
2754        memcpy(p->id, vpd + PCI_VPD_LRDT_TAG_SIZE, id_len);
2755        strim(p->id);
2756        memcpy(p->ec, vpd + ec, EC_LEN);
2757        strim(p->ec);
2758        i = pci_vpd_info_field_size(vpd + sn - PCI_VPD_INFO_FLD_HDR_SIZE);
2759        memcpy(p->sn, vpd + sn, min(i, SERNUM_LEN));
2760        strim(p->sn);
2761        i = pci_vpd_info_field_size(vpd + pn - PCI_VPD_INFO_FLD_HDR_SIZE);
2762        memcpy(p->pn, vpd + pn, min(i, PN_LEN));
2763        strim(p->pn);
2764        memcpy(p->na, vpd + na, min(i, MACADDR_LEN));
2765        strim((char *)p->na);
2766
2767out:
2768        vfree(vpd);
2769        return ret < 0 ? ret : 0;
2770}
2771
2772/**
2773 *      t4_get_vpd_params - read VPD parameters & retrieve Core Clock
2774 *      @adapter: adapter to read
2775 *      @p: where to store the parameters
2776 *
2777 *      Reads card parameters stored in VPD EEPROM and retrieves the Core
2778 *      Clock.  This can only be called after a connection to the firmware
2779 *      is established.
2780 */
2781int t4_get_vpd_params(struct adapter *adapter, struct vpd_params *p)
2782{
2783        u32 cclk_param, cclk_val;
2784        int ret;
2785
2786        /* Grab the raw VPD parameters.
2787         */
2788        ret = t4_get_raw_vpd_params(adapter, p);
2789        if (ret)
2790                return ret;
2791
2792        /* Ask firmware for the Core Clock since it knows how to translate the
2793         * Reference Clock ('V2') VPD field into a Core Clock value ...
2794         */
2795        cclk_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
2796                      FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CCLK));
2797        ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
2798                              1, &cclk_param, &cclk_val);
2799
2800        if (ret)
2801                return ret;
2802        p->cclk = cclk_val;
2803
2804        return 0;
2805}
2806
2807/* serial flash and firmware constants */
2808enum {
2809        SF_ATTEMPTS = 10,             /* max retries for SF operations */
2810
2811        /* flash command opcodes */
2812        SF_PROG_PAGE    = 2,          /* program page */
2813        SF_WR_DISABLE   = 4,          /* disable writes */
2814        SF_RD_STATUS    = 5,          /* read status register */
2815        SF_WR_ENABLE    = 6,          /* enable writes */
2816        SF_RD_DATA_FAST = 0xb,        /* read flash */
2817        SF_RD_ID        = 0x9f,       /* read ID */
2818        SF_ERASE_SECTOR = 0xd8,       /* erase sector */
2819
2820        FW_MAX_SIZE = 16 * SF_SEC_SIZE,
2821};
2822
2823/**
2824 *      sf1_read - read data from the serial flash
2825 *      @adapter: the adapter
2826 *      @byte_cnt: number of bytes to read
2827 *      @cont: whether another operation will be chained
2828 *      @lock: whether to lock SF for PL access only
2829 *      @valp: where to store the read data
2830 *
2831 *      Reads up to 4 bytes of data from the serial flash.  The location of
2832 *      the read needs to be specified prior to calling this by issuing the
2833 *      appropriate commands to the serial flash.
2834 */
2835static int sf1_read(struct adapter *adapter, unsigned int byte_cnt, int cont,
2836                    int lock, u32 *valp)
2837{
2838        int ret;
2839
2840        if (!byte_cnt || byte_cnt > 4)
2841                return -EINVAL;
2842        if (t4_read_reg(adapter, SF_OP_A) & SF_BUSY_F)
2843                return -EBUSY;
2844        t4_write_reg(adapter, SF_OP_A, SF_LOCK_V(lock) |
2845                     SF_CONT_V(cont) | BYTECNT_V(byte_cnt - 1));
2846        ret = t4_wait_op_done(adapter, SF_OP_A, SF_BUSY_F, 0, SF_ATTEMPTS, 5);
2847        if (!ret)
2848                *valp = t4_read_reg(adapter, SF_DATA_A);
2849        return ret;
2850}
2851
2852/**
2853 *      sf1_write - write data to the serial flash
2854 *      @adapter: the adapter
2855 *      @byte_cnt: number of bytes to write
2856 *      @cont: whether another operation will be chained
2857 *      @lock: whether to lock SF for PL access only
2858 *      @val: value to write
2859 *
2860 *      Writes up to 4 bytes of data to the serial flash.  The location of
2861 *      the write needs to be specified prior to calling this by issuing the
2862 *      appropriate commands to the serial flash.
2863 */
2864static int sf1_write(struct adapter *adapter, unsigned int byte_cnt, int cont,
2865                     int lock, u32 val)
2866{
2867        if (!byte_cnt || byte_cnt > 4)
2868                return -EINVAL;
2869        if (t4_read_reg(adapter, SF_OP_A) & SF_BUSY_F)
2870                return -EBUSY;
2871        t4_write_reg(adapter, SF_DATA_A, val);
2872        t4_write_reg(adapter, SF_OP_A, SF_LOCK_V(lock) |
2873                     SF_CONT_V(cont) | BYTECNT_V(byte_cnt - 1) | OP_V(1));
2874        return t4_wait_op_done(adapter, SF_OP_A, SF_BUSY_F, 0, SF_ATTEMPTS, 5);
2875}
2876
2877/**
2878 *      flash_wait_op - wait for a flash operation to complete
2879 *      @adapter: the adapter
2880 *      @attempts: max number of polls of the status register
2881 *      @delay: delay between polls in ms
2882 *
2883 *      Wait for a flash operation to complete by polling the status register.
2884 */
2885static int flash_wait_op(struct adapter *adapter, int attempts, int delay)
2886{
2887        int ret;
2888        u32 status;
2889
2890        while (1) {
2891                if ((ret = sf1_write(adapter, 1, 1, 1, SF_RD_STATUS)) != 0 ||
2892                    (ret = sf1_read(adapter, 1, 0, 1, &status)) != 0)
2893                        return ret;
2894                if (!(status & 1))
2895                        return 0;
2896                if (--attempts == 0)
2897                        return -EAGAIN;
2898                if (delay)
2899                        msleep(delay);
2900        }
2901}
2902
2903/**
2904 *      t4_read_flash - read words from serial flash
2905 *      @adapter: the adapter
2906 *      @addr: the start address for the read
2907 *      @nwords: how many 32-bit words to read
2908 *      @data: where to store the read data
2909 *      @byte_oriented: whether to store data as bytes or as words
2910 *
2911 *      Read the specified number of 32-bit words from the serial flash.
2912 *      If @byte_oriented is set the read data is stored as a byte array
2913 *      (i.e., big-endian), otherwise as 32-bit words in the platform's
2914 *      natural endianess.
2915 */
2916int t4_read_flash(struct adapter *adapter, unsigned int addr,
2917                  unsigned int nwords, u32 *data, int byte_oriented)
2918{
2919        int ret;
2920
2921        if (addr + nwords * sizeof(u32) > adapter->params.sf_size || (addr & 3))
2922                return -EINVAL;
2923
2924        addr = swab32(addr) | SF_RD_DATA_FAST;
2925
2926        if ((ret = sf1_write(adapter, 4, 1, 0, addr)) != 0 ||
2927            (ret = sf1_read(adapter, 1, 1, 0, data)) != 0)
2928                return ret;
2929
2930        for ( ; nwords; nwords--, data++) {
2931                ret = sf1_read(adapter, 4, nwords > 1, nwords == 1, data);
2932                if (nwords == 1)
2933                        t4_write_reg(adapter, SF_OP_A, 0);    /* unlock SF */
2934                if (ret)
2935                        return ret;
2936                if (byte_oriented)
2937                        *data = (__force __u32)(cpu_to_be32(*data));
2938        }
2939        return 0;
2940}
2941
2942/**
2943 *      t4_write_flash - write up to a page of data to the serial flash
2944 *      @adapter: the adapter
2945 *      @addr: the start address to write
2946 *      @n: length of data to write in bytes
2947 *      @data: the data to write
2948 *
2949 *      Writes up to a page of data (256 bytes) to the serial flash starting
2950 *      at the given address.  All the data must be written to the same page.
2951 */
2952static int t4_write_flash(struct adapter *adapter, unsigned int addr,
2953                          unsigned int n, const u8 *data)
2954{
2955        int ret;
2956        u32 buf[64];
2957        unsigned int i, c, left, val, offset = addr & 0xff;
2958
2959        if (addr >= adapter->params.sf_size || offset + n > SF_PAGE_SIZE)
2960                return -EINVAL;
2961
2962        val = swab32(addr) | SF_PROG_PAGE;
2963
2964        if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
2965            (ret = sf1_write(adapter, 4, 1, 1, val)) != 0)
2966                goto unlock;
2967
2968        for (left = n; left; left -= c) {
2969                c = min(left, 4U);
2970                for (val = 0, i = 0; i < c; ++i)
2971                        val = (val << 8) + *data++;
2972
2973                ret = sf1_write(adapter, c, c != left, 1, val);
2974                if (ret)
2975                        goto unlock;
2976        }
2977        ret = flash_wait_op(adapter, 8, 1);
2978        if (ret)
2979                goto unlock;
2980
2981        t4_write_reg(adapter, SF_OP_A, 0);    /* unlock SF */
2982
2983        /* Read the page to verify the write succeeded */
2984        ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1);
2985        if (ret)
2986                return ret;
2987
2988        if (memcmp(data - n, (u8 *)buf + offset, n)) {
2989                dev_err(adapter->pdev_dev,
2990                        "failed to correctly write the flash page at %#x\n",
2991                        addr);
2992                return -EIO;
2993        }
2994        return 0;
2995
2996unlock:
2997        t4_write_reg(adapter, SF_OP_A, 0);    /* unlock SF */
2998        return ret;
2999}
3000

3001/**
3002 *      t4_get_fw_version - read the firmware version
3003 *      @adapter: the adapter
3004 *      @vers: where to place the version
3005 *
3006 *      Reads the FW version from flash.
3007 */
3008int t4_get_fw_version(struct adapter *adapter, u32 *vers)
3009{
3010        return t4_read_flash(adapter, FLASH_FW_START +
3011                             offsetof(struct fw_hdr, fw_ver), 1,
3012                             vers, 0);
3013}
3014
3015/**
3016 *      t4_get_bs_version - read the firmware bootstrap version
3017 *      @adapter: the adapter
3018 *      @vers: where to place the version
3019 *
3020 *      Reads the FW Bootstrap version from flash.
3021 */
3022int t4_get_bs_version(struct adapter *adapter, u32 *vers)
3023{
3024        return t4_read_flash(adapter, FLASH_FWBOOTSTRAP_START +
3025                             offsetof(struct fw_hdr, fw_ver), 1,
3026                             vers, 0);
3027}
3028
3029/**
3030 *      t4_get_tp_version - read the TP microcode version
3031 *      @adapter: the adapter
3032 *      @vers: where to place the version
3033 *
3034 *      Reads the TP microcode version from flash.
3035 */
3036int t4_get_tp_version(struct adapter *adapter, u32 *vers)
3037{
3038        return t4_read_flash(adapter, FLASH_FW_START +
3039                             offsetof(struct fw_hdr, tp_microcode_ver),
3040                             1, vers, 0);
3041}
3042
3043/**
3044 *      t4_get_exprom_version - return the Expansion ROM version (if any)
3045 *      @adapter: the adapter
3046 *      @vers: where to place the version
3047 *
3048 *      Reads the Expansion ROM header from FLASH and returns the version
3049 *      number (if present) through the @vers return value pointer.  We return
3050 *      this in the Firmware Version Format since it's convenient.  Return
3051 *      0 on success, -ENOENT if no Expansion ROM is present.
3052 */
3053int t4_get_exprom_version(struct adapter *adap, u32 *vers)
3054{
3055        struct exprom_header {
3056                unsigned char hdr_arr[16];      /* must start with 0x55aa */
3057                unsigned char hdr_ver[4];       /* Expansion ROM version */
3058        } *hdr;
3059        u32 exprom_header_buf[DIV_ROUND_UP(sizeof(struct exprom_header),
3060                                           sizeof(u32))];
3061        int ret;
3062
3063        ret = t4_read_flash(adap, FLASH_EXP_ROM_START,
3064                            ARRAY_SIZE(exprom_header_buf), exprom_header_buf,
3065                            0);
3066        if (ret)
3067                return ret;
3068
3069        hdr = (struct exprom_header *)exprom_header_buf;
3070        if (hdr->hdr_arr[0] != 0x55 || hdr->hdr_arr[1] != 0xaa)
3071                return -ENOENT;
3072
3073        *vers = (FW_HDR_FW_VER_MAJOR_V(hdr->hdr_ver[0]) |
3074                 FW_HDR_FW_VER_MINOR_V(hdr->hdr_ver[1]) |
3075                 FW_HDR_FW_VER_MICRO_V(hdr->hdr_ver[2]) |
3076                 FW_HDR_FW_VER_BUILD_V(hdr->hdr_ver[3]));
3077        return 0;
3078}
3079
3080/**
3081 *      t4_get_vpd_version - return the VPD version
3082 *      @adapter: the adapter
3083 *      @vers: where to place the version
3084 *
3085 *      Reads the VPD via the Firmware interface (thus this can only be called
3086 *      once we're ready to issue Firmware commands).  The format of the
3087 *      VPD version is adapter specific.  Returns 0 on success, an error on
3088 *      failure.
3089 *
3090 *      Note that early versions of the Firmware didn't include the ability
3091 *      to retrieve the VPD version, so we zero-out the return-value parameter
3092 *      in that case to avoid leaving it with garbage in it.
3093 *
3094 *      Also note that the Firmware will return its cached copy of the VPD
3095 *      Revision ID, not the actual Revision ID as written in the Serial
3096 *      EEPROM.  This is only an issue if a new VPD has been written and the
3097 *      Firmware/Chip haven't yet gone through a RESET sequence.  So it's best
3098 *      to defer calling this routine till after a FW_RESET_CMD has been issued
3099 *      if the Host Driver will be performing a full adapter initialization.
3100 */
3101int t4_get_vpd_version(struct adapter *adapter, u32 *vers)
3102{
3103        u32 vpdrev_param;
3104        int ret;
3105
3106        vpdrev_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3107                        FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_VPDREV));
3108        ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3109                              1, &vpdrev_param, vers);
3110        if (ret)
3111                *vers = 0;
3112        return ret;
3113}
3114
3115/**
3116 *      t4_get_scfg_version - return the Serial Configuration version
3117 *      @adapter: the adapter
3118 *      @vers: where to place the version
3119 *
3120 *      Reads the Serial Configuration Version via the Firmware interface
3121 *      (thus this can only be called once we're ready to issue Firmware
3122 *      commands).  The format of the Serial Configuration version is
3123 *      adapter specific.  Returns 0 on success, an error on failure.
3124 *
3125 *      Note that early versions of the Firmware didn't include the ability
3126 *      to retrieve the Serial Configuration version, so we zero-out the
3127 *      return-value parameter in that case to avoid leaving it with
3128 *      garbage in it.
3129 *
3130 *      Also note that the Firmware will return its cached copy of the Serial
3131 *      Initialization Revision ID, not the actual Revision ID as written in
3132 *      the Serial EEPROM.  This is only an issue if a new VPD has been written
3133 *      and the Firmware/Chip haven't yet gone through a RESET sequence.  So
3134 *      it's best to defer calling this routine till after a FW_RESET_CMD has
3135 *      been issued if the Host Driver will be performing a full adapter
3136 *      initialization.
3137 */
3138int t4_get_scfg_version(struct adapter *adapter, u32 *vers)
3139{
3140        u32 scfgrev_param;
3141        int ret;
3142
3143        scfgrev_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3144                         FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_SCFGREV));
3145        ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3146                              1, &scfgrev_param, vers);
3147        if (ret)
3148                *vers = 0;
3149        return ret;
3150}
3151
3152/**
3153 *      t4_get_version_info - extract various chip/firmware version information
3154 *      @adapter: the adapter
3155 *
3156 *      Reads various chip/firmware version numbers and stores them into the
3157 *      adapter Adapter Parameters structure.  If any of the efforts fails
3158 *      the first failure will be returned, but all of the version numbers
3159 *      will be read.
3160 */
3161int t4_get_version_info(struct adapter *adapter)
3162{
3163        int ret = 0;
3164
3165        #define FIRST_RET(__getvinfo) \
3166        do { \
3167                int __ret = __getvinfo; \
3168                if (__ret && !ret) \
3169                        ret = __ret; \
3170        } while (0)
3171
3172        FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers));
3173        FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers));
3174        FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers));
3175        FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers));
3176        FIRST_RET(t4_get_scfg_version(adapter, &adapter->params.scfg_vers));
3177        FIRST_RET(t4_get_vpd_version(adapter, &adapter->params.vpd_vers));
3178
3179        #undef FIRST_RET
3180        return ret;
3181}
3182
3183/**
3184 *      t4_dump_version_info - dump all of the adapter configuration IDs
3185 *      @adapter: the adapter
3186 *
3187 *      Dumps all of the various bits of adapter configuration version/revision
3188 *      IDs information.  This is typically called at some point after
3189 *      t4_get_version_info() has been called.
3190 */
3191void t4_dump_version_info(struct adapter *adapter)
3192{
3193        /* Device information */
3194        dev_info(adapter->pdev_dev, "Chelsio %s rev %d\n",
3195                 adapter->params.vpd.id,
3196                 CHELSIO_CHIP_RELEASE(adapter->params.chip));
3197        dev_info(adapter->pdev_dev, "S/N: %s, P/N: %s\n",
3198                 adapter->params.vpd.sn, adapter->params.vpd.pn);
3199
3200        /* Firmware Version */
3201        if (!adapter->params.fw_vers)
3202                dev_warn(adapter->pdev_dev, "No firmware loaded\n");
3203        else
3204                dev_info(adapter->pdev_dev, "Firmware version: %u.%u.%u.%u\n",
3205                         FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
3206                         FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
3207                         FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
3208                         FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers));
3209
3210        /* Bootstrap Firmware Version. (Some adapters don't have Bootstrap
3211         * Firmware, so dev_info() is more appropriate here.)
3212         */
3213        if (!adapter->params.bs_vers)
3214                dev_info(adapter->pdev_dev, "No bootstrap loaded\n");
3215        else
3216                dev_info(adapter->pdev_dev, "Bootstrap version: %u.%u.%u.%u\n",
3217                         FW_HDR_FW_VER_MAJOR_G(adapter->params.bs_vers),
3218                         FW_HDR_FW_VER_MINOR_G(adapter->params.bs_vers),
3219                         FW_HDR_FW_VER_MICRO_G(adapter->params.bs_vers),
3220                         FW_HDR_FW_VER_BUILD_G(adapter->params.bs_vers));
3221
3222        /* TP Microcode Version */
3223        if (!adapter->params.tp_vers)
3224                dev_warn(adapter->pdev_dev, "No TP Microcode loaded\n");
3225        else
3226                dev_info(adapter->pdev_dev,
3227                         "TP Microcode version: %u.%u.%u.%u\n",
3228                         FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
3229                         FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
3230                         FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
3231                         FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
3232
3233        /* Expansion ROM version */
3234        if (!adapter->params.er_vers)
3235                dev_info(adapter->pdev_dev, "No Expansion ROM loaded\n");
3236        else
3237                dev_info(adapter->pdev_dev,
3238                         "Expansion ROM version: %u.%u.%u.%u\n",
3239                         FW_HDR_FW_VER_MAJOR_G(adapter->params.er_vers),
3240                         FW_HDR_FW_VER_MINOR_G(adapter->params.er_vers),
3241                         FW_HDR_FW_VER_MICRO_G(adapter->params.er_vers),
3242                         FW_HDR_FW_VER_BUILD_G(adapter->params.er_vers));
3243
3244        /* Serial Configuration version */
3245        dev_info(adapter->pdev_dev, "Serial Configuration version: %#x\n",
3246                 adapter->params.scfg_vers);
3247
3248        /* VPD Version */
3249        dev_info(adapter->pdev_dev, "VPD version: %#x\n",
3250                 adapter->params.vpd_vers);
3251}
3252
3253/**
3254 *      t4_check_fw_version - check if the FW is supported with this driver
3255 *      @adap: the adapter
3256 *
3257 *      Checks if an adapter's FW is compatible with the driver.  Returns 0
3258 *      if there's exact match, a negative error if the version could not be
3259 *      read or there's a major version mismatch
3260 */
3261int t4_check_fw_version(struct adapter *adap)
3262{
3263        int i, ret, major, minor, micro;
3264        int exp_major, exp_minor, exp_micro;
3265        unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip);
3266
3267        ret = t4_get_fw_version(adap, &adap->params.fw_vers);
3268        /* Try multiple times before returning error */
3269        for (i = 0; (ret == -EBUSY || ret == -EAGAIN) && i < 3; i++)
3270                ret = t4_get_fw_version(adap, &adap->params.fw_vers);
3271
3272        if (ret)
3273                return ret;
3274
3275        major = FW_HDR_FW_VER_MAJOR_G(adap->params.fw_vers);
3276        minor = FW_HDR_FW_VER_MINOR_G(adap->params.fw_vers);
3277        micro = FW_HDR_FW_VER_MICRO_G(adap->params.fw_vers);
3278
3279        switch (chip_version) {
3280        case CHELSIO_T4:
3281                exp_major = T4FW_MIN_VERSION_MAJOR;
3282                exp_minor = T4FW_MIN_VERSION_MINOR;
3283                exp_micro = T4FW_MIN_VERSION_MICRO;
3284                break;
3285        case CHELSIO_T5:
3286                exp_major = T5FW_MIN_VERSION_MAJOR;
3287                exp_minor = T5FW_MIN_VERSION_MINOR;
3288                exp_micro = T5FW_MIN_VERSION_MICRO;
3289                break;
3290        case CHELSIO_T6:
3291                exp_major = T6FW_MIN_VERSION_MAJOR;
3292                exp_minor = T6FW_MIN_VERSION_MINOR;
3293                exp_micro = T6FW_MIN_VERSION_MICRO;
3294                break;
3295        default:
3296                dev_err(adap->pdev_dev, "Unsupported chip type, %x\n",
3297                        adap->chip);
3298                return -EINVAL;
3299        }
3300
3301        if (major < exp_major || (major == exp_major && minor < exp_minor) ||
3302            (major == exp_major && minor == exp_minor && micro < exp_micro)) {
3303                dev_err(adap->pdev_dev,
3304                        "Card has firmware version %u.%u.%u, minimum "
3305                        "supported firmware is %u.%u.%u.\n", major, minor,
3306                        micro, exp_major, exp_minor, exp_micro);
3307                return -EFAULT;
3308        }
3309        return 0;
3310}
3311
3312/* Is the given firmware API compatible with the one the driver was compiled
3313 * with?
3314 */
3315static int fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
3316{
3317
3318        /* short circuit if it's the exact same firmware version */
3319        if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
3320                return 1;
3321
3322#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
3323        if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
3324            SAME_INTF(ri) && SAME_INTF(iscsi) && SAME_INTF(fcoe))
3325                return 1;
3326#undef SAME_INTF
3327
3328        return 0;
3329}
3330
3331/* The firmware in the filesystem is usable, but should it be installed?
3332 * This routine explains itself in detail if it indicates the filesystem
3333 * firmware should be installed.
3334 */
3335static int should_install_fs_fw(struct adapter *adap, int card_fw_usable,
3336                                int k, int c)
3337{
3338        const char *reason;
3339
3340        if (!card_fw_usable) {
3341                reason = "incompatible or unusable";
3342                goto install;
3343        }
3344
3345        if (k > c) {
3346                reason = "older than the version supported with this driver";
3347                goto install;
3348        }
3349
3350        return 0;
3351
3352install:
3353        dev_err(adap->pdev_dev, "firmware on card (%u.%u.%u.%u) is %s, "
3354                "installing firmware %u.%u.%u.%u on card.\n",
3355                FW_HDR_FW_VER_MAJOR_G(c), FW_HDR_FW_VER_MINOR_G(c),
3356                FW_HDR_FW_VER_MICRO_G(c), FW_HDR_FW_VER_BUILD_G(c), reason,
3357                FW_HDR_FW_VER_MAJOR_G(k), FW_HDR_FW_VER_MINOR_G(k),
3358                FW_HDR_FW_VER_MICRO_G(k), FW_HDR_FW_VER_BUILD_G(k));
3359
3360        return 1;
3361}
3362
3363int t4_prep_fw(struct adapter *adap, struct fw_info *fw_info,
3364               const u8 *fw_data, unsigned int fw_size,
3365               struct fw_hdr *card_fw, enum dev_state state,
3366               int *reset)
3367{
3368        int ret, card_fw_usable, fs_fw_usable;
3369        const struct fw_hdr *fs_fw;
3370        const struct fw_hdr *drv_fw;
3371
3372        drv_fw = &fw_info->fw_hdr;
3373
3374        /* Read the header of the firmware on the card */
3375        ret = -t4_read_flash(adap, FLASH_FW_START,
3376                            sizeof(*card_fw) / sizeof(uint32_t),
3377                            (uint32_t *)card_fw, 1);
3378        if (ret == 0) {
3379                card_fw_usable = fw_compatible(drv_fw, (const void *)card_fw);
3380        } else {
3381                dev_err(adap->pdev_dev,
3382                        "Unable to read card's firmware header: %d\n", ret);
3383                card_fw_usable = 0;
3384        }
3385
3386        if (fw_data != NULL) {
3387                fs_fw = (const void *)fw_data;
3388                fs_fw_usable = fw_compatible(drv_fw, fs_fw);
3389        } else {
3390                fs_fw = NULL;
3391                fs_fw_usable = 0;
3392        }
3393
3394        if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
3395            (!fs_fw_usable || fs_fw->fw_ver == drv_fw->fw_ver)) {
3396                /* Common case: the firmware on the card is an exact match and
3397                 * the filesystem one is an exact match too, or the filesystem
3398                 * one is absent/incompatible.
3399                 */
3400        } else if (fs_fw_usable && state == DEV_STATE_UNINIT &&
3401                   should_install_fs_fw(adap, card_fw_usable,
3402                                        be32_to_cpu(fs_fw->fw_ver),
3403                                        be32_to_cpu(card_fw->fw_ver))) {
3404                ret = -t4_fw_upgrade(adap, adap->mbox, fw_data,
3405                                     fw_size, 0);
3406                if (ret != 0) {
3407                        dev_err(adap->pdev_dev,
3408                                "failed to install firmware: %d\n", ret);
3409                        goto bye;
3410                }
3411
3412                /* Installed successfully, update the cached header too. */
3413                *card_fw = *fs_fw;
3414                card_fw_usable = 1;
3415                *reset = 0;     /* already reset as part of load_fw */
3416        }
3417
3418        if (!card_fw_usable) {
3419                uint32_t d, c, k;
3420
3421                d = be32_to_cpu(drv_fw->fw_ver);
3422                c = be32_to_cpu(card_fw->fw_ver);
3423                k = fs_fw ? be32_to_cpu(fs_fw->fw_ver) : 0;
3424
3425                dev_err(adap->pdev_dev, "Cannot find a usable firmware: "
3426                        "chip state %d, "
3427                        "driver compiled with %d.%d.%d.%d, "
3428                        "card has %d.%d.%d.%d, filesystem has %d.%d.%d.%d\n",
3429                        state,
3430                        FW_HDR_FW_VER_MAJOR_G(d), FW_HDR_FW_VER_MINOR_G(d),
3431                        FW_HDR_FW_VER_MICRO_G(d), FW_HDR_FW_VER_BUILD_G(d),
3432                        FW_HDR_FW_VER_MAJOR_G(c), FW_HDR_FW_VER_MINOR_G(c),
3433                        FW_HDR_FW_VER_MICRO_G(c), FW_HDR_FW_VER_BUILD_G(c),
3434                        FW_HDR_FW_VER_MAJOR_G(k), FW_HDR_FW_VER_MINOR_G(k),
3435                        FW_HDR_FW_VER_MICRO_G(k), FW_HDR_FW_VER_BUILD_G(k));
3436                ret = EINVAL;
3437                goto bye;
3438        }
3439
3440        /* We're using whatever's on the card and it's known to be good. */
3441        adap->params.fw_vers = be32_to_cpu(card_fw->fw_ver);
3442        adap->params.tp_vers = be32_to_cpu(card_fw->tp_microcode_ver);
3443
3444bye:
3445        return ret;
3446}
3447
3448/**
3449 *      t4_flash_erase_sectors - erase a range of flash sectors
3450 *      @adapter: the adapter
3451 *      @start: the first sector to erase
3452 *      @end: the last sector to erase
3453 *
3454 *      Erases the sectors in the given inclusive range.
3455 */
3456static int t4_flash_erase_sectors(struct adapter *adapter, int start, int end)
3457{
3458        int ret = 0;
3459
3460        if (end >= adapter->params.sf_nsec)
3461                return -EINVAL;
3462
3463        while (start <= end) {
3464                if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
3465                    (ret = sf1_write(adapter, 4, 0, 1,
3466                                     SF_ERASE_SECTOR | (start << 8))) != 0 ||
3467                    (ret = flash_wait_op(adapter, 14, 500)) != 0) {
3468                        dev_err(adapter->pdev_dev,
3469                                "erase of flash sector %d failed, error %d\n",
3470                                start, ret);
3471                        break;
3472                }
3473                start++;
3474        }
3475        t4_write_reg(adapter, SF_OP_A, 0);    /* unlock SF */
3476        return ret;
3477}
3478
3479/**
3480 *      t4_flash_cfg_addr - return the address of the flash configuration file
3481 *      @adapter: the adapter
3482 *
3483 *      Return the address within the flash where the Firmware Configuration
3484 *      File is stored.
3485 */
3486unsigned int t4_flash_cfg_addr(struct adapter *adapter)
3487{
3488        if (adapter->params.sf_size == 0x100000)
3489                return FLASH_FPGA_CFG_START;
3490        else
3491                return FLASH_CFG_START;
3492}
3493
3494/* Return TRUE if the specified firmware matches the adapter.  I.e. T4
3495 * firmware for T4 adapters, T5 firmware for T5 adapters, etc.  We go ahead
3496 * and emit an error message for mismatched firmware to save our caller the
3497 * effort ...
3498 */
3499static bool t4_fw_matches_chip(const struct adapter *adap,
3500                               const struct fw_hdr *hdr)
3501{
3502        /* The expression below will return FALSE for any unsupported adapter
3503         * which will keep us "honest" in the future ...
3504         */
3505        if ((is_t4(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T4) ||
3506            (is_t5(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T5) ||
3507            (is_t6(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T6))
3508                return true;
3509
3510        dev_err(adap->pdev_dev,
3511                "FW image (%d) is not suitable for this adapter (%d)\n",
3512                hdr->chip, CHELSIO_CHIP_VERSION(adap->params.chip));
3513        return false;
3514}
3515
3516/**
3517 *      t4_load_fw - download firmware
3518 *      @adap: the adapter
3519 *      @fw_data: the firmware image to write
3520 *      @size: image size
3521 *
3522 *      Write the supplied firmware image to the card's serial flash.
3523 */
3524int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size)
3525{
3526        u32 csum;
3527        int ret, addr;
3528        unsigned int i;
3529        u8 first_page[SF_PAGE_SIZE];
3530        const __be32 *p = (const __be32 *)fw_data;
3531        const struct fw_hdr *hdr = (const struct fw_hdr *)fw_data;
3532        unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
3533        unsigned int fw_img_start = adap->params.sf_fw_start;
3534        unsigned int fw_start_sec = fw_img_start / sf_sec_size;
3535
3536        if (!size) {
3537                dev_err(adap->pdev_dev, "FW image has no data\n");
3538                return -EINVAL;
3539        }
3540        if (size & 511) {
3541                dev_err(adap->pdev_dev,
3542                        "FW image size not multiple of 512 bytes\n");
3543                return -EINVAL;
3544        }
3545        if ((unsigned int)be16_to_cpu(hdr->len512) * 512 != size) {
3546                dev_err(adap->pdev_dev,
3547                        "FW image size differs from size in FW header\n");
3548                return -EINVAL;
3549        }
3550        if (size > FW_MAX_SIZE) {
3551                dev_err(adap->pdev_dev, "FW image too large, max is %u bytes\n",
3552                        FW_MAX_SIZE);
3553                return -EFBIG;
3554        }
3555        if (!t4_fw_matches_chip(adap, hdr))
3556                return -EINVAL;
3557
3558        for (csum = 0, i = 0; i < size / sizeof(csum); i++)
3559                csum += be32_to_cpu(p[i]);
3560
3561        if (csum != 0xffffffff) {
3562                dev_err(adap->pdev_dev,
3563                        "corrupted firmware image, checksum %#x\n", csum);
3564                return -EINVAL;
3565        }
3566
3567        i = DIV_ROUND_UP(size, sf_sec_size);        /* # of sectors spanned */
3568        ret = t4_flash_erase_sectors(adap, fw_start_sec, fw_start_sec + i - 1);
3569        if (ret)
3570                goto out;
3571
3572        /*
3573         * We write the correct version at the end so the driver can see a bad
3574         * version if the FW write fails.  Start by writing a copy of the
3575         * first page with a bad version.
3576         */
3577        memcpy(first_page, fw_data, SF_PAGE_SIZE);
3578        ((struct fw_hdr *)first_page)->fw_ver = cpu_to_be32(0xffffffff);
3579        ret = t4_write_flash(adap, fw_img_start, SF_PAGE_SIZE, first_page);
3580        if (ret)
3581                goto out;
3582
3583        addr = fw_img_start;
3584        for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
3585                addr += SF_PAGE_SIZE;
3586                fw_data += SF_PAGE_SIZE;
3587                ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data);
3588                if (ret)
3589                        goto out;
3590        }
3591
3592        ret = t4_write_flash(adap,
3593                             fw_img_start + offsetof(struct fw_hdr, fw_ver),
3594                             sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver);
3595out:
3596        if (ret)
3597                dev_err(adap->pdev_dev, "firmware download failed, error %d\n",
3598                        ret);
3599        else
3600                ret = t4_get_fw_version(adap, &adap->params.fw_vers);
3601        return ret;
3602}
3603
3604/**
3605 *      t4_phy_fw_ver - return current PHY firmware version
3606 *      @adap: the adapter
3607 *      @phy_fw_ver: return value buffer for PHY firmware version
3608 *
3609 *      Returns the current version of external PHY firmware on the
3610 *      adapter.
3611 */
3612int t4_phy_fw_ver(struct adapter *adap, int *phy_fw_ver)
3613{
3614        u32 param, val;
3615        int ret;
3616
3617        param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3618                 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) |
3619                 FW_PARAMS_PARAM_Y_V(adap->params.portvec) |
3620                 FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_VERSION));
3621        ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
3622                              &param, &val);
3623        if (ret < 0)
3624                return ret;
3625        *phy_fw_ver = val;
3626        return 0;
3627}
3628
3629/**
3630 *      t4_load_phy_fw - download port PHY firmware
3631 *      @adap: the adapter
3632 *      @win: the PCI-E Memory Window index to use for t4_memory_rw()
3633 *      @win_lock: the lock to use to guard the memory copy
3634 *      @phy_fw_version: function to check PHY firmware versions
3635 *      @phy_fw_data: the PHY firmware image to write
3636 *      @phy_fw_size: image size
3637 *
3638 *      Transfer the specified PHY firmware to the adapter.  If a non-NULL
3639 *      @phy_fw_version is supplied, then it will be used to determine if
3640 *      it's necessary to perform the transfer by comparing the version
3641 *      of any existing adapter PHY firmware with that of the passed in
3642 *      PHY firmware image.  If @win_lock is non-NULL then it will be used
3643 *      around the call to t4_memory_rw() which transfers the PHY firmware
3644 *      to the adapter.
3645 *
3646 *      A negative error number will be returned if an error occurs.  If
3647 *      version number support is available and there's no need to upgrade
3648 *      the firmware, 0 will be returned.  If firmware is successfully
3649 *      transferred to the adapter, 1 will be retured.
3650 *
3651 *      NOTE: some adapters only have local RAM to store the PHY firmware.  As
3652 *      a result, a RESET of the adapter would cause that RAM to lose its
3653 *      contents.  Thus, loading PHY firmware on such adapters must happen
3654 *      after any FW_RESET_CMDs ...
3655 */
3656int t4_load_phy_fw(struct adapter *adap,
3657                   int win, spinlock_t *win_lock,
3658                   int (*phy_fw_version)(const u8 *, size_t),
3659                   const u8 *phy_fw_data, size_t phy_fw_size)
3660{
3661        unsigned long mtype = 0, maddr = 0;
3662        u32 param, val;
3663        int cur_phy_fw_ver = 0, new_phy_fw_vers = 0;
3664        int ret;
3665
3666        /* If we have version number support, then check to see if the adapter
3667         * already has up-to-date PHY firmware loaded.
3668         */
3669         if (phy_fw_version) {
3670                new_phy_fw_vers = phy_fw_version(phy_fw_data, phy_fw_size);
3671                ret = t4_phy_fw_ver(adap, &cur_phy_fw_ver);
3672                if (ret < 0)
3673                        return ret;
3674
3675                if (cur_phy_fw_ver >= new_phy_fw_vers) {
3676                        CH_WARN(adap, "PHY Firmware already up-to-date, "
3677                                "version %#x\n", cur_phy_fw_ver);
3678                        return 0;
3679                }
3680        }
3681
3682        /* Ask the firmware where it wants us to copy the PHY firmware image.
3683         * The size of the file requires a special version of the READ coommand
3684         * which will pass the file size via the values field in PARAMS_CMD and
3685         * retrieve the return value from firmware and place it in the same
3686         * buffer values
3687         */
3688        param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3689                 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) |
3690                 FW_PARAMS_PARAM_Y_V(adap->params.portvec) |
3691                 FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD));
3692        val = phy_fw_size;
3693        ret = t4_query_params_rw(adap, adap->mbox, adap->pf, 0, 1,
3694                                 &param, &val, 1, true);
3695        if (ret < 0)
3696                return ret;
3697        mtype = val >> 8;
3698        maddr = (val & 0xff) << 16;
3699
3700        /* Copy the supplied PHY Firmware image to the adapter memory location
3701         * allocated by the adapter firmware.
3702         */
3703        if (win_lock)
3704                spin_lock_bh(win_lock);
3705        ret = t4_memory_rw(adap, win, mtype, maddr,
3706                           phy_fw_size, (__be32 *)phy_fw_data,
3707                           T4_MEMORY_WRITE);
3708        if (win_lock)
3709                spin_unlock_bh(win_lock);
3710        if (ret)
3711                return ret;
3712
3713        /* Tell the firmware that the PHY firmware image has been written to
3714         * RAM and it can now start copying it over to the PHYs.  The chip
3715         * firmware will RESET the affected PHYs as part of this operation
3716         * leaving them running the new PHY firmware image.
3717         */
3718        param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3719                 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) |
3720                 FW_PARAMS_PARAM_Y_V(adap->params.portvec) |
3721                 FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD));
3722        ret = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1,
3723                                    &param, &val, 30000);
3724
3725        /* If we have version number support, then check to see that the new
3726         * firmware got loaded properly.
3727         */
3728        if (phy_fw_version) {
3729                ret = t4_phy_fw_ver(adap, &cur_phy_fw_ver);
3730                if (ret < 0)
3731                        return ret;
3732
3733                if (cur_phy_fw_ver != new_phy_fw_vers) {
3734                        CH_WARN(adap, "PHY Firmware did not update: "
3735                                "version on adapter %#x, "
3736                                "version flashed %#x\n",
3737                                cur_phy_fw_ver, new_phy_fw_vers);
3738                        return -ENXIO;
3739                }
3740        }
3741
3742        return 1;
3743}
3744
3745/**
3746 *      t4_fwcache - firmware cache operation
3747 *      @adap: the adapter
3748 *      @op  : the operation (flush or flush and invalidate)
3749 */
3750int t4_fwcache(struct adapter *adap, enum fw_params_param_dev_fwcache op)
3751{
3752        struct fw_params_cmd c;
3753
3754        memset(&c, 0, sizeof(c));
3755        c.op_to_vfn =
3756                cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) |
3757                            FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
3758                            FW_PARAMS_CMD_PFN_V(adap->pf) |
3759                            FW_PARAMS_CMD_VFN_V(0));
3760        c.retval_len16 = cpu_to_be32(FW_LEN16(c));
3761        c.param[0].mnem =
3762                cpu_to_be32(FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3763                            FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_FWCACHE));
3764        c.param[0].val = (__force __be32)op;
3765
3766        return t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), NULL);
3767}
3768
3769void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp,
3770                        unsigned int *pif_req_wrptr,
3771                        unsigned int *pif_rsp_wrptr)
3772{
3773        int i, j;
3774        u32 cfg, val, req, rsp;
3775
3776        cfg = t4_read_reg(adap, CIM_DEBUGCFG_A);
3777        if (cfg & LADBGEN_F)
3778                t4_write_reg(adap, CIM_DEBUGCFG_A, cfg ^ LADBGEN_F);
3779
3780        val = t4_read_reg(adap, CIM_DEBUGSTS_A);
3781        req = POLADBGWRPTR_G(val);
3782        rsp = PILADBGWRPTR_G(val);
3783        if (pif_req_wrptr)
3784                *pif_req_wrptr = req;
3785        if (pif_rsp_wrptr)
3786                *pif_rsp_wrptr = rsp;
3787
3788        for (i = 0; i < CIM_PIFLA_SIZE; i++) {
3789                for (j = 0; j < 6; j++) {
3790                        t4_write_reg(adap, CIM_DEBUGCFG_A, POLADBGRDPTR_V(req) |
3791                                     PILADBGRDPTR_V(rsp));
3792                        *pif_req++ = t4_read_reg(adap, CIM_PO_LA_DEBUGDATA_A);
3793                        *pif_rsp++ = t4_read_reg(adap, CIM_PI_LA_DEBUGDATA_A);
3794                        req++;
3795                        rsp++;
3796                }
3797                req = (req + 2) & POLADBGRDPTR_M;
3798                rsp = (rsp + 2) & PILADBGRDPTR_M;
3799        }
3800        t4_write_reg(adap, CIM_DEBUGCFG_A, cfg);
3801}
3802
3803void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp)
3804{
3805        u32 cfg;
3806        int i, j, idx;
3807
3808        cfg = t4_read_reg(adap, CIM_DEBUGCFG_A);
3809        if (cfg & LADBGEN_F)
3810                t4_write_reg(adap, CIM_DEBUGCFG_A, cfg ^ LADBGEN_F);
3811
3812        for (i = 0; i < CIM_MALA_SIZE; i++) {
3813                for (j = 0; j < 5; j++) {
3814                        idx = 8 * i + j;
3815                        t4_write_reg(adap, CIM_DEBUGCFG_A, POLADBGRDPTR_V(idx) |
3816                                     PILADBGRDPTR_V(idx));
3817                        *ma_req++ = t4_read_reg(adap, CIM_PO_LA_MADEBUGDATA_A);
3818                        *ma_rsp++ = t4_read_reg(adap, CIM_PI_LA_MADEBUGDATA_A);
3819                }
3820        }
3821        t4_write_reg(adap, CIM_DEBUGCFG_A, cfg);
3822}
3823
3824void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf)
3825{
3826        unsigned int i, j;
3827
3828        for (i = 0; i < 8; i++) {
3829                u32 *p = la_buf + i;
3830
3831                t4_write_reg(adap, ULP_RX_LA_CTL_A, i);
3832                j = t4_read_reg(adap, ULP_RX_LA_WRPTR_A);
3833                t4_write_reg(adap, ULP_RX_LA_RDPTR_A, j);
3834                for (j = 0; j < ULPRX_LA_SIZE; j++, p += 8)
3835                        *p = t4_read_reg(adap, ULP_RX_LA_RDDATA_A);
3836        }
3837}
3838
3839#define ADVERT_MASK (FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G |\
3840                     FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_25G | \
3841                     FW_PORT_CAP_SPEED_40G | FW_PORT_CAP_SPEED_100G | \
3842                     FW_PORT_CAP_ANEG)
3843
3844/**
3845 *      t4_link_l1cfg - apply link configuration to MAC/PHY
3846 *      @phy: the PHY to setup
3847 *      @mac: the MAC to setup
3848 *      @lc: the requested link configuration
3849 *
3850 *      Set up a port's MAC and PHY according to a desired link configuration.
3851 *      - If the PHY can auto-negotiate first decide what to advertise, then
3852 *        enable/disable auto-negotiation as desired, and reset.
3853 *      - If the PHY does not auto-negotiate just reset it.
3854 *      - If auto-negotiation is off set the MAC to the proper speed/duplex/FC,
3855 *        otherwise do it later based on the outcome of auto-negotiation.
3856 */
3857int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port,
3858                  struct link_config *lc)
3859{
3860        struct fw_port_cmd c;
3861        unsigned int mdi = FW_PORT_CAP_MDI_V(FW_PORT_CAP_MDI_AUTO);
3862        unsigned int fc = 0, fec = 0, fw_fec = 0;
3863
3864        lc->link_ok = 0;
3865        if (lc->requested_fc & PAUSE_RX)
3866                fc |= FW_PORT_CAP_FC_RX;
3867        if (lc->requested_fc & PAUSE_TX)
3868                fc |= FW_PORT_CAP_FC_TX;
3869
3870        fec = lc->requested_fec & FEC_AUTO ? lc->auto_fec : lc->requested_fec;
3871
3872        if (fec & FEC_RS)
3873                fw_fec |= FW_PORT_CAP_FEC_RS;
3874        if (fec & FEC_BASER_RS)
3875                fw_fec |= FW_PORT_CAP_FEC_BASER_RS;
3876
3877        memset(&c, 0, sizeof(c));
3878        c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
3879                                     FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
3880                                     FW_PORT_CMD_PORTID_V(port));
3881        c.action_to_len16 =
3882                cpu_to_be32(FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_L1_CFG) |
3883                            FW_LEN16(c));
3884
3885        if (!(lc->supported & FW_PORT_CAP_ANEG)) {
3886                c.u.l1cfg.rcap = cpu_to_be32((lc->supported & ADVERT_MASK) |
3887                                             fc | fw_fec);
3888                lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
3889        } else if (lc->autoneg == AUTONEG_DISABLE) {
3890                c.u.l1cfg.rcap = cpu_to_be32(lc->requested_speed | fc |
3891                                             fw_fec | mdi);
3892                lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
3893        } else
3894                c.u.l1cfg.rcap = cpu_to_be32(lc->advertising | fc |
3895                                             fw_fec | mdi);
3896
3897        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
3898}
3899
3900/**
3901 *      t4_restart_aneg - restart autonegotiation
3902 *      @adap: the adapter
3903 *      @mbox: mbox to use for the FW command
3904 *      @port: the port id
3905 *
3906 *      Restarts autonegotiation for the selected port.
3907 */
3908int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port)
3909{
3910        struct fw_port_cmd c;
3911
3912        memset(&c, 0, sizeof(c));
3913        c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
3914                                     FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
3915                                     FW_PORT_CMD_PORTID_V(port));
3916        c.action_to_len16 =
3917                cpu_to_be32(FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_L1_CFG) |
3918                            FW_LEN16(c));
3919        c.u.l1cfg.rcap = cpu_to_be32(FW_PORT_CAP_ANEG);
3920        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
3921}
3922
3923typedef void (*int_handler_t)(struct adapter *adap);
3924
3925struct intr_info {
3926        unsigned int mask;       /* bits to check in interrupt status */
3927        const char *msg;         /* message to print or NULL */
3928        short stat_idx;          /* stat counter to increment or -1 */
3929        unsigned short fatal;    /* whether the condition reported is fatal */
3930        int_handler_t int_handler; /* platform-specific int handler */
3931};
3932
3933/**
3934 *      t4_handle_intr_status - table driven interrupt handler
3935 *      @adapter: the adapter that generated the interrupt
3936 *      @reg: the interrupt status register to process
3937 *      @acts: table of interrupt actions
3938 *
3939 *      A table driven interrupt handler that applies a set of masks to an
3940 *      interrupt status word and performs the corresponding actions if the
3941 *      interrupts described by the mask have occurred.  The actions include
3942 *      optionally emitting a warning or alert message.  The table is terminated
3943 *      by an entry specifying mask 0.  Returns the number of fatal interrupt
3944 *      conditions.
3945 */
3946static int t4_handle_intr_status(struct adapter *adapter, unsigned int reg,
3947                                 const struct intr_info *acts)
3948{
3949        int fatal = 0;
3950        unsigned int mask = 0;
3951        unsigned int status = t4_read_reg(adapter, reg);
3952
3953        for ( ; acts->mask; ++acts) {
3954                if (!(status & acts->mask))
3955                        continue;
3956                if (acts->fatal) {
3957                        fatal++;
3958                        dev_alert(adapter->pdev_dev, "%s (0x%x)\n", acts->msg,
3959                                  status & acts->mask);
3960                } else if (acts->msg && printk_ratelimit())
3961                        dev_warn(adapter->pdev_dev, "%s (0x%x)\n", acts->msg,
3962                                 status & acts->mask);
3963                if (acts->int_handler)
3964                        acts->int_handler(adapter);
3965                mask |= acts->mask;
3966        }
3967        status &= mask;
3968        if (status)                           /* clear processed interrupts */
3969                t4_write_reg(adapter, reg, status);
3970        return fatal;
3971}
3972
3973/*
3974 * Interrupt handler for the PCIE module.
3975 */
3976static void pcie_intr_handler(struct adapter *adapter)
3977{
3978        static const struct intr_info sysbus_intr_info[] = {
3979                { RNPP_F, "RXNP array parity error", -1, 1 },
3980                { RPCP_F, "RXPC array parity error", -1, 1 },
3981                { RCIP_F, "RXCIF array parity error", -1, 1 },
3982                { RCCP_F, "Rx completions control array parity error", -1, 1 },
3983                { RFTP_F, "RXFT array parity error", -1, 1 },
3984                { 0 }
3985        };
3986        static const struct intr_info pcie_port_intr_info[] = {
3987                { TPCP_F, "TXPC array parity error", -1, 1 },
3988                { TNPP_F, "TXNP array parity error", -1, 1 },
3989                { TFTP_F, "TXFT array parity error", -1, 1 },
3990                { TCAP_F, "TXCA array parity error", -1, 1 },
3991                { TCIP_F, "TXCIF array parity error", -1, 1 },
3992                { RCAP_F, "RXCA array parity error", -1, 1 },
3993                { OTDD_F, "outbound request TLP discarded", -1, 1 },
3994                { RDPE_F, "Rx data parity error", -1, 1 },
3995                { TDUE_F, "Tx uncorrectable data error", -1, 1 },
3996                { 0 }
3997        };
3998        static const struct intr_info pcie_intr_info[] = {
3999                { MSIADDRLPERR_F, "MSI AddrL parity error", -1, 1 },
4000                { MSIADDRHPERR_F, "MSI AddrH parity error", -1, 1 },

4001                { MSIDATAPERR_F, "MSI data parity error", -1, 1 },
4002                { MSIXADDRLPERR_F, "MSI-X AddrL parity error", -1, 1 },
4003                { MSIXADDRHPERR_F, "MSI-X AddrH parity error", -1, 1 },
4004                { MSIXDATAPERR_F, "MSI-X data parity error", -1, 1 },
4005                { MSIXDIPERR_F, "MSI-X DI parity error", -1, 1 },
4006                { PIOCPLPERR_F, "PCI PIO completion FIFO parity error", -1, 1 },
4007                { PIOREQPERR_F, "PCI PIO request FIFO parity error", -1, 1 },
4008                { TARTAGPERR_F, "PCI PCI target tag FIFO parity error", -1, 1 },
4009                { CCNTPERR_F, "PCI CMD channel count parity error", -1, 1 },
4010                { CREQPERR_F, "PCI CMD channel request parity error", -1, 1 },
4011                { CRSPPERR_F, "PCI CMD channel response parity error", -1, 1 },
4012                { DCNTPERR_F, "PCI DMA channel count parity error", -1, 1 },
4013                { DREQPERR_F, "PCI DMA channel request parity error", -1, 1 },
4014                { DRSPPERR_F, "PCI DMA channel response parity error", -1, 1 },
4015                { HCNTPERR_F, "PCI HMA channel count parity error", -1, 1 },
4016                { HREQPERR_F, "PCI HMA channel request parity error", -1, 1 },
4017                { HRSPPERR_F, "PCI HMA channel response parity error", -1, 1 },
4018                { CFGSNPPERR_F, "PCI config snoop FIFO parity error", -1, 1 },
4019                { FIDPERR_F, "PCI FID parity error", -1, 1 },
4020                { INTXCLRPERR_F, "PCI INTx clear parity error", -1, 1 },
4021                { MATAGPERR_F, "PCI MA tag parity error", -1, 1 },
4022                { PIOTAGPERR_F, "PCI PIO tag parity error", -1, 1 },
4023                { RXCPLPERR_F, "PCI Rx completion parity error", -1, 1 },
4024                { RXWRPERR_F, "PCI Rx write parity error", -1, 1 },
4025                { RPLPERR_F, "PCI replay buffer parity error", -1, 1 },
4026                { PCIESINT_F, "PCI core secondary fault", -1, 1 },
4027                { PCIEPINT_F, "PCI core primary fault", -1, 1 },
4028                { UNXSPLCPLERR_F, "PCI unexpected split completion error",
4029                  -1, 0 },
4030                { 0 }
4031        };
4032
4033        static struct intr_info t5_pcie_intr_info[] = {
4034                { MSTGRPPERR_F, "Master Response Read Queue parity error",
4035                  -1, 1 },
4036                { MSTTIMEOUTPERR_F, "Master Timeout FIFO parity error", -1, 1 },
4037                { MSIXSTIPERR_F, "MSI-X STI SRAM parity error", -1, 1 },
4038                { MSIXADDRLPERR_F, "MSI-X AddrL parity error", -1, 1 },
4039                { MSIXADDRHPERR_F, "MSI-X AddrH parity error", -1, 1 },
4040                { MSIXDATAPERR_F, "MSI-X data parity error", -1, 1 },
4041                { MSIXDIPERR_F, "MSI-X DI parity error", -1, 1 },
4042                { PIOCPLGRPPERR_F, "PCI PIO completion Group FIFO parity error",
4043                  -1, 1 },
4044                { PIOREQGRPPERR_F, "PCI PIO request Group FIFO parity error",
4045                  -1, 1 },
4046                { TARTAGPERR_F, "PCI PCI target tag FIFO parity error", -1, 1 },
4047                { MSTTAGQPERR_F, "PCI master tag queue parity error", -1, 1 },
4048                { CREQPERR_F, "PCI CMD channel request parity error", -1, 1 },
4049                { CRSPPERR_F, "PCI CMD channel response parity error", -1, 1 },
4050                { DREQWRPERR_F, "PCI DMA channel write request parity error",
4051                  -1, 1 },
4052                { DREQPERR_F, "PCI DMA channel request parity error", -1, 1 },
4053                { DRSPPERR_F, "PCI DMA channel response parity error", -1, 1 },
4054                { HREQWRPERR_F, "PCI HMA channel count parity error", -1, 1 },
4055                { HREQPERR_F, "PCI HMA channel request parity error", -1, 1 },
4056                { HRSPPERR_F, "PCI HMA channel response parity error", -1, 1 },
4057                { CFGSNPPERR_F, "PCI config snoop FIFO parity error", -1, 1 },
4058                { FIDPERR_F, "PCI FID parity error", -1, 1 },
4059                { VFIDPERR_F, "PCI INTx clear parity error", -1, 1 },
4060                { MAGRPPERR_F, "PCI MA group FIFO parity error", -1, 1 },
4061                { PIOTAGPERR_F, "PCI PIO tag parity error", -1, 1 },
4062                { IPRXHDRGRPPERR_F, "PCI IP Rx header group parity error",
4063                  -1, 1 },
4064                { IPRXDATAGRPPERR_F, "PCI IP Rx data group parity error",
4065                  -1, 1 },
4066                { RPLPERR_F, "PCI IP replay buffer parity error", -1, 1 },
4067                { IPSOTPERR_F, "PCI IP SOT buffer parity error", -1, 1 },
4068                { TRGT1GRPPERR_F, "PCI TRGT1 group FIFOs parity error", -1, 1 },
4069                { READRSPERR_F, "Outbound read error", -1, 0 },
4070                { 0 }
4071        };
4072
4073        int fat;
4074
4075        if (is_t4(adapter->params.chip))
4076                fat = t4_handle_intr_status(adapter,
4077                                PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS_A,
4078                                sysbus_intr_info) +
4079                        t4_handle_intr_status(adapter,
4080                                        PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS_A,
4081                                        pcie_port_intr_info) +
4082                        t4_handle_intr_status(adapter, PCIE_INT_CAUSE_A,
4083                                              pcie_intr_info);
4084        else
4085                fat = t4_handle_intr_status(adapter, PCIE_INT_CAUSE_A,
4086                                            t5_pcie_intr_info);
4087
4088        if (fat)
4089                t4_fatal_err(adapter);
4090}
4091
4092/*
4093 * TP interrupt handler.
4094 */
4095static void tp_intr_handler(struct adapter *adapter)
4096{
4097        static const struct intr_info tp_intr_info[] = {
4098                { 0x3fffffff, "TP parity error", -1, 1 },
4099                { FLMTXFLSTEMPTY_F, "TP out of Tx pages", -1, 1 },
4100                { 0 }
4101        };
4102
4103        if (t4_handle_intr_status(adapter, TP_INT_CAUSE_A, tp_intr_info))
4104                t4_fatal_err(adapter);
4105}
4106
4107/*
4108 * SGE interrupt handler.
4109 */
4110static void sge_intr_handler(struct adapter *adapter)
4111{
4112        u64 v;
4113        u32 err;
4114
4115        static const struct intr_info sge_intr_info[] = {
4116                { ERR_CPL_EXCEED_IQE_SIZE_F,
4117                  "SGE received CPL exceeding IQE size", -1, 1 },
4118                { ERR_INVALID_CIDX_INC_F,
4119                  "SGE GTS CIDX increment too large", -1, 0 },
4120                { ERR_CPL_OPCODE_0_F, "SGE received 0-length CPL", -1, 0 },
4121                { DBFIFO_LP_INT_F, NULL, -1, 0, t4_db_full },
4122                { ERR_DATA_CPL_ON_HIGH_QID1_F | ERR_DATA_CPL_ON_HIGH_QID0_F,
4123                  "SGE IQID > 1023 received CPL for FL", -1, 0 },
4124                { ERR_BAD_DB_PIDX3_F, "SGE DBP 3 pidx increment too large", -1,
4125                  0 },
4126                { ERR_BAD_DB_PIDX2_F, "SGE DBP 2 pidx increment too large", -1,
4127                  0 },
4128                { ERR_BAD_DB_PIDX1_F, "SGE DBP 1 pidx increment too large", -1,
4129                  0 },
4130                { ERR_BAD_DB_PIDX0_F, "SGE DBP 0 pidx increment too large", -1,
4131                  0 },
4132                { ERR_ING_CTXT_PRIO_F,
4133                  "SGE too many priority ingress contexts", -1, 0 },
4134                { INGRESS_SIZE_ERR_F, "SGE illegal ingress QID", -1, 0 },
4135                { EGRESS_SIZE_ERR_F, "SGE illegal egress QID", -1, 0 },
4136                { 0 }
4137        };
4138
4139        static struct intr_info t4t5_sge_intr_info[] = {
4140                { ERR_DROPPED_DB_F, NULL, -1, 0, t4_db_dropped },
4141                { DBFIFO_HP_INT_F, NULL, -1, 0, t4_db_full },
4142                { ERR_EGR_CTXT_PRIO_F,
4143                  "SGE too many priority egress contexts", -1, 0 },
4144                { 0 }
4145        };
4146
4147        v = (u64)t4_read_reg(adapter, SGE_INT_CAUSE1_A) |
4148                ((u64)t4_read_reg(adapter, SGE_INT_CAUSE2_A) << 32);
4149        if (v) {
4150                dev_alert(adapter->pdev_dev, "SGE parity error (%#llx)\n",
4151                                (unsigned long long)v);
4152                t4_write_reg(adapter, SGE_INT_CAUSE1_A, v);
4153                t4_write_reg(adapter, SGE_INT_CAUSE2_A, v >> 32);
4154        }
4155
4156        v |= t4_handle_intr_status(adapter, SGE_INT_CAUSE3_A, sge_intr_info);
4157        if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
4158                v |= t4_handle_intr_status(adapter, SGE_INT_CAUSE3_A,
4159                                           t4t5_sge_intr_info);
4160
4161        err = t4_read_reg(adapter, SGE_ERROR_STATS_A);
4162        if (err & ERROR_QID_VALID_F) {
4163                dev_err(adapter->pdev_dev, "SGE error for queue %u\n",
4164                        ERROR_QID_G(err));
4165                if (err & UNCAPTURED_ERROR_F)
4166                        dev_err(adapter->pdev_dev,
4167                                "SGE UNCAPTURED_ERROR set (clearing)\n");
4168                t4_write_reg(adapter, SGE_ERROR_STATS_A, ERROR_QID_VALID_F |
4169                             UNCAPTURED_ERROR_F);
4170        }
4171
4172        if (v != 0)
4173                t4_fatal_err(adapter);
4174}
4175
4176#define CIM_OBQ_INTR (OBQULP0PARERR_F | OBQULP1PARERR_F | OBQULP2PARERR_F |\
4177                      OBQULP3PARERR_F | OBQSGEPARERR_F | OBQNCSIPARERR_F)
4178#define CIM_IBQ_INTR (IBQTP0PARERR_F | IBQTP1PARERR_F | IBQULPPARERR_F |\
4179                      IBQSGEHIPARERR_F | IBQSGELOPARERR_F | IBQNCSIPARERR_F)
4180
4181/*
4182 * CIM interrupt handler.
4183 */
4184static void cim_intr_handler(struct adapter *adapter)
4185{
4186        static const struct intr_info cim_intr_info[] = {
4187                { PREFDROPINT_F, "CIM control register prefetch drop", -1, 1 },
4188                { CIM_OBQ_INTR, "CIM OBQ parity error", -1, 1 },
4189                { CIM_IBQ_INTR, "CIM IBQ parity error", -1, 1 },
4190                { MBUPPARERR_F, "CIM mailbox uP parity error", -1, 1 },
4191                { MBHOSTPARERR_F, "CIM mailbox host parity error", -1, 1 },
4192                { TIEQINPARERRINT_F, "CIM TIEQ outgoing parity error", -1, 1 },
4193                { TIEQOUTPARERRINT_F, "CIM TIEQ incoming parity error", -1, 1 },
4194                { TIMER0INT_F, "CIM TIMER0 interrupt", -1, 1 },
4195                { 0 }
4196        };
4197        static const struct intr_info cim_upintr_info[] = {
4198                { RSVDSPACEINT_F, "CIM reserved space access", -1, 1 },
4199                { ILLTRANSINT_F, "CIM illegal transaction", -1, 1 },
4200                { ILLWRINT_F, "CIM illegal write", -1, 1 },
4201                { ILLRDINT_F, "CIM illegal read", -1, 1 },
4202                { ILLRDBEINT_F, "CIM illegal read BE", -1, 1 },
4203                { ILLWRBEINT_F, "CIM illegal write BE", -1, 1 },
4204                { SGLRDBOOTINT_F, "CIM single read from boot space", -1, 1 },
4205                { SGLWRBOOTINT_F, "CIM single write to boot space", -1, 1 },
4206                { BLKWRBOOTINT_F, "CIM block write to boot space", -1, 1 },
4207                { SGLRDFLASHINT_F, "CIM single read from flash space", -1, 1 },
4208                { SGLWRFLASHINT_F, "CIM single write to flash space", -1, 1 },
4209                { BLKWRFLASHINT_F, "CIM block write to flash space", -1, 1 },
4210                { SGLRDEEPROMINT_F, "CIM single EEPROM read", -1, 1 },
4211                { SGLWREEPROMINT_F, "CIM single EEPROM write", -1, 1 },
4212                { BLKRDEEPROMINT_F, "CIM block EEPROM read", -1, 1 },
4213                { BLKWREEPROMINT_F, "CIM block EEPROM write", -1, 1 },
4214                { SGLRDCTLINT_F, "CIM single read from CTL space", -1, 1 },
4215                { SGLWRCTLINT_F, "CIM single write to CTL space", -1, 1 },
4216                { BLKRDCTLINT_F, "CIM block read from CTL space", -1, 1 },
4217                { BLKWRCTLINT_F, "CIM block write to CTL space", -1, 1 },
4218                { SGLRDPLINT_F, "CIM single read from PL space", -1, 1 },
4219                { SGLWRPLINT_F, "CIM single write to PL space", -1, 1 },
4220                { BLKRDPLINT_F, "CIM block read from PL space", -1, 1 },
4221                { BLKWRPLINT_F, "CIM block write to PL space", -1, 1 },
4222                { REQOVRLOOKUPINT_F, "CIM request FIFO overwrite", -1, 1 },
4223                { RSPOVRLOOKUPINT_F, "CIM response FIFO overwrite", -1, 1 },
4224                { TIMEOUTINT_F, "CIM PIF timeout", -1, 1 },
4225                { TIMEOUTMAINT_F, "CIM PIF MA timeout", -1, 1 },
4226                { 0 }
4227        };
4228
4229        u32 val, fw_err;
4230        int fat;
4231
4232        fw_err = t4_read_reg(adapter, PCIE_FW_A);
4233        if (fw_err & PCIE_FW_ERR_F)
4234                t4_report_fw_error(adapter);
4235
4236        /* When the Firmware detects an internal error which normally
4237         * wouldn't raise a Host Interrupt, it forces a CIM Timer0 interrupt
4238         * in order to make sure the Host sees the Firmware Crash.  So
4239         * if we have a Timer0 interrupt and don't see a Firmware Crash,
4240         * ignore the Timer0 interrupt.
4241         */
4242
4243        val = t4_read_reg(adapter, CIM_HOST_INT_CAUSE_A);
4244        if (val & TIMER0INT_F)
4245                if (!(fw_err & PCIE_FW_ERR_F) ||
4246                    (PCIE_FW_EVAL_G(fw_err) != PCIE_FW_EVAL_CRASH))
4247                        t4_write_reg(adapter, CIM_HOST_INT_CAUSE_A,
4248                                     TIMER0INT_F);
4249
4250        fat = t4_handle_intr_status(adapter, CIM_HOST_INT_CAUSE_A,
4251                                    cim_intr_info) +
4252              t4_handle_intr_status(adapter, CIM_HOST_UPACC_INT_CAUSE_A,
4253                                    cim_upintr_info);
4254        if (fat)
4255                t4_fatal_err(adapter);
4256}
4257
4258/*
4259 * ULP RX interrupt handler.
4260 */
4261static void ulprx_intr_handler(struct adapter *adapter)
4262{
4263        static const struct intr_info ulprx_intr_info[] = {
4264                { 0x1800000, "ULPRX context error", -1, 1 },
4265                { 0x7fffff, "ULPRX parity error", -1, 1 },
4266                { 0 }
4267        };
4268
4269        if (t4_handle_intr_status(adapter, ULP_RX_INT_CAUSE_A, ulprx_intr_info))
4270                t4_fatal_err(adapter);
4271}
4272
4273/*
4274 * ULP TX interrupt handler.
4275 */
4276static void ulptx_intr_handler(struct adapter *adapter)
4277{
4278        static const struct intr_info ulptx_intr_info[] = {
4279                { PBL_BOUND_ERR_CH3_F, "ULPTX channel 3 PBL out of bounds", -1,
4280                  0 },
4281                { PBL_BOUND_ERR_CH2_F, "ULPTX channel 2 PBL out of bounds", -1,
4282                  0 },
4283                { PBL_BOUND_ERR_CH1_F, "ULPTX channel 1 PBL out of bounds", -1,
4284                  0 },
4285                { PBL_BOUND_ERR_CH0_F, "ULPTX channel 0 PBL out of bounds", -1,
4286                  0 },
4287                { 0xfffffff, "ULPTX parity error", -1, 1 },
4288                { 0 }
4289        };
4290
4291        if (t4_handle_intr_status(adapter, ULP_TX_INT_CAUSE_A, ulptx_intr_info))
4292                t4_fatal_err(adapter);
4293}
4294
4295/*
4296 * PM TX interrupt handler.
4297 */
4298static void pmtx_intr_handler(struct adapter *adapter)
4299{
4300        static const struct intr_info pmtx_intr_info[] = {
4301                { PCMD_LEN_OVFL0_F, "PMTX channel 0 pcmd too large", -1, 1 },
4302                { PCMD_LEN_OVFL1_F, "PMTX channel 1 pcmd too large", -1, 1 },
4303                { PCMD_LEN_OVFL2_F, "PMTX channel 2 pcmd too large", -1, 1 },
4304                { ZERO_C_CMD_ERROR_F, "PMTX 0-length pcmd", -1, 1 },
4305                { PMTX_FRAMING_ERROR_F, "PMTX framing error", -1, 1 },
4306                { OESPI_PAR_ERROR_F, "PMTX oespi parity error", -1, 1 },
4307                { DB_OPTIONS_PAR_ERROR_F, "PMTX db_options parity error",
4308                  -1, 1 },
4309                { ICSPI_PAR_ERROR_F, "PMTX icspi parity error", -1, 1 },
4310                { PMTX_C_PCMD_PAR_ERROR_F, "PMTX c_pcmd parity error", -1, 1},
4311                { 0 }
4312        };
4313
4314        if (t4_handle_intr_status(adapter, PM_TX_INT_CAUSE_A, pmtx_intr_info))
4315                t4_fatal_err(adapter);
4316}
4317
4318/*
4319 * PM RX interrupt handler.
4320 */
4321static void pmrx_intr_handler(struct adapter *adapter)
4322{
4323        static const struct intr_info pmrx_intr_info[] = {
4324                { ZERO_E_CMD_ERROR_F, "PMRX 0-length pcmd", -1, 1 },
4325                { PMRX_FRAMING_ERROR_F, "PMRX framing error", -1, 1 },
4326                { OCSPI_PAR_ERROR_F, "PMRX ocspi parity error", -1, 1 },
4327                { DB_OPTIONS_PAR_ERROR_F, "PMRX db_options parity error",
4328                  -1, 1 },
4329                { IESPI_PAR_ERROR_F, "PMRX iespi parity error", -1, 1 },
4330                { PMRX_E_PCMD_PAR_ERROR_F, "PMRX e_pcmd parity error", -1, 1},
4331                { 0 }
4332        };
4333
4334        if (t4_handle_intr_status(adapter, PM_RX_INT_CAUSE_A, pmrx_intr_info))
4335                t4_fatal_err(adapter);
4336}
4337
4338/*
4339 * CPL switch interrupt handler.
4340 */
4341static void cplsw_intr_handler(struct adapter *adapter)
4342{
4343        static const struct intr_info cplsw_intr_info[] = {
4344                { CIM_OP_MAP_PERR_F, "CPLSW CIM op_map parity error", -1, 1 },
4345                { CIM_OVFL_ERROR_F, "CPLSW CIM overflow", -1, 1 },
4346                { TP_FRAMING_ERROR_F, "CPLSW TP framing error", -1, 1 },
4347                { SGE_FRAMING_ERROR_F, "CPLSW SGE framing error", -1, 1 },
4348                { CIM_FRAMING_ERROR_F, "CPLSW CIM framing error", -1, 1 },
4349                { ZERO_SWITCH_ERROR_F, "CPLSW no-switch error", -1, 1 },
4350                { 0 }
4351        };
4352
4353        if (t4_handle_intr_status(adapter, CPL_INTR_CAUSE_A, cplsw_intr_info))
4354                t4_fatal_err(adapter);
4355}
4356
4357/*
4358 * LE interrupt handler.
4359 */
4360static void le_intr_handler(struct adapter *adap)
4361{
4362        enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip);
4363        static const struct intr_info le_intr_info[] = {
4364                { LIPMISS_F, "LE LIP miss", -1, 0 },
4365                { LIP0_F, "LE 0 LIP error", -1, 0 },
4366                { PARITYERR_F, "LE parity error", -1, 1 },
4367                { UNKNOWNCMD_F, "LE unknown command", -1, 1 },
4368                { REQQPARERR_F, "LE request queue parity error", -1, 1 },
4369                { 0 }
4370        };
4371
4372        static struct intr_info t6_le_intr_info[] = {
4373                { T6_LIPMISS_F, "LE LIP miss", -1, 0 },
4374                { T6_LIP0_F, "LE 0 LIP error", -1, 0 },
4375                { TCAMINTPERR_F, "LE parity error", -1, 1 },
4376                { T6_UNKNOWNCMD_F, "LE unknown command", -1, 1 },
4377                { SSRAMINTPERR_F, "LE request queue parity error", -1, 1 },
4378                { 0 }
4379        };
4380
4381        if (t4_handle_intr_status(adap, LE_DB_INT_CAUSE_A,
4382                                  (chip <= CHELSIO_T5) ?
4383                                  le_intr_info : t6_le_intr_info))
4384                t4_fatal_err(adap);
4385}
4386
4387/*
4388 * MPS interrupt handler.
4389 */
4390static void mps_intr_handler(struct adapter *adapter)
4391{
4392        static const struct intr_info mps_rx_intr_info[] = {
4393                { 0xffffff, "MPS Rx parity error", -1, 1 },
4394                { 0 }
4395        };
4396        static const struct intr_info mps_tx_intr_info[] = {
4397                { TPFIFO_V(TPFIFO_M), "MPS Tx TP FIFO parity error", -1, 1 },
4398                { NCSIFIFO_F, "MPS Tx NC-SI FIFO parity error", -1, 1 },
4399                { TXDATAFIFO_V(TXDATAFIFO_M), "MPS Tx data FIFO parity error",
4400                  -1, 1 },
4401                { TXDESCFIFO_V(TXDESCFIFO_M), "MPS Tx desc FIFO parity error",
4402                  -1, 1 },
4403                { BUBBLE_F, "MPS Tx underflow", -1, 1 },
4404                { SECNTERR_F, "MPS Tx SOP/EOP error", -1, 1 },
4405                { FRMERR_F, "MPS Tx framing error", -1, 1 },
4406                { 0 }
4407        };
4408        static const struct intr_info t6_mps_tx_intr_info[] = {
4409                { TPFIFO_V(TPFIFO_M), "MPS Tx TP FIFO parity error", -1, 1 },
4410                { NCSIFIFO_F, "MPS Tx NC-SI FIFO parity error", -1, 1 },
4411                { TXDATAFIFO_V(TXDATAFIFO_M), "MPS Tx data FIFO parity error",
4412                  -1, 1 },
4413                { TXDESCFIFO_V(TXDESCFIFO_M), "MPS Tx desc FIFO parity error",
4414                  -1, 1 },
4415                /* MPS Tx Bubble is normal for T6 */
4416                { SECNTERR_F, "MPS Tx SOP/EOP error", -1, 1 },
4417                { FRMERR_F, "MPS Tx framing error", -1, 1 },
4418                { 0 }
4419        };
4420        static const struct intr_info mps_trc_intr_info[] = {
4421                { FILTMEM_V(FILTMEM_M), "MPS TRC filter parity error", -1, 1 },
4422                { PKTFIFO_V(PKTFIFO_M), "MPS TRC packet FIFO parity error",
4423                  -1, 1 },
4424                { MISCPERR_F, "MPS TRC misc parity error", -1, 1 },
4425                { 0 }
4426        };
4427        static const struct intr_info mps_stat_sram_intr_info[] = {
4428                { 0x1fffff, "MPS statistics SRAM parity error", -1, 1 },
4429                { 0 }
4430        };
4431        static const struct intr_info mps_stat_tx_intr_info[] = {
4432                { 0xfffff, "MPS statistics Tx FIFO parity error", -1, 1 },
4433                { 0 }
4434        };
4435        static const struct intr_info mps_stat_rx_intr_info[] = {
4436                { 0xffffff, "MPS statistics Rx FIFO parity error", -1, 1 },
4437                { 0 }
4438        };
4439        static const struct intr_info mps_cls_intr_info[] = {
4440                { MATCHSRAM_F, "MPS match SRAM parity error", -1, 1 },
4441                { MATCHTCAM_F, "MPS match TCAM parity error", -1, 1 },
4442                { HASHSRAM_F, "MPS hash SRAM parity error", -1, 1 },
4443                { 0 }
4444        };
4445
4446        int fat;
4447
4448        fat = t4_handle_intr_status(adapter, MPS_RX_PERR_INT_CAUSE_A,
4449                                    mps_rx_intr_info) +
4450              t4_handle_intr_status(adapter, MPS_TX_INT_CAUSE_A,
4451                                    is_t6(adapter->params.chip)
4452                                    ? t6_mps_tx_intr_info
4453                                    : mps_tx_intr_info) +
4454              t4_handle_intr_status(adapter, MPS_TRC_INT_CAUSE_A,
4455                                    mps_trc_intr_info) +
4456              t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_SRAM_A,
4457                                    mps_stat_sram_intr_info) +
4458              t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_TX_FIFO_A,
4459                                    mps_stat_tx_intr_info) +
4460              t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_RX_FIFO_A,
4461                                    mps_stat_rx_intr_info) +
4462              t4_handle_intr_status(adapter, MPS_CLS_INT_CAUSE_A,
4463                                    mps_cls_intr_info);
4464
4465        t4_write_reg(adapter, MPS_INT_CAUSE_A, 0);
4466        t4_read_reg(adapter, MPS_INT_CAUSE_A);                    /* flush */
4467        if (fat)
4468                t4_fatal_err(adapter);
4469}
4470
4471#define MEM_INT_MASK (PERR_INT_CAUSE_F | ECC_CE_INT_CAUSE_F | \
4472                      ECC_UE_INT_CAUSE_F)
4473
4474/*
4475 * EDC/MC interrupt handler.
4476 */
4477static void mem_intr_handler(struct adapter *adapter, int idx)
4478{
4479        static const char name[4][7] = { "EDC0", "EDC1", "MC/MC0", "MC1" };
4480
4481        unsigned int addr, cnt_addr, v;
4482
4483        if (idx <= MEM_EDC1) {
4484                addr = EDC_REG(EDC_INT_CAUSE_A, idx);
4485                cnt_addr = EDC_REG(EDC_ECC_STATUS_A, idx);
4486        } else if (idx == MEM_MC) {
4487                if (is_t4(adapter->params.chip)) {
4488                        addr = MC_INT_CAUSE_A;
4489                        cnt_addr = MC_ECC_STATUS_A;
4490                } else {
4491                        addr = MC_P_INT_CAUSE_A;
4492                        cnt_addr = MC_P_ECC_STATUS_A;
4493                }
4494        } else {
4495                addr = MC_REG(MC_P_INT_CAUSE_A, 1);
4496                cnt_addr = MC_REG(MC_P_ECC_STATUS_A, 1);
4497        }
4498
4499        v = t4_read_reg(adapter, addr) & MEM_INT_MASK;
4500        if (v & PERR_INT_CAUSE_F)
4501                dev_alert(adapter->pdev_dev, "%s FIFO parity error\n",
4502                          name[idx]);
4503        if (v & ECC_CE_INT_CAUSE_F) {
4504                u32 cnt = ECC_CECNT_G(t4_read_reg(adapter, cnt_addr));
4505
4506                t4_edc_err_read(adapter, idx);
4507
4508                t4_write_reg(adapter, cnt_addr, ECC_CECNT_V(ECC_CECNT_M));
4509                if (printk_ratelimit())
4510                        dev_warn(adapter->pdev_dev,
4511                                 "%u %s correctable ECC data error%s\n",
4512                                 cnt, name[idx], cnt > 1 ? "s" : "");
4513        }
4514        if (v & ECC_UE_INT_CAUSE_F)
4515                dev_alert(adapter->pdev_dev,
4516                          "%s uncorrectable ECC data error\n", name[idx]);
4517
4518        t4_write_reg(adapter, addr, v);
4519        if (v & (PERR_INT_CAUSE_F | ECC_UE_INT_CAUSE_F))
4520                t4_fatal_err(adapter);
4521}
4522
4523/*
4524 * MA interrupt handler.
4525 */
4526static void ma_intr_handler(struct adapter *adap)
4527{
4528        u32 v, status = t4_read_reg(adap, MA_INT_CAUSE_A);
4529
4530        if (status & MEM_PERR_INT_CAUSE_F) {
4531                dev_alert(adap->pdev_dev,
4532                          "MA parity error, parity status %#x\n",
4533                          t4_read_reg(adap, MA_PARITY_ERROR_STATUS1_A));
4534                if (is_t5(adap->params.chip))
4535                        dev_alert(adap->pdev_dev,
4536                                  "MA parity error, parity status %#x\n",
4537                                  t4_read_reg(adap,
4538                                              MA_PARITY_ERROR_STATUS2_A));
4539        }
4540        if (status & MEM_WRAP_INT_CAUSE_F) {
4541                v = t4_read_reg(adap, MA_INT_WRAP_STATUS_A);
4542                dev_alert(adap->pdev_dev, "MA address wrap-around error by "
4543                          "client %u to address %#x\n",
4544                          MEM_WRAP_CLIENT_NUM_G(v),
4545                          MEM_WRAP_ADDRESS_G(v) << 4);
4546        }
4547        t4_write_reg(adap, MA_INT_CAUSE_A, status);
4548        t4_fatal_err(adap);
4549}
4550
4551/*
4552 * SMB interrupt handler.
4553 */
4554static void smb_intr_handler(struct adapter *adap)
4555{
4556        static const struct intr_info smb_intr_info[] = {
4557                { MSTTXFIFOPARINT_F, "SMB master Tx FIFO parity error", -1, 1 },
4558                { MSTRXFIFOPARINT_F, "SMB master Rx FIFO parity error", -1, 1 },
4559                { SLVFIFOPARINT_F, "SMB slave FIFO parity error", -1, 1 },
4560                { 0 }
4561        };
4562
4563        if (t4_handle_intr_status(adap, SMB_INT_CAUSE_A, smb_intr_info))
4564                t4_fatal_err(adap);
4565}
4566
4567/*
4568 * NC-SI interrupt handler.
4569 */
4570static void ncsi_intr_handler(struct adapter *adap)
4571{
4572        static const struct intr_info ncsi_intr_info[] = {
4573                { CIM_DM_PRTY_ERR_F, "NC-SI CIM parity error", -1, 1 },
4574                { MPS_DM_PRTY_ERR_F, "NC-SI MPS parity error", -1, 1 },
4575                { TXFIFO_PRTY_ERR_F, "NC-SI Tx FIFO parity error", -1, 1 },
4576                { RXFIFO_PRTY_ERR_F, "NC-SI Rx FIFO parity error", -1, 1 },
4577                { 0 }
4578        };
4579
4580        if (t4_handle_intr_status(adap, NCSI_INT_CAUSE_A, ncsi_intr_info))
4581                t4_fatal_err(adap);
4582}
4583
4584/*
4585 * XGMAC interrupt handler.
4586 */
4587static void xgmac_intr_handler(struct adapter *adap, int port)
4588{
4589        u32 v, int_cause_reg;
4590
4591        if (is_t4(adap->params.chip))
4592                int_cause_reg = PORT_REG(port, XGMAC_PORT_INT_CAUSE_A);
4593        else
4594                int_cause_reg = T5_PORT_REG(port, MAC_PORT_INT_CAUSE_A);
4595
4596        v = t4_read_reg(adap, int_cause_reg);
4597
4598        v &= TXFIFO_PRTY_ERR_F | RXFIFO_PRTY_ERR_F;
4599        if (!v)
4600                return;
4601
4602        if (v & TXFIFO_PRTY_ERR_F)
4603                dev_alert(adap->pdev_dev, "XGMAC %d Tx FIFO parity error\n",
4604                          port);
4605        if (v & RXFIFO_PRTY_ERR_F)
4606                dev_alert(adap->pdev_dev, "XGMAC %d Rx FIFO parity error\n",
4607                          port);
4608        t4_write_reg(adap, PORT_REG(port, XGMAC_PORT_INT_CAUSE_A), v);
4609        t4_fatal_err(adap);
4610}
4611
4612/*
4613 * PL interrupt handler.
4614 */
4615static void pl_intr_handler(struct adapter *adap)
4616{
4617        static const struct intr_info pl_intr_info[] = {
4618                { FATALPERR_F, "T4 fatal parity error", -1, 1 },
4619                { PERRVFID_F, "PL VFID_MAP parity error", -1, 1 },
4620                { 0 }
4621        };
4622
4623        if (t4_handle_intr_status(adap, PL_PL_INT_CAUSE_A, pl_intr_info))
4624                t4_fatal_err(adap);
4625}
4626
4627#define PF_INTR_MASK (PFSW_F)
4628#define GLBL_INTR_MASK (CIM_F | MPS_F | PL_F | PCIE_F | MC_F | EDC0_F | \
4629                EDC1_F | LE_F | TP_F | MA_F | PM_TX_F | PM_RX_F | ULP_RX_F | \
4630                CPL_SWITCH_F | SGE_F | ULP_TX_F | SF_F)
4631
4632/**
4633 *      t4_slow_intr_handler - control path interrupt handler
4634 *      @adapter: the adapter
4635 *
4636 *      T4 interrupt handler for non-data global interrupt events, e.g., errors.
4637 *      The designation 'slow' is because it involves register reads, while
4638 *      data interrupts typically don't involve any MMIOs.
4639 */
4640int t4_slow_intr_handler(struct adapter *adapter)
4641{
4642        u32 cause = t4_read_reg(adapter, PL_INT_CAUSE_A);
4643
4644        if (!(cause & GLBL_INTR_MASK))
4645                return 0;
4646        if (cause & CIM_F)
4647                cim_intr_handler(adapter);
4648        if (cause & MPS_F)
4649                mps_intr_handler(adapter);
4650        if (cause & NCSI_F)
4651                ncsi_intr_handler(adapter);
4652        if (cause & PL_F)
4653                pl_intr_handler(adapter);
4654        if (cause & SMB_F)
4655                smb_intr_handler(adapter);
4656        if (cause & XGMAC0_F)
4657                xgmac_intr_handler(adapter, 0);
4658        if (cause & XGMAC1_F)
4659                xgmac_intr_handler(adapter, 1);
4660        if (cause & XGMAC_KR0_F)
4661                xgmac_intr_handler(adapter, 2);
4662        if (cause & XGMAC_KR1_F)
4663                xgmac_intr_handler(adapter, 3);
4664        if (cause & PCIE_F)
4665                pcie_intr_handler(adapter);
4666        if (cause & MC_F)
4667                mem_intr_handler(adapter, MEM_MC);
4668        if (is_t5(adapter->params.chip) && (cause & MC1_F))
4669                mem_intr_handler(adapter, MEM_MC1);
4670        if (cause & EDC0_F)
4671                mem_intr_handler(adapter, MEM_EDC0);
4672        if (cause & EDC1_F)
4673                mem_intr_handler(adapter, MEM_EDC1);
4674        if (cause & LE_F)
4675                le_intr_handler(adapter);
4676        if (cause & TP_F)
4677                tp_intr_handler(adapter);
4678        if (cause & MA_F)
4679                ma_intr_handler(adapter);
4680        if (cause & PM_TX_F)
4681                pmtx_intr_handler(adapter);
4682        if (cause & PM_RX_F)
4683                pmrx_intr_handler(adapter);
4684        if (cause & ULP_RX_F)
4685                ulprx_intr_handler(adapter);
4686        if (cause & CPL_SWITCH_F)
4687                cplsw_intr_handler(adapter);
4688        if (cause & SGE_F)
4689                sge_intr_handler(adapter);
4690        if (cause & ULP_TX_F)
4691                ulptx_intr_handler(adapter);
4692
4693        /* Clear the interrupts just processed for which we are the master. */
4694        t4_write_reg(adapter, PL_INT_CAUSE_A, cause & GLBL_INTR_MASK);
4695        (void)t4_read_reg(adapter, PL_INT_CAUSE_A); /* flush */
4696        return 1;
4697}
4698
4699/**
4700 *      t4_intr_enable - enable interrupts
4701 *      @adapter: the adapter whose interrupts should be enabled
4702 *
4703 *      Enable PF-specific interrupts for the calling function and the top-level
4704 *      interrupt concentrator for global interrupts.  Interrupts are already
4705 *      enabled at each module, here we just enable the roots of the interrupt
4706 *      hierarchies.
4707 *
4708 *      Note: this function should be called only when the driver manages
4709 *      non PF-specific interrupts from the various HW modules.  Only one PCI
4710 *      function at a time should be doing this.
4711 */
4712void t4_intr_enable(struct adapter *adapter)
4713{
4714        u32 val = 0;
4715        u32 whoami = t4_read_reg(adapter, PL_WHOAMI_A);
4716        u32 pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ?
4717                        SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
4718
4719        if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
4720                val = ERR_DROPPED_DB_F | ERR_EGR_CTXT_PRIO_F | DBFIFO_HP_INT_F;
4721        t4_write_reg(adapter, SGE_INT_ENABLE3_A, ERR_CPL_EXCEED_IQE_SIZE_F |
4722                     ERR_INVALID_CIDX_INC_F | ERR_CPL_OPCODE_0_F |
4723                     ERR_DATA_CPL_ON_HIGH_QID1_F | INGRESS_SIZE_ERR_F |
4724                     ERR_DATA_CPL_ON_HIGH_QID0_F | ERR_BAD_DB_PIDX3_F |
4725                     ERR_BAD_DB_PIDX2_F | ERR_BAD_DB_PIDX1_F |
4726                     ERR_BAD_DB_PIDX0_F | ERR_ING_CTXT_PRIO_F |
4727                     DBFIFO_LP_INT_F | EGRESS_SIZE_ERR_F | val);
4728        t4_write_reg(adapter, MYPF_REG(PL_PF_INT_ENABLE_A), PF_INTR_MASK);
4729        t4_set_reg_field(adapter, PL_INT_MAP0_A, 0, 1 << pf);
4730}
4731
4732/**
4733 *      t4_intr_disable - disable interrupts
4734 *      @adapter: the adapter whose interrupts should be disabled
4735 *
4736 *      Disable interrupts.  We only disable the top-level interrupt
4737 *      concentrators.  The caller must be a PCI function managing global
4738 *      interrupts.
4739 */
4740void t4_intr_disable(struct adapter *adapter)
4741{
4742        u32 whoami, pf;
4743
4744        if (pci_channel_offline(adapter->pdev))
4745                return;
4746
4747        whoami = t4_read_reg(adapter, PL_WHOAMI_A);
4748        pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ?
4749                        SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
4750
4751        t4_write_reg(adapter, MYPF_REG(PL_PF_INT_ENABLE_A), 0);
4752        t4_set_reg_field(adapter, PL_INT_MAP0_A, 1 << pf, 0);
4753}
4754
4755/**
4756 *      t4_config_rss_range - configure a portion of the RSS mapping table
4757 *      @adapter: the adapter
4758 *      @mbox: mbox to use for the FW command
4759 *      @viid: virtual interface whose RSS subtable is to be written
4760 *      @start: start entry in the table to write
4761 *      @n: how many table entries to write
4762 *      @rspq: values for the response queue lookup table
4763 *      @nrspq: number of values in @rspq
4764 *
4765 *      Programs the selected part of the VI's RSS mapping table with the
4766 *      provided values.  If @nrspq < @n the supplied values are used repeatedly
4767 *      until the full table range is populated.
4768 *
4769 *      The caller must ensure the values in @rspq are in the range allowed for
4770 *      @viid.
4771 */
4772int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
4773                        int start, int n, const u16 *rspq, unsigned int nrspq)
4774{
4775        int ret;
4776        const u16 *rsp = rspq;
4777        const u16 *rsp_end = rspq + nrspq;
4778        struct fw_rss_ind_tbl_cmd cmd;
4779
4780        memset(&cmd, 0, sizeof(cmd));
4781        cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_IND_TBL_CMD) |
4782                               FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
4783                               FW_RSS_IND_TBL_CMD_VIID_V(viid));
4784        cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
4785
4786        /* each fw_rss_ind_tbl_cmd takes up to 32 entries */
4787        while (n > 0) {
4788                int nq = min(n, 32);
4789                __be32 *qp = &cmd.iq0_to_iq2;
4790
4791                cmd.niqid = cpu_to_be16(nq);
4792                cmd.startidx = cpu_to_be16(start);
4793
4794                start += nq;
4795                n -= nq;
4796
4797                while (nq > 0) {
4798                        unsigned int v;
4799
4800                        v = FW_RSS_IND_TBL_CMD_IQ0_V(*rsp);
4801                        if (++rsp >= rsp_end)
4802                                rsp = rspq;
4803                        v |= FW_RSS_IND_TBL_CMD_IQ1_V(*rsp);
4804                        if (++rsp >= rsp_end)
4805                                rsp = rspq;
4806                        v |= FW_RSS_IND_TBL_CMD_IQ2_V(*rsp);
4807                        if (++rsp >= rsp_end)
4808                                rsp = rspq;
4809
4810                        *qp++ = cpu_to_be32(v);
4811                        nq -= 3;
4812                }
4813
4814                ret = t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL);
4815                if (ret)
4816                        return ret;
4817        }
4818        return 0;
4819}
4820
4821/**
4822 *      t4_config_glbl_rss - configure the global RSS mode
4823 *      @adapter: the adapter
4824 *      @mbox: mbox to use for the FW command
4825 *      @mode: global RSS mode
4826 *      @flags: mode-specific flags
4827 *
4828 *      Sets the global RSS mode.
4829 */
4830int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode,
4831                       unsigned int flags)
4832{
4833        struct fw_rss_glb_config_cmd c;
4834
4835        memset(&c, 0, sizeof(c));
4836        c.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_RSS_GLB_CONFIG_CMD) |
4837                                    FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
4838        c.retval_len16 = cpu_to_be32(FW_LEN16(c));
4839        if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_MANUAL) {
4840                c.u.manual.mode_pkd =
4841                        cpu_to_be32(FW_RSS_GLB_CONFIG_CMD_MODE_V(mode));
4842        } else if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
4843                c.u.basicvirtual.mode_pkd =
4844                        cpu_to_be32(FW_RSS_GLB_CONFIG_CMD_MODE_V(mode));
4845                c.u.basicvirtual.synmapen_to_hashtoeplitz = cpu_to_be32(flags);
4846        } else
4847                return -EINVAL;
4848        return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
4849}
4850
4851/**
4852 *      t4_config_vi_rss - configure per VI RSS settings
4853 *      @adapter: the adapter
4854 *      @mbox: mbox to use for the FW command
4855 *      @viid: the VI id
4856 *      @flags: RSS flags
4857 *      @defq: id of the default RSS queue for the VI.
4858 *
4859 *      Configures VI-specific RSS properties.
4860 */
4861int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid,
4862                     unsigned int flags, unsigned int defq)
4863{
4864        struct fw_rss_vi_config_cmd c;
4865
4866        memset(&c, 0, sizeof(c));
4867        c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_VI_CONFIG_CMD) |
4868                                   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
4869                                   FW_RSS_VI_CONFIG_CMD_VIID_V(viid));
4870        c.retval_len16 = cpu_to_be32(FW_LEN16(c));
4871        c.u.basicvirtual.defaultq_to_udpen = cpu_to_be32(flags |
4872                                        FW_RSS_VI_CONFIG_CMD_DEFAULTQ_V(defq));
4873        return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
4874}
4875
4876/* Read an RSS table row */
4877static int rd_rss_row(struct adapter *adap, int row, u32 *val)
4878{
4879        t4_write_reg(adap, TP_RSS_LKP_TABLE_A, 0xfff00000 | row);
4880        return t4_wait_op_done_val(adap, TP_RSS_LKP_TABLE_A, LKPTBLROWVLD_F, 1,
4881                                   5, 0, val);
4882}
4883
4884/**
4885 *      t4_read_rss - read the contents of the RSS mapping table
4886 *      @adapter: the adapter
4887 *      @map: holds the contents of the RSS mapping table
4888 *
4889 *      Reads the contents of the RSS hash->queue mapping table.
4890 */
4891int t4_read_rss(struct adapter *adapter, u16 *map)
4892{
4893        u32 val;
4894        int i, ret;
4895
4896        for (i = 0; i < RSS_NENTRIES / 2; ++i) {
4897                ret = rd_rss_row(adapter, i, &val);
4898                if (ret)
4899                        return ret;
4900                *map++ = LKPTBLQUEUE0_G(val);
4901                *map++ = LKPTBLQUEUE1_G(val);
4902        }
4903        return 0;
4904}
4905
4906static unsigned int t4_use_ldst(struct adapter *adap)
4907{
4908        return (adap->flags & FW_OK) || !adap->use_bd;
4909}
4910
4911/**
4912 *      t4_fw_tp_pio_rw - Access TP PIO through LDST
4913 *      @adap: the adapter
4914 *      @vals: where the indirect register values are stored/written
4915 *      @nregs: how many indirect registers to read/write
4916 *      @start_idx: index of first indirect register to read/write
4917 *      @rw: Read (1) or Write (0)
4918 *
4919 *      Access TP PIO registers through LDST
4920 */
4921static void t4_fw_tp_pio_rw(struct adapter *adap, u32 *vals, unsigned int nregs,
4922                            unsigned int start_index, unsigned int rw)
4923{
4924        int ret, i;
4925        int cmd = FW_LDST_ADDRSPC_TP_PIO;
4926        struct fw_ldst_cmd c;
4927
4928        for (i = 0 ; i < nregs; i++) {
4929                memset(&c, 0, sizeof(c));
4930                c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
4931                                                FW_CMD_REQUEST_F |
4932                                                (rw ? FW_CMD_READ_F :
4933                                                      FW_CMD_WRITE_F) |
4934                                                FW_LDST_CMD_ADDRSPACE_V(cmd));
4935                c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
4936
4937                c.u.addrval.addr = cpu_to_be32(start_index + i);
4938                c.u.addrval.val  = rw ? 0 : cpu_to_be32(vals[i]);
4939                ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
4940                if (!ret && rw)
4941                        vals[i] = be32_to_cpu(c.u.addrval.val);
4942        }
4943}
4944
4945/**
4946 *      t4_read_rss_key - read the global RSS key
4947 *      @adap: the adapter
4948 *      @key: 10-entry array holding the 320-bit RSS key
4949 *
4950 *      Reads the global 320-bit RSS key.
4951 */
4952void t4_read_rss_key(struct adapter *adap, u32 *key)
4953{
4954        if (t4_use_ldst(adap))
4955                t4_fw_tp_pio_rw(adap, key, 10, TP_RSS_SECRET_KEY0_A, 1);
4956        else
4957                t4_read_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, key, 10,
4958                                 TP_RSS_SECRET_KEY0_A);
4959}
4960
4961/**
4962 *      t4_write_rss_key - program one of the RSS keys
4963 *      @adap: the adapter
4964 *      @key: 10-entry array holding the 320-bit RSS key
4965 *      @idx: which RSS key to write
4966 *
4967 *      Writes one of the RSS keys with the given 320-bit value.  If @idx is
4968 *      0..15 the corresponding entry in the RSS key table is written,
4969 *      otherwise the global RSS key is written.
4970 */
4971void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx)
4972{
4973        u8 rss_key_addr_cnt = 16;
4974        u32 vrt = t4_read_reg(adap, TP_RSS_CONFIG_VRT_A);
4975
4976        /* T6 and later: for KeyMode 3 (per-vf and per-vf scramble),
4977         * allows access to key addresses 16-63 by using KeyWrAddrX
4978         * as index[5:4](upper 2) into key table
4979         */
4980        if ((CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) &&
4981            (vrt & KEYEXTEND_F) && (KEYMODE_G(vrt) == 3))
4982                rss_key_addr_cnt = 32;
4983
4984        if (t4_use_ldst(adap))
4985                t4_fw_tp_pio_rw(adap, (void *)key, 10, TP_RSS_SECRET_KEY0_A, 0);
4986        else
4987                t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, key, 10,
4988                                  TP_RSS_SECRET_KEY0_A);
4989
4990        if (idx >= 0 && idx < rss_key_addr_cnt) {
4991                if (rss_key_addr_cnt > 16)
4992                        t4_write_reg(adap, TP_RSS_CONFIG_VRT_A,
4993                                     KEYWRADDRX_V(idx >> 4) |
4994                                     T6_VFWRADDR_V(idx) | KEYWREN_F);
4995                else
4996                        t4_write_reg(adap, TP_RSS_CONFIG_VRT_A,
4997                                     KEYWRADDR_V(idx) | KEYWREN_F);
4998        }
4999}
5000

5001/**
5002 *      t4_read_rss_pf_config - read PF RSS Configuration Table
5003 *      @adapter: the adapter
5004 *      @index: the entry in the PF RSS table to read
5005 *      @valp: where to store the returned value
5006 *
5007 *      Reads the PF RSS Configuration Table at the specified index and returns
5008 *      the value found there.
5009 */
5010void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index,
5011                           u32 *valp)
5012{
5013        if (t4_use_ldst(adapter))
5014                t4_fw_tp_pio_rw(adapter, valp, 1,
5015                                TP_RSS_PF0_CONFIG_A + index, 1);
5016        else
5017                t4_read_indirect(adapter, TP_PIO_ADDR_A, TP_PIO_DATA_A,
5018                                 valp, 1, TP_RSS_PF0_CONFIG_A + index);
5019}
5020
5021/**
5022 *      t4_read_rss_vf_config - read VF RSS Configuration Table
5023 *      @adapter: the adapter
5024 *      @index: the entry in the VF RSS table to read
5025 *      @vfl: where to store the returned VFL
5026 *      @vfh: where to store the returned VFH
5027 *
5028 *      Reads the VF RSS Configuration Table at the specified index and returns
5029 *      the (VFL, VFH) values found there.
5030 */
5031void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
5032                           u32 *vfl, u32 *vfh)
5033{
5034        u32 vrt, mask, data;
5035
5036        if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) {
5037                mask = VFWRADDR_V(VFWRADDR_M);
5038                data = VFWRADDR_V(index);
5039        } else {
5040                 mask =  T6_VFWRADDR_V(T6_VFWRADDR_M);
5041                 data = T6_VFWRADDR_V(index);
5042        }
5043
5044        /* Request that the index'th VF Table values be read into VFL/VFH.
5045         */
5046        vrt = t4_read_reg(adapter, TP_RSS_CONFIG_VRT_A);
5047        vrt &= ~(VFRDRG_F | VFWREN_F | KEYWREN_F | mask);
5048        vrt |= data | VFRDEN_F;
5049        t4_write_reg(adapter, TP_RSS_CONFIG_VRT_A, vrt);
5050
5051        /* Grab the VFL/VFH values ...
5052         */
5053        if (t4_use_ldst(adapter)) {
5054                t4_fw_tp_pio_rw(adapter, vfl, 1, TP_RSS_VFL_CONFIG_A, 1);
5055                t4_fw_tp_pio_rw(adapter, vfh, 1, TP_RSS_VFH_CONFIG_A, 1);
5056        } else {
5057                t4_read_indirect(adapter, TP_PIO_ADDR_A, TP_PIO_DATA_A,
5058                                 vfl, 1, TP_RSS_VFL_CONFIG_A);
5059                t4_read_indirect(adapter, TP_PIO_ADDR_A, TP_PIO_DATA_A,
5060                                 vfh, 1, TP_RSS_VFH_CONFIG_A);
5061        }
5062}
5063
5064/**
5065 *      t4_read_rss_pf_map - read PF RSS Map
5066 *      @adapter: the adapter
5067 *
5068 *      Reads the PF RSS Map register and returns its value.
5069 */
5070u32 t4_read_rss_pf_map(struct adapter *adapter)
5071{
5072        u32 pfmap;
5073
5074        if (t4_use_ldst(adapter))
5075                t4_fw_tp_pio_rw(adapter, &pfmap, 1, TP_RSS_PF_MAP_A, 1);
5076        else
5077                t4_read_indirect(adapter, TP_PIO_ADDR_A, TP_PIO_DATA_A,
5078                                 &pfmap, 1, TP_RSS_PF_MAP_A);
5079        return pfmap;
5080}
5081
5082/**
5083 *      t4_read_rss_pf_mask - read PF RSS Mask
5084 *      @adapter: the adapter
5085 *
5086 *      Reads the PF RSS Mask register and returns its value.
5087 */
5088u32 t4_read_rss_pf_mask(struct adapter *adapter)
5089{
5090        u32 pfmask;
5091
5092        if (t4_use_ldst(adapter))
5093                t4_fw_tp_pio_rw(adapter, &pfmask, 1, TP_RSS_PF_MSK_A, 1);
5094        else
5095                t4_read_indirect(adapter, TP_PIO_ADDR_A, TP_PIO_DATA_A,
5096                                 &pfmask, 1, TP_RSS_PF_MSK_A);
5097        return pfmask;
5098}
5099
5100/**
5101 *      t4_tp_get_tcp_stats - read TP's TCP MIB counters
5102 *      @adap: the adapter
5103 *      @v4: holds the TCP/IP counter values
5104 *      @v6: holds the TCP/IPv6 counter values
5105 *
5106 *      Returns the values of TP's TCP/IP and TCP/IPv6 MIB counters.
5107 *      Either @v4 or @v6 may be %NULL to skip the corresponding stats.
5108 */
5109void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4,
5110                         struct tp_tcp_stats *v6)
5111{
5112        u32 val[TP_MIB_TCP_RXT_SEG_LO_A - TP_MIB_TCP_OUT_RST_A + 1];
5113
5114#define STAT_IDX(x) ((TP_MIB_TCP_##x##_A) - TP_MIB_TCP_OUT_RST_A)
5115#define STAT(x)     val[STAT_IDX(x)]
5116#define STAT64(x)   (((u64)STAT(x##_HI) << 32) | STAT(x##_LO))
5117
5118        if (v4) {
5119                t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, val,
5120                                 ARRAY_SIZE(val), TP_MIB_TCP_OUT_RST_A);
5121                v4->tcp_out_rsts = STAT(OUT_RST);
5122                v4->tcp_in_segs  = STAT64(IN_SEG);
5123                v4->tcp_out_segs = STAT64(OUT_SEG);
5124                v4->tcp_retrans_segs = STAT64(RXT_SEG);
5125        }
5126        if (v6) {
5127                t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, val,
5128                                 ARRAY_SIZE(val), TP_MIB_TCP_V6OUT_RST_A);
5129                v6->tcp_out_rsts = STAT(OUT_RST);
5130                v6->tcp_in_segs  = STAT64(IN_SEG);
5131                v6->tcp_out_segs = STAT64(OUT_SEG);
5132                v6->tcp_retrans_segs = STAT64(RXT_SEG);
5133        }
5134#undef STAT64
5135#undef STAT
5136#undef STAT_IDX
5137}
5138
5139/**
5140 *      t4_tp_get_err_stats - read TP's error MIB counters
5141 *      @adap: the adapter
5142 *      @st: holds the counter values
5143 *
5144 *      Returns the values of TP's error counters.
5145 */
5146void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st)
5147{
5148        int nchan = adap->params.arch.nchan;
5149
5150        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5151                         st->mac_in_errs, nchan, TP_MIB_MAC_IN_ERR_0_A);
5152        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5153                         st->hdr_in_errs, nchan, TP_MIB_HDR_IN_ERR_0_A);
5154        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5155                         st->tcp_in_errs, nchan, TP_MIB_TCP_IN_ERR_0_A);
5156        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5157                         st->tnl_cong_drops, nchan, TP_MIB_TNL_CNG_DROP_0_A);
5158        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5159                         st->ofld_chan_drops, nchan, TP_MIB_OFD_CHN_DROP_0_A);
5160        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5161                         st->tnl_tx_drops, nchan, TP_MIB_TNL_DROP_0_A);
5162        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5163                         st->ofld_vlan_drops, nchan, TP_MIB_OFD_VLN_DROP_0_A);
5164        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5165                         st->tcp6_in_errs, nchan, TP_MIB_TCP_V6IN_ERR_0_A);
5166
5167        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A,
5168                         &st->ofld_no_neigh, 2, TP_MIB_OFD_ARP_DROP_A);
5169}
5170
5171/**
5172 *      t4_tp_get_cpl_stats - read TP's CPL MIB counters
5173 *      @adap: the adapter
5174 *      @st: holds the counter values
5175 *
5176 *      Returns the values of TP's CPL counters.
5177 */
5178void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st)
5179{
5180        int nchan = adap->params.arch.nchan;
5181
5182        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, st->req,
5183                         nchan, TP_MIB_CPL_IN_REQ_0_A);
5184        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, st->rsp,
5185                         nchan, TP_MIB_CPL_OUT_RSP_0_A);
5186
5187}
5188
5189/**
5190 *      t4_tp_get_rdma_stats - read TP's RDMA MIB counters
5191 *      @adap: the adapter
5192 *      @st: holds the counter values
5193 *
5194 *      Returns the values of TP's RDMA counters.
5195 */
5196void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st)
5197{
5198        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, &st->rqe_dfr_pkt,
5199                         2, TP_MIB_RQE_DFR_PKT_A);
5200}
5201
5202/**
5203 *      t4_get_fcoe_stats - read TP's FCoE MIB counters for a port
5204 *      @adap: the adapter
5205 *      @idx: the port index
5206 *      @st: holds the counter values
5207 *
5208 *      Returns the values of TP's FCoE counters for the selected port.
5209 */
5210void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx,
5211                       struct tp_fcoe_stats *st)
5212{
5213        u32 val[2];
5214
5215        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, &st->frames_ddp,
5216                         1, TP_MIB_FCOE_DDP_0_A + idx);
5217        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, &st->frames_drop,
5218                         1, TP_MIB_FCOE_DROP_0_A + idx);
5219        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, val,
5220                         2, TP_MIB_FCOE_BYTE_0_HI_A + 2 * idx);
5221        st->octets_ddp = ((u64)val[0] << 32) | val[1];
5222}
5223
5224/**
5225 *      t4_get_usm_stats - read TP's non-TCP DDP MIB counters
5226 *      @adap: the adapter
5227 *      @st: holds the counter values
5228 *
5229 *      Returns the values of TP's counters for non-TCP directly-placed packets.
5230 */
5231void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st)
5232{
5233        u32 val[4];
5234
5235        t4_read_indirect(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, val, 4,
5236                         TP_MIB_USM_PKTS_A);
5237        st->frames = val[0];
5238        st->drops = val[1];
5239        st->octets = ((u64)val[2] << 32) | val[3];
5240}
5241
5242/**
5243 *      t4_read_mtu_tbl - returns the values in the HW path MTU table
5244 *      @adap: the adapter
5245 *      @mtus: where to store the MTU values
5246 *      @mtu_log: where to store the MTU base-2 log (may be %NULL)
5247 *
5248 *      Reads the HW path MTU table.
5249 */
5250void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log)
5251{
5252        u32 v;
5253        int i;
5254
5255        for (i = 0; i < NMTUS; ++i) {
5256                t4_write_reg(adap, TP_MTU_TABLE_A,
5257                             MTUINDEX_V(0xff) | MTUVALUE_V(i));
5258                v = t4_read_reg(adap, TP_MTU_TABLE_A);
5259                mtus[i] = MTUVALUE_G(v);
5260                if (mtu_log)
5261                        mtu_log[i] = MTUWIDTH_G(v);
5262        }
5263}
5264
5265/**
5266 *      t4_read_cong_tbl - reads the congestion control table
5267 *      @adap: the adapter
5268 *      @incr: where to store the alpha values
5269 *
5270 *      Reads the additive increments programmed into the HW congestion
5271 *      control table.
5272 */
5273void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN])
5274{
5275        unsigned int mtu, w;
5276
5277        for (mtu = 0; mtu < NMTUS; ++mtu)
5278                for (w = 0; w < NCCTRL_WIN; ++w) {
5279                        t4_write_reg(adap, TP_CCTRL_TABLE_A,
5280                                     ROWINDEX_V(0xffff) | (mtu << 5) | w);
5281                        incr[mtu][w] = (u16)t4_read_reg(adap,
5282                                                TP_CCTRL_TABLE_A) & 0x1fff;
5283                }
5284}
5285
5286/**
5287 *      t4_tp_wr_bits_indirect - set/clear bits in an indirect TP register
5288 *      @adap: the adapter
5289 *      @addr: the indirect TP register address
5290 *      @mask: specifies the field within the register to modify
5291 *      @val: new value for the field
5292 *
5293 *      Sets a field of an indirect TP register to the given value.
5294 */
5295void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr,
5296                            unsigned int mask, unsigned int val)
5297{
5298        t4_write_reg(adap, TP_PIO_ADDR_A, addr);
5299        val |= t4_read_reg(adap, TP_PIO_DATA_A) & ~mask;
5300        t4_write_reg(adap, TP_PIO_DATA_A, val);
5301}
5302
5303/**
5304 *      init_cong_ctrl - initialize congestion control parameters
5305 *      @a: the alpha values for congestion control
5306 *      @b: the beta values for congestion control
5307 *
5308 *      Initialize the congestion control parameters.
5309 */
5310static void init_cong_ctrl(unsigned short *a, unsigned short *b)
5311{
5312        a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1;
5313        a[9] = 2;
5314        a[10] = 3;
5315        a[11] = 4;
5316        a[12] = 5;
5317        a[13] = 6;
5318        a[14] = 7;
5319        a[15] = 8;
5320        a[16] = 9;
5321        a[17] = 10;
5322        a[18] = 14;
5323        a[19] = 17;
5324        a[20] = 21;
5325        a[21] = 25;
5326        a[22] = 30;
5327        a[23] = 35;
5328        a[24] = 45;
5329        a[25] = 60;
5330        a[26] = 80;
5331        a[27] = 100;
5332        a[28] = 200;
5333        a[29] = 300;
5334        a[30] = 400;
5335        a[31] = 500;
5336
5337        b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0;
5338        b[9] = b[10] = 1;
5339        b[11] = b[12] = 2;
5340        b[13] = b[14] = b[15] = b[16] = 3;
5341        b[17] = b[18] = b[19] = b[20] = b[21] = 4;
5342        b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5;
5343        b[28] = b[29] = 6;
5344        b[30] = b[31] = 7;
5345}
5346
5347/* The minimum additive increment value for the congestion control table */
5348#define CC_MIN_INCR 2U
5349
5350/**
5351 *      t4_load_mtus - write the MTU and congestion control HW tables
5352 *      @adap: the adapter
5353 *      @mtus: the values for the MTU table
5354 *      @alpha: the values for the congestion control alpha parameter
5355 *      @beta: the values for the congestion control beta parameter
5356 *
5357 *      Write the HW MTU table with the supplied MTUs and the high-speed
5358 *      congestion control table with the supplied alpha, beta, and MTUs.
5359 *      We write the two tables together because the additive increments
5360 *      depend on the MTUs.
5361 */
5362void t4_load_mtus(struct adapter *adap, const unsigned short *mtus,
5363                  const unsigned short *alpha, const unsigned short *beta)
5364{
5365        static const unsigned int avg_pkts[NCCTRL_WIN] = {
5366                2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640,
5367                896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480,
5368                28672, 40960, 57344, 81920, 114688, 163840, 229376
5369        };
5370
5371        unsigned int i, w;
5372
5373        for (i = 0; i < NMTUS; ++i) {
5374                unsigned int mtu = mtus[i];
5375                unsigned int log2 = fls(mtu);
5376
5377                if (!(mtu & ((1 << log2) >> 2)))     /* round */
5378                        log2--;
5379                t4_write_reg(adap, TP_MTU_TABLE_A, MTUINDEX_V(i) |
5380                             MTUWIDTH_V(log2) | MTUVALUE_V(mtu));
5381
5382                for (w = 0; w < NCCTRL_WIN; ++w) {
5383                        unsigned int inc;
5384
5385                        inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w],
5386                                  CC_MIN_INCR);
5387
5388                        t4_write_reg(adap, TP_CCTRL_TABLE_A, (i << 21) |
5389                                     (w << 16) | (beta[w] << 13) | inc);
5390                }
5391        }
5392}
5393
5394/* Calculates a rate in bytes/s given the number of 256-byte units per 4K core
5395 * clocks.  The formula is
5396 *
5397 * bytes/s = bytes256 * 256 * ClkFreq / 4096
5398 *
5399 * which is equivalent to
5400 *
5401 * bytes/s = 62.5 * bytes256 * ClkFreq_ms
5402 */
5403static u64 chan_rate(struct adapter *adap, unsigned int bytes256)
5404{
5405        u64 v = bytes256 * adap->params.vpd.cclk;
5406
5407        return v * 62 + v / 2;
5408}
5409
5410/**
5411 *      t4_get_chan_txrate - get the current per channel Tx rates
5412 *      @adap: the adapter
5413 *      @nic_rate: rates for NIC traffic
5414 *      @ofld_rate: rates for offloaded traffic
5415 *
5416 *      Return the current Tx rates in bytes/s for NIC and offloaded traffic
5417 *      for each channel.
5418 */
5419void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate)
5420{
5421        u32 v;
5422
5423        v = t4_read_reg(adap, TP_TX_TRATE_A);
5424        nic_rate[0] = chan_rate(adap, TNLRATE0_G(v));
5425        nic_rate[1] = chan_rate(adap, TNLRATE1_G(v));
5426        if (adap->params.arch.nchan == NCHAN) {
5427                nic_rate[2] = chan_rate(adap, TNLRATE2_G(v));
5428                nic_rate[3] = chan_rate(adap, TNLRATE3_G(v));
5429        }
5430
5431        v = t4_read_reg(adap, TP_TX_ORATE_A);
5432        ofld_rate[0] = chan_rate(adap, OFDRATE0_G(v));
5433        ofld_rate[1] = chan_rate(adap, OFDRATE1_G(v));
5434        if (adap->params.arch.nchan == NCHAN) {
5435                ofld_rate[2] = chan_rate(adap, OFDRATE2_G(v));
5436                ofld_rate[3] = chan_rate(adap, OFDRATE3_G(v));
5437        }
5438}
5439
5440/**
5441 *      t4_set_trace_filter - configure one of the tracing filters
5442 *      @adap: the adapter
5443 *      @tp: the desired trace filter parameters
5444 *      @idx: which filter to configure
5445 *      @enable: whether to enable or disable the filter
5446 *
5447 *      Configures one of the tracing filters available in HW.  If @enable is
5448 *      %0 @tp is not examined and may be %NULL. The user is responsible to
5449 *      set the single/multiple trace mode by writing to MPS_TRC_CFG_A register
5450 */
5451int t4_set_trace_filter(struct adapter *adap, const struct trace_params *tp,
5452                        int idx, int enable)
5453{
5454        int i, ofst = idx * 4;
5455        u32 data_reg, mask_reg, cfg;
5456        u32 multitrc = TRCMULTIFILTER_F;
5457
5458        if (!enable) {
5459                t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst, 0);
5460                return 0;
5461        }
5462
5463        cfg = t4_read_reg(adap, MPS_TRC_CFG_A);
5464        if (cfg & TRCMULTIFILTER_F) {
5465                /* If multiple tracers are enabled, then maximum
5466                 * capture size is 2.5KB (FIFO size of a single channel)
5467                 * minus 2 flits for CPL_TRACE_PKT header.
5468                 */
5469                if (tp->snap_len > ((10 * 1024 / 4) - (2 * 8)))
5470                        return -EINVAL;
5471        } else {
5472                /* If multiple tracers are disabled, to avoid deadlocks
5473                 * maximum packet capture size of 9600 bytes is recommended.
5474                 * Also in this mode, only trace0 can be enabled and running.
5475                 */
5476                multitrc = 0;
5477                if (tp->snap_len > 9600 || idx)
5478                        return -EINVAL;
5479        }
5480
5481        if (tp->port > (is_t4(adap->params.chip) ? 11 : 19) || tp->invert > 1 ||
5482            tp->skip_len > TFLENGTH_M || tp->skip_ofst > TFOFFSET_M ||
5483            tp->min_len > TFMINPKTSIZE_M)
5484                return -EINVAL;
5485
5486        /* stop the tracer we'll be changing */
5487        t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst, 0);
5488
5489        idx *= (MPS_TRC_FILTER1_MATCH_A - MPS_TRC_FILTER0_MATCH_A);
5490        data_reg = MPS_TRC_FILTER0_MATCH_A + idx;
5491        mask_reg = MPS_TRC_FILTER0_DONT_CARE_A + idx;
5492
5493        for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
5494                t4_write_reg(adap, data_reg, tp->data[i]);
5495                t4_write_reg(adap, mask_reg, ~tp->mask[i]);
5496        }
5497        t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_B_A + ofst,
5498                     TFCAPTUREMAX_V(tp->snap_len) |
5499                     TFMINPKTSIZE_V(tp->min_len));
5500        t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst,
5501                     TFOFFSET_V(tp->skip_ofst) | TFLENGTH_V(tp->skip_len) |
5502                     (is_t4(adap->params.chip) ?
5503                     TFPORT_V(tp->port) | TFEN_F | TFINVERTMATCH_V(tp->invert) :
5504                     T5_TFPORT_V(tp->port) | T5_TFEN_F |
5505                     T5_TFINVERTMATCH_V(tp->invert)));
5506
5507        return 0;
5508}
5509
5510/**
5511 *      t4_get_trace_filter - query one of the tracing filters
5512 *      @adap: the adapter
5513 *      @tp: the current trace filter parameters
5514 *      @idx: which trace filter to query
5515 *      @enabled: non-zero if the filter is enabled
5516 *
5517 *      Returns the current settings of one of the HW tracing filters.
5518 */
5519void t4_get_trace_filter(struct adapter *adap, struct trace_params *tp, int idx,
5520                         int *enabled)
5521{
5522        u32 ctla, ctlb;
5523        int i, ofst = idx * 4;
5524        u32 data_reg, mask_reg;
5525
5526        ctla = t4_read_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst);
5527        ctlb = t4_read_reg(adap, MPS_TRC_FILTER_MATCH_CTL_B_A + ofst);
5528
5529        if (is_t4(adap->params.chip)) {
5530                *enabled = !!(ctla & TFEN_F);
5531                tp->port =  TFPORT_G(ctla);
5532                tp->invert = !!(ctla & TFINVERTMATCH_F);
5533        } else {
5534                *enabled = !!(ctla & T5_TFEN_F);
5535                tp->port = T5_TFPORT_G(ctla);
5536                tp->invert = !!(ctla & T5_TFINVERTMATCH_F);
5537        }
5538        tp->snap_len = TFCAPTUREMAX_G(ctlb);
5539        tp->min_len = TFMINPKTSIZE_G(ctlb);
5540        tp->skip_ofst = TFOFFSET_G(ctla);
5541        tp->skip_len = TFLENGTH_G(ctla);
5542
5543        ofst = (MPS_TRC_FILTER1_MATCH_A - MPS_TRC_FILTER0_MATCH_A) * idx;
5544        data_reg = MPS_TRC_FILTER0_MATCH_A + ofst;
5545        mask_reg = MPS_TRC_FILTER0_DONT_CARE_A + ofst;
5546
5547        for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
5548                tp->mask[i] = ~t4_read_reg(adap, mask_reg);
5549                tp->data[i] = t4_read_reg(adap, data_reg) & tp->mask[i];
5550        }
5551}
5552
5553/**
5554 *      t4_pmtx_get_stats - returns the HW stats from PMTX
5555 *      @adap: the adapter
5556 *      @cnt: where to store the count statistics
5557 *      @cycles: where to store the cycle statistics
5558 *
5559 *      Returns performance statistics from PMTX.
5560 */
5561void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
5562{
5563        int i;
5564        u32 data[2];
5565
5566        for (i = 0; i < adap->params.arch.pm_stats_cnt; i++) {
5567                t4_write_reg(adap, PM_TX_STAT_CONFIG_A, i + 1);
5568                cnt[i] = t4_read_reg(adap, PM_TX_STAT_COUNT_A);
5569                if (is_t4(adap->params.chip)) {
5570                        cycles[i] = t4_read_reg64(adap, PM_TX_STAT_LSB_A);
5571                } else {
5572                        t4_read_indirect(adap, PM_TX_DBG_CTRL_A,
5573                                         PM_TX_DBG_DATA_A, data, 2,
5574                                         PM_TX_DBG_STAT_MSB_A);
5575                        cycles[i] = (((u64)data[0] << 32) | data[1]);
5576                }
5577        }
5578}
5579
5580/**
5581 *      t4_pmrx_get_stats - returns the HW stats from PMRX
5582 *      @adap: the adapter
5583 *      @cnt: where to store the count statistics
5584 *      @cycles: where to store the cycle statistics
5585 *
5586 *      Returns performance statistics from PMRX.
5587 */
5588void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
5589{
5590        int i;
5591        u32 data[2];
5592
5593        for (i = 0; i < adap->params.arch.pm_stats_cnt; i++) {
5594                t4_write_reg(adap, PM_RX_STAT_CONFIG_A, i + 1);
5595                cnt[i] = t4_read_reg(adap, PM_RX_STAT_COUNT_A);
5596                if (is_t4(adap->params.chip)) {
5597                        cycles[i] = t4_read_reg64(adap, PM_RX_STAT_LSB_A);
5598                } else {
5599                        t4_read_indirect(adap, PM_RX_DBG_CTRL_A,
5600                                         PM_RX_DBG_DATA_A, data, 2,
5601                                         PM_RX_DBG_STAT_MSB_A);
5602                        cycles[i] = (((u64)data[0] << 32) | data[1]);
5603                }
5604        }
5605}
5606
5607/**
5608 *      compute_mps_bg_map - compute the MPS Buffer Group Map for a Port
5609 *      @adap: the adapter
5610 *      @pidx: the port index
5611 *
5612 *      Computes and returns a bitmap indicating which MPS buffer groups are
5613 *      associated with the given Port.  Bit i is set if buffer group i is
5614 *      used by the Port.
5615 */
5616static inline unsigned int compute_mps_bg_map(struct adapter *adapter,
5617                                              int pidx)
5618{
5619        unsigned int chip_version, nports;
5620
5621        chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);
5622        nports = 1 << NUMPORTS_G(t4_read_reg(adapter, MPS_CMN_CTL_A));
5623
5624        switch (chip_version) {
5625        case CHELSIO_T4:
5626        case CHELSIO_T5:
5627                switch (nports) {
5628                case 1: return 0xf;
5629                case 2: return 3 << (2 * pidx);
5630                case 4: return 1 << pidx;
5631                }
5632                break;
5633
5634        case CHELSIO_T6:
5635                switch (nports) {
5636                case 2: return 1 << (2 * pidx);
5637                }
5638                break;
5639        }
5640
5641        dev_err(adapter->pdev_dev, "Need MPS Buffer Group Map for Chip %0x, Nports %d\n",
5642                chip_version, nports);
5643
5644        return 0;
5645}
5646
5647/**
5648 *      t4_get_mps_bg_map - return the buffer groups associated with a port
5649 *      @adapter: the adapter
5650 *      @pidx: the port index
5651 *
5652 *      Returns a bitmap indicating which MPS buffer groups are associated
5653 *      with the given Port.  Bit i is set if buffer group i is used by the
5654 *      Port.
5655 */
5656unsigned int t4_get_mps_bg_map(struct adapter *adapter, int pidx)
5657{
5658        u8 *mps_bg_map;
5659        unsigned int nports;
5660
5661        nports = 1 << NUMPORTS_G(t4_read_reg(adapter, MPS_CMN_CTL_A));
5662        if (pidx >= nports) {
5663                CH_WARN(adapter, "MPS Port Index %d >= Nports %d\n",
5664                        pidx, nports);
5665                return 0;
5666        }
5667
5668        /* If we've already retrieved/computed this, just return the result.
5669         */
5670        mps_bg_map = adapter->params.mps_bg_map;
5671        if (mps_bg_map[pidx])
5672                return mps_bg_map[pidx];
5673
5674        /* Newer Firmware can tell us what the MPS Buffer Group Map is.
5675         * If we're talking to such Firmware, let it tell us.  If the new
5676         * API isn't supported, revert back to old hardcoded way.  The value
5677         * obtained from Firmware is encoded in below format:
5678         *
5679         * val = (( MPSBGMAP[Port 3] << 24 ) |
5680         *        ( MPSBGMAP[Port 2] << 16 ) |
5681         *        ( MPSBGMAP[Port 1] <<  8 ) |
5682         *        ( MPSBGMAP[Port 0] <<  0 ))
5683         */
5684        if (adapter->flags & FW_OK) {
5685                u32 param, val;
5686                int ret;
5687
5688                param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5689                         FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_MPSBGMAP));
5690                ret = t4_query_params_ns(adapter, adapter->mbox, adapter->pf,
5691                                         0, 1, &param, &val);
5692                if (!ret) {
5693                        int p;
5694
5695                        /* Store the BG Map for all of the Ports in order to
5696                         * avoid more calls to the Firmware in the future.
5697                         */
5698                        for (p = 0; p < MAX_NPORTS; p++, val >>= 8)
5699                                mps_bg_map[p] = val & 0xff;
5700
5701                        return mps_bg_map[pidx];
5702                }
5703        }
5704
5705        /* Either we're not talking to the Firmware or we're dealing with
5706         * older Firmware which doesn't support the new API to get the MPS
5707         * Buffer Group Map.  Fall back to computing it ourselves.
5708         */
5709        mps_bg_map[pidx] = compute_mps_bg_map(adapter, pidx);
5710        return mps_bg_map[pidx];
5711}
5712
5713/**
5714 *      t4_get_tp_ch_map - return TP ingress channels associated with a port
5715 *      @adapter: the adapter
5716 *      @pidx: the port index
5717 *
5718 *      Returns a bitmap indicating which TP Ingress Channels are associated
5719 *      with a given Port.  Bit i is set if TP Ingress Channel i is used by
5720 *      the Port.
5721 */
5722unsigned int t4_get_tp_ch_map(struct adapter *adap, int pidx)
5723{
5724        unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip);
5725        unsigned int nports = 1 << NUMPORTS_G(t4_read_reg(adap, MPS_CMN_CTL_A));
5726
5727        if (pidx >= nports) {
5728                dev_warn(adap->pdev_dev, "TP Port Index %d >= Nports %d\n",
5729                         pidx, nports);
5730                return 0;
5731        }
5732
5733        switch (chip_version) {
5734        case CHELSIO_T4:
5735        case CHELSIO_T5:
5736                /* Note that this happens to be the same values as the MPS
5737                 * Buffer Group Map for these Chips.  But we replicate the code
5738                 * here because they're really separate concepts.
5739                 */
5740                switch (nports) {
5741                case 1: return 0xf;
5742                case 2: return 3 << (2 * pidx);
5743                case 4: return 1 << pidx;
5744                }
5745                break;
5746
5747        case CHELSIO_T6:
5748                switch (nports) {
5749                case 2: return 1 << pidx;
5750                }
5751                break;
5752        }
5753
5754        dev_err(adap->pdev_dev, "Need TP Channel Map for Chip %0x, Nports %d\n",
5755                chip_version, nports);
5756        return 0;
5757}
5758
5759/**
5760 *      t4_get_port_type_description - return Port Type string description
5761 *      @port_type: firmware Port Type enumeration
5762 */
5763const char *t4_get_port_type_description(enum fw_port_type port_type)
5764{
5765        static const char *const port_type_description[] = {
5766                "Fiber_XFI",
5767                "Fiber_XAUI",
5768                "BT_SGMII",
5769                "BT_XFI",
5770                "BT_XAUI",
5771                "KX4",
5772                "CX4",
5773                "KX",
5774                "KR",
5775                "SFP",
5776                "BP_AP",
5777                "BP4_AP",
5778                "QSFP_10G",
5779                "QSA",
5780                "QSFP",
5781                "BP40_BA",
5782                "KR4_100G",
5783                "CR4_QSFP",
5784                "CR_QSFP",
5785                "CR2_QSFP",
5786                "SFP28",
5787                "KR_SFP28",
5788        };
5789
5790        if (port_type < ARRAY_SIZE(port_type_description))
5791                return port_type_description[port_type];
5792        return "UNKNOWN";
5793}
5794
5795/**
5796 *      t4_get_port_stats_offset - collect port stats relative to a previous
5797 *                                 snapshot
5798 *      @adap: The adapter
5799 *      @idx: The port
5800 *      @stats: Current stats to fill
5801 *      @offset: Previous stats snapshot
5802 */
5803void t4_get_port_stats_offset(struct adapter *adap, int idx,
5804                              struct port_stats *stats,
5805                              struct port_stats *offset)
5806{
5807        u64 *s, *o;
5808        int i;
5809
5810        t4_get_port_stats(adap, idx, stats);
5811        for (i = 0, s = (u64 *)stats, o = (u64 *)offset;
5812                        i < (sizeof(struct port_stats) / sizeof(u64));
5813                        i++, s++, o++)
5814                *s -= *o;
5815}
5816
5817/**
5818 *      t4_get_port_stats - collect port statistics
5819 *      @adap: the adapter
5820 *      @idx: the port index
5821 *      @p: the stats structure to fill
5822 *
5823 *      Collect statistics related to the given port from HW.
5824 */
5825void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
5826{
5827        u32 bgmap = t4_get_mps_bg_map(adap, idx);
5828        u32 stat_ctl = t4_read_reg(adap, MPS_STAT_CTL_A);
5829
5830#define GET_STAT(name) \
5831        t4_read_reg64(adap, \
5832        (is_t4(adap->params.chip) ? PORT_REG(idx, MPS_PORT_STAT_##name##_L) : \
5833        T5_PORT_REG(idx, MPS_PORT_STAT_##name##_L)))
5834#define GET_STAT_COM(name) t4_read_reg64(adap, MPS_STAT_##name##_L)
5835
5836        p->tx_octets           = GET_STAT(TX_PORT_BYTES);
5837        p->tx_frames           = GET_STAT(TX_PORT_FRAMES);
5838        p->tx_bcast_frames     = GET_STAT(TX_PORT_BCAST);
5839        p->tx_mcast_frames     = GET_STAT(TX_PORT_MCAST);
5840        p->tx_ucast_frames     = GET_STAT(TX_PORT_UCAST);
5841        p->tx_error_frames     = GET_STAT(TX_PORT_ERROR);
5842        p->tx_frames_64        = GET_STAT(TX_PORT_64B);
5843        p->tx_frames_65_127    = GET_STAT(TX_PORT_65B_127B);
5844        p->tx_frames_128_255   = GET_STAT(TX_PORT_128B_255B);
5845        p->tx_frames_256_511   = GET_STAT(TX_PORT_256B_511B);
5846        p->tx_frames_512_1023  = GET_STAT(TX_PORT_512B_1023B);
5847        p->tx_frames_1024_1518 = GET_STAT(TX_PORT_1024B_1518B);
5848        p->tx_frames_1519_max  = GET_STAT(TX_PORT_1519B_MAX);
5849        p->tx_drop             = GET_STAT(TX_PORT_DROP);
5850        p->tx_pause            = GET_STAT(TX_PORT_PAUSE);
5851        p->tx_ppp0             = GET_STAT(TX_PORT_PPP0);
5852        p->tx_ppp1             = GET_STAT(TX_PORT_PPP1);
5853        p->tx_ppp2             = GET_STAT(TX_PORT_PPP2);
5854        p->tx_ppp3             = GET_STAT(TX_PORT_PPP3);
5855        p->tx_ppp4             = GET_STAT(TX_PORT_PPP4);
5856        p->tx_ppp5             = GET_STAT(TX_PORT_PPP5);
5857        p->tx_ppp6             = GET_STAT(TX_PORT_PPP6);
5858        p->tx_ppp7             = GET_STAT(TX_PORT_PPP7);
5859
5860        if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) {
5861                if (stat_ctl & COUNTPAUSESTATTX_F)
5862                        p->tx_frames_64 -= p->tx_pause;
5863                if (stat_ctl & COUNTPAUSEMCTX_F)
5864                        p->tx_mcast_frames -= p->tx_pause;
5865        }
5866        p->rx_octets           = GET_STAT(RX_PORT_BYTES);
5867        p->rx_frames           = GET_STAT(RX_PORT_FRAMES);
5868        p->rx_bcast_frames     = GET_STAT(RX_PORT_BCAST);
5869        p->rx_mcast_frames     = GET_STAT(RX_PORT_MCAST);
5870        p->rx_ucast_frames     = GET_STAT(RX_PORT_UCAST);
5871        p->rx_too_long         = GET_STAT(RX_PORT_MTU_ERROR);
5872        p->rx_jabber           = GET_STAT(RX_PORT_MTU_CRC_ERROR);
5873        p->rx_fcs_err          = GET_STAT(RX_PORT_CRC_ERROR);
5874        p->rx_len_err          = GET_STAT(RX_PORT_LEN_ERROR);
5875        p->rx_symbol_err       = GET_STAT(RX_PORT_SYM_ERROR);
5876        p->rx_runt             = GET_STAT(RX_PORT_LESS_64B);
5877        p->rx_frames_64        = GET_STAT(RX_PORT_64B);
5878        p->rx_frames_65_127    = GET_STAT(RX_PORT_65B_127B);
5879        p->rx_frames_128_255   = GET_STAT(RX_PORT_128B_255B);
5880        p->rx_frames_256_511   = GET_STAT(RX_PORT_256B_511B);
5881        p->rx_frames_512_1023  = GET_STAT(RX_PORT_512B_1023B);
5882        p->rx_frames_1024_1518 = GET_STAT(RX_PORT_1024B_1518B);
5883        p->rx_frames_1519_max  = GET_STAT(RX_PORT_1519B_MAX);
5884        p->rx_pause            = GET_STAT(RX_PORT_PAUSE);
5885        p->rx_ppp0             = GET_STAT(RX_PORT_PPP0);
5886        p->rx_ppp1             = GET_STAT(RX_PORT_PPP1);
5887        p->rx_ppp2             = GET_STAT(RX_PORT_PPP2);
5888        p->rx_ppp3             = GET_STAT(RX_PORT_PPP3);
5889        p->rx_ppp4             = GET_STAT(RX_PORT_PPP4);
5890        p->rx_ppp5             = GET_STAT(RX_PORT_PPP5);
5891        p->rx_ppp6             = GET_STAT(RX_PORT_PPP6);
5892        p->rx_ppp7             = GET_STAT(RX_PORT_PPP7);
5893
5894        if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) {
5895                if (stat_ctl & COUNTPAUSESTATRX_F)
5896                        p->rx_frames_64 -= p->rx_pause;
5897                if (stat_ctl & COUNTPAUSEMCRX_F)
5898                        p->rx_mcast_frames -= p->rx_pause;
5899        }
5900
5901        p->rx_ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0;
5902        p->rx_ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0;
5903        p->rx_ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0;
5904        p->rx_ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0;
5905        p->rx_trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0;
5906        p->rx_trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0;
5907        p->rx_trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0;
5908        p->rx_trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0;
5909
5910#undef GET_STAT
5911#undef GET_STAT_COM
5912}
5913
5914/**
5915 *      t4_get_lb_stats - collect loopback port statistics
5916 *      @adap: the adapter
5917 *      @idx: the loopback port index
5918 *      @p: the stats structure to fill
5919 *
5920 *      Return HW statistics for the given loopback port.
5921 */
5922void t4_get_lb_stats(struct adapter *adap, int idx, struct lb_port_stats *p)
5923{
5924        u32 bgmap = t4_get_mps_bg_map(adap, idx);
5925
5926#define GET_STAT(name) \
5927        t4_read_reg64(adap, \
5928        (is_t4(adap->params.chip) ? \
5929        PORT_REG(idx, MPS_PORT_STAT_LB_PORT_##name##_L) : \
5930        T5_PORT_REG(idx, MPS_PORT_STAT_LB_PORT_##name##_L)))
5931#define GET_STAT_COM(name) t4_read_reg64(adap, MPS_STAT_##name##_L)
5932
5933        p->octets           = GET_STAT(BYTES);
5934        p->frames           = GET_STAT(FRAMES);
5935        p->bcast_frames     = GET_STAT(BCAST);
5936        p->mcast_frames     = GET_STAT(MCAST);
5937        p->ucast_frames     = GET_STAT(UCAST);
5938        p->error_frames     = GET_STAT(ERROR);
5939
5940        p->frames_64        = GET_STAT(64B);
5941        p->frames_65_127    = GET_STAT(65B_127B);
5942        p->frames_128_255   = GET_STAT(128B_255B);
5943        p->frames_256_511   = GET_STAT(256B_511B);
5944        p->frames_512_1023  = GET_STAT(512B_1023B);
5945        p->frames_1024_1518 = GET_STAT(1024B_1518B);
5946        p->frames_1519_max  = GET_STAT(1519B_MAX);
5947        p->drop             = GET_STAT(DROP_FRAMES);
5948
5949        p->ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_DROP_FRAME) : 0;
5950        p->ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_DROP_FRAME) : 0;
5951        p->ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_DROP_FRAME) : 0;
5952        p->ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_DROP_FRAME) : 0;
5953        p->trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_TRUNC_FRAME) : 0;
5954        p->trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_TRUNC_FRAME) : 0;
5955        p->trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_TRUNC_FRAME) : 0;
5956        p->trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_TRUNC_FRAME) : 0;
5957
5958#undef GET_STAT
5959#undef GET_STAT_COM
5960}
5961
5962/*     t4_mk_filtdelwr - create a delete filter WR
5963 *     @ftid: the filter ID
5964 *     @wr: the filter work request to populate
5965 *     @qid: ingress queue to receive the delete notification
5966 *
5967 *     Creates a filter work request to delete the supplied filter.  If @qid is
5968 *     negative the delete notification is suppressed.
5969 */
5970void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid)
5971{
5972        memset(wr, 0, sizeof(*wr));
5973        wr->op_pkd = cpu_to_be32(FW_WR_OP_V(FW_FILTER_WR));
5974        wr->len16_pkd = cpu_to_be32(FW_WR_LEN16_V(sizeof(*wr) / 16));
5975        wr->tid_to_iq = cpu_to_be32(FW_FILTER_WR_TID_V(ftid) |
5976                                    FW_FILTER_WR_NOREPLY_V(qid < 0));
5977        wr->del_filter_to_l2tix = cpu_to_be32(FW_FILTER_WR_DEL_FILTER_F);
5978        if (qid >= 0)
5979                wr->rx_chan_rx_rpl_iq =
5980                        cpu_to_be16(FW_FILTER_WR_RX_RPL_IQ_V(qid));
5981}
5982
5983#define INIT_CMD(var, cmd, rd_wr) do { \
5984        (var).op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_##cmd##_CMD) | \
5985                                        FW_CMD_REQUEST_F | \
5986                                        FW_CMD_##rd_wr##_F); \
5987        (var).retval_len16 = cpu_to_be32(FW_LEN16(var)); \
5988} while (0)
5989
5990int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox,
5991                          u32 addr, u32 val)
5992{
5993        u32 ldst_addrspace;
5994        struct fw_ldst_cmd c;
5995
5996        memset(&c, 0, sizeof(c));
5997        ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FIRMWARE);
5998        c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
5999                                        FW_CMD_REQUEST_F |
6000                                        FW_CMD_WRITE_F |

6001                                        ldst_addrspace);
6002        c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6003        c.u.addrval.addr = cpu_to_be32(addr);
6004        c.u.addrval.val = cpu_to_be32(val);
6005
6006        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6007}
6008
6009/**
6010 *      t4_mdio_rd - read a PHY register through MDIO
6011 *      @adap: the adapter
6012 *      @mbox: mailbox to use for the FW command
6013 *      @phy_addr: the PHY address
6014 *      @mmd: the PHY MMD to access (0 for clause 22 PHYs)
6015 *      @reg: the register to read
6016 *      @valp: where to store the value
6017 *
6018 *      Issues a FW command through the given mailbox to read a PHY register.
6019 */
6020int t4_mdio_rd(struct adapter *adap, unsigned int mbox, unsigned int phy_addr,
6021               unsigned int mmd, unsigned int reg, u16 *valp)
6022{
6023        int ret;
6024        u32 ldst_addrspace;
6025        struct fw_ldst_cmd c;
6026
6027        memset(&c, 0, sizeof(c));
6028        ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_MDIO);
6029        c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
6030                                        FW_CMD_REQUEST_F | FW_CMD_READ_F |
6031                                        ldst_addrspace);
6032        c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6033        c.u.mdio.paddr_mmd = cpu_to_be16(FW_LDST_CMD_PADDR_V(phy_addr) |
6034                                         FW_LDST_CMD_MMD_V(mmd));
6035        c.u.mdio.raddr = cpu_to_be16(reg);
6036
6037        ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
6038        if (ret == 0)
6039                *valp = be16_to_cpu(c.u.mdio.rval);
6040        return ret;
6041}
6042
6043/**
6044 *      t4_mdio_wr - write a PHY register through MDIO
6045 *      @adap: the adapter
6046 *      @mbox: mailbox to use for the FW command
6047 *      @phy_addr: the PHY address
6048 *      @mmd: the PHY MMD to access (0 for clause 22 PHYs)
6049 *      @reg: the register to write
6050 *      @valp: value to write
6051 *
6052 *      Issues a FW command through the given mailbox to write a PHY register.
6053 */
6054int t4_mdio_wr(struct adapter *adap, unsigned int mbox, unsigned int phy_addr,
6055               unsigned int mmd, unsigned int reg, u16 val)
6056{
6057        u32 ldst_addrspace;
6058        struct fw_ldst_cmd c;
6059
6060        memset(&c, 0, sizeof(c));
6061        ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_MDIO);
6062        c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
6063                                        FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
6064                                        ldst_addrspace);
6065        c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6066        c.u.mdio.paddr_mmd = cpu_to_be16(FW_LDST_CMD_PADDR_V(phy_addr) |
6067                                         FW_LDST_CMD_MMD_V(mmd));
6068        c.u.mdio.raddr = cpu_to_be16(reg);
6069        c.u.mdio.rval = cpu_to_be16(val);
6070
6071        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6072}
6073
6074/**
6075 *      t4_sge_decode_idma_state - decode the idma state
6076 *      @adap: the adapter
6077 *      @state: the state idma is stuck in
6078 */
6079void t4_sge_decode_idma_state(struct adapter *adapter, int state)
6080{
6081        static const char * const t4_decode[] = {
6082                "IDMA_IDLE",
6083                "IDMA_PUSH_MORE_CPL_FIFO",
6084                "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO",
6085                "Not used",
6086                "IDMA_PHYSADDR_SEND_PCIEHDR",
6087                "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST",
6088                "IDMA_PHYSADDR_SEND_PAYLOAD",
6089                "IDMA_SEND_FIFO_TO_IMSG",
6090                "IDMA_FL_REQ_DATA_FL_PREP",
6091                "IDMA_FL_REQ_DATA_FL",
6092                "IDMA_FL_DROP",
6093                "IDMA_FL_H_REQ_HEADER_FL",
6094                "IDMA_FL_H_SEND_PCIEHDR",
6095                "IDMA_FL_H_PUSH_CPL_FIFO",
6096                "IDMA_FL_H_SEND_CPL",
6097                "IDMA_FL_H_SEND_IP_HDR_FIRST",
6098                "IDMA_FL_H_SEND_IP_HDR",
6099                "IDMA_FL_H_REQ_NEXT_HEADER_FL",
6100                "IDMA_FL_H_SEND_NEXT_PCIEHDR",
6101                "IDMA_FL_H_SEND_IP_HDR_PADDING",
6102                "IDMA_FL_D_SEND_PCIEHDR",
6103                "IDMA_FL_D_SEND_CPL_AND_IP_HDR",
6104                "IDMA_FL_D_REQ_NEXT_DATA_FL",
6105                "IDMA_FL_SEND_PCIEHDR",
6106                "IDMA_FL_PUSH_CPL_FIFO",
6107                "IDMA_FL_SEND_CPL",
6108                "IDMA_FL_SEND_PAYLOAD_FIRST",
6109                "IDMA_FL_SEND_PAYLOAD",
6110                "IDMA_FL_REQ_NEXT_DATA_FL",
6111                "IDMA_FL_SEND_NEXT_PCIEHDR",
6112                "IDMA_FL_SEND_PADDING",
6113                "IDMA_FL_SEND_COMPLETION_TO_IMSG",
6114                "IDMA_FL_SEND_FIFO_TO_IMSG",
6115                "IDMA_FL_REQ_DATAFL_DONE",
6116                "IDMA_FL_REQ_HEADERFL_DONE",
6117        };
6118        static const char * const t5_decode[] = {
6119                "IDMA_IDLE",
6120                "IDMA_ALMOST_IDLE",
6121                "IDMA_PUSH_MORE_CPL_FIFO",
6122                "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO",
6123                "IDMA_SGEFLRFLUSH_SEND_PCIEHDR",
6124                "IDMA_PHYSADDR_SEND_PCIEHDR",
6125                "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST",
6126                "IDMA_PHYSADDR_SEND_PAYLOAD",
6127                "IDMA_SEND_FIFO_TO_IMSG",
6128                "IDMA_FL_REQ_DATA_FL",
6129                "IDMA_FL_DROP",
6130                "IDMA_FL_DROP_SEND_INC",
6131                "IDMA_FL_H_REQ_HEADER_FL",
6132                "IDMA_FL_H_SEND_PCIEHDR",
6133                "IDMA_FL_H_PUSH_CPL_FIFO",
6134                "IDMA_FL_H_SEND_CPL",
6135                "IDMA_FL_H_SEND_IP_HDR_FIRST",
6136                "IDMA_FL_H_SEND_IP_HDR",
6137                "IDMA_FL_H_REQ_NEXT_HEADER_FL",
6138                "IDMA_FL_H_SEND_NEXT_PCIEHDR",
6139                "IDMA_FL_H_SEND_IP_HDR_PADDING",
6140                "IDMA_FL_D_SEND_PCIEHDR",
6141                "IDMA_FL_D_SEND_CPL_AND_IP_HDR",
6142                "IDMA_FL_D_REQ_NEXT_DATA_FL",
6143                "IDMA_FL_SEND_PCIEHDR",
6144                "IDMA_FL_PUSH_CPL_FIFO",
6145                "IDMA_FL_SEND_CPL",
6146                "IDMA_FL_SEND_PAYLOAD_FIRST",
6147                "IDMA_FL_SEND_PAYLOAD",
6148                "IDMA_FL_REQ_NEXT_DATA_FL",
6149                "IDMA_FL_SEND_NEXT_PCIEHDR",
6150                "IDMA_FL_SEND_PADDING",
6151                "IDMA_FL_SEND_COMPLETION_TO_IMSG",
6152        };
6153        static const char * const t6_decode[] = {
6154                "IDMA_IDLE",
6155                "IDMA_PUSH_MORE_CPL_FIFO",
6156                "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO",
6157                "IDMA_SGEFLRFLUSH_SEND_PCIEHDR",
6158                "IDMA_PHYSADDR_SEND_PCIEHDR",
6159                "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST",
6160                "IDMA_PHYSADDR_SEND_PAYLOAD",
6161                "IDMA_FL_REQ_DATA_FL",
6162                "IDMA_FL_DROP",
6163                "IDMA_FL_DROP_SEND_INC",
6164                "IDMA_FL_H_REQ_HEADER_FL",
6165                "IDMA_FL_H_SEND_PCIEHDR",
6166                "IDMA_FL_H_PUSH_CPL_FIFO",
6167                "IDMA_FL_H_SEND_CPL",
6168                "IDMA_FL_H_SEND_IP_HDR_FIRST",
6169                "IDMA_FL_H_SEND_IP_HDR",
6170                "IDMA_FL_H_REQ_NEXT_HEADER_FL",
6171                "IDMA_FL_H_SEND_NEXT_PCIEHDR",
6172                "IDMA_FL_H_SEND_IP_HDR_PADDING",
6173                "IDMA_FL_D_SEND_PCIEHDR",
6174                "IDMA_FL_D_SEND_CPL_AND_IP_HDR",
6175                "IDMA_FL_D_REQ_NEXT_DATA_FL",
6176                "IDMA_FL_SEND_PCIEHDR",
6177                "IDMA_FL_PUSH_CPL_FIFO",
6178                "IDMA_FL_SEND_CPL",
6179                "IDMA_FL_SEND_PAYLOAD_FIRST",
6180                "IDMA_FL_SEND_PAYLOAD",
6181                "IDMA_FL_REQ_NEXT_DATA_FL",
6182                "IDMA_FL_SEND_NEXT_PCIEHDR",
6183                "IDMA_FL_SEND_PADDING",
6184                "IDMA_FL_SEND_COMPLETION_TO_IMSG",
6185        };
6186        static const u32 sge_regs[] = {
6187                SGE_DEBUG_DATA_LOW_INDEX_2_A,
6188                SGE_DEBUG_DATA_LOW_INDEX_3_A,
6189                SGE_DEBUG_DATA_HIGH_INDEX_10_A,
6190        };
6191        const char **sge_idma_decode;
6192        int sge_idma_decode_nstates;
6193        int i;
6194        unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);
6195
6196        /* Select the right set of decode strings to dump depending on the
6197         * adapter chip type.
6198         */
6199        switch (chip_version) {
6200        case CHELSIO_T4:
6201                sge_idma_decode = (const char **)t4_decode;
6202                sge_idma_decode_nstates = ARRAY_SIZE(t4_decode);
6203                break;
6204
6205        case CHELSIO_T5:
6206                sge_idma_decode = (const char **)t5_decode;
6207                sge_idma_decode_nstates = ARRAY_SIZE(t5_decode);
6208                break;
6209
6210        case CHELSIO_T6:
6211                sge_idma_decode = (const char **)t6_decode;
6212                sge_idma_decode_nstates = ARRAY_SIZE(t6_decode);
6213                break;
6214
6215        default:
6216                dev_err(adapter->pdev_dev,
6217                        "Unsupported chip version %d\n", chip_version);
6218                return;
6219        }
6220
6221        if (is_t4(adapter->params.chip)) {
6222                sge_idma_decode = (const char **)t4_decode;
6223                sge_idma_decode_nstates = ARRAY_SIZE(t4_decode);
6224        } else {
6225                sge_idma_decode = (const char **)t5_decode;
6226                sge_idma_decode_nstates = ARRAY_SIZE(t5_decode);
6227        }
6228
6229        if (state < sge_idma_decode_nstates)
6230                CH_WARN(adapter, "idma state %s\n", sge_idma_decode[state]);
6231        else
6232                CH_WARN(adapter, "idma state %d unknown\n", state);
6233
6234        for (i = 0; i < ARRAY_SIZE(sge_regs); i++)
6235                CH_WARN(adapter, "SGE register %#x value %#x\n",
6236                        sge_regs[i], t4_read_reg(adapter, sge_regs[i]));
6237}
6238
6239/**
6240 *      t4_sge_ctxt_flush - flush the SGE context cache
6241 *      @adap: the adapter
6242 *      @mbox: mailbox to use for the FW command
6243 *
6244 *      Issues a FW command through the given mailbox to flush the
6245 *      SGE context cache.
6246 */
6247int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox)
6248{
6249        int ret;
6250        u32 ldst_addrspace;
6251        struct fw_ldst_cmd c;
6252
6253        memset(&c, 0, sizeof(c));
6254        ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_SGE_EGRC);
6255        c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
6256                                        FW_CMD_REQUEST_F | FW_CMD_READ_F |
6257                                        ldst_addrspace);
6258        c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6259        c.u.idctxt.msg_ctxtflush = cpu_to_be32(FW_LDST_CMD_CTXTFLUSH_F);
6260
6261        ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
6262        return ret;
6263}
6264
6265/**
6266 *      t4_fw_hello - establish communication with FW
6267 *      @adap: the adapter
6268 *      @mbox: mailbox to use for the FW command
6269 *      @evt_mbox: mailbox to receive async FW events
6270 *      @master: specifies the caller's willingness to be the device master
6271 *      @state: returns the current device state (if non-NULL)
6272 *
6273 *      Issues a command to establish communication with FW.  Returns either
6274 *      an error (negative integer) or the mailbox of the Master PF.
6275 */
6276int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox,
6277                enum dev_master master, enum dev_state *state)
6278{
6279        int ret;
6280        struct fw_hello_cmd c;
6281        u32 v;
6282        unsigned int master_mbox;
6283        int retries = FW_CMD_HELLO_RETRIES;
6284
6285retry:
6286        memset(&c, 0, sizeof(c));
6287        INIT_CMD(c, HELLO, WRITE);
6288        c.err_to_clearinit = cpu_to_be32(
6289                FW_HELLO_CMD_MASTERDIS_V(master == MASTER_CANT) |
6290                FW_HELLO_CMD_MASTERFORCE_V(master == MASTER_MUST) |
6291                FW_HELLO_CMD_MBMASTER_V(master == MASTER_MUST ?
6292                                        mbox : FW_HELLO_CMD_MBMASTER_M) |
6293                FW_HELLO_CMD_MBASYNCNOT_V(evt_mbox) |
6294                FW_HELLO_CMD_STAGE_V(fw_hello_cmd_stage_os) |
6295                FW_HELLO_CMD_CLEARINIT_F);
6296
6297        /*
6298         * Issue the HELLO command to the firmware.  If it's not successful
6299         * but indicates that we got a "busy" or "timeout" condition, retry
6300         * the HELLO until we exhaust our retry limit.  If we do exceed our
6301         * retry limit, check to see if the firmware left us any error
6302         * information and report that if so.
6303         */
6304        ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
6305        if (ret < 0) {
6306                if ((ret == -EBUSY || ret == -ETIMEDOUT) && retries-- > 0)
6307                        goto retry;
6308                if (t4_read_reg(adap, PCIE_FW_A) & PCIE_FW_ERR_F)
6309                        t4_report_fw_error(adap);
6310                return ret;
6311        }
6312
6313        v = be32_to_cpu(c.err_to_clearinit);
6314        master_mbox = FW_HELLO_CMD_MBMASTER_G(v);
6315        if (state) {
6316                if (v & FW_HELLO_CMD_ERR_F)
6317                        *state = DEV_STATE_ERR;
6318                else if (v & FW_HELLO_CMD_INIT_F)
6319                        *state = DEV_STATE_INIT;
6320                else
6321                        *state = DEV_STATE_UNINIT;
6322        }
6323
6324        /*
6325         * If we're not the Master PF then we need to wait around for the
6326         * Master PF Driver to finish setting up the adapter.
6327         *
6328         * Note that we also do this wait if we're a non-Master-capable PF and
6329         * there is no current Master PF; a Master PF may show up momentarily
6330         * and we wouldn't want to fail pointlessly.  (This can happen when an
6331         * OS loads lots of different drivers rapidly at the same time).  In
6332         * this case, the Master PF returned by the firmware will be
6333         * PCIE_FW_MASTER_M so the test below will work ...
6334         */
6335        if ((v & (FW_HELLO_CMD_ERR_F|FW_HELLO_CMD_INIT_F)) == 0 &&
6336            master_mbox != mbox) {
6337                int waiting = FW_CMD_HELLO_TIMEOUT;
6338
6339                /*
6340                 * Wait for the firmware to either indicate an error or
6341                 * initialized state.  If we see either of these we bail out
6342                 * and report the issue to the caller.  If we exhaust the
6343                 * "hello timeout" and we haven't exhausted our retries, try
6344                 * again.  Otherwise bail with a timeout error.
6345                 */
6346                for (;;) {
6347                        u32 pcie_fw;
6348
6349                        msleep(50);
6350                        waiting -= 50;
6351
6352                        /*
6353                         * If neither Error nor Initialialized are indicated
6354                         * by the firmware keep waiting till we exaust our
6355                         * timeout ... and then retry if we haven't exhausted
6356                         * our retries ...
6357                         */
6358                        pcie_fw = t4_read_reg(adap, PCIE_FW_A);
6359                        if (!(pcie_fw & (PCIE_FW_ERR_F|PCIE_FW_INIT_F))) {
6360                                if (waiting <= 0) {
6361                                        if (retries-- > 0)
6362                                                goto retry;
6363
6364                                        return -ETIMEDOUT;
6365                                }
6366                                continue;
6367                        }
6368
6369                        /*
6370                         * We either have an Error or Initialized condition
6371                         * report errors preferentially.
6372                         */
6373                        if (state) {
6374                                if (pcie_fw & PCIE_FW_ERR_F)
6375                                        *state = DEV_STATE_ERR;
6376                                else if (pcie_fw & PCIE_FW_INIT_F)
6377                                        *state = DEV_STATE_INIT;
6378                        }
6379
6380                        /*
6381                         * If we arrived before a Master PF was selected and
6382                         * there's not a valid Master PF, grab its identity
6383                         * for our caller.
6384                         */
6385                        if (master_mbox == PCIE_FW_MASTER_M &&
6386                            (pcie_fw & PCIE_FW_MASTER_VLD_F))
6387                                master_mbox = PCIE_FW_MASTER_G(pcie_fw);
6388                        break;
6389                }
6390        }
6391
6392        return master_mbox;
6393}
6394
6395/**
6396 *      t4_fw_bye - end communication with FW
6397 *      @adap: the adapter
6398 *      @mbox: mailbox to use for the FW command
6399 *
6400 *      Issues a command to terminate communication with FW.
6401 */
6402int t4_fw_bye(struct adapter *adap, unsigned int mbox)
6403{
6404        struct fw_bye_cmd c;
6405
6406        memset(&c, 0, sizeof(c));
6407        INIT_CMD(c, BYE, WRITE);
6408        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6409}
6410
6411/**
6412 *      t4_init_cmd - ask FW to initialize the device
6413 *      @adap: the adapter
6414 *      @mbox: mailbox to use for the FW command
6415 *
6416 *      Issues a command to FW to partially initialize the device.  This
6417 *      performs initialization that generally doesn't depend on user input.
6418 */
6419int t4_early_init(struct adapter *adap, unsigned int mbox)
6420{
6421        struct fw_initialize_cmd c;
6422
6423        memset(&c, 0, sizeof(c));
6424        INIT_CMD(c, INITIALIZE, WRITE);
6425        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6426}
6427
6428/**
6429 *      t4_fw_reset - issue a reset to FW
6430 *      @adap: the adapter
6431 *      @mbox: mailbox to use for the FW command
6432 *      @reset: specifies the type of reset to perform
6433 *
6434 *      Issues a reset command of the specified type to FW.
6435 */
6436int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset)
6437{
6438        struct fw_reset_cmd c;
6439
6440        memset(&c, 0, sizeof(c));
6441        INIT_CMD(c, RESET, WRITE);
6442        c.val = cpu_to_be32(reset);
6443        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6444}
6445
6446/**
6447 *      t4_fw_halt - issue a reset/halt to FW and put uP into RESET
6448 *      @adap: the adapter
6449 *      @mbox: mailbox to use for the FW RESET command (if desired)
6450 *      @force: force uP into RESET even if FW RESET command fails
6451 *
6452 *      Issues a RESET command to firmware (if desired) with a HALT indication
6453 *      and then puts the microprocessor into RESET state.  The RESET command
6454 *      will only be issued if a legitimate mailbox is provided (mbox <=
6455 *      PCIE_FW_MASTER_M).
6456 *
6457 *      This is generally used in order for the host to safely manipulate the
6458 *      adapter without fear of conflicting with whatever the firmware might
6459 *      be doing.  The only way out of this state is to RESTART the firmware
6460 *      ...
6461 */
6462static int t4_fw_halt(struct adapter *adap, unsigned int mbox, int force)
6463{
6464        int ret = 0;
6465
6466        /*
6467         * If a legitimate mailbox is provided, issue a RESET command
6468         * with a HALT indication.
6469         */
6470        if (mbox <= PCIE_FW_MASTER_M) {
6471                struct fw_reset_cmd c;
6472
6473                memset(&c, 0, sizeof(c));
6474                INIT_CMD(c, RESET, WRITE);
6475                c.val = cpu_to_be32(PIORST_F | PIORSTMODE_F);
6476                c.halt_pkd = cpu_to_be32(FW_RESET_CMD_HALT_F);
6477                ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6478        }
6479
6480        /*
6481         * Normally we won't complete the operation if the firmware RESET
6482         * command fails but if our caller insists we'll go ahead and put the
6483         * uP into RESET.  This can be useful if the firmware is hung or even
6484         * missing ...  We'll have to take the risk of putting the uP into
6485         * RESET without the cooperation of firmware in that case.
6486         *
6487         * We also force the firmware's HALT flag to be on in case we bypassed
6488         * the firmware RESET command above or we're dealing with old firmware
6489         * which doesn't have the HALT capability.  This will serve as a flag
6490         * for the incoming firmware to know that it's coming out of a HALT
6491         * rather than a RESET ... if it's new enough to understand that ...
6492         */
6493        if (ret == 0 || force) {
6494                t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, UPCRST_F);
6495                t4_set_reg_field(adap, PCIE_FW_A, PCIE_FW_HALT_F,
6496                                 PCIE_FW_HALT_F);
6497        }
6498
6499        /*
6500         * And we always return the result of the firmware RESET command
6501         * even when we force the uP into RESET ...
6502         */
6503        return ret;
6504}
6505
6506/**
6507 *      t4_fw_restart - restart the firmware by taking the uP out of RESET
6508 *      @adap: the adapter
6509 *      @reset: if we want to do a RESET to restart things
6510 *
6511 *      Restart firmware previously halted by t4_fw_halt().  On successful
6512 *      return the previous PF Master remains as the new PF Master and there
6513 *      is no need to issue a new HELLO command, etc.
6514 *
6515 *      We do this in two ways:
6516 *
6517 *       1. If we're dealing with newer firmware we'll simply want to take
6518 *          the chip's microprocessor out of RESET.  This will cause the
6519 *          firmware to start up from its start vector.  And then we'll loop
6520 *          until the firmware indicates it's started again (PCIE_FW.HALT
6521 *          reset to 0) or we timeout.
6522 *
6523 *       2. If we're dealing with older firmware then we'll need to RESET
6524 *          the chip since older firmware won't recognize the PCIE_FW.HALT
6525 *          flag and automatically RESET itself on startup.
6526 */
6527static int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset)
6528{
6529        if (reset) {
6530                /*
6531                 * Since we're directing the RESET instead of the firmware
6532                 * doing it automatically, we need to clear the PCIE_FW.HALT
6533                 * bit.
6534                 */
6535                t4_set_reg_field(adap, PCIE_FW_A, PCIE_FW_HALT_F, 0);
6536
6537                /*
6538                 * If we've been given a valid mailbox, first try to get the
6539                 * firmware to do the RESET.  If that works, great and we can
6540                 * return success.  Otherwise, if we haven't been given a
6541                 * valid mailbox or the RESET command failed, fall back to
6542                 * hitting the chip with a hammer.
6543                 */
6544                if (mbox <= PCIE_FW_MASTER_M) {
6545                        t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, 0);
6546                        msleep(100);
6547                        if (t4_fw_reset(adap, mbox,
6548                                        PIORST_F | PIORSTMODE_F) == 0)
6549                                return 0;
6550                }
6551
6552                t4_write_reg(adap, PL_RST_A, PIORST_F | PIORSTMODE_F);
6553                msleep(2000);
6554        } else {
6555                int ms;
6556
6557                t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, 0);
6558                for (ms = 0; ms < FW_CMD_MAX_TIMEOUT; ) {
6559                        if (!(t4_read_reg(adap, PCIE_FW_A) & PCIE_FW_HALT_F))
6560                                return 0;
6561                        msleep(100);
6562                        ms += 100;
6563                }
6564                return -ETIMEDOUT;
6565        }
6566        return 0;
6567}
6568
6569/**
6570 *      t4_fw_upgrade - perform all of the steps necessary to upgrade FW
6571 *      @adap: the adapter
6572 *      @mbox: mailbox to use for the FW RESET command (if desired)
6573 *      @fw_data: the firmware image to write
6574 *      @size: image size
6575 *      @force: force upgrade even if firmware doesn't cooperate
6576 *
6577 *      Perform all of the steps necessary for upgrading an adapter's
6578 *      firmware image.  Normally this requires the cooperation of the
6579 *      existing firmware in order to halt all existing activities
6580 *      but if an invalid mailbox token is passed in we skip that step
6581 *      (though we'll still put the adapter microprocessor into RESET in
6582 *      that case).
6583 *
6584 *      On successful return the new firmware will have been loaded and
6585 *      the adapter will have been fully RESET losing all previous setup
6586 *      state.  On unsuccessful return the adapter may be completely hosed ...
6587 *      positive errno indicates that the adapter is ~probably~ intact, a
6588 *      negative errno indicates that things are looking bad ...
6589 */
6590int t4_fw_upgrade(struct adapter *adap, unsigned int mbox,
6591                  const u8 *fw_data, unsigned int size, int force)
6592{
6593        const struct fw_hdr *fw_hdr = (const struct fw_hdr *)fw_data;
6594        int reset, ret;
6595
6596        if (!t4_fw_matches_chip(adap, fw_hdr))
6597                return -EINVAL;
6598
6599        /* Disable FW_OK flag so that mbox commands with FW_OK flag set
6600         * wont be sent when we are flashing FW.
6601         */
6602        adap->flags &= ~FW_OK;
6603
6604        ret = t4_fw_halt(adap, mbox, force);
6605        if (ret < 0 && !force)
6606                goto out;
6607
6608        ret = t4_load_fw(adap, fw_data, size);
6609        if (ret < 0)
6610                goto out;
6611
6612        /*
6613         * If there was a Firmware Configuration File stored in FLASH,
6614         * there's a good chance that it won't be compatible with the new
6615         * Firmware.  In order to prevent difficult to diagnose adapter
6616         * initialization issues, we clear out the Firmware Configuration File
6617         * portion of the FLASH .  The user will need to re-FLASH a new
6618         * Firmware Configuration File which is compatible with the new
6619         * Firmware if that's desired.
6620         */
6621        (void)t4_load_cfg(adap, NULL, 0);
6622
6623        /*
6624         * Older versions of the firmware don't understand the new
6625         * PCIE_FW.HALT flag and so won't know to perform a RESET when they
6626         * restart.  So for newly loaded older firmware we'll have to do the
6627         * RESET for it so it starts up on a clean slate.  We can tell if
6628         * the newly loaded firmware will handle this right by checking
6629         * its header flags to see if it advertises the capability.
6630         */
6631        reset = ((be32_to_cpu(fw_hdr->flags) & FW_HDR_FLAGS_RESET_HALT) == 0);
6632        ret = t4_fw_restart(adap, mbox, reset);
6633
6634        /* Grab potentially new Firmware Device Log parameters so we can see
6635         * how healthy the new Firmware is.  It's okay to contact the new
6636         * Firmware for these parameters even though, as far as it's
6637         * concerned, we've never said "HELLO" to it ...
6638         */
6639        (void)t4_init_devlog_params(adap);
6640out:
6641        adap->flags |= FW_OK;
6642        return ret;
6643}
6644
6645/**
6646 *      t4_fl_pkt_align - return the fl packet alignment
6647 *      @adap: the adapter
6648 *
6649 *      T4 has a single field to specify the packing and padding boundary.
6650 *      T5 onwards has separate fields for this and hence the alignment for
6651 *      next packet offset is maximum of these two.
6652 *
6653 */
6654int t4_fl_pkt_align(struct adapter *adap)
6655{
6656        u32 sge_control, sge_control2;
6657        unsigned int ingpadboundary, ingpackboundary, fl_align, ingpad_shift;
6658
6659        sge_control = t4_read_reg(adap, SGE_CONTROL_A);
6660
6661        /* T4 uses a single control field to specify both the PCIe Padding and
6662         * Packing Boundary.  T5 introduced the ability to specify these
6663         * separately.  The actual Ingress Packet Data alignment boundary
6664         * within Packed Buffer Mode is the maximum of these two
6665         * specifications.  (Note that it makes no real practical sense to
6666         * have the Pading Boudary be larger than the Packing Boundary but you
6667         * could set the chip up that way and, in fact, legacy T4 code would
6668         * end doing this because it would initialize the Padding Boundary and
6669         * leave the Packing Boundary initialized to 0 (16 bytes).)
6670         * Padding Boundary values in T6 starts from 8B,
6671         * where as it is 32B for T4 and T5.
6672         */
6673        if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
6674                ingpad_shift = INGPADBOUNDARY_SHIFT_X;
6675        else
6676                ingpad_shift = T6_INGPADBOUNDARY_SHIFT_X;
6677
6678        ingpadboundary = 1 << (INGPADBOUNDARY_G(sge_control) + ingpad_shift);
6679
6680        fl_align = ingpadboundary;
6681        if (!is_t4(adap->params.chip)) {
6682                /* T5 has a weird interpretation of one of the PCIe Packing
6683                 * Boundary values.  No idea why ...
6684                 */
6685                sge_control2 = t4_read_reg(adap, SGE_CONTROL2_A);
6686                ingpackboundary = INGPACKBOUNDARY_G(sge_control2);
6687                if (ingpackboundary == INGPACKBOUNDARY_16B_X)
6688                        ingpackboundary = 16;
6689                else
6690                        ingpackboundary = 1 << (ingpackboundary +
6691                                                INGPACKBOUNDARY_SHIFT_X);
6692
6693                fl_align = max(ingpadboundary, ingpackboundary);
6694        }
6695        return fl_align;
6696}
6697
6698/**
6699 *      t4_fixup_host_params - fix up host-dependent parameters
6700 *      @adap: the adapter
6701 *      @page_size: the host's Base Page Size
6702 *      @cache_line_size: the host's Cache Line Size
6703 *
6704 *      Various registers in T4 contain values which are dependent on the
6705 *      host's Base Page and Cache Line Sizes.  This function will fix all of
6706 *      those registers with the appropriate values as passed in ...
6707 */
6708int t4_fixup_host_params(struct adapter *adap, unsigned int page_size,
6709                         unsigned int cache_line_size)
6710{
6711        unsigned int page_shift = fls(page_size) - 1;
6712        unsigned int sge_hps = page_shift - 10;
6713        unsigned int stat_len = cache_line_size > 64 ? 128 : 64;
6714        unsigned int fl_align = cache_line_size < 32 ? 32 : cache_line_size;
6715        unsigned int fl_align_log = fls(fl_align) - 1;
6716
6717        t4_write_reg(adap, SGE_HOST_PAGE_SIZE_A,
6718                     HOSTPAGESIZEPF0_V(sge_hps) |
6719                     HOSTPAGESIZEPF1_V(sge_hps) |
6720                     HOSTPAGESIZEPF2_V(sge_hps) |
6721                     HOSTPAGESIZEPF3_V(sge_hps) |
6722                     HOSTPAGESIZEPF4_V(sge_hps) |
6723                     HOSTPAGESIZEPF5_V(sge_hps) |
6724                     HOSTPAGESIZEPF6_V(sge_hps) |
6725                     HOSTPAGESIZEPF7_V(sge_hps));
6726
6727        if (is_t4(adap->params.chip)) {
6728                t4_set_reg_field(adap, SGE_CONTROL_A,
6729                                 INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
6730                                 EGRSTATUSPAGESIZE_F,
6731                                 INGPADBOUNDARY_V(fl_align_log -
6732                                                  INGPADBOUNDARY_SHIFT_X) |
6733                                 EGRSTATUSPAGESIZE_V(stat_len != 64));
6734        } else {
6735                unsigned int pack_align;
6736                unsigned int ingpad, ingpack;
6737                unsigned int pcie_cap;
6738
6739                /* T5 introduced the separation of the Free List Padding and
6740                 * Packing Boundaries.  Thus, we can select a smaller Padding
6741                 * Boundary to avoid uselessly chewing up PCIe Link and Memory
6742                 * Bandwidth, and use a Packing Boundary which is large enough
6743                 * to avoid false sharing between CPUs, etc.
6744                 *
6745                 * For the PCI Link, the smaller the Padding Boundary the
6746                 * better.  For the Memory Controller, a smaller Padding
6747                 * Boundary is better until we cross under the Memory Line
6748                 * Size (the minimum unit of transfer to/from Memory).  If we
6749                 * have a Padding Boundary which is smaller than the Memory
6750                 * Line Size, that'll involve a Read-Modify-Write cycle on the
6751                 * Memory Controller which is never good.
6752                 */
6753
6754                /* We want the Packing Boundary to be based on the Cache Line
6755                 * Size in order to help avoid False Sharing performance
6756                 * issues between CPUs, etc.  We also want the Packing
6757                 * Boundary to incorporate the PCI-E Maximum Payload Size.  We
6758                 * get best performance when the Packing Boundary is a
6759                 * multiple of the Maximum Payload Size.
6760                 */
6761                pack_align = fl_align;
6762                pcie_cap = pci_find_capability(adap->pdev, PCI_CAP_ID_EXP);
6763                if (pcie_cap) {
6764                        unsigned int mps, mps_log;
6765                        u16 devctl;
6766
6767                        /* The PCIe Device Control Maximum Payload Size field
6768                         * [bits 7:5] encodes sizes as powers of 2 starting at
6769                         * 128 bytes.
6770                         */
6771                        pci_read_config_word(adap->pdev,
6772                                             pcie_cap + PCI_EXP_DEVCTL,
6773                                             &devctl);
6774                        mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7;
6775                        mps = 1 << mps_log;
6776                        if (mps > pack_align)
6777                                pack_align = mps;
6778                }
6779
6780                /* N.B. T5/T6 have a crazy special interpretation of the "0"
6781                 * value for the Packing Boundary.  This corresponds to 16
6782                 * bytes instead of the expected 32 bytes.  So if we want 32
6783                 * bytes, the best we can really do is 64 bytes ...
6784                 */
6785                if (pack_align <= 16) {
6786                        ingpack = INGPACKBOUNDARY_16B_X;
6787                        fl_align = 16;
6788                } else if (pack_align == 32) {
6789                        ingpack = INGPACKBOUNDARY_64B_X;
6790                        fl_align = 64;
6791                } else {
6792                        unsigned int pack_align_log = fls(pack_align) - 1;
6793
6794                        ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X;
6795                        fl_align = pack_align;
6796                }
6797
6798                /* Use the smallest Ingress Padding which isn't smaller than
6799                 * the Memory Controller Read/Write Size.  We'll take that as
6800                 * being 8 bytes since we don't know of any system with a
6801                 * wider Memory Controller Bus Width.
6802                 */
6803                if (is_t5(adap->params.chip))
6804                        ingpad = INGPADBOUNDARY_32B_X;
6805                else
6806                        ingpad = T6_INGPADBOUNDARY_8B_X;
6807
6808                t4_set_reg_field(adap, SGE_CONTROL_A,
6809                                 INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
6810                                 EGRSTATUSPAGESIZE_F,
6811                                 INGPADBOUNDARY_V(ingpad) |
6812                                 EGRSTATUSPAGESIZE_V(stat_len != 64));
6813                t4_set_reg_field(adap, SGE_CONTROL2_A,
6814                                 INGPACKBOUNDARY_V(INGPACKBOUNDARY_M),
6815                                 INGPACKBOUNDARY_V(ingpack));
6816        }
6817        /*
6818         * Adjust various SGE Free List Host Buffer Sizes.
6819         *
6820         * This is something of a crock since we're using fixed indices into
6821         * the array which are also known by the sge.c code and the T4
6822         * Firmware Configuration File.  We need to come up with a much better
6823         * approach to managing this array.  For now, the first four entries
6824         * are:
6825         *
6826         *   0: Host Page Size
6827         *   1: 64KB
6828         *   2: Buffer size corresponding to 1500 byte MTU (unpacked mode)
6829         *   3: Buffer size corresponding to 9000 byte MTU (unpacked mode)
6830         *
6831         * For the single-MTU buffers in unpacked mode we need to include
6832         * space for the SGE Control Packet Shift, 14 byte Ethernet header,
6833         * possible 4 byte VLAN tag, all rounded up to the next Ingress Packet
6834         * Padding boundry.  All of these are accommodated in the Factory
6835         * Default Firmware Configuration File but we need to adjust it for
6836         * this host's cache line size.
6837         */
6838        t4_write_reg(adap, SGE_FL_BUFFER_SIZE0_A, page_size);
6839        t4_write_reg(adap, SGE_FL_BUFFER_SIZE2_A,
6840                     (t4_read_reg(adap, SGE_FL_BUFFER_SIZE2_A) + fl_align-1)
6841                     & ~(fl_align-1));
6842        t4_write_reg(adap, SGE_FL_BUFFER_SIZE3_A,
6843                     (t4_read_reg(adap, SGE_FL_BUFFER_SIZE3_A) + fl_align-1)
6844                     & ~(fl_align-1));
6845
6846        t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(page_shift - 12));
6847
6848        return 0;
6849}
6850
6851/**
6852 *      t4_fw_initialize - ask FW to initialize the device
6853 *      @adap: the adapter
6854 *      @mbox: mailbox to use for the FW command
6855 *
6856 *      Issues a command to FW to partially initialize the device.  This
6857 *      performs initialization that generally doesn't depend on user input.
6858 */
6859int t4_fw_initialize(struct adapter *adap, unsigned int mbox)
6860{
6861        struct fw_initialize_cmd c;
6862
6863        memset(&c, 0, sizeof(c));
6864        INIT_CMD(c, INITIALIZE, WRITE);
6865        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6866}
6867
6868/**
6869 *      t4_query_params_rw - query FW or device parameters
6870 *      @adap: the adapter
6871 *      @mbox: mailbox to use for the FW command
6872 *      @pf: the PF
6873 *      @vf: the VF
6874 *      @nparams: the number of parameters
6875 *      @params: the parameter names
6876 *      @val: the parameter values
6877 *      @rw: Write and read flag
6878 *      @sleep_ok: if true, we may sleep awaiting mbox cmd completion
6879 *
6880 *      Reads the value of FW or device parameters.  Up to 7 parameters can be
6881 *      queried at once.
6882 */
6883int t4_query_params_rw(struct adapter *adap, unsigned int mbox, unsigned int pf,
6884                       unsigned int vf, unsigned int nparams, const u32 *params,
6885                       u32 *val, int rw, bool sleep_ok)
6886{
6887        int i, ret;
6888        struct fw_params_cmd c;
6889        __be32 *p = &c.param[0].mnem;
6890
6891        if (nparams > 7)
6892                return -EINVAL;
6893
6894        memset(&c, 0, sizeof(c));
6895        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) |
6896                                  FW_CMD_REQUEST_F | FW_CMD_READ_F |
6897                                  FW_PARAMS_CMD_PFN_V(pf) |
6898                                  FW_PARAMS_CMD_VFN_V(vf));
6899        c.retval_len16 = cpu_to_be32(FW_LEN16(c));
6900
6901        for (i = 0; i < nparams; i++) {
6902                *p++ = cpu_to_be32(*params++);
6903                if (rw)
6904                        *p = cpu_to_be32(*(val + i));
6905                p++;
6906        }
6907
6908        ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok);
6909        if (ret == 0)
6910                for (i = 0, p = &c.param[0].val; i < nparams; i++, p += 2)
6911                        *val++ = be32_to_cpu(*p);
6912        return ret;
6913}
6914
6915int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
6916                    unsigned int vf, unsigned int nparams, const u32 *params,
6917                    u32 *val)
6918{
6919        return t4_query_params_rw(adap, mbox, pf, vf, nparams, params, val, 0,
6920                                  true);
6921}
6922
6923int t4_query_params_ns(struct adapter *adap, unsigned int mbox, unsigned int pf,
6924                       unsigned int vf, unsigned int nparams, const u32 *params,
6925                       u32 *val)
6926{
6927        return t4_query_params_rw(adap, mbox, pf, vf, nparams, params, val, 0,
6928                                  false);
6929}
6930
6931/**
6932 *      t4_set_params_timeout - sets FW or device parameters
6933 *      @adap: the adapter
6934 *      @mbox: mailbox to use for the FW command
6935 *      @pf: the PF
6936 *      @vf: the VF
6937 *      @nparams: the number of parameters
6938 *      @params: the parameter names
6939 *      @val: the parameter values
6940 *      @timeout: the timeout time
6941 *
6942 *      Sets the value of FW or device parameters.  Up to 7 parameters can be
6943 *      specified at once.
6944 */
6945int t4_set_params_timeout(struct adapter *adap, unsigned int mbox,
6946                          unsigned int pf, unsigned int vf,
6947                          unsigned int nparams, const u32 *params,
6948                          const u32 *val, int timeout)
6949{
6950        struct fw_params_cmd c;
6951        __be32 *p = &c.param[0].mnem;
6952
6953        if (nparams > 7)
6954                return -EINVAL;
6955
6956        memset(&c, 0, sizeof(c));
6957        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) |
6958                                  FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
6959                                  FW_PARAMS_CMD_PFN_V(pf) |
6960                                  FW_PARAMS_CMD_VFN_V(vf));
6961        c.retval_len16 = cpu_to_be32(FW_LEN16(c));
6962
6963        while (nparams--) {
6964                *p++ = cpu_to_be32(*params++);
6965                *p++ = cpu_to_be32(*val++);
6966        }
6967
6968        return t4_wr_mbox_timeout(adap, mbox, &c, sizeof(c), NULL, timeout);
6969}
6970
6971/**
6972 *      t4_set_params - sets FW or device parameters
6973 *      @adap: the adapter
6974 *      @mbox: mailbox to use for the FW command
6975 *      @pf: the PF
6976 *      @vf: the VF
6977 *      @nparams: the number of parameters
6978 *      @params: the parameter names
6979 *      @val: the parameter values
6980 *
6981 *      Sets the value of FW or device parameters.  Up to 7 parameters can be
6982 *      specified at once.
6983 */
6984int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
6985                  unsigned int vf, unsigned int nparams, const u32 *params,
6986                  const u32 *val)
6987{
6988        return t4_set_params_timeout(adap, mbox, pf, vf, nparams, params, val,
6989                                     FW_CMD_MAX_TIMEOUT);
6990}
6991
6992/**
6993 *      t4_cfg_pfvf - configure PF/VF resource limits
6994 *      @adap: the adapter
6995 *      @mbox: mailbox to use for the FW command
6996 *      @pf: the PF being configured
6997 *      @vf: the VF being configured
6998 *      @txq: the max number of egress queues
6999 *      @txq_eth_ctrl: the max number of egress Ethernet or control queues
7000 *      @rxqi: the max number of interrupt-capable ingress queues

7001 *      @rxq: the max number of interruptless ingress queues
7002 *      @tc: the PCI traffic class
7003 *      @vi: the max number of virtual interfaces
7004 *      @cmask: the channel access rights mask for the PF/VF
7005 *      @pmask: the port access rights mask for the PF/VF
7006 *      @nexact: the maximum number of exact MPS filters
7007 *      @rcaps: read capabilities
7008 *      @wxcaps: write/execute capabilities
7009 *
7010 *      Configures resource limits and capabilities for a physical or virtual
7011 *      function.
7012 */
7013int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf,
7014                unsigned int vf, unsigned int txq, unsigned int txq_eth_ctrl,
7015                unsigned int rxqi, unsigned int rxq, unsigned int tc,
7016                unsigned int vi, unsigned int cmask, unsigned int pmask,
7017                unsigned int nexact, unsigned int rcaps, unsigned int wxcaps)
7018{
7019        struct fw_pfvf_cmd c;
7020
7021        memset(&c, 0, sizeof(c));
7022        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) | FW_CMD_REQUEST_F |
7023                                  FW_CMD_WRITE_F | FW_PFVF_CMD_PFN_V(pf) |
7024                                  FW_PFVF_CMD_VFN_V(vf));
7025        c.retval_len16 = cpu_to_be32(FW_LEN16(c));
7026        c.niqflint_niq = cpu_to_be32(FW_PFVF_CMD_NIQFLINT_V(rxqi) |
7027                                     FW_PFVF_CMD_NIQ_V(rxq));
7028        c.type_to_neq = cpu_to_be32(FW_PFVF_CMD_CMASK_V(cmask) |
7029                                    FW_PFVF_CMD_PMASK_V(pmask) |
7030                                    FW_PFVF_CMD_NEQ_V(txq));
7031        c.tc_to_nexactf = cpu_to_be32(FW_PFVF_CMD_TC_V(tc) |
7032                                      FW_PFVF_CMD_NVI_V(vi) |
7033                                      FW_PFVF_CMD_NEXACTF_V(nexact));
7034        c.r_caps_to_nethctrl = cpu_to_be32(FW_PFVF_CMD_R_CAPS_V(rcaps) |
7035                                        FW_PFVF_CMD_WX_CAPS_V(wxcaps) |
7036                                        FW_PFVF_CMD_NETHCTRL_V(txq_eth_ctrl));
7037        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7038}
7039
7040/**
7041 *      t4_alloc_vi - allocate a virtual interface
7042 *      @adap: the adapter
7043 *      @mbox: mailbox to use for the FW command
7044 *      @port: physical port associated with the VI
7045 *      @pf: the PF owning the VI
7046 *      @vf: the VF owning the VI
7047 *      @nmac: number of MAC addresses needed (1 to 5)
7048 *      @mac: the MAC addresses of the VI
7049 *      @rss_size: size of RSS table slice associated with this VI
7050 *
7051 *      Allocates a virtual interface for the given physical port.  If @mac is
7052 *      not %NULL it contains the MAC addresses of the VI as assigned by FW.
7053 *      @mac should be large enough to hold @nmac Ethernet addresses, they are
7054 *      stored consecutively so the space needed is @nmac * 6 bytes.
7055 *      Returns a negative error number or the non-negative VI id.
7056 */
7057int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port,
7058                unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac,
7059                unsigned int *rss_size)
7060{
7061        int ret;
7062        struct fw_vi_cmd c;
7063
7064        memset(&c, 0, sizeof(c));
7065        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) | FW_CMD_REQUEST_F |
7066                                  FW_CMD_WRITE_F | FW_CMD_EXEC_F |
7067                                  FW_VI_CMD_PFN_V(pf) | FW_VI_CMD_VFN_V(vf));
7068        c.alloc_to_len16 = cpu_to_be32(FW_VI_CMD_ALLOC_F | FW_LEN16(c));
7069        c.portid_pkd = FW_VI_CMD_PORTID_V(port);
7070        c.nmac = nmac - 1;
7071
7072        ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
7073        if (ret)
7074                return ret;
7075
7076        if (mac) {
7077                memcpy(mac, c.mac, sizeof(c.mac));
7078                switch (nmac) {
7079                case 5:
7080                        memcpy(mac + 24, c.nmac3, sizeof(c.nmac3));
7081                case 4:
7082                        memcpy(mac + 18, c.nmac2, sizeof(c.nmac2));
7083                case 3:
7084                        memcpy(mac + 12, c.nmac1, sizeof(c.nmac1));
7085                case 2:
7086                        memcpy(mac + 6,  c.nmac0, sizeof(c.nmac0));
7087                }
7088        }
7089        if (rss_size)
7090                *rss_size = FW_VI_CMD_RSSSIZE_G(be16_to_cpu(c.rsssize_pkd));
7091        return FW_VI_CMD_VIID_G(be16_to_cpu(c.type_viid));
7092}
7093
7094/**
7095 *      t4_free_vi - free a virtual interface
7096 *      @adap: the adapter
7097 *      @mbox: mailbox to use for the FW command
7098 *      @pf: the PF owning the VI
7099 *      @vf: the VF owning the VI
7100 *      @viid: virtual interface identifiler
7101 *
7102 *      Free a previously allocated virtual interface.
7103 */
7104int t4_free_vi(struct adapter *adap, unsigned int mbox, unsigned int pf,
7105               unsigned int vf, unsigned int viid)
7106{
7107        struct fw_vi_cmd c;
7108
7109        memset(&c, 0, sizeof(c));
7110        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) |
7111                                  FW_CMD_REQUEST_F |
7112                                  FW_CMD_EXEC_F |
7113                                  FW_VI_CMD_PFN_V(pf) |
7114                                  FW_VI_CMD_VFN_V(vf));
7115        c.alloc_to_len16 = cpu_to_be32(FW_VI_CMD_FREE_F | FW_LEN16(c));
7116        c.type_viid = cpu_to_be16(FW_VI_CMD_VIID_V(viid));
7117
7118        return t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
7119}
7120
7121/**
7122 *      t4_set_rxmode - set Rx properties of a virtual interface
7123 *      @adap: the adapter
7124 *      @mbox: mailbox to use for the FW command
7125 *      @viid: the VI id
7126 *      @mtu: the new MTU or -1
7127 *      @promisc: 1 to enable promiscuous mode, 0 to disable it, -1 no change
7128 *      @all_multi: 1 to enable all-multi mode, 0 to disable it, -1 no change
7129 *      @bcast: 1 to enable broadcast Rx, 0 to disable it, -1 no change
7130 *      @vlanex: 1 to enable HW VLAN extraction, 0 to disable it, -1 no change
7131 *      @sleep_ok: if true we may sleep while awaiting command completion
7132 *
7133 *      Sets Rx properties of a virtual interface.
7134 */
7135int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
7136                  int mtu, int promisc, int all_multi, int bcast, int vlanex,
7137                  bool sleep_ok)
7138{
7139        struct fw_vi_rxmode_cmd c;
7140
7141        /* convert to FW values */
7142        if (mtu < 0)
7143                mtu = FW_RXMODE_MTU_NO_CHG;
7144        if (promisc < 0)
7145                promisc = FW_VI_RXMODE_CMD_PROMISCEN_M;
7146        if (all_multi < 0)
7147                all_multi = FW_VI_RXMODE_CMD_ALLMULTIEN_M;
7148        if (bcast < 0)
7149                bcast = FW_VI_RXMODE_CMD_BROADCASTEN_M;
7150        if (vlanex < 0)
7151                vlanex = FW_VI_RXMODE_CMD_VLANEXEN_M;
7152
7153        memset(&c, 0, sizeof(c));
7154        c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_RXMODE_CMD) |
7155                                   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7156                                   FW_VI_RXMODE_CMD_VIID_V(viid));
7157        c.retval_len16 = cpu_to_be32(FW_LEN16(c));
7158        c.mtu_to_vlanexen =
7159                cpu_to_be32(FW_VI_RXMODE_CMD_MTU_V(mtu) |
7160                            FW_VI_RXMODE_CMD_PROMISCEN_V(promisc) |
7161                            FW_VI_RXMODE_CMD_ALLMULTIEN_V(all_multi) |
7162                            FW_VI_RXMODE_CMD_BROADCASTEN_V(bcast) |
7163                            FW_VI_RXMODE_CMD_VLANEXEN_V(vlanex));
7164        return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok);
7165}
7166
7167/**
7168 *      t4_alloc_mac_filt - allocates exact-match filters for MAC addresses
7169 *      @adap: the adapter
7170 *      @mbox: mailbox to use for the FW command
7171 *      @viid: the VI id
7172 *      @free: if true any existing filters for this VI id are first removed
7173 *      @naddr: the number of MAC addresses to allocate filters for (up to 7)
7174 *      @addr: the MAC address(es)
7175 *      @idx: where to store the index of each allocated filter
7176 *      @hash: pointer to hash address filter bitmap
7177 *      @sleep_ok: call is allowed to sleep
7178 *
7179 *      Allocates an exact-match filter for each of the supplied addresses and
7180 *      sets it to the corresponding address.  If @idx is not %NULL it should
7181 *      have at least @naddr entries, each of which will be set to the index of
7182 *      the filter allocated for the corresponding MAC address.  If a filter
7183 *      could not be allocated for an address its index is set to 0xffff.
7184 *      If @hash is not %NULL addresses that fail to allocate an exact filter
7185 *      are hashed and update the hash filter bitmap pointed at by @hash.
7186 *
7187 *      Returns a negative error number or the number of filters allocated.
7188 */
7189int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox,
7190                      unsigned int viid, bool free, unsigned int naddr,
7191                      const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok)
7192{
7193        int offset, ret = 0;
7194        struct fw_vi_mac_cmd c;
7195        unsigned int nfilters = 0;
7196        unsigned int max_naddr = adap->params.arch.mps_tcam_size;
7197        unsigned int rem = naddr;
7198
7199        if (naddr > max_naddr)
7200                return -EINVAL;
7201
7202        for (offset = 0; offset < naddr ; /**/) {
7203                unsigned int fw_naddr = (rem < ARRAY_SIZE(c.u.exact) ?
7204                                         rem : ARRAY_SIZE(c.u.exact));
7205                size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd,
7206                                                     u.exact[fw_naddr]), 16);
7207                struct fw_vi_mac_exact *p;
7208                int i;
7209
7210                memset(&c, 0, sizeof(c));
7211                c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7212                                           FW_CMD_REQUEST_F |
7213                                           FW_CMD_WRITE_F |
7214                                           FW_CMD_EXEC_V(free) |
7215                                           FW_VI_MAC_CMD_VIID_V(viid));
7216                c.freemacs_to_len16 =
7217                        cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(free) |
7218                                    FW_CMD_LEN16_V(len16));
7219
7220                for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) {
7221                        p->valid_to_idx =
7222                                cpu_to_be16(FW_VI_MAC_CMD_VALID_F |
7223                                            FW_VI_MAC_CMD_IDX_V(
7224                                                    FW_VI_MAC_ADD_MAC));
7225                        memcpy(p->macaddr, addr[offset + i],
7226                               sizeof(p->macaddr));
7227                }
7228
7229                /* It's okay if we run out of space in our MAC address arena.
7230                 * Some of the addresses we submit may get stored so we need
7231                 * to run through the reply to see what the results were ...
7232                 */
7233                ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok);
7234                if (ret && ret != -FW_ENOMEM)
7235                        break;
7236
7237                for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) {
7238                        u16 index = FW_VI_MAC_CMD_IDX_G(
7239                                        be16_to_cpu(p->valid_to_idx));
7240
7241                        if (idx)
7242                                idx[offset + i] = (index >= max_naddr ?
7243                                                   0xffff : index);
7244                        if (index < max_naddr)
7245                                nfilters++;
7246                        else if (hash)
7247                                *hash |= (1ULL <<
7248                                          hash_mac_addr(addr[offset + i]));
7249                }
7250
7251                free = false;
7252                offset += fw_naddr;
7253                rem -= fw_naddr;
7254        }
7255
7256        if (ret == 0 || ret == -FW_ENOMEM)
7257                ret = nfilters;
7258        return ret;
7259}
7260
7261/**
7262 *      t4_free_mac_filt - frees exact-match filters of given MAC addresses
7263 *      @adap: the adapter
7264 *      @mbox: mailbox to use for the FW command
7265 *      @viid: the VI id
7266 *      @naddr: the number of MAC addresses to allocate filters for (up to 7)
7267 *      @addr: the MAC address(es)
7268 *      @sleep_ok: call is allowed to sleep
7269 *
7270 *      Frees the exact-match filter for each of the supplied addresses
7271 *
7272 *      Returns a negative error number or the number of filters freed.
7273 */
7274int t4_free_mac_filt(struct adapter *adap, unsigned int mbox,
7275                     unsigned int viid, unsigned int naddr,
7276                     const u8 **addr, bool sleep_ok)
7277{
7278        int offset, ret = 0;
7279        struct fw_vi_mac_cmd c;
7280        unsigned int nfilters = 0;
7281        unsigned int max_naddr = is_t4(adap->params.chip) ?
7282                                       NUM_MPS_CLS_SRAM_L_INSTANCES :
7283                                       NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
7284        unsigned int rem = naddr;
7285
7286        if (naddr > max_naddr)
7287                return -EINVAL;
7288
7289        for (offset = 0; offset < (int)naddr ; /**/) {
7290                unsigned int fw_naddr = (rem < ARRAY_SIZE(c.u.exact)
7291                                         ? rem
7292                                         : ARRAY_SIZE(c.u.exact));
7293                size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd,
7294                                                     u.exact[fw_naddr]), 16);
7295                struct fw_vi_mac_exact *p;
7296                int i;
7297
7298                memset(&c, 0, sizeof(c));
7299                c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7300                                     FW_CMD_REQUEST_F |
7301                                     FW_CMD_WRITE_F |
7302                                     FW_CMD_EXEC_V(0) |
7303                                     FW_VI_MAC_CMD_VIID_V(viid));
7304                c.freemacs_to_len16 =
7305                                cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) |
7306                                            FW_CMD_LEN16_V(len16));
7307
7308                for (i = 0, p = c.u.exact; i < (int)fw_naddr; i++, p++) {
7309                        p->valid_to_idx = cpu_to_be16(
7310                                FW_VI_MAC_CMD_VALID_F |
7311                                FW_VI_MAC_CMD_IDX_V(FW_VI_MAC_MAC_BASED_FREE));
7312                        memcpy(p->macaddr, addr[offset+i], sizeof(p->macaddr));
7313                }
7314
7315                ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok);
7316                if (ret)
7317                        break;
7318
7319                for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) {
7320                        u16 index = FW_VI_MAC_CMD_IDX_G(
7321                                                be16_to_cpu(p->valid_to_idx));
7322
7323                        if (index < max_naddr)
7324                                nfilters++;
7325                }
7326
7327                offset += fw_naddr;
7328                rem -= fw_naddr;
7329        }
7330
7331        if (ret == 0)
7332                ret = nfilters;
7333        return ret;
7334}
7335
7336/**
7337 *      t4_change_mac - modifies the exact-match filter for a MAC address
7338 *      @adap: the adapter
7339 *      @mbox: mailbox to use for the FW command
7340 *      @viid: the VI id
7341 *      @idx: index of existing filter for old value of MAC address, or -1
7342 *      @addr: the new MAC address value
7343 *      @persist: whether a new MAC allocation should be persistent
7344 *      @add_smt: if true also add the address to the HW SMT
7345 *
7346 *      Modifies an exact-match filter and sets it to the new MAC address.
7347 *      Note that in general it is not possible to modify the value of a given
7348 *      filter so the generic way to modify an address filter is to free the one
7349 *      being used by the old address value and allocate a new filter for the
7350 *      new address value.  @idx can be -1 if the address is a new addition.
7351 *
7352 *      Returns a negative error number or the index of the filter with the new
7353 *      MAC value.
7354 */
7355int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid,
7356                  int idx, const u8 *addr, bool persist, bool add_smt)
7357{
7358        int ret, mode;
7359        struct fw_vi_mac_cmd c;
7360        struct fw_vi_mac_exact *p = c.u.exact;
7361        unsigned int max_mac_addr = adap->params.arch.mps_tcam_size;
7362
7363        if (idx < 0)                             /* new allocation */
7364                idx = persist ? FW_VI_MAC_ADD_PERSIST_MAC : FW_VI_MAC_ADD_MAC;
7365        mode = add_smt ? FW_VI_MAC_SMT_AND_MPSTCAM : FW_VI_MAC_MPS_TCAM_ENTRY;
7366
7367        memset(&c, 0, sizeof(c));
7368        c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7369                                   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7370                                   FW_VI_MAC_CMD_VIID_V(viid));
7371        c.freemacs_to_len16 = cpu_to_be32(FW_CMD_LEN16_V(1));
7372        p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F |
7373                                      FW_VI_MAC_CMD_SMAC_RESULT_V(mode) |
7374                                      FW_VI_MAC_CMD_IDX_V(idx));
7375        memcpy(p->macaddr, addr, sizeof(p->macaddr));
7376
7377        ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
7378        if (ret == 0) {
7379                ret = FW_VI_MAC_CMD_IDX_G(be16_to_cpu(p->valid_to_idx));
7380                if (ret >= max_mac_addr)
7381                        ret = -ENOMEM;
7382        }
7383        return ret;
7384}
7385
7386/**
7387 *      t4_set_addr_hash - program the MAC inexact-match hash filter
7388 *      @adap: the adapter
7389 *      @mbox: mailbox to use for the FW command
7390 *      @viid: the VI id
7391 *      @ucast: whether the hash filter should also match unicast addresses
7392 *      @vec: the value to be written to the hash filter
7393 *      @sleep_ok: call is allowed to sleep
7394 *
7395 *      Sets the 64-bit inexact-match hash filter for a virtual interface.
7396 */
7397int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid,
7398                     bool ucast, u64 vec, bool sleep_ok)
7399{
7400        struct fw_vi_mac_cmd c;
7401
7402        memset(&c, 0, sizeof(c));
7403        c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7404                                   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7405                                   FW_VI_ENABLE_CMD_VIID_V(viid));
7406        c.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_HASHVECEN_F |
7407                                          FW_VI_MAC_CMD_HASHUNIEN_V(ucast) |
7408                                          FW_CMD_LEN16_V(1));
7409        c.u.hash.hashvec = cpu_to_be64(vec);
7410        return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok);
7411}
7412
7413/**
7414 *      t4_enable_vi_params - enable/disable a virtual interface
7415 *      @adap: the adapter
7416 *      @mbox: mailbox to use for the FW command
7417 *      @viid: the VI id
7418 *      @rx_en: 1=enable Rx, 0=disable Rx
7419 *      @tx_en: 1=enable Tx, 0=disable Tx
7420 *      @dcb_en: 1=enable delivery of Data Center Bridging messages.
7421 *
7422 *      Enables/disables a virtual interface.  Note that setting DCB Enable
7423 *      only makes sense when enabling a Virtual Interface ...
7424 */
7425int t4_enable_vi_params(struct adapter *adap, unsigned int mbox,
7426                        unsigned int viid, bool rx_en, bool tx_en, bool dcb_en)
7427{
7428        struct fw_vi_enable_cmd c;
7429
7430        memset(&c, 0, sizeof(c));
7431        c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_ENABLE_CMD) |
7432                                   FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
7433                                   FW_VI_ENABLE_CMD_VIID_V(viid));
7434        c.ien_to_len16 = cpu_to_be32(FW_VI_ENABLE_CMD_IEN_V(rx_en) |
7435                                     FW_VI_ENABLE_CMD_EEN_V(tx_en) |
7436                                     FW_VI_ENABLE_CMD_DCB_INFO_V(dcb_en) |
7437                                     FW_LEN16(c));
7438        return t4_wr_mbox_ns(adap, mbox, &c, sizeof(c), NULL);
7439}
7440
7441/**
7442 *      t4_enable_vi - enable/disable a virtual interface
7443 *      @adap: the adapter
7444 *      @mbox: mailbox to use for the FW command
7445 *      @viid: the VI id
7446 *      @rx_en: 1=enable Rx, 0=disable Rx
7447 *      @tx_en: 1=enable Tx, 0=disable Tx
7448 *
7449 *      Enables/disables a virtual interface.
7450 */
7451int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid,
7452                 bool rx_en, bool tx_en)
7453{
7454        return t4_enable_vi_params(adap, mbox, viid, rx_en, tx_en, 0);
7455}
7456
7457/**
7458 *      t4_identify_port - identify a VI's port by blinking its LED
7459 *      @adap: the adapter
7460 *      @mbox: mailbox to use for the FW command
7461 *      @viid: the VI id
7462 *      @nblinks: how many times to blink LED at 2.5 Hz
7463 *
7464 *      Identifies a VI's port by blinking its LED.
7465 */
7466int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid,
7467                     unsigned int nblinks)
7468{
7469        struct fw_vi_enable_cmd c;
7470
7471        memset(&c, 0, sizeof(c));
7472        c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_ENABLE_CMD) |
7473                                   FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
7474                                   FW_VI_ENABLE_CMD_VIID_V(viid));
7475        c.ien_to_len16 = cpu_to_be32(FW_VI_ENABLE_CMD_LED_F | FW_LEN16(c));
7476        c.blinkdur = cpu_to_be16(nblinks);
7477        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7478}
7479
7480/**
7481 *      t4_iq_stop - stop an ingress queue and its FLs
7482 *      @adap: the adapter
7483 *      @mbox: mailbox to use for the FW command
7484 *      @pf: the PF owning the queues
7485 *      @vf: the VF owning the queues
7486 *      @iqtype: the ingress queue type (FW_IQ_TYPE_FL_INT_CAP, etc.)
7487 *      @iqid: ingress queue id
7488 *      @fl0id: FL0 queue id or 0xffff if no attached FL0
7489 *      @fl1id: FL1 queue id or 0xffff if no attached FL1
7490 *
7491 *      Stops an ingress queue and its associated FLs, if any.  This causes
7492 *      any current or future data/messages destined for these queues to be
7493 *      tossed.
7494 */
7495int t4_iq_stop(struct adapter *adap, unsigned int mbox, unsigned int pf,
7496               unsigned int vf, unsigned int iqtype, unsigned int iqid,
7497               unsigned int fl0id, unsigned int fl1id)
7498{
7499        struct fw_iq_cmd c;
7500
7501        memset(&c, 0, sizeof(c));
7502        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F |
7503                                  FW_CMD_EXEC_F | FW_IQ_CMD_PFN_V(pf) |
7504                                  FW_IQ_CMD_VFN_V(vf));
7505        c.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_IQSTOP_F | FW_LEN16(c));
7506        c.type_to_iqandstindex = cpu_to_be32(FW_IQ_CMD_TYPE_V(iqtype));
7507        c.iqid = cpu_to_be16(iqid);
7508        c.fl0id = cpu_to_be16(fl0id);
7509        c.fl1id = cpu_to_be16(fl1id);
7510        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7511}
7512
7513/**
7514 *      t4_iq_free - free an ingress queue and its FLs
7515 *      @adap: the adapter
7516 *      @mbox: mailbox to use for the FW command
7517 *      @pf: the PF owning the queues
7518 *      @vf: the VF owning the queues
7519 *      @iqtype: the ingress queue type
7520 *      @iqid: ingress queue id
7521 *      @fl0id: FL0 queue id or 0xffff if no attached FL0
7522 *      @fl1id: FL1 queue id or 0xffff if no attached FL1
7523 *
7524 *      Frees an ingress queue and its associated FLs, if any.
7525 */
7526int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
7527               unsigned int vf, unsigned int iqtype, unsigned int iqid,
7528               unsigned int fl0id, unsigned int fl1id)
7529{
7530        struct fw_iq_cmd c;
7531
7532        memset(&c, 0, sizeof(c));
7533        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F |
7534                                  FW_CMD_EXEC_F | FW_IQ_CMD_PFN_V(pf) |
7535                                  FW_IQ_CMD_VFN_V(vf));
7536        c.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_FREE_F | FW_LEN16(c));
7537        c.type_to_iqandstindex = cpu_to_be32(FW_IQ_CMD_TYPE_V(iqtype));
7538        c.iqid = cpu_to_be16(iqid);
7539        c.fl0id = cpu_to_be16(fl0id);
7540        c.fl1id = cpu_to_be16(fl1id);
7541        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7542}
7543
7544/**
7545 *      t4_eth_eq_free - free an Ethernet egress queue
7546 *      @adap: the adapter
7547 *      @mbox: mailbox to use for the FW command
7548 *      @pf: the PF owning the queue
7549 *      @vf: the VF owning the queue
7550 *      @eqid: egress queue id
7551 *
7552 *      Frees an Ethernet egress queue.
7553 */
7554int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
7555                   unsigned int vf, unsigned int eqid)
7556{
7557        struct fw_eq_eth_cmd c;
7558
7559        memset(&c, 0, sizeof(c));
7560        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_ETH_CMD) |
7561                                  FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
7562                                  FW_EQ_ETH_CMD_PFN_V(pf) |
7563                                  FW_EQ_ETH_CMD_VFN_V(vf));
7564        c.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_FREE_F | FW_LEN16(c));
7565        c.eqid_pkd = cpu_to_be32(FW_EQ_ETH_CMD_EQID_V(eqid));
7566        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7567}
7568
7569/**
7570 *      t4_ctrl_eq_free - free a control egress queue
7571 *      @adap: the adapter
7572 *      @mbox: mailbox to use for the FW command
7573 *      @pf: the PF owning the queue
7574 *      @vf: the VF owning the queue
7575 *      @eqid: egress queue id
7576 *
7577 *      Frees a control egress queue.
7578 */
7579int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
7580                    unsigned int vf, unsigned int eqid)
7581{
7582        struct fw_eq_ctrl_cmd c;
7583
7584        memset(&c, 0, sizeof(c));
7585        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_CTRL_CMD) |
7586                                  FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
7587                                  FW_EQ_CTRL_CMD_PFN_V(pf) |
7588                                  FW_EQ_CTRL_CMD_VFN_V(vf));
7589        c.alloc_to_len16 = cpu_to_be32(FW_EQ_CTRL_CMD_FREE_F | FW_LEN16(c));
7590        c.cmpliqid_eqid = cpu_to_be32(FW_EQ_CTRL_CMD_EQID_V(eqid));
7591        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7592}
7593
7594/**
7595 *      t4_ofld_eq_free - free an offload egress queue
7596 *      @adap: the adapter
7597 *      @mbox: mailbox to use for the FW command
7598 *      @pf: the PF owning the queue
7599 *      @vf: the VF owning the queue
7600 *      @eqid: egress queue id
7601 *
7602 *      Frees a control egress queue.
7603 */
7604int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
7605                    unsigned int vf, unsigned int eqid)
7606{
7607        struct fw_eq_ofld_cmd c;
7608
7609        memset(&c, 0, sizeof(c));
7610        c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_OFLD_CMD) |
7611                                  FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
7612                                  FW_EQ_OFLD_CMD_PFN_V(pf) |
7613                                  FW_EQ_OFLD_CMD_VFN_V(vf));
7614        c.alloc_to_len16 = cpu_to_be32(FW_EQ_OFLD_CMD_FREE_F | FW_LEN16(c));
7615        c.eqid_pkd = cpu_to_be32(FW_EQ_OFLD_CMD_EQID_V(eqid));
7616        return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7617}
7618
7619/**
7620 *      t4_link_down_rc_str - return a string for a Link Down Reason Code
7621 *      @adap: the adapter
7622 *      @link_down_rc: Link Down Reason Code
7623 *
7624 *      Returns a string representation of the Link Down Reason Code.
7625 */
7626static const char *t4_link_down_rc_str(unsigned char link_down_rc)
7627{
7628        static const char * const reason[] = {
7629                "Link Down",
7630                "Remote Fault",
7631                "Auto-negotiation Failure",
7632                "Reserved",
7633                "Insufficient Airflow",
7634                "Unable To Determine Reason",
7635                "No RX Signal Detected",
7636                "Reserved",
7637        };
7638
7639        if (link_down_rc >= ARRAY_SIZE(reason))
7640                return "Bad Reason Code";
7641
7642        return reason[link_down_rc];
7643}
7644
7645/**
7646 *      t4_handle_get_port_info - process a FW reply message
7647 *      @pi: the port info
7648 *      @rpl: start of the FW message
7649 *
7650 *      Processes a GET_PORT_INFO FW reply message.
7651 */
7652void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
7653{
7654        const struct fw_port_cmd *p = (const void *)rpl;
7655        struct adapter *adap = pi->adapter;
7656
7657        /* link/module state change message */
7658        int speed = 0, fc = 0;
7659        struct link_config *lc;
7660        u32 stat = be32_to_cpu(p->u.info.lstatus_to_modtype);
7661        int link_ok = (stat & FW_PORT_CMD_LSTATUS_F) != 0;
7662        u32 mod = FW_PORT_CMD_MODTYPE_G(stat);
7663
7664        if (stat & FW_PORT_CMD_RXPAUSE_F)
7665                fc |= PAUSE_RX;
7666        if (stat & FW_PORT_CMD_TXPAUSE_F)
7667                fc |= PAUSE_TX;
7668        if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M))
7669                speed = 100;
7670        else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G))
7671                speed = 1000;
7672        else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
7673                speed = 10000;
7674        else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
7675                speed = 25000;
7676        else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
7677                speed = 40000;
7678        else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
7679                speed = 100000;
7680
7681        lc = &pi->link_cfg;
7682
7683        if (mod != pi->mod_type) {
7684                pi->mod_type = mod;
7685                t4_os_portmod_changed(adap, pi->port_id);
7686        }
7687        if (link_ok != lc->link_ok || speed != lc->speed ||
7688            fc != lc->fc) {     /* something changed */
7689                if (!link_ok && lc->link_ok) {
7690                        unsigned char rc = FW_PORT_CMD_LINKDNRC_G(stat);
7691
7692                        lc->link_down_rc = rc;
7693                        dev_warn(adap->pdev_dev,
7694                                 "Port %d link down, reason: %s\n",
7695                                 pi->port_id, t4_link_down_rc_str(rc));
7696                }
7697                lc->link_ok = link_ok;
7698                lc->speed = speed;
7699                lc->fc = fc;
7700                lc->supported = be16_to_cpu(p->u.info.pcap);
7701                lc->lp_advertising = be16_to_cpu(p->u.info.lpacap);
7702
7703                t4_os_link_changed(adap, pi->port_id, link_ok);
7704        }
7705}
7706
7707/**
7708 *      t4_update_port_info - retrieve and update port information if changed
7709 *      @pi: the port_info
7710 *
7711 *      We issue a Get Port Information Command to the Firmware and, if
7712 *      successful, we check to see if anything is different from what we
7713 *      last recorded and update things accordingly.
7714 */
7715int t4_update_port_info(struct port_info *pi)
7716{
7717        struct fw_port_cmd port_cmd;
7718        int ret;
7719
7720        memset(&port_cmd, 0, sizeof(port_cmd));
7721        port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
7722                                            FW_CMD_REQUEST_F | FW_CMD_READ_F |
7723                                            FW_PORT_CMD_PORTID_V(pi->port_id));
7724        port_cmd.action_to_len16 = cpu_to_be32(
7725                FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_GET_PORT_INFO) |
7726                FW_LEN16(port_cmd));
7727        ret = t4_wr_mbox(pi->adapter, pi->adapter->mbox,
7728                         &port_cmd, sizeof(port_cmd), &port_cmd);
7729        if (ret)
7730                return ret;
7731
7732        t4_handle_get_port_info(pi, (__be64 *)&port_cmd);
7733        return 0;
7734}
7735
7736/**
7737 *      t4_handle_fw_rpl - process a FW reply message
7738 *      @adap: the adapter
7739 *      @rpl: start of the FW message
7740 *
7741 *      Processes a FW message, such as link state change messages.
7742 */
7743int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl)
7744{
7745        u8 opcode = *(const u8 *)rpl;
7746
7747        /* This might be a port command ... this simplifies the following
7748         * conditionals ...  We can get away with pre-dereferencing
7749         * action_to_len16 because it's in the first 16 bytes and all messages
7750         * will be at least that long.
7751         */
7752        const struct fw_port_cmd *p = (const void *)rpl;
7753        unsigned int action =
7754                FW_PORT_CMD_ACTION_G(be32_to_cpu(p->action_to_len16));
7755
7756        if (opcode == FW_PORT_CMD && action == FW_PORT_ACTION_GET_PORT_INFO) {
7757                int i;
7758                int chan = FW_PORT_CMD_PORTID_G(be32_to_cpu(p->op_to_portid));
7759                struct port_info *pi = NULL;
7760
7761                for_each_port(adap, i) {
7762                        pi = adap2pinfo(adap, i);
7763                        if (pi->tx_chan == chan)
7764                                break;
7765                }
7766
7767                t4_handle_get_port_info(pi, rpl);
7768        } else {
7769                dev_warn(adap->pdev_dev, "Unknown firmware reply %d\n", opcode);
7770                return -EINVAL;
7771        }
7772        return 0;
7773}
7774
7775static void get_pci_mode(struct adapter *adapter, struct pci_params *p)
7776{
7777        u16 val;
7778
7779        if (pci_is_pcie(adapter->pdev)) {
7780                pcie_capability_read_word(adapter->pdev, PCI_EXP_LNKSTA, &val);
7781                p->speed = val & PCI_EXP_LNKSTA_CLS;
7782                p->width = (val & PCI_EXP_LNKSTA_NLW) >> 4;
7783        }
7784}
7785
7786/**
7787 *      init_link_config - initialize a link's SW state
7788 *      @lc: structure holding the link state
7789 *      @caps: link capabilities
7790 *
7791 *      Initializes the SW state maintained for each link, including the link's
7792 *      capabilities and default speed/flow-control/autonegotiation settings.
7793 */
7794static void init_link_config(struct link_config *lc, unsigned int pcaps,
7795                             unsigned int acaps)
7796{
7797        lc->supported = pcaps;
7798        lc->lp_advertising = 0;
7799        lc->requested_speed = 0;
7800        lc->speed = 0;
7801        lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX;
7802        lc->auto_fec = 0;
7803
7804        /* For Forward Error Control, we default to whatever the Firmware
7805         * tells us the Link is currently advertising.
7806         */
7807        if (acaps & FW_PORT_CAP_FEC_RS)
7808                lc->auto_fec |= FEC_RS;
7809        if (acaps & FW_PORT_CAP_FEC_BASER_RS)
7810                lc->auto_fec |= FEC_BASER_RS;
7811        lc->requested_fec = FEC_AUTO;
7812        lc->fec = lc->auto_fec;
7813
7814        if (lc->supported & FW_PORT_CAP_ANEG) {
7815                lc->advertising = lc->supported & ADVERT_MASK;
7816                lc->autoneg = AUTONEG_ENABLE;
7817                lc->requested_fc |= PAUSE_AUTONEG;
7818        } else {
7819                lc->advertising = 0;
7820                lc->autoneg = AUTONEG_DISABLE;
7821        }
7822}
7823
7824#define CIM_PF_NOACCESS 0xeeeeeeee
7825
7826int t4_wait_dev_ready(void __iomem *regs)
7827{
7828        u32 whoami;
7829
7830        whoami = readl(regs + PL_WHOAMI_A);
7831        if (whoami != 0xffffffff && whoami != CIM_PF_NOACCESS)
7832                return 0;
7833
7834        msleep(500);
7835        whoami = readl(regs + PL_WHOAMI_A);
7836        return (whoami != 0xffffffff && whoami != CIM_PF_NOACCESS ? 0 : -EIO);
7837}
7838
7839struct flash_desc {
7840        u32 vendor_and_model_id;
7841        u32 size_mb;
7842};
7843
7844static int get_flash_params(struct adapter *adap)
7845{
7846        /* Table for non-Numonix supported flash parts.  Numonix parts are left
7847         * to the preexisting code.  All flash parts have 64KB sectors.
7848         */
7849        static struct flash_desc supported_flash[] = {
7850                { 0x150201, 4 << 20 },       /* Spansion 4MB S25FL032P */
7851        };
7852
7853        int ret;
7854        u32 info;
7855
7856        ret = sf1_write(adap, 1, 1, 0, SF_RD_ID);
7857        if (!ret)
7858                ret = sf1_read(adap, 3, 0, 1, &info);
7859        t4_write_reg(adap, SF_OP_A, 0);                    /* unlock SF */
7860        if (ret)
7861                return ret;
7862
7863        for (ret = 0; ret < ARRAY_SIZE(supported_flash); ++ret)
7864                if (supported_flash[ret].vendor_and_model_id == info) {
7865                        adap->params.sf_size = supported_flash[ret].size_mb;
7866                        adap->params.sf_nsec =
7867                                adap->params.sf_size / SF_SEC_SIZE;
7868                        return 0;
7869                }
7870
7871        if ((info & 0xff) != 0x20)             /* not a Numonix flash */
7872                return -EINVAL;
7873        info >>= 16;                           /* log2 of size */
7874        if (info >= 0x14 && info < 0x18)
7875                adap->params.sf_nsec = 1 << (info - 16);
7876        else if (info == 0x18)
7877                adap->params.sf_nsec = 64;
7878        else
7879                return -EINVAL;
7880        adap->params.sf_size = 1 << info;
7881        adap->params.sf_fw_start =
7882                t4_read_reg(adap, CIM_BOOT_CFG_A) & BOOTADDR_M;
7883
7884        if (adap->params.sf_size < FLASH_MIN_SIZE)
7885                dev_warn(adap->pdev_dev, "WARNING!!! FLASH size %#x < %#x!!!\n",
7886                         adap->params.sf_size, FLASH_MIN_SIZE);
7887        return 0;
7888}
7889
7890static void set_pcie_completion_timeout(struct adapter *adapter, u8 range)
7891{
7892        u16 val;
7893        u32 pcie_cap;
7894
7895        pcie_cap = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7896        if (pcie_cap) {
7897                pci_read_config_word(adapter->pdev,
7898                                     pcie_cap + PCI_EXP_DEVCTL2, &val);
7899                val &= ~PCI_EXP_DEVCTL2_COMP_TIMEOUT;
7900                val |= range;
7901                pci_write_config_word(adapter->pdev,
7902                                      pcie_cap + PCI_EXP_DEVCTL2, val);
7903        }
7904}
7905
7906/**
7907 *      t4_prep_adapter - prepare SW and HW for operation
7908 *      @adapter: the adapter
7909 *      @reset: if true perform a HW reset
7910 *
7911 *      Initialize adapter SW state for the various HW modules, set initial
7912 *      values for some adapter tunables, take PHYs out of reset, and
7913 *      initialize the MDIO interface.
7914 */
7915int t4_prep_adapter(struct adapter *adapter)
7916{
7917        int ret, ver;
7918        uint16_t device_id;
7919        u32 pl_rev;
7920
7921        get_pci_mode(adapter, &adapter->params.pci);
7922        pl_rev = REV_G(t4_read_reg(adapter, PL_REV_A));
7923
7924        ret = get_flash_params(adapter);
7925        if (ret < 0) {
7926                dev_err(adapter->pdev_dev, "error %d identifying flash\n", ret);
7927                return ret;
7928        }
7929
7930        /* Retrieve adapter's device ID
7931         */
7932        pci_read_config_word(adapter->pdev, PCI_DEVICE_ID, &device_id);
7933        ver = device_id >> 12;
7934        adapter->params.chip = 0;
7935        switch (ver) {
7936        case CHELSIO_T4:
7937                adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev);
7938                adapter->params.arch.sge_fl_db = DBPRIO_F;
7939                adapter->params.arch.mps_tcam_size =
7940                                 NUM_MPS_CLS_SRAM_L_INSTANCES;
7941                adapter->params.arch.mps_rplc_size = 128;
7942                adapter->params.arch.nchan = NCHAN;
7943                adapter->params.arch.pm_stats_cnt = PM_NSTATS;
7944                adapter->params.arch.vfcount = 128;
7945                /* Congestion map is for 4 channels so that
7946                 * MPS can have 4 priority per port.
7947                 */
7948                adapter->params.arch.cng_ch_bits_log = 2;
7949                break;
7950        case CHELSIO_T5:
7951                adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev);
7952                adapter->params.arch.sge_fl_db = DBPRIO_F | DBTYPE_F;
7953                adapter->params.arch.mps_tcam_size =
7954                                 NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
7955                adapter->params.arch.mps_rplc_size = 128;
7956                adapter->params.arch.nchan = NCHAN;
7957                adapter->params.arch.pm_stats_cnt = PM_NSTATS;
7958                adapter->params.arch.vfcount = 128;
7959                adapter->params.arch.cng_ch_bits_log = 2;
7960                break;
7961        case CHELSIO_T6:
7962                adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev);
7963                adapter->params.arch.sge_fl_db = 0;
7964                adapter->params.arch.mps_tcam_size =
7965                                 NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
7966                adapter->params.arch.mps_rplc_size = 256;
7967                adapter->params.arch.nchan = 2;
7968                adapter->params.arch.pm_stats_cnt = T6_PM_NSTATS;
7969                adapter->params.arch.vfcount = 256;
7970                /* Congestion map will be for 2 channels so that
7971                 * MPS can have 8 priority per port.
7972                 */
7973                adapter->params.arch.cng_ch_bits_log = 3;
7974                break;
7975        default:
7976                dev_err(adapter->pdev_dev, "Device %d is not supported\n",
7977                        device_id);
7978                return -EINVAL;
7979        }
7980
7981        adapter->params.cim_la_size = CIMLA_SIZE;
7982        init_cong_ctrl(adapter->params.a_wnd, adapter->params.b_wnd);
7983
7984        /*
7985         * Default port for debugging in case we can't reach FW.
7986         */
7987        adapter->params.nports = 1;
7988        adapter->params.portvec = 1;
7989        adapter->params.vpd.cclk = 50000;
7990
7991        /* Set pci completion timeout value to 4 seconds. */
7992        set_pcie_completion_timeout(adapter, 0xd);
7993        return 0;
7994}
7995
7996/**
7997 *      t4_shutdown_adapter - shut down adapter, host & wire
7998 *      @adapter: the adapter
7999 *
8000 *      Perform an emergency shutdown of the adapter and stop it from

8001 *      continuing any further communication on the ports or DMA to the
8002 *      host.  This is typically used when the adapter and/or firmware
8003 *      have crashed and we want to prevent any further accidental
8004 *      communication with the rest of the world.  This will also force
8005 *      the port Link Status to go down -- if register writes work --
8006 *      which should help our peers figure out that we're down.
8007 */
8008int t4_shutdown_adapter(struct adapter *adapter)
8009{
8010        int port;
8011
8012        t4_intr_disable(adapter);
8013        t4_write_reg(adapter, DBG_GPIO_EN_A, 0);
8014        for_each_port(adapter, port) {
8015                u32 a_port_cfg = is_t4(adapter->params.chip) ?
8016                                       PORT_REG(port, XGMAC_PORT_CFG_A) :
8017                                       T5_PORT_REG(port, MAC_PORT_CFG_A);
8018
8019                t4_write_reg(adapter, a_port_cfg,
8020                             t4_read_reg(adapter, a_port_cfg)
8021                             & ~SIGNAL_DET_V(1));
8022        }
8023        t4_set_reg_field(adapter, SGE_CONTROL_A, GLOBALENABLE_F, 0);
8024
8025        return 0;
8026}
8027
8028/**
8029 *      t4_bar2_sge_qregs - return BAR2 SGE Queue register information
8030 *      @adapter: the adapter
8031 *      @qid: the Queue ID
8032 *      @qtype: the Ingress or Egress type for @qid
8033 *      @user: true if this request is for a user mode queue
8034 *      @pbar2_qoffset: BAR2 Queue Offset
8035 *      @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
8036 *
8037 *      Returns the BAR2 SGE Queue Registers information associated with the
8038 *      indicated Absolute Queue ID.  These are passed back in return value
8039 *      pointers.  @qtype should be T4_BAR2_QTYPE_EGRESS for Egress Queue
8040 *      and T4_BAR2_QTYPE_INGRESS for Ingress Queues.
8041 *
8042 *      This may return an error which indicates that BAR2 SGE Queue
8043 *      registers aren't available.  If an error is not returned, then the
8044 *      following values are returned:
8045 *
8046 *        *@pbar2_qoffset: the BAR2 Offset of the @qid Registers
8047 *        *@pbar2_qid: the BAR2 SGE Queue ID or 0 of @qid
8048 *
8049 *      If the returned BAR2 Queue ID is 0, then BAR2 SGE registers which
8050 *      require the "Inferred Queue ID" ability may be used.  E.g. the
8051 *      Write Combining Doorbell Buffer. If the BAR2 Queue ID is not 0,
8052 *      then these "Inferred Queue ID" register may not be used.
8053 */
8054int t4_bar2_sge_qregs(struct adapter *adapter,
8055                      unsigned int qid,
8056                      enum t4_bar2_qtype qtype,
8057                      int user,
8058                      u64 *pbar2_qoffset,
8059                      unsigned int *pbar2_qid)
8060{
8061        unsigned int page_shift, page_size, qpp_shift, qpp_mask;
8062        u64 bar2_page_offset, bar2_qoffset;
8063        unsigned int bar2_qid, bar2_qid_offset, bar2_qinferred;
8064
8065        /* T4 doesn't support BAR2 SGE Queue registers for kernel mode queues */
8066        if (!user && is_t4(adapter->params.chip))
8067                return -EINVAL;
8068
8069        /* Get our SGE Page Size parameters.
8070         */
8071        page_shift = adapter->params.sge.hps + 10;
8072        page_size = 1 << page_shift;
8073
8074        /* Get the right Queues per Page parameters for our Queue.
8075         */
8076        qpp_shift = (qtype == T4_BAR2_QTYPE_EGRESS
8077                     ? adapter->params.sge.eq_qpp
8078                     : adapter->params.sge.iq_qpp);
8079        qpp_mask = (1 << qpp_shift) - 1;
8080
8081        /*  Calculate the basics of the BAR2 SGE Queue register area:
8082         *  o The BAR2 page the Queue registers will be in.
8083         *  o The BAR2 Queue ID.
8084         *  o The BAR2 Queue ID Offset into the BAR2 page.
8085         */
8086        bar2_page_offset = ((u64)(qid >> qpp_shift) << page_shift);
8087        bar2_qid = qid & qpp_mask;
8088        bar2_qid_offset = bar2_qid * SGE_UDB_SIZE;
8089
8090        /* If the BAR2 Queue ID Offset is less than the Page Size, then the
8091         * hardware will infer the Absolute Queue ID simply from the writes to
8092         * the BAR2 Queue ID Offset within the BAR2 Page (and we need to use a
8093         * BAR2 Queue ID of 0 for those writes).  Otherwise, we'll simply
8094         * write to the first BAR2 SGE Queue Area within the BAR2 Page with
8095         * the BAR2 Queue ID and the hardware will infer the Absolute Queue ID
8096         * from the BAR2 Page and BAR2 Queue ID.
8097         *
8098         * One important censequence of this is that some BAR2 SGE registers
8099         * have a "Queue ID" field and we can write the BAR2 SGE Queue ID
8100         * there.  But other registers synthesize the SGE Queue ID purely
8101         * from the writes to the registers -- the Write Combined Doorbell
8102         * Buffer is a good example.  These BAR2 SGE Registers are only
8103         * available for those BAR2 SGE Register areas where the SGE Absolute
8104         * Queue ID can be inferred from simple writes.
8105         */
8106        bar2_qoffset = bar2_page_offset;
8107        bar2_qinferred = (bar2_qid_offset < page_size);
8108        if (bar2_qinferred) {
8109                bar2_qoffset += bar2_qid_offset;
8110                bar2_qid = 0;
8111        }
8112
8113        *pbar2_qoffset = bar2_qoffset;
8114        *pbar2_qid = bar2_qid;
8115        return 0;
8116}
8117
8118/**
8119 *      t4_init_devlog_params - initialize adapter->params.devlog
8120 *      @adap: the adapter
8121 *
8122 *      Initialize various fields of the adapter's Firmware Device Log
8123 *      Parameters structure.
8124 */
8125int t4_init_devlog_params(struct adapter *adap)
8126{
8127        struct devlog_params *dparams = &adap->params.devlog;
8128        u32 pf_dparams;
8129        unsigned int devlog_meminfo;
8130        struct fw_devlog_cmd devlog_cmd;
8131        int ret;
8132
8133        /* If we're dealing with newer firmware, the Device Log Paramerters
8134         * are stored in a designated register which allows us to access the
8135         * Device Log even if we can't talk to the firmware.
8136         */
8137        pf_dparams =
8138                t4_read_reg(adap, PCIE_FW_REG(PCIE_FW_PF_A, PCIE_FW_PF_DEVLOG));
8139        if (pf_dparams) {
8140                unsigned int nentries, nentries128;
8141
8142                dparams->memtype = PCIE_FW_PF_DEVLOG_MEMTYPE_G(pf_dparams);
8143                dparams->start = PCIE_FW_PF_DEVLOG_ADDR16_G(pf_dparams) << 4;
8144
8145                nentries128 = PCIE_FW_PF_DEVLOG_NENTRIES128_G(pf_dparams);
8146                nentries = (nentries128 + 1) * 128;
8147                dparams->size = nentries * sizeof(struct fw_devlog_e);
8148
8149                return 0;
8150        }
8151
8152        /* Otherwise, ask the firmware for it's Device Log Parameters.
8153         */
8154        memset(&devlog_cmd, 0, sizeof(devlog_cmd));
8155        devlog_cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_DEVLOG_CMD) |
8156                                             FW_CMD_REQUEST_F | FW_CMD_READ_F);
8157        devlog_cmd.retval_len16 = cpu_to_be32(FW_LEN16(devlog_cmd));
8158        ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd),
8159                         &devlog_cmd);
8160        if (ret)
8161                return ret;
8162
8163        devlog_meminfo =
8164                be32_to_cpu(devlog_cmd.memtype_devlog_memaddr16_devlog);
8165        dparams->memtype = FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo);
8166        dparams->start = FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4;
8167        dparams->size = be32_to_cpu(devlog_cmd.memsize_devlog);
8168
8169        return 0;
8170}
8171
8172/**
8173 *      t4_init_sge_params - initialize adap->params.sge
8174 *      @adapter: the adapter
8175 *
8176 *      Initialize various fields of the adapter's SGE Parameters structure.
8177 */
8178int t4_init_sge_params(struct adapter *adapter)
8179{
8180        struct sge_params *sge_params = &adapter->params.sge;
8181        u32 hps, qpp;
8182        unsigned int s_hps, s_qpp;
8183
8184        /* Extract the SGE Page Size for our PF.
8185         */
8186        hps = t4_read_reg(adapter, SGE_HOST_PAGE_SIZE_A);
8187        s_hps = (HOSTPAGESIZEPF0_S +
8188                 (HOSTPAGESIZEPF1_S - HOSTPAGESIZEPF0_S) * adapter->pf);
8189        sge_params->hps = ((hps >> s_hps) & HOSTPAGESIZEPF0_M);
8190
8191        /* Extract the SGE Egress and Ingess Queues Per Page for our PF.
8192         */
8193        s_qpp = (QUEUESPERPAGEPF0_S +
8194                (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * adapter->pf);
8195        qpp = t4_read_reg(adapter, SGE_EGRESS_QUEUES_PER_PAGE_PF_A);
8196        sge_params->eq_qpp = ((qpp >> s_qpp) & QUEUESPERPAGEPF0_M);
8197        qpp = t4_read_reg(adapter, SGE_INGRESS_QUEUES_PER_PAGE_PF_A);
8198        sge_params->iq_qpp = ((qpp >> s_qpp) & QUEUESPERPAGEPF0_M);
8199
8200        return 0;
8201}
8202
8203/**
8204 *      t4_init_tp_params - initialize adap->params.tp
8205 *      @adap: the adapter
8206 *
8207 *      Initialize various fields of the adapter's TP Parameters structure.
8208 */
8209int t4_init_tp_params(struct adapter *adap)
8210{
8211        int chan;
8212        u32 v;
8213
8214        v = t4_read_reg(adap, TP_TIMER_RESOLUTION_A);
8215        adap->params.tp.tre = TIMERRESOLUTION_G(v);
8216        adap->params.tp.dack_re = DELAYEDACKRESOLUTION_G(v);
8217
8218        /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
8219        for (chan = 0; chan < NCHAN; chan++)
8220                adap->params.tp.tx_modq[chan] = chan;
8221
8222        /* Cache the adapter's Compressed Filter Mode and global Incress
8223         * Configuration.
8224         */
8225        if (t4_use_ldst(adap)) {
8226                t4_fw_tp_pio_rw(adap, &adap->params.tp.vlan_pri_map, 1,
8227                                TP_VLAN_PRI_MAP_A, 1);
8228                t4_fw_tp_pio_rw(adap, &adap->params.tp.ingress_config, 1,
8229                                TP_INGRESS_CONFIG_A, 1);
8230        } else {
8231                t4_read_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
8232                                 &adap->params.tp.vlan_pri_map, 1,
8233                                 TP_VLAN_PRI_MAP_A);
8234                t4_read_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
8235                                 &adap->params.tp.ingress_config, 1,
8236                                 TP_INGRESS_CONFIG_A);
8237        }
8238        /* For T6, cache the adapter's compressed error vector
8239         * and passing outer header info for encapsulated packets.
8240         */
8241        if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) {
8242                v = t4_read_reg(adap, TP_OUT_CONFIG_A);
8243                adap->params.tp.rx_pkt_encap = (v & CRXPKTENC_F) ? 1 : 0;
8244        }
8245
8246        /* Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field
8247         * shift positions of several elements of the Compressed Filter Tuple
8248         * for this adapter which we need frequently ...
8249         */
8250        adap->params.tp.vlan_shift = t4_filter_field_shift(adap, VLAN_F);
8251        adap->params.tp.vnic_shift = t4_filter_field_shift(adap, VNIC_ID_F);
8252        adap->params.tp.port_shift = t4_filter_field_shift(adap, PORT_F);
8253        adap->params.tp.protocol_shift = t4_filter_field_shift(adap,
8254                                                               PROTOCOL_F);
8255
8256        /* If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID
8257         * represents the presense of an Outer VLAN instead of a VNIC ID.
8258         */
8259        if ((adap->params.tp.ingress_config & VNIC_F) == 0)
8260                adap->params.tp.vnic_shift = -1;
8261
8262        return 0;
8263}
8264
8265/**
8266 *      t4_filter_field_shift - calculate filter field shift
8267 *      @adap: the adapter
8268 *      @filter_sel: the desired field (from TP_VLAN_PRI_MAP bits)
8269 *
8270 *      Return the shift position of a filter field within the Compressed
8271 *      Filter Tuple.  The filter field is specified via its selection bit
8272 *      within TP_VLAN_PRI_MAL (filter mode).  E.g. F_VLAN.
8273 */
8274int t4_filter_field_shift(const struct adapter *adap, int filter_sel)
8275{
8276        unsigned int filter_mode = adap->params.tp.vlan_pri_map;
8277        unsigned int sel;
8278        int field_shift;
8279
8280        if ((filter_mode & filter_sel) == 0)
8281                return -1;
8282
8283        for (sel = 1, field_shift = 0; sel < filter_sel; sel <<= 1) {
8284                switch (filter_mode & sel) {
8285                case FCOE_F:
8286                        field_shift += FT_FCOE_W;
8287                        break;
8288                case PORT_F:
8289                        field_shift += FT_PORT_W;
8290                        break;
8291                case VNIC_ID_F:
8292                        field_shift += FT_VNIC_ID_W;
8293                        break;
8294                case VLAN_F:
8295                        field_shift += FT_VLAN_W;
8296                        break;
8297                case TOS_F:
8298                        field_shift += FT_TOS_W;
8299                        break;
8300                case PROTOCOL_F:
8301                        field_shift += FT_PROTOCOL_W;
8302                        break;
8303                case ETHERTYPE_F:
8304                        field_shift += FT_ETHERTYPE_W;
8305                        break;
8306                case MACMATCH_F:
8307                        field_shift += FT_MACMATCH_W;
8308                        break;
8309                case MPSHITTYPE_F:
8310                        field_shift += FT_MPSHITTYPE_W;
8311                        break;
8312                case FRAGMENTATION_F:
8313                        field_shift += FT_FRAGMENTATION_W;
8314                        break;
8315                }
8316        }
8317        return field_shift;
8318}
8319
8320int t4_init_rss_mode(struct adapter *adap, int mbox)
8321{
8322        int i, ret;
8323        struct fw_rss_vi_config_cmd rvc;
8324
8325        memset(&rvc, 0, sizeof(rvc));
8326
8327        for_each_port(adap, i) {
8328                struct port_info *p = adap2pinfo(adap, i);
8329
8330                rvc.op_to_viid =
8331                        cpu_to_be32(FW_CMD_OP_V(FW_RSS_VI_CONFIG_CMD) |
8332                                    FW_CMD_REQUEST_F | FW_CMD_READ_F |
8333                                    FW_RSS_VI_CONFIG_CMD_VIID_V(p->viid));
8334                rvc.retval_len16 = cpu_to_be32(FW_LEN16(rvc));
8335                ret = t4_wr_mbox(adap, mbox, &rvc, sizeof(rvc), &rvc);
8336                if (ret)
8337                        return ret;
8338                p->rss_mode = be32_to_cpu(rvc.u.basicvirtual.defaultq_to_udpen);
8339        }
8340        return 0;
8341}
8342
8343/**
8344 *      t4_init_portinfo - allocate a virtual interface amd initialize port_info
8345 *      @pi: the port_info
8346 *      @mbox: mailbox to use for the FW command
8347 *      @port: physical port associated with the VI
8348 *      @pf: the PF owning the VI
8349 *      @vf: the VF owning the VI
8350 *      @mac: the MAC address of the VI
8351 *
8352 *      Allocates a virtual interface for the given physical port.  If @mac is
8353 *      not %NULL it contains the MAC address of the VI as assigned by FW.
8354 *      @mac should be large enough to hold an Ethernet address.
8355 *      Returns < 0 on error.
8356 */
8357int t4_init_portinfo(struct port_info *pi, int mbox,
8358                     int port, int pf, int vf, u8 mac[])
8359{
8360        int ret;
8361        struct fw_port_cmd c;
8362        unsigned int rss_size;
8363
8364        memset(&c, 0, sizeof(c));
8365        c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
8366                                     FW_CMD_REQUEST_F | FW_CMD_READ_F |
8367                                     FW_PORT_CMD_PORTID_V(port));
8368        c.action_to_len16 = cpu_to_be32(
8369                FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_GET_PORT_INFO) |
8370                FW_LEN16(c));
8371        ret = t4_wr_mbox(pi->adapter, mbox, &c, sizeof(c), &c);
8372        if (ret)
8373                return ret;
8374
8375        ret = t4_alloc_vi(pi->adapter, mbox, port, pf, vf, 1, mac, &rss_size);
8376        if (ret < 0)
8377                return ret;
8378
8379        pi->viid = ret;
8380        pi->tx_chan = port;
8381        pi->lport = port;
8382        pi->rss_size = rss_size;
8383
8384        ret = be32_to_cpu(c.u.info.lstatus_to_modtype);
8385        pi->mdio_addr = (ret & FW_PORT_CMD_MDIOCAP_F) ?
8386                FW_PORT_CMD_MDIOADDR_G(ret) : -1;
8387        pi->port_type = FW_PORT_CMD_PTYPE_G(ret);
8388        pi->mod_type = FW_PORT_MOD_TYPE_NA;
8389
8390        init_link_config(&pi->link_cfg, be16_to_cpu(c.u.info.pcap),
8391                         be16_to_cpu(c.u.info.acap));
8392        return 0;
8393}
8394
8395int t4_port_init(struct adapter *adap, int mbox, int pf, int vf)
8396{
8397        u8 addr[6];
8398        int ret, i, j = 0;
8399
8400        for_each_port(adap, i) {
8401                struct port_info *pi = adap2pinfo(adap, i);
8402
8403                while ((adap->params.portvec & (1 << j)) == 0)
8404                        j++;
8405
8406                ret = t4_init_portinfo(pi, mbox, j, pf, vf, addr);
8407                if (ret)
8408                        return ret;
8409
8410                memcpy(adap->port[i]->dev_addr, addr, ETH_ALEN);
8411                j++;
8412        }
8413        return 0;
8414}
8415
8416/**
8417 *      t4_read_cimq_cfg - read CIM queue configuration
8418 *      @adap: the adapter
8419 *      @base: holds the queue base addresses in bytes
8420 *      @size: holds the queue sizes in bytes
8421 *      @thres: holds the queue full thresholds in bytes
8422 *
8423 *      Returns the current configuration of the CIM queues, starting with
8424 *      the IBQs, then the OBQs.
8425 */
8426void t4_read_cimq_cfg(struct adapter *adap, u16 *base, u16 *size, u16 *thres)
8427{
8428        unsigned int i, v;
8429        int cim_num_obq = is_t4(adap->params.chip) ?
8430                                CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
8431
8432        for (i = 0; i < CIM_NUM_IBQ; i++) {
8433                t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, IBQSELECT_F |
8434                             QUENUMSELECT_V(i));
8435                v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A);
8436                /* value is in 256-byte units */
8437                *base++ = CIMQBASE_G(v) * 256;
8438                *size++ = CIMQSIZE_G(v) * 256;
8439                *thres++ = QUEFULLTHRSH_G(v) * 8; /* 8-byte unit */
8440        }
8441        for (i = 0; i < cim_num_obq; i++) {
8442                t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, OBQSELECT_F |
8443                             QUENUMSELECT_V(i));
8444                v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A);
8445                /* value is in 256-byte units */
8446                *base++ = CIMQBASE_G(v) * 256;
8447                *size++ = CIMQSIZE_G(v) * 256;
8448        }
8449}
8450
8451/**
8452 *      t4_read_cim_ibq - read the contents of a CIM inbound queue
8453 *      @adap: the adapter
8454 *      @qid: the queue index
8455 *      @data: where to store the queue contents
8456 *      @n: capacity of @data in 32-bit words
8457 *
8458 *      Reads the contents of the selected CIM queue starting at address 0 up
8459 *      to the capacity of @data.  @n must be a multiple of 4.  Returns < 0 on
8460 *      error and the number of 32-bit words actually read on success.
8461 */
8462int t4_read_cim_ibq(struct adapter *adap, unsigned int qid, u32 *data, size_t n)
8463{
8464        int i, err, attempts;
8465        unsigned int addr;
8466        const unsigned int nwords = CIM_IBQ_SIZE * 4;
8467
8468        if (qid > 5 || (n & 3))
8469                return -EINVAL;
8470
8471        addr = qid * nwords;
8472        if (n > nwords)
8473                n = nwords;
8474
8475        /* It might take 3-10ms before the IBQ debug read access is allowed.
8476         * Wait for 1 Sec with a delay of 1 usec.
8477         */
8478        attempts = 1000000;
8479
8480        for (i = 0; i < n; i++, addr++) {
8481                t4_write_reg(adap, CIM_IBQ_DBG_CFG_A, IBQDBGADDR_V(addr) |
8482                             IBQDBGEN_F);
8483                err = t4_wait_op_done(adap, CIM_IBQ_DBG_CFG_A, IBQDBGBUSY_F, 0,
8484                                      attempts, 1);
8485                if (err)
8486                        return err;
8487                *data++ = t4_read_reg(adap, CIM_IBQ_DBG_DATA_A);
8488        }
8489        t4_write_reg(adap, CIM_IBQ_DBG_CFG_A, 0);
8490        return i;
8491}
8492
8493/**
8494 *      t4_read_cim_obq - read the contents of a CIM outbound queue
8495 *      @adap: the adapter
8496 *      @qid: the queue index
8497 *      @data: where to store the queue contents
8498 *      @n: capacity of @data in 32-bit words
8499 *
8500 *      Reads the contents of the selected CIM queue starting at address 0 up
8501 *      to the capacity of @data.  @n must be a multiple of 4.  Returns < 0 on
8502 *      error and the number of 32-bit words actually read on success.
8503 */
8504int t4_read_cim_obq(struct adapter *adap, unsigned int qid, u32 *data, size_t n)
8505{
8506        int i, err;
8507        unsigned int addr, v, nwords;
8508        int cim_num_obq = is_t4(adap->params.chip) ?
8509                                CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
8510
8511        if ((qid > (cim_num_obq - 1)) || (n & 3))
8512                return -EINVAL;
8513
8514        t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, OBQSELECT_F |
8515                     QUENUMSELECT_V(qid));
8516        v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A);
8517
8518        addr = CIMQBASE_G(v) * 64;    /* muliple of 256 -> muliple of 4 */
8519        nwords = CIMQSIZE_G(v) * 64;  /* same */
8520        if (n > nwords)
8521                n = nwords;
8522
8523        for (i = 0; i < n; i++, addr++) {
8524                t4_write_reg(adap, CIM_OBQ_DBG_CFG_A, OBQDBGADDR_V(addr) |
8525                             OBQDBGEN_F);
8526                err = t4_wait_op_done(adap, CIM_OBQ_DBG_CFG_A, OBQDBGBUSY_F, 0,
8527                                      2, 1);
8528                if (err)
8529                        return err;
8530                *data++ = t4_read_reg(adap, CIM_OBQ_DBG_DATA_A);
8531        }
8532        t4_write_reg(adap, CIM_OBQ_DBG_CFG_A, 0);
8533        return i;
8534}
8535
8536/**
8537 *      t4_cim_read - read a block from CIM internal address space
8538 *      @adap: the adapter
8539 *      @addr: the start address within the CIM address space
8540 *      @n: number of words to read
8541 *      @valp: where to store the result
8542 *
8543 *      Reads a block of 4-byte words from the CIM intenal address space.
8544 */
8545int t4_cim_read(struct adapter *adap, unsigned int addr, unsigned int n,
8546                unsigned int *valp)
8547{
8548        int ret = 0;
8549
8550        if (t4_read_reg(adap, CIM_HOST_ACC_CTRL_A) & HOSTBUSY_F)
8551                return -EBUSY;
8552
8553        for ( ; !ret && n--; addr += 4) {
8554                t4_write_reg(adap, CIM_HOST_ACC_CTRL_A, addr);
8555                ret = t4_wait_op_done(adap, CIM_HOST_ACC_CTRL_A, HOSTBUSY_F,
8556                                      0, 5, 2);
8557                if (!ret)
8558                        *valp++ = t4_read_reg(adap, CIM_HOST_ACC_DATA_A);
8559        }
8560        return ret;
8561}
8562
8563/**
8564 *      t4_cim_write - write a block into CIM internal address space
8565 *      @adap: the adapter
8566 *      @addr: the start address within the CIM address space
8567 *      @n: number of words to write
8568 *      @valp: set of values to write
8569 *
8570 *      Writes a block of 4-byte words into the CIM intenal address space.
8571 */
8572int t4_cim_write(struct adapter *adap, unsigned int addr, unsigned int n,
8573                 const unsigned int *valp)
8574{
8575        int ret = 0;
8576
8577        if (t4_read_reg(adap, CIM_HOST_ACC_CTRL_A) & HOSTBUSY_F)
8578                return -EBUSY;
8579
8580        for ( ; !ret && n--; addr += 4) {
8581                t4_write_reg(adap, CIM_HOST_ACC_DATA_A, *valp++);
8582                t4_write_reg(adap, CIM_HOST_ACC_CTRL_A, addr | HOSTWRITE_F);
8583                ret = t4_wait_op_done(adap, CIM_HOST_ACC_CTRL_A, HOSTBUSY_F,
8584                                      0, 5, 2);
8585        }
8586        return ret;
8587}
8588
8589static int t4_cim_write1(struct adapter *adap, unsigned int addr,
8590                         unsigned int val)
8591{
8592        return t4_cim_write(adap, addr, 1, &val);
8593}
8594
8595/**
8596 *      t4_cim_read_la - read CIM LA capture buffer
8597 *      @adap: the adapter
8598 *      @la_buf: where to store the LA data
8599 *      @wrptr: the HW write pointer within the capture buffer
8600 *
8601 *      Reads the contents of the CIM LA buffer with the most recent entry at
8602 *      the end of the returned data and with the entry at @wrptr first.
8603 *      We try to leave the LA in the running state we find it in.
8604 */
8605int t4_cim_read_la(struct adapter *adap, u32 *la_buf, unsigned int *wrptr)
8606{
8607        int i, ret;
8608        unsigned int cfg, val, idx;
8609
8610        ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &cfg);
8611        if (ret)
8612                return ret;
8613
8614        if (cfg & UPDBGLAEN_F) {        /* LA is running, freeze it */
8615                ret = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A, 0);
8616                if (ret)
8617                        return ret;
8618        }
8619
8620        ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &val);
8621        if (ret)
8622                goto restart;
8623
8624        idx = UPDBGLAWRPTR_G(val);
8625        if (wrptr)
8626                *wrptr = idx;
8627
8628        for (i = 0; i < adap->params.cim_la_size; i++) {
8629                ret = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A,
8630                                    UPDBGLARDPTR_V(idx) | UPDBGLARDEN_F);
8631                if (ret)
8632                        break;
8633                ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &val);
8634                if (ret)
8635                        break;
8636                if (val & UPDBGLARDEN_F) {
8637                        ret = -ETIMEDOUT;
8638                        break;
8639                }
8640                ret = t4_cim_read(adap, UP_UP_DBG_LA_DATA_A, 1, &la_buf[i]);
8641                if (ret)
8642                        break;
8643
8644                /* Bits 0-3 of UpDbgLaRdPtr can be between 0000 to 1001 to
8645                 * identify the 32-bit portion of the full 312-bit data
8646                 */
8647                if (is_t6(adap->params.chip) && (idx & 0xf) >= 9)
8648                        idx = (idx & 0xff0) + 0x10;
8649                else
8650                        idx++;
8651                /* address can't exceed 0xfff */
8652                idx &= UPDBGLARDPTR_M;
8653        }
8654restart:
8655        if (cfg & UPDBGLAEN_F) {
8656                int r = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A,
8657                                      cfg & ~UPDBGLARDEN_F);
8658                if (!ret)
8659                        ret = r;
8660        }
8661        return ret;
8662}
8663
8664/**
8665 *      t4_tp_read_la - read TP LA capture buffer
8666 *      @adap: the adapter
8667 *      @la_buf: where to store the LA data
8668 *      @wrptr: the HW write pointer within the capture buffer
8669 *
8670 *      Reads the contents of the TP LA buffer with the most recent entry at
8671 *      the end of the returned data and with the entry at @wrptr first.
8672 *      We leave the LA in the running state we find it in.
8673 */
8674void t4_tp_read_la(struct adapter *adap, u64 *la_buf, unsigned int *wrptr)
8675{
8676        bool last_incomplete;
8677        unsigned int i, cfg, val, idx;
8678
8679        cfg = t4_read_reg(adap, TP_DBG_LA_CONFIG_A) & 0xffff;
8680        if (cfg & DBGLAENABLE_F)                        /* freeze LA */
8681                t4_write_reg(adap, TP_DBG_LA_CONFIG_A,
8682                             adap->params.tp.la_mask | (cfg ^ DBGLAENABLE_F));
8683
8684        val = t4_read_reg(adap, TP_DBG_LA_CONFIG_A);
8685        idx = DBGLAWPTR_G(val);
8686        last_incomplete = DBGLAMODE_G(val) >= 2 && (val & DBGLAWHLF_F) == 0;
8687        if (last_incomplete)
8688                idx = (idx + 1) & DBGLARPTR_M;
8689        if (wrptr)
8690                *wrptr = idx;
8691
8692        val &= 0xffff;
8693        val &= ~DBGLARPTR_V(DBGLARPTR_M);
8694        val |= adap->params.tp.la_mask;
8695
8696        for (i = 0; i < TPLA_SIZE; i++) {
8697                t4_write_reg(adap, TP_DBG_LA_CONFIG_A, DBGLARPTR_V(idx) | val);
8698                la_buf[i] = t4_read_reg64(adap, TP_DBG_LA_DATAL_A);
8699                idx = (idx + 1) & DBGLARPTR_M;
8700        }
8701
8702        /* Wipe out last entry if it isn't valid */
8703        if (last_incomplete)
8704                la_buf[TPLA_SIZE - 1] = ~0ULL;
8705
8706        if (cfg & DBGLAENABLE_F)                    /* restore running state */
8707                t4_write_reg(adap, TP_DBG_LA_CONFIG_A,
8708                             cfg | adap->params.tp.la_mask);
8709}
8710
8711/* SGE Hung Ingress DMA Warning Threshold time and Warning Repeat Rate (in
8712 * seconds).  If we find one of the SGE Ingress DMA State Machines in the same
8713 * state for more than the Warning Threshold then we'll issue a warning about
8714 * a potential hang.  We'll repeat the warning as the SGE Ingress DMA Channel
8715 * appears to be hung every Warning Repeat second till the situation clears.
8716 * If the situation clears, we'll note that as well.
8717 */
8718#define SGE_IDMA_WARN_THRESH 1
8719#define SGE_IDMA_WARN_REPEAT 300
8720
8721/**
8722 *      t4_idma_monitor_init - initialize SGE Ingress DMA Monitor
8723 *      @adapter: the adapter
8724 *      @idma: the adapter IDMA Monitor state
8725 *
8726 *      Initialize the state of an SGE Ingress DMA Monitor.
8727 */
8728void t4_idma_monitor_init(struct adapter *adapter,
8729                          struct sge_idma_monitor_state *idma)
8730{
8731        /* Initialize the state variables for detecting an SGE Ingress DMA
8732         * hang.  The SGE has internal counters which count up on each clock
8733         * tick whenever the SGE finds its Ingress DMA State Engines in the
8734         * same state they were on the previous clock tick.  The clock used is
8735         * the Core Clock so we have a limit on the maximum "time" they can
8736         * record; typically a very small number of seconds.  For instance,
8737         * with a 600MHz Core Clock, we can only count up to a bit more than
8738         * 7s.  So we'll synthesize a larger counter in order to not run the
8739         * risk of having the "timers" overflow and give us the flexibility to
8740         * maintain a Hung SGE State Machine of our own which operates across
8741         * a longer time frame.
8742         */
8743        idma->idma_1s_thresh = core_ticks_per_usec(adapter) * 1000000; /* 1s */
8744        idma->idma_stalled[0] = 0;
8745        idma->idma_stalled[1] = 0;
8746}
8747
8748/**
8749 *      t4_idma_monitor - monitor SGE Ingress DMA state
8750 *      @adapter: the adapter
8751 *      @idma: the adapter IDMA Monitor state
8752 *      @hz: number of ticks/second
8753 *      @ticks: number of ticks since the last IDMA Monitor call
8754 */
8755void t4_idma_monitor(struct adapter *adapter,
8756                     struct sge_idma_monitor_state *idma,
8757                     int hz, int ticks)
8758{
8759        int i, idma_same_state_cnt[2];
8760
8761         /* Read the SGE Debug Ingress DMA Same State Count registers.  These
8762          * are counters inside the SGE which count up on each clock when the
8763          * SGE finds its Ingress DMA State Engines in the same states they
8764          * were in the previous clock.  The counters will peg out at
8765          * 0xffffffff without wrapping around so once they pass the 1s
8766          * threshold they'll stay above that till the IDMA state changes.
8767          */
8768        t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 13);
8769        idma_same_state_cnt[0] = t4_read_reg(adapter, SGE_DEBUG_DATA_HIGH_A);
8770        idma_same_state_cnt[1] = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A);
8771
8772        for (i = 0; i < 2; i++) {
8773                u32 debug0, debug11;
8774
8775                /* If the Ingress DMA Same State Counter ("timer") is less
8776                 * than 1s, then we can reset our synthesized Stall Timer and
8777                 * continue.  If we have previously emitted warnings about a
8778                 * potential stalled Ingress Queue, issue a note indicating
8779                 * that the Ingress Queue has resumed forward progress.
8780                 */
8781                if (idma_same_state_cnt[i] < idma->idma_1s_thresh) {
8782                        if (idma->idma_stalled[i] >= SGE_IDMA_WARN_THRESH * hz)
8783                                dev_warn(adapter->pdev_dev, "SGE idma%d, queue %u, "
8784                                         "resumed after %d seconds\n",
8785                                         i, idma->idma_qid[i],
8786                                         idma->idma_stalled[i] / hz);
8787                        idma->idma_stalled[i] = 0;
8788                        continue;
8789                }
8790
8791                /* Synthesize an SGE Ingress DMA Same State Timer in the Hz
8792                 * domain.  The first time we get here it'll be because we
8793                 * passed the 1s Threshold; each additional time it'll be
8794                 * because the RX Timer Callback is being fired on its regular
8795                 * schedule.
8796                 *
8797                 * If the stall is below our Potential Hung Ingress Queue
8798                 * Warning Threshold, continue.
8799                 */
8800                if (idma->idma_stalled[i] == 0) {
8801                        idma->idma_stalled[i] = hz;
8802                        idma->idma_warn[i] = 0;
8803                } else {
8804                        idma->idma_stalled[i] += ticks;
8805                        idma->idma_warn[i] -= ticks;
8806                }
8807
8808                if (idma->idma_stalled[i] < SGE_IDMA_WARN_THRESH * hz)
8809                        continue;
8810
8811                /* We'll issue a warning every SGE_IDMA_WARN_REPEAT seconds.
8812                 */
8813                if (idma->idma_warn[i] > 0)
8814                        continue;
8815                idma->idma_warn[i] = SGE_IDMA_WARN_REPEAT * hz;
8816
8817                /* Read and save the SGE IDMA State and Queue ID information.
8818                 * We do this every time in case it changes across time ...
8819                 * can't be too careful ...
8820                 */
8821                t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 0);
8822                debug0 = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A);
8823                idma->idma_state[i] = (debug0 >> (i * 9)) & 0x3f;
8824
8825                t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 11);
8826                debug11 = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A);
8827                idma->idma_qid[i] = (debug11 >> (i * 16)) & 0xffff;
8828
8829                dev_warn(adapter->pdev_dev, "SGE idma%u, queue %u, potentially stuck in "
8830                         "state %u for %d seconds (debug0=%#x, debug11=%#x)\n",
8831                         i, idma->idma_qid[i], idma->idma_state[i],
8832                         idma->idma_stalled[i] / hz,
8833                         debug0, debug11);
8834                t4_sge_decode_idma_state(adapter, idma->idma_state[i]);
8835        }
8836}
8837
8838/**
8839 *      t4_load_cfg - download config file
8840 *      @adap: the adapter
8841 *      @cfg_data: the cfg text file to write
8842 *      @size: text file size
8843 *
8844 *      Write the supplied config text file to the card's serial flash.
8845 */
8846int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size)
8847{
8848        int ret, i, n, cfg_addr;
8849        unsigned int addr;
8850        unsigned int flash_cfg_start_sec;
8851        unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
8852
8853        cfg_addr = t4_flash_cfg_addr(adap);
8854        if (cfg_addr < 0)
8855                return cfg_addr;
8856
8857        addr = cfg_addr;
8858        flash_cfg_start_sec = addr / SF_SEC_SIZE;
8859
8860        if (size > FLASH_CFG_MAX_SIZE) {
8861                dev_err(adap->pdev_dev, "cfg file too large, max is %u bytes\n",
8862                        FLASH_CFG_MAX_SIZE);
8863                return -EFBIG;
8864        }
8865
8866        i = DIV_ROUND_UP(FLASH_CFG_MAX_SIZE,    /* # of sectors spanned */
8867                         sf_sec_size);
8868        ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec,
8869                                     flash_cfg_start_sec + i - 1);
8870        /* If size == 0 then we're simply erasing the FLASH sectors associated
8871         * with the on-adapter Firmware Configuration File.
8872         */
8873        if (ret || size == 0)
8874                goto out;
8875
8876        /* this will write to the flash up to SF_PAGE_SIZE at a time */
8877        for (i = 0; i < size; i += SF_PAGE_SIZE) {
8878                if ((size - i) <  SF_PAGE_SIZE)
8879                        n = size - i;
8880                else
8881                        n = SF_PAGE_SIZE;
8882                ret = t4_write_flash(adap, addr, n, cfg_data);
8883                if (ret)
8884                        goto out;
8885
8886                addr += SF_PAGE_SIZE;
8887                cfg_data += SF_PAGE_SIZE;
8888        }
8889
8890out:
8891        if (ret)
8892                dev_err(adap->pdev_dev, "config file %s failed %d\n",
8893                        (size == 0 ? "clear" : "download"), ret);
8894        return ret;
8895}
8896
8897/**
8898 *      t4_set_vf_mac - Set MAC address for the specified VF
8899 *      @adapter: The adapter
8900 *      @vf: one of the VFs instantiated by the specified PF
8901 *      @naddr: the number of MAC addresses
8902 *      @addr: the MAC address(es) to be set to the specified VF
8903 */
8904int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf,
8905                      unsigned int naddr, u8 *addr)
8906{
8907        struct fw_acl_mac_cmd cmd;
8908
8909        memset(&cmd, 0, sizeof(cmd));
8910        cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_MAC_CMD) |
8911                                    FW_CMD_REQUEST_F |
8912                                    FW_CMD_WRITE_F |
8913                                    FW_ACL_MAC_CMD_PFN_V(adapter->pf) |
8914                                    FW_ACL_MAC_CMD_VFN_V(vf));
8915
8916        /* Note: Do not enable the ACL */
8917        cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd));
8918        cmd.nmac = naddr;
8919
8920        switch (adapter->pf) {
8921        case 3:
8922                memcpy(cmd.macaddr3, addr, sizeof(cmd.macaddr3));
8923                break;
8924        case 2:
8925                memcpy(cmd.macaddr2, addr, sizeof(cmd.macaddr2));
8926                break;
8927        case 1:
8928                memcpy(cmd.macaddr1, addr, sizeof(cmd.macaddr1));
8929                break;
8930        case 0:
8931                memcpy(cmd.macaddr0, addr, sizeof(cmd.macaddr0));
8932                break;
8933        }
8934
8935        return t4_wr_mbox(adapter, adapter->mbox, &cmd, sizeof(cmd), &cmd);
8936}
8937
8938int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
8939                    int rateunit, int ratemode, int channel, int class,
8940                    int minrate, int maxrate, int weight, int pktsize)
8941{
8942        struct fw_sched_cmd cmd;
8943
8944        memset(&cmd, 0, sizeof(cmd));
8945        cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_SCHED_CMD) |
8946                                      FW_CMD_REQUEST_F |
8947                                      FW_CMD_WRITE_F);
8948        cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
8949
8950        cmd.u.params.sc = FW_SCHED_SC_PARAMS;
8951        cmd.u.params.type = type;
8952        cmd.u.params.level = level;
8953        cmd.u.params.mode = mode;
8954        cmd.u.params.ch = channel;
8955        cmd.u.params.cl = class;
8956        cmd.u.params.unit = rateunit;
8957        cmd.u.params.rate = ratemode;
8958        cmd.u.params.min = cpu_to_be32(minrate);
8959        cmd.u.params.max = cpu_to_be32(maxrate);
8960        cmd.u.params.weight = cpu_to_be16(weight);
8961        cmd.u.params.pktsize = cpu_to_be16(pktsize);
8962
8963        return t4_wr_mbox_meat(adapter, adapter->mbox, &cmd, sizeof(cmd),
8964                               NULL, 1);
8965}
8966