linux/drivers/ntb/hw/idt/ntb_hw_idt.c
<<
>>
Prefs
   1/*
   2 *   This file is provided under a GPLv2 license.  When using or
   3 *   redistributing this file, you may do so under that license.
   4 *
   5 *   GPL LICENSE SUMMARY
   6 *
   7 *   Copyright (C) 2016-2018 T-Platforms JSC All Rights Reserved.
   8 *
   9 *   This program is free software; you can redistribute it and/or modify it
  10 *   under the terms and conditions of the GNU General Public License,
  11 *   version 2, as published by the Free Software Foundation.
  12 *
  13 *   This program is distributed in the hope that it will be useful, but
  14 *   WITHOUT ANY WARRANTY; without even the implied warranty of
  15 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
  16 *   Public License for more details.
  17 *
  18 *   You should have received a copy of the GNU General Public License along
  19 *   with this program; if not, one can be found http://www.gnu.org/licenses/.
  20 *
  21 *   The full GNU General Public License is included in this distribution in
  22 *   the file called "COPYING".
  23 *
  24 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  27 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  28 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  30 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  34 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35 *
  36 * IDT PCIe-switch NTB Linux driver
  37 *
  38 * Contact Information:
  39 * Serge Semin <fancer.lancer@gmail.com>, <Sergey.Semin@t-platforms.ru>
  40 */
  41
  42#include <linux/stddef.h>
  43#include <linux/types.h>
  44#include <linux/kernel.h>
  45#include <linux/bitops.h>
  46#include <linux/sizes.h>
  47#include <linux/module.h>
  48#include <linux/moduleparam.h>
  49#include <linux/init.h>
  50#include <linux/interrupt.h>
  51#include <linux/spinlock.h>
  52#include <linux/mutex.h>
  53#include <linux/pci.h>
  54#include <linux/aer.h>
  55#include <linux/slab.h>
  56#include <linux/list.h>
  57#include <linux/debugfs.h>
  58#include <linux/hwmon.h>
  59#include <linux/hwmon-sysfs.h>
  60#include <linux/ntb.h>
  61
  62#include "ntb_hw_idt.h"
  63
  64#define NTB_NAME        "ntb_hw_idt"
  65#define NTB_DESC        "IDT PCI-E Non-Transparent Bridge Driver"
  66#define NTB_VER         "2.0"
  67#define NTB_IRQNAME     "ntb_irq_idt"
  68
  69MODULE_DESCRIPTION(NTB_DESC);
  70MODULE_VERSION(NTB_VER);
  71MODULE_LICENSE("GPL v2");
  72MODULE_AUTHOR("T-platforms");
  73
  74/*
  75 * NT Endpoint registers table simplifying a loop access to the functionally
  76 * related registers
  77 */
  78static const struct idt_ntb_regs ntdata_tbl = {
  79        { {IDT_NT_BARSETUP0,    IDT_NT_BARLIMIT0,
  80           IDT_NT_BARLTBASE0,   IDT_NT_BARUTBASE0},
  81          {IDT_NT_BARSETUP1,    IDT_NT_BARLIMIT1,
  82           IDT_NT_BARLTBASE1,   IDT_NT_BARUTBASE1},
  83          {IDT_NT_BARSETUP2,    IDT_NT_BARLIMIT2,
  84           IDT_NT_BARLTBASE2,   IDT_NT_BARUTBASE2},
  85          {IDT_NT_BARSETUP3,    IDT_NT_BARLIMIT3,
  86           IDT_NT_BARLTBASE3,   IDT_NT_BARUTBASE3},
  87          {IDT_NT_BARSETUP4,    IDT_NT_BARLIMIT4,
  88           IDT_NT_BARLTBASE4,   IDT_NT_BARUTBASE4},
  89          {IDT_NT_BARSETUP5,    IDT_NT_BARLIMIT5,
  90           IDT_NT_BARLTBASE5,   IDT_NT_BARUTBASE5} },
  91        { {IDT_NT_INMSG0,       IDT_NT_OUTMSG0, IDT_NT_INMSGSRC0},
  92          {IDT_NT_INMSG1,       IDT_NT_OUTMSG1, IDT_NT_INMSGSRC1},
  93          {IDT_NT_INMSG2,       IDT_NT_OUTMSG2, IDT_NT_INMSGSRC2},
  94          {IDT_NT_INMSG3,       IDT_NT_OUTMSG3, IDT_NT_INMSGSRC3} }
  95};
  96
  97/*
  98 * NT Endpoint ports data table with the corresponding pcie command, link
  99 * status, control and BAR-related registers
 100 */
 101static const struct idt_ntb_port portdata_tbl[IDT_MAX_NR_PORTS] = {
 102/*0*/   { IDT_SW_NTP0_PCIECMDSTS,       IDT_SW_NTP0_PCIELCTLSTS,
 103          IDT_SW_NTP0_NTCTL,
 104          IDT_SW_SWPORT0CTL,            IDT_SW_SWPORT0STS,
 105          { {IDT_SW_NTP0_BARSETUP0,     IDT_SW_NTP0_BARLIMIT0,
 106             IDT_SW_NTP0_BARLTBASE0,    IDT_SW_NTP0_BARUTBASE0},
 107            {IDT_SW_NTP0_BARSETUP1,     IDT_SW_NTP0_BARLIMIT1,
 108             IDT_SW_NTP0_BARLTBASE1,    IDT_SW_NTP0_BARUTBASE1},
 109            {IDT_SW_NTP0_BARSETUP2,     IDT_SW_NTP0_BARLIMIT2,
 110             IDT_SW_NTP0_BARLTBASE2,    IDT_SW_NTP0_BARUTBASE2},
 111            {IDT_SW_NTP0_BARSETUP3,     IDT_SW_NTP0_BARLIMIT3,
 112             IDT_SW_NTP0_BARLTBASE3,    IDT_SW_NTP0_BARUTBASE3},
 113            {IDT_SW_NTP0_BARSETUP4,     IDT_SW_NTP0_BARLIMIT4,
 114             IDT_SW_NTP0_BARLTBASE4,    IDT_SW_NTP0_BARUTBASE4},
 115            {IDT_SW_NTP0_BARSETUP5,     IDT_SW_NTP0_BARLIMIT5,
 116             IDT_SW_NTP0_BARLTBASE5,    IDT_SW_NTP0_BARUTBASE5} } },
 117/*1*/   {0},
 118/*2*/   { IDT_SW_NTP2_PCIECMDSTS,       IDT_SW_NTP2_PCIELCTLSTS,
 119          IDT_SW_NTP2_NTCTL,
 120          IDT_SW_SWPORT2CTL,            IDT_SW_SWPORT2STS,
 121          { {IDT_SW_NTP2_BARSETUP0,     IDT_SW_NTP2_BARLIMIT0,
 122             IDT_SW_NTP2_BARLTBASE0,    IDT_SW_NTP2_BARUTBASE0},
 123            {IDT_SW_NTP2_BARSETUP1,     IDT_SW_NTP2_BARLIMIT1,
 124             IDT_SW_NTP2_BARLTBASE1,    IDT_SW_NTP2_BARUTBASE1},
 125            {IDT_SW_NTP2_BARSETUP2,     IDT_SW_NTP2_BARLIMIT2,
 126             IDT_SW_NTP2_BARLTBASE2,    IDT_SW_NTP2_BARUTBASE2},
 127            {IDT_SW_NTP2_BARSETUP3,     IDT_SW_NTP2_BARLIMIT3,
 128             IDT_SW_NTP2_BARLTBASE3,    IDT_SW_NTP2_BARUTBASE3},
 129            {IDT_SW_NTP2_BARSETUP4,     IDT_SW_NTP2_BARLIMIT4,
 130             IDT_SW_NTP2_BARLTBASE4,    IDT_SW_NTP2_BARUTBASE4},
 131            {IDT_SW_NTP2_BARSETUP5,     IDT_SW_NTP2_BARLIMIT5,
 132             IDT_SW_NTP2_BARLTBASE5,    IDT_SW_NTP2_BARUTBASE5} } },
 133/*3*/   {0},
 134/*4*/   { IDT_SW_NTP4_PCIECMDSTS,       IDT_SW_NTP4_PCIELCTLSTS,
 135          IDT_SW_NTP4_NTCTL,
 136          IDT_SW_SWPORT4CTL,            IDT_SW_SWPORT4STS,
 137          { {IDT_SW_NTP4_BARSETUP0,     IDT_SW_NTP4_BARLIMIT0,
 138             IDT_SW_NTP4_BARLTBASE0,    IDT_SW_NTP4_BARUTBASE0},
 139            {IDT_SW_NTP4_BARSETUP1,     IDT_SW_NTP4_BARLIMIT1,
 140             IDT_SW_NTP4_BARLTBASE1,    IDT_SW_NTP4_BARUTBASE1},
 141            {IDT_SW_NTP4_BARSETUP2,     IDT_SW_NTP4_BARLIMIT2,
 142             IDT_SW_NTP4_BARLTBASE2,    IDT_SW_NTP4_BARUTBASE2},
 143            {IDT_SW_NTP4_BARSETUP3,     IDT_SW_NTP4_BARLIMIT3,
 144             IDT_SW_NTP4_BARLTBASE3,    IDT_SW_NTP4_BARUTBASE3},
 145            {IDT_SW_NTP4_BARSETUP4,     IDT_SW_NTP4_BARLIMIT4,
 146             IDT_SW_NTP4_BARLTBASE4,    IDT_SW_NTP4_BARUTBASE4},
 147            {IDT_SW_NTP4_BARSETUP5,     IDT_SW_NTP4_BARLIMIT5,
 148             IDT_SW_NTP4_BARLTBASE5,    IDT_SW_NTP4_BARUTBASE5} } },
 149/*5*/   {0},
 150/*6*/   { IDT_SW_NTP6_PCIECMDSTS,       IDT_SW_NTP6_PCIELCTLSTS,
 151          IDT_SW_NTP6_NTCTL,
 152          IDT_SW_SWPORT6CTL,            IDT_SW_SWPORT6STS,
 153          { {IDT_SW_NTP6_BARSETUP0,     IDT_SW_NTP6_BARLIMIT0,
 154             IDT_SW_NTP6_BARLTBASE0,    IDT_SW_NTP6_BARUTBASE0},
 155            {IDT_SW_NTP6_BARSETUP1,     IDT_SW_NTP6_BARLIMIT1,
 156             IDT_SW_NTP6_BARLTBASE1,    IDT_SW_NTP6_BARUTBASE1},
 157            {IDT_SW_NTP6_BARSETUP2,     IDT_SW_NTP6_BARLIMIT2,
 158             IDT_SW_NTP6_BARLTBASE2,    IDT_SW_NTP6_BARUTBASE2},
 159            {IDT_SW_NTP6_BARSETUP3,     IDT_SW_NTP6_BARLIMIT3,
 160             IDT_SW_NTP6_BARLTBASE3,    IDT_SW_NTP6_BARUTBASE3},
 161            {IDT_SW_NTP6_BARSETUP4,     IDT_SW_NTP6_BARLIMIT4,
 162             IDT_SW_NTP6_BARLTBASE4,    IDT_SW_NTP6_BARUTBASE4},
 163            {IDT_SW_NTP6_BARSETUP5,     IDT_SW_NTP6_BARLIMIT5,
 164             IDT_SW_NTP6_BARLTBASE5,    IDT_SW_NTP6_BARUTBASE5} } },
 165/*7*/   {0},
 166/*8*/   { IDT_SW_NTP8_PCIECMDSTS,       IDT_SW_NTP8_PCIELCTLSTS,
 167          IDT_SW_NTP8_NTCTL,
 168          IDT_SW_SWPORT8CTL,            IDT_SW_SWPORT8STS,
 169          { {IDT_SW_NTP8_BARSETUP0,     IDT_SW_NTP8_BARLIMIT0,
 170             IDT_SW_NTP8_BARLTBASE0,    IDT_SW_NTP8_BARUTBASE0},
 171            {IDT_SW_NTP8_BARSETUP1,     IDT_SW_NTP8_BARLIMIT1,
 172             IDT_SW_NTP8_BARLTBASE1,    IDT_SW_NTP8_BARUTBASE1},
 173            {IDT_SW_NTP8_BARSETUP2,     IDT_SW_NTP8_BARLIMIT2,
 174             IDT_SW_NTP8_BARLTBASE2,    IDT_SW_NTP8_BARUTBASE2},
 175            {IDT_SW_NTP8_BARSETUP3,     IDT_SW_NTP8_BARLIMIT3,
 176             IDT_SW_NTP8_BARLTBASE3,    IDT_SW_NTP8_BARUTBASE3},
 177            {IDT_SW_NTP8_BARSETUP4,     IDT_SW_NTP8_BARLIMIT4,
 178             IDT_SW_NTP8_BARLTBASE4,    IDT_SW_NTP8_BARUTBASE4},
 179            {IDT_SW_NTP8_BARSETUP5,     IDT_SW_NTP8_BARLIMIT5,
 180             IDT_SW_NTP8_BARLTBASE5,    IDT_SW_NTP8_BARUTBASE5} } },
 181/*9*/   {0},
 182/*10*/  {0},
 183/*11*/  {0},
 184/*12*/  { IDT_SW_NTP12_PCIECMDSTS,      IDT_SW_NTP12_PCIELCTLSTS,
 185          IDT_SW_NTP12_NTCTL,
 186          IDT_SW_SWPORT12CTL,           IDT_SW_SWPORT12STS,
 187          { {IDT_SW_NTP12_BARSETUP0,    IDT_SW_NTP12_BARLIMIT0,
 188             IDT_SW_NTP12_BARLTBASE0,   IDT_SW_NTP12_BARUTBASE0},
 189            {IDT_SW_NTP12_BARSETUP1,    IDT_SW_NTP12_BARLIMIT1,
 190             IDT_SW_NTP12_BARLTBASE1,   IDT_SW_NTP12_BARUTBASE1},
 191            {IDT_SW_NTP12_BARSETUP2,    IDT_SW_NTP12_BARLIMIT2,
 192             IDT_SW_NTP12_BARLTBASE2,   IDT_SW_NTP12_BARUTBASE2},
 193            {IDT_SW_NTP12_BARSETUP3,    IDT_SW_NTP12_BARLIMIT3,
 194             IDT_SW_NTP12_BARLTBASE3,   IDT_SW_NTP12_BARUTBASE3},
 195            {IDT_SW_NTP12_BARSETUP4,    IDT_SW_NTP12_BARLIMIT4,
 196             IDT_SW_NTP12_BARLTBASE4,   IDT_SW_NTP12_BARUTBASE4},
 197            {IDT_SW_NTP12_BARSETUP5,    IDT_SW_NTP12_BARLIMIT5,
 198             IDT_SW_NTP12_BARLTBASE5,   IDT_SW_NTP12_BARUTBASE5} } },
 199/*13*/  {0},
 200/*14*/  {0},
 201/*15*/  {0},
 202/*16*/  { IDT_SW_NTP16_PCIECMDSTS,      IDT_SW_NTP16_PCIELCTLSTS,
 203          IDT_SW_NTP16_NTCTL,
 204          IDT_SW_SWPORT16CTL,           IDT_SW_SWPORT16STS,
 205          { {IDT_SW_NTP16_BARSETUP0,    IDT_SW_NTP16_BARLIMIT0,
 206             IDT_SW_NTP16_BARLTBASE0,   IDT_SW_NTP16_BARUTBASE0},
 207            {IDT_SW_NTP16_BARSETUP1,    IDT_SW_NTP16_BARLIMIT1,
 208             IDT_SW_NTP16_BARLTBASE1,   IDT_SW_NTP16_BARUTBASE1},
 209            {IDT_SW_NTP16_BARSETUP2,    IDT_SW_NTP16_BARLIMIT2,
 210             IDT_SW_NTP16_BARLTBASE2,   IDT_SW_NTP16_BARUTBASE2},
 211            {IDT_SW_NTP16_BARSETUP3,    IDT_SW_NTP16_BARLIMIT3,
 212             IDT_SW_NTP16_BARLTBASE3,   IDT_SW_NTP16_BARUTBASE3},
 213            {IDT_SW_NTP16_BARSETUP4,    IDT_SW_NTP16_BARLIMIT4,
 214             IDT_SW_NTP16_BARLTBASE4,   IDT_SW_NTP16_BARUTBASE4},
 215            {IDT_SW_NTP16_BARSETUP5,    IDT_SW_NTP16_BARLIMIT5,
 216             IDT_SW_NTP16_BARLTBASE5,   IDT_SW_NTP16_BARUTBASE5} } },
 217/*17*/  {0},
 218/*18*/  {0},
 219/*19*/  {0},
 220/*20*/  { IDT_SW_NTP20_PCIECMDSTS,      IDT_SW_NTP20_PCIELCTLSTS,
 221          IDT_SW_NTP20_NTCTL,
 222          IDT_SW_SWPORT20CTL,           IDT_SW_SWPORT20STS,
 223          { {IDT_SW_NTP20_BARSETUP0,    IDT_SW_NTP20_BARLIMIT0,
 224             IDT_SW_NTP20_BARLTBASE0,   IDT_SW_NTP20_BARUTBASE0},
 225            {IDT_SW_NTP20_BARSETUP1,    IDT_SW_NTP20_BARLIMIT1,
 226             IDT_SW_NTP20_BARLTBASE1,   IDT_SW_NTP20_BARUTBASE1},
 227            {IDT_SW_NTP20_BARSETUP2,    IDT_SW_NTP20_BARLIMIT2,
 228             IDT_SW_NTP20_BARLTBASE2,   IDT_SW_NTP20_BARUTBASE2},
 229            {IDT_SW_NTP20_BARSETUP3,    IDT_SW_NTP20_BARLIMIT3,
 230             IDT_SW_NTP20_BARLTBASE3,   IDT_SW_NTP20_BARUTBASE3},
 231            {IDT_SW_NTP20_BARSETUP4,    IDT_SW_NTP20_BARLIMIT4,
 232             IDT_SW_NTP20_BARLTBASE4,   IDT_SW_NTP20_BARUTBASE4},
 233            {IDT_SW_NTP20_BARSETUP5,    IDT_SW_NTP20_BARLIMIT5,
 234             IDT_SW_NTP20_BARLTBASE5,   IDT_SW_NTP20_BARUTBASE5} } },
 235/*21*/  {0},
 236/*22*/  {0},
 237/*23*/  {0}
 238};
 239
 240/*
 241 * IDT PCIe-switch partitions table with the corresponding control, status
 242 * and messages control registers
 243 */
 244static const struct idt_ntb_part partdata_tbl[IDT_MAX_NR_PARTS] = {
 245/*0*/   { IDT_SW_SWPART0CTL,    IDT_SW_SWPART0STS,
 246          {IDT_SW_SWP0MSGCTL0,  IDT_SW_SWP0MSGCTL1,
 247           IDT_SW_SWP0MSGCTL2,  IDT_SW_SWP0MSGCTL3} },
 248/*1*/   { IDT_SW_SWPART1CTL,    IDT_SW_SWPART1STS,
 249          {IDT_SW_SWP1MSGCTL0,  IDT_SW_SWP1MSGCTL1,
 250           IDT_SW_SWP1MSGCTL2,  IDT_SW_SWP1MSGCTL3} },
 251/*2*/   { IDT_SW_SWPART2CTL,    IDT_SW_SWPART2STS,
 252          {IDT_SW_SWP2MSGCTL0,  IDT_SW_SWP2MSGCTL1,
 253           IDT_SW_SWP2MSGCTL2,  IDT_SW_SWP2MSGCTL3} },
 254/*3*/   { IDT_SW_SWPART3CTL,    IDT_SW_SWPART3STS,
 255          {IDT_SW_SWP3MSGCTL0,  IDT_SW_SWP3MSGCTL1,
 256           IDT_SW_SWP3MSGCTL2,  IDT_SW_SWP3MSGCTL3} },
 257/*4*/   { IDT_SW_SWPART4CTL,    IDT_SW_SWPART4STS,
 258          {IDT_SW_SWP4MSGCTL0,  IDT_SW_SWP4MSGCTL1,
 259           IDT_SW_SWP4MSGCTL2,  IDT_SW_SWP4MSGCTL3} },
 260/*5*/   { IDT_SW_SWPART5CTL,    IDT_SW_SWPART5STS,
 261          {IDT_SW_SWP5MSGCTL0,  IDT_SW_SWP5MSGCTL1,
 262           IDT_SW_SWP5MSGCTL2,  IDT_SW_SWP5MSGCTL3} },
 263/*6*/   { IDT_SW_SWPART6CTL,    IDT_SW_SWPART6STS,
 264          {IDT_SW_SWP6MSGCTL0,  IDT_SW_SWP6MSGCTL1,
 265           IDT_SW_SWP6MSGCTL2,  IDT_SW_SWP6MSGCTL3} },
 266/*7*/   { IDT_SW_SWPART7CTL,    IDT_SW_SWPART7STS,
 267          {IDT_SW_SWP7MSGCTL0,  IDT_SW_SWP7MSGCTL1,
 268           IDT_SW_SWP7MSGCTL2,  IDT_SW_SWP7MSGCTL3} }
 269};
 270
 271/*
 272 * DebugFS directory to place the driver debug file
 273 */
 274static struct dentry *dbgfs_topdir;
 275
 276/*=============================================================================
 277 *                1. IDT PCIe-switch registers IO-functions
 278 *
 279 *    Beside ordinary configuration space registers IDT PCIe-switch expose
 280 * global configuration registers, which are used to determine state of other
 281 * device ports as well as being notified of some switch-related events.
 282 * Additionally all the configuration space registers of all the IDT
 283 * PCIe-switch functions are mapped to the Global Address space, so each
 284 * function can determine a configuration of any other PCI-function.
 285 *    Functions declared in this chapter are created to encapsulate access
 286 * to configuration and global registers, so the driver code just need to
 287 * provide IDT NTB hardware descriptor and a register address.
 288 *=============================================================================
 289 */
 290
 291/*
 292 * idt_nt_write() - PCI configuration space registers write method
 293 * @ndev:       IDT NTB hardware driver descriptor
 294 * @reg:        Register to write data to
 295 * @data:       Value to write to the register
 296 *
 297 * IDT PCIe-switch registers are all Little endian.
 298 */
 299static void idt_nt_write(struct idt_ntb_dev *ndev,
 300                         const unsigned int reg, const u32 data)
 301{
 302        /*
 303         * It's obvious bug to request a register exceeding the maximum possible
 304         * value as well as to have it unaligned.
 305         */
 306        if (WARN_ON(reg > IDT_REG_PCI_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN)))
 307                return;
 308
 309        /* Just write the value to the specified register */
 310        iowrite32(data, ndev->cfgspc + (ptrdiff_t)reg);
 311}
 312
 313/*
 314 * idt_nt_read() - PCI configuration space registers read method
 315 * @ndev:       IDT NTB hardware driver descriptor
 316 * @reg:        Register to write data to
 317 *
 318 * IDT PCIe-switch Global configuration registers are all Little endian.
 319 *
 320 * Return: register value
 321 */
 322static u32 idt_nt_read(struct idt_ntb_dev *ndev, const unsigned int reg)
 323{
 324        /*
 325         * It's obvious bug to request a register exceeding the maximum possible
 326         * value as well as to have it unaligned.
 327         */
 328        if (WARN_ON(reg > IDT_REG_PCI_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN)))
 329                return ~0;
 330
 331        /* Just read the value from the specified register */
 332        return ioread32(ndev->cfgspc + (ptrdiff_t)reg);
 333}
 334
 335/*
 336 * idt_sw_write() - Global registers write method
 337 * @ndev:       IDT NTB hardware driver descriptor
 338 * @reg:        Register to write data to
 339 * @data:       Value to write to the register
 340 *
 341 * IDT PCIe-switch Global configuration registers are all Little endian.
 342 */
 343static void idt_sw_write(struct idt_ntb_dev *ndev,
 344                         const unsigned int reg, const u32 data)
 345{
 346        unsigned long irqflags;
 347
 348        /*
 349         * It's obvious bug to request a register exceeding the maximum possible
 350         * value as well as to have it unaligned.
 351         */
 352        if (WARN_ON(reg > IDT_REG_SW_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN)))
 353                return;
 354
 355        /* Lock GASA registers operations */
 356        spin_lock_irqsave(&ndev->gasa_lock, irqflags);
 357        /* Set the global register address */
 358        iowrite32((u32)reg, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASAADDR);
 359        /* Put the new value of the register */
 360        iowrite32(data, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASADATA);
 361        /* Unlock GASA registers operations */
 362        spin_unlock_irqrestore(&ndev->gasa_lock, irqflags);
 363}
 364
 365/*
 366 * idt_sw_read() - Global registers read method
 367 * @ndev:       IDT NTB hardware driver descriptor
 368 * @reg:        Register to write data to
 369 *
 370 * IDT PCIe-switch Global configuration registers are all Little endian.
 371 *
 372 * Return: register value
 373 */
 374static u32 idt_sw_read(struct idt_ntb_dev *ndev, const unsigned int reg)
 375{
 376        unsigned long irqflags;
 377        u32 data;
 378
 379        /*
 380         * It's obvious bug to request a register exceeding the maximum possible
 381         * value as well as to have it unaligned.
 382         */
 383        if (WARN_ON(reg > IDT_REG_SW_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN)))
 384                return ~0;
 385
 386        /* Lock GASA registers operations */
 387        spin_lock_irqsave(&ndev->gasa_lock, irqflags);
 388        /* Set the global register address */
 389        iowrite32((u32)reg, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASAADDR);
 390        /* Get the data of the register (read ops acts as MMIO barrier) */
 391        data = ioread32(ndev->cfgspc + (ptrdiff_t)IDT_NT_GASADATA);
 392        /* Unlock GASA registers operations */
 393        spin_unlock_irqrestore(&ndev->gasa_lock, irqflags);
 394
 395        return data;
 396}
 397
 398/*
 399 * idt_reg_set_bits() - set bits of a passed register
 400 * @ndev:       IDT NTB hardware driver descriptor
 401 * @reg:        Register to change bits of
 402 * @reg_lock:   Register access spin lock
 403 * @valid_mask: Mask of valid bits
 404 * @set_bits:   Bitmask to set
 405 *
 406 * Helper method to check whether a passed bitfield is valid and set
 407 * corresponding bits of a register.
 408 *
 409 * WARNING! Make sure the passed register isn't accessed over plane
 410 * idt_nt_write() method (read method is ok to be used concurrently).
 411 *
 412 * Return: zero on success, negative error on invalid bitmask.
 413 */
 414static inline int idt_reg_set_bits(struct idt_ntb_dev *ndev, unsigned int reg,
 415                                   spinlock_t *reg_lock,
 416                                   u64 valid_mask, u64 set_bits)
 417{
 418        unsigned long irqflags;
 419        u32 data;
 420
 421        if (set_bits & ~(u64)valid_mask)
 422                return -EINVAL;
 423
 424        /* Lock access to the register unless the change is written back */
 425        spin_lock_irqsave(reg_lock, irqflags);
 426        data = idt_nt_read(ndev, reg) | (u32)set_bits;
 427        idt_nt_write(ndev, reg, data);
 428        /* Unlock the register */
 429        spin_unlock_irqrestore(reg_lock, irqflags);
 430
 431        return 0;
 432}
 433
 434/*
 435 * idt_reg_clear_bits() - clear bits of a passed register
 436 * @ndev:       IDT NTB hardware driver descriptor
 437 * @reg:        Register to change bits of
 438 * @reg_lock:   Register access spin lock
 439 * @set_bits:   Bitmask to clear
 440 *
 441 * Helper method to check whether a passed bitfield is valid and clear
 442 * corresponding bits of a register.
 443 *
 444 * NOTE! Invalid bits are always considered cleared so it's not an error
 445 * to clear them over.
 446 *
 447 * WARNING! Make sure the passed register isn't accessed over plane
 448 * idt_nt_write() method (read method is ok to use concurrently).
 449 */
 450static inline void idt_reg_clear_bits(struct idt_ntb_dev *ndev,
 451                                     unsigned int reg, spinlock_t *reg_lock,
 452                                     u64 clear_bits)
 453{
 454        unsigned long irqflags;
 455        u32 data;
 456
 457        /* Lock access to the register unless the change is written back */
 458        spin_lock_irqsave(reg_lock, irqflags);
 459        data = idt_nt_read(ndev, reg) & ~(u32)clear_bits;
 460        idt_nt_write(ndev, reg, data);
 461        /* Unlock the register */
 462        spin_unlock_irqrestore(reg_lock, irqflags);
 463}
 464
 465/*===========================================================================
 466 *                           2. Ports operations
 467 *
 468 *    IDT PCIe-switches can have from 3 up to 8 ports with possible
 469 * NT-functions enabled. So all the possible ports need to be scanned looking
 470 * for NTB activated. NTB API will have enumerated only the ports with NTB.
 471 *===========================================================================
 472 */
 473
 474/*
 475 * idt_scan_ports() - scan IDT PCIe-switch ports collecting info in the tables
 476 * @ndev:       Pointer to the PCI device descriptor
 477 *
 478 * Return: zero on success, otherwise a negative error number.
 479 */
 480static int idt_scan_ports(struct idt_ntb_dev *ndev)
 481{
 482        unsigned char pidx, port, part;
 483        u32 data, portsts, partsts;
 484
 485        /* Retrieve the local port number */
 486        data = idt_nt_read(ndev, IDT_NT_PCIELCAP);
 487        ndev->port = GET_FIELD(PCIELCAP_PORTNUM, data);
 488
 489        /* Retrieve the local partition number */
 490        portsts = idt_sw_read(ndev, portdata_tbl[ndev->port].sts);
 491        ndev->part = GET_FIELD(SWPORTxSTS_SWPART, portsts);
 492
 493        /* Initialize port/partition -> index tables with invalid values */
 494        memset(ndev->port_idx_map, -EINVAL, sizeof(ndev->port_idx_map));
 495        memset(ndev->part_idx_map, -EINVAL, sizeof(ndev->part_idx_map));
 496
 497        /*
 498         * Walk over all the possible ports checking whether any of them has
 499         * NT-function activated
 500         */
 501        ndev->peer_cnt = 0;
 502        for (pidx = 0; pidx < ndev->swcfg->port_cnt; pidx++) {
 503                port = ndev->swcfg->ports[pidx];
 504                /* Skip local port */
 505                if (port == ndev->port)
 506                        continue;
 507
 508                /* Read the port status register to get it partition */
 509                portsts = idt_sw_read(ndev, portdata_tbl[port].sts);
 510                part = GET_FIELD(SWPORTxSTS_SWPART, portsts);
 511
 512                /* Retrieve the partition status */
 513                partsts = idt_sw_read(ndev, partdata_tbl[part].sts);
 514                /* Check if partition state is active and port has NTB */
 515                if (IS_FLD_SET(SWPARTxSTS_STATE, partsts, ACT) &&
 516                    (IS_FLD_SET(SWPORTxSTS_MODE, portsts, NT) ||
 517                     IS_FLD_SET(SWPORTxSTS_MODE, portsts, USNT) ||
 518                     IS_FLD_SET(SWPORTxSTS_MODE, portsts, USNTDMA) ||
 519                     IS_FLD_SET(SWPORTxSTS_MODE, portsts, NTDMA))) {
 520                        /* Save the port and partition numbers */
 521                        ndev->peers[ndev->peer_cnt].port = port;
 522                        ndev->peers[ndev->peer_cnt].part = part;
 523                        /* Fill in the port/partition -> index tables */
 524                        ndev->port_idx_map[port] = ndev->peer_cnt;
 525                        ndev->part_idx_map[part] = ndev->peer_cnt;
 526                        ndev->peer_cnt++;
 527                }
 528        }
 529
 530        dev_dbg(&ndev->ntb.pdev->dev, "Local port: %hhu, num of peers: %hhu\n",
 531                ndev->port, ndev->peer_cnt);
 532
 533        /* It's useless to have this driver loaded if there is no any peer */
 534        if (ndev->peer_cnt == 0) {
 535                dev_warn(&ndev->ntb.pdev->dev, "No active peer found\n");
 536                return -ENODEV;
 537        }
 538
 539        return 0;
 540}
 541
 542/*
 543 * idt_ntb_port_number() - get the local port number
 544 * @ntb:        NTB device context.
 545 *
 546 * Return: the local port number
 547 */
 548static int idt_ntb_port_number(struct ntb_dev *ntb)
 549{
 550        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 551
 552        return ndev->port;
 553}
 554
 555/*
 556 * idt_ntb_peer_port_count() - get the number of peer ports
 557 * @ntb:        NTB device context.
 558 *
 559 * Return the count of detected peer NT-functions.
 560 *
 561 * Return: number of peer ports
 562 */
 563static int idt_ntb_peer_port_count(struct ntb_dev *ntb)
 564{
 565        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 566
 567        return ndev->peer_cnt;
 568}
 569
 570/*
 571 * idt_ntb_peer_port_number() - get peer port by given index
 572 * @ntb:        NTB device context.
 573 * @pidx:       Peer port index.
 574 *
 575 * Return: peer port or negative error
 576 */
 577static int idt_ntb_peer_port_number(struct ntb_dev *ntb, int pidx)
 578{
 579        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 580
 581        if (pidx < 0 || ndev->peer_cnt <= pidx)
 582                return -EINVAL;
 583
 584        /* Return the detected NT-function port number */
 585        return ndev->peers[pidx].port;
 586}
 587
 588/*
 589 * idt_ntb_peer_port_idx() - get peer port index by given port number
 590 * @ntb:        NTB device context.
 591 * @port:       Peer port number.
 592 *
 593 * Internal port -> index table is pre-initialized with -EINVAL values,
 594 * so we just need to return it value
 595 *
 596 * Return: peer NT-function port index or negative error
 597 */
 598static int idt_ntb_peer_port_idx(struct ntb_dev *ntb, int port)
 599{
 600        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 601
 602        if (port < 0 || IDT_MAX_NR_PORTS <= port)
 603                return -EINVAL;
 604
 605        return ndev->port_idx_map[port];
 606}
 607
 608/*===========================================================================
 609 *                         3. Link status operations
 610 *    There is no any ready-to-use method to have peer ports notified if NTB
 611 * link is set up or got down. Instead global signal can be used instead.
 612 * In case if any one of ports changes local NTB link state, it sends
 613 * global signal and clears corresponding global state bit. Then all the ports
 614 * receive a notification of that, so to make client driver being aware of
 615 * possible NTB link change.
 616 *    Additionally each of active NT-functions is subscribed to PCIe-link
 617 * state changes of peer ports.
 618 *===========================================================================
 619 */
 620
 621static void idt_ntb_local_link_disable(struct idt_ntb_dev *ndev);
 622
 623/*
 624 * idt_init_link() - Initialize NTB link state notification subsystem
 625 * @ndev:       IDT NTB hardware driver descriptor
 626 *
 627 * Function performs the basic initialization of some global registers
 628 * needed to enable IRQ-based notifications of PCIe Link Up/Down and
 629 * Global Signal events.
 630 * NOTE Since it's not possible to determine when all the NTB peer drivers are
 631 * unloaded as well as have those registers accessed concurrently, we must
 632 * preinitialize them with the same value and leave it uncleared on local
 633 * driver unload.
 634 */
 635static void idt_init_link(struct idt_ntb_dev *ndev)
 636{
 637        u32 part_mask, port_mask, se_mask;
 638        unsigned char pidx;
 639
 640        /* Initialize spin locker of Mapping Table access registers */
 641        spin_lock_init(&ndev->mtbl_lock);
 642
 643        /* Walk over all detected peers collecting port and partition masks */
 644        port_mask = ~BIT(ndev->port);
 645        part_mask = ~BIT(ndev->part);
 646        for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {
 647                port_mask &= ~BIT(ndev->peers[pidx].port);
 648                part_mask &= ~BIT(ndev->peers[pidx].part);
 649        }
 650
 651        /* Clean the Link Up/Down and GLobal Signal status registers */
 652        idt_sw_write(ndev, IDT_SW_SELINKUPSTS, (u32)-1);
 653        idt_sw_write(ndev, IDT_SW_SELINKDNSTS, (u32)-1);
 654        idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)-1);
 655
 656        /* Unmask NT-activated partitions to receive Global Switch events */
 657        idt_sw_write(ndev, IDT_SW_SEPMSK, part_mask);
 658
 659        /* Enable PCIe Link Up events of NT-activated ports */
 660        idt_sw_write(ndev, IDT_SW_SELINKUPMSK, port_mask);
 661
 662        /* Enable PCIe Link Down events of NT-activated ports */
 663        idt_sw_write(ndev, IDT_SW_SELINKDNMSK, port_mask);
 664
 665        /* Unmask NT-activated partitions to receive Global Signal events */
 666        idt_sw_write(ndev, IDT_SW_SEGSIGMSK, part_mask);
 667
 668        /* Unmask Link Up/Down and Global Switch Events */
 669        se_mask = ~(IDT_SEMSK_LINKUP | IDT_SEMSK_LINKDN | IDT_SEMSK_GSIGNAL);
 670        idt_sw_write(ndev, IDT_SW_SEMSK, se_mask);
 671
 672        dev_dbg(&ndev->ntb.pdev->dev, "NTB link status events initialized");
 673}
 674
 675/*
 676 * idt_deinit_link() - deinitialize link subsystem
 677 * @ndev:       IDT NTB hardware driver descriptor
 678 *
 679 * Just disable the link back.
 680 */
 681static void idt_deinit_link(struct idt_ntb_dev *ndev)
 682{
 683        /* Disable the link */
 684        idt_ntb_local_link_disable(ndev);
 685
 686        dev_dbg(&ndev->ntb.pdev->dev, "NTB link status events deinitialized");
 687}
 688
 689/*
 690 * idt_se_isr() - switch events ISR
 691 * @ndev:       IDT NTB hardware driver descriptor
 692 * @ntint_sts:  NT-function interrupt status
 693 *
 694 * This driver doesn't support IDT PCIe-switch dynamic reconfigurations,
 695 * Failover capability, etc, so switch events are utilized to notify of
 696 * PCIe and NTB link events.
 697 * The method is called from PCIe ISR bottom-half routine.
 698 */
 699static void idt_se_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)
 700{
 701        u32 sests;
 702
 703        /* Read Switch Events status */
 704        sests = idt_sw_read(ndev, IDT_SW_SESTS);
 705
 706        /* Clean the Link Up/Down and Global Signal status registers */
 707        idt_sw_write(ndev, IDT_SW_SELINKUPSTS, (u32)-1);
 708        idt_sw_write(ndev, IDT_SW_SELINKDNSTS, (u32)-1);
 709        idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)-1);
 710
 711        /* Clean the corresponding interrupt bit */
 712        idt_nt_write(ndev, IDT_NT_NTINTSTS, IDT_NTINTSTS_SEVENT);
 713
 714        dev_dbg(&ndev->ntb.pdev->dev, "SE IRQ detected %#08x (SESTS %#08x)",
 715                          ntint_sts, sests);
 716
 717        /* Notify the client driver of possible link state change */
 718        ntb_link_event(&ndev->ntb);
 719}
 720
 721/*
 722 * idt_ntb_local_link_enable() - enable the local NTB link.
 723 * @ndev:       IDT NTB hardware driver descriptor
 724 *
 725 * In order to enable the NTB link we need:
 726 * - enable Completion TLPs translation
 727 * - initialize mapping table to enable the Request ID translation
 728 * - notify peers of NTB link state change
 729 */
 730static void idt_ntb_local_link_enable(struct idt_ntb_dev *ndev)
 731{
 732        u32 reqid, mtbldata = 0;
 733        unsigned long irqflags;
 734
 735        /* Enable the ID protection and Completion TLPs translation */
 736        idt_nt_write(ndev, IDT_NT_NTCTL, IDT_NTCTL_CPEN);
 737
 738        /* Retrieve the current Requester ID (Bus:Device:Function) */
 739        reqid = idt_nt_read(ndev, IDT_NT_REQIDCAP);
 740
 741        /*
 742         * Set the corresponding NT Mapping table entry of port partition index
 743         * with the data to perform the Request ID translation
 744         */
 745        mtbldata = SET_FIELD(NTMTBLDATA_REQID, 0, reqid) |
 746                   SET_FIELD(NTMTBLDATA_PART, 0, ndev->part) |
 747                   IDT_NTMTBLDATA_VALID;
 748        spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
 749        idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);
 750        idt_nt_write(ndev, IDT_NT_NTMTBLDATA, mtbldata);
 751        spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
 752
 753        /* Notify the peers by setting and clearing the global signal bit */
 754        idt_nt_write(ndev, IDT_NT_NTGSIGNAL, IDT_NTGSIGNAL_SET);
 755        idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)1 << ndev->part);
 756}
 757
 758/*
 759 * idt_ntb_local_link_disable() - disable the local NTB link.
 760 * @ndev:       IDT NTB hardware driver descriptor
 761 *
 762 * In order to enable the NTB link we need:
 763 * - disable Completion TLPs translation
 764 * - clear corresponding mapping table entry
 765 * - notify peers of NTB link state change
 766 */
 767static void idt_ntb_local_link_disable(struct idt_ntb_dev *ndev)
 768{
 769        unsigned long irqflags;
 770
 771        /* Disable Completion TLPs translation */
 772        idt_nt_write(ndev, IDT_NT_NTCTL, 0);
 773
 774        /* Clear the corresponding NT Mapping table entry */
 775        spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
 776        idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);
 777        idt_nt_write(ndev, IDT_NT_NTMTBLDATA, 0);
 778        spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
 779
 780        /* Notify the peers by setting and clearing the global signal bit */
 781        idt_nt_write(ndev, IDT_NT_NTGSIGNAL, IDT_NTGSIGNAL_SET);
 782        idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)1 << ndev->part);
 783}
 784
 785/*
 786 * idt_ntb_local_link_is_up() - test wethter local NTB link is up
 787 * @ndev:       IDT NTB hardware driver descriptor
 788 *
 789 * Local link is up under the following conditions:
 790 * - Bus mastering is enabled
 791 * - NTCTL has Completion TLPs translation enabled
 792 * - Mapping table permits Request TLPs translation
 793 * NOTE: We don't need to check PCIe link state since it's obviously
 794 * up while we are able to communicate with IDT PCIe-switch
 795 *
 796 * Return: true if link is up, otherwise false
 797 */
 798static bool idt_ntb_local_link_is_up(struct idt_ntb_dev *ndev)
 799{
 800        unsigned long irqflags;
 801        u32 data;
 802
 803        /* Read the local Bus Master Enable status */
 804        data = idt_nt_read(ndev, IDT_NT_PCICMDSTS);
 805        if (!(data & IDT_PCICMDSTS_BME))
 806                return false;
 807
 808        /* Read the local Completion TLPs translation enable status */
 809        data = idt_nt_read(ndev, IDT_NT_NTCTL);
 810        if (!(data & IDT_NTCTL_CPEN))
 811                return false;
 812
 813        /* Read Mapping table entry corresponding to the local partition */
 814        spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
 815        idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);
 816        data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);
 817        spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
 818
 819        return !!(data & IDT_NTMTBLDATA_VALID);
 820}
 821
 822/*
 823 * idt_ntb_peer_link_is_up() - test whether peer NTB link is up
 824 * @ndev:       IDT NTB hardware driver descriptor
 825 * @pidx:       Peer port index
 826 *
 827 * Peer link is up under the following conditions:
 828 * - PCIe link is up
 829 * - Bus mastering is enabled
 830 * - NTCTL has Completion TLPs translation enabled
 831 * - Mapping table permits Request TLPs translation
 832 *
 833 * Return: true if link is up, otherwise false
 834 */
 835static bool idt_ntb_peer_link_is_up(struct idt_ntb_dev *ndev, int pidx)
 836{
 837        unsigned long irqflags;
 838        unsigned char port;
 839        u32 data;
 840
 841        /* Retrieve the device port number */
 842        port = ndev->peers[pidx].port;
 843
 844        /* Check whether PCIe link is up */
 845        data = idt_sw_read(ndev, portdata_tbl[port].sts);
 846        if (!(data & IDT_SWPORTxSTS_LINKUP))
 847                return false;
 848
 849        /* Check whether bus mastering is enabled on the peer port */
 850        data = idt_sw_read(ndev, portdata_tbl[port].pcicmdsts);
 851        if (!(data & IDT_PCICMDSTS_BME))
 852                return false;
 853
 854        /* Check if Completion TLPs translation is enabled on the peer port */
 855        data = idt_sw_read(ndev, portdata_tbl[port].ntctl);
 856        if (!(data & IDT_NTCTL_CPEN))
 857                return false;
 858
 859        /* Read Mapping table entry corresponding to the peer partition */
 860        spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
 861        idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->peers[pidx].part);
 862        data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);
 863        spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
 864
 865        return !!(data & IDT_NTMTBLDATA_VALID);
 866}
 867
 868/*
 869 * idt_ntb_link_is_up() - get the current ntb link state (NTB API callback)
 870 * @ntb:        NTB device context.
 871 * @speed:      OUT - The link speed expressed as PCIe generation number.
 872 * @width:      OUT - The link width expressed as the number of PCIe lanes.
 873 *
 874 * Get the bitfield of NTB link states for all peer ports
 875 *
 876 * Return: bitfield of indexed ports link state: bit is set/cleared if the
 877 *         link is up/down respectively.
 878 */
 879static u64 idt_ntb_link_is_up(struct ntb_dev *ntb,
 880                              enum ntb_speed *speed, enum ntb_width *width)
 881{
 882        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 883        unsigned char pidx;
 884        u64 status;
 885        u32 data;
 886
 887        /* Retrieve the local link speed and width */
 888        if (speed != NULL || width != NULL) {
 889                data = idt_nt_read(ndev, IDT_NT_PCIELCTLSTS);
 890                if (speed != NULL)
 891                        *speed = GET_FIELD(PCIELCTLSTS_CLS, data);
 892                if (width != NULL)
 893                        *width = GET_FIELD(PCIELCTLSTS_NLW, data);
 894        }
 895
 896        /* If local NTB link isn't up then all the links are considered down */
 897        if (!idt_ntb_local_link_is_up(ndev))
 898                return 0;
 899
 900        /* Collect all the peer ports link states into the bitfield */
 901        status = 0;
 902        for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {
 903                if (idt_ntb_peer_link_is_up(ndev, pidx))
 904                        status |= ((u64)1 << pidx);
 905        }
 906
 907        return status;
 908}
 909
 910/*
 911 * idt_ntb_link_enable() - enable local port ntb link (NTB API callback)
 912 * @ntb:        NTB device context.
 913 * @max_speed:  The maximum link speed expressed as PCIe generation number.
 914 * @max_width:  The maximum link width expressed as the number of PCIe lanes.
 915 *
 916 * Enable just local NTB link. PCIe link parameters are ignored.
 917 *
 918 * Return: always zero.
 919 */
 920static int idt_ntb_link_enable(struct ntb_dev *ntb, enum ntb_speed speed,
 921                               enum ntb_width width)
 922{
 923        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 924
 925        /* Just enable the local NTB link */
 926        idt_ntb_local_link_enable(ndev);
 927
 928        dev_dbg(&ndev->ntb.pdev->dev, "Local NTB link enabled");
 929
 930        return 0;
 931}
 932
 933/*
 934 * idt_ntb_link_disable() - disable local port ntb link (NTB API callback)
 935 * @ntb:        NTB device context.
 936 *
 937 * Disable just local NTB link.
 938 *
 939 * Return: always zero.
 940 */
 941static int idt_ntb_link_disable(struct ntb_dev *ntb)
 942{
 943        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 944
 945        /* Just disable the local NTB link */
 946        idt_ntb_local_link_disable(ndev);
 947
 948        dev_dbg(&ndev->ntb.pdev->dev, "Local NTB link disabled");
 949
 950        return 0;
 951}
 952
 953/*=============================================================================
 954 *                         4. Memory Window operations
 955 *
 956 *    IDT PCIe-switches have two types of memory windows: MWs with direct
 957 * address translation and MWs with LUT based translation. The first type of
 958 * MWs is simple map of corresponding BAR address space to a memory space
 959 * of specified target port. So it implemets just ont-to-one mapping. Lookup
 960 * table in its turn can map one BAR address space to up to 24 different
 961 * memory spaces of different ports.
 962 *    NT-functions BARs can be turned on to implement either direct or lookup
 963 * table based address translations, so:
 964 * BAR0 - NT configuration registers space/direct address translation
 965 * BAR1 - direct address translation/upper address of BAR0x64
 966 * BAR2 - direct address translation/Lookup table with either 12 or 24 entries
 967 * BAR3 - direct address translation/upper address of BAR2x64
 968 * BAR4 - direct address translation/Lookup table with either 12 or 24 entries
 969 * BAR5 - direct address translation/upper address of BAR4x64
 970 *    Additionally BAR2 and BAR4 can't have 24-entries LUT enabled at the same
 971 * time. Since the BARs setup can be rather complicated this driver implements
 972 * a scanning algorithm to have all the possible memory windows configuration
 973 * covered.
 974 *
 975 * NOTE 1 BAR setup must be done before Linux kernel enumerated NT-function
 976 * of any port, so this driver would have memory windows configurations fixed.
 977 * In this way all initializations must be performed either by platform BIOS
 978 * or using EEPROM connected to IDT PCIe-switch master SMBus.
 979 *
 980 * NOTE 2 This driver expects BAR0 mapping NT-function configuration space.
 981 * Easy calculation can give us an upper boundary of 29 possible memory windows
 982 * per each NT-function if all the BARs are of 32bit type.
 983 *=============================================================================
 984 */
 985
 986/*
 987 * idt_get_mw_count() - get memory window count
 988 * @mw_type:    Memory window type
 989 *
 990 * Return: number of memory windows with respect to the BAR type
 991 */
 992static inline unsigned char idt_get_mw_count(enum idt_mw_type mw_type)
 993{
 994        switch (mw_type) {
 995        case IDT_MW_DIR:
 996                return 1;
 997        case IDT_MW_LUT12:
 998                return 12;
 999        case IDT_MW_LUT24:
1000                return 24;
1001        default:
1002                break;
1003        }
1004
1005        return 0;
1006}
1007
1008/*
1009 * idt_get_mw_name() - get memory window name
1010 * @mw_type:    Memory window type
1011 *
1012 * Return: pointer to a string with name
1013 */
1014static inline char *idt_get_mw_name(enum idt_mw_type mw_type)
1015{
1016        switch (mw_type) {
1017        case IDT_MW_DIR:
1018                return "DIR  ";
1019        case IDT_MW_LUT12:
1020                return "LUT12";
1021        case IDT_MW_LUT24:
1022                return "LUT24";
1023        default:
1024                break;
1025        }
1026
1027        return "unknown";
1028}
1029
1030/*
1031 * idt_scan_mws() - scan memory windows of the port
1032 * @ndev:       IDT NTB hardware driver descriptor
1033 * @port:       Port to get number of memory windows for
1034 * @mw_cnt:     Out - number of memory windows
1035 *
1036 * It walks over BAR setup registers of the specified port and determines
1037 * the memory windows parameters if any activated.
1038 *
1039 * Return: array of memory windows
1040 */
1041static struct idt_mw_cfg *idt_scan_mws(struct idt_ntb_dev *ndev, int port,
1042                                       unsigned char *mw_cnt)
1043{
1044        struct idt_mw_cfg mws[IDT_MAX_NR_MWS], *ret_mws;
1045        const struct idt_ntb_bar *bars;
1046        enum idt_mw_type mw_type;
1047        unsigned char widx, bidx, en_cnt;
1048        bool bar_64bit = false;
1049        int aprt_size;
1050        u32 data;
1051
1052        /* Retrieve the array of the BARs registers */
1053        bars = portdata_tbl[port].bars;
1054
1055        /* Scan all the BARs belonging to the port */
1056        *mw_cnt = 0;
1057        for (bidx = 0; bidx < IDT_BAR_CNT; bidx += 1 + bar_64bit) {
1058                /* Read BARSETUP register value */
1059                data = idt_sw_read(ndev, bars[bidx].setup);
1060
1061                /* Skip disabled BARs */
1062                if (!(data & IDT_BARSETUP_EN)) {
1063                        bar_64bit = false;
1064                        continue;
1065                }
1066
1067                /* Skip next BARSETUP if current one has 64bit addressing */
1068                bar_64bit = IS_FLD_SET(BARSETUP_TYPE, data, 64);
1069
1070                /* Skip configuration space mapping BARs */
1071                if (data & IDT_BARSETUP_MODE_CFG)
1072                        continue;
1073
1074                /* Retrieve MW type/entries count and aperture size */
1075                mw_type = GET_FIELD(BARSETUP_ATRAN, data);
1076                en_cnt = idt_get_mw_count(mw_type);
1077                aprt_size = (u64)1 << GET_FIELD(BARSETUP_SIZE, data);
1078
1079                /* Save configurations of all available memory windows */
1080                for (widx = 0; widx < en_cnt; widx++, (*mw_cnt)++) {
1081                        /*
1082                         * IDT can expose a limited number of MWs, so it's bug
1083                         * to have more than the driver expects
1084                         */
1085                        if (*mw_cnt >= IDT_MAX_NR_MWS)
1086                                return ERR_PTR(-EINVAL);
1087
1088                        /* Save basic MW info */
1089                        mws[*mw_cnt].type = mw_type;
1090                        mws[*mw_cnt].bar = bidx;
1091                        mws[*mw_cnt].idx = widx;
1092                        /* It's always DWORD aligned */
1093                        mws[*mw_cnt].addr_align = IDT_TRANS_ALIGN;
1094                        /* DIR and LUT approachs differently configure MWs */
1095                        if (mw_type == IDT_MW_DIR)
1096                                mws[*mw_cnt].size_max = aprt_size;
1097                        else if (mw_type == IDT_MW_LUT12)
1098                                mws[*mw_cnt].size_max = aprt_size / 16;
1099                        else
1100                                mws[*mw_cnt].size_max = aprt_size / 32;
1101                        mws[*mw_cnt].size_align = (mw_type == IDT_MW_DIR) ?
1102                                IDT_DIR_SIZE_ALIGN : mws[*mw_cnt].size_max;
1103                }
1104        }
1105
1106        /* Allocate memory for memory window descriptors */
1107        ret_mws = devm_kcalloc(&ndev->ntb.pdev->dev, *mw_cnt, sizeof(*ret_mws),
1108                               GFP_KERNEL);
1109        if (!ret_mws)
1110                return ERR_PTR(-ENOMEM);
1111
1112        /* Copy the info of detected memory windows */
1113        memcpy(ret_mws, mws, (*mw_cnt)*sizeof(*ret_mws));
1114
1115        return ret_mws;
1116}
1117
1118/*
1119 * idt_init_mws() - initialize memory windows subsystem
1120 * @ndev:       IDT NTB hardware driver descriptor
1121 *
1122 * Scan BAR setup registers of local and peer ports to determine the
1123 * outbound and inbound memory windows parameters
1124 *
1125 * Return: zero on success, otherwise a negative error number
1126 */
1127static int idt_init_mws(struct idt_ntb_dev *ndev)
1128{
1129        struct idt_ntb_peer *peer;
1130        unsigned char pidx;
1131
1132        /* Scan memory windows of the local port */
1133        ndev->mws = idt_scan_mws(ndev, ndev->port, &ndev->mw_cnt);
1134        if (IS_ERR(ndev->mws)) {
1135                dev_err(&ndev->ntb.pdev->dev,
1136                        "Failed to scan mws of local port %hhu", ndev->port);
1137                return PTR_ERR(ndev->mws);
1138        }
1139
1140        /* Scan memory windows of the peer ports */
1141        for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {
1142                peer = &ndev->peers[pidx];
1143                peer->mws = idt_scan_mws(ndev, peer->port, &peer->mw_cnt);
1144                if (IS_ERR(peer->mws)) {
1145                        dev_err(&ndev->ntb.pdev->dev,
1146                                "Failed to scan mws of port %hhu", peer->port);
1147                        return PTR_ERR(peer->mws);
1148                }
1149        }
1150
1151        /* Initialize spin locker of the LUT registers */
1152        spin_lock_init(&ndev->lut_lock);
1153
1154        dev_dbg(&ndev->ntb.pdev->dev, "Outbound and inbound MWs initialized");
1155
1156        return 0;
1157}
1158
1159/*
1160 * idt_ntb_mw_count() - number of inbound memory windows (NTB API callback)
1161 * @ntb:        NTB device context.
1162 * @pidx:       Port index of peer device.
1163 *
1164 * The value is returned for the specified peer, so generally speaking it can
1165 * be different for different port depending on the IDT PCIe-switch
1166 * initialization.
1167 *
1168 * Return: the number of memory windows.
1169 */
1170static int idt_ntb_mw_count(struct ntb_dev *ntb, int pidx)
1171{
1172        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1173
1174        if (pidx < 0 || ndev->peer_cnt <= pidx)
1175                return -EINVAL;
1176
1177        return ndev->peers[pidx].mw_cnt;
1178}
1179
1180/*
1181 * idt_ntb_mw_get_align() - inbound memory window parameters (NTB API callback)
1182 * @ntb:        NTB device context.
1183 * @pidx:       Port index of peer device.
1184 * @widx:       Memory window index.
1185 * @addr_align: OUT - the base alignment for translating the memory window
1186 * @size_align: OUT - the size alignment for translating the memory window
1187 * @size_max:   OUT - the maximum size of the memory window
1188 *
1189 * The peer memory window parameters have already been determined, so just
1190 * return the corresponding values, which mustn't change within session.
1191 *
1192 * Return: Zero on success, otherwise a negative error number.
1193 */
1194static int idt_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx,
1195                                resource_size_t *addr_align,
1196                                resource_size_t *size_align,
1197                                resource_size_t *size_max)
1198{
1199        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1200        struct idt_ntb_peer *peer;
1201
1202        if (pidx < 0 || ndev->peer_cnt <= pidx)
1203                return -EINVAL;
1204
1205        peer = &ndev->peers[pidx];
1206
1207        if (widx < 0 || peer->mw_cnt <= widx)
1208                return -EINVAL;
1209
1210        if (addr_align != NULL)
1211                *addr_align = peer->mws[widx].addr_align;
1212
1213        if (size_align != NULL)
1214                *size_align = peer->mws[widx].size_align;
1215
1216        if (size_max != NULL)
1217                *size_max = peer->mws[widx].size_max;
1218
1219        return 0;
1220}
1221
1222/*
1223 * idt_ntb_peer_mw_count() - number of outbound memory windows
1224 *                           (NTB API callback)
1225 * @ntb:        NTB device context.
1226 *
1227 * Outbound memory windows parameters have been determined based on the
1228 * BAR setup registers value, which are mostly constants within one session.
1229 *
1230 * Return: the number of memory windows.
1231 */
1232static int idt_ntb_peer_mw_count(struct ntb_dev *ntb)
1233{
1234        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1235
1236        return ndev->mw_cnt;
1237}
1238
1239/*
1240 * idt_ntb_peer_mw_get_addr() - get map address of an outbound memory window
1241 *                              (NTB API callback)
1242 * @ntb:        NTB device context.
1243 * @widx:       Memory window index (within ntb_peer_mw_count() return value).
1244 * @base:       OUT - the base address of mapping region.
1245 * @size:       OUT - the size of mapping region.
1246 *
1247 * Return just parameters of BAR resources mapping. Size reflects just the size
1248 * of the resource
1249 *
1250 * Return: Zero on success, otherwise a negative error number.
1251 */
1252static int idt_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int widx,
1253                                    phys_addr_t *base, resource_size_t *size)
1254{
1255        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1256
1257        if (widx < 0 || ndev->mw_cnt <= widx)
1258                return -EINVAL;
1259
1260        /* Mapping address is just properly shifted BAR resource start */
1261        if (base != NULL)
1262                *base = pci_resource_start(ntb->pdev, ndev->mws[widx].bar) +
1263                        ndev->mws[widx].idx * ndev->mws[widx].size_max;
1264
1265        /* Mapping size has already been calculated at MWs scanning */
1266        if (size != NULL)
1267                *size = ndev->mws[widx].size_max;
1268
1269        return 0;
1270}
1271
1272/*
1273 * idt_ntb_peer_mw_set_trans() - set a translation address of a memory window
1274 *                               (NTB API callback)
1275 * @ntb:        NTB device context.
1276 * @pidx:       Port index of peer device the translation address received from.
1277 * @widx:       Memory window index.
1278 * @addr:       The dma address of the shared memory to access.
1279 * @size:       The size of the shared memory to access.
1280 *
1281 * The Direct address translation and LUT base translation is initialized a
1282 * bit differenet. Although the parameters restriction are now determined by
1283 * the same code.
1284 *
1285 * Return: Zero on success, otherwise an error number.
1286 */
1287static int idt_ntb_peer_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
1288                                     u64 addr, resource_size_t size)
1289{
1290        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1291        struct idt_mw_cfg *mw_cfg;
1292        u32 data = 0, lutoff = 0;
1293
1294        if (pidx < 0 || ndev->peer_cnt <= pidx)
1295                return -EINVAL;
1296
1297        if (widx < 0 || ndev->mw_cnt <= widx)
1298                return -EINVAL;
1299
1300        /*
1301         * Retrieve the memory window config to make sure the passed arguments
1302         * fit it restrictions
1303         */
1304        mw_cfg = &ndev->mws[widx];
1305        if (!IS_ALIGNED(addr, mw_cfg->addr_align))
1306                return -EINVAL;
1307        if (!IS_ALIGNED(size, mw_cfg->size_align) || size > mw_cfg->size_max)
1308                return -EINVAL;
1309
1310        /* DIR and LUT based translations are initialized differently */
1311        if (mw_cfg->type == IDT_MW_DIR) {
1312                const struct idt_ntb_bar *bar = &ntdata_tbl.bars[mw_cfg->bar];
1313                u64 limit;
1314                /* Set destination partition of translation */
1315                data = idt_nt_read(ndev, bar->setup);
1316                data = SET_FIELD(BARSETUP_TPART, data, ndev->peers[pidx].part);
1317                idt_nt_write(ndev, bar->setup, data);
1318                /* Set translation base address */
1319                idt_nt_write(ndev, bar->ltbase, (u32)addr);
1320                idt_nt_write(ndev, bar->utbase, (u32)(addr >> 32));
1321                /* Set the custom BAR aperture limit */
1322                limit = pci_bus_address(ntb->pdev, mw_cfg->bar) + size;
1323                idt_nt_write(ndev, bar->limit, (u32)limit);
1324                if (IS_FLD_SET(BARSETUP_TYPE, data, 64))
1325                        idt_nt_write(ndev, (bar + 1)->limit, (limit >> 32));
1326        } else {
1327                unsigned long irqflags;
1328                /* Initialize corresponding LUT entry */
1329                lutoff = SET_FIELD(LUTOFFSET_INDEX, 0, mw_cfg->idx) |
1330                         SET_FIELD(LUTOFFSET_BAR, 0, mw_cfg->bar);
1331                data = SET_FIELD(LUTUDATA_PART, 0, ndev->peers[pidx].part) |
1332                        IDT_LUTUDATA_VALID;
1333                spin_lock_irqsave(&ndev->lut_lock, irqflags);
1334                idt_nt_write(ndev, IDT_NT_LUTOFFSET, lutoff);
1335                idt_nt_write(ndev, IDT_NT_LUTLDATA, (u32)addr);
1336                idt_nt_write(ndev, IDT_NT_LUTMDATA, (u32)(addr >> 32));
1337                idt_nt_write(ndev, IDT_NT_LUTUDATA, data);
1338                spin_unlock_irqrestore(&ndev->lut_lock, irqflags);
1339                /* Limit address isn't specified since size is fixed for LUT */
1340        }
1341
1342        return 0;
1343}
1344
1345/*
1346 * idt_ntb_peer_mw_clear_trans() - clear the outbound MW translation address
1347 *                                 (NTB API callback)
1348 * @ntb:        NTB device context.
1349 * @pidx:       Port index of peer device.
1350 * @widx:       Memory window index.
1351 *
1352 * It effectively disables the translation over the specified outbound MW.
1353 *
1354 * Return: Zero on success, otherwise an error number.
1355 */
1356static int idt_ntb_peer_mw_clear_trans(struct ntb_dev *ntb, int pidx,
1357                                        int widx)
1358{
1359        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1360        struct idt_mw_cfg *mw_cfg;
1361
1362        if (pidx < 0 || ndev->peer_cnt <= pidx)
1363                return -EINVAL;
1364
1365        if (widx < 0 || ndev->mw_cnt <= widx)
1366                return -EINVAL;
1367
1368        mw_cfg = &ndev->mws[widx];
1369
1370        /* DIR and LUT based translations are initialized differently */
1371        if (mw_cfg->type == IDT_MW_DIR) {
1372                const struct idt_ntb_bar *bar = &ntdata_tbl.bars[mw_cfg->bar];
1373                u32 data;
1374                /* Read BARSETUP to check BAR type */
1375                data = idt_nt_read(ndev, bar->setup);
1376                /* Disable translation by specifying zero BAR limit */
1377                idt_nt_write(ndev, bar->limit, 0);
1378                if (IS_FLD_SET(BARSETUP_TYPE, data, 64))
1379                        idt_nt_write(ndev, (bar + 1)->limit, 0);
1380        } else {
1381                unsigned long irqflags;
1382                u32 lutoff;
1383                /* Clear the corresponding LUT entry up */
1384                lutoff = SET_FIELD(LUTOFFSET_INDEX, 0, mw_cfg->idx) |
1385                         SET_FIELD(LUTOFFSET_BAR, 0, mw_cfg->bar);
1386                spin_lock_irqsave(&ndev->lut_lock, irqflags);
1387                idt_nt_write(ndev, IDT_NT_LUTOFFSET, lutoff);
1388                idt_nt_write(ndev, IDT_NT_LUTLDATA, 0);
1389                idt_nt_write(ndev, IDT_NT_LUTMDATA, 0);
1390                idt_nt_write(ndev, IDT_NT_LUTUDATA, 0);
1391                spin_unlock_irqrestore(&ndev->lut_lock, irqflags);
1392        }
1393
1394        return 0;
1395}
1396
1397/*=============================================================================
1398 *                          5. Doorbell operations
1399 *
1400 *    Doorbell functionality of IDT PCIe-switches is pretty unusual. First of
1401 * all there is global doorbell register which state can be changed by any
1402 * NT-function of the IDT device in accordance with global permissions. These
1403 * permissions configs are not supported by NTB API, so it must be done by
1404 * either BIOS or EEPROM settings. In the same way the state of the global
1405 * doorbell is reflected to the NT-functions local inbound doorbell registers.
1406 * It can lead to situations when client driver sets some peer doorbell bits
1407 * and get them bounced back to local inbound doorbell if permissions are
1408 * granted.
1409 *    Secondly there is just one IRQ vector for Doorbell, Message, Temperature
1410 * and Switch events, so if client driver left any of Doorbell bits set and
1411 * some other event occurred, the driver will be notified of Doorbell event
1412 * again.
1413 *=============================================================================
1414 */
1415
1416/*
1417 * idt_db_isr() - doorbell event ISR
1418 * @ndev:       IDT NTB hardware driver descriptor
1419 * @ntint_sts:  NT-function interrupt status
1420 *
1421 * Doorbell event happans when DBELL bit of NTINTSTS switches from 0 to 1.
1422 * It happens only when unmasked doorbell bits are set to ones on completely
1423 * zeroed doorbell register.
1424 * The method is called from PCIe ISR bottom-half routine.
1425 */
1426static void idt_db_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)
1427{
1428        /*
1429         * Doorbell IRQ status will be cleaned only when client
1430         * driver unsets all the doorbell bits.
1431         */
1432        dev_dbg(&ndev->ntb.pdev->dev, "DB IRQ detected %#08x", ntint_sts);
1433
1434        /* Notify the client driver of possible doorbell state change */
1435        ntb_db_event(&ndev->ntb, 0);
1436}
1437
1438/*
1439 * idt_ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb
1440 *                           (NTB API callback)
1441 * @ntb:        NTB device context.
1442 *
1443 * IDT PCIe-switches expose just one Doorbell register of DWORD size.
1444 *
1445 * Return: A mask of doorbell bits supported by the ntb.
1446 */
1447static u64 idt_ntb_db_valid_mask(struct ntb_dev *ntb)
1448{
1449        return IDT_DBELL_MASK;
1450}
1451
1452/*
1453 * idt_ntb_db_read() - read the local doorbell register (NTB API callback)
1454 * @ntb:        NTB device context.
1455 *
1456 * There is just on inbound doorbell register of each NT-function, so
1457 * this method return it value.
1458 *
1459 * Return: The bits currently set in the local doorbell register.
1460 */
1461static u64 idt_ntb_db_read(struct ntb_dev *ntb)
1462{
1463        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1464
1465        return idt_nt_read(ndev, IDT_NT_INDBELLSTS);
1466}
1467
1468/*
1469 * idt_ntb_db_clear() - clear bits in the local doorbell register
1470 *                      (NTB API callback)
1471 * @ntb:        NTB device context.
1472 * @db_bits:    Doorbell bits to clear.
1473 *
1474 * Clear bits of inbound doorbell register by writing ones to it.
1475 *
1476 * NOTE! Invalid bits are always considered cleared so it's not an error
1477 * to clear them over.
1478 *
1479 * Return: always zero as success.
1480 */
1481static int idt_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1482{
1483        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1484
1485        idt_nt_write(ndev, IDT_NT_INDBELLSTS, (u32)db_bits);
1486
1487        return 0;
1488}
1489
1490/*
1491 * idt_ntb_db_read_mask() - read the local doorbell mask (NTB API callback)
1492 * @ntb:        NTB device context.
1493 *
1494 * Each inbound doorbell bit can be masked from generating IRQ by setting
1495 * the corresponding bit in inbound doorbell mask. So this method returns
1496 * the value of the register.
1497 *
1498 * Return: The bits currently set in the local doorbell mask register.
1499 */
1500static u64 idt_ntb_db_read_mask(struct ntb_dev *ntb)
1501{
1502        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1503
1504        return idt_nt_read(ndev, IDT_NT_INDBELLMSK);
1505}
1506
1507/*
1508 * idt_ntb_db_set_mask() - set bits in the local doorbell mask
1509 *                         (NTB API callback)
1510 * @ntb:        NTB device context.
1511 * @db_bits:    Doorbell mask bits to set.
1512 *
1513 * The inbound doorbell register mask value must be read, then OR'ed with
1514 * passed field and only then set back.
1515 *
1516 * Return: zero on success, negative error if invalid argument passed.
1517 */
1518static int idt_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1519{
1520        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1521
1522        return idt_reg_set_bits(ndev, IDT_NT_INDBELLMSK, &ndev->db_mask_lock,
1523                                IDT_DBELL_MASK, db_bits);
1524}
1525
1526/*
1527 * idt_ntb_db_clear_mask() - clear bits in the local doorbell mask
1528 *                           (NTB API callback)
1529 * @ntb:        NTB device context.
1530 * @db_bits:    Doorbell bits to clear.
1531 *
1532 * The method just clears the set bits up in accordance with the passed
1533 * bitfield. IDT PCIe-switch shall generate an interrupt if there hasn't
1534 * been any unmasked bit set before current unmasking. Otherwise IRQ won't
1535 * be generated since there is only one IRQ vector for all doorbells.
1536 *
1537 * Return: always zero as success
1538 */
1539static int idt_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1540{
1541        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1542
1543        idt_reg_clear_bits(ndev, IDT_NT_INDBELLMSK, &ndev->db_mask_lock,
1544                           db_bits);
1545
1546        return 0;
1547}
1548
1549/*
1550 * idt_ntb_peer_db_set() - set bits in the peer doorbell register
1551 *                         (NTB API callback)
1552 * @ntb:        NTB device context.
1553 * @db_bits:    Doorbell bits to set.
1554 *
1555 * IDT PCIe-switches exposes local outbound doorbell register to change peer
1556 * inbound doorbell register state.
1557 *
1558 * Return: zero on success, negative error if invalid argument passed.
1559 */
1560static int idt_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1561{
1562        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1563
1564        if (db_bits & ~(u64)IDT_DBELL_MASK)
1565                return -EINVAL;
1566
1567        idt_nt_write(ndev, IDT_NT_OUTDBELLSET, (u32)db_bits);
1568        return 0;
1569}
1570
1571/*=============================================================================
1572 *                          6. Messaging operations
1573 *
1574 *    Each NT-function of IDT PCIe-switch has four inbound and four outbound
1575 * message registers. Each outbound message register can be connected to one or
1576 * even more than one peer inbound message registers by setting global
1577 * configurations. Since NTB API permits one-on-one message registers mapping
1578 * only, the driver acts in according with that restriction.
1579 *=============================================================================
1580 */
1581
1582/*
1583 * idt_init_msg() - initialize messaging interface
1584 * @ndev:       IDT NTB hardware driver descriptor
1585 *
1586 * Just initialize the message registers routing tables locker.
1587 */
1588static void idt_init_msg(struct idt_ntb_dev *ndev)
1589{
1590        unsigned char midx;
1591
1592        /* Init the messages routing table lockers */
1593        for (midx = 0; midx < IDT_MSG_CNT; midx++)
1594                spin_lock_init(&ndev->msg_locks[midx]);
1595
1596        dev_dbg(&ndev->ntb.pdev->dev, "NTB Messaging initialized");
1597}
1598
1599/*
1600 * idt_msg_isr() - message event ISR
1601 * @ndev:       IDT NTB hardware driver descriptor
1602 * @ntint_sts:  NT-function interrupt status
1603 *
1604 * Message event happens when MSG bit of NTINTSTS switches from 0 to 1.
1605 * It happens only when unmasked message status bits are set to ones on
1606 * completely zeroed message status register.
1607 * The method is called from PCIe ISR bottom-half routine.
1608 */
1609static void idt_msg_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)
1610{
1611        /*
1612         * Message IRQ status will be cleaned only when client
1613         * driver unsets all the message status bits.
1614         */
1615        dev_dbg(&ndev->ntb.pdev->dev, "Message IRQ detected %#08x", ntint_sts);
1616
1617        /* Notify the client driver of possible message status change */
1618        ntb_msg_event(&ndev->ntb);
1619}
1620
1621/*
1622 * idt_ntb_msg_count() - get the number of message registers (NTB API callback)
1623 * @ntb:        NTB device context.
1624 *
1625 * IDT PCIe-switches support four message registers.
1626 *
1627 * Return: the number of message registers.
1628 */
1629static int idt_ntb_msg_count(struct ntb_dev *ntb)
1630{
1631        return IDT_MSG_CNT;
1632}
1633
1634/*
1635 * idt_ntb_msg_inbits() - get a bitfield of inbound message registers status
1636 *                        (NTB API callback)
1637 * @ntb:        NTB device context.
1638 *
1639 * NT message status register is shared between inbound and outbound message
1640 * registers status
1641 *
1642 * Return: bitfield of inbound message registers.
1643 */
1644static u64 idt_ntb_msg_inbits(struct ntb_dev *ntb)
1645{
1646        return (u64)IDT_INMSG_MASK;
1647}
1648
1649/*
1650 * idt_ntb_msg_outbits() - get a bitfield of outbound message registers status
1651 *                        (NTB API callback)
1652 * @ntb:        NTB device context.
1653 *
1654 * NT message status register is shared between inbound and outbound message
1655 * registers status
1656 *
1657 * Return: bitfield of outbound message registers.
1658 */
1659static u64 idt_ntb_msg_outbits(struct ntb_dev *ntb)
1660{
1661        return (u64)IDT_OUTMSG_MASK;
1662}
1663
1664/*
1665 * idt_ntb_msg_read_sts() - read the message registers status (NTB API callback)
1666 * @ntb:        NTB device context.
1667 *
1668 * IDT PCIe-switches expose message status registers to notify drivers of
1669 * incoming data and failures in case if peer message register isn't freed.
1670 *
1671 * Return: status bits of message registers
1672 */
1673static u64 idt_ntb_msg_read_sts(struct ntb_dev *ntb)
1674{
1675        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1676
1677        return idt_nt_read(ndev, IDT_NT_MSGSTS);
1678}
1679
1680/*
1681 * idt_ntb_msg_clear_sts() - clear status bits of message registers
1682 *                           (NTB API callback)
1683 * @ntb:        NTB device context.
1684 * @sts_bits:   Status bits to clear.
1685 *
1686 * Clear bits in the status register by writing ones.
1687 *
1688 * NOTE! Invalid bits are always considered cleared so it's not an error
1689 * to clear them over.
1690 *
1691 * Return: always zero as success.
1692 */
1693static int idt_ntb_msg_clear_sts(struct ntb_dev *ntb, u64 sts_bits)
1694{
1695        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1696
1697        idt_nt_write(ndev, IDT_NT_MSGSTS, sts_bits);
1698
1699        return 0;
1700}
1701
1702/*
1703 * idt_ntb_msg_set_mask() - set mask of message register status bits
1704 *                          (NTB API callback)
1705 * @ntb:        NTB device context.
1706 * @mask_bits:  Mask bits.
1707 *
1708 * Mask the message status bits from raising an IRQ.
1709 *
1710 * Return: zero on success, negative error if invalid argument passed.
1711 */
1712static int idt_ntb_msg_set_mask(struct ntb_dev *ntb, u64 mask_bits)
1713{
1714        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1715
1716        return idt_reg_set_bits(ndev, IDT_NT_MSGSTSMSK, &ndev->msg_mask_lock,
1717                                IDT_MSG_MASK, mask_bits);
1718}
1719
1720/*
1721 * idt_ntb_msg_clear_mask() - clear message registers mask
1722 *                            (NTB API callback)
1723 * @ntb:        NTB device context.
1724 * @mask_bits:  Mask bits.
1725 *
1726 * Clear mask of message status bits IRQs.
1727 *
1728 * Return: always zero as success.
1729 */
1730static int idt_ntb_msg_clear_mask(struct ntb_dev *ntb, u64 mask_bits)
1731{
1732        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1733
1734        idt_reg_clear_bits(ndev, IDT_NT_MSGSTSMSK, &ndev->msg_mask_lock,
1735                           mask_bits);
1736
1737        return 0;
1738}
1739
1740/*
1741 * idt_ntb_msg_read() - read message register with specified index
1742 *                      (NTB API callback)
1743 * @ntb:        NTB device context.
1744 * @pidx:       OUT - Port index of peer device a message retrieved from
1745 * @midx:       Message register index
1746 *
1747 * Read data from the specified message register and source register.
1748 *
1749 * Return: inbound message register value.
1750 */
1751static u32 idt_ntb_msg_read(struct ntb_dev *ntb, int *pidx, int midx)
1752{
1753        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1754
1755        if (midx < 0 || IDT_MSG_CNT <= midx)
1756                return ~(u32)0;
1757
1758        /* Retrieve source port index of the message */
1759        if (pidx != NULL) {
1760                u32 srcpart;
1761
1762                srcpart = idt_nt_read(ndev, ntdata_tbl.msgs[midx].src);
1763                *pidx = ndev->part_idx_map[srcpart];
1764
1765                /* Sanity check partition index (for initial case) */
1766                if (*pidx == -EINVAL)
1767                        *pidx = 0;
1768        }
1769
1770        /* Retrieve data of the corresponding message register */
1771        return idt_nt_read(ndev, ntdata_tbl.msgs[midx].in);
1772}
1773
1774/*
1775 * idt_ntb_peer_msg_write() - write data to the specified message register
1776 *                            (NTB API callback)
1777 * @ntb:        NTB device context.
1778 * @pidx:       Port index of peer device a message being sent to
1779 * @midx:       Message register index
1780 * @msg:        Data to send
1781 *
1782 * Just try to send data to a peer. Message status register should be
1783 * checked by client driver.
1784 *
1785 * Return: zero on success, negative error if invalid argument passed.
1786 */
1787static int idt_ntb_peer_msg_write(struct ntb_dev *ntb, int pidx, int midx,
1788                                  u32 msg)
1789{
1790        struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1791        unsigned long irqflags;
1792        u32 swpmsgctl = 0;
1793
1794        if (midx < 0 || IDT_MSG_CNT <= midx)
1795                return -EINVAL;
1796
1797        if (pidx < 0 || ndev->peer_cnt <= pidx)
1798                return -EINVAL;
1799
1800        /* Collect the routing information */
1801        swpmsgctl = SET_FIELD(SWPxMSGCTL_REG, 0, midx) |
1802                    SET_FIELD(SWPxMSGCTL_PART, 0, ndev->peers[pidx].part);
1803
1804        /* Lock the messages routing table of the specified register */
1805        spin_lock_irqsave(&ndev->msg_locks[midx], irqflags);
1806        /* Set the route and send the data */
1807        idt_sw_write(ndev, partdata_tbl[ndev->part].msgctl[midx], swpmsgctl);
1808        idt_nt_write(ndev, ntdata_tbl.msgs[midx].out, msg);
1809        /* Unlock the messages routing table */
1810        spin_unlock_irqrestore(&ndev->msg_locks[midx], irqflags);
1811
1812        /* Client driver shall check the status register */
1813        return 0;
1814}
1815
1816/*=============================================================================
1817 *                      7. Temperature sensor operations
1818 *
1819 *    IDT PCIe-switch has an embedded temperature sensor, which can be used to
1820 * check current chip core temperature. Since a workload environment can be
1821 * different on different platforms, an offset and ADC/filter settings can be
1822 * specified. Although the offset configuration is only exposed to the sysfs
1823 * hwmon interface at the moment. The rest of the settings can be adjusted
1824 * for instance by the BIOS/EEPROM firmware.
1825 *=============================================================================
1826 */
1827
1828/*
1829 * idt_get_deg() - convert millidegree Celsius value to just degree
1830 * @mdegC:      IN - millidegree Celsius value
1831 *
1832 * Return: Degree corresponding to the passed millidegree value
1833 */
1834static inline s8 idt_get_deg(long mdegC)
1835{
1836        return mdegC / 1000;
1837}
1838
1839/*
1840 * idt_get_frac() - retrieve 0/0.5 fraction of the millidegree Celsius value
1841 * @mdegC:      IN - millidegree Celsius value
1842 *
1843 * Return: 0/0.5 degree fraction of the passed millidegree value
1844 */
1845static inline u8 idt_get_deg_frac(long mdegC)
1846{
1847        return (mdegC % 1000) >= 500 ? 5 : 0;
1848}
1849
1850/*
1851 * idt_get_temp_fmt() - convert millidegree Celsius value to 0:7:1 format
1852 * @mdegC:      IN - millidegree Celsius value
1853 *
1854 * Return: 0:7:1 format acceptable by the IDT temperature sensor
1855 */
1856static inline u8 idt_temp_get_fmt(long mdegC)
1857{
1858        return (idt_get_deg(mdegC) << 1) | (idt_get_deg_frac(mdegC) ? 1 : 0);
1859}
1860
1861/*
1862 * idt_get_temp_sval() - convert temp sample to signed millidegree Celsius
1863 * @data:       IN - shifted to LSB 8-bits temperature sample
1864 *
1865 * Return: signed millidegree Celsius
1866 */
1867static inline long idt_get_temp_sval(u32 data)
1868{
1869        return ((s8)data / 2) * 1000 + (data & 0x1 ? 500 : 0);
1870}
1871
1872/*
1873 * idt_get_temp_sval() - convert temp sample to unsigned millidegree Celsius
1874 * @data:       IN - shifted to LSB 8-bits temperature sample
1875 *
1876 * Return: unsigned millidegree Celsius
1877 */
1878static inline long idt_get_temp_uval(u32 data)
1879{
1880        return (data / 2) * 1000 + (data & 0x1 ? 500 : 0);
1881}
1882
1883/*
1884 * idt_read_temp() - read temperature from chip sensor
1885 * @ntb:        NTB device context.
1886 * @type:       IN - type of the temperature value to read
1887 * @val:        OUT - integer value of temperature in millidegree Celsius
1888 */
1889static void idt_read_temp(struct idt_ntb_dev *ndev,
1890                          const enum idt_temp_val type, long *val)
1891{
1892        u32 data;
1893
1894        /* Alter the temperature field in accordance with the passed type */
1895        switch (type) {
1896        case IDT_TEMP_CUR:
1897                data = GET_FIELD(TMPSTS_TEMP,
1898                                 idt_sw_read(ndev, IDT_SW_TMPSTS));
1899                break;
1900        case IDT_TEMP_LOW:
1901                data = GET_FIELD(TMPSTS_LTEMP,
1902                                 idt_sw_read(ndev, IDT_SW_TMPSTS));
1903                break;
1904        case IDT_TEMP_HIGH:
1905                data = GET_FIELD(TMPSTS_HTEMP,
1906                                 idt_sw_read(ndev, IDT_SW_TMPSTS));
1907                break;
1908        case IDT_TEMP_OFFSET:
1909                /* This is the only field with signed 0:7:1 format */
1910                data = GET_FIELD(TMPADJ_OFFSET,
1911                                 idt_sw_read(ndev, IDT_SW_TMPADJ));
1912                *val = idt_get_temp_sval(data);
1913                return;
1914        default:
1915                data = GET_FIELD(TMPSTS_TEMP,
1916                                 idt_sw_read(ndev, IDT_SW_TMPSTS));
1917                break;
1918        }
1919
1920        /* The rest of the fields accept unsigned 0:7:1 format */
1921        *val = idt_get_temp_uval(data);
1922}
1923
1924/*
1925 * idt_write_temp() - write temperature to the chip sensor register
1926 * @ntb:        NTB device context.
1927 * @type:       IN - type of the temperature value to change
1928 * @val:        IN - integer value of temperature in millidegree Celsius
1929 */
1930static void idt_write_temp(struct idt_ntb_dev *ndev,
1931                           const enum idt_temp_val type, const long val)
1932{
1933        unsigned int reg;
1934        u32 data;
1935        u8 fmt;
1936
1937        /* Retrieve the properly formatted temperature value */
1938        fmt = idt_temp_get_fmt(val);
1939
1940        mutex_lock(&ndev->hwmon_mtx);
1941        switch (type) {
1942        case IDT_TEMP_LOW:
1943                reg = IDT_SW_TMPALARM;
1944                data = SET_FIELD(TMPALARM_LTEMP, idt_sw_read(ndev, reg), fmt) &
1945                        ~IDT_TMPALARM_IRQ_MASK;
1946                break;
1947        case IDT_TEMP_HIGH:
1948                reg = IDT_SW_TMPALARM;
1949                data = SET_FIELD(TMPALARM_HTEMP, idt_sw_read(ndev, reg), fmt) &
1950                        ~IDT_TMPALARM_IRQ_MASK;
1951                break;
1952        case IDT_TEMP_OFFSET:
1953                reg = IDT_SW_TMPADJ;
1954                data = SET_FIELD(TMPADJ_OFFSET, idt_sw_read(ndev, reg), fmt);
1955                break;
1956        default:
1957                goto inval_spin_unlock;
1958        }
1959
1960        idt_sw_write(ndev, reg, data);
1961
1962inval_spin_unlock:
1963        mutex_unlock(&ndev->hwmon_mtx);
1964}
1965
1966/*
1967 * idt_sysfs_show_temp() - printout corresponding temperature value
1968 * @dev:        Pointer to the NTB device structure
1969 * @da:         Sensor device attribute structure
1970 * @buf:        Buffer to print temperature out
1971 *
1972 * Return: Number of written symbols or negative error
1973 */
1974static ssize_t idt_sysfs_show_temp(struct device *dev,
1975                                   struct device_attribute *da, char *buf)
1976{
1977        struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
1978        struct idt_ntb_dev *ndev = dev_get_drvdata(dev);
1979        enum idt_temp_val type = attr->index;
1980        long mdeg;
1981
1982        idt_read_temp(ndev, type, &mdeg);
1983        return sprintf(buf, "%ld\n", mdeg);
1984}
1985
1986/*
1987 * idt_sysfs_set_temp() - set corresponding temperature value
1988 * @dev:        Pointer to the NTB device structure
1989 * @da:         Sensor device attribute structure
1990 * @buf:        Buffer to print temperature out
1991 * @count:      Size of the passed buffer
1992 *
1993 * Return: Number of written symbols or negative error
1994 */
1995static ssize_t idt_sysfs_set_temp(struct device *dev,
1996                                  struct device_attribute *da, const char *buf,
1997                                  size_t count)
1998{
1999        struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
2000        struct idt_ntb_dev *ndev = dev_get_drvdata(dev);
2001        enum idt_temp_val type = attr->index;
2002        long mdeg;
2003        int ret;
2004
2005        ret = kstrtol(buf, 10, &mdeg);
2006        if (ret)
2007                return ret;
2008
2009        /* Clamp the passed value in accordance with the type */
2010        if (type == IDT_TEMP_OFFSET)
2011                mdeg = clamp_val(mdeg, IDT_TEMP_MIN_OFFSET,
2012                                 IDT_TEMP_MAX_OFFSET);
2013        else
2014                mdeg = clamp_val(mdeg, IDT_TEMP_MIN_MDEG, IDT_TEMP_MAX_MDEG);
2015
2016        idt_write_temp(ndev, type, mdeg);
2017
2018        return count;
2019}
2020
2021/*
2022 * idt_sysfs_reset_hist() - reset temperature history
2023 * @dev:        Pointer to the NTB device structure
2024 * @da:         Sensor device attribute structure
2025 * @buf:        Buffer to print temperature out
2026 * @count:      Size of the passed buffer
2027 *
2028 * Return: Number of written symbols or negative error
2029 */
2030static ssize_t idt_sysfs_reset_hist(struct device *dev,
2031                                    struct device_attribute *da,
2032                                    const char *buf, size_t count)
2033{
2034        struct idt_ntb_dev *ndev = dev_get_drvdata(dev);
2035
2036        /* Just set the maximal value to the lowest temperature field and
2037         * minimal value to the highest temperature field
2038         */
2039        idt_write_temp(ndev, IDT_TEMP_LOW, IDT_TEMP_MAX_MDEG);
2040        idt_write_temp(ndev, IDT_TEMP_HIGH, IDT_TEMP_MIN_MDEG);
2041
2042        return count;
2043}
2044
2045/*
2046 * Hwmon IDT sysfs attributes
2047 */
2048static SENSOR_DEVICE_ATTR(temp1_input, 0444, idt_sysfs_show_temp, NULL,
2049                          IDT_TEMP_CUR);
2050static SENSOR_DEVICE_ATTR(temp1_lowest, 0444, idt_sysfs_show_temp, NULL,
2051                          IDT_TEMP_LOW);
2052static SENSOR_DEVICE_ATTR(temp1_highest, 0444, idt_sysfs_show_temp, NULL,
2053                          IDT_TEMP_HIGH);
2054static SENSOR_DEVICE_ATTR(temp1_offset, 0644, idt_sysfs_show_temp,
2055                          idt_sysfs_set_temp, IDT_TEMP_OFFSET);
2056static DEVICE_ATTR(temp1_reset_history, 0200, NULL, idt_sysfs_reset_hist);
2057
2058/*
2059 * Hwmon IDT sysfs attributes group
2060 */
2061static struct attribute *idt_temp_attrs[] = {
2062        &sensor_dev_attr_temp1_input.dev_attr.attr,
2063        &sensor_dev_attr_temp1_lowest.dev_attr.attr,
2064        &sensor_dev_attr_temp1_highest.dev_attr.attr,
2065        &sensor_dev_attr_temp1_offset.dev_attr.attr,
2066        &dev_attr_temp1_reset_history.attr,
2067        NULL
2068};
2069ATTRIBUTE_GROUPS(idt_temp);
2070
2071/*
2072 * idt_init_temp() - initialize temperature sensor interface
2073 * @ndev:       IDT NTB hardware driver descriptor
2074 *
2075 * Simple sensor initializarion method is responsible for device switching
2076 * on and resource management based hwmon interface registration. Note, that
2077 * since the device is shared we won't disable it on remove, but leave it
2078 * working until the system is powered off.
2079 */
2080static void idt_init_temp(struct idt_ntb_dev *ndev)
2081{
2082        struct device *hwmon;
2083
2084        /* Enable sensor if it hasn't been already */
2085        idt_sw_write(ndev, IDT_SW_TMPCTL, 0x0);
2086
2087        /* Initialize hwmon interface fields */
2088        mutex_init(&ndev->hwmon_mtx);
2089
2090        hwmon = devm_hwmon_device_register_with_groups(&ndev->ntb.pdev->dev,
2091                ndev->swcfg->name, ndev, idt_temp_groups);
2092        if (IS_ERR(hwmon)) {
2093                dev_err(&ndev->ntb.pdev->dev, "Couldn't create hwmon device");
2094                return;
2095        }
2096
2097        dev_dbg(&ndev->ntb.pdev->dev, "Temperature HWmon interface registered");
2098}
2099
2100/*=============================================================================
2101 *                           8. ISRs related operations
2102 *
2103 *    IDT PCIe-switch has strangely developed IRQ system. There is just one
2104 * interrupt vector for doorbell and message registers. So the hardware driver
2105 * can't determine actual source of IRQ if, for example, message event happened
2106 * while any of unmasked doorbell is still set. The similar situation may be if
2107 * switch or temperature sensor events pop up. The difference is that SEVENT
2108 * and TMPSENSOR bits of NT interrupt status register can be cleaned by
2109 * IRQ handler so a next interrupt request won't have false handling of
2110 * corresponding events.
2111 *    The hardware driver has only bottom-half handler of the IRQ, since if any
2112 * of events happened the device won't raise it again before the last one is
2113 * handled by clearing of corresponding NTINTSTS bit.
2114 *=============================================================================
2115 */
2116
2117static irqreturn_t idt_thread_isr(int irq, void *devid);
2118
2119/*
2120 * idt_init_isr() - initialize PCIe interrupt handler
2121 * @ndev:       IDT NTB hardware driver descriptor
2122 *
2123 * Return: zero on success, otherwise a negative error number.
2124 */
2125static int idt_init_isr(struct idt_ntb_dev *ndev)
2126{
2127        struct pci_dev *pdev = ndev->ntb.pdev;
2128        u32 ntint_mask;
2129        int ret;
2130
2131        /* Allocate just one interrupt vector for the ISR */
2132        ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_LEGACY);
2133        if (ret != 1) {
2134                dev_err(&pdev->dev, "Failed to allocate IRQ vector");
2135                return ret;
2136        }
2137
2138        /* Retrieve the IRQ vector */
2139        ret = pci_irq_vector(pdev, 0);
2140        if (ret < 0) {
2141                dev_err(&pdev->dev, "Failed to get IRQ vector");
2142                goto err_free_vectors;
2143        }
2144
2145        /* Set the IRQ handler */
2146        ret = devm_request_threaded_irq(&pdev->dev, ret, NULL, idt_thread_isr,
2147                                        IRQF_ONESHOT, NTB_IRQNAME, ndev);
2148        if (ret != 0) {
2149                dev_err(&pdev->dev, "Failed to set MSI IRQ handler, %d", ret);
2150                goto err_free_vectors;
2151        }
2152
2153        /* Unmask Message/Doorbell/SE interrupts */
2154        ntint_mask = idt_nt_read(ndev, IDT_NT_NTINTMSK) & ~IDT_NTINTMSK_ALL;
2155        idt_nt_write(ndev, IDT_NT_NTINTMSK, ntint_mask);
2156
2157        /* From now on the interrupts are enabled */
2158        dev_dbg(&pdev->dev, "NTB interrupts initialized");
2159
2160        return 0;
2161
2162err_free_vectors:
2163        pci_free_irq_vectors(pdev);
2164
2165        return ret;
2166}
2167
2168/*
2169 * idt_deinit_ist() - deinitialize PCIe interrupt handler
2170 * @ndev:       IDT NTB hardware driver descriptor
2171 *
2172 * Disable corresponding interrupts and free allocated IRQ vectors.
2173 */
2174static void idt_deinit_isr(struct idt_ntb_dev *ndev)
2175{
2176        struct pci_dev *pdev = ndev->ntb.pdev;
2177        u32 ntint_mask;
2178
2179        /* Mask interrupts back */
2180        ntint_mask = idt_nt_read(ndev, IDT_NT_NTINTMSK) | IDT_NTINTMSK_ALL;
2181        idt_nt_write(ndev, IDT_NT_NTINTMSK, ntint_mask);
2182
2183        /* Manually free IRQ otherwise PCI free irq vectors will fail */
2184        devm_free_irq(&pdev->dev, pci_irq_vector(pdev, 0), ndev);
2185
2186        /* Free allocated IRQ vectors */
2187        pci_free_irq_vectors(pdev);
2188
2189        dev_dbg(&pdev->dev, "NTB interrupts deinitialized");
2190}
2191
2192/*
2193 * idt_thread_isr() - NT function interrupts handler
2194 * @irq:        IRQ number
2195 * @devid:      Custom buffer
2196 *
2197 * It reads current NT interrupts state register and handles all the event
2198 * it declares.
2199 * The method is bottom-half routine of actual default PCIe IRQ handler.
2200 */
2201static irqreturn_t idt_thread_isr(int irq, void *devid)
2202{
2203        struct idt_ntb_dev *ndev = devid;
2204        bool handled = false;
2205        u32 ntint_sts;
2206
2207        /* Read the NT interrupts status register */
2208        ntint_sts = idt_nt_read(ndev, IDT_NT_NTINTSTS);
2209
2210        /* Handle messaging interrupts */
2211        if (ntint_sts & IDT_NTINTSTS_MSG) {
2212                idt_msg_isr(ndev, ntint_sts);
2213                handled = true;
2214        }
2215
2216        /* Handle doorbell interrupts */
2217        if (ntint_sts & IDT_NTINTSTS_DBELL) {
2218                idt_db_isr(ndev, ntint_sts);
2219                handled = true;
2220        }
2221
2222        /* Handle switch event interrupts */
2223        if (ntint_sts & IDT_NTINTSTS_SEVENT) {
2224                idt_se_isr(ndev, ntint_sts);
2225                handled = true;
2226        }
2227
2228        dev_dbg(&ndev->ntb.pdev->dev, "IDT IRQs 0x%08x handled", ntint_sts);
2229
2230        return handled ? IRQ_HANDLED : IRQ_NONE;
2231}
2232
2233/*===========================================================================
2234 *                     9. NTB hardware driver initialization
2235 *===========================================================================
2236 */
2237
2238/*
2239 * NTB API operations
2240 */
2241static const struct ntb_dev_ops idt_ntb_ops = {
2242        .port_number            = idt_ntb_port_number,
2243        .peer_port_count        = idt_ntb_peer_port_count,
2244        .peer_port_number       = idt_ntb_peer_port_number,
2245        .peer_port_idx          = idt_ntb_peer_port_idx,
2246        .link_is_up             = idt_ntb_link_is_up,
2247        .link_enable            = idt_ntb_link_enable,
2248        .link_disable           = idt_ntb_link_disable,
2249        .mw_count               = idt_ntb_mw_count,
2250        .mw_get_align           = idt_ntb_mw_get_align,
2251        .peer_mw_count          = idt_ntb_peer_mw_count,
2252        .peer_mw_get_addr       = idt_ntb_peer_mw_get_addr,
2253        .peer_mw_set_trans      = idt_ntb_peer_mw_set_trans,
2254        .peer_mw_clear_trans    = idt_ntb_peer_mw_clear_trans,
2255        .db_valid_mask          = idt_ntb_db_valid_mask,
2256        .db_read                = idt_ntb_db_read,
2257        .db_clear               = idt_ntb_db_clear,
2258        .db_read_mask           = idt_ntb_db_read_mask,
2259        .db_set_mask            = idt_ntb_db_set_mask,
2260        .db_clear_mask          = idt_ntb_db_clear_mask,
2261        .peer_db_set            = idt_ntb_peer_db_set,
2262        .msg_count              = idt_ntb_msg_count,
2263        .msg_inbits             = idt_ntb_msg_inbits,
2264        .msg_outbits            = idt_ntb_msg_outbits,
2265        .msg_read_sts           = idt_ntb_msg_read_sts,
2266        .msg_clear_sts          = idt_ntb_msg_clear_sts,
2267        .msg_set_mask           = idt_ntb_msg_set_mask,
2268        .msg_clear_mask         = idt_ntb_msg_clear_mask,
2269        .msg_read               = idt_ntb_msg_read,
2270        .peer_msg_write         = idt_ntb_peer_msg_write
2271};
2272
2273/*
2274 * idt_register_device() - register IDT NTB device
2275 * @ndev:       IDT NTB hardware driver descriptor
2276 *
2277 * Return: zero on success, otherwise a negative error number.
2278 */
2279static int idt_register_device(struct idt_ntb_dev *ndev)
2280{
2281        int ret;
2282
2283        /* Initialize the rest of NTB device structure and register it */
2284        ndev->ntb.ops = &idt_ntb_ops;
2285        ndev->ntb.topo = NTB_TOPO_SWITCH;
2286
2287        ret = ntb_register_device(&ndev->ntb);
2288        if (ret != 0) {
2289                dev_err(&ndev->ntb.pdev->dev, "Failed to register NTB device");
2290                return ret;
2291        }
2292
2293        dev_dbg(&ndev->ntb.pdev->dev, "NTB device successfully registered");
2294
2295        return 0;
2296}
2297
2298/*
2299 * idt_unregister_device() - unregister IDT NTB device
2300 * @ndev:       IDT NTB hardware driver descriptor
2301 */
2302static void idt_unregister_device(struct idt_ntb_dev *ndev)
2303{
2304        /* Just unregister the NTB device */
2305        ntb_unregister_device(&ndev->ntb);
2306
2307        dev_dbg(&ndev->ntb.pdev->dev, "NTB device unregistered");
2308}
2309
2310/*=============================================================================
2311 *                        10. DebugFS node initialization
2312 *=============================================================================
2313 */
2314
2315static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf,
2316                                   size_t count, loff_t *offp);
2317
2318/*
2319 * Driver DebugFS info file operations
2320 */
2321static const struct file_operations idt_dbgfs_info_ops = {
2322        .owner = THIS_MODULE,
2323        .open = simple_open,
2324        .read = idt_dbgfs_info_read
2325};
2326
2327/*
2328 * idt_dbgfs_info_read() - DebugFS read info node callback
2329 * @file:       File node descriptor.
2330 * @ubuf:       User-space buffer to put data to
2331 * @count:      Size of the buffer
2332 * @offp:       Offset within the buffer
2333 */
2334static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf,
2335                                   size_t count, loff_t *offp)
2336{
2337        struct idt_ntb_dev *ndev = filp->private_data;
2338        unsigned char idx, pidx, cnt;
2339        unsigned long irqflags, mdeg;
2340        ssize_t ret = 0, off = 0;
2341        enum ntb_speed speed;
2342        enum ntb_width width;
2343        char *strbuf;
2344        size_t size;
2345        u32 data;
2346
2347        /* Lets limit the buffer size the way the Intel/AMD drivers do */
2348        size = min_t(size_t, count, 0x1000U);
2349
2350        /* Allocate the memory for the buffer */
2351        strbuf = kmalloc(size, GFP_KERNEL);
2352        if (strbuf == NULL)
2353                return -ENOMEM;
2354
2355        /* Put the data into the string buffer */
2356        off += scnprintf(strbuf + off, size - off,
2357                "\n\t\tIDT NTB device Information:\n\n");
2358
2359        /* General local device configurations */
2360        off += scnprintf(strbuf + off, size - off,
2361                "Local Port %hhu, Partition %hhu\n", ndev->port, ndev->part);
2362
2363        /* Peer ports information */
2364        off += scnprintf(strbuf + off, size - off, "Peers:\n");
2365        for (idx = 0; idx < ndev->peer_cnt; idx++) {
2366                off += scnprintf(strbuf + off, size - off,
2367                        "\t%hhu. Port %hhu, Partition %hhu\n",
2368                        idx, ndev->peers[idx].port, ndev->peers[idx].part);
2369        }
2370
2371        /* Links status */
2372        data = idt_ntb_link_is_up(&ndev->ntb, &speed, &width);
2373        off += scnprintf(strbuf + off, size - off,
2374                "NTB link status\t- 0x%08x, ", data);
2375        off += scnprintf(strbuf + off, size - off, "PCIe Gen %d x%d lanes\n",
2376                speed, width);
2377
2378        /* Mapping table entries */
2379        off += scnprintf(strbuf + off, size - off, "NTB Mapping Table:\n");
2380        for (idx = 0; idx < IDT_MTBL_ENTRY_CNT; idx++) {
2381                spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
2382                idt_nt_write(ndev, IDT_NT_NTMTBLADDR, idx);
2383                data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);
2384                spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
2385
2386                /* Print valid entries only */
2387                if (data & IDT_NTMTBLDATA_VALID) {
2388                        off += scnprintf(strbuf + off, size - off,
2389                                "\t%hhu. Partition %d, Requester ID 0x%04x\n",
2390                                idx, GET_FIELD(NTMTBLDATA_PART, data),
2391                                GET_FIELD(NTMTBLDATA_REQID, data));
2392                }
2393        }
2394        off += scnprintf(strbuf + off, size - off, "\n");
2395
2396        /* Outbound memory windows information */
2397        off += scnprintf(strbuf + off, size - off,
2398                "Outbound Memory Windows:\n");
2399        for (idx = 0; idx < ndev->mw_cnt; idx += cnt) {
2400                data = ndev->mws[idx].type;
2401                cnt = idt_get_mw_count(data);
2402
2403                /* Print Memory Window information */
2404                if (data == IDT_MW_DIR)
2405                        off += scnprintf(strbuf + off, size - off,
2406                                "\t%hhu.\t", idx);
2407                else
2408                        off += scnprintf(strbuf + off, size - off,
2409                                "\t%hhu-%hhu.\t", idx, idx + cnt - 1);
2410
2411                off += scnprintf(strbuf + off, size - off, "%s BAR%hhu, ",
2412                        idt_get_mw_name(data), ndev->mws[idx].bar);
2413
2414                off += scnprintf(strbuf + off, size - off,
2415                        "Address align 0x%08llx, ", ndev->mws[idx].addr_align);
2416
2417                off += scnprintf(strbuf + off, size - off,
2418                        "Size align 0x%08llx, Size max %llu\n",
2419                        ndev->mws[idx].size_align, ndev->mws[idx].size_max);
2420        }
2421
2422        /* Inbound memory windows information */
2423        for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {
2424                off += scnprintf(strbuf + off, size - off,
2425                        "Inbound Memory Windows for peer %hhu (Port %hhu):\n",
2426                        pidx, ndev->peers[pidx].port);
2427
2428                /* Print Memory Windows information */
2429                for (idx = 0; idx < ndev->peers[pidx].mw_cnt; idx += cnt) {
2430                        data = ndev->peers[pidx].mws[idx].type;
2431                        cnt = idt_get_mw_count(data);
2432
2433                        if (data == IDT_MW_DIR)
2434                                off += scnprintf(strbuf + off, size - off,
2435                                        "\t%hhu.\t", idx);
2436                        else
2437                                off += scnprintf(strbuf + off, size - off,
2438                                        "\t%hhu-%hhu.\t", idx, idx + cnt - 1);
2439
2440                        off += scnprintf(strbuf + off, size - off,
2441                                "%s BAR%hhu, ", idt_get_mw_name(data),
2442                                ndev->peers[pidx].mws[idx].bar);
2443
2444                        off += scnprintf(strbuf + off, size - off,
2445                                "Address align 0x%08llx, ",
2446                                ndev->peers[pidx].mws[idx].addr_align);
2447
2448                        off += scnprintf(strbuf + off, size - off,
2449                                "Size align 0x%08llx, Size max %llu\n",
2450                                ndev->peers[pidx].mws[idx].size_align,
2451                                ndev->peers[pidx].mws[idx].size_max);
2452                }
2453        }
2454        off += scnprintf(strbuf + off, size - off, "\n");
2455
2456        /* Doorbell information */
2457        data = idt_sw_read(ndev, IDT_SW_GDBELLSTS);
2458        off += scnprintf(strbuf + off, size - off,
2459                 "Global Doorbell state\t- 0x%08x\n", data);
2460        data = idt_ntb_db_read(&ndev->ntb);
2461        off += scnprintf(strbuf + off, size - off,
2462                 "Local  Doorbell state\t- 0x%08x\n", data);
2463        data = idt_nt_read(ndev, IDT_NT_INDBELLMSK);
2464        off += scnprintf(strbuf + off, size - off,
2465                 "Local  Doorbell mask\t- 0x%08x\n", data);
2466        off += scnprintf(strbuf + off, size - off, "\n");
2467
2468        /* Messaging information */
2469        off += scnprintf(strbuf + off, size - off,
2470                 "Message event valid\t- 0x%08x\n", IDT_MSG_MASK);
2471        data = idt_ntb_msg_read_sts(&ndev->ntb);
2472        off += scnprintf(strbuf + off, size - off,
2473                 "Message event status\t- 0x%08x\n", data);
2474        data = idt_nt_read(ndev, IDT_NT_MSGSTSMSK);
2475        off += scnprintf(strbuf + off, size - off,
2476                 "Message event mask\t- 0x%08x\n", data);
2477        off += scnprintf(strbuf + off, size - off,
2478                 "Message data:\n");
2479        for (idx = 0; idx < IDT_MSG_CNT; idx++) {
2480                int src;
2481                data = idt_ntb_msg_read(&ndev->ntb, &src, idx);
2482                off += scnprintf(strbuf + off, size - off,
2483                        "\t%hhu. 0x%08x from peer %hhu (Port %hhu)\n",
2484                        idx, data, src, ndev->peers[src].port);
2485        }
2486        off += scnprintf(strbuf + off, size - off, "\n");
2487
2488        /* Current temperature */
2489        idt_read_temp(ndev, IDT_TEMP_CUR, &mdeg);
2490        off += scnprintf(strbuf + off, size - off,
2491                "Switch temperature\t\t- %hhd.%hhuC\n",
2492                idt_get_deg(mdeg), idt_get_deg_frac(mdeg));
2493
2494        /* Copy the buffer to the User Space */
2495        ret = simple_read_from_buffer(ubuf, count, offp, strbuf, off);
2496        kfree(strbuf);
2497
2498        return ret;
2499}
2500
2501/*
2502 * idt_init_dbgfs() - initialize DebugFS node
2503 * @ndev:       IDT NTB hardware driver descriptor
2504 *
2505 * Return: zero on success, otherwise a negative error number.
2506 */
2507static int idt_init_dbgfs(struct idt_ntb_dev *ndev)
2508{
2509        char devname[64];
2510
2511        /* If the top directory is not created then do nothing */
2512        if (IS_ERR_OR_NULL(dbgfs_topdir)) {
2513                dev_info(&ndev->ntb.pdev->dev, "Top DebugFS directory absent");
2514                return PTR_ERR(dbgfs_topdir);
2515        }
2516
2517        /* Create the info file node */
2518        snprintf(devname, 64, "info:%s", pci_name(ndev->ntb.pdev));
2519        ndev->dbgfs_info = debugfs_create_file(devname, 0400, dbgfs_topdir,
2520                ndev, &idt_dbgfs_info_ops);
2521        if (IS_ERR(ndev->dbgfs_info)) {
2522                dev_dbg(&ndev->ntb.pdev->dev, "Failed to create DebugFS node");
2523                return PTR_ERR(ndev->dbgfs_info);
2524        }
2525
2526        dev_dbg(&ndev->ntb.pdev->dev, "NTB device DebugFS node created");
2527
2528        return 0;
2529}
2530
2531/*
2532 * idt_deinit_dbgfs() - deinitialize DebugFS node
2533 * @ndev:       IDT NTB hardware driver descriptor
2534 *
2535 * Just discard the info node from DebugFS
2536 */
2537static void idt_deinit_dbgfs(struct idt_ntb_dev *ndev)
2538{
2539        debugfs_remove(ndev->dbgfs_info);
2540
2541        dev_dbg(&ndev->ntb.pdev->dev, "NTB device DebugFS node discarded");
2542}
2543
2544/*=============================================================================
2545 *                     11. Basic PCIe device initialization
2546 *=============================================================================
2547 */
2548
2549/*
2550 * idt_check_setup() - Check whether the IDT PCIe-swtich is properly
2551 *                     pre-initialized
2552 * @pdev:       Pointer to the PCI device descriptor
2553 *
2554 * Return: zero on success, otherwise a negative error number.
2555 */
2556static int idt_check_setup(struct pci_dev *pdev)
2557{
2558        u32 data;
2559        int ret;
2560
2561        /* Read the BARSETUP0 */
2562        ret = pci_read_config_dword(pdev, IDT_NT_BARSETUP0, &data);
2563        if (ret != 0) {
2564                dev_err(&pdev->dev,
2565                        "Failed to read BARSETUP0 config register");
2566                return ret;
2567        }
2568
2569        /* Check whether the BAR0 register is enabled to be of config space */
2570        if (!(data & IDT_BARSETUP_EN) || !(data & IDT_BARSETUP_MODE_CFG)) {
2571                dev_err(&pdev->dev, "BAR0 doesn't map config space");
2572                return -EINVAL;
2573        }
2574
2575        /* Configuration space BAR0 must have certain size */
2576        if ((data & IDT_BARSETUP_SIZE_MASK) != IDT_BARSETUP_SIZE_CFG) {
2577                dev_err(&pdev->dev, "Invalid size of config space");
2578                return -EINVAL;
2579        }
2580
2581        dev_dbg(&pdev->dev, "NTB device pre-initialized correctly");
2582
2583        return 0;
2584}
2585
2586/*
2587 * Create the IDT PCIe-switch driver descriptor
2588 * @pdev:       Pointer to the PCI device descriptor
2589 * @id:         IDT PCIe-device configuration
2590 *
2591 * It just allocates a memory for IDT PCIe-switch device structure and
2592 * initializes some commonly used fields.
2593 *
2594 * No need of release method, since managed device resource is used for
2595 * memory allocation.
2596 *
2597 * Return: pointer to the descriptor, otherwise a negative error number.
2598 */
2599static struct idt_ntb_dev *idt_create_dev(struct pci_dev *pdev,
2600                                          const struct pci_device_id *id)
2601{
2602        struct idt_ntb_dev *ndev;
2603
2604        /* Allocate memory for the IDT PCIe-device descriptor */
2605        ndev = devm_kzalloc(&pdev->dev, sizeof(*ndev), GFP_KERNEL);
2606        if (!ndev) {
2607                dev_err(&pdev->dev, "Memory allocation failed for descriptor");
2608                return ERR_PTR(-ENOMEM);
2609        }
2610
2611        /* Save the IDT PCIe-switch ports configuration */
2612        ndev->swcfg = (struct idt_89hpes_cfg *)id->driver_data;
2613        /* Save the PCI-device pointer inside the NTB device structure */
2614        ndev->ntb.pdev = pdev;
2615
2616        /* Initialize spin locker of Doorbell, Message and GASA registers */
2617        spin_lock_init(&ndev->db_mask_lock);
2618        spin_lock_init(&ndev->msg_mask_lock);
2619        spin_lock_init(&ndev->gasa_lock);
2620
2621        dev_info(&pdev->dev, "IDT %s discovered", ndev->swcfg->name);
2622
2623        dev_dbg(&pdev->dev, "NTB device descriptor created");
2624
2625        return ndev;
2626}
2627
2628/*
2629 * idt_init_pci() - initialize the basic PCI-related subsystem
2630 * @ndev:       Pointer to the IDT PCIe-switch driver descriptor
2631 *
2632 * Managed device resources will be freed automatically in case of failure or
2633 * driver detachment.
2634 *
2635 * Return: zero on success, otherwise negative error number.
2636 */
2637static int idt_init_pci(struct idt_ntb_dev *ndev)
2638{
2639        struct pci_dev *pdev = ndev->ntb.pdev;
2640        int ret;
2641
2642        /* Initialize the bit mask of PCI/NTB DMA */
2643        ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2644        if (ret != 0) {
2645                ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2646                if (ret != 0) {
2647                        dev_err(&pdev->dev, "Failed to set DMA bit mask\n");
2648                        return ret;
2649                }
2650                dev_warn(&pdev->dev, "Cannot set DMA highmem bit mask\n");
2651        }
2652        ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2653        if (ret != 0) {
2654                ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
2655                if (ret != 0) {
2656                        dev_err(&pdev->dev,
2657                                "Failed to set consistent DMA bit mask\n");
2658                        return ret;
2659                }
2660                dev_warn(&pdev->dev,
2661                        "Cannot set consistent DMA highmem bit mask\n");
2662        }
2663        ret = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
2664                                           dma_get_mask(&pdev->dev));
2665        if (ret != 0) {
2666                dev_err(&pdev->dev, "Failed to set NTB device DMA bit mask\n");
2667                return ret;
2668        }
2669
2670        /*
2671         * Enable the device advanced error reporting. It's not critical to
2672         * have AER disabled in the kernel.
2673         */
2674        ret = pci_enable_pcie_error_reporting(pdev);
2675        if (ret != 0)
2676                dev_warn(&pdev->dev, "PCIe AER capability disabled\n");
2677        else /* Cleanup uncorrectable error status before getting to init */
2678                pci_cleanup_aer_uncorrect_error_status(pdev);
2679
2680        /* First enable the PCI device */
2681        ret = pcim_enable_device(pdev);
2682        if (ret != 0) {
2683                dev_err(&pdev->dev, "Failed to enable PCIe device\n");
2684                goto err_disable_aer;
2685        }
2686
2687        /*
2688         * Enable the bus mastering, which effectively enables MSI IRQs and
2689         * Request TLPs translation
2690         */
2691        pci_set_master(pdev);
2692
2693        /* Request all BARs resources and map BAR0 only */
2694        ret = pcim_iomap_regions_request_all(pdev, 1, NTB_NAME);
2695        if (ret != 0) {
2696                dev_err(&pdev->dev, "Failed to request resources\n");
2697                goto err_clear_master;
2698        }
2699
2700        /* Retrieve virtual address of BAR0 - PCI configuration space */
2701        ndev->cfgspc = pcim_iomap_table(pdev)[0];
2702
2703        /* Put the IDT driver data pointer to the PCI-device private pointer */
2704        pci_set_drvdata(pdev, ndev);
2705
2706        dev_dbg(&pdev->dev, "NT-function PCIe interface initialized");
2707
2708        return 0;
2709
2710err_clear_master:
2711        pci_clear_master(pdev);
2712err_disable_aer:
2713        (void)pci_disable_pcie_error_reporting(pdev);
2714
2715        return ret;
2716}
2717
2718/*
2719 * idt_deinit_pci() - deinitialize the basic PCI-related subsystem
2720 * @ndev:       Pointer to the IDT PCIe-switch driver descriptor
2721 *
2722 * Managed resources will be freed on the driver detachment
2723 */
2724static void idt_deinit_pci(struct idt_ntb_dev *ndev)
2725{
2726        struct pci_dev *pdev = ndev->ntb.pdev;
2727
2728        /* Clean up the PCI-device private data pointer */
2729        pci_set_drvdata(pdev, NULL);
2730
2731        /* Clear the bus master disabling the Request TLPs translation */
2732        pci_clear_master(pdev);
2733
2734        /* Disable the AER capability */
2735        (void)pci_disable_pcie_error_reporting(pdev);
2736
2737        dev_dbg(&pdev->dev, "NT-function PCIe interface cleared");
2738}
2739
2740/*===========================================================================
2741 *                       12. PCI bus callback functions
2742 *===========================================================================
2743 */
2744
2745/*
2746 * idt_pci_probe() - PCI device probe callback
2747 * @pdev:       Pointer to PCI device structure
2748 * @id:         PCIe device custom descriptor
2749 *
2750 * Return: zero on success, otherwise negative error number
2751 */
2752static int idt_pci_probe(struct pci_dev *pdev,
2753                         const struct pci_device_id *id)
2754{
2755        struct idt_ntb_dev *ndev;
2756        int ret;
2757
2758        /* Check whether IDT PCIe-switch is properly pre-initialized */
2759        ret = idt_check_setup(pdev);
2760        if (ret != 0)
2761                return ret;
2762
2763        /* Allocate the memory for IDT NTB device data */
2764        ndev = idt_create_dev(pdev, id);
2765        if (IS_ERR_OR_NULL(ndev))
2766                return PTR_ERR(ndev);
2767
2768        /* Initialize the basic PCI subsystem of the device */
2769        ret = idt_init_pci(ndev);
2770        if (ret != 0)
2771                return ret;
2772
2773        /* Scan ports of the IDT PCIe-switch */
2774        (void)idt_scan_ports(ndev);
2775
2776        /* Initialize NTB link events subsystem */
2777        idt_init_link(ndev);
2778
2779        /* Initialize MWs subsystem */
2780        ret = idt_init_mws(ndev);
2781        if (ret != 0)
2782                goto err_deinit_link;
2783
2784        /* Initialize Messaging subsystem */
2785        idt_init_msg(ndev);
2786
2787        /* Initialize hwmon interface */
2788        idt_init_temp(ndev);
2789
2790        /* Initialize IDT interrupts handler */
2791        ret = idt_init_isr(ndev);
2792        if (ret != 0)
2793                goto err_deinit_link;
2794
2795        /* Register IDT NTB devices on the NTB bus */
2796        ret = idt_register_device(ndev);
2797        if (ret != 0)
2798                goto err_deinit_isr;
2799
2800        /* Initialize DebugFS info node */
2801        (void)idt_init_dbgfs(ndev);
2802
2803        /* IDT PCIe-switch NTB driver is finally initialized */
2804        dev_info(&pdev->dev, "IDT NTB device is ready");
2805
2806        /* May the force be with us... */
2807        return 0;
2808
2809err_deinit_isr:
2810        idt_deinit_isr(ndev);
2811err_deinit_link:
2812        idt_deinit_link(ndev);
2813        idt_deinit_pci(ndev);
2814
2815        return ret;
2816}
2817
2818/*
2819 * idt_pci_probe() - PCI device remove callback
2820 * @pdev:       Pointer to PCI device structure
2821 */
2822static void idt_pci_remove(struct pci_dev *pdev)
2823{
2824        struct idt_ntb_dev *ndev = pci_get_drvdata(pdev);
2825
2826        /* Deinit the DebugFS node */
2827        idt_deinit_dbgfs(ndev);
2828
2829        /* Unregister NTB device */
2830        idt_unregister_device(ndev);
2831
2832        /* Stop the interrupts handling */
2833        idt_deinit_isr(ndev);
2834
2835        /* Deinitialize link event subsystem */
2836        idt_deinit_link(ndev);
2837
2838        /* Deinit basic PCI subsystem */
2839        idt_deinit_pci(ndev);
2840
2841        /* IDT PCIe-switch NTB driver is finally initialized */
2842        dev_info(&pdev->dev, "IDT NTB device is removed");
2843
2844        /* Sayonara... */
2845}
2846
2847/*
2848 * IDT PCIe-switch models ports configuration structures
2849 */
2850static const struct idt_89hpes_cfg idt_89hpes24nt6ag2_config = {
2851        .name = "89HPES24NT6AG2",
2852        .port_cnt = 6, .ports = {0, 2, 4, 6, 8, 12}
2853};
2854static const struct idt_89hpes_cfg idt_89hpes32nt8ag2_config = {
2855        .name = "89HPES32NT8AG2",
2856        .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2857};
2858static const struct idt_89hpes_cfg idt_89hpes32nt8bg2_config = {
2859        .name = "89HPES32NT8BG2",
2860        .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2861};
2862static const struct idt_89hpes_cfg idt_89hpes12nt12g2_config = {
2863        .name = "89HPES12NT12G2",
2864        .port_cnt = 3, .ports = {0, 8, 16}
2865};
2866static const struct idt_89hpes_cfg idt_89hpes16nt16g2_config = {
2867        .name = "89HPES16NT16G2",
2868        .port_cnt = 4, .ports = {0, 8, 12, 16}
2869};
2870static const struct idt_89hpes_cfg idt_89hpes24nt24g2_config = {
2871        .name = "89HPES24NT24G2",
2872        .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2873};
2874static const struct idt_89hpes_cfg idt_89hpes32nt24ag2_config = {
2875        .name = "89HPES32NT24AG2",
2876        .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2877};
2878static const struct idt_89hpes_cfg idt_89hpes32nt24bg2_config = {
2879        .name = "89HPES32NT24BG2",
2880        .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2881};
2882
2883/*
2884 * PCI-ids table of the supported IDT PCIe-switch devices
2885 */
2886static const struct pci_device_id idt_pci_tbl[] = {
2887        {IDT_PCI_DEVICE_IDS(89HPES24NT6AG2,  idt_89hpes24nt6ag2_config)},
2888        {IDT_PCI_DEVICE_IDS(89HPES32NT8AG2,  idt_89hpes32nt8ag2_config)},
2889        {IDT_PCI_DEVICE_IDS(89HPES32NT8BG2,  idt_89hpes32nt8bg2_config)},
2890        {IDT_PCI_DEVICE_IDS(89HPES12NT12G2,  idt_89hpes12nt12g2_config)},
2891        {IDT_PCI_DEVICE_IDS(89HPES16NT16G2,  idt_89hpes16nt16g2_config)},
2892        {IDT_PCI_DEVICE_IDS(89HPES24NT24G2,  idt_89hpes24nt24g2_config)},
2893        {IDT_PCI_DEVICE_IDS(89HPES32NT24AG2, idt_89hpes32nt24ag2_config)},
2894        {IDT_PCI_DEVICE_IDS(89HPES32NT24BG2, idt_89hpes32nt24bg2_config)},
2895        {0}
2896};
2897MODULE_DEVICE_TABLE(pci, idt_pci_tbl);
2898
2899/*
2900 * IDT PCIe-switch NT-function device driver structure definition
2901 */
2902static struct pci_driver idt_pci_driver = {
2903        .name           = KBUILD_MODNAME,
2904        .probe          = idt_pci_probe,
2905        .remove         = idt_pci_remove,
2906        .id_table       = idt_pci_tbl,
2907};
2908
2909static int __init idt_pci_driver_init(void)
2910{
2911        pr_info("%s %s\n", NTB_DESC, NTB_VER);
2912
2913        /* Create the top DebugFS directory if the FS is initialized */
2914        if (debugfs_initialized())
2915                dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2916
2917        /* Register the NTB hardware driver to handle the PCI device */
2918        return pci_register_driver(&idt_pci_driver);
2919}
2920module_init(idt_pci_driver_init);
2921
2922static void __exit idt_pci_driver_exit(void)
2923{
2924        /* Unregister the NTB hardware driver */
2925        pci_unregister_driver(&idt_pci_driver);
2926
2927        /* Discard the top DebugFS directory */
2928        debugfs_remove_recursive(dbgfs_topdir);
2929}
2930module_exit(idt_pci_driver_exit);
2931
2932