LXR linux/drivers/crypto/cavium/nitrox/nitrox

   1// SPDX-License-Identifier: GPL-2.0-only
   2#include <linux/aer.h>
   3#include <linux/delay.h>
   4#include <linux/firmware.h>
   5#include <linux/list.h>
   6#include <linux/module.h>
   7#include <linux/mutex.h>
   8#include <linux/pci.h>
   9#include <linux/pci_ids.h>
  10
  11#include "nitrox_dev.h"
  12#include "nitrox_common.h"
  13#include "nitrox_csr.h"
  14#include "nitrox_hal.h"
  15#include "nitrox_isr.h"
  16#include "nitrox_debugfs.h"
  17
  18#define CNN55XX_DEV_ID  0x12
  19#define UCODE_HLEN 48
  20#define SE_GROUP 0
  21
  22#define DRIVER_VERSION "1.1"
  23#define FW_DIR "cavium/"
  24/* SE microcode */
  25#define SE_FW   FW_DIR "cnn55xx_se.fw"
  26
  27static const char nitrox_driver_name[] = "CNN55XX";
  28
  29static LIST_HEAD(ndevlist);
  30static DEFINE_MUTEX(devlist_lock);
  31static unsigned int num_devices;
  32
  33/**
  34 * nitrox_pci_tbl - PCI Device ID Table
  35 */
  36static const struct pci_device_id nitrox_pci_tbl[] = {
  37        {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
  38        /* required last entry */
  39        {0, }
  40};
  41MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
  42
  43static unsigned int qlen = DEFAULT_CMD_QLEN;
  44module_param(qlen, uint, 0644);
  45MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
  46
  47#ifdef CONFIG_PCI_IOV
  48int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
  49#else
  50int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
  51{
  52        return 0;
  53}
  54#endif
  55
  56/**
  57 * struct ucode - Firmware Header
  58 * @id: microcode ID
  59 * @version: firmware version
  60 * @code_size: code section size
  61 * @raz: alignment
  62 * @code: code section
  63 */
  64struct ucode {
  65        u8 id;
  66        char version[VERSION_LEN - 1];
  67        __be32 code_size;
  68        u8 raz[12];
  69        u64 code[0];
  70};
  71
  72/**
  73 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
  74 */
  75static void write_to_ucd_unit(struct nitrox_device *ndev,
  76                              struct ucode *ucode)
  77{
  78        u32 code_size = be32_to_cpu(ucode->code_size) * 2;
  79        u64 offset, data;
  80        int i = 0;
  81
  82        /*
  83         * UCD structure
  84         *
  85         *  -------------
  86         *  |    BLK 7  |
  87         *  -------------
  88         *  |    BLK 6  |
  89         *  -------------
  90         *  |    ...    |
  91         *  -------------
  92         *  |    BLK 0  |
  93         *  -------------
  94         *  Total of 8 blocks, each size 32KB
  95         */
  96
  97        /* set the block number */
  98        offset = UCD_UCODE_LOAD_BLOCK_NUM;
  99        nitrox_write_csr(ndev, offset, 0);
 100
 101        code_size = roundup(code_size, 8);
 102        while (code_size) {
 103                data = ucode->code[i];
 104                /* write 8 bytes at a time */
 105                offset = UCD_UCODE_LOAD_IDX_DATAX(i);
 106                nitrox_write_csr(ndev, offset, data);
 107                code_size -= 8;
 108                i++;
 109        }
 110
 111        /* put all SE cores in group 0 */
 112        offset = POM_GRP_EXECMASKX(SE_GROUP);
 113        nitrox_write_csr(ndev, offset, (~0ULL));
 114
 115        for (i = 0; i < ndev->hw.se_cores; i++) {
 116                /*
 117                 * write block number and firware length
 118                 * bit:<2:0> block number
 119                 * bit:3 is set SE uses 32KB microcode
 120                 * bit:3 is clear SE uses 64KB microcode
 121                 */
 122                offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
 123                nitrox_write_csr(ndev, offset, 0x8);
 124        }
 125        usleep_range(300, 400);
 126}
 127
 128static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
 129{
 130        const struct firmware *fw;
 131        struct ucode *ucode;
 132        int ret;
 133
 134        dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 135
 136        ret = request_firmware(&fw, fw_name, DEV(ndev));
 137        if (ret < 0) {
 138                dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 139                return ret;
 140        }
 141
 142        ucode = (struct ucode *)fw->data;
 143        /* copy the firmware version */
 144        memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
 145        ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
 146
 147        write_to_ucd_unit(ndev, ucode);
 148        release_firmware(fw);
 149
 150        return 0;
 151}
 152
 153/**
 154 * nitrox_add_to_devlist - add NITROX device to global device list
 155 * @ndev: NITROX device
 156 */
 157static int nitrox_add_to_devlist(struct nitrox_device *ndev)
 158{
 159        struct nitrox_device *dev;
 160        int ret = 0;
 161
 162        INIT_LIST_HEAD(&ndev->list);
 163        refcount_set(&ndev->refcnt, 1);
 164
 165        mutex_lock(&devlist_lock);
 166        list_for_each_entry(dev, &ndevlist, list) {
 167                if (dev == ndev) {
 168                        ret = -EEXIST;
 169                        goto unlock;
 170                }
 171        }
 172        ndev->idx = num_devices++;
 173        list_add_tail(&ndev->list, &ndevlist);
 174unlock:
 175        mutex_unlock(&devlist_lock);
 176        return ret;
 177}
 178
 179/**
 180 * nitrox_remove_from_devlist - remove NITROX device from
 181 *   global device list
 182 * @ndev: NITROX device
 183 */
 184static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
 185{
 186        mutex_lock(&devlist_lock);
 187        list_del(&ndev->list);
 188        num_devices--;
 189        mutex_unlock(&devlist_lock);
 190}
 191
 192struct nitrox_device *nitrox_get_first_device(void)
 193{
 194        struct nitrox_device *ndev = NULL;
 195
 196        mutex_lock(&devlist_lock);
 197        list_for_each_entry(ndev, &ndevlist, list) {
 198                if (nitrox_ready(ndev))
 199                        break;
 200        }
 201        mutex_unlock(&devlist_lock);
 202        if (!ndev)
 203                return NULL;
 204
 205        refcount_inc(&ndev->refcnt);
 206        /* barrier to sync with other cpus */
 207        smp_mb__after_atomic();
 208        return ndev;
 209}
 210
 211void nitrox_put_device(struct nitrox_device *ndev)
 212{
 213        if (!ndev)
 214                return;
 215
 216        refcount_dec(&ndev->refcnt);
 217        /* barrier to sync with other cpus */
 218        smp_mb__after_atomic();
 219}
 220
 221static int nitrox_device_flr(struct pci_dev *pdev)
 222{
 223        int pos = 0;
 224
 225        pos = pci_save_state(pdev);
 226        if (pos) {
 227                dev_err(&pdev->dev, "Failed to save pci state\n");
 228                return -ENOMEM;
 229        }
 230
 231        /* check flr support */
 232        if (pcie_has_flr(pdev))
 233                pcie_flr(pdev);
 234
 235        pci_restore_state(pdev);
 236
 237        return 0;
 238}
 239
 240static int nitrox_pf_sw_init(struct nitrox_device *ndev)
 241{
 242        int err;
 243
 244        err = nitrox_common_sw_init(ndev);
 245        if (err)
 246                return err;
 247
 248        err = nitrox_register_interrupts(ndev);
 249        if (err)
 250                nitrox_common_sw_cleanup(ndev);
 251
 252        return err;
 253}
 254
 255static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
 256{
 257        nitrox_unregister_interrupts(ndev);
 258        nitrox_common_sw_cleanup(ndev);
 259}
 260
 261/**
 262 * nitrox_bist_check - Check NITORX BIST registers status
 263 * @ndev: NITROX device
 264 */
 265static int nitrox_bist_check(struct nitrox_device *ndev)
 266{
 267        u64 value = 0;
 268        int i;
 269
 270        for (i = 0; i < NR_CLUSTERS; i++) {
 271                value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
 272                value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
 273        }
 274        value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
 275        value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
 276        value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
 277        value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
 278        value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
 279        value += nitrox_read_csr(ndev, POM_BIST_REG);
 280        value += nitrox_read_csr(ndev, BMI_BIST_REG);
 281        value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
 282        value += nitrox_read_csr(ndev, BMO_BIST_REG);
 283        value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
 284        value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
 285        if (value)
 286                return -EIO;
 287        return 0;
 288}
 289
 290static int nitrox_pf_hw_init(struct nitrox_device *ndev)
 291{
 292        int err;
 293
 294        err = nitrox_bist_check(ndev);
 295        if (err) {
 296                dev_err(&ndev->pdev->dev, "BIST check failed\n");
 297                return err;
 298        }
 299        /* get cores information */
 300        nitrox_get_hwinfo(ndev);
 301
 302        nitrox_config_nps_unit(ndev);
 303        nitrox_config_pom_unit(ndev);
 304        nitrox_config_efl_unit(ndev);
 305        /* configure IO units */
 306        nitrox_config_bmi_unit(ndev);
 307        nitrox_config_bmo_unit(ndev);
 308        /* configure Local Buffer Cache */
 309        nitrox_config_lbc_unit(ndev);
 310        nitrox_config_rand_unit(ndev);
 311
 312        /* load firmware on SE cores */
 313        err = nitrox_load_fw(ndev, SE_FW);
 314        if (err)
 315                return err;
 316
 317        nitrox_config_emu_unit(ndev);
 318
 319        return 0;
 320}
 321
 322/**
 323 * nitrox_probe - NITROX Initialization function.
 324 * @pdev: PCI device information struct
 325 * @id: entry in nitrox_pci_tbl
 326 *
 327 * Return: 0, if the driver is bound to the device, or
 328 *         a negative error if there is failure.
 329 */
 330static int nitrox_probe(struct pci_dev *pdev,
 331                        const struct pci_device_id *id)
 332{
 333        struct nitrox_device *ndev;
 334        int err;
 335
 336        dev_info_once(&pdev->dev, "%s driver version %s\n",
 337                      nitrox_driver_name, DRIVER_VERSION);
 338
 339        err = pci_enable_device_mem(pdev);
 340        if (err)
 341                return err;
 342
 343        /* do FLR */
 344        err = nitrox_device_flr(pdev);
 345        if (err) {
 346                dev_err(&pdev->dev, "FLR failed\n");
 347                pci_disable_device(pdev);
 348                return err;
 349        }
 350
 351        if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
 352                dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
 353        } else {
 354                err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 355                if (err) {
 356                        dev_err(&pdev->dev, "DMA configuration failed\n");
 357                        pci_disable_device(pdev);
 358                        return err;
 359                }
 360        }
 361
 362        err = pci_request_mem_regions(pdev, nitrox_driver_name);
 363        if (err) {
 364                pci_disable_device(pdev);
 365                return err;
 366        }
 367        pci_set_master(pdev);
 368
 369        ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 370        if (!ndev) {
 371                err = -ENOMEM;
 372                goto ndev_fail;
 373        }
 374
 375        pci_set_drvdata(pdev, ndev);
 376        ndev->pdev = pdev;
 377
 378        /* add to device list */
 379        nitrox_add_to_devlist(ndev);
 380
 381        ndev->hw.vendor_id = pdev->vendor;
 382        ndev->hw.device_id = pdev->device;
 383        ndev->hw.revision_id = pdev->revision;
 384        /* command timeout in jiffies */
 385        ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
 386        ndev->node = dev_to_node(&pdev->dev);
 387        if (ndev->node == NUMA_NO_NODE)
 388                ndev->node = 0;
 389
 390        ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
 391                                 pci_resource_len(pdev, 0));
 392        if (!ndev->bar_addr) {
 393                err = -EIO;
 394                goto ioremap_err;
 395        }
 396        /* allocate command queus based on cpus, max queues are 64 */
 397        ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
 398        ndev->qlen = qlen;
 399
 400        err = nitrox_pf_sw_init(ndev);
 401        if (err)
 402                goto ioremap_err;
 403
 404        err = nitrox_pf_hw_init(ndev);
 405        if (err)
 406                goto pf_hw_fail;
 407
 408        nitrox_debugfs_init(ndev);
 409
 410        /* clear the statistics */
 411        atomic64_set(&ndev->stats.posted, 0);
 412        atomic64_set(&ndev->stats.completed, 0);
 413        atomic64_set(&ndev->stats.dropped, 0);
 414
 415        atomic_set(&ndev->state, __NDEV_READY);
 416        /* barrier to sync with other cpus */
 417        smp_mb__after_atomic();
 418
 419        err = nitrox_crypto_register();
 420        if (err)
 421                goto crypto_fail;
 422
 423        return 0;
 424
 425crypto_fail:
 426        nitrox_debugfs_exit(ndev);
 427        atomic_set(&ndev->state, __NDEV_NOT_READY);
 428        /* barrier to sync with other cpus */
 429        smp_mb__after_atomic();
 430pf_hw_fail:
 431        nitrox_pf_sw_cleanup(ndev);
 432ioremap_err:
 433        nitrox_remove_from_devlist(ndev);
 434        kfree(ndev);
 435        pci_set_drvdata(pdev, NULL);
 436ndev_fail:
 437        pci_release_mem_regions(pdev);
 438        pci_disable_device(pdev);
 439        return err;
 440}
 441
 442/**
 443 * nitrox_remove - Unbind the driver from the device.
 444 * @pdev: PCI device information struct
 445 */
 446static void nitrox_remove(struct pci_dev *pdev)
 447{
 448        struct nitrox_device *ndev = pci_get_drvdata(pdev);
 449
 450        if (!ndev)
 451                return;
 452
 453        if (!refcount_dec_and_test(&ndev->refcnt)) {
 454                dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
 455                        refcount_read(&ndev->refcnt));
 456                return;
 457        }
 458
 459        dev_info(DEV(ndev), "Removing Device %x:%x\n",
 460                 ndev->hw.vendor_id, ndev->hw.device_id);
 461
 462        atomic_set(&ndev->state, __NDEV_NOT_READY);
 463        /* barrier to sync with other cpus */
 464        smp_mb__after_atomic();
 465
 466        nitrox_remove_from_devlist(ndev);
 467
 468#ifdef CONFIG_PCI_IOV
 469        /* disable SR-IOV */
 470        nitrox_sriov_configure(pdev, 0);
 471#endif
 472        nitrox_crypto_unregister();
 473        nitrox_debugfs_exit(ndev);
 474        nitrox_pf_sw_cleanup(ndev);
 475
 476        iounmap(ndev->bar_addr);
 477        kfree(ndev);
 478
 479        pci_set_drvdata(pdev, NULL);
 480        pci_release_mem_regions(pdev);
 481        pci_disable_device(pdev);
 482}
 483
 484static void nitrox_shutdown(struct pci_dev *pdev)
 485{
 486        pci_set_drvdata(pdev, NULL);
 487        pci_release_mem_regions(pdev);
 488        pci_disable_device(pdev);
 489}
 490
 491static struct pci_driver nitrox_driver = {
 492        .name = nitrox_driver_name,
 493        .id_table = nitrox_pci_tbl,
 494        .probe = nitrox_probe,
 495        .remove = nitrox_remove,
 496        .shutdown = nitrox_shutdown,
 497#ifdef CONFIG_PCI_IOV
 498        .sriov_configure = nitrox_sriov_configure,
 499#endif
 500};
 501
 502module_pci_driver(nitrox_driver);
 503
 504MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
 505MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
 506MODULE_LICENSE("GPL");
 507MODULE_VERSION(DRIVER_VERSION);
 508MODULE_FIRMWARE(SE_FW);
 509