dpdk/drivers/crypto/ccp/ccp_dev.c
<<
>>
Prefs
   1/*   SPDX-License-Identifier: BSD-3-Clause
   2 *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3 */
   4
   5#include <dirent.h>
   6#include <fcntl.h>
   7#include <stdio.h>
   8#include <string.h>
   9#include <sys/mman.h>
  10#include <sys/queue.h>
  11#include <sys/types.h>
  12#include <sys/file.h>
  13#include <unistd.h>
  14
  15#include <rte_hexdump.h>
  16#include <rte_memzone.h>
  17#include <rte_malloc.h>
  18#include <rte_memory.h>
  19#include <rte_spinlock.h>
  20#include <rte_string_fns.h>
  21
  22#include "ccp_dev.h"
  23#include "ccp_pci.h"
  24#include "ccp_pmd_private.h"
  25
  26int iommu_mode;
  27struct ccp_list ccp_list = TAILQ_HEAD_INITIALIZER(ccp_list);
  28static int ccp_dev_id;
  29
  30int
  31ccp_dev_start(struct rte_cryptodev *dev)
  32{
  33        struct ccp_private *priv = dev->data->dev_private;
  34
  35        priv->last_dev = TAILQ_FIRST(&ccp_list);
  36        return 0;
  37}
  38
  39struct ccp_queue *
  40ccp_allot_queue(struct rte_cryptodev *cdev, int slot_req)
  41{
  42        int i, ret = 0;
  43        struct ccp_device *dev;
  44        struct ccp_private *priv = cdev->data->dev_private;
  45
  46        dev = TAILQ_NEXT(priv->last_dev, next);
  47        if (unlikely(dev == NULL))
  48                dev = TAILQ_FIRST(&ccp_list);
  49        priv->last_dev = dev;
  50        if (dev->qidx >= dev->cmd_q_count)
  51                dev->qidx = 0;
  52        ret = rte_atomic64_read(&dev->cmd_q[dev->qidx].free_slots);
  53        if (ret >= slot_req)
  54                return &dev->cmd_q[dev->qidx];
  55        for (i = 0; i < dev->cmd_q_count; i++) {
  56                dev->qidx++;
  57                if (dev->qidx >= dev->cmd_q_count)
  58                        dev->qidx = 0;
  59                ret = rte_atomic64_read(&dev->cmd_q[dev->qidx].free_slots);
  60                if (ret >= slot_req)
  61                        return &dev->cmd_q[dev->qidx];
  62        }
  63        return NULL;
  64}
  65
  66int
  67ccp_read_hwrng(uint32_t *value)
  68{
  69        struct ccp_device *dev;
  70
  71        TAILQ_FOREACH(dev, &ccp_list, next) {
  72                void *vaddr = (void *)(dev->pci.mem_resource[2].addr);
  73
  74                while (dev->hwrng_retries++ < CCP_MAX_TRNG_RETRIES) {
  75                        *value = CCP_READ_REG(vaddr, TRNG_OUT_REG);
  76                        if (*value) {
  77                                dev->hwrng_retries = 0;
  78                                return 0;
  79                        }
  80                }
  81                dev->hwrng_retries = 0;
  82        }
  83        return -1;
  84}
  85
  86static const struct rte_memzone *
  87ccp_queue_dma_zone_reserve(const char *queue_name,
  88                           uint32_t queue_size,
  89                           int socket_id)
  90{
  91        const struct rte_memzone *mz;
  92
  93        mz = rte_memzone_lookup(queue_name);
  94        if (mz != 0) {
  95                if (((size_t)queue_size <= mz->len) &&
  96                    ((socket_id == SOCKET_ID_ANY) ||
  97                     (socket_id == mz->socket_id))) {
  98                        CCP_LOG_INFO("re-use memzone already "
  99                                     "allocated for %s", queue_name);
 100                        return mz;
 101                }
 102                CCP_LOG_ERR("Incompatible memzone already "
 103                            "allocated %s, size %u, socket %d. "
 104                            "Requested size %u, socket %u",
 105                            queue_name, (uint32_t)mz->len,
 106                            mz->socket_id, queue_size, socket_id);
 107                return NULL;
 108        }
 109
 110        CCP_LOG_INFO("Allocate memzone for %s, size %u on socket %u",
 111                     queue_name, queue_size, socket_id);
 112
 113        return rte_memzone_reserve_aligned(queue_name, queue_size,
 114                        socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 115}
 116
 117/* bitmap support apis */
 118static inline void
 119ccp_set_bit(unsigned long *bitmap, int n)
 120{
 121        __sync_fetch_and_or(&bitmap[WORD_OFFSET(n)], (1UL << BIT_OFFSET(n)));
 122}
 123
 124static inline void
 125ccp_clear_bit(unsigned long *bitmap, int n)
 126{
 127        __sync_fetch_and_and(&bitmap[WORD_OFFSET(n)], ~(1UL << BIT_OFFSET(n)));
 128}
 129
 130static inline uint32_t
 131ccp_get_bit(unsigned long *bitmap, int n)
 132{
 133        return ((bitmap[WORD_OFFSET(n)] & (1 << BIT_OFFSET(n))) != 0);
 134}
 135
 136
 137static inline uint32_t
 138ccp_ffz(unsigned long word)
 139{
 140        unsigned long first_zero;
 141
 142        first_zero = __builtin_ffsl(~word);
 143        return first_zero ? (first_zero - 1) :
 144                BITS_PER_WORD;
 145}
 146
 147static inline uint32_t
 148ccp_find_first_zero_bit(unsigned long *addr, uint32_t limit)
 149{
 150        uint32_t i;
 151        uint32_t nwords = 0;
 152
 153        nwords = (limit - 1) / BITS_PER_WORD + 1;
 154        for (i = 0; i < nwords; i++) {
 155                if (addr[i] == 0UL)
 156                        return i * BITS_PER_WORD;
 157                if (addr[i] < ~(0UL))
 158                        break;
 159        }
 160        return (i == nwords) ? limit : i * BITS_PER_WORD + ccp_ffz(addr[i]);
 161}
 162
 163static void
 164ccp_bitmap_set(unsigned long *map, unsigned int start, int len)
 165{
 166        unsigned long *p = map + WORD_OFFSET(start);
 167        const unsigned int size = start + len;
 168        int bits_to_set = BITS_PER_WORD - (start % BITS_PER_WORD);
 169        unsigned long mask_to_set = CCP_BITMAP_FIRST_WORD_MASK(start);
 170
 171        while (len - bits_to_set >= 0) {
 172                *p |= mask_to_set;
 173                len -= bits_to_set;
 174                bits_to_set = BITS_PER_WORD;
 175                mask_to_set = ~0UL;
 176                p++;
 177        }
 178        if (len) {
 179                mask_to_set &= CCP_BITMAP_LAST_WORD_MASK(size);
 180                *p |= mask_to_set;
 181        }
 182}
 183
 184static void
 185ccp_bitmap_clear(unsigned long *map, unsigned int start, int len)
 186{
 187        unsigned long *p = map + WORD_OFFSET(start);
 188        const unsigned int size = start + len;
 189        int bits_to_clear = BITS_PER_WORD - (start % BITS_PER_WORD);
 190        unsigned long mask_to_clear = CCP_BITMAP_FIRST_WORD_MASK(start);
 191
 192        while (len - bits_to_clear >= 0) {
 193                *p &= ~mask_to_clear;
 194                len -= bits_to_clear;
 195                bits_to_clear = BITS_PER_WORD;
 196                mask_to_clear = ~0UL;
 197                p++;
 198        }
 199        if (len) {
 200                mask_to_clear &= CCP_BITMAP_LAST_WORD_MASK(size);
 201                *p &= ~mask_to_clear;
 202        }
 203}
 204
 205
 206static unsigned long
 207_ccp_find_next_bit(const unsigned long *addr,
 208                   unsigned long nbits,
 209                   unsigned long start,
 210                   unsigned long invert)
 211{
 212        unsigned long tmp;
 213
 214        if (!nbits || start >= nbits)
 215                return nbits;
 216
 217        tmp = addr[start / BITS_PER_WORD] ^ invert;
 218
 219        /* Handle 1st word. */
 220        tmp &= CCP_BITMAP_FIRST_WORD_MASK(start);
 221        start = ccp_round_down(start, BITS_PER_WORD);
 222
 223        while (!tmp) {
 224                start += BITS_PER_WORD;
 225                if (start >= nbits)
 226                        return nbits;
 227
 228                tmp = addr[start / BITS_PER_WORD] ^ invert;
 229        }
 230
 231        return RTE_MIN(start + (ffs(tmp) - 1), nbits);
 232}
 233
 234static unsigned long
 235ccp_find_next_bit(const unsigned long *addr,
 236                  unsigned long size,
 237                  unsigned long offset)
 238{
 239        return _ccp_find_next_bit(addr, size, offset, 0UL);
 240}
 241
 242static unsigned long
 243ccp_find_next_zero_bit(const unsigned long *addr,
 244                       unsigned long size,
 245                       unsigned long offset)
 246{
 247        return _ccp_find_next_bit(addr, size, offset, ~0UL);
 248}
 249
 250/**
 251 * bitmap_find_next_zero_area - find a contiguous aligned zero area
 252 * @map: The address to base the search on
 253 * @size: The bitmap size in bits
 254 * @start: The bitnumber to start searching at
 255 * @nr: The number of zeroed bits we're looking for
 256 */
 257static unsigned long
 258ccp_bitmap_find_next_zero_area(unsigned long *map,
 259                               unsigned long size,
 260                               unsigned long start,
 261                               unsigned int nr)
 262{
 263        unsigned long index, end, i;
 264
 265again:
 266        index = ccp_find_next_zero_bit(map, size, start);
 267
 268        end = index + nr;
 269        if (end > size)
 270                return end;
 271        i = ccp_find_next_bit(map, end, index);
 272        if (i < end) {
 273                start = i + 1;
 274                goto again;
 275        }
 276        return index;
 277}
 278
 279static uint32_t
 280ccp_lsb_alloc(struct ccp_queue *cmd_q, unsigned int count)
 281{
 282        struct ccp_device *ccp;
 283        int start;
 284
 285        /* First look at the map for the queue */
 286        if (cmd_q->lsb >= 0) {
 287                start = (uint32_t)ccp_bitmap_find_next_zero_area(cmd_q->lsbmap,
 288                                                                 LSB_SIZE, 0,
 289                                                                 count);
 290                if (start < LSB_SIZE) {
 291                        ccp_bitmap_set(cmd_q->lsbmap, start, count);
 292                        return start + cmd_q->lsb * LSB_SIZE;
 293                }
 294        }
 295
 296        /* try to get an entry from the shared blocks */
 297        ccp = cmd_q->dev;
 298
 299        rte_spinlock_lock(&ccp->lsb_lock);
 300
 301        start = (uint32_t)ccp_bitmap_find_next_zero_area(ccp->lsbmap,
 302                                                    MAX_LSB_CNT * LSB_SIZE,
 303                                                    0, count);
 304        if (start <= MAX_LSB_CNT * LSB_SIZE) {
 305                ccp_bitmap_set(ccp->lsbmap, start, count);
 306                rte_spinlock_unlock(&ccp->lsb_lock);
 307                return start * LSB_ITEM_SIZE;
 308        }
 309        CCP_LOG_ERR("NO LSBs available");
 310
 311        rte_spinlock_unlock(&ccp->lsb_lock);
 312
 313        return 0;
 314}
 315
 316static void __rte_unused
 317ccp_lsb_free(struct ccp_queue *cmd_q,
 318             unsigned int start,
 319             unsigned int count)
 320{
 321        int lsbno = start / LSB_SIZE;
 322
 323        if (!start)
 324                return;
 325
 326        if (cmd_q->lsb == lsbno) {
 327                /* An entry from the private LSB */
 328                ccp_bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
 329        } else {
 330                /* From the shared LSBs */
 331                struct ccp_device *ccp = cmd_q->dev;
 332
 333                rte_spinlock_lock(&ccp->lsb_lock);
 334                ccp_bitmap_clear(ccp->lsbmap, start, count);
 335                rte_spinlock_unlock(&ccp->lsb_lock);
 336        }
 337}
 338
 339static int
 340ccp_find_lsb_regions(struct ccp_queue *cmd_q, uint64_t status)
 341{
 342        int q_mask = 1 << cmd_q->id;
 343        int weight = 0;
 344        int j;
 345
 346        /* Build a bit mask to know which LSBs
 347         * this queue has access to.
 348         * Don't bother with segment 0
 349         * as it has special
 350         * privileges.
 351         */
 352        cmd_q->lsbmask = 0;
 353        status >>= LSB_REGION_WIDTH;
 354        for (j = 1; j < MAX_LSB_CNT; j++) {
 355                if (status & q_mask)
 356                        ccp_set_bit(&cmd_q->lsbmask, j);
 357
 358                status >>= LSB_REGION_WIDTH;
 359        }
 360
 361        for (j = 0; j < MAX_LSB_CNT; j++)
 362                if (ccp_get_bit(&cmd_q->lsbmask, j))
 363                        weight++;
 364
 365        printf("Queue %d can access %d LSB regions  of mask  %lu\n",
 366               (int)cmd_q->id, weight, cmd_q->lsbmask);
 367
 368        return weight ? 0 : -EINVAL;
 369}
 370
 371static int
 372ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
 373                             int lsb_cnt, int n_lsbs,
 374                             unsigned long *lsb_pub)
 375{
 376        unsigned long qlsb = 0;
 377        int bitno = 0;
 378        int qlsb_wgt = 0;
 379        int i, j;
 380
 381        /* For each queue:
 382         * If the count of potential LSBs available to a queue matches the
 383         * ordinal given to us in lsb_cnt:
 384         * Copy the mask of possible LSBs for this queue into "qlsb";
 385         * For each bit in qlsb, see if the corresponding bit in the
 386         * aggregation mask is set; if so, we have a match.
 387         *     If we have a match, clear the bit in the aggregation to
 388         *     mark it as no longer available.
 389         *     If there is no match, clear the bit in qlsb and keep looking.
 390         */
 391        for (i = 0; i < ccp->cmd_q_count; i++) {
 392                struct ccp_queue *cmd_q = &ccp->cmd_q[i];
 393
 394                qlsb_wgt = 0;
 395                for (j = 0; j < MAX_LSB_CNT; j++)
 396                        if (ccp_get_bit(&cmd_q->lsbmask, j))
 397                                qlsb_wgt++;
 398
 399                if (qlsb_wgt == lsb_cnt) {
 400                        qlsb = cmd_q->lsbmask;
 401
 402                        bitno = ffs(qlsb) - 1;
 403                        while (bitno < MAX_LSB_CNT) {
 404                                if (ccp_get_bit(lsb_pub, bitno)) {
 405                                        /* We found an available LSB
 406                                         * that this queue can access
 407                                         */
 408                                        cmd_q->lsb = bitno;
 409                                        ccp_clear_bit(lsb_pub, bitno);
 410                                        break;
 411                                }
 412                                ccp_clear_bit(&qlsb, bitno);
 413                                bitno = ffs(qlsb) - 1;
 414                        }
 415                        if (bitno >= MAX_LSB_CNT)
 416                                return -EINVAL;
 417                        n_lsbs--;
 418                }
 419        }
 420        return n_lsbs;
 421}
 422
 423/* For each queue, from the most- to least-constrained:
 424 * find an LSB that can be assigned to the queue. If there are N queues that
 425 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
 426 * dedicated LSB. Remaining LSB regions become a shared resource.
 427 * If we have fewer LSBs than queues, all LSB regions become shared
 428 * resources.
 429 */
 430static int
 431ccp_assign_lsbs(struct ccp_device *ccp)
 432{
 433        unsigned long lsb_pub = 0, qlsb = 0;
 434        int n_lsbs = 0;
 435        int bitno;
 436        int i, lsb_cnt;
 437        int rc = 0;
 438
 439        rte_spinlock_init(&ccp->lsb_lock);
 440
 441        /* Create an aggregate bitmap to get a total count of available LSBs */
 442        for (i = 0; i < ccp->cmd_q_count; i++)
 443                lsb_pub |= ccp->cmd_q[i].lsbmask;
 444
 445        for (i = 0; i < MAX_LSB_CNT; i++)
 446                if (ccp_get_bit(&lsb_pub, i))
 447                        n_lsbs++;
 448
 449        if (n_lsbs >= ccp->cmd_q_count) {
 450                /* We have enough LSBS to give every queue a private LSB.
 451                 * Brute force search to start with the queues that are more
 452                 * constrained in LSB choice. When an LSB is privately
 453                 * assigned, it is removed from the public mask.
 454                 * This is an ugly N squared algorithm with some optimization.
 455                 */
 456                for (lsb_cnt = 1; n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
 457                     lsb_cnt++) {
 458                        rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
 459                                                          &lsb_pub);
 460                        if (rc < 0)
 461                                return -EINVAL;
 462                        n_lsbs = rc;
 463                }
 464        }
 465
 466        rc = 0;
 467        /* What's left of the LSBs, according to the public mask, now become
 468         * shared. Any zero bits in the lsb_pub mask represent an LSB region
 469         * that can't be used as a shared resource, so mark the LSB slots for
 470         * them as "in use".
 471         */
 472        qlsb = lsb_pub;
 473        bitno = ccp_find_first_zero_bit(&qlsb, MAX_LSB_CNT);
 474        while (bitno < MAX_LSB_CNT) {
 475                ccp_bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
 476                ccp_set_bit(&qlsb, bitno);
 477                bitno = ccp_find_first_zero_bit(&qlsb, MAX_LSB_CNT);
 478        }
 479
 480        return rc;
 481}
 482
 483static int
 484ccp_add_device(struct ccp_device *dev, int type)
 485{
 486        int i;
 487        uint32_t qmr, status_lo, status_hi, dma_addr_lo, dma_addr_hi;
 488        uint64_t status;
 489        struct ccp_queue *cmd_q;
 490        const struct rte_memzone *q_mz;
 491        void *vaddr;
 492
 493        if (dev == NULL)
 494                return -1;
 495
 496        dev->id = ccp_dev_id++;
 497        dev->qidx = 0;
 498        vaddr = (void *)(dev->pci.mem_resource[2].addr);
 499
 500        if (type == CCP_VERSION_5B) {
 501                CCP_WRITE_REG(vaddr, CMD_TRNG_CTL_OFFSET, 0x00012D57);
 502                CCP_WRITE_REG(vaddr, CMD_CONFIG_0_OFFSET, 0x00000003);
 503                for (i = 0; i < 12; i++) {
 504                        CCP_WRITE_REG(vaddr, CMD_AES_MASK_OFFSET,
 505                                      CCP_READ_REG(vaddr, TRNG_OUT_REG));
 506                }
 507                CCP_WRITE_REG(vaddr, CMD_QUEUE_MASK_OFFSET, 0x0000001F);
 508                CCP_WRITE_REG(vaddr, CMD_QUEUE_PRIO_OFFSET, 0x00005B6D);
 509                CCP_WRITE_REG(vaddr, CMD_CMD_TIMEOUT_OFFSET, 0x00000000);
 510
 511                CCP_WRITE_REG(vaddr, LSB_PRIVATE_MASK_LO_OFFSET, 0x3FFFFFFF);
 512                CCP_WRITE_REG(vaddr, LSB_PRIVATE_MASK_HI_OFFSET, 0x000003FF);
 513
 514                CCP_WRITE_REG(vaddr, CMD_CLK_GATE_CTL_OFFSET, 0x00108823);
 515        }
 516        CCP_WRITE_REG(vaddr, CMD_REQID_CONFIG_OFFSET, 0x0);
 517
 518        /* Copy the private LSB mask to the public registers */
 519        status_lo = CCP_READ_REG(vaddr, LSB_PRIVATE_MASK_LO_OFFSET);
 520        status_hi = CCP_READ_REG(vaddr, LSB_PRIVATE_MASK_HI_OFFSET);
 521        CCP_WRITE_REG(vaddr, LSB_PUBLIC_MASK_LO_OFFSET, status_lo);
 522        CCP_WRITE_REG(vaddr, LSB_PUBLIC_MASK_HI_OFFSET, status_hi);
 523        status = ((uint64_t)status_hi<<30) | ((uint64_t)status_lo);
 524
 525        dev->cmd_q_count = 0;
 526        /* Find available queues */
 527        qmr = CCP_READ_REG(vaddr, Q_MASK_REG);
 528        for (i = 0; i < MAX_HW_QUEUES; i++) {
 529                if (!(qmr & (1 << i)))
 530                        continue;
 531                cmd_q = &dev->cmd_q[dev->cmd_q_count++];
 532                cmd_q->dev = dev;
 533                cmd_q->id = i;
 534                cmd_q->qidx = 0;
 535                cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
 536
 537                cmd_q->reg_base = (uint8_t *)vaddr +
 538                        CMD_Q_STATUS_INCR * (i + 1);
 539
 540                /* CCP queue memory */
 541                snprintf(cmd_q->memz_name, sizeof(cmd_q->memz_name),
 542                         "%s_%d_%s_%d_%s",
 543                         "ccp_dev",
 544                         (int)dev->id, "queue",
 545                         (int)cmd_q->id, "mem");
 546                q_mz = ccp_queue_dma_zone_reserve(cmd_q->memz_name,
 547                                                  cmd_q->qsize, SOCKET_ID_ANY);
 548                cmd_q->qbase_addr = (void *)q_mz->addr;
 549                cmd_q->qbase_desc = (void *)q_mz->addr;
 550                cmd_q->qbase_phys_addr =  q_mz->iova;
 551
 552                cmd_q->qcontrol = 0;
 553                /* init control reg to zero */
 554                CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
 555                              cmd_q->qcontrol);
 556
 557                /* Disable the interrupts */
 558                CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_INT_ENABLE_BASE, 0x00);
 559                CCP_READ_REG(cmd_q->reg_base, CMD_Q_INT_STATUS_BASE);
 560                CCP_READ_REG(cmd_q->reg_base, CMD_Q_STATUS_BASE);
 561
 562                /* Clear the interrupts */
 563                CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_INTERRUPT_STATUS_BASE,
 564                              ALL_INTERRUPTS);
 565
 566                /* Configure size of each virtual queue accessible to host */
 567                cmd_q->qcontrol &= ~(CMD_Q_SIZE << CMD_Q_SHIFT);
 568                cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD_Q_SHIFT;
 569
 570                dma_addr_lo = low32_value(cmd_q->qbase_phys_addr);
 571                CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE,
 572                              (uint32_t)dma_addr_lo);
 573                CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_HEAD_LO_BASE,
 574                              (uint32_t)dma_addr_lo);
 575
 576                dma_addr_hi = high32_value(cmd_q->qbase_phys_addr);
 577                cmd_q->qcontrol |= (dma_addr_hi << 16);
 578                CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
 579                              cmd_q->qcontrol);
 580
 581                /* create LSB Mask map */
 582                if (ccp_find_lsb_regions(cmd_q, status))
 583                        CCP_LOG_ERR("queue doesn't have lsb regions");
 584                cmd_q->lsb = -1;
 585
 586                rte_atomic64_init(&cmd_q->free_slots);
 587                rte_atomic64_set(&cmd_q->free_slots, (COMMANDS_PER_QUEUE - 1));
 588                /* unused slot barrier b/w H&T */
 589        }
 590
 591        if (ccp_assign_lsbs(dev))
 592                CCP_LOG_ERR("Unable to assign lsb region");
 593
 594        /* pre-allocate LSB slots */
 595        for (i = 0; i < dev->cmd_q_count; i++) {
 596                dev->cmd_q[i].sb_key =
 597                        ccp_lsb_alloc(&dev->cmd_q[i], 1);
 598                dev->cmd_q[i].sb_iv =
 599                        ccp_lsb_alloc(&dev->cmd_q[i], 1);
 600                dev->cmd_q[i].sb_sha =
 601                        ccp_lsb_alloc(&dev->cmd_q[i], 2);
 602                dev->cmd_q[i].sb_hmac =
 603                        ccp_lsb_alloc(&dev->cmd_q[i], 2);
 604        }
 605
 606        TAILQ_INSERT_TAIL(&ccp_list, dev, next);
 607        return 0;
 608}
 609
 610static void
 611ccp_remove_device(struct ccp_device *dev)
 612{
 613        if (dev == NULL)
 614                return;
 615
 616        TAILQ_REMOVE(&ccp_list, dev, next);
 617}
 618
 619static int
 620is_ccp_device(const char *dirname,
 621              const struct rte_pci_id *ccp_id,
 622              int *type)
 623{
 624        char filename[PATH_MAX];
 625        const struct rte_pci_id *id;
 626        uint16_t vendor, device_id;
 627        int i;
 628        unsigned long tmp;
 629
 630        /* get vendor id */
 631        snprintf(filename, sizeof(filename), "%s/vendor", dirname);
 632        if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 633                return 0;
 634        vendor = (uint16_t)tmp;
 635
 636        /* get device id */
 637        snprintf(filename, sizeof(filename), "%s/device", dirname);
 638        if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 639                return 0;
 640        device_id = (uint16_t)tmp;
 641
 642        for (id = ccp_id, i = 0; id->vendor_id != 0; id++, i++) {
 643                if (vendor == id->vendor_id &&
 644                    device_id == id->device_id) {
 645                        *type = i;
 646                        return 1; /* Matched device */
 647                }
 648        }
 649        return 0;
 650}
 651
 652static int
 653ccp_probe_device(int ccp_type, struct rte_pci_device *pci_dev)
 654{
 655        struct ccp_device *ccp_dev = NULL;
 656        int uio_fd = -1;
 657
 658        ccp_dev = rte_zmalloc("ccp_device", sizeof(*ccp_dev),
 659                              RTE_CACHE_LINE_SIZE);
 660        if (ccp_dev == NULL)
 661                goto fail;
 662
 663        ccp_dev->pci = *pci_dev;
 664
 665        /* device is valid, add in list */
 666        if (ccp_add_device(ccp_dev, ccp_type)) {
 667                ccp_remove_device(ccp_dev);
 668                goto fail;
 669        }
 670
 671        return 0;
 672fail:
 673        CCP_LOG_ERR("CCP Device probe failed");
 674        if (uio_fd >= 0)
 675                close(uio_fd);
 676        rte_free(ccp_dev);
 677        return -1;
 678}
 679
 680int
 681ccp_probe_devices(struct rte_pci_device *pci_dev,
 682                const struct rte_pci_id *ccp_id)
 683{
 684        int dev_cnt = 0;
 685        int ccp_type = 0;
 686        struct dirent *d;
 687        DIR *dir;
 688        int ret = 0;
 689        int module_idx = 0;
 690        uint16_t domain;
 691        uint8_t bus, devid, function;
 692        char dirname[PATH_MAX];
 693
 694        module_idx = ccp_check_pci_uio_module();
 695        if (module_idx < 0)
 696                return -1;
 697
 698        iommu_mode = module_idx;
 699        TAILQ_INIT(&ccp_list);
 700        dir = opendir(SYSFS_PCI_DEVICES);
 701        if (dir == NULL)
 702                return -1;
 703        while ((d = readdir(dir)) != NULL) {
 704                if (d->d_name[0] == '.')
 705                        continue;
 706                if (ccp_parse_pci_addr_format(d->d_name, sizeof(d->d_name),
 707                                        &domain, &bus, &devid, &function) != 0)
 708                        continue;
 709                snprintf(dirname, sizeof(dirname), "%s/%s",
 710                             SYSFS_PCI_DEVICES, d->d_name);
 711                if (is_ccp_device(dirname, ccp_id, &ccp_type)) {
 712                        printf("CCP : Detected CCP device with ID = 0x%x\n",
 713                               ccp_id[ccp_type].device_id);
 714                        ret = ccp_probe_device(ccp_type, pci_dev);
 715                        if (ret == 0)
 716                                dev_cnt++;
 717                }
 718        }
 719        closedir(dir);
 720        return dev_cnt;
 721}
 722