LXR dpdk/drivers/compress/isal/isal_compress_pmd

   1/* SPDX-License-Identifier: BSD-3-Clause
   2 * Copyright(c) 2018 Intel Corporation
   3 */
   4#include <isa-l.h>
   5
   6#include <rte_common.h>
   7#include <rte_compressdev_pmd.h>
   8#include <rte_malloc.h>
   9
  10#include "isal_compress_pmd_private.h"
  11
  12static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
  13        {
  14                .algo = RTE_COMP_ALGO_DEFLATE,
  15                .comp_feature_flags =   RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
  16                                        RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
  17                                        RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
  18                                        RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
  19                                        RTE_COMP_FF_HUFFMAN_FIXED |
  20                                        RTE_COMP_FF_HUFFMAN_DYNAMIC |
  21                                        RTE_COMP_FF_CRC32_CHECKSUM |
  22                                        RTE_COMP_FF_ADLER32_CHECKSUM,
  23                .window_size = {
  24                        .min = 15,
  25                        .max = 15,
  26                        .increment = 0
  27                },
  28        },
  29        RTE_COMP_END_OF_CAPABILITIES_LIST()
  30};
  31
  32/** Configure device */
  33static int
  34isal_comp_pmd_config(struct rte_compressdev *dev,
  35                struct rte_compressdev_config *config)
  36{
  37        int ret = 0;
  38        unsigned int n;
  39        char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN];
  40        unsigned int elt_size = sizeof(struct isal_priv_xform);
  41        struct isal_comp_private *internals = dev->data->dev_private;
  42
  43        n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp",
  44                        dev->data->dev_id);
  45        if (n > sizeof(mp_name)) {
  46                ISAL_PMD_LOG(ERR,
  47                        "Unable to create unique name for xform mempool");
  48                return -ENOMEM;
  49        }
  50
  51        internals->priv_xform_mp = rte_mempool_lookup(mp_name);
  52
  53        if (internals->priv_xform_mp != NULL) {
  54                if (((internals->priv_xform_mp)->elt_size != elt_size) ||
  55                                ((internals->priv_xform_mp)->size <
  56                                        config->max_nb_priv_xforms)) {
  57
  58                        ISAL_PMD_LOG(ERR, "%s mempool already exists with different"
  59                                " initialization parameters", mp_name);
  60                        internals->priv_xform_mp = NULL;
  61                        return -ENOMEM;
  62                }
  63        } else { /* First time configuration */
  64                internals->priv_xform_mp = rte_mempool_create(
  65                                mp_name, /* mempool name */
  66                                /* number of elements*/
  67                                config->max_nb_priv_xforms,
  68                                elt_size, /* element size*/
  69                                0, /* Cache size*/
  70                                0, /* private data size */
  71                                NULL, /* obj initialization constructor */
  72                                NULL, /* obj initialization constructor arg */
  73                                NULL, /**< obj constructor*/
  74                                NULL, /* obj constructor arg */
  75                                config->socket_id, /* socket id */
  76                                0); /* flags */
  77        }
  78
  79        if (internals->priv_xform_mp == NULL) {
  80                ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name);
  81                return -ENOMEM;
  82        }
  83
  84        dev->data->dev_private = internals;
  85
  86        return ret;
  87}
  88
  89/** Start device */
  90static int
  91isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev)
  92{
  93        return 0;
  94}
  95
  96/** Stop device */
  97static void
  98isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev)
  99{
 100}
 101
 102/** Close device */
 103static int
 104isal_comp_pmd_close(struct rte_compressdev *dev)
 105{
 106        /* Free private data */
 107        struct isal_comp_private *internals = dev->data->dev_private;
 108
 109        rte_mempool_free(internals->priv_xform_mp);
 110        return 0;
 111}
 112
 113/** Get device statistics */
 114static void
 115isal_comp_pmd_stats_get(struct rte_compressdev *dev,
 116                struct rte_compressdev_stats *stats)
 117{
 118        uint16_t qp_id;
 119
 120        for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 121                struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 122
 123                stats->enqueued_count += qp->qp_stats.enqueued_count;
 124                stats->dequeued_count += qp->qp_stats.dequeued_count;
 125
 126                stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
 127                stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
 128        }
 129}
 130
 131/** Get device info */
 132static void
 133isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused,
 134                struct rte_compressdev_info *dev_info)
 135{
 136        if (dev_info != NULL) {
 137                dev_info->capabilities = isal_pmd_capabilities;
 138
 139                /* Check CPU for supported vector instruction and set
 140                 * feature_flags
 141                 */
 142                if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
 143                        dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX512;
 144                else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
 145                        dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX2;
 146                else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
 147                        dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX;
 148                else
 149                        dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_SSE;
 150        }
 151}
 152
 153/** Reset device statistics */
 154static void
 155isal_comp_pmd_stats_reset(struct rte_compressdev *dev)
 156{
 157        uint16_t qp_id;
 158
 159        for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 160                struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 161                memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
 162        }
 163}
 164
 165/** Release queue pair */
 166static int
 167isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
 168{
 169        struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 170
 171        if (qp == NULL)
 172                return -EINVAL;
 173
 174        if (qp->stream)
 175                rte_free(qp->stream->level_buf);
 176
 177        rte_free(qp->state);
 178        rte_ring_free(qp->processed_pkts);
 179        rte_free(qp->stream);
 180        rte_free(qp);
 181        dev->data->queue_pairs[qp_id] = NULL;
 182
 183        return 0;
 184}
 185
 186/** Create a ring to place process packets on */
 187static struct rte_ring *
 188isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp,
 189                unsigned int ring_size, int socket_id)
 190{
 191        struct rte_ring *r;
 192
 193        r = rte_ring_lookup(qp->name);
 194        if (r) {
 195                if (rte_ring_get_size(r) >= ring_size) {
 196                        ISAL_PMD_LOG(DEBUG,
 197                                "Reusing existing ring %s for processed packets",
 198                                qp->name);
 199                        return r;
 200                }
 201
 202                        ISAL_PMD_LOG(ERR,
 203                                "Unable to reuse existing ring %s"
 204                                " for processed packets",
 205                         qp->name);
 206                return NULL;
 207        }
 208
 209        return rte_ring_create(qp->name, ring_size, socket_id,
 210                        RING_F_SP_ENQ | RING_F_SC_DEQ);
 211}
 212
 213/** set a unique name for the queue pair based on its name, dev_id and qp_id */
 214static int
 215isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev,
 216struct isal_comp_qp *qp)
 217{
 218        unsigned int n = snprintf(qp->name, sizeof(qp->name),
 219                        "isal_comp_pmd_%u_qp_%u",
 220                        dev->data->dev_id, qp->id);
 221
 222        if (n >= sizeof(qp->name))
 223                return -1;
 224
 225        return 0;
 226}
 227
 228/* Setup a queue pair */
 229static int
 230isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 231                uint32_t max_inflight_ops, int socket_id)
 232{
 233        struct isal_comp_qp *qp = NULL;
 234        int retval;
 235
 236        /* Free memory prior to re-allocation if needed. */
 237        if (dev->data->queue_pairs[qp_id] != NULL)
 238                isal_comp_pmd_qp_release(dev, qp_id);
 239
 240        /* Allocate the queue pair data structure. */
 241        qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp),
 242                                        RTE_CACHE_LINE_SIZE, socket_id);
 243        if (qp == NULL) {
 244                ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory");
 245                return (-ENOMEM);
 246        }
 247
 248        /* Initialize memory for compression stream structure */
 249        qp->stream = rte_zmalloc_socket("Isa-l compression stream ",
 250                        sizeof(struct isal_zstream),  RTE_CACHE_LINE_SIZE,
 251                        socket_id);
 252        if (qp->stream == NULL) {
 253                ISAL_PMD_LOG(ERR, "Failed to allocate compression stream memory");
 254                goto qp_setup_cleanup;
 255        }
 256        /* Initialize memory for compression level buffer */
 257        qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf",
 258                        ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE,
 259                        socket_id);
 260        if (qp->stream->level_buf == NULL) {
 261                ISAL_PMD_LOG(ERR, "Failed to allocate compression level_buf memory");
 262                goto qp_setup_cleanup;
 263        }
 264
 265        /* Initialize memory for decompression state structure */
 266        qp->state = rte_zmalloc_socket("Isa-l decompression state",
 267                        sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE,
 268                        socket_id);
 269        if (qp->state == NULL) {
 270                ISAL_PMD_LOG(ERR, "Failed to allocate decompression state memory");
 271                goto qp_setup_cleanup;
 272        }
 273
 274        qp->id = qp_id;
 275        dev->data->queue_pairs[qp_id] = qp;
 276
 277        retval = isal_comp_pmd_qp_set_unique_name(dev, qp);
 278        if (retval) {
 279                ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
 280                                "compression device");
 281                goto qp_setup_cleanup;
 282        }
 283
 284        qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp,
 285                        max_inflight_ops, socket_id);
 286        if (qp->processed_pkts == NULL) {
 287                ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
 288                                "compression device");
 289                goto qp_setup_cleanup;
 290        }
 291
 292        qp->num_free_elements = rte_ring_free_count(qp->processed_pkts);
 293
 294        memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
 295        return 0;
 296
 297qp_setup_cleanup:
 298        if (qp->stream)
 299                rte_free(qp->stream->level_buf);
 300        rte_free(qp->stream);
 301        rte_free(qp->state);
 302        rte_free(qp);
 303
 304        return -1;
 305}
 306
 307/** Set private xform data*/
 308static int
 309isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev,
 310                        const struct rte_comp_xform *xform, void **priv_xform)
 311{
 312        int ret;
 313        struct isal_comp_private *internals = dev->data->dev_private;
 314
 315        if (xform == NULL) {
 316                ISAL_PMD_LOG(ERR, "Invalid Xform struct");
 317                return -EINVAL;
 318        }
 319
 320        if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) {
 321                ISAL_PMD_LOG(ERR,
 322                        "Couldn't get object from private xform mempool");
 323                return -ENOMEM;
 324        }
 325
 326        ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform);
 327        if (ret != 0) {
 328                ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters");
 329
 330                /* Return private xform to mempool */
 331                rte_mempool_put(internals->priv_xform_mp, priv_xform);
 332                return ret;
 333        }
 334        return 0;
 335}
 336
 337/** Clear memory of the private xform so it doesn't leave key material behind */
 338static int
 339isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform)
 340{
 341        struct isal_comp_private *internals = dev->data->dev_private;
 342
 343        /* Zero out the whole structure */
 344        if (priv_xform) {
 345                memset(priv_xform, 0, sizeof(struct isal_priv_xform));
 346                rte_mempool_put(internals->priv_xform_mp, priv_xform);
 347        }
 348        return 0;
 349}
 350
 351struct rte_compressdev_ops isal_pmd_ops = {
 352                .dev_configure          = isal_comp_pmd_config,
 353                .dev_start              = isal_comp_pmd_start,
 354                .dev_stop               = isal_comp_pmd_stop,
 355                .dev_close              = isal_comp_pmd_close,
 356
 357                .stats_get              = isal_comp_pmd_stats_get,
 358                .stats_reset            = isal_comp_pmd_stats_reset,
 359
 360                .dev_infos_get          = isal_comp_pmd_info_get,
 361
 362                .queue_pair_setup       = isal_comp_pmd_qp_setup,
 363                .queue_pair_release     = isal_comp_pmd_qp_release,
 364
 365                .private_xform_create   = isal_comp_pmd_priv_xform_create,
 366                .private_xform_free     = isal_comp_pmd_priv_xform_free,
 367};
 368
 369struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops;
 370