LXR dpdk/drivers/net/mlx5/mlx5_flow

   1// SPDX-License-Identifier: BSD-3-Clause
   2/*
   3 * Copyright 2018 Mellanox Technologies, Ltd
   4 */
   5#include <math.h>
   6
   7#include <rte_tailq.h>
   8#include <rte_malloc.h>
   9#include <rte_mtr.h>
  10#include <rte_mtr_driver.h>
  11
  12#include <mlx5_devx_cmds.h>
  13#include <mlx5_malloc.h>
  14
  15#include "mlx5.h"
  16#include "mlx5_flow.h"
  17
  18/**
  19 * Create the meter action.
  20 *
  21 * @param priv
  22 *   Pointer to mlx5_priv.
  23 * @param[in] fm
  24 *   Pointer to flow meter to be converted.
  25 *
  26 * @return
  27 *   Pointer to the meter action on success, NULL otherwise.
  28 */
  29static void *
  30mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                              struct mlx5_flow_meter_info *fm)
  32{
  33#ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34        struct mlx5dv_dr_flow_meter_attr mtr_init;
  35        uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36        struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                     &fm->profile->srtcm_prm;
  38        uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39        uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40        uint32_t val;
  41        enum mlx5_meter_domain domain =
  42                fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                        fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                MLX5_MTR_DOMAIN_INGRESS;
  45        struct mlx5_flow_meter_def_policy *def_policy =
  46                priv->sh->mtrmng->def_policy[domain];
  47
  48        memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49        MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50        MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51        MLX5_SET(flow_meter_parameters, fmp,
  52                start_color, MLX5_FLOW_COLOR_GREEN);
  53        MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54        val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55        MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56        val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57        MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58        val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59        MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60        val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61        MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62        val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63        MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64        val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65        MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66        mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67        mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68        mtr_init.flow_meter_parameter = fmp;
  69        mtr_init.flow_meter_parameter_sz =
  70                MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71        mtr_init.active = fm->active_state;
  72        return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73#else
  74        (void)priv;
  75        (void)fm;
  76        return NULL;
  77#endif
  78}
  79
  80/**
  81 * Find meter profile by id.
  82 *
  83 * @param priv
  84 *   Pointer to mlx5_priv.
  85 * @param meter_profile_id
  86 *   Meter profile id.
  87 *
  88 * @return
  89 *   Pointer to the profile found on success, NULL otherwise.
  90 */
  91static struct mlx5_flow_meter_profile *
  92mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93{
  94        struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
  95        struct mlx5_flow_meter_profile *fmp;
  96
  97        TAILQ_FOREACH(fmp, fmps, next)
  98                if (meter_profile_id == fmp->id)
  99                        return fmp;
 100        return NULL;
 101}
 102
 103/**
 104 * Validate the MTR profile.
 105 *
 106 * @param[in] dev
 107 *   Pointer to Ethernet device.
 108 * @param[in] meter_profile_id
 109 *   Meter profile id.
 110 * @param[in] profile
 111 *   Pointer to meter profile detail.
 112 * @param[out] error
 113 *   Pointer to the error structure.
 114 *
 115 * @return
 116 *   0 on success, a negative errno value otherwise and rte_errno is set.
 117 */
 118static int
 119mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 120                                 uint32_t meter_profile_id,
 121                                 struct rte_mtr_meter_profile *profile,
 122                                 struct rte_mtr_error *error)
 123{
 124        struct mlx5_priv *priv = dev->data->dev_private;
 125        struct mlx5_flow_meter_profile *fmp;
 126
 127        /* Profile must not be NULL. */
 128        if (profile == NULL)
 129                return -rte_mtr_error_set(error, EINVAL,
 130                                          RTE_MTR_ERROR_TYPE_METER_PROFILE,
 131                                          NULL, "Meter profile is null.");
 132        /* Meter profile ID must be valid. */
 133        if (meter_profile_id == UINT32_MAX)
 134                return -rte_mtr_error_set(error, EINVAL,
 135                                          RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 136                                          NULL, "Meter profile id not valid.");
 137        /* Meter profile must not exist. */
 138        fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 139        if (fmp)
 140                return -rte_mtr_error_set(error, EEXIST,
 141                                          RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 142                                          NULL,
 143                                          "Meter profile already exists.");
 144        if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 145                if (priv->config.hca_attr.qos.flow_meter_old) {
 146                        /* Verify support for flow meter parameters. */
 147                        if (priv->sh->meter_aso_en && profile->packet_mode) {
 148                                if (profile->srtcm_rfc2697.cir > 0 &&
 149                                        (profile->srtcm_rfc2697.cir <<
 150                                        MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 151                                        <= MLX5_SRTCM_CIR_MAX &&
 152                                        profile->srtcm_rfc2697.cbs > 0 &&
 153                                        (profile->srtcm_rfc2697.cbs <<
 154                                        MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 155                                        <= MLX5_SRTCM_CBS_MAX &&
 156                                        (profile->srtcm_rfc2697.ebs <<
 157                                        MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                        <= MLX5_SRTCM_EBS_MAX)
 159                                        return 0;
 160                                return -rte_mtr_error_set
 161                                             (error, ENOTSUP,
 162                                              RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 163                                              NULL,
 164                                              profile->srtcm_rfc2697.ebs ?
 165                                              "Metering value ebs must be 0." :
 166                                              "Invalid metering parameters.");
 167                        }
 168                        if (profile->srtcm_rfc2697.cir > 0 &&
 169                                profile->srtcm_rfc2697.cir <=
 170                                                MLX5_SRTCM_CIR_MAX &&
 171                                profile->srtcm_rfc2697.cbs > 0 &&
 172                                profile->srtcm_rfc2697.cbs <=
 173                                                MLX5_SRTCM_CBS_MAX &&
 174                                profile->srtcm_rfc2697.ebs <=
 175                                                MLX5_SRTCM_EBS_MAX)
 176                                return 0;
 177                        return -rte_mtr_error_set(error, ENOTSUP,
 178                                        RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 179                                        NULL,
 180                                        profile->srtcm_rfc2697.ebs ?
 181                                        "Metering value ebs must be 0." :
 182                                        "Invalid metering parameters.");
 183                }
 184        }
 185        return -rte_mtr_error_set(error, ENOTSUP,
 186                                  RTE_MTR_ERROR_TYPE_METER_PROFILE,
 187                                  NULL, "Metering algorithm not supported.");
 188}
 189
 190/**
 191 * Calculate mantissa and exponent for cir.
 192 *
 193 * @param[in] cir
 194 *   Value to be calculated.
 195 * @param[out] man
 196 *   Pointer to the mantissa.
 197 * @param[out] exp
 198 *   Pointer to the exp.
 199 */
 200static void
 201mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 202{
 203        int64_t _cir;
 204        int64_t delta = INT64_MAX;
 205        uint8_t _man = 0;
 206        uint8_t _exp = 0;
 207        uint64_t m, e;
 208
 209        for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 210                for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 211                        _cir = (1000000000ULL * m) >> e;
 212                        if (llabs(cir - _cir) <= delta) {
 213                                delta = llabs(cir - _cir);
 214                                _man = m;
 215                                _exp = e;
 216                        }
 217                }
 218        }
 219        *man = _man;
 220        *exp = _exp;
 221}
 222
 223/**
 224 * Calculate mantissa and exponent for xbs.
 225 *
 226 * @param[in] xbs
 227 *   Value to be calculated.
 228 * @param[out] man
 229 *   Pointer to the mantissa.
 230 * @param[out] exp
 231 *   Pointer to the exp.
 232 */
 233static void
 234mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 235{
 236        int _exp;
 237        double _man;
 238
 239        /* Special case xbs == 0 ? both exp and matissa are 0. */
 240        if (xbs == 0) {
 241                *man = 0;
 242                *exp = 0;
 243                return;
 244        }
 245        /* xbs = xbs_mantissa * 2^xbs_exponent */
 246        _man = frexp(xbs, &_exp);
 247        _man = _man * pow(2, MLX5_MAN_WIDTH);
 248        _exp = _exp - MLX5_MAN_WIDTH;
 249        *man = (uint8_t)ceil(_man);
 250        *exp = _exp;
 251}
 252
 253/**
 254 * Fill the prm meter parameter.
 255 *
 256 * @param[in,out] fmp
 257 *   Pointer to meter profie to be converted.
 258 * @param[out] error
 259 *   Pointer to the error structure.
 260 *
 261 * @return
 262 *   0 on success, a negative errno value otherwise and rte_errno is set.
 263 */
 264static int
 265mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 266                        struct mlx5_priv *priv, struct rte_mtr_error *error)
 267{
 268        struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 269        uint8_t man, exp;
 270        uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 271        uint32_t ebs_exp, ebs_man;
 272        uint64_t cir, cbs, ebs;
 273
 274        if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 275                return -rte_mtr_error_set(error, ENOTSUP,
 276                                RTE_MTR_ERROR_TYPE_METER_PROFILE,
 277                                NULL, "Metering algorithm not supported.");
 278        if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
 279                return -rte_mtr_error_set(error, ENOTSUP,
 280                        RTE_MTR_ERROR_TYPE_METER_PROFILE,
 281                        NULL, "Metering algorithm packet mode not supported.");
 282        if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
 283                cir = fmp->profile.srtcm_rfc2697.cir <<
 284                                MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 285                cbs = fmp->profile.srtcm_rfc2697.cbs <<
 286                                MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 287                ebs = fmp->profile.srtcm_rfc2697.ebs <<
 288                                MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 289        } else {
 290                cir = fmp->profile.srtcm_rfc2697.cir;
 291                cbs = fmp->profile.srtcm_rfc2697.cbs;
 292                ebs = fmp->profile.srtcm_rfc2697.ebs;
 293        }
 294        /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 295        mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
 296        /* Check if cir mantissa is too large. */
 297        if (exp > ASO_DSEG_CIR_EXP_MASK)
 298                return -rte_mtr_error_set(error, ENOTSUP,
 299                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 300                                          "meter profile parameter cir is"
 301                                          " not supported.");
 302        cir_man = man;
 303        cir_exp = exp;
 304         /* cbs = cbs_mantissa * 2^cbs_exponent */
 305        mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 306        /* Check if cbs mantissa is too large. */
 307        if (exp > ASO_DSEG_EXP_MASK)
 308                return -rte_mtr_error_set(error, ENOTSUP,
 309                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 310                                          "meter profile parameter cbs is"
 311                                          " not supported.");
 312        cbs_man = man;
 313        cbs_exp = exp;
 314        srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 315                                cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 316                                cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 317                                cir_man);
 318        mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 319        /* Check if ebs mantissa is too large. */
 320        if (exp > ASO_DSEG_EXP_MASK)
 321                return -rte_mtr_error_set(error, ENOTSUP,
 322                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 323                                          "meter profile parameter ebs is"
 324                                          " not supported.");
 325        ebs_man = man;
 326        ebs_exp = exp;
 327        srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 328                                        ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 329        return 0;
 330}
 331
 332/**
 333 * Callback to get MTR capabilities.
 334 *
 335 * @param[in] dev
 336 *   Pointer to Ethernet device.
 337 * @param[out] cap
 338 *   Pointer to save MTR capabilities.
 339 * @param[out] error
 340 *   Pointer to the error structure.
 341 *
 342 * @return
 343 *   0 on success, a negative errno value otherwise and rte_errno is set.
 344 */
 345static int
 346mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 347                 struct rte_mtr_capabilities *cap,
 348                 struct rte_mtr_error *error __rte_unused)
 349{
 350        struct mlx5_priv *priv = dev->data->dev_private;
 351        struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 352
 353        if (!priv->mtr_en)
 354                return -rte_mtr_error_set(error, ENOTSUP,
 355                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 356                                          "Meter is not supported");
 357        memset(cap, 0, sizeof(*cap));
 358        if (priv->sh->meter_aso_en) {
 359                /* 2 meters per one ASO cache line. */
 360                cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 361                cap->srtcm_rfc2697_packet_mode_supported = 1;
 362        } else {
 363                cap->n_max = 1 << qattr->log_max_flow_meter;
 364                cap->srtcm_rfc2697_packet_mode_supported = 0;
 365        }
 366        cap->srtcm_rfc2697_byte_mode_supported = 1;
 367        cap->n_shared_max = cap->n_max;
 368        cap->identical = 1;
 369        cap->shared_identical = 1;
 370        cap->shared_n_flows_per_mtr_max = 4 << 20;
 371        /* 2M flows can share the same meter. */
 372        cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 373        cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 374        cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 375        cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 376                          RTE_MTR_STATS_N_PKTS_DROPPED;
 377        return 0;
 378}
 379
 380/**
 381 * Callback to add MTR profile.
 382 *
 383 * @param[in] dev
 384 *   Pointer to Ethernet device.
 385 * @param[in] meter_profile_id
 386 *   Meter profile id.
 387 * @param[in] profile
 388 *   Pointer to meter profile detail.
 389 * @param[out] error
 390 *   Pointer to the error structure.
 391 *
 392 * @return
 393 *   0 on success, a negative errno value otherwise and rte_errno is set.
 394 */
 395static int
 396mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 397                       uint32_t meter_profile_id,
 398                       struct rte_mtr_meter_profile *profile,
 399                       struct rte_mtr_error *error)
 400{
 401        struct mlx5_priv *priv = dev->data->dev_private;
 402        struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
 403        struct mlx5_flow_meter_profile *fmp;
 404        int ret;
 405
 406        if (!priv->mtr_en)
 407                return -rte_mtr_error_set(error, ENOTSUP,
 408                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 409                                          "Meter is not supported");
 410        /* Check input params. */
 411        ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 412                                               profile, error);
 413        if (ret)
 414                return ret;
 415        /* Meter profile memory allocation. */
 416        fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 417                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 418        if (fmp == NULL)
 419                return -rte_mtr_error_set(error, ENOMEM,
 420                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 421                                          NULL, "Meter profile memory "
 422                                          "alloc failed.");
 423        /* Fill profile info. */
 424        fmp->id = meter_profile_id;
 425        fmp->profile = *profile;
 426        /* Fill the flow meter parameters for the PRM. */
 427        ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 428        if (ret)
 429                goto error;
 430        /* Add to list. */
 431        TAILQ_INSERT_TAIL(fmps, fmp, next);
 432        return 0;
 433error:
 434        mlx5_free(fmp);
 435        return ret;
 436}
 437
 438/**
 439 * Callback to delete MTR profile.
 440 *
 441 * @param[in] dev
 442 *   Pointer to Ethernet device.
 443 * @param[in] meter_profile_id
 444 *   Meter profile id.
 445 * @param[out] error
 446 *   Pointer to the error structure.
 447 *
 448 * @return
 449 *   0 on success, a negative errno value otherwise and rte_errno is set.
 450 */
 451static int
 452mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 453                          uint32_t meter_profile_id,
 454                          struct rte_mtr_error *error)
 455{
 456        struct mlx5_priv *priv = dev->data->dev_private;
 457        struct mlx5_flow_meter_profile *fmp;
 458
 459        if (!priv->mtr_en)
 460                return -rte_mtr_error_set(error, ENOTSUP,
 461                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 462                                          "Meter is not supported");
 463        /* Meter profile must exist. */
 464        fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 465        if (fmp == NULL)
 466                return -rte_mtr_error_set(error, ENOENT,
 467                                          RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 468                                          &meter_profile_id,
 469                                          "Meter profile id is invalid.");
 470        /* Check profile is unused. */
 471        if (fmp->ref_cnt)
 472                return -rte_mtr_error_set(error, EBUSY,
 473                                          RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 474                                          NULL, "Meter profile is in use.");
 475        /* Remove from list. */
 476        TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
 477        mlx5_free(fmp);
 478        return 0;
 479}
 480
 481/**
 482 * Find policy by id.
 483 *
 484 * @param[in] dev
 485 *   Pointer to Ethernet device.
 486 * @param policy_id
 487 *   Policy id.
 488 *
 489 * @return
 490 *   Pointer to the policy found on success, NULL otherwise.
 491 */
 492struct mlx5_flow_meter_policy *
 493mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 494                            uint32_t policy_id,
 495                            uint32_t *policy_idx)
 496{
 497        struct mlx5_priv *priv = dev->data->dev_private;
 498        struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 499        union mlx5_l3t_data data;
 500
 501        if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM ||
 502                !priv->sh->mtrmng->policy_idx_tbl)
 503                return NULL;
 504        if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl,
 505                                policy_id, &data) ||
 506                                !data.dword)
 507                return NULL;
 508        if (policy_idx)
 509                *policy_idx = data.dword;
 510        sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 511                                        data.dword);
 512        /* Remove reference taken by the mlx5_l3t_get_entry. */
 513        mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 514                                policy_id);
 515        if (sub_policy)
 516                if (sub_policy->main_policy_id)
 517                        return sub_policy->main_policy;
 518        return NULL;
 519}
 520
 521/**
 522 * Callback to check MTR policy action validate
 523 *
 524 * @param[in] dev
 525 *   Pointer to Ethernet device.
 526 * @param[in] actions
 527 *   Pointer to meter policy action detail.
 528 * @param[out] error
 529 *   Pointer to the error structure.
 530 *
 531 * @return
 532 *   0 on success, a negative errno value otherwise and rte_errno is set.
 533 */
 534static int
 535mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 536        struct rte_mtr_meter_policy_params *policy,
 537        struct rte_mtr_error *error)
 538{
 539        struct mlx5_priv *priv = dev->data->dev_private;
 540        struct rte_flow_attr attr = { .transfer =
 541                        priv->config.dv_esw_en ? 1 : 0};
 542        bool is_rss = false;
 543        bool is_def_policy = false;
 544        uint8_t domain_bitmap;
 545        int ret;
 546
 547        if (!priv->mtr_en || !priv->sh->meter_aso_en)
 548                return -rte_mtr_error_set(error, ENOTSUP,
 549                                RTE_MTR_ERROR_TYPE_METER_POLICY,
 550                                NULL, "meter policy unsupported.");
 551        ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 552                        &is_rss, &domain_bitmap, &is_def_policy, error);
 553        if (ret)
 554                return ret;
 555        return 0;
 556}
 557
 558static int
 559__mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 560                        uint32_t policy_id,
 561                        struct mlx5_flow_meter_policy *mtr_policy,
 562                        struct rte_mtr_error *error)
 563{
 564        struct mlx5_priv *priv = dev->data->dev_private;
 565        struct mlx5_flow_meter_sub_policy *sub_policy;
 566        uint32_t i, j;
 567        uint16_t sub_policy_num;
 568
 569        rte_spinlock_lock(&mtr_policy->sl);
 570        if (mtr_policy->ref_cnt) {
 571                rte_spinlock_unlock(&mtr_policy->sl);
 572                return -rte_mtr_error_set(error, EBUSY,
 573                                RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 574                                 NULL,
 575                                "Meter policy object is being used.");
 576        }
 577        mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 578        mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 579        for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 580                sub_policy_num = (mtr_policy->sub_policy_num >>
 581                        (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 582                        MLX5_MTR_SUB_POLICY_NUM_MASK;
 583                if (sub_policy_num) {
 584                        for (j = 0; j < sub_policy_num; j++) {
 585                                sub_policy = mtr_policy->sub_policys[i][j];
 586                                if (sub_policy)
 587                                        mlx5_ipool_free
 588                                        (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 589                                        sub_policy->idx);
 590                        }
 591                }
 592        }
 593        if (priv->sh->mtrmng->policy_idx_tbl) {
 594                if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 595                                        policy_id)) {
 596                        rte_spinlock_unlock(&mtr_policy->sl);
 597                        return -rte_mtr_error_set(error, ENOTSUP,
 598                                RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 599                                "Fail to delete policy in index table.");
 600                }
 601        }
 602        rte_spinlock_unlock(&mtr_policy->sl);
 603        return 0;
 604}
 605
 606/**
 607 * Callback to add MTR policy.
 608 *
 609 * @param[in] dev
 610 *   Pointer to Ethernet device.
 611 * @param[out] policy_id
 612 *   Pointer to policy id
 613 * @param[in] actions
 614 *   Pointer to meter policy action detail.
 615 * @param[out] error
 616 *   Pointer to the error structure.
 617 *
 618 * @return
 619 *   0 on success, a negative errno value otherwise and rte_errno is set.
 620 */
 621static int
 622mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 623                        uint32_t policy_id,
 624                        struct rte_mtr_meter_policy_params *policy,
 625                        struct rte_mtr_error *error)
 626{
 627        struct mlx5_priv *priv = dev->data->dev_private;
 628        struct rte_flow_attr attr = { .transfer =
 629                        priv->config.dv_esw_en ? 1 : 0};
 630        uint32_t sub_policy_idx = 0;
 631        uint32_t policy_idx = 0;
 632        struct mlx5_flow_meter_policy *mtr_policy = NULL;
 633        struct mlx5_flow_meter_sub_policy *sub_policy;
 634        bool is_rss = false;
 635        bool is_def_policy = false;
 636        uint32_t i;
 637        int ret;
 638        uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 639        uint16_t sub_policy_num;
 640        uint8_t domain_bitmap = 0;
 641        union mlx5_l3t_data data;
 642
 643        if (!priv->mtr_en)
 644                return -rte_mtr_error_set(error, ENOTSUP,
 645                                          RTE_MTR_ERROR_TYPE_METER_POLICY,
 646                                          NULL, "meter policy unsupported.");
 647        if (policy_id == MLX5_INVALID_POLICY_ID)
 648                return -rte_mtr_error_set(error, ENOTSUP,
 649                        RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 650                        "policy ID is invalid. ");
 651        if (policy_id == priv->sh->mtrmng->def_policy_id)
 652                return -rte_mtr_error_set(error, EEXIST,
 653                        RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 654                        "policy ID exists. ");
 655        mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id,
 656                                &policy_idx);
 657        if (mtr_policy)
 658                return -rte_mtr_error_set(error, EEXIST,
 659                        RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 660                        "policy ID exists. ");
 661        ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 662                        &is_rss, &domain_bitmap, &is_def_policy, error);
 663        if (ret)
 664                return ret;
 665        if (!domain_bitmap)
 666                return -rte_mtr_error_set(error, ENOTSUP,
 667                                RTE_MTR_ERROR_TYPE_METER_POLICY,
 668                                NULL, "fail to find policy domain.");
 669        if (is_def_policy) {
 670                if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 671                        return -rte_mtr_error_set(error, EEXIST,
 672                                RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 673                                NULL, "a policy with similar actions "
 674                                "is already configured");
 675                if (mlx5_flow_create_def_policy(dev))
 676                        return -rte_mtr_error_set(error, ENOTSUP,
 677                                RTE_MTR_ERROR_TYPE_METER_POLICY,
 678                                NULL,
 679                                "fail to create non-terminated policy.");
 680                priv->sh->mtrmng->def_policy_id = policy_id;
 681                return 0;
 682        }
 683        if (!priv->sh->meter_aso_en)
 684                return -rte_mtr_error_set(error, ENOTSUP,
 685                        RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 686                        "no ASO capability to support the policy ");
 687        for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 688                if (!(domain_bitmap & (1 << i)))
 689                        continue;
 690                if (is_rss) {
 691                        policy_size +=
 692                        sizeof(struct mlx5_flow_meter_sub_policy *) *
 693                        MLX5_MTR_RSS_MAX_SUB_POLICY;
 694                        break;
 695                }
 696                policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 697        }
 698        mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 699                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 700        if (!mtr_policy)
 701                return -rte_mtr_error_set(error, ENOMEM,
 702                                RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 703                                "Memory alloc failed for meter policy.");
 704        policy_size = sizeof(struct mlx5_flow_meter_policy);
 705        for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 706                if (!(domain_bitmap & (1 << i)))
 707                        continue;
 708                if (i == MLX5_MTR_DOMAIN_INGRESS)
 709                        mtr_policy->ingress = 1;
 710                if (i == MLX5_MTR_DOMAIN_EGRESS)
 711                        mtr_policy->egress = 1;
 712                if (i == MLX5_MTR_DOMAIN_TRANSFER)
 713                        mtr_policy->transfer = 1;
 714                sub_policy = mlx5_ipool_zmalloc
 715                                (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 716                                &sub_policy_idx);
 717                if (!sub_policy)
 718                        goto policy_add_err;
 719                if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 720                        goto policy_add_err;
 721                sub_policy->idx = sub_policy_idx;
 722                sub_policy->main_policy = mtr_policy;
 723                if (!policy_idx) {
 724                        policy_idx = sub_policy_idx;
 725                        sub_policy->main_policy_id = 1;
 726                }
 727                mtr_policy->sub_policys[i] =
 728                (struct mlx5_flow_meter_sub_policy **)
 729                        ((uint8_t *)mtr_policy + policy_size);
 730                mtr_policy->sub_policys[i][0] = sub_policy;
 731                sub_policy_num = (mtr_policy->sub_policy_num >>
 732                        (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 733                        MLX5_MTR_SUB_POLICY_NUM_MASK;
 734                sub_policy_num++;
 735                mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 736                        (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 737                mtr_policy->sub_policy_num |=
 738                        (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 739                        (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 740                if (is_rss) {
 741                        mtr_policy->is_rss = 1;
 742                        break;
 743                }
 744                policy_size += sizeof(struct mlx5_flow_meter_sub_policy  *);
 745        }
 746        rte_spinlock_init(&mtr_policy->sl);
 747        ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 748                                        policy->actions, error);
 749        if (ret)
 750                goto policy_add_err;
 751        if (!is_rss && !mtr_policy->is_queue) {
 752                /* Create policy rules in HW. */
 753                ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 754                if (ret)
 755                        goto policy_add_err;
 756        }
 757        data.dword = policy_idx;
 758        if (!priv->sh->mtrmng->policy_idx_tbl) {
 759                priv->sh->mtrmng->policy_idx_tbl =
 760                                mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 761                if (!priv->sh->mtrmng->policy_idx_tbl)
 762                        goto policy_add_err;
 763        }
 764        if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl,
 765                                policy_id, &data))
 766                goto policy_add_err;
 767        return 0;
 768policy_add_err:
 769        if (mtr_policy) {
 770                ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 771                        mtr_policy, error);
 772                mlx5_free(mtr_policy);
 773                if (ret)
 774                        return ret;
 775        }
 776        return -rte_mtr_error_set(error, ENOTSUP,
 777                                  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 778                                  NULL, "Failed to create devx policy.");
 779}
 780
 781/**
 782 * Callback to delete MTR policy.
 783 *
 784 * @param[in] dev
 785 *   Pointer to Ethernet device.
 786 * @param[in] policy_id
 787 *   Meter policy id.
 788 * @param[out] error
 789 *   Pointer to the error structure.
 790 *
 791 * @return
 792 *   0 on success, a negative errno value otherwise and rte_errno is set.
 793 */
 794static int
 795mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 796                          uint32_t policy_id,
 797                          struct rte_mtr_error *error)
 798{
 799        struct mlx5_priv *priv = dev->data->dev_private;
 800        struct mlx5_flow_meter_policy *mtr_policy;
 801        uint32_t policy_idx;
 802        int ret;
 803
 804        if (policy_id == priv->sh->mtrmng->def_policy_id) {
 805                if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 806                        return -rte_mtr_error_set(error, ENOTSUP,
 807                                RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 808                                "Meter policy object is being used.");
 809                priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 810                return 0;
 811        }
 812        mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 813        if (!mtr_policy)
 814                return -rte_mtr_error_set(error, ENOTSUP,
 815                        RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 816                        "Meter policy id is invalid. ");
 817        ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 818                                                error);
 819        if (ret)
 820                return ret;
 821        mlx5_free(mtr_policy);
 822        return 0;
 823}
 824
 825/**
 826 * Check meter validation.
 827 *
 828 * @param[in] priv
 829 *   Pointer to mlx5 private data structure.
 830 * @param[in] meter_id
 831 *   Meter id.
 832 * @param[in] params
 833 *   Pointer to rte meter parameters.
 834 * @param[out] error
 835 *   Pointer to rte meter error structure.
 836 *
 837 * @return
 838 *   0 on success, a negative errno value otherwise and rte_errno is set.
 839 */
 840static int
 841mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 842                         struct rte_mtr_params *params,
 843                         struct rte_mtr_error *error)
 844{
 845        /* Meter must use global drop action. */
 846        if (!priv->sh->dr_drop_action)
 847                return -rte_mtr_error_set(error, ENOTSUP,
 848                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 849                                          NULL,
 850                                          "No drop action ready for meter.");
 851        /* Meter params must not be NULL. */
 852        if (params == NULL)
 853                return -rte_mtr_error_set(error, EINVAL,
 854                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 855                                          NULL, "Meter object params null.");
 856        /* Previous meter color is not supported. */
 857        if (params->use_prev_mtr_color)
 858                return -rte_mtr_error_set(error, ENOTSUP,
 859                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 860                                          NULL,
 861                                          "Previous meter color "
 862                                          "not supported.");
 863        if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 864                return -rte_mtr_error_set(error, ENOENT,
 865                                RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 866                                NULL, "Meter policy id not valid.");
 867        /* Validate meter id. */
 868        if (mlx5_flow_meter_find(priv, meter_id, NULL))
 869                return -rte_mtr_error_set(error, EEXIST,
 870                        RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 871                        "Meter object already exists.");
 872        return 0;
 873}
 874
 875/**
 876 * Modify the flow meter action.
 877 *
 878 * @param[in] priv
 879 *   Pointer to mlx5 private data structure.
 880 * @param[in] fm
 881 *   Pointer to flow meter to be modified.
 882 * @param[in] srtcm
 883 *   Pointer to meter srtcm description parameter.
 884 * @param[in] modify_bits
 885 *   The bit in srtcm to be updated.
 886 * @param[in] active_state
 887 *   The state to be updated.
 888 * @return
 889 *   0 on success, o negative value otherwise.
 890 */
 891static int
 892mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 893                struct mlx5_flow_meter_info *fm,
 894                const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 895                uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 896{
 897#ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 898        uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 899        uint32_t *attr;
 900        struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 901        int ret;
 902        struct mlx5_aso_mtr *aso_mtr = NULL;
 903        uint32_t cbs_cir, ebs_eir, val;
 904
 905        if (priv->sh->meter_aso_en) {
 906                fm->is_enable = !!is_enable;
 907                aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 908                ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 909                if (ret)
 910                        return ret;
 911                ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 912                if (ret)
 913                        return ret;
 914        } else {
 915                /* Fill command parameters. */
 916                mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 917                mod_attr.flow_meter_parameter = in;
 918                mod_attr.flow_meter_parameter_sz =
 919                                MLX5_ST_SZ_BYTES(flow_meter_parameters);
 920                if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 921                        mod_attr.active = !!active_state;
 922                else
 923                        mod_attr.active = 0;
 924                attr = in;
 925                cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 926                ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 927                if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 928                        val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 929                                ASO_DSEG_EXP_MASK;
 930                        MLX5_SET(flow_meter_parameters, attr,
 931                                cbs_exponent, val);
 932                        val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 933                                ASO_DSEG_MAN_MASK;
 934                        MLX5_SET(flow_meter_parameters, attr,
 935                                cbs_mantissa, val);
 936                }
 937                if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 938                        val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
 939                                ASO_DSEG_EXP_MASK;
 940                        MLX5_SET(flow_meter_parameters, attr,
 941                                cir_exponent, val);
 942                        val = cbs_cir & ASO_DSEG_MAN_MASK;
 943                        MLX5_SET(flow_meter_parameters, attr,
 944                                cir_mantissa, val);
 945                }
 946                if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 947                        val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
 948                                ASO_DSEG_EXP_MASK;
 949                        MLX5_SET(flow_meter_parameters, attr,
 950                                ebs_exponent, val);
 951                        val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
 952                                ASO_DSEG_MAN_MASK;
 953                        MLX5_SET(flow_meter_parameters, attr,
 954                                ebs_mantissa, val);
 955                }
 956                /* Apply modifications to meter only if it was created. */
 957                if (fm->meter_action) {
 958                        ret = mlx5_glue->dv_modify_flow_action_meter
 959                                        (fm->meter_action, &mod_attr,
 960                                        rte_cpu_to_be_64(modify_bits));
 961                        if (ret)
 962                                return ret;
 963                }
 964                /* Update succeedded modify meter parameters. */
 965                if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 966                        fm->active_state = !!active_state;
 967        }
 968        return 0;
 969#else
 970        (void)priv;
 971        (void)fm;
 972        (void)srtcm;
 973        (void)modify_bits;
 974        (void)active_state;
 975        (void)is_enable;
 976        return -ENOTSUP;
 977#endif
 978}
 979
 980static int
 981mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
 982                                struct mlx5_flow_meter_info *fm,
 983                                uint64_t stats_mask)
 984{
 985        fm->bytes_dropped =
 986                (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
 987        fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
 988        if (fm->bytes_dropped || fm->pkts_dropped) {
 989                if (!fm->drop_cnt) {
 990                        /* Alloc policer counters. */
 991                        fm->drop_cnt = mlx5_counter_alloc(dev);
 992                        if (!fm->drop_cnt)
 993                                return -1;
 994                }
 995        } else {
 996                if (fm->drop_cnt) {
 997                        mlx5_counter_free(dev, fm->drop_cnt);
 998                        fm->drop_cnt = 0;
 999                }
1000        }

1001        return 0;
1002}
1003
1004/**
1005 * Create meter rules.
1006 *
1007 * @param[in] dev
1008 *   Pointer to Ethernet device.
1009 * @param[in] meter_id
1010 *   Meter id.
1011 * @param[in] params
1012 *   Pointer to rte meter parameters.
1013 * @param[in] shared
1014 *   Meter shared with other flow or not.
1015 * @param[out] error
1016 *   Pointer to rte meter error structure.
1017 *
1018 * @return
1019 *   0 on success, a negative errno value otherwise and rte_errno is set.
1020 */
1021static int
1022mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1023                       struct rte_mtr_params *params, int shared,
1024                       struct rte_mtr_error *error)
1025{
1026        struct mlx5_priv *priv = dev->data->dev_private;
1027        struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1028        struct mlx5_flow_meter_profile *fmp;
1029        struct mlx5_flow_meter_info *fm;
1030        struct mlx5_legacy_flow_meter *legacy_fm;
1031        struct mlx5_flow_meter_policy *mtr_policy = NULL;
1032        struct mlx5_indexed_pool_config flow_ipool_cfg = {
1033                .size = 0,
1034                .trunk_size = 64,
1035                .need_lock = 1,
1036                .type = "mlx5_flow_mtr_flow_id_pool",
1037        };
1038        struct mlx5_aso_mtr *aso_mtr;
1039        uint32_t mtr_idx, policy_idx;
1040        union mlx5_l3t_data data;
1041        int ret;
1042        uint8_t domain_bitmap;
1043        uint8_t mtr_id_bits;
1044        uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1045                                MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1046
1047        if (!priv->mtr_en)
1048                return -rte_mtr_error_set(error, ENOTSUP,
1049                                        RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1050                                        "Meter is not supported");
1051        /* Validate the parameters. */
1052        ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1053        if (ret)
1054                return ret;
1055        /* Meter profile must exist. */
1056        fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1057        if (fmp == NULL)
1058                return -rte_mtr_error_set(error, ENOENT,
1059                        RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1060                        NULL, "Meter profile id not valid.");
1061        /* Meter policy must exist. */
1062        if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1063                __atomic_add_fetch
1064                        (&priv->sh->mtrmng->def_policy_ref_cnt,
1065                        1, __ATOMIC_RELAXED);
1066                domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1067                if (!priv->config.dv_esw_en)
1068                        domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1069        } else {
1070                mtr_policy = mlx5_flow_meter_policy_find(dev,
1071                                params->meter_policy_id, &policy_idx);
1072                if (!priv->sh->meter_aso_en)
1073                        return -rte_mtr_error_set(error, ENOTSUP,
1074                                RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1075                                "Part of the policies cannot be "
1076                                "supported without ASO ");
1077                if (!mtr_policy)
1078                        return -rte_mtr_error_set(error, ENOENT,
1079                                RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1080                                NULL, "Meter policy id not valid.");
1081                domain_bitmap = (mtr_policy->ingress ?
1082                                        MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1083                                (mtr_policy->egress ?
1084                                        MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1085                                (mtr_policy->transfer ?
1086                                        MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1087        }
1088        /* Allocate the flow meter memory. */
1089        if (priv->sh->meter_aso_en) {
1090                mtr_idx = mlx5_flow_mtr_alloc(dev);
1091                if (!mtr_idx)
1092                        return -rte_mtr_error_set(error, ENOMEM,
1093                                RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1094                                "Memory alloc failed for meter.");
1095                aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1096                fm = &aso_mtr->fm;
1097        } else {
1098                legacy_fm = mlx5_ipool_zmalloc
1099                                (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1100                if (legacy_fm == NULL)
1101                        return -rte_mtr_error_set(error, ENOMEM,
1102                                RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1103                                "Memory alloc failed for meter.");
1104                legacy_fm->idx = mtr_idx;
1105                fm = &legacy_fm->fm;
1106        }
1107        mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1108        if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1109            mtr_reg_bits) {
1110                DRV_LOG(ERR, "Meter number exceeds max limit.");
1111                goto error;
1112        }
1113        if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1114                priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1115        /* Fill the flow meter parameters. */
1116        fm->meter_id = meter_id;
1117        fm->policy_id = params->meter_policy_id;
1118        fm->profile = fmp;
1119        if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1120                goto error;
1121        if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1122                goto error;
1123        /* Add to the flow meter list. */
1124        if (!priv->sh->meter_aso_en)
1125                TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1126        /* Add to the flow meter list. */
1127        fm->active_state = 1; /* Config meter starts as active. */
1128        fm->is_enable = 1;
1129        fm->shared = !!shared;
1130        __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1131        if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1132                fm->def_policy = 1;
1133                fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1134                if (!fm->flow_ipool)
1135                        goto error;
1136        }
1137        rte_spinlock_init(&fm->sl);
1138        /* If ASO meter supported, update ASO flow meter by wqe. */
1139        if (priv->sh->meter_aso_en) {
1140                aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1141                ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1142                if (ret)
1143                        goto error;
1144                if (!priv->mtr_idx_tbl) {
1145                        priv->mtr_idx_tbl =
1146                                mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1147                        if (!priv->mtr_idx_tbl)
1148                                goto error;
1149                }
1150                data.dword = mtr_idx;
1151                if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1152                        goto error;
1153        }
1154        if (mtr_policy)
1155                __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1156        return 0;
1157error:
1158        mlx5_flow_destroy_mtr_tbls(dev, fm);
1159        /* Free policer counters. */
1160        if (fm->drop_cnt)
1161                mlx5_counter_free(dev, fm->drop_cnt);
1162        if (priv->sh->meter_aso_en)
1163                mlx5_flow_mtr_free(dev, mtr_idx);
1164        else
1165                mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1166        return -rte_mtr_error_set(error, ENOTSUP,
1167                RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1168                NULL, "Failed to create devx meter.");
1169}
1170
1171static int
1172mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1173                        struct mlx5_flow_meter_info *fm,
1174                        uint32_t mtr_idx)
1175{
1176        struct mlx5_priv *priv = dev->data->dev_private;
1177        struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1178        struct mlx5_flow_meter_profile *fmp;
1179        struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1180        struct mlx5_flow_meter_policy *mtr_policy;
1181
1182        /* Meter object must not have any owner. */
1183        MLX5_ASSERT(!fm->ref_cnt);
1184        /* Get meter profile. */
1185        fmp = fm->profile;
1186        if (fmp == NULL)
1187                return -1;
1188        /* Update dependencies. */
1189        __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1190        fm->profile = NULL;
1191        /* Remove from list. */
1192        if (!priv->sh->meter_aso_en) {
1193                legacy_fm = container_of(fm,
1194                        struct mlx5_legacy_flow_meter, fm);
1195                TAILQ_REMOVE(fms, legacy_fm, next);
1196        }
1197        /* Free drop counters. */
1198        if (fm->drop_cnt)
1199                mlx5_counter_free(dev, fm->drop_cnt);
1200        /* Free meter flow table. */
1201        if (fm->flow_ipool) {
1202                mlx5_ipool_destroy(fm->flow_ipool);
1203                fm->flow_ipool = 0;
1204        }
1205        mlx5_flow_destroy_mtr_tbls(dev, fm);
1206        if (fm->def_policy)
1207                __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1208                                1, __ATOMIC_RELAXED);
1209        if (priv->sh->meter_aso_en) {
1210                if (!fm->def_policy) {
1211                        mtr_policy = mlx5_flow_meter_policy_find(dev,
1212                                                fm->policy_id, NULL);
1213                        if (mtr_policy)
1214                                __atomic_sub_fetch(&mtr_policy->ref_cnt,
1215                                                1, __ATOMIC_RELAXED);
1216                        fm->policy_id = 0;
1217                }
1218                fm->def_policy = 0;
1219                if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1220                        return -1;
1221                mlx5_flow_mtr_free(dev, mtr_idx);
1222        } else {
1223                mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1224                                        legacy_fm->idx);
1225        }
1226        return 0;
1227}
1228
1229/**
1230 * Destroy meter rules.
1231 *
1232 * @param[in] dev
1233 *   Pointer to Ethernet device.
1234 * @param[in] meter_id
1235 *   Meter id.
1236 * @param[out] error
1237 *   Pointer to rte meter error structure.
1238 *
1239 * @return
1240 *   0 on success, a negative errno value otherwise and rte_errno is set.
1241 */
1242static int
1243mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1244                        struct rte_mtr_error *error)
1245{
1246        struct mlx5_priv *priv = dev->data->dev_private;
1247        struct mlx5_flow_meter_info *fm;
1248        uint32_t mtr_idx = 0;
1249
1250        if (!priv->mtr_en)
1251                return -rte_mtr_error_set(error, ENOTSUP,
1252                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1253                                          NULL,
1254                                          "Meter is not supported");
1255        /* Meter object must exist. */
1256        fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1257        if (fm == NULL)
1258                return -rte_mtr_error_set(error, ENOENT,
1259                                          RTE_MTR_ERROR_TYPE_MTR_ID,
1260                                          NULL,
1261                                          "Meter object id not valid.");
1262        /* Meter object must not have any owner. */
1263        if (fm->ref_cnt > 0)
1264                return -rte_mtr_error_set(error, EBUSY,
1265                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1266                                          NULL,
1267                                          "Meter object is being used.");
1268        /* Destroy the meter profile. */
1269        if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1270                return -rte_mtr_error_set(error, EINVAL,
1271                                        RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1272                                        NULL,
1273                                        "MTR object meter profile invalid.");
1274        return 0;
1275}
1276
1277/**
1278 * Modify meter state.
1279 *
1280 * @param[in] priv
1281 *   Pointer to mlx5 private data structure.
1282 * @param[in] fm
1283 *   Pointer to flow meter.
1284 * @param[in] new_state
1285 *   New state to update.
1286 * @param[out] error
1287 *   Pointer to rte meter error structure.
1288 *
1289 * @return
1290 *   0 on success, a negative errno value otherwise and rte_errno is set.
1291 */
1292static int
1293mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1294                             struct mlx5_flow_meter_info *fm,
1295                             uint32_t new_state,
1296                             struct rte_mtr_error *error)
1297{
1298        static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1299                .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1300                .ebs_eir = 0,
1301        };
1302        uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1303                               MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1304        int ret;
1305
1306        if (new_state == MLX5_FLOW_METER_DISABLE)
1307                ret = mlx5_flow_meter_action_modify(priv, fm,
1308                                &srtcm, modify_bits, 0, 0);
1309        else
1310                ret = mlx5_flow_meter_action_modify(priv, fm,
1311                                                   &fm->profile->srtcm_prm,
1312                                                    modify_bits, 0, 1);
1313        if (ret)
1314                return -rte_mtr_error_set(error, -ret,
1315                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1316                                          NULL,
1317                                          new_state ?
1318                                          "Failed to enable meter." :
1319                                          "Failed to disable meter.");
1320        return 0;
1321}
1322
1323/**
1324 * Callback to enable flow meter.
1325 *
1326 * @param[in] dev
1327 *   Pointer to Ethernet device.
1328 * @param[in] meter_id
1329 *   Meter id.
1330 * @param[out] error
1331 *   Pointer to rte meter error structure.
1332 *
1333 * @return
1334 *   0 on success, a negative errno value otherwise and rte_errno is set.
1335 */
1336static int
1337mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1338                       uint32_t meter_id,
1339                       struct rte_mtr_error *error)
1340{
1341        struct mlx5_priv *priv = dev->data->dev_private;
1342        struct mlx5_flow_meter_info *fm;
1343        int ret;
1344
1345        if (!priv->mtr_en)
1346                return -rte_mtr_error_set(error, ENOTSUP,
1347                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1348                                          "Meter is not supported");
1349        /* Meter object must exist. */
1350        fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1351        if (fm == NULL)
1352                return -rte_mtr_error_set(error, ENOENT,
1353                                          RTE_MTR_ERROR_TYPE_MTR_ID,
1354                                          NULL, "Meter not found.");
1355        if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1356                return 0;
1357        ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1358                                           error);
1359        if (!ret)
1360                fm->active_state = MLX5_FLOW_METER_ENABLE;
1361        return ret;
1362}
1363
1364/**
1365 * Callback to disable flow meter.
1366 *
1367 * @param[in] dev
1368 *   Pointer to Ethernet device.
1369 * @param[in] meter_id
1370 *   Meter id.
1371 * @param[out] error
1372 *   Pointer to rte meter error structure.
1373 *
1374 * @return
1375 *   0 on success, a negative errno value otherwise and rte_errno is set.
1376 */
1377static int
1378mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1379                        uint32_t meter_id,
1380                        struct rte_mtr_error *error)
1381{
1382        struct mlx5_priv *priv = dev->data->dev_private;
1383        struct mlx5_flow_meter_info *fm;
1384        int ret;
1385
1386        if (!priv->mtr_en)
1387                return -rte_mtr_error_set(error, ENOTSUP,
1388                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1389                                          "Meter is not supported");
1390        /* Meter object must exist. */
1391        fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1392        if (fm == NULL)
1393                return -rte_mtr_error_set(error, ENOENT,
1394                                          RTE_MTR_ERROR_TYPE_MTR_ID,
1395                                          NULL, "Meter not found.");
1396        if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1397                return 0;
1398        ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1399                                           error);
1400        if (!ret)
1401                fm->active_state = MLX5_FLOW_METER_DISABLE;
1402        return ret;
1403}
1404
1405/**
1406 * Callback to update meter profile.
1407 *
1408 * @param[in] dev
1409 *   Pointer to Ethernet device.
1410 * @param[in] meter_id
1411 *   Meter id.
1412 * @param[in] meter_profile_id
1413 *   To be updated meter profile id.
1414 * @param[out] error
1415 *   Pointer to rte meter error structure.
1416 *
1417 * @return
1418 *   0 on success, a negative errno value otherwise and rte_errno is set.
1419 */
1420static int
1421mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1422                               uint32_t meter_id,
1423                               uint32_t meter_profile_id,
1424                               struct rte_mtr_error *error)
1425{
1426        struct mlx5_priv *priv = dev->data->dev_private;
1427        struct mlx5_flow_meter_profile *fmp;
1428        struct mlx5_flow_meter_profile *old_fmp;
1429        struct mlx5_flow_meter_info *fm;
1430        uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1431                               MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1432        int ret;
1433
1434        if (!priv->mtr_en)
1435                return -rte_mtr_error_set(error, ENOTSUP,
1436                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1437                                          "Meter is not supported");
1438        /* Meter profile must exist. */
1439        fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1440        if (fmp == NULL)
1441                return -rte_mtr_error_set(error, ENOENT,
1442                                          RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1443                                          NULL, "Meter profile not found.");
1444        /* Meter object must exist. */
1445        fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1446        if (fm == NULL)
1447                return -rte_mtr_error_set(error, ENOENT,
1448                                          RTE_MTR_ERROR_TYPE_MTR_ID,
1449                                          NULL, "Meter not found.");
1450        /* MTR object already set to meter profile id. */
1451        old_fmp = fm->profile;
1452        if (fmp == old_fmp)
1453                return 0;
1454        /* Update the profile. */
1455        fm->profile = fmp;
1456        /* Update meter params in HW (if not disabled). */
1457        if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1458                return 0;
1459        ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1460                                              modify_bits, fm->active_state, 1);
1461        if (ret) {
1462                fm->profile = old_fmp;
1463                return -rte_mtr_error_set(error, -ret,
1464                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1465                                          NULL, "Failed to update meter"
1466                                          " parmeters in hardware.");
1467        }
1468        old_fmp->ref_cnt--;
1469        fmp->ref_cnt++;
1470        return 0;
1471}
1472
1473/**
1474 * Callback to update meter stats mask.
1475 *
1476 * @param[in] dev
1477 *   Pointer to Ethernet device.
1478 * @param[in] meter_id
1479 *   Meter id.
1480 * @param[in] stats_mask
1481 *   To be updated stats_mask.
1482 * @param[out] error
1483 *   Pointer to rte meter error structure.
1484 *
1485 * @return
1486 *   0 on success, a negative errno value otherwise and rte_errno is set.
1487 */
1488static int
1489mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1490                             uint32_t meter_id,
1491                             uint64_t stats_mask,
1492                             struct rte_mtr_error *error)
1493{
1494        struct mlx5_priv *priv = dev->data->dev_private;
1495        struct mlx5_flow_meter_info *fm;
1496
1497        if (!priv->mtr_en)
1498                return -rte_mtr_error_set(error, ENOTSUP,
1499                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1500                                          "Meter is not supported");
1501        /* Meter object must exist. */
1502        fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1503        if (fm == NULL)
1504                return -rte_mtr_error_set(error, ENOENT,
1505                                          RTE_MTR_ERROR_TYPE_MTR_ID,
1506                                          NULL, "Meter object id not valid.");
1507        if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1508                return -rte_mtr_error_set(error, ENOENT,
1509                                          RTE_MTR_ERROR_TYPE_MTR_ID,
1510                                          NULL, "Fail to allocate "
1511                                          "counter for meter.");
1512        return 0;
1513}
1514
1515/**
1516 * Callback to read meter statistics.
1517 *
1518 * @param[in] dev
1519 *   Pointer to Ethernet device.
1520 * @param[in] meter_id
1521 *   Meter id.
1522 * @param[out] stats
1523 *   Pointer to store the statistics.
1524 * @param[out] stats_mask
1525 *   Pointer to store the stats_mask.
1526 * @param[in] clear
1527 *   Statistic to be cleared after read or not.
1528 * @param[out] error
1529 *   Pointer to rte meter error structure.
1530 *
1531 * @return
1532 *   0 on success, a negative errno value otherwise and rte_errno is set.
1533 */
1534static int
1535mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1536                           uint32_t meter_id,
1537                           struct rte_mtr_stats *stats,
1538                           uint64_t *stats_mask,
1539                           int clear,
1540                           struct rte_mtr_error *error)
1541{
1542        struct mlx5_priv *priv = dev->data->dev_private;
1543        struct mlx5_flow_meter_info *fm;
1544        uint64_t pkts;
1545        uint64_t bytes;
1546        int ret = 0;
1547
1548        if (!priv->mtr_en)
1549                return -rte_mtr_error_set(error, ENOTSUP,
1550                                          RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1551                                          "Meter is not supported");
1552        /* Meter object must exist. */
1553        fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1554        if (fm == NULL)
1555                return -rte_mtr_error_set(error, ENOENT,
1556                                          RTE_MTR_ERROR_TYPE_MTR_ID,
1557                                          NULL, "Meter object id not valid.");
1558        *stats_mask = 0;
1559        if (fm->bytes_dropped)
1560                *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1561        if (fm->pkts_dropped)
1562                *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1563        memset(stats, 0, sizeof(*stats));
1564        if (fm->drop_cnt) {
1565                ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1566                                                 &bytes);
1567                if (ret)
1568                        goto error;
1569                /* If need to read the packets, set it. */
1570                if (fm->pkts_dropped)
1571                        stats->n_pkts_dropped = pkts;
1572                /* If need to read the bytes, set it. */
1573                if (fm->bytes_dropped)
1574                        stats->n_bytes_dropped = bytes;
1575        }
1576        return 0;
1577error:
1578        return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1579                                 "Failed to read meter drop counters.");
1580}
1581
1582static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1583        .capabilities_get = mlx5_flow_mtr_cap_get,
1584        .meter_profile_add = mlx5_flow_meter_profile_add,
1585        .meter_profile_delete = mlx5_flow_meter_profile_delete,
1586        .meter_policy_validate = mlx5_flow_meter_policy_validate,
1587        .meter_policy_add = mlx5_flow_meter_policy_add,
1588        .meter_policy_delete = mlx5_flow_meter_policy_delete,
1589        .create = mlx5_flow_meter_create,
1590        .destroy = mlx5_flow_meter_destroy,
1591        .meter_enable = mlx5_flow_meter_enable,
1592        .meter_disable = mlx5_flow_meter_disable,
1593        .meter_profile_update = mlx5_flow_meter_profile_update,
1594        .meter_dscp_table_update = NULL,
1595        .stats_update = mlx5_flow_meter_stats_update,
1596        .stats_read = mlx5_flow_meter_stats_read,
1597};
1598
1599/**
1600 * Get meter operations.
1601 *
1602 * @param dev
1603 *   Pointer to Ethernet device structure.
1604 * @param arg
1605 *   Pointer to set the mtr operations.
1606 *
1607 * @return
1608 *   Always 0.
1609 */
1610int
1611mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1612{
1613        *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1614        return 0;
1615}
1616
1617/**
1618 * Find meter by id.
1619 *
1620 * @param priv
1621 *   Pointer to mlx5_priv.
1622 * @param meter_id
1623 *   Meter id.
1624 * @param mtr_idx
1625 *   Pointer to Meter index.
1626 *
1627 * @return
1628 *   Pointer to the meter info found on success, NULL otherwise.
1629 */
1630struct mlx5_flow_meter_info *
1631mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1632                uint32_t *mtr_idx)
1633{
1634        struct mlx5_legacy_flow_meter *legacy_fm;
1635        struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1636        struct mlx5_aso_mtr *aso_mtr;
1637        struct mlx5_aso_mtr_pools_mng *pools_mng =
1638                                &priv->sh->mtrmng->pools_mng;
1639        union mlx5_l3t_data data;
1640
1641        if (priv->sh->meter_aso_en) {
1642                rte_spinlock_lock(&pools_mng->mtrsl);
1643                if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1644                        rte_spinlock_unlock(&pools_mng->mtrsl);
1645                        return NULL;
1646                }
1647                if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1648                        !data.dword) {
1649                        rte_spinlock_unlock(&pools_mng->mtrsl);
1650                        return NULL;
1651                }
1652                if (mtr_idx)
1653                        *mtr_idx = data.dword;
1654                aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1655                /* Remove reference taken by the mlx5_l3t_get_entry. */
1656                mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1657                rte_spinlock_unlock(&pools_mng->mtrsl);
1658                if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1659                        return NULL;
1660                return &aso_mtr->fm;
1661        }
1662        TAILQ_FOREACH(legacy_fm, fms, next)
1663                if (meter_id == legacy_fm->fm.meter_id) {
1664                        if (mtr_idx)
1665                                *mtr_idx = legacy_fm->idx;
1666                        return &legacy_fm->fm;
1667                }
1668        return NULL;
1669}
1670
1671/**
1672 * Find meter by index.
1673 *
1674 * @param priv
1675 *   Pointer to mlx5_priv.
1676 * @param idx
1677 *   Meter index.
1678 *
1679 * @return
1680 *   Pointer to the meter info found on success, NULL otherwise.
1681 */
1682struct mlx5_flow_meter_info *
1683flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1684{
1685        struct mlx5_aso_mtr *aso_mtr;
1686
1687        if (priv->sh->meter_aso_en) {
1688                aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1689                if (!aso_mtr)
1690                        return NULL;
1691                return &aso_mtr->fm;
1692        } else {
1693                return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1694        }
1695}
1696
1697/**
1698 * Attach meter to flow.
1699 * Unidirectional Meter creation can only be done
1700 * when flow direction is known, i.e. when calling meter_attach.
1701 *
1702 * @param [in] priv
1703 *  Pointer to mlx5 private data.
1704 * @param[in] fm
1705 *   Pointer to flow meter.
1706 * @param [in] attr
1707 *  Pointer to flow attributes.
1708 * @param [out] error
1709 *  Pointer to error structure.
1710 *
1711 * @return
1712 *   0 on success, a negative errno value otherwise and rte_errno is set.
1713 */
1714int
1715mlx5_flow_meter_attach(struct mlx5_priv *priv,
1716                       struct mlx5_flow_meter_info *fm,
1717                       const struct rte_flow_attr *attr,
1718                       struct rte_flow_error *error)
1719{
1720        int ret = 0;
1721
1722        if (priv->sh->meter_aso_en) {
1723                struct mlx5_aso_mtr *aso_mtr;
1724
1725                aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1726                if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1727                        return rte_flow_error_set(error, ENOENT,
1728                                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1729                                        NULL,
1730                                        "Timeout in meter configuration");
1731                }
1732                rte_spinlock_lock(&fm->sl);
1733                if (fm->shared || !fm->ref_cnt) {
1734                        fm->ref_cnt++;
1735                } else {
1736                        rte_flow_error_set(error, EINVAL,
1737                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1738                                   "Meter cannot be shared");
1739                        ret = -1;
1740                }
1741                rte_spinlock_unlock(&fm->sl);
1742        } else {
1743                rte_spinlock_lock(&fm->sl);
1744                if (fm->meter_action) {
1745                        if (fm->shared &&
1746                            attr->transfer == fm->transfer &&
1747                            attr->ingress == fm->ingress &&
1748                            attr->egress == fm->egress) {
1749                                fm->ref_cnt++;
1750                        } else {
1751                                rte_flow_error_set(error, EINVAL,
1752                                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1753                                        fm->shared ?
1754                                        "Meter attr not match." :
1755                                        "Meter cannot be shared.");
1756                                ret = -1;
1757                        }
1758                } else {
1759                        fm->ingress = attr->ingress;
1760                        fm->egress = attr->egress;
1761                        fm->transfer = attr->transfer;
1762                        fm->ref_cnt = 1;
1763                        /* This also creates the meter object. */
1764                        fm->meter_action = mlx5_flow_meter_action_create(priv,
1765                                                                         fm);
1766                        if (!fm->meter_action) {
1767                                fm->ref_cnt = 0;
1768                                fm->ingress = 0;
1769                                fm->egress = 0;
1770                                fm->transfer = 0;
1771                                rte_flow_error_set(error, EINVAL,
1772                                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1773                                        "Meter action create failed.");
1774                                ret = -1;
1775                        }
1776                }
1777                rte_spinlock_unlock(&fm->sl);
1778        }
1779        return ret ? -rte_errno : 0;
1780}
1781
1782/**
1783 * Detach meter from flow.
1784 *
1785 * @param [in] priv
1786 *  Pointer to mlx5 private data.
1787 * @param [in] fm
1788 *  Pointer to flow meter.
1789 */
1790void
1791mlx5_flow_meter_detach(struct mlx5_priv *priv,
1792                       struct mlx5_flow_meter_info *fm)
1793{
1794#ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1795        rte_spinlock_lock(&fm->sl);
1796        MLX5_ASSERT(fm->ref_cnt);
1797        if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1798                mlx5_glue->destroy_flow_action(fm->meter_action);
1799                fm->meter_action = NULL;
1800                fm->ingress = 0;
1801                fm->egress = 0;
1802                fm->transfer = 0;
1803        }
1804        rte_spinlock_unlock(&fm->sl);
1805#else
1806        (void)priv;
1807        (void)fm;
1808#endif
1809}
1810
1811/**
1812 * Flush meter with Rx queue configuration.
1813 *
1814 * @param[in] dev
1815 *   Pointer to Ethernet device.
1816 */
1817void
1818mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1819{
1820        struct mlx5_priv *priv = dev->data->dev_private;
1821        struct mlx5_flow_meter_sub_policy *sub_policy;
1822        struct mlx5_flow_meter_policy *mtr_policy;
1823        void *entry;
1824        uint32_t i, policy_idx;
1825
1826        if (!priv->mtr_en)
1827                return;
1828        if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1829                MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1830                                        i, entry) {
1831                        policy_idx = *(uint32_t *)entry;
1832                        sub_policy = mlx5_ipool_get
1833                                (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1834                                policy_idx);
1835                        if (!sub_policy || !sub_policy->main_policy)
1836                                continue;
1837                        mtr_policy = sub_policy->main_policy;
1838                        if (mtr_policy->is_queue || mtr_policy->is_rss)
1839                                mlx5_flow_destroy_sub_policy_with_rxq(dev,
1840                                        mtr_policy);
1841                }
1842        }
1843}
1844
1845/**
1846 * Flush meter configuration.
1847 *
1848 * @param[in] dev
1849 *   Pointer to Ethernet device.
1850 * @param[out] error
1851 *   Pointer to rte meter error structure.
1852 *
1853 * @return
1854 *   0 on success, a negative errno value otherwise and rte_errno is set.
1855 */
1856int
1857mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1858{
1859        struct mlx5_priv *priv = dev->data->dev_private;
1860        struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1861        struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1862        struct mlx5_flow_meter_profile *fmp;
1863        struct mlx5_legacy_flow_meter *legacy_fm;
1864        struct mlx5_flow_meter_info *fm;
1865        struct mlx5_flow_meter_sub_policy *sub_policy;
1866        void *tmp;
1867        uint32_t i, mtr_idx, policy_idx;
1868        void *entry;
1869        struct mlx5_aso_mtr *aso_mtr;
1870
1871        if (!priv->mtr_en)
1872                return 0;
1873        if (priv->sh->meter_aso_en) {
1874                if (priv->mtr_idx_tbl) {
1875                        MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1876                                mtr_idx = *(uint32_t *)entry;
1877                                if (mtr_idx) {
1878                                        aso_mtr =
1879                                        mlx5_aso_meter_by_idx(priv, mtr_idx);
1880                                        fm = &aso_mtr->fm;
1881                                        (void)mlx5_flow_meter_params_flush(dev,
1882                                                fm, mtr_idx);
1883                                }
1884                        }
1885                        mlx5_l3t_destroy(priv->mtr_idx_tbl);
1886                        priv->mtr_idx_tbl = NULL;
1887                }
1888        } else {
1889                TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1890                        fm = &legacy_fm->fm;
1891                        if (mlx5_flow_meter_params_flush(dev, fm, 0))
1892                                return -rte_mtr_error_set(error, EINVAL,
1893                                RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1894                                NULL, "MTR object meter profile invalid.");
1895                }
1896        }
1897        if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1898                MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1899                                        i, entry) {
1900                        policy_idx = *(uint32_t *)entry;
1901                        sub_policy = mlx5_ipool_get
1902                                (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1903                                policy_idx);
1904                        if (!sub_policy)
1905                                return -rte_mtr_error_set(error,
1906                                                EINVAL,
1907                                        RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1908                                                NULL, "MTR object "
1909                                                "meter policy invalid.");
1910                        if (__mlx5_flow_meter_policy_delete(dev, i,
1911                                                sub_policy->main_policy,
1912                                                error))
1913                                return -rte_mtr_error_set(error,
1914                                                EINVAL,
1915                                        RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1916                                                NULL, "MTR object "
1917                                                "meter policy invalid.");
1918                        mlx5_free(sub_policy->main_policy);
1919                }
1920                mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl);
1921                priv->sh->mtrmng->policy_idx_tbl = NULL;
1922        }
1923        TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1924                /* Check unused. */
1925                MLX5_ASSERT(!fmp->ref_cnt);
1926                /* Remove from list. */
1927                TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1928                mlx5_free(fmp);
1929        }
1930        /* Delete default policy table. */
1931        mlx5_flow_destroy_def_policy(dev);
1932        if (priv->sh->refcnt == 1)
1933                mlx5_flow_destroy_mtr_drop_tbls(dev);
1934        return 0;
1935}
1936