linux/drivers/infiniband/hw/mlx5/mr.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33
  34#include <linux/kref.h>
  35#include <linux/random.h>
  36#include <linux/debugfs.h>
  37#include <linux/export.h>
  38#include <linux/delay.h>
  39#include <rdma/ib_umem.h>
  40#include <rdma/ib_umem_odp.h>
  41#include <rdma/ib_verbs.h>
  42#include "mlx5_ib.h"
  43
  44enum {
  45        MAX_PENDING_REG_MR = 8,
  46};
  47
  48#define MLX5_UMR_ALIGN 2048
  49
  50static void
  51create_mkey_callback(int status, struct mlx5_async_work *context);
  52
  53static void
  54assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
  55                    u32 *in)
  56{
  57        u8 key = atomic_inc_return(&dev->mkey_var);
  58        void *mkc;
  59
  60        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  61        MLX5_SET(mkc, mkc, mkey_7_0, key);
  62        mkey->key = key;
  63}
  64
  65static int
  66mlx5_ib_create_mkey(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
  67                    u32 *in, int inlen)
  68{
  69        assign_mkey_variant(dev, mkey, in);
  70        return mlx5_core_create_mkey(dev->mdev, mkey, in, inlen);
  71}
  72
  73static int
  74mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev,
  75                       struct mlx5_core_mkey *mkey,
  76                       struct mlx5_async_ctx *async_ctx,
  77                       u32 *in, int inlen, u32 *out, int outlen,
  78                       struct mlx5_async_work *context)
  79{
  80        MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
  81        assign_mkey_variant(dev, mkey, in);
  82        return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen,
  83                                create_mkey_callback, context);
  84}
  85
  86static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
  87static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
  88static int mr_cache_max_order(struct mlx5_ib_dev *dev);
  89static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
  90
  91static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
  92{
  93        return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled);
  94}
  95
  96static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
  97{
  98        WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
  99
 100        return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
 101}
 102
 103static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length)
 104{
 105        return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
 106                length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1));
 107}
 108
 109static void create_mkey_callback(int status, struct mlx5_async_work *context)
 110{
 111        struct mlx5_ib_mr *mr =
 112                container_of(context, struct mlx5_ib_mr, cb_work);
 113        struct mlx5_ib_dev *dev = mr->dev;
 114        struct mlx5_cache_ent *ent = mr->cache_ent;
 115        unsigned long flags;
 116
 117        if (status) {
 118                mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
 119                kfree(mr);
 120                spin_lock_irqsave(&ent->lock, flags);
 121                ent->pending--;
 122                WRITE_ONCE(dev->fill_delay, 1);
 123                spin_unlock_irqrestore(&ent->lock, flags);
 124                mod_timer(&dev->delay_timer, jiffies + HZ);
 125                return;
 126        }
 127
 128        mr->mmkey.type = MLX5_MKEY_MR;
 129        mr->mmkey.key |= mlx5_idx_to_mkey(
 130                MLX5_GET(create_mkey_out, mr->out, mkey_index));
 131
 132        WRITE_ONCE(dev->cache.last_add, jiffies);
 133
 134        spin_lock_irqsave(&ent->lock, flags);
 135        list_add_tail(&mr->list, &ent->head);
 136        ent->available_mrs++;
 137        ent->total_mrs++;
 138        /* If we are doing fill_to_high_water then keep going. */
 139        queue_adjust_cache_locked(ent);
 140        ent->pending--;
 141        spin_unlock_irqrestore(&ent->lock, flags);
 142}
 143
 144static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc)
 145{
 146        struct mlx5_ib_mr *mr;
 147
 148        mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 149        if (!mr)
 150                return NULL;
 151        mr->order = ent->order;
 152        mr->cache_ent = ent;
 153        mr->dev = ent->dev;
 154
 155        MLX5_SET(mkc, mkc, free, 1);
 156        MLX5_SET(mkc, mkc, umr_en, 1);
 157        MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
 158        MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7);
 159
 160        MLX5_SET(mkc, mkc, qpn, 0xffffff);
 161        MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
 162        MLX5_SET(mkc, mkc, log_page_size, ent->page);
 163        return mr;
 164}
 165
 166/* Asynchronously schedule new MRs to be populated in the cache. */
 167static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
 168{
 169        size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 170        struct mlx5_ib_mr *mr;
 171        void *mkc;
 172        u32 *in;
 173        int err = 0;
 174        int i;
 175
 176        in = kzalloc(inlen, GFP_KERNEL);
 177        if (!in)
 178                return -ENOMEM;
 179
 180        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 181        for (i = 0; i < num; i++) {
 182                mr = alloc_cache_mr(ent, mkc);
 183                if (!mr) {
 184                        err = -ENOMEM;
 185                        break;
 186                }
 187                spin_lock_irq(&ent->lock);
 188                if (ent->pending >= MAX_PENDING_REG_MR) {
 189                        err = -EAGAIN;
 190                        spin_unlock_irq(&ent->lock);
 191                        kfree(mr);
 192                        break;
 193                }
 194                ent->pending++;
 195                spin_unlock_irq(&ent->lock);
 196                err = mlx5_ib_create_mkey_cb(ent->dev, &mr->mmkey,
 197                                             &ent->dev->async_ctx, in, inlen,
 198                                             mr->out, sizeof(mr->out),
 199                                             &mr->cb_work);
 200                if (err) {
 201                        spin_lock_irq(&ent->lock);
 202                        ent->pending--;
 203                        spin_unlock_irq(&ent->lock);
 204                        mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err);
 205                        kfree(mr);
 206                        break;
 207                }
 208        }
 209
 210        kfree(in);
 211        return err;
 212}
 213
 214/* Synchronously create a MR in the cache */
 215static struct mlx5_ib_mr *create_cache_mr(struct mlx5_cache_ent *ent)
 216{
 217        size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 218        struct mlx5_ib_mr *mr;
 219        void *mkc;
 220        u32 *in;
 221        int err;
 222
 223        in = kzalloc(inlen, GFP_KERNEL);
 224        if (!in)
 225                return ERR_PTR(-ENOMEM);
 226        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 227
 228        mr = alloc_cache_mr(ent, mkc);
 229        if (!mr) {
 230                err = -ENOMEM;
 231                goto free_in;
 232        }
 233
 234        err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey, in, inlen);
 235        if (err)
 236                goto free_mr;
 237
 238        mr->mmkey.type = MLX5_MKEY_MR;
 239        WRITE_ONCE(ent->dev->cache.last_add, jiffies);
 240        spin_lock_irq(&ent->lock);
 241        ent->total_mrs++;
 242        spin_unlock_irq(&ent->lock);
 243        kfree(in);
 244        return mr;
 245free_mr:
 246        kfree(mr);
 247free_in:
 248        kfree(in);
 249        return ERR_PTR(err);
 250}
 251
 252static void remove_cache_mr_locked(struct mlx5_cache_ent *ent)
 253{
 254        struct mlx5_ib_mr *mr;
 255
 256        lockdep_assert_held(&ent->lock);
 257        if (list_empty(&ent->head))
 258                return;
 259        mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
 260        list_del(&mr->list);
 261        ent->available_mrs--;
 262        ent->total_mrs--;
 263        spin_unlock_irq(&ent->lock);
 264        mlx5_core_destroy_mkey(ent->dev->mdev, &mr->mmkey);
 265        kfree(mr);
 266        spin_lock_irq(&ent->lock);
 267}
 268
 269static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target,
 270                                bool limit_fill)
 271{
 272        int err;
 273
 274        lockdep_assert_held(&ent->lock);
 275
 276        while (true) {
 277                if (limit_fill)
 278                        target = ent->limit * 2;
 279                if (target == ent->available_mrs + ent->pending)
 280                        return 0;
 281                if (target > ent->available_mrs + ent->pending) {
 282                        u32 todo = target - (ent->available_mrs + ent->pending);
 283
 284                        spin_unlock_irq(&ent->lock);
 285                        err = add_keys(ent, todo);
 286                        if (err == -EAGAIN)
 287                                usleep_range(3000, 5000);
 288                        spin_lock_irq(&ent->lock);
 289                        if (err) {
 290                                if (err != -EAGAIN)
 291                                        return err;
 292                        } else
 293                                return 0;
 294                } else {
 295                        remove_cache_mr_locked(ent);
 296                }
 297        }
 298}
 299
 300static ssize_t size_write(struct file *filp, const char __user *buf,
 301                          size_t count, loff_t *pos)
 302{
 303        struct mlx5_cache_ent *ent = filp->private_data;
 304        u32 target;
 305        int err;
 306
 307        err = kstrtou32_from_user(buf, count, 0, &target);
 308        if (err)
 309                return err;
 310
 311        /*
 312         * Target is the new value of total_mrs the user requests, however we
 313         * cannot free MRs that are in use. Compute the target value for
 314         * available_mrs.
 315         */
 316        spin_lock_irq(&ent->lock);
 317        if (target < ent->total_mrs - ent->available_mrs) {
 318                err = -EINVAL;
 319                goto err_unlock;
 320        }
 321        target = target - (ent->total_mrs - ent->available_mrs);
 322        if (target < ent->limit || target > ent->limit*2) {
 323                err = -EINVAL;
 324                goto err_unlock;
 325        }
 326        err = resize_available_mrs(ent, target, false);
 327        if (err)
 328                goto err_unlock;
 329        spin_unlock_irq(&ent->lock);
 330
 331        return count;
 332
 333err_unlock:
 334        spin_unlock_irq(&ent->lock);
 335        return err;
 336}
 337
 338static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
 339                         loff_t *pos)
 340{
 341        struct mlx5_cache_ent *ent = filp->private_data;
 342        char lbuf[20];
 343        int err;
 344
 345        err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->total_mrs);
 346        if (err < 0)
 347                return err;
 348
 349        return simple_read_from_buffer(buf, count, pos, lbuf, err);
 350}
 351
 352static const struct file_operations size_fops = {
 353        .owner  = THIS_MODULE,
 354        .open   = simple_open,
 355        .write  = size_write,
 356        .read   = size_read,
 357};
 358
 359static ssize_t limit_write(struct file *filp, const char __user *buf,
 360                           size_t count, loff_t *pos)
 361{
 362        struct mlx5_cache_ent *ent = filp->private_data;
 363        u32 var;
 364        int err;
 365
 366        err = kstrtou32_from_user(buf, count, 0, &var);
 367        if (err)
 368                return err;
 369
 370        /*
 371         * Upon set we immediately fill the cache to high water mark implied by
 372         * the limit.
 373         */
 374        spin_lock_irq(&ent->lock);
 375        ent->limit = var;
 376        err = resize_available_mrs(ent, 0, true);
 377        spin_unlock_irq(&ent->lock);
 378        if (err)
 379                return err;
 380        return count;
 381}
 382
 383static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
 384                          loff_t *pos)
 385{
 386        struct mlx5_cache_ent *ent = filp->private_data;
 387        char lbuf[20];
 388        int err;
 389
 390        err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
 391        if (err < 0)
 392                return err;
 393
 394        return simple_read_from_buffer(buf, count, pos, lbuf, err);
 395}
 396
 397static const struct file_operations limit_fops = {
 398        .owner  = THIS_MODULE,
 399        .open   = simple_open,
 400        .write  = limit_write,
 401        .read   = limit_read,
 402};
 403
 404static bool someone_adding(struct mlx5_mr_cache *cache)
 405{
 406        unsigned int i;
 407
 408        for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
 409                struct mlx5_cache_ent *ent = &cache->ent[i];
 410                bool ret;
 411
 412                spin_lock_irq(&ent->lock);
 413                ret = ent->available_mrs < ent->limit;
 414                spin_unlock_irq(&ent->lock);
 415                if (ret)
 416                        return true;
 417        }
 418        return false;
 419}
 420
 421/*
 422 * Check if the bucket is outside the high/low water mark and schedule an async
 423 * update. The cache refill has hysteresis, once the low water mark is hit it is
 424 * refilled up to the high mark.
 425 */
 426static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
 427{
 428        lockdep_assert_held(&ent->lock);
 429
 430        if (ent->disabled || READ_ONCE(ent->dev->fill_delay))
 431                return;
 432        if (ent->available_mrs < ent->limit) {
 433                ent->fill_to_high_water = true;
 434                queue_work(ent->dev->cache.wq, &ent->work);
 435        } else if (ent->fill_to_high_water &&
 436                   ent->available_mrs + ent->pending < 2 * ent->limit) {
 437                /*
 438                 * Once we start populating due to hitting a low water mark
 439                 * continue until we pass the high water mark.
 440                 */
 441                queue_work(ent->dev->cache.wq, &ent->work);
 442        } else if (ent->available_mrs == 2 * ent->limit) {
 443                ent->fill_to_high_water = false;
 444        } else if (ent->available_mrs > 2 * ent->limit) {
 445                /* Queue deletion of excess entries */
 446                ent->fill_to_high_water = false;
 447                if (ent->pending)
 448                        queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
 449                                           msecs_to_jiffies(1000));
 450                else
 451                        queue_work(ent->dev->cache.wq, &ent->work);
 452        }
 453}
 454
 455static void __cache_work_func(struct mlx5_cache_ent *ent)
 456{
 457        struct mlx5_ib_dev *dev = ent->dev;
 458        struct mlx5_mr_cache *cache = &dev->cache;
 459        int err;
 460
 461        spin_lock_irq(&ent->lock);
 462        if (ent->disabled)
 463                goto out;
 464
 465        if (ent->fill_to_high_water &&
 466            ent->available_mrs + ent->pending < 2 * ent->limit &&
 467            !READ_ONCE(dev->fill_delay)) {
 468                spin_unlock_irq(&ent->lock);
 469                err = add_keys(ent, 1);
 470                spin_lock_irq(&ent->lock);
 471                if (ent->disabled)
 472                        goto out;
 473                if (err) {
 474                        /*
 475                         * EAGAIN only happens if pending is positive, so we
 476                         * will be rescheduled from reg_mr_callback(). The only
 477                         * failure path here is ENOMEM.
 478                         */
 479                        if (err != -EAGAIN) {
 480                                mlx5_ib_warn(
 481                                        dev,
 482                                        "command failed order %d, err %d\n",
 483                                        ent->order, err);
 484                                queue_delayed_work(cache->wq, &ent->dwork,
 485                                                   msecs_to_jiffies(1000));
 486                        }
 487                }
 488        } else if (ent->available_mrs > 2 * ent->limit) {
 489                bool need_delay;
 490
 491                /*
 492                 * The remove_cache_mr() logic is performed as garbage
 493                 * collection task. Such task is intended to be run when no
 494                 * other active processes are running.
 495                 *
 496                 * The need_resched() will return TRUE if there are user tasks
 497                 * to be activated in near future.
 498                 *
 499                 * In such case, we don't execute remove_cache_mr() and postpone
 500                 * the garbage collection work to try to run in next cycle, in
 501                 * order to free CPU resources to other tasks.
 502                 */
 503                spin_unlock_irq(&ent->lock);
 504                need_delay = need_resched() || someone_adding(cache) ||
 505                             time_after(jiffies,
 506                                        READ_ONCE(cache->last_add) + 300 * HZ);
 507                spin_lock_irq(&ent->lock);
 508                if (ent->disabled)
 509                        goto out;
 510                if (need_delay)
 511                        queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
 512                remove_cache_mr_locked(ent);
 513                queue_adjust_cache_locked(ent);
 514        }
 515out:
 516        spin_unlock_irq(&ent->lock);
 517}
 518
 519static void delayed_cache_work_func(struct work_struct *work)
 520{
 521        struct mlx5_cache_ent *ent;
 522
 523        ent = container_of(work, struct mlx5_cache_ent, dwork.work);
 524        __cache_work_func(ent);
 525}
 526
 527static void cache_work_func(struct work_struct *work)
 528{
 529        struct mlx5_cache_ent *ent;
 530
 531        ent = container_of(work, struct mlx5_cache_ent, work);
 532        __cache_work_func(ent);
 533}
 534
 535/* Allocate a special entry from the cache */
 536struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
 537                                       unsigned int entry)
 538{
 539        struct mlx5_mr_cache *cache = &dev->cache;
 540        struct mlx5_cache_ent *ent;
 541        struct mlx5_ib_mr *mr;
 542
 543        if (WARN_ON(entry <= MR_CACHE_LAST_STD_ENTRY ||
 544                    entry >= ARRAY_SIZE(cache->ent)))
 545                return ERR_PTR(-EINVAL);
 546
 547        ent = &cache->ent[entry];
 548        spin_lock_irq(&ent->lock);
 549        if (list_empty(&ent->head)) {
 550                spin_unlock_irq(&ent->lock);
 551                mr = create_cache_mr(ent);
 552                if (IS_ERR(mr))
 553                        return mr;
 554        } else {
 555                mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
 556                list_del(&mr->list);
 557                ent->available_mrs--;
 558                queue_adjust_cache_locked(ent);
 559                spin_unlock_irq(&ent->lock);
 560        }
 561        return mr;
 562}
 563
 564/* Return a MR already available in the cache */
 565static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent)
 566{
 567        struct mlx5_ib_dev *dev = req_ent->dev;
 568        struct mlx5_ib_mr *mr = NULL;
 569        struct mlx5_cache_ent *ent = req_ent;
 570
 571        /* Try larger MR pools from the cache to satisfy the allocation */
 572        for (; ent != &dev->cache.ent[MR_CACHE_LAST_STD_ENTRY + 1]; ent++) {
 573                mlx5_ib_dbg(dev, "order %u, cache index %zu\n", ent->order,
 574                            ent - dev->cache.ent);
 575
 576                spin_lock_irq(&ent->lock);
 577                if (!list_empty(&ent->head)) {
 578                        mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
 579                                              list);
 580                        list_del(&mr->list);
 581                        ent->available_mrs--;
 582                        queue_adjust_cache_locked(ent);
 583                        spin_unlock_irq(&ent->lock);
 584                        break;
 585                }
 586                queue_adjust_cache_locked(ent);
 587                spin_unlock_irq(&ent->lock);
 588        }
 589
 590        if (!mr)
 591                req_ent->miss++;
 592
 593        return mr;
 594}
 595
 596static void detach_mr_from_cache(struct mlx5_ib_mr *mr)
 597{
 598        struct mlx5_cache_ent *ent = mr->cache_ent;
 599
 600        mr->cache_ent = NULL;
 601        spin_lock_irq(&ent->lock);
 602        ent->total_mrs--;
 603        spin_unlock_irq(&ent->lock);
 604}
 605
 606void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 607{
 608        struct mlx5_cache_ent *ent = mr->cache_ent;
 609
 610        if (!ent)
 611                return;
 612
 613        if (mlx5_mr_cache_invalidate(mr)) {
 614                detach_mr_from_cache(mr);
 615                destroy_mkey(dev, mr);
 616                return;
 617        }
 618
 619        spin_lock_irq(&ent->lock);
 620        list_add_tail(&mr->list, &ent->head);
 621        ent->available_mrs++;
 622        queue_adjust_cache_locked(ent);
 623        spin_unlock_irq(&ent->lock);
 624}
 625
 626static void clean_keys(struct mlx5_ib_dev *dev, int c)
 627{
 628        struct mlx5_mr_cache *cache = &dev->cache;
 629        struct mlx5_cache_ent *ent = &cache->ent[c];
 630        struct mlx5_ib_mr *tmp_mr;
 631        struct mlx5_ib_mr *mr;
 632        LIST_HEAD(del_list);
 633
 634        cancel_delayed_work(&ent->dwork);
 635        while (1) {
 636                spin_lock_irq(&ent->lock);
 637                if (list_empty(&ent->head)) {
 638                        spin_unlock_irq(&ent->lock);
 639                        break;
 640                }
 641                mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
 642                list_move(&mr->list, &del_list);
 643                ent->available_mrs--;
 644                ent->total_mrs--;
 645                spin_unlock_irq(&ent->lock);
 646                mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
 647        }
 648
 649        list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
 650                list_del(&mr->list);
 651                kfree(mr);
 652        }
 653}
 654
 655static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
 656{
 657        if (!mlx5_debugfs_root || dev->is_rep)
 658                return;
 659
 660        debugfs_remove_recursive(dev->cache.root);
 661        dev->cache.root = NULL;
 662}
 663
 664static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
 665{
 666        struct mlx5_mr_cache *cache = &dev->cache;
 667        struct mlx5_cache_ent *ent;
 668        struct dentry *dir;
 669        int i;
 670
 671        if (!mlx5_debugfs_root || dev->is_rep)
 672                return;
 673
 674        cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
 675
 676        for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
 677                ent = &cache->ent[i];
 678                sprintf(ent->name, "%d", ent->order);
 679                dir = debugfs_create_dir(ent->name, cache->root);
 680                debugfs_create_file("size", 0600, dir, ent, &size_fops);
 681                debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
 682                debugfs_create_u32("cur", 0400, dir, &ent->available_mrs);
 683                debugfs_create_u32("miss", 0600, dir, &ent->miss);
 684        }
 685}
 686
 687static void delay_time_func(struct timer_list *t)
 688{
 689        struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
 690
 691        WRITE_ONCE(dev->fill_delay, 0);
 692}
 693
 694int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
 695{
 696        struct mlx5_mr_cache *cache = &dev->cache;
 697        struct mlx5_cache_ent *ent;
 698        int i;
 699
 700        mutex_init(&dev->slow_path_mutex);
 701        cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
 702        if (!cache->wq) {
 703                mlx5_ib_warn(dev, "failed to create work queue\n");
 704                return -ENOMEM;
 705        }
 706
 707        mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
 708        timer_setup(&dev->delay_timer, delay_time_func, 0);
 709        for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
 710                ent = &cache->ent[i];
 711                INIT_LIST_HEAD(&ent->head);
 712                spin_lock_init(&ent->lock);
 713                ent->order = i + 2;
 714                ent->dev = dev;
 715                ent->limit = 0;
 716
 717                INIT_WORK(&ent->work, cache_work_func);
 718                INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
 719
 720                if (i > MR_CACHE_LAST_STD_ENTRY) {
 721                        mlx5_odp_init_mr_cache_entry(ent);
 722                        continue;
 723                }
 724
 725                if (ent->order > mr_cache_max_order(dev))
 726                        continue;
 727
 728                ent->page = PAGE_SHIFT;
 729                ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
 730                           MLX5_IB_UMR_OCTOWORD;
 731                ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
 732                if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
 733                    !dev->is_rep &&
 734                    mlx5_core_is_pf(dev->mdev))
 735                        ent->limit = dev->mdev->profile->mr_cache[i].limit;
 736                else
 737                        ent->limit = 0;
 738                spin_lock_irq(&ent->lock);
 739                queue_adjust_cache_locked(ent);
 740                spin_unlock_irq(&ent->lock);
 741        }
 742
 743        mlx5_mr_cache_debugfs_init(dev);
 744
 745        return 0;
 746}
 747
 748int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
 749{
 750        unsigned int i;
 751
 752        if (!dev->cache.wq)
 753                return 0;
 754
 755        for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
 756                struct mlx5_cache_ent *ent = &dev->cache.ent[i];
 757
 758                spin_lock_irq(&ent->lock);
 759                ent->disabled = true;
 760                spin_unlock_irq(&ent->lock);
 761                cancel_work_sync(&ent->work);
 762                cancel_delayed_work_sync(&ent->dwork);
 763        }
 764
 765        mlx5_mr_cache_debugfs_cleanup(dev);
 766        mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
 767
 768        for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
 769                clean_keys(dev, i);
 770
 771        destroy_workqueue(dev->cache.wq);
 772        del_timer_sync(&dev->delay_timer);
 773
 774        return 0;
 775}
 776
 777static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
 778                                          struct ib_pd *pd)
 779{
 780        struct mlx5_ib_dev *dev = to_mdev(pd->device);
 781
 782        MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
 783        MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
 784        MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
 785        MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
 786        MLX5_SET(mkc, mkc, lr, 1);
 787
 788        if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
 789                MLX5_SET(mkc, mkc, relaxed_ordering_write,
 790                         !!(acc & IB_ACCESS_RELAXED_ORDERING));
 791        if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
 792                MLX5_SET(mkc, mkc, relaxed_ordering_read,
 793                         !!(acc & IB_ACCESS_RELAXED_ORDERING));
 794
 795        MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
 796        MLX5_SET(mkc, mkc, qpn, 0xffffff);
 797        MLX5_SET64(mkc, mkc, start_addr, start_addr);
 798}
 799
 800struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
 801{
 802        struct mlx5_ib_dev *dev = to_mdev(pd->device);
 803        int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 804        struct mlx5_ib_mr *mr;
 805        void *mkc;
 806        u32 *in;
 807        int err;
 808
 809        mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 810        if (!mr)
 811                return ERR_PTR(-ENOMEM);
 812
 813        in = kzalloc(inlen, GFP_KERNEL);
 814        if (!in) {
 815                err = -ENOMEM;
 816                goto err_free;
 817        }
 818
 819        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 820
 821        MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
 822        MLX5_SET(mkc, mkc, length64, 1);
 823        set_mkc_access_pd_addr_fields(mkc, acc, 0, pd);
 824
 825        err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
 826        if (err)
 827                goto err_in;
 828
 829        kfree(in);
 830        mr->mmkey.type = MLX5_MKEY_MR;
 831        mr->ibmr.lkey = mr->mmkey.key;
 832        mr->ibmr.rkey = mr->mmkey.key;
 833        mr->umem = NULL;
 834
 835        return &mr->ibmr;
 836
 837err_in:
 838        kfree(in);
 839
 840err_free:
 841        kfree(mr);
 842
 843        return ERR_PTR(err);
 844}
 845
 846static int get_octo_len(u64 addr, u64 len, int page_shift)
 847{
 848        u64 page_size = 1ULL << page_shift;
 849        u64 offset;
 850        int npages;
 851
 852        offset = addr & (page_size - 1);
 853        npages = ALIGN(len + offset, page_size) >> page_shift;
 854        return (npages + 1) / 2;
 855}
 856
 857static int mr_cache_max_order(struct mlx5_ib_dev *dev)
 858{
 859        if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
 860                return MR_CACHE_LAST_STD_ENTRY + 2;
 861        return MLX5_MAX_UMR_SHIFT;
 862}
 863
 864static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
 865                       int access_flags, struct ib_umem **umem, int *npages,
 866                       int *page_shift, int *ncont, int *order)
 867{
 868        struct ib_umem *u;
 869
 870        *umem = NULL;
 871
 872        if (access_flags & IB_ACCESS_ON_DEMAND) {
 873                struct ib_umem_odp *odp;
 874
 875                odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags,
 876                                      &mlx5_mn_ops);
 877                if (IS_ERR(odp)) {
 878                        mlx5_ib_dbg(dev, "umem get failed (%ld)\n",
 879                                    PTR_ERR(odp));
 880                        return PTR_ERR(odp);
 881                }
 882
 883                u = &odp->umem;
 884
 885                *page_shift = odp->page_shift;
 886                *ncont = ib_umem_odp_num_pages(odp);
 887                *npages = *ncont << (*page_shift - PAGE_SHIFT);
 888                if (order)
 889                        *order = ilog2(roundup_pow_of_two(*ncont));
 890        } else {
 891                u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
 892                if (IS_ERR(u)) {
 893                        mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
 894                        return PTR_ERR(u);
 895                }
 896
 897                mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
 898                                   page_shift, ncont, order);
 899        }
 900
 901        if (!*npages) {
 902                mlx5_ib_warn(dev, "avoid zero region\n");
 903                ib_umem_release(u);
 904                return -EINVAL;
 905        }
 906
 907        *umem = u;
 908
 909        mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
 910                    *npages, *ncont, *order, *page_shift);
 911
 912        return 0;
 913}
 914
 915static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
 916{
 917        struct mlx5_ib_umr_context *context =
 918                container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe);
 919
 920        context->status = wc->status;
 921        complete(&context->done);
 922}
 923
 924static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
 925{
 926        context->cqe.done = mlx5_ib_umr_done;
 927        context->status = -1;
 928        init_completion(&context->done);
 929}
 930
 931static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,
 932                                  struct mlx5_umr_wr *umrwr)
 933{
 934        struct umr_common *umrc = &dev->umrc;
 935        const struct ib_send_wr *bad;
 936        int err;
 937        struct mlx5_ib_umr_context umr_context;
 938
 939        mlx5_ib_init_umr_context(&umr_context);
 940        umrwr->wr.wr_cqe = &umr_context.cqe;
 941
 942        down(&umrc->sem);
 943        err = ib_post_send(umrc->qp, &umrwr->wr, &bad);
 944        if (err) {
 945                mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err);
 946        } else {
 947                wait_for_completion(&umr_context.done);
 948                if (umr_context.status != IB_WC_SUCCESS) {
 949                        mlx5_ib_warn(dev, "reg umr failed (%u)\n",
 950                                     umr_context.status);
 951                        err = -EFAULT;
 952                }
 953        }
 954        up(&umrc->sem);
 955        return err;
 956}
 957
 958static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
 959                                                      unsigned int order)
 960{
 961        struct mlx5_mr_cache *cache = &dev->cache;
 962
 963        if (order < cache->ent[0].order)
 964                return &cache->ent[0];
 965        order = order - cache->ent[0].order;
 966        if (order > MR_CACHE_LAST_STD_ENTRY)
 967                return NULL;
 968        return &cache->ent[order];
 969}
 970
 971static struct mlx5_ib_mr *
 972alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
 973                    u64 len, int npages, int page_shift, unsigned int order,
 974                    int access_flags)
 975{
 976        struct mlx5_ib_dev *dev = to_mdev(pd->device);
 977        struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);
 978        struct mlx5_ib_mr *mr;
 979
 980        if (!ent)
 981                return ERR_PTR(-E2BIG);
 982        mr = get_cache_mr(ent);
 983        if (!mr) {
 984                mr = create_cache_mr(ent);
 985                if (IS_ERR(mr))
 986                        return mr;
 987        }
 988
 989        mr->ibmr.pd = pd;
 990        mr->umem = umem;
 991        mr->access_flags = access_flags;
 992        mr->desc_size = sizeof(struct mlx5_mtt);
 993        mr->mmkey.iova = virt_addr;
 994        mr->mmkey.size = len;
 995        mr->mmkey.pd = to_mpd(pd)->pdn;
 996
 997        return mr;
 998}
 999
1000#define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \
1001                            MLX5_UMR_MTT_ALIGNMENT)
1002#define MLX5_SPARE_UMR_CHUNK 0x10000
1003
1004int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
1005                       int page_shift, int flags)
1006{
1007        struct mlx5_ib_dev *dev = mr->dev;
1008        struct device *ddev = dev->ib_dev.dev.parent;
1009        int size;
1010        void *xlt;
1011        dma_addr_t dma;
1012        struct mlx5_umr_wr wr;
1013        struct ib_sge sg;
1014        int err = 0;
1015        int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
1016                               ? sizeof(struct mlx5_klm)
1017                               : sizeof(struct mlx5_mtt);
1018        const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size;
1019        const int page_mask = page_align - 1;
1020        size_t pages_mapped = 0;
1021        size_t pages_to_map = 0;
1022        size_t pages_iter = 0;
1023        size_t size_to_map = 0;
1024        gfp_t gfp;
1025        bool use_emergency_page = false;
1026
1027        if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
1028            !umr_can_use_indirect_mkey(dev))
1029                return -EPERM;
1030
1031        /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
1032         * so we need to align the offset and length accordingly
1033         */
1034        if (idx & page_mask) {
1035                npages += idx & page_mask;
1036                idx &= ~page_mask;
1037        }
1038
1039        gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL;
1040        gfp |= __GFP_ZERO | __GFP_NOWARN;
1041
1042        pages_to_map = ALIGN(npages, page_align);
1043        size = desc_size * pages_to_map;
1044        size = min_t(int, size, MLX5_MAX_UMR_CHUNK);
1045
1046        xlt = (void *)__get_free_pages(gfp, get_order(size));
1047        if (!xlt && size > MLX5_SPARE_UMR_CHUNK) {
1048                mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n",
1049                            size, get_order(size), MLX5_SPARE_UMR_CHUNK);
1050
1051                size = MLX5_SPARE_UMR_CHUNK;
1052                xlt = (void *)__get_free_pages(gfp, get_order(size));
1053        }
1054
1055        if (!xlt) {
1056                mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
1057                xlt = (void *)mlx5_ib_get_xlt_emergency_page();
1058                size = PAGE_SIZE;
1059                memset(xlt, 0, size);
1060                use_emergency_page = true;
1061        }
1062        pages_iter = size / desc_size;
1063        dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
1064        if (dma_mapping_error(ddev, dma)) {
1065                mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
1066                err = -ENOMEM;
1067                goto free_xlt;
1068        }
1069
1070        if (mr->umem->is_odp) {
1071                if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) {
1072                        struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
1073                        size_t max_pages = ib_umem_odp_num_pages(odp) - idx;
1074
1075                        pages_to_map = min_t(size_t, pages_to_map, max_pages);
1076                }
1077        }
1078
1079        sg.addr = dma;
1080        sg.lkey = dev->umrc.pd->local_dma_lkey;
1081
1082        memset(&wr, 0, sizeof(wr));
1083        wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;
1084        if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
1085                wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1086        wr.wr.sg_list = &sg;
1087        wr.wr.num_sge = 1;
1088        wr.wr.opcode = MLX5_IB_WR_UMR;
1089
1090        wr.pd = mr->ibmr.pd;
1091        wr.mkey = mr->mmkey.key;
1092        wr.length = mr->mmkey.size;
1093        wr.virt_addr = mr->mmkey.iova;
1094        wr.access_flags = mr->access_flags;
1095        wr.page_shift = page_shift;
1096
1097        for (pages_mapped = 0;
1098             pages_mapped < pages_to_map && !err;
1099             pages_mapped += pages_iter, idx += pages_iter) {
1100                npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
1101                size_to_map = npages * desc_size;
1102                dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE);
1103                if (mr->umem->is_odp) {
1104                        mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
1105                } else {
1106                        __mlx5_ib_populate_pas(dev, mr->umem, page_shift, idx,
1107                                               npages, xlt,
1108                                               MLX5_IB_MTT_PRESENT);
1109                        /* Clear padding after the pages
1110                         * brought from the umem.
1111                         */
1112                        memset(xlt + size_to_map, 0, size - size_to_map);
1113                }
1114                dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
1115
1116                sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT);
1117
1118                if (pages_mapped + pages_iter >= pages_to_map) {
1119                        if (flags & MLX5_IB_UPD_XLT_ENABLE)
1120                                wr.wr.send_flags |=
1121                                        MLX5_IB_SEND_UMR_ENABLE_MR |
1122                                        MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |
1123                                        MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1124                        if (flags & MLX5_IB_UPD_XLT_PD ||
1125                            flags & MLX5_IB_UPD_XLT_ACCESS)
1126                                wr.wr.send_flags |=
1127                                        MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1128                        if (flags & MLX5_IB_UPD_XLT_ADDR)
1129                                wr.wr.send_flags |=
1130                                        MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1131                }
1132
1133                wr.offset = idx * desc_size;
1134                wr.xlt_size = sg.length;
1135
1136                err = mlx5_ib_post_send_wait(dev, &wr);
1137        }
1138        dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1139
1140free_xlt:
1141        if (use_emergency_page)
1142                mlx5_ib_put_xlt_emergency_page();
1143        else
1144                free_pages((unsigned long)xlt, get_order(size));
1145
1146        return err;
1147}
1148
1149/*
1150 * If ibmr is NULL it will be allocated by reg_create.
1151 * Else, the given ibmr will be used.
1152 */
1153static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
1154                                     u64 virt_addr, u64 length,
1155                                     struct ib_umem *umem, int npages,
1156                                     int page_shift, int access_flags,
1157                                     bool populate)
1158{
1159        struct mlx5_ib_dev *dev = to_mdev(pd->device);
1160        struct mlx5_ib_mr *mr;
1161        __be64 *pas;
1162        void *mkc;
1163        int inlen;
1164        u32 *in;
1165        int err;
1166        bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
1167
1168        mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);
1169        if (!mr)
1170                return ERR_PTR(-ENOMEM);
1171
1172        mr->ibmr.pd = pd;
1173        mr->access_flags = access_flags;
1174
1175        inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1176        if (populate)
1177                inlen += sizeof(*pas) * roundup(npages, 2);
1178        in = kvzalloc(inlen, GFP_KERNEL);
1179        if (!in) {
1180                err = -ENOMEM;
1181                goto err_1;
1182        }
1183        pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
1184        if (populate && !(access_flags & IB_ACCESS_ON_DEMAND))
1185                mlx5_ib_populate_pas(dev, umem, page_shift, pas,
1186                                     pg_cap ? MLX5_IB_MTT_PRESENT : 0);
1187
1188        /* The pg_access bit allows setting the access flags
1189         * in the page list submitted with the command. */
1190        MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
1191
1192        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1193        MLX5_SET(mkc, mkc, free, !populate);
1194        MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
1195        if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
1196                MLX5_SET(mkc, mkc, relaxed_ordering_write,
1197                         !!(access_flags & IB_ACCESS_RELAXED_ORDERING));
1198        if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
1199                MLX5_SET(mkc, mkc, relaxed_ordering_read,
1200                         !!(access_flags & IB_ACCESS_RELAXED_ORDERING));
1201        MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
1202        MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
1203        MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
1204        MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE));
1205        MLX5_SET(mkc, mkc, lr, 1);
1206        MLX5_SET(mkc, mkc, umr_en, 1);
1207
1208        MLX5_SET64(mkc, mkc, start_addr, virt_addr);
1209        MLX5_SET64(mkc, mkc, len, length);
1210        MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1211        MLX5_SET(mkc, mkc, bsf_octword_size, 0);
1212        MLX5_SET(mkc, mkc, translations_octword_size,
1213                 get_octo_len(virt_addr, length, page_shift));
1214        MLX5_SET(mkc, mkc, log_page_size, page_shift);
1215        MLX5_SET(mkc, mkc, qpn, 0xffffff);
1216        if (populate) {
1217                MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
1218                         get_octo_len(virt_addr, length, page_shift));
1219        }
1220
1221        err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1222        if (err) {
1223                mlx5_ib_warn(dev, "create mkey failed\n");
1224                goto err_2;
1225        }
1226        mr->mmkey.type = MLX5_MKEY_MR;
1227        mr->desc_size = sizeof(struct mlx5_mtt);
1228        mr->dev = dev;
1229        kvfree(in);
1230
1231        mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
1232
1233        return mr;
1234
1235err_2:
1236        kvfree(in);
1237
1238err_1:
1239        if (!ibmr)
1240                kfree(mr);
1241
1242        return ERR_PTR(err);
1243}
1244
1245static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
1246                          int npages, u64 length, int access_flags)
1247{
1248        mr->npages = npages;
1249        atomic_add(npages, &dev->mdev->priv.reg_pages);
1250        mr->ibmr.lkey = mr->mmkey.key;
1251        mr->ibmr.rkey = mr->mmkey.key;
1252        mr->ibmr.length = length;
1253        mr->access_flags = access_flags;
1254}
1255
1256static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,
1257                                       u64 length, int acc, int mode)
1258{
1259        struct mlx5_ib_dev *dev = to_mdev(pd->device);
1260        int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1261        struct mlx5_ib_mr *mr;
1262        void *mkc;
1263        u32 *in;
1264        int err;
1265
1266        mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1267        if (!mr)
1268                return ERR_PTR(-ENOMEM);
1269
1270        in = kzalloc(inlen, GFP_KERNEL);
1271        if (!in) {
1272                err = -ENOMEM;
1273                goto err_free;
1274        }
1275
1276        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1277
1278        MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
1279        MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
1280        MLX5_SET64(mkc, mkc, len, length);
1281        set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd);
1282
1283        err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1284        if (err)
1285                goto err_in;
1286
1287        kfree(in);
1288
1289        mr->umem = NULL;
1290        set_mr_fields(dev, mr, 0, length, acc);
1291
1292        return &mr->ibmr;
1293
1294err_in:
1295        kfree(in);
1296
1297err_free:
1298        kfree(mr);
1299
1300        return ERR_PTR(err);
1301}
1302
1303int mlx5_ib_advise_mr(struct ib_pd *pd,
1304                      enum ib_uverbs_advise_mr_advice advice,
1305                      u32 flags,
1306                      struct ib_sge *sg_list,
1307                      u32 num_sge,
1308                      struct uverbs_attr_bundle *attrs)
1309{
1310        if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH &&
1311            advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE)
1312                return -EOPNOTSUPP;
1313
1314        return mlx5_ib_advise_mr_prefetch(pd, advice, flags,
1315                                         sg_list, num_sge);
1316}
1317
1318struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
1319                                struct ib_dm_mr_attr *attr,
1320                                struct uverbs_attr_bundle *attrs)
1321{
1322        struct mlx5_ib_dm *mdm = to_mdm(dm);
1323        struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev;
1324        u64 start_addr = mdm->dev_addr + attr->offset;
1325        int mode;
1326
1327        switch (mdm->type) {
1328        case MLX5_IB_UAPI_DM_TYPE_MEMIC:
1329                if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS)
1330                        return ERR_PTR(-EINVAL);
1331
1332                mode = MLX5_MKC_ACCESS_MODE_MEMIC;
1333                start_addr -= pci_resource_start(dev->pdev, 0);
1334                break;
1335        case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
1336        case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
1337                if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS)
1338                        return ERR_PTR(-EINVAL);
1339
1340                mode = MLX5_MKC_ACCESS_MODE_SW_ICM;
1341                break;
1342        default:
1343                return ERR_PTR(-EINVAL);
1344        }
1345
1346        return mlx5_ib_get_dm_mr(pd, start_addr, attr->length,
1347                                 attr->access_flags, mode);
1348}
1349
1350struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1351                                  u64 virt_addr, int access_flags,
1352                                  struct ib_udata *udata)
1353{
1354        struct mlx5_ib_dev *dev = to_mdev(pd->device);
1355        struct mlx5_ib_mr *mr = NULL;
1356        bool use_umr;
1357        struct ib_umem *umem;
1358        int page_shift;
1359        int npages;
1360        int ncont;
1361        int order;
1362        int err;
1363
1364        if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
1365                return ERR_PTR(-EOPNOTSUPP);
1366
1367        mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1368                    start, virt_addr, length, access_flags);
1369
1370        if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && !start &&
1371            length == U64_MAX) {
1372                if (virt_addr != start)
1373                        return ERR_PTR(-EINVAL);
1374                if (!(access_flags & IB_ACCESS_ON_DEMAND) ||
1375                    !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1376                        return ERR_PTR(-EINVAL);
1377
1378                mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags);
1379                if (IS_ERR(mr))
1380                        return ERR_CAST(mr);
1381                return &mr->ibmr;
1382        }
1383
1384        err = mr_umem_get(dev, start, length, access_flags, &umem,
1385                          &npages, &page_shift, &ncont, &order);
1386
1387        if (err < 0)
1388                return ERR_PTR(err);
1389
1390        use_umr = mlx5_ib_can_use_umr(dev, true, access_flags);
1391
1392        if (order <= mr_cache_max_order(dev) && use_umr) {
1393                mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
1394                                         page_shift, order, access_flags);
1395                if (PTR_ERR(mr) == -EAGAIN) {
1396                        mlx5_ib_dbg(dev, "cache empty for order %d\n", order);
1397                        mr = NULL;
1398                }
1399        } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) {
1400                if (access_flags & IB_ACCESS_ON_DEMAND) {
1401                        err = -EINVAL;
1402                        pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n");
1403                        goto error;
1404                }
1405                use_umr = false;
1406        }
1407
1408        if (!mr) {
1409                mutex_lock(&dev->slow_path_mutex);
1410                mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
1411                                page_shift, access_flags, !use_umr);
1412                mutex_unlock(&dev->slow_path_mutex);
1413        }
1414
1415        if (IS_ERR(mr)) {
1416                err = PTR_ERR(mr);
1417                goto error;
1418        }
1419
1420        mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);
1421
1422        mr->umem = umem;
1423        set_mr_fields(dev, mr, npages, length, access_flags);
1424
1425        if (use_umr) {
1426                int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
1427
1428                if (access_flags & IB_ACCESS_ON_DEMAND)
1429                        update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
1430
1431                err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
1432                                         update_xlt_flags);
1433
1434                if (err) {
1435                        dereg_mr(dev, mr);
1436                        return ERR_PTR(err);
1437                }
1438        }
1439
1440        if (is_odp_mr(mr)) {
1441                to_ib_umem_odp(mr->umem)->private = mr;
1442                init_waitqueue_head(&mr->q_deferred_work);
1443                atomic_set(&mr->num_deferred_work, 0);
1444                err = xa_err(xa_store(&dev->odp_mkeys,
1445                                      mlx5_base_mkey(mr->mmkey.key), &mr->mmkey,
1446                                      GFP_KERNEL));
1447                if (err) {
1448                        dereg_mr(dev, mr);
1449                        return ERR_PTR(err);
1450                }
1451        }
1452
1453        return &mr->ibmr;
1454error:
1455        ib_umem_release(umem);
1456        return ERR_PTR(err);
1457}
1458
1459/**
1460 * mlx5_mr_cache_invalidate - Fence all DMA on the MR
1461 * @mr: The MR to fence
1462 *
1463 * Upon return the NIC will not be doing any DMA to the pages under the MR,
1464 * and any DMA inprogress will be completed. Failure of this function
1465 * indicates the HW has failed catastrophically.
1466 */
1467int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr)
1468{
1469        struct mlx5_umr_wr umrwr = {};
1470
1471        if (mr->dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1472                return 0;
1473
1474        umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
1475                              MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1476        umrwr.wr.opcode = MLX5_IB_WR_UMR;
1477        umrwr.pd = mr->dev->umrc.pd;
1478        umrwr.mkey = mr->mmkey.key;
1479        umrwr.ignore_free_state = 1;
1480
1481        return mlx5_ib_post_send_wait(mr->dev, &umrwr);
1482}
1483
1484static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1485                     int access_flags, int flags)
1486{
1487        struct mlx5_ib_dev *dev = to_mdev(pd->device);
1488        struct mlx5_umr_wr umrwr = {};
1489        int err;
1490
1491        umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1492
1493        umrwr.wr.opcode = MLX5_IB_WR_UMR;
1494        umrwr.mkey = mr->mmkey.key;
1495
1496        if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
1497                umrwr.pd = pd;
1498                umrwr.access_flags = access_flags;
1499                umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1500        }
1501
1502        err = mlx5_ib_post_send_wait(dev, &umrwr);
1503
1504        return err;
1505}
1506
1507int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1508                          u64 length, u64 virt_addr, int new_access_flags,
1509                          struct ib_pd *new_pd, struct ib_udata *udata)
1510{
1511        struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
1512        struct mlx5_ib_mr *mr = to_mmr(ib_mr);
1513        struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd;
1514        int access_flags = flags & IB_MR_REREG_ACCESS ?
1515                            new_access_flags :
1516                            mr->access_flags;
1517        int page_shift = 0;
1518        int upd_flags = 0;
1519        int npages = 0;
1520        int ncont = 0;
1521        int order = 0;
1522        u64 addr, len;
1523        int err;
1524
1525        mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1526                    start, virt_addr, length, access_flags);
1527
1528        atomic_sub(mr->npages, &dev->mdev->priv.reg_pages);
1529
1530        if (!mr->umem)
1531                return -EINVAL;
1532
1533        if (is_odp_mr(mr))
1534                return -EOPNOTSUPP;
1535
1536        if (flags & IB_MR_REREG_TRANS) {
1537                addr = virt_addr;
1538                len = length;
1539        } else {
1540                addr = mr->umem->address;
1541                len = mr->umem->length;
1542        }
1543
1544        if (flags != IB_MR_REREG_PD) {
1545                /*
1546                 * Replace umem. This needs to be done whether or not UMR is
1547                 * used.
1548                 */
1549                flags |= IB_MR_REREG_TRANS;
1550                ib_umem_release(mr->umem);
1551                mr->umem = NULL;
1552                err = mr_umem_get(dev, addr, len, access_flags, &mr->umem,
1553                                  &npages, &page_shift, &ncont, &order);
1554                if (err)
1555                        goto err;
1556        }
1557
1558        if (!mlx5_ib_can_use_umr(dev, true, access_flags) ||
1559            (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len))) {
1560                /*
1561                 * UMR can't be used - MKey needs to be replaced.
1562                 */
1563                if (mr->cache_ent)
1564                        detach_mr_from_cache(mr);
1565                err = destroy_mkey(dev, mr);
1566                if (err)
1567                        goto err;
1568
1569                mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
1570                                page_shift, access_flags, true);
1571
1572                if (IS_ERR(mr)) {
1573                        err = PTR_ERR(mr);
1574                        mr = to_mmr(ib_mr);
1575                        goto err;
1576                }
1577        } else {
1578                /*
1579                 * Send a UMR WQE
1580                 */
1581                mr->ibmr.pd = pd;
1582                mr->access_flags = access_flags;
1583                mr->mmkey.iova = addr;
1584                mr->mmkey.size = len;
1585                mr->mmkey.pd = to_mpd(pd)->pdn;
1586
1587                if (flags & IB_MR_REREG_TRANS) {
1588                        upd_flags = MLX5_IB_UPD_XLT_ADDR;
1589                        if (flags & IB_MR_REREG_PD)
1590                                upd_flags |= MLX5_IB_UPD_XLT_PD;
1591                        if (flags & IB_MR_REREG_ACCESS)
1592                                upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
1593                        err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
1594                                                 upd_flags);
1595                } else {
1596                        err = rereg_umr(pd, mr, access_flags, flags);
1597                }
1598
1599                if (err)
1600                        goto err;
1601        }
1602
1603        set_mr_fields(dev, mr, npages, len, access_flags);
1604
1605        return 0;
1606
1607err:
1608        ib_umem_release(mr->umem);
1609        mr->umem = NULL;
1610
1611        clean_mr(dev, mr);
1612        return err;
1613}
1614
1615static int
1616mlx5_alloc_priv_descs(struct ib_device *device,
1617                      struct mlx5_ib_mr *mr,
1618                      int ndescs,
1619                      int desc_size)
1620{
1621        int size = ndescs * desc_size;
1622        int add_size;
1623        int ret;
1624
1625        add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0);
1626
1627        mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL);
1628        if (!mr->descs_alloc)
1629                return -ENOMEM;
1630
1631        mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN);
1632
1633        mr->desc_map = dma_map_single(device->dev.parent, mr->descs,
1634                                      size, DMA_TO_DEVICE);
1635        if (dma_mapping_error(device->dev.parent, mr->desc_map)) {
1636                ret = -ENOMEM;
1637                goto err;
1638        }
1639
1640        return 0;
1641err:
1642        kfree(mr->descs_alloc);
1643
1644        return ret;
1645}
1646
1647static void
1648mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
1649{
1650        if (mr->descs) {
1651                struct ib_device *device = mr->ibmr.device;
1652                int size = mr->max_descs * mr->desc_size;
1653
1654                dma_unmap_single(device->dev.parent, mr->desc_map,
1655                                 size, DMA_TO_DEVICE);
1656                kfree(mr->descs_alloc);
1657                mr->descs = NULL;
1658        }
1659}
1660
1661static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1662{
1663        if (mr->sig) {
1664                if (mlx5_core_destroy_psv(dev->mdev,
1665                                          mr->sig->psv_memory.psv_idx))
1666                        mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1667                                     mr->sig->psv_memory.psv_idx);
1668                if (mlx5_core_destroy_psv(dev->mdev,
1669                                          mr->sig->psv_wire.psv_idx))
1670                        mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1671                                     mr->sig->psv_wire.psv_idx);
1672                xa_erase(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key));
1673                kfree(mr->sig);
1674                mr->sig = NULL;
1675        }
1676
1677        if (!mr->cache_ent) {
1678                destroy_mkey(dev, mr);
1679                mlx5_free_priv_descs(mr);
1680        }
1681}
1682
1683static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1684{
1685        int npages = mr->npages;
1686        struct ib_umem *umem = mr->umem;
1687
1688        /* Stop all DMA */
1689        if (is_odp_mr(mr))
1690                mlx5_ib_fence_odp_mr(mr);
1691        else
1692                clean_mr(dev, mr);
1693
1694        if (mr->cache_ent)
1695                mlx5_mr_cache_free(dev, mr);
1696        else
1697                kfree(mr);
1698
1699        ib_umem_release(umem);
1700        atomic_sub(npages, &dev->mdev->priv.reg_pages);
1701
1702}
1703
1704int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1705{
1706        struct mlx5_ib_mr *mmr = to_mmr(ibmr);
1707
1708        if (ibmr->type == IB_MR_TYPE_INTEGRITY) {
1709                dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr);
1710                dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr);
1711        }
1712
1713        if (is_odp_mr(mmr) && to_ib_umem_odp(mmr->umem)->is_implicit_odp) {
1714                mlx5_ib_free_implicit_mr(mmr);
1715                return 0;
1716        }
1717
1718        dereg_mr(to_mdev(ibmr->device), mmr);
1719
1720        return 0;
1721}
1722
1723static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs,
1724                                   int access_mode, int page_shift)
1725{
1726        void *mkc;
1727
1728        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1729
1730        MLX5_SET(mkc, mkc, free, 1);
1731        MLX5_SET(mkc, mkc, qpn, 0xffffff);
1732        MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1733        MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1734        MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);
1735        MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);
1736        MLX5_SET(mkc, mkc, umr_en, 1);
1737        MLX5_SET(mkc, mkc, log_page_size, page_shift);
1738}
1739
1740static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1741                                  int ndescs, int desc_size, int page_shift,
1742                                  int access_mode, u32 *in, int inlen)
1743{
1744        struct mlx5_ib_dev *dev = to_mdev(pd->device);
1745        int err;
1746
1747        mr->access_mode = access_mode;
1748        mr->desc_size = desc_size;
1749        mr->max_descs = ndescs;
1750
1751        err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size);
1752        if (err)
1753                return err;
1754
1755        mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift);
1756
1757        err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1758        if (err)
1759                goto err_free_descs;
1760
1761        mr->mmkey.type = MLX5_MKEY_MR;
1762        mr->ibmr.lkey = mr->mmkey.key;
1763        mr->ibmr.rkey = mr->mmkey.key;
1764
1765        return 0;
1766
1767err_free_descs:
1768        mlx5_free_priv_descs(mr);
1769        return err;
1770}
1771
1772static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,
1773                                u32 max_num_sg, u32 max_num_meta_sg,
1774                                int desc_size, int access_mode)
1775{
1776        int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1777        int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);
1778        int page_shift = 0;
1779        struct mlx5_ib_mr *mr;
1780        u32 *in;
1781        int err;
1782
1783        mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1784        if (!mr)
1785                return ERR_PTR(-ENOMEM);
1786
1787        mr->ibmr.pd = pd;
1788        mr->ibmr.device = pd->device;
1789
1790        in = kzalloc(inlen, GFP_KERNEL);
1791        if (!in) {
1792                err = -ENOMEM;
1793                goto err_free;
1794        }
1795
1796        if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)
1797                page_shift = PAGE_SHIFT;
1798
1799        err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift,
1800                                     access_mode, in, inlen);
1801        if (err)
1802                goto err_free_in;
1803
1804        mr->umem = NULL;
1805        kfree(in);
1806
1807        return mr;
1808
1809err_free_in:
1810        kfree(in);
1811err_free:
1812        kfree(mr);
1813        return ERR_PTR(err);
1814}
1815
1816static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1817                                    int ndescs, u32 *in, int inlen)
1818{
1819        return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt),
1820                                      PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in,
1821                                      inlen);
1822}
1823
1824static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1825                                    int ndescs, u32 *in, int inlen)
1826{
1827        return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm),
1828                                      0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
1829}
1830
1831static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1832                                      int max_num_sg, int max_num_meta_sg,
1833                                      u32 *in, int inlen)
1834{
1835        struct mlx5_ib_dev *dev = to_mdev(pd->device);
1836        u32 psv_index[2];
1837        void *mkc;
1838        int err;
1839
1840        mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
1841        if (!mr->sig)
1842                return -ENOMEM;
1843
1844        /* create mem & wire PSVs */
1845        err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index);
1846        if (err)
1847                goto err_free_sig;
1848
1849        mr->sig->psv_memory.psv_idx = psv_index[0];
1850        mr->sig->psv_wire.psv_idx = psv_index[1];
1851
1852        mr->sig->sig_status_checked = true;
1853        mr->sig->sig_err_exists = false;
1854        /* Next UMR, Arm SIGERR */
1855        ++mr->sig->sigerr_count;
1856        mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
1857                                         sizeof(struct mlx5_klm),
1858                                         MLX5_MKC_ACCESS_MODE_KLMS);
1859        if (IS_ERR(mr->klm_mr)) {
1860                err = PTR_ERR(mr->klm_mr);
1861                goto err_destroy_psv;
1862        }
1863        mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
1864                                         sizeof(struct mlx5_mtt),
1865                                         MLX5_MKC_ACCESS_MODE_MTT);
1866        if (IS_ERR(mr->mtt_mr)) {
1867                err = PTR_ERR(mr->mtt_mr);
1868                goto err_free_klm_mr;
1869        }
1870
1871        /* Set bsf descriptors for mkey */
1872        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1873        MLX5_SET(mkc, mkc, bsf_en, 1);
1874        MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
1875
1876        err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0,
1877                                     MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
1878        if (err)
1879                goto err_free_mtt_mr;
1880
1881        err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key),
1882                              mr->sig, GFP_KERNEL));
1883        if (err)
1884                goto err_free_descs;
1885        return 0;
1886
1887err_free_descs:
1888        destroy_mkey(dev, mr);
1889        mlx5_free_priv_descs(mr);
1890err_free_mtt_mr:
1891        dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);
1892        mr->mtt_mr = NULL;
1893err_free_klm_mr:
1894        dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr);
1895        mr->klm_mr = NULL;
1896err_destroy_psv:
1897        if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx))
1898                mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1899                             mr->sig->psv_memory.psv_idx);
1900        if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx))
1901                mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1902                             mr->sig->psv_wire.psv_idx);
1903err_free_sig:
1904        kfree(mr->sig);
1905
1906        return err;
1907}
1908
1909static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,
1910                                        enum ib_mr_type mr_type, u32 max_num_sg,
1911                                        u32 max_num_meta_sg)
1912{
1913        struct mlx5_ib_dev *dev = to_mdev(pd->device);
1914        int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1915        int ndescs = ALIGN(max_num_sg, 4);
1916        struct mlx5_ib_mr *mr;
1917        u32 *in;
1918        int err;
1919
1920        mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1921        if (!mr)
1922                return ERR_PTR(-ENOMEM);
1923
1924        in = kzalloc(inlen, GFP_KERNEL);
1925        if (!in) {
1926                err = -ENOMEM;
1927                goto err_free;
1928        }
1929
1930        mr->ibmr.device = pd->device;
1931        mr->umem = NULL;
1932
1933        switch (mr_type) {
1934        case IB_MR_TYPE_MEM_REG:
1935                err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen);
1936                break;
1937        case IB_MR_TYPE_SG_GAPS:
1938                err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen);
1939                break;
1940        case IB_MR_TYPE_INTEGRITY:
1941                err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg,
1942                                                 max_num_meta_sg, in, inlen);
1943                break;
1944        default:
1945                mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
1946                err = -EINVAL;
1947        }
1948
1949        if (err)
1950                goto err_free_in;
1951
1952        kfree(in);
1953
1954        return &mr->ibmr;
1955
1956err_free_in:
1957        kfree(in);
1958err_free:
1959        kfree(mr);
1960        return ERR_PTR(err);
1961}
1962
1963struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1964                               u32 max_num_sg, struct ib_udata *udata)
1965{
1966        return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);
1967}
1968
1969struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
1970                                         u32 max_num_sg, u32 max_num_meta_sg)
1971{
1972        return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg,
1973                                  max_num_meta_sg);
1974}
1975
1976struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
1977                               struct ib_udata *udata)
1978{
1979        struct mlx5_ib_dev *dev = to_mdev(pd->device);
1980        int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1981        struct mlx5_ib_mw *mw = NULL;
1982        u32 *in = NULL;
1983        void *mkc;
1984        int ndescs;
1985        int err;
1986        struct mlx5_ib_alloc_mw req = {};
1987        struct {
1988                __u32   comp_mask;
1989                __u32   response_length;
1990        } resp = {};
1991
1992        err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1993        if (err)
1994                return ERR_PTR(err);
1995
1996        if (req.comp_mask || req.reserved1 || req.reserved2)
1997                return ERR_PTR(-EOPNOTSUPP);
1998
1999        if (udata->inlen > sizeof(req) &&
2000            !ib_is_udata_cleared(udata, sizeof(req),
2001                                 udata->inlen - sizeof(req)))
2002                return ERR_PTR(-EOPNOTSUPP);
2003
2004        ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
2005
2006        mw = kzalloc(sizeof(*mw), GFP_KERNEL);
2007        in = kzalloc(inlen, GFP_KERNEL);
2008        if (!mw || !in) {
2009                err = -ENOMEM;
2010                goto free;
2011        }
2012
2013        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
2014
2015        MLX5_SET(mkc, mkc, free, 1);
2016        MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
2017        MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
2018        MLX5_SET(mkc, mkc, umr_en, 1);
2019        MLX5_SET(mkc, mkc, lr, 1);
2020        MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
2021        MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2)));
2022        MLX5_SET(mkc, mkc, qpn, 0xffffff);
2023
2024        err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen);
2025        if (err)
2026                goto free;
2027
2028        mw->mmkey.type = MLX5_MKEY_MW;
2029        mw->ibmw.rkey = mw->mmkey.key;
2030        mw->ndescs = ndescs;
2031
2032        resp.response_length = min(offsetof(typeof(resp), response_length) +
2033                                   sizeof(resp.response_length), udata->outlen);
2034        if (resp.response_length) {
2035                err = ib_copy_to_udata(udata, &resp, resp.response_length);
2036                if (err) {
2037                        mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
2038                        goto free;
2039                }
2040        }
2041
2042        if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
2043                err = xa_err(xa_store(&dev->odp_mkeys,
2044                                      mlx5_base_mkey(mw->mmkey.key), &mw->mmkey,
2045                                      GFP_KERNEL));
2046                if (err)
2047                        goto free_mkey;
2048        }
2049
2050        kfree(in);
2051        return &mw->ibmw;
2052
2053free_mkey:
2054        mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
2055free:
2056        kfree(mw);
2057        kfree(in);
2058        return ERR_PTR(err);
2059}
2060
2061int mlx5_ib_dealloc_mw(struct ib_mw *mw)
2062{
2063        struct mlx5_ib_dev *dev = to_mdev(mw->device);
2064        struct mlx5_ib_mw *mmw = to_mmw(mw);
2065        int err;
2066
2067        if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
2068                xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key));
2069                /*
2070                 * pagefault_single_data_segment() may be accessing mmw under
2071                 * SRCU if the user bound an ODP MR to this MW.
2072                 */
2073                synchronize_srcu(&dev->odp_srcu);
2074        }
2075
2076        err = mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey);
2077        if (err)
2078                return err;
2079        kfree(mmw);
2080        return 0;
2081}
2082
2083int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
2084                            struct ib_mr_status *mr_status)
2085{
2086        struct mlx5_ib_mr *mmr = to_mmr(ibmr);
2087        int ret = 0;
2088
2089        if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
2090                pr_err("Invalid status check mask\n");
2091                ret = -EINVAL;
2092                goto done;
2093        }
2094
2095        mr_status->fail_status = 0;
2096        if (check_mask & IB_MR_CHECK_SIG_STATUS) {
2097                if (!mmr->sig) {
2098                        ret = -EINVAL;
2099                        pr_err("signature status check requested on a non-signature enabled MR\n");
2100                        goto done;
2101                }
2102
2103                mmr->sig->sig_status_checked = true;
2104                if (!mmr->sig->sig_err_exists)
2105                        goto done;
2106
2107                if (ibmr->lkey == mmr->sig->err_item.key)
2108                        memcpy(&mr_status->sig_err, &mmr->sig->err_item,
2109                               sizeof(mr_status->sig_err));
2110                else {
2111                        mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
2112                        mr_status->sig_err.sig_err_offset = 0;
2113                        mr_status->sig_err.key = mmr->sig->err_item.key;
2114                }
2115
2116                mmr->sig->sig_err_exists = false;
2117                mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
2118        }
2119
2120done:
2121        return ret;
2122}
2123
2124static int
2125mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2126                        int data_sg_nents, unsigned int *data_sg_offset,
2127                        struct scatterlist *meta_sg, int meta_sg_nents,
2128                        unsigned int *meta_sg_offset)
2129{
2130        struct mlx5_ib_mr *mr = to_mmr(ibmr);
2131        unsigned int sg_offset = 0;
2132        int n = 0;
2133
2134        mr->meta_length = 0;
2135        if (data_sg_nents == 1) {
2136                n++;
2137                mr->ndescs = 1;
2138                if (data_sg_offset)
2139                        sg_offset = *data_sg_offset;
2140                mr->data_length = sg_dma_len(data_sg) - sg_offset;
2141                mr->data_iova = sg_dma_address(data_sg) + sg_offset;
2142                if (meta_sg_nents == 1) {
2143                        n++;
2144                        mr->meta_ndescs = 1;
2145                        if (meta_sg_offset)
2146                                sg_offset = *meta_sg_offset;
2147                        else
2148                                sg_offset = 0;
2149                        mr->meta_length = sg_dma_len(meta_sg) - sg_offset;
2150                        mr->pi_iova = sg_dma_address(meta_sg) + sg_offset;
2151                }
2152                ibmr->length = mr->data_length + mr->meta_length;
2153        }
2154
2155        return n;
2156}
2157
2158static int
2159mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
2160                   struct scatterlist *sgl,
2161                   unsigned short sg_nents,
2162                   unsigned int *sg_offset_p,
2163                   struct scatterlist *meta_sgl,
2164                   unsigned short meta_sg_nents,
2165                   unsigned int *meta_sg_offset_p)
2166{
2167        struct scatterlist *sg = sgl;
2168        struct mlx5_klm *klms = mr->descs;
2169        unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
2170        u32 lkey = mr->ibmr.pd->local_dma_lkey;
2171        int i, j = 0;
2172
2173        mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
2174        mr->ibmr.length = 0;
2175
2176        for_each_sg(sgl, sg, sg_nents, i) {
2177                if (unlikely(i >= mr->max_descs))
2178                        break;
2179                klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset);
2180                klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset);
2181                klms[i].key = cpu_to_be32(lkey);
2182                mr->ibmr.length += sg_dma_len(sg) - sg_offset;
2183
2184                sg_offset = 0;
2185        }
2186
2187        if (sg_offset_p)
2188                *sg_offset_p = sg_offset;
2189
2190        mr->ndescs = i;
2191        mr->data_length = mr->ibmr.length;
2192
2193        if (meta_sg_nents) {
2194                sg = meta_sgl;
2195                sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0;
2196                for_each_sg(meta_sgl, sg, meta_sg_nents, j) {
2197                        if (unlikely(i + j >= mr->max_descs))
2198                                break;
2199                        klms[i + j].va = cpu_to_be64(sg_dma_address(sg) +
2200                                                     sg_offset);
2201                        klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) -
2202                                                         sg_offset);
2203                        klms[i + j].key = cpu_to_be32(lkey);
2204                        mr->ibmr.length += sg_dma_len(sg) - sg_offset;
2205
2206                        sg_offset = 0;
2207                }
2208                if (meta_sg_offset_p)
2209                        *meta_sg_offset_p = sg_offset;
2210
2211                mr->meta_ndescs = j;
2212                mr->meta_length = mr->ibmr.length - mr->data_length;
2213        }
2214
2215        return i + j;
2216}
2217
2218static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
2219{
2220        struct mlx5_ib_mr *mr = to_mmr(ibmr);
2221        __be64 *descs;
2222
2223        if (unlikely(mr->ndescs == mr->max_descs))
2224                return -ENOMEM;
2225
2226        descs = mr->descs;
2227        descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
2228
2229        return 0;
2230}
2231
2232static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr)
2233{
2234        struct mlx5_ib_mr *mr = to_mmr(ibmr);
2235        __be64 *descs;
2236
2237        if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs))
2238                return -ENOMEM;
2239
2240        descs = mr->descs;
2241        descs[mr->ndescs + mr->meta_ndescs++] =
2242                cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
2243
2244        return 0;
2245}
2246
2247static int
2248mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2249                         int data_sg_nents, unsigned int *data_sg_offset,
2250                         struct scatterlist *meta_sg, int meta_sg_nents,
2251                         unsigned int *meta_sg_offset)
2252{
2253        struct mlx5_ib_mr *mr = to_mmr(ibmr);
2254        struct mlx5_ib_mr *pi_mr = mr->mtt_mr;
2255        int n;
2256
2257        pi_mr->ndescs = 0;
2258        pi_mr->meta_ndescs = 0;
2259        pi_mr->meta_length = 0;
2260
2261        ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
2262                                   pi_mr->desc_size * pi_mr->max_descs,
2263                                   DMA_TO_DEVICE);
2264
2265        pi_mr->ibmr.page_size = ibmr->page_size;
2266        n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset,
2267                           mlx5_set_page);
2268        if (n != data_sg_nents)
2269                return n;
2270
2271        pi_mr->data_iova = pi_mr->ibmr.iova;
2272        pi_mr->data_length = pi_mr->ibmr.length;
2273        pi_mr->ibmr.length = pi_mr->data_length;
2274        ibmr->length = pi_mr->data_length;
2275
2276        if (meta_sg_nents) {
2277                u64 page_mask = ~((u64)ibmr->page_size - 1);
2278                u64 iova = pi_mr->data_iova;
2279
2280                n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents,
2281                                    meta_sg_offset, mlx5_set_page_pi);
2282
2283                pi_mr->meta_length = pi_mr->ibmr.length;
2284                /*
2285                 * PI address for the HW is the offset of the metadata address
2286                 * relative to the first data page address.
2287                 * It equals to first data page address + size of data pages +
2288                 * metadata offset at the first metadata page
2289                 */
2290                pi_mr->pi_iova = (iova & page_mask) +
2291                                 pi_mr->ndescs * ibmr->page_size +
2292                                 (pi_mr->ibmr.iova & ~page_mask);
2293                /*
2294                 * In order to use one MTT MR for data and metadata, we register
2295                 * also the gaps between the end of the data and the start of
2296                 * the metadata (the sig MR will verify that the HW will access
2297                 * to right addresses). This mapping is safe because we use
2298                 * internal mkey for the registration.
2299                 */
2300                pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova;
2301                pi_mr->ibmr.iova = iova;
2302                ibmr->length += pi_mr->meta_length;
2303        }
2304
2305        ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
2306                                      pi_mr->desc_size * pi_mr->max_descs,
2307                                      DMA_TO_DEVICE);
2308
2309        return n;
2310}
2311
2312static int
2313mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2314                         int data_sg_nents, unsigned int *data_sg_offset,
2315                         struct scatterlist *meta_sg, int meta_sg_nents,
2316                         unsigned int *meta_sg_offset)
2317{
2318        struct mlx5_ib_mr *mr = to_mmr(ibmr);
2319        struct mlx5_ib_mr *pi_mr = mr->klm_mr;
2320        int n;
2321
2322        pi_mr->ndescs = 0;
2323        pi_mr->meta_ndescs = 0;
2324        pi_mr->meta_length = 0;
2325
2326        ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
2327                                   pi_mr->desc_size * pi_mr->max_descs,
2328                                   DMA_TO_DEVICE);
2329
2330        n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset,
2331                               meta_sg, meta_sg_nents, meta_sg_offset);
2332
2333        ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
2334                                      pi_mr->desc_size * pi_mr->max_descs,
2335                                      DMA_TO_DEVICE);
2336
2337        /* This is zero-based memory region */
2338        pi_mr->data_iova = 0;
2339        pi_mr->ibmr.iova = 0;
2340        pi_mr->pi_iova = pi_mr->data_length;
2341        ibmr->length = pi_mr->ibmr.length;
2342
2343        return n;
2344}
2345
2346int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2347                         int data_sg_nents, unsigned int *data_sg_offset,
2348                         struct scatterlist *meta_sg, int meta_sg_nents,
2349                         unsigned int *meta_sg_offset)
2350{
2351        struct mlx5_ib_mr *mr = to_mmr(ibmr);
2352        struct mlx5_ib_mr *pi_mr = NULL;
2353        int n;
2354
2355        WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY);
2356
2357        mr->ndescs = 0;
2358        mr->data_length = 0;
2359        mr->data_iova = 0;
2360        mr->meta_ndescs = 0;
2361        mr->pi_iova = 0;
2362        /*
2363         * As a performance optimization, if possible, there is no need to
2364         * perform UMR operation to register the data/metadata buffers.
2365         * First try to map the sg lists to PA descriptors with local_dma_lkey.
2366         * Fallback to UMR only in case of a failure.
2367         */
2368        n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2369                                    data_sg_offset, meta_sg, meta_sg_nents,
2370                                    meta_sg_offset);
2371        if (n == data_sg_nents + meta_sg_nents)
2372                goto out;
2373        /*
2374         * As a performance optimization, if possible, there is no need to map
2375         * the sg lists to KLM descriptors. First try to map the sg lists to MTT
2376         * descriptors and fallback to KLM only in case of a failure.
2377         * It's more efficient for the HW to work with MTT descriptors
2378         * (especially in high load).
2379         * Use KLM (indirect access) only if it's mandatory.
2380         */
2381        pi_mr = mr->mtt_mr;
2382        n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2383                                     data_sg_offset, meta_sg, meta_sg_nents,
2384                                     meta_sg_offset);
2385        if (n == data_sg_nents + meta_sg_nents)
2386                goto out;
2387
2388        pi_mr = mr->klm_mr;
2389        n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2390                                     data_sg_offset, meta_sg, meta_sg_nents,
2391                                     meta_sg_offset);
2392        if (unlikely(n != data_sg_nents + meta_sg_nents))
2393                return -ENOMEM;
2394
2395out:
2396        /* This is zero-based memory region */
2397        ibmr->iova = 0;
2398        mr->pi_mr = pi_mr;
2399        if (pi_mr)
2400                ibmr->sig_attrs->meta_length = pi_mr->meta_length;
2401        else
2402                ibmr->sig_attrs->meta_length = mr->meta_length;
2403
2404        return 0;
2405}
2406
2407int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
2408                      unsigned int *sg_offset)
2409{
2410        struct mlx5_ib_mr *mr = to_mmr(ibmr);
2411        int n;
2412
2413        mr->ndescs = 0;
2414
2415        ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map,
2416                                   mr->desc_size * mr->max_descs,
2417                                   DMA_TO_DEVICE);
2418
2419        if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
2420                n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0,
2421                                       NULL);
2422        else
2423                n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
2424                                mlx5_set_page);
2425
2426        ib_dma_sync_single_for_device(ibmr->device, mr->desc_map,
2427                                      mr->desc_size * mr->max_descs,
2428                                      DMA_TO_DEVICE);
2429
2430        return n;
2431}
2432