1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/kref.h>
35#include <linux/random.h>
36#include <linux/debugfs.h>
37#include <linux/export.h>
38#include <linux/delay.h>
39#include <rdma/ib_umem.h>
40#include <rdma/ib_umem_odp.h>
41#include <rdma/ib_verbs.h>
42#include "mlx5_ib.h"
43
44enum {
45 MAX_PENDING_REG_MR = 8,
46};
47
48#define MLX5_UMR_ALIGN 2048
49
50static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
51static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
52static int mr_cache_max_order(struct mlx5_ib_dev *dev);
53static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
54static bool umr_can_modify_entity_size(struct mlx5_ib_dev *dev)
55{
56 return !MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled);
57}
58
59static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
60{
61 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled);
62}
63
64static bool use_umr(struct mlx5_ib_dev *dev, int order)
65{
66 return order <= mr_cache_max_order(dev) &&
67 umr_can_modify_entity_size(dev);
68}
69
70static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
71{
72 int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
73
74#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
75
76 synchronize_srcu(&dev->mr_srcu);
77#endif
78
79 return err;
80}
81
82static int order2idx(struct mlx5_ib_dev *dev, int order)
83{
84 struct mlx5_mr_cache *cache = &dev->cache;
85
86 if (order < cache->ent[0].order)
87 return 0;
88 else
89 return order - cache->ent[0].order;
90}
91
92static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length)
93{
94 return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
95 length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1));
96}
97
98#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
99static void update_odp_mr(struct mlx5_ib_mr *mr)
100{
101 if (mr->umem->odp_data) {
102
103
104
105
106
107
108
109 smp_wmb();
110 mr->umem->odp_data->private = mr;
111
112
113
114
115
116
117
118
119
120
121 smp_wmb();
122 }
123}
124#endif
125
126static void reg_mr_callback(int status, void *context)
127{
128 struct mlx5_ib_mr *mr = context;
129 struct mlx5_ib_dev *dev = mr->dev;
130 struct mlx5_mr_cache *cache = &dev->cache;
131 int c = order2idx(dev, mr->order);
132 struct mlx5_cache_ent *ent = &cache->ent[c];
133 u8 key;
134 unsigned long flags;
135 struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table;
136 int err;
137
138 spin_lock_irqsave(&ent->lock, flags);
139 ent->pending--;
140 spin_unlock_irqrestore(&ent->lock, flags);
141 if (status) {
142 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
143 kfree(mr);
144 dev->fill_delay = 1;
145 mod_timer(&dev->delay_timer, jiffies + HZ);
146 return;
147 }
148
149 mr->mmkey.type = MLX5_MKEY_MR;
150 spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags);
151 key = dev->mdev->priv.mkey_key++;
152 spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags);
153 mr->mmkey.key = mlx5_idx_to_mkey(MLX5_GET(create_mkey_out, mr->out, mkey_index)) | key;
154
155 cache->last_add = jiffies;
156
157 spin_lock_irqsave(&ent->lock, flags);
158 list_add_tail(&mr->list, &ent->head);
159 ent->cur++;
160 ent->size++;
161 spin_unlock_irqrestore(&ent->lock, flags);
162
163 write_lock_irqsave(&table->lock, flags);
164 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmkey.key),
165 &mr->mmkey);
166 if (err)
167 pr_err("Error inserting to mkey tree. 0x%x\n", -err);
168 write_unlock_irqrestore(&table->lock, flags);
169
170 if (!completion_done(&ent->compl))
171 complete(&ent->compl);
172}
173
174static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
175{
176 struct mlx5_mr_cache *cache = &dev->cache;
177 struct mlx5_cache_ent *ent = &cache->ent[c];
178 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
179 struct mlx5_ib_mr *mr;
180 void *mkc;
181 u32 *in;
182 int err = 0;
183 int i;
184
185 in = kzalloc(inlen, GFP_KERNEL);
186 if (!in)
187 return -ENOMEM;
188
189 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
190 for (i = 0; i < num; i++) {
191 if (ent->pending >= MAX_PENDING_REG_MR) {
192 err = -EAGAIN;
193 break;
194 }
195
196 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
197 if (!mr) {
198 err = -ENOMEM;
199 break;
200 }
201 mr->order = ent->order;
202 mr->allocated_from_cache = 1;
203 mr->dev = dev;
204
205 MLX5_SET(mkc, mkc, free, 1);
206 MLX5_SET(mkc, mkc, umr_en, 1);
207 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
208 MLX5_SET(mkc, mkc, access_mode_4_2,
209 (ent->access_mode >> 2) & 0x7);
210
211 MLX5_SET(mkc, mkc, qpn, 0xffffff);
212 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
213 MLX5_SET(mkc, mkc, log_page_size, ent->page);
214
215 spin_lock_irq(&ent->lock);
216 ent->pending++;
217 spin_unlock_irq(&ent->lock);
218 err = mlx5_core_create_mkey_cb(dev->mdev, &mr->mmkey,
219 in, inlen,
220 mr->out, sizeof(mr->out),
221 reg_mr_callback, mr);
222 if (err) {
223 spin_lock_irq(&ent->lock);
224 ent->pending--;
225 spin_unlock_irq(&ent->lock);
226 mlx5_ib_warn(dev, "create mkey failed %d\n", err);
227 kfree(mr);
228 break;
229 }
230 }
231
232 kfree(in);
233 return err;
234}
235
236static void remove_keys(struct mlx5_ib_dev *dev, int c, int num)
237{
238 struct mlx5_mr_cache *cache = &dev->cache;
239 struct mlx5_cache_ent *ent = &cache->ent[c];
240 struct mlx5_ib_mr *tmp_mr;
241 struct mlx5_ib_mr *mr;
242 LIST_HEAD(del_list);
243 int i;
244
245 for (i = 0; i < num; i++) {
246 spin_lock_irq(&ent->lock);
247 if (list_empty(&ent->head)) {
248 spin_unlock_irq(&ent->lock);
249 break;
250 }
251 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
252 list_move(&mr->list, &del_list);
253 ent->cur--;
254 ent->size--;
255 spin_unlock_irq(&ent->lock);
256 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
257 }
258
259#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
260 synchronize_srcu(&dev->mr_srcu);
261#endif
262
263 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
264 list_del(&mr->list);
265 kfree(mr);
266 }
267}
268
269static ssize_t size_write(struct file *filp, const char __user *buf,
270 size_t count, loff_t *pos)
271{
272 struct mlx5_cache_ent *ent = filp->private_data;
273 struct mlx5_ib_dev *dev = ent->dev;
274 char lbuf[20] = {0};
275 u32 var;
276 int err;
277 int c;
278
279 count = min(count, sizeof(lbuf) - 1);
280 if (copy_from_user(lbuf, buf, count))
281 return -EFAULT;
282
283 c = order2idx(dev, ent->order);
284
285 if (sscanf(lbuf, "%u", &var) != 1)
286 return -EINVAL;
287
288 if (var < ent->limit)
289 return -EINVAL;
290
291 if (var > ent->size) {
292 do {
293 err = add_keys(dev, c, var - ent->size);
294 if (err && err != -EAGAIN)
295 return err;
296
297 usleep_range(3000, 5000);
298 } while (err);
299 } else if (var < ent->size) {
300 remove_keys(dev, c, ent->size - var);
301 }
302
303 return count;
304}
305
306static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
307 loff_t *pos)
308{
309 struct mlx5_cache_ent *ent = filp->private_data;
310 char lbuf[20];
311 int err;
312
313 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size);
314 if (err < 0)
315 return err;
316
317 return simple_read_from_buffer(buf, count, pos, lbuf, err);
318}
319
320static const struct file_operations size_fops = {
321 .owner = THIS_MODULE,
322 .open = simple_open,
323 .write = size_write,
324 .read = size_read,
325};
326
327static ssize_t limit_write(struct file *filp, const char __user *buf,
328 size_t count, loff_t *pos)
329{
330 struct mlx5_cache_ent *ent = filp->private_data;
331 struct mlx5_ib_dev *dev = ent->dev;
332 char lbuf[20] = {0};
333 u32 var;
334 int err;
335 int c;
336
337 count = min(count, sizeof(lbuf) - 1);
338 if (copy_from_user(lbuf, buf, count))
339 return -EFAULT;
340
341 c = order2idx(dev, ent->order);
342
343 if (sscanf(lbuf, "%u", &var) != 1)
344 return -EINVAL;
345
346 if (var > ent->size)
347 return -EINVAL;
348
349 ent->limit = var;
350
351 if (ent->cur < ent->limit) {
352 err = add_keys(dev, c, 2 * ent->limit - ent->cur);
353 if (err)
354 return err;
355 }
356
357 return count;
358}
359
360static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
361 loff_t *pos)
362{
363 struct mlx5_cache_ent *ent = filp->private_data;
364 char lbuf[20];
365 int err;
366
367 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
368 if (err < 0)
369 return err;
370
371 return simple_read_from_buffer(buf, count, pos, lbuf, err);
372}
373
374static const struct file_operations limit_fops = {
375 .owner = THIS_MODULE,
376 .open = simple_open,
377 .write = limit_write,
378 .read = limit_read,
379};
380
381static int someone_adding(struct mlx5_mr_cache *cache)
382{
383 int i;
384
385 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
386 if (cache->ent[i].cur < cache->ent[i].limit)
387 return 1;
388 }
389
390 return 0;
391}
392
393static void __cache_work_func(struct mlx5_cache_ent *ent)
394{
395 struct mlx5_ib_dev *dev = ent->dev;
396 struct mlx5_mr_cache *cache = &dev->cache;
397 int i = order2idx(dev, ent->order);
398 int err;
399
400 if (cache->stopped)
401 return;
402
403 ent = &dev->cache.ent[i];
404 if (ent->cur < 2 * ent->limit && !dev->fill_delay) {
405 err = add_keys(dev, i, 1);
406 if (ent->cur < 2 * ent->limit) {
407 if (err == -EAGAIN) {
408 mlx5_ib_dbg(dev, "returned eagain, order %d\n",
409 i + 2);
410 queue_delayed_work(cache->wq, &ent->dwork,
411 msecs_to_jiffies(3));
412 } else if (err) {
413 mlx5_ib_warn(dev, "command failed order %d, err %d\n",
414 i + 2, err);
415 queue_delayed_work(cache->wq, &ent->dwork,
416 msecs_to_jiffies(1000));
417 } else {
418 queue_work(cache->wq, &ent->work);
419 }
420 }
421 } else if (ent->cur > 2 * ent->limit) {
422
423
424
425
426
427
428
429
430
431
432
433
434 if (!need_resched() && !someone_adding(cache) &&
435 time_after(jiffies, cache->last_add + 300 * HZ)) {
436 remove_keys(dev, i, 1);
437 if (ent->cur > ent->limit)
438 queue_work(cache->wq, &ent->work);
439 } else {
440 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
441 }
442 }
443}
444
445static void delayed_cache_work_func(struct work_struct *work)
446{
447 struct mlx5_cache_ent *ent;
448
449 ent = container_of(work, struct mlx5_cache_ent, dwork.work);
450 __cache_work_func(ent);
451}
452
453static void cache_work_func(struct work_struct *work)
454{
455 struct mlx5_cache_ent *ent;
456
457 ent = container_of(work, struct mlx5_cache_ent, work);
458 __cache_work_func(ent);
459}
460
461struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry)
462{
463 struct mlx5_mr_cache *cache = &dev->cache;
464 struct mlx5_cache_ent *ent;
465 struct mlx5_ib_mr *mr;
466 int err;
467
468 if (entry < 0 || entry >= MAX_MR_CACHE_ENTRIES) {
469 mlx5_ib_err(dev, "cache entry %d is out of range\n", entry);
470 return NULL;
471 }
472
473 ent = &cache->ent[entry];
474 while (1) {
475 spin_lock_irq(&ent->lock);
476 if (list_empty(&ent->head)) {
477 spin_unlock_irq(&ent->lock);
478
479 err = add_keys(dev, entry, 1);
480 if (err && err != -EAGAIN)
481 return ERR_PTR(err);
482
483 wait_for_completion(&ent->compl);
484 } else {
485 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
486 list);
487 list_del(&mr->list);
488 ent->cur--;
489 spin_unlock_irq(&ent->lock);
490 if (ent->cur < ent->limit)
491 queue_work(cache->wq, &ent->work);
492 return mr;
493 }
494 }
495}
496
497static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
498{
499 struct mlx5_mr_cache *cache = &dev->cache;
500 struct mlx5_ib_mr *mr = NULL;
501 struct mlx5_cache_ent *ent;
502 int last_umr_cache_entry;
503 int c;
504 int i;
505
506 c = order2idx(dev, order);
507 last_umr_cache_entry = order2idx(dev, mr_cache_max_order(dev));
508 if (c < 0 || c > last_umr_cache_entry) {
509 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
510 return NULL;
511 }
512
513 for (i = c; i <= last_umr_cache_entry; i++) {
514 ent = &cache->ent[i];
515
516 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
517
518 spin_lock_irq(&ent->lock);
519 if (!list_empty(&ent->head)) {
520 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
521 list);
522 list_del(&mr->list);
523 ent->cur--;
524 spin_unlock_irq(&ent->lock);
525 if (ent->cur < ent->limit)
526 queue_work(cache->wq, &ent->work);
527 break;
528 }
529 spin_unlock_irq(&ent->lock);
530
531 queue_work(cache->wq, &ent->work);
532 }
533
534 if (!mr)
535 cache->ent[c].miss++;
536
537 return mr;
538}
539
540void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
541{
542 struct mlx5_mr_cache *cache = &dev->cache;
543 struct mlx5_cache_ent *ent;
544 int shrink = 0;
545 int c;
546
547 if (!mr->allocated_from_cache)
548 return;
549
550 c = order2idx(dev, mr->order);
551 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
552 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c);
553 return;
554 }
555
556 if (unreg_umr(dev, mr))
557 return;
558
559 ent = &cache->ent[c];
560 spin_lock_irq(&ent->lock);
561 list_add_tail(&mr->list, &ent->head);
562 ent->cur++;
563 if (ent->cur > 2 * ent->limit)
564 shrink = 1;
565 spin_unlock_irq(&ent->lock);
566
567 if (shrink)
568 queue_work(cache->wq, &ent->work);
569}
570
571static void clean_keys(struct mlx5_ib_dev *dev, int c)
572{
573 struct mlx5_mr_cache *cache = &dev->cache;
574 struct mlx5_cache_ent *ent = &cache->ent[c];
575 struct mlx5_ib_mr *tmp_mr;
576 struct mlx5_ib_mr *mr;
577 LIST_HEAD(del_list);
578
579 cancel_delayed_work(&ent->dwork);
580 while (1) {
581 spin_lock_irq(&ent->lock);
582 if (list_empty(&ent->head)) {
583 spin_unlock_irq(&ent->lock);
584 break;
585 }
586 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
587 list_move(&mr->list, &del_list);
588 ent->cur--;
589 ent->size--;
590 spin_unlock_irq(&ent->lock);
591 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
592 }
593
594#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
595 synchronize_srcu(&dev->mr_srcu);
596#endif
597
598 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
599 list_del(&mr->list);
600 kfree(mr);
601 }
602}
603
604static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
605{
606 if (!mlx5_debugfs_root || dev->rep)
607 return;
608
609 debugfs_remove_recursive(dev->cache.root);
610 dev->cache.root = NULL;
611}
612
613static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
614{
615 struct mlx5_mr_cache *cache = &dev->cache;
616 struct mlx5_cache_ent *ent;
617 int i;
618
619 if (!mlx5_debugfs_root || dev->rep)
620 return 0;
621
622 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
623 if (!cache->root)
624 return -ENOMEM;
625
626 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
627 ent = &cache->ent[i];
628 sprintf(ent->name, "%d", ent->order);
629 ent->dir = debugfs_create_dir(ent->name, cache->root);
630 if (!ent->dir)
631 goto err;
632
633 ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent,
634 &size_fops);
635 if (!ent->fsize)
636 goto err;
637
638 ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent,
639 &limit_fops);
640 if (!ent->flimit)
641 goto err;
642
643 ent->fcur = debugfs_create_u32("cur", 0400, ent->dir,
644 &ent->cur);
645 if (!ent->fcur)
646 goto err;
647
648 ent->fmiss = debugfs_create_u32("miss", 0600, ent->dir,
649 &ent->miss);
650 if (!ent->fmiss)
651 goto err;
652 }
653
654 return 0;
655err:
656 mlx5_mr_cache_debugfs_cleanup(dev);
657
658 return -ENOMEM;
659}
660
661static void delay_time_func(struct timer_list *t)
662{
663 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
664
665 dev->fill_delay = 0;
666}
667
668int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
669{
670 struct mlx5_mr_cache *cache = &dev->cache;
671 struct mlx5_cache_ent *ent;
672 int err;
673 int i;
674
675 mutex_init(&dev->slow_path_mutex);
676 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
677 if (!cache->wq) {
678 mlx5_ib_warn(dev, "failed to create work queue\n");
679 return -ENOMEM;
680 }
681
682 timer_setup(&dev->delay_timer, delay_time_func, 0);
683 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
684 ent = &cache->ent[i];
685 INIT_LIST_HEAD(&ent->head);
686 spin_lock_init(&ent->lock);
687 ent->order = i + 2;
688 ent->dev = dev;
689 ent->limit = 0;
690
691 init_completion(&ent->compl);
692 INIT_WORK(&ent->work, cache_work_func);
693 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
694
695 if (i > MR_CACHE_LAST_STD_ENTRY) {
696 mlx5_odp_init_mr_cache_entry(ent);
697 continue;
698 }
699
700 if (ent->order > mr_cache_max_order(dev))
701 continue;
702
703 ent->page = PAGE_SHIFT;
704 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
705 MLX5_IB_UMR_OCTOWORD;
706 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
707 if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
708 !dev->rep &&
709 mlx5_core_is_pf(dev->mdev))
710 ent->limit = dev->mdev->profile->mr_cache[i].limit;
711 else
712 ent->limit = 0;
713 queue_work(cache->wq, &ent->work);
714 }
715
716 err = mlx5_mr_cache_debugfs_init(dev);
717 if (err)
718 mlx5_ib_warn(dev, "cache debugfs failure\n");
719
720
721
722
723
724
725 return 0;
726}
727
728static void wait_for_async_commands(struct mlx5_ib_dev *dev)
729{
730 struct mlx5_mr_cache *cache = &dev->cache;
731 struct mlx5_cache_ent *ent;
732 int total = 0;
733 int i;
734 int j;
735
736 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
737 ent = &cache->ent[i];
738 for (j = 0 ; j < 1000; j++) {
739 if (!ent->pending)
740 break;
741 msleep(50);
742 }
743 }
744 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
745 ent = &cache->ent[i];
746 total += ent->pending;
747 }
748
749 if (total)
750 mlx5_ib_warn(dev, "aborted while there are %d pending mr requests\n", total);
751 else
752 mlx5_ib_warn(dev, "done with all pending requests\n");
753}
754
755int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
756{
757 int i;
758
759 if (!dev->cache.wq)
760 return 0;
761
762 dev->cache.stopped = 1;
763 flush_workqueue(dev->cache.wq);
764
765 mlx5_mr_cache_debugfs_cleanup(dev);
766
767 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
768 clean_keys(dev, i);
769
770 destroy_workqueue(dev->cache.wq);
771 wait_for_async_commands(dev);
772 del_timer_sync(&dev->delay_timer);
773
774 return 0;
775}
776
777struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
778{
779 struct mlx5_ib_dev *dev = to_mdev(pd->device);
780 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
781 struct mlx5_core_dev *mdev = dev->mdev;
782 struct mlx5_ib_mr *mr;
783 void *mkc;
784 u32 *in;
785 int err;
786
787 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
788 if (!mr)
789 return ERR_PTR(-ENOMEM);
790
791 in = kzalloc(inlen, GFP_KERNEL);
792 if (!in) {
793 err = -ENOMEM;
794 goto err_free;
795 }
796
797 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
798
799 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
800 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
801 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
802 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
803 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
804 MLX5_SET(mkc, mkc, lr, 1);
805
806 MLX5_SET(mkc, mkc, length64, 1);
807 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
808 MLX5_SET(mkc, mkc, qpn, 0xffffff);
809 MLX5_SET64(mkc, mkc, start_addr, 0);
810
811 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen);
812 if (err)
813 goto err_in;
814
815 kfree(in);
816 mr->mmkey.type = MLX5_MKEY_MR;
817 mr->ibmr.lkey = mr->mmkey.key;
818 mr->ibmr.rkey = mr->mmkey.key;
819 mr->umem = NULL;
820
821 return &mr->ibmr;
822
823err_in:
824 kfree(in);
825
826err_free:
827 kfree(mr);
828
829 return ERR_PTR(err);
830}
831
832static int get_octo_len(u64 addr, u64 len, int page_shift)
833{
834 u64 page_size = 1ULL << page_shift;
835 u64 offset;
836 int npages;
837
838 offset = addr & (page_size - 1);
839 npages = ALIGN(len + offset, page_size) >> page_shift;
840 return (npages + 1) / 2;
841}
842
843static int mr_cache_max_order(struct mlx5_ib_dev *dev)
844{
845 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
846 return MR_CACHE_LAST_STD_ENTRY + 2;
847 return MLX5_MAX_UMR_SHIFT;
848}
849
850static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
851 int access_flags, struct ib_umem **umem,
852 int *npages, int *page_shift, int *ncont,
853 int *order)
854{
855 struct mlx5_ib_dev *dev = to_mdev(pd->device);
856 struct ib_umem *u;
857 int err;
858
859 *umem = NULL;
860
861 u = ib_umem_get(pd->uobject->context, start, length, access_flags, 0);
862 err = PTR_ERR_OR_ZERO(u);
863 if (err) {
864 mlx5_ib_dbg(dev, "umem get failed (%d)\n", err);
865 return err;
866 }
867
868 mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
869 page_shift, ncont, order);
870 if (!*npages) {
871 mlx5_ib_warn(dev, "avoid zero region\n");
872 ib_umem_release(u);
873 return -EINVAL;
874 }
875
876 *umem = u;
877
878 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
879 *npages, *ncont, *order, *page_shift);
880
881 return 0;
882}
883
884static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
885{
886 struct mlx5_ib_umr_context *context =
887 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe);
888
889 context->status = wc->status;
890 complete(&context->done);
891}
892
893static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
894{
895 context->cqe.done = mlx5_ib_umr_done;
896 context->status = -1;
897 init_completion(&context->done);
898}
899
900static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,
901 struct mlx5_umr_wr *umrwr)
902{
903 struct umr_common *umrc = &dev->umrc;
904 struct ib_send_wr *bad;
905 int err;
906 struct mlx5_ib_umr_context umr_context;
907
908 mlx5_ib_init_umr_context(&umr_context);
909 umrwr->wr.wr_cqe = &umr_context.cqe;
910
911 down(&umrc->sem);
912 err = ib_post_send(umrc->qp, &umrwr->wr, &bad);
913 if (err) {
914 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err);
915 } else {
916 wait_for_completion(&umr_context.done);
917 if (umr_context.status != IB_WC_SUCCESS) {
918 mlx5_ib_warn(dev, "reg umr failed (%u)\n",
919 umr_context.status);
920 err = -EFAULT;
921 }
922 }
923 up(&umrc->sem);
924 return err;
925}
926
927static struct mlx5_ib_mr *alloc_mr_from_cache(
928 struct ib_pd *pd, struct ib_umem *umem,
929 u64 virt_addr, u64 len, int npages,
930 int page_shift, int order, int access_flags)
931{
932 struct mlx5_ib_dev *dev = to_mdev(pd->device);
933 struct mlx5_ib_mr *mr;
934 int err = 0;
935 int i;
936
937 for (i = 0; i < 1; i++) {
938 mr = alloc_cached_mr(dev, order);
939 if (mr)
940 break;
941
942 err = add_keys(dev, order2idx(dev, order), 1);
943 if (err && err != -EAGAIN) {
944 mlx5_ib_warn(dev, "add_keys failed, err %d\n", err);
945 break;
946 }
947 }
948
949 if (!mr)
950 return ERR_PTR(-EAGAIN);
951
952 mr->ibmr.pd = pd;
953 mr->umem = umem;
954 mr->access_flags = access_flags;
955 mr->desc_size = sizeof(struct mlx5_mtt);
956 mr->mmkey.iova = virt_addr;
957 mr->mmkey.size = len;
958 mr->mmkey.pd = to_mpd(pd)->pdn;
959
960 return mr;
961}
962
963static inline int populate_xlt(struct mlx5_ib_mr *mr, int idx, int npages,
964 void *xlt, int page_shift, size_t size,
965 int flags)
966{
967 struct mlx5_ib_dev *dev = mr->dev;
968 struct ib_umem *umem = mr->umem;
969
970 if (flags & MLX5_IB_UPD_XLT_INDIRECT) {
971 if (!umr_can_use_indirect_mkey(dev))
972 return -EPERM;
973 mlx5_odp_populate_klm(xlt, idx, npages, mr, flags);
974 return npages;
975 }
976
977 npages = min_t(size_t, npages, ib_umem_num_pages(umem) - idx);
978
979 if (!(flags & MLX5_IB_UPD_XLT_ZAP)) {
980 __mlx5_ib_populate_pas(dev, umem, page_shift,
981 idx, npages, xlt,
982 MLX5_IB_MTT_PRESENT);
983
984
985
986 memset(xlt + (npages * sizeof(struct mlx5_mtt)), 0,
987 size - npages * sizeof(struct mlx5_mtt));
988 }
989
990 return npages;
991}
992
993#define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \
994 MLX5_UMR_MTT_ALIGNMENT)
995#define MLX5_SPARE_UMR_CHUNK 0x10000
996
997int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
998 int page_shift, int flags)
999{
1000 struct mlx5_ib_dev *dev = mr->dev;
1001 struct device *ddev = dev->ib_dev.dev.parent;
1002 int size;
1003 void *xlt;
1004 dma_addr_t dma;
1005 struct mlx5_umr_wr wr;
1006 struct ib_sge sg;
1007 int err = 0;
1008 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
1009 ? sizeof(struct mlx5_klm)
1010 : sizeof(struct mlx5_mtt);
1011 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size;
1012 const int page_mask = page_align - 1;
1013 size_t pages_mapped = 0;
1014 size_t pages_to_map = 0;
1015 size_t pages_iter = 0;
1016 gfp_t gfp;
1017 bool use_emergency_page = false;
1018
1019 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
1020 !umr_can_use_indirect_mkey(dev))
1021 return -EPERM;
1022
1023
1024
1025
1026 if (idx & page_mask) {
1027 npages += idx & page_mask;
1028 idx &= ~page_mask;
1029 }
1030
1031 gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL;
1032 gfp |= __GFP_ZERO | __GFP_NOWARN;
1033
1034 pages_to_map = ALIGN(npages, page_align);
1035 size = desc_size * pages_to_map;
1036 size = min_t(int, size, MLX5_MAX_UMR_CHUNK);
1037
1038 xlt = (void *)__get_free_pages(gfp, get_order(size));
1039 if (!xlt && size > MLX5_SPARE_UMR_CHUNK) {
1040 mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n",
1041 size, get_order(size), MLX5_SPARE_UMR_CHUNK);
1042
1043 size = MLX5_SPARE_UMR_CHUNK;
1044 xlt = (void *)__get_free_pages(gfp, get_order(size));
1045 }
1046
1047 if (!xlt) {
1048 mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
1049 xlt = (void *)mlx5_ib_get_xlt_emergency_page();
1050 size = PAGE_SIZE;
1051 memset(xlt, 0, size);
1052 use_emergency_page = true;
1053 }
1054 pages_iter = size / desc_size;
1055 dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
1056 if (dma_mapping_error(ddev, dma)) {
1057 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
1058 err = -ENOMEM;
1059 goto free_xlt;
1060 }
1061
1062 sg.addr = dma;
1063 sg.lkey = dev->umrc.pd->local_dma_lkey;
1064
1065 memset(&wr, 0, sizeof(wr));
1066 wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;
1067 if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
1068 wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1069 wr.wr.sg_list = &sg;
1070 wr.wr.num_sge = 1;
1071 wr.wr.opcode = MLX5_IB_WR_UMR;
1072
1073 wr.pd = mr->ibmr.pd;
1074 wr.mkey = mr->mmkey.key;
1075 wr.length = mr->mmkey.size;
1076 wr.virt_addr = mr->mmkey.iova;
1077 wr.access_flags = mr->access_flags;
1078 wr.page_shift = page_shift;
1079
1080 for (pages_mapped = 0;
1081 pages_mapped < pages_to_map && !err;
1082 pages_mapped += pages_iter, idx += pages_iter) {
1083 npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
1084 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE);
1085 npages = populate_xlt(mr, idx, npages, xlt,
1086 page_shift, size, flags);
1087
1088 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
1089
1090 sg.length = ALIGN(npages * desc_size,
1091 MLX5_UMR_MTT_ALIGNMENT);
1092
1093 if (pages_mapped + pages_iter >= pages_to_map) {
1094 if (flags & MLX5_IB_UPD_XLT_ENABLE)
1095 wr.wr.send_flags |=
1096 MLX5_IB_SEND_UMR_ENABLE_MR |
1097 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |
1098 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1099 if (flags & MLX5_IB_UPD_XLT_PD ||
1100 flags & MLX5_IB_UPD_XLT_ACCESS)
1101 wr.wr.send_flags |=
1102 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1103 if (flags & MLX5_IB_UPD_XLT_ADDR)
1104 wr.wr.send_flags |=
1105 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1106 }
1107
1108 wr.offset = idx * desc_size;
1109 wr.xlt_size = sg.length;
1110
1111 err = mlx5_ib_post_send_wait(dev, &wr);
1112 }
1113 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1114
1115free_xlt:
1116 if (use_emergency_page)
1117 mlx5_ib_put_xlt_emergency_page();
1118 else
1119 free_pages((unsigned long)xlt, get_order(size));
1120
1121 return err;
1122}
1123
1124
1125
1126
1127
1128static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
1129 u64 virt_addr, u64 length,
1130 struct ib_umem *umem, int npages,
1131 int page_shift, int access_flags,
1132 bool populate)
1133{
1134 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1135 struct mlx5_ib_mr *mr;
1136 __be64 *pas;
1137 void *mkc;
1138 int inlen;
1139 u32 *in;
1140 int err;
1141 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
1142
1143 mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);
1144 if (!mr)
1145 return ERR_PTR(-ENOMEM);
1146
1147 mr->ibmr.pd = pd;
1148 mr->access_flags = access_flags;
1149
1150 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1151 if (populate)
1152 inlen += sizeof(*pas) * roundup(npages, 2);
1153 in = kvzalloc(inlen, GFP_KERNEL);
1154 if (!in) {
1155 err = -ENOMEM;
1156 goto err_1;
1157 }
1158 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
1159 if (populate && !(access_flags & IB_ACCESS_ON_DEMAND))
1160 mlx5_ib_populate_pas(dev, umem, page_shift, pas,
1161 pg_cap ? MLX5_IB_MTT_PRESENT : 0);
1162
1163
1164
1165 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
1166
1167 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1168 MLX5_SET(mkc, mkc, free, !populate);
1169 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
1170 MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
1171 MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
1172 MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
1173 MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE));
1174 MLX5_SET(mkc, mkc, lr, 1);
1175 MLX5_SET(mkc, mkc, umr_en, 1);
1176
1177 MLX5_SET64(mkc, mkc, start_addr, virt_addr);
1178 MLX5_SET64(mkc, mkc, len, length);
1179 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1180 MLX5_SET(mkc, mkc, bsf_octword_size, 0);
1181 MLX5_SET(mkc, mkc, translations_octword_size,
1182 get_octo_len(virt_addr, length, page_shift));
1183 MLX5_SET(mkc, mkc, log_page_size, page_shift);
1184 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1185 if (populate) {
1186 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
1187 get_octo_len(virt_addr, length, page_shift));
1188 }
1189
1190 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
1191 if (err) {
1192 mlx5_ib_warn(dev, "create mkey failed\n");
1193 goto err_2;
1194 }
1195 mr->mmkey.type = MLX5_MKEY_MR;
1196 mr->desc_size = sizeof(struct mlx5_mtt);
1197 mr->dev = dev;
1198 kvfree(in);
1199
1200 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
1201
1202 return mr;
1203
1204err_2:
1205 kvfree(in);
1206
1207err_1:
1208 if (!ibmr)
1209 kfree(mr);
1210
1211 return ERR_PTR(err);
1212}
1213
1214static void set_mr_fileds(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
1215 int npages, u64 length, int access_flags)
1216{
1217 mr->npages = npages;
1218 atomic_add(npages, &dev->mdev->priv.reg_pages);
1219 mr->ibmr.lkey = mr->mmkey.key;
1220 mr->ibmr.rkey = mr->mmkey.key;
1221 mr->ibmr.length = length;
1222 mr->access_flags = access_flags;
1223}
1224
1225static struct ib_mr *mlx5_ib_get_memic_mr(struct ib_pd *pd, u64 memic_addr,
1226 u64 length, int acc)
1227{
1228 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1229 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1230 struct mlx5_core_dev *mdev = dev->mdev;
1231 struct mlx5_ib_mr *mr;
1232 void *mkc;
1233 u32 *in;
1234 int err;
1235
1236 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1237 if (!mr)
1238 return ERR_PTR(-ENOMEM);
1239
1240 in = kzalloc(inlen, GFP_KERNEL);
1241 if (!in) {
1242 err = -ENOMEM;
1243 goto err_free;
1244 }
1245
1246 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1247
1248 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MEMIC & 0x3);
1249 MLX5_SET(mkc, mkc, access_mode_4_2,
1250 (MLX5_MKC_ACCESS_MODE_MEMIC >> 2) & 0x7);
1251 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
1252 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
1253 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
1254 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
1255 MLX5_SET(mkc, mkc, lr, 1);
1256
1257 MLX5_SET64(mkc, mkc, len, length);
1258 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1259 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1260 MLX5_SET64(mkc, mkc, start_addr,
1261 memic_addr - pci_resource_start(dev->mdev->pdev, 0));
1262
1263 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen);
1264 if (err)
1265 goto err_in;
1266
1267 kfree(in);
1268
1269 mr->umem = NULL;
1270 set_mr_fileds(dev, mr, 0, length, acc);
1271
1272 return &mr->ibmr;
1273
1274err_in:
1275 kfree(in);
1276
1277err_free:
1278 kfree(mr);
1279
1280 return ERR_PTR(err);
1281}
1282
1283struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
1284 struct ib_dm_mr_attr *attr,
1285 struct uverbs_attr_bundle *attrs)
1286{
1287 struct mlx5_ib_dm *mdm = to_mdm(dm);
1288 u64 memic_addr;
1289
1290 if (attr->access_flags & ~MLX5_IB_DM_ALLOWED_ACCESS)
1291 return ERR_PTR(-EINVAL);
1292
1293 memic_addr = mdm->dev_addr + attr->offset;
1294
1295 return mlx5_ib_get_memic_mr(pd, memic_addr, attr->length,
1296 attr->access_flags);
1297}
1298
1299struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1300 u64 virt_addr, int access_flags,
1301 struct ib_udata *udata)
1302{
1303 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1304 struct mlx5_ib_mr *mr = NULL;
1305 bool populate_mtts = false;
1306 struct ib_umem *umem;
1307 int page_shift;
1308 int npages;
1309 int ncont;
1310 int order;
1311 int err;
1312
1313 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
1314 return ERR_PTR(-EOPNOTSUPP);
1315
1316 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1317 start, virt_addr, length, access_flags);
1318
1319#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1320 if (!start && length == U64_MAX) {
1321 if (!(access_flags & IB_ACCESS_ON_DEMAND) ||
1322 !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1323 return ERR_PTR(-EINVAL);
1324
1325 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), access_flags);
1326 if (IS_ERR(mr))
1327 return ERR_CAST(mr);
1328 return &mr->ibmr;
1329 }
1330#endif
1331
1332 err = mr_umem_get(pd, start, length, access_flags, &umem, &npages,
1333 &page_shift, &ncont, &order);
1334
1335 if (err < 0)
1336 return ERR_PTR(err);
1337
1338 if (use_umr(dev, order)) {
1339 mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
1340 page_shift, order, access_flags);
1341 if (PTR_ERR(mr) == -EAGAIN) {
1342 mlx5_ib_dbg(dev, "cache empty for order %d\n", order);
1343 mr = NULL;
1344 }
1345 populate_mtts = false;
1346 } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) {
1347 if (access_flags & IB_ACCESS_ON_DEMAND) {
1348 err = -EINVAL;
1349 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n");
1350 goto error;
1351 }
1352 populate_mtts = true;
1353 }
1354
1355 if (!mr) {
1356 if (!umr_can_modify_entity_size(dev))
1357 populate_mtts = true;
1358 mutex_lock(&dev->slow_path_mutex);
1359 mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
1360 page_shift, access_flags, populate_mtts);
1361 mutex_unlock(&dev->slow_path_mutex);
1362 }
1363
1364 if (IS_ERR(mr)) {
1365 err = PTR_ERR(mr);
1366 goto error;
1367 }
1368
1369 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);
1370
1371 mr->umem = umem;
1372 set_mr_fileds(dev, mr, npages, length, access_flags);
1373
1374#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1375 update_odp_mr(mr);
1376#endif
1377
1378 if (!populate_mtts) {
1379 int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
1380
1381 if (access_flags & IB_ACCESS_ON_DEMAND)
1382 update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
1383
1384 err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
1385 update_xlt_flags);
1386
1387 if (err) {
1388 dereg_mr(dev, mr);
1389 return ERR_PTR(err);
1390 }
1391 }
1392
1393#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1394 mr->live = 1;
1395#endif
1396 return &mr->ibmr;
1397error:
1398 ib_umem_release(umem);
1399 return ERR_PTR(err);
1400}
1401
1402static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1403{
1404 struct mlx5_core_dev *mdev = dev->mdev;
1405 struct mlx5_umr_wr umrwr = {};
1406
1407 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1408 return 0;
1409
1410 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
1411 MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1412 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1413 umrwr.mkey = mr->mmkey.key;
1414
1415 return mlx5_ib_post_send_wait(dev, &umrwr);
1416}
1417
1418static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1419 int access_flags, int flags)
1420{
1421 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1422 struct mlx5_umr_wr umrwr = {};
1423 int err;
1424
1425 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1426
1427 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1428 umrwr.mkey = mr->mmkey.key;
1429
1430 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
1431 umrwr.pd = pd;
1432 umrwr.access_flags = access_flags;
1433 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1434 }
1435
1436 err = mlx5_ib_post_send_wait(dev, &umrwr);
1437
1438 return err;
1439}
1440
1441int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1442 u64 length, u64 virt_addr, int new_access_flags,
1443 struct ib_pd *new_pd, struct ib_udata *udata)
1444{
1445 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
1446 struct mlx5_ib_mr *mr = to_mmr(ib_mr);
1447 struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd;
1448 int access_flags = flags & IB_MR_REREG_ACCESS ?
1449 new_access_flags :
1450 mr->access_flags;
1451 int page_shift = 0;
1452 int upd_flags = 0;
1453 int npages = 0;
1454 int ncont = 0;
1455 int order = 0;
1456 u64 addr, len;
1457 int err;
1458
1459 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1460 start, virt_addr, length, access_flags);
1461
1462 atomic_sub(mr->npages, &dev->mdev->priv.reg_pages);
1463
1464 if (!mr->umem)
1465 return -EINVAL;
1466
1467 if (flags & IB_MR_REREG_TRANS) {
1468 addr = virt_addr;
1469 len = length;
1470 } else {
1471 addr = mr->umem->address;
1472 len = mr->umem->length;
1473 }
1474
1475 if (flags != IB_MR_REREG_PD) {
1476
1477
1478
1479
1480 flags |= IB_MR_REREG_TRANS;
1481 ib_umem_release(mr->umem);
1482 mr->umem = NULL;
1483 err = mr_umem_get(pd, addr, len, access_flags, &mr->umem,
1484 &npages, &page_shift, &ncont, &order);
1485 if (err)
1486 goto err;
1487 }
1488
1489 if (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len)) {
1490
1491
1492
1493 if (mr->allocated_from_cache)
1494 err = unreg_umr(dev, mr);
1495 else
1496 err = destroy_mkey(dev, mr);
1497 if (err)
1498 goto err;
1499
1500 mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
1501 page_shift, access_flags, true);
1502
1503 if (IS_ERR(mr)) {
1504 err = PTR_ERR(mr);
1505 mr = to_mmr(ib_mr);
1506 goto err;
1507 }
1508
1509 mr->allocated_from_cache = 0;
1510#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1511 mr->live = 1;
1512#endif
1513 } else {
1514
1515
1516
1517 mr->ibmr.pd = pd;
1518 mr->access_flags = access_flags;
1519 mr->mmkey.iova = addr;
1520 mr->mmkey.size = len;
1521 mr->mmkey.pd = to_mpd(pd)->pdn;
1522
1523 if (flags & IB_MR_REREG_TRANS) {
1524 upd_flags = MLX5_IB_UPD_XLT_ADDR;
1525 if (flags & IB_MR_REREG_PD)
1526 upd_flags |= MLX5_IB_UPD_XLT_PD;
1527 if (flags & IB_MR_REREG_ACCESS)
1528 upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
1529 err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
1530 upd_flags);
1531 } else {
1532 err = rereg_umr(pd, mr, access_flags, flags);
1533 }
1534
1535 if (err)
1536 goto err;
1537 }
1538
1539 set_mr_fileds(dev, mr, npages, len, access_flags);
1540
1541#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1542 update_odp_mr(mr);
1543#endif
1544 return 0;
1545
1546err:
1547 if (mr->umem) {
1548 ib_umem_release(mr->umem);
1549 mr->umem = NULL;
1550 }
1551 clean_mr(dev, mr);
1552 return err;
1553}
1554
1555static int
1556mlx5_alloc_priv_descs(struct ib_device *device,
1557 struct mlx5_ib_mr *mr,
1558 int ndescs,
1559 int desc_size)
1560{
1561 int size = ndescs * desc_size;
1562 int add_size;
1563 int ret;
1564
1565 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0);
1566
1567 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL);
1568 if (!mr->descs_alloc)
1569 return -ENOMEM;
1570
1571 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN);
1572
1573 mr->desc_map = dma_map_single(device->dev.parent, mr->descs,
1574 size, DMA_TO_DEVICE);
1575 if (dma_mapping_error(device->dev.parent, mr->desc_map)) {
1576 ret = -ENOMEM;
1577 goto err;
1578 }
1579
1580 return 0;
1581err:
1582 kfree(mr->descs_alloc);
1583
1584 return ret;
1585}
1586
1587static void
1588mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
1589{
1590 if (mr->descs) {
1591 struct ib_device *device = mr->ibmr.device;
1592 int size = mr->max_descs * mr->desc_size;
1593
1594 dma_unmap_single(device->dev.parent, mr->desc_map,
1595 size, DMA_TO_DEVICE);
1596 kfree(mr->descs_alloc);
1597 mr->descs = NULL;
1598 }
1599}
1600
1601static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1602{
1603 int allocated_from_cache = mr->allocated_from_cache;
1604
1605 if (mr->sig) {
1606 if (mlx5_core_destroy_psv(dev->mdev,
1607 mr->sig->psv_memory.psv_idx))
1608 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1609 mr->sig->psv_memory.psv_idx);
1610 if (mlx5_core_destroy_psv(dev->mdev,
1611 mr->sig->psv_wire.psv_idx))
1612 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1613 mr->sig->psv_wire.psv_idx);
1614 kfree(mr->sig);
1615 mr->sig = NULL;
1616 }
1617
1618 mlx5_free_priv_descs(mr);
1619
1620 if (!allocated_from_cache)
1621 destroy_mkey(dev, mr);
1622}
1623
1624static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1625{
1626 int npages = mr->npages;
1627 struct ib_umem *umem = mr->umem;
1628
1629#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1630 if (umem && umem->odp_data) {
1631
1632 mr->live = 0;
1633
1634 synchronize_srcu(&dev->mr_srcu);
1635
1636 if (umem->odp_data->page_list)
1637 mlx5_ib_invalidate_range(umem, ib_umem_start(umem),
1638 ib_umem_end(umem));
1639 else
1640 mlx5_ib_free_implicit_mr(mr);
1641
1642
1643
1644
1645
1646 ib_umem_release(umem);
1647 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1648
1649
1650 umem = NULL;
1651 }
1652#endif
1653 clean_mr(dev, mr);
1654
1655
1656
1657
1658
1659 mlx5_mr_cache_free(dev, mr);
1660 if (umem) {
1661 ib_umem_release(umem);
1662 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1663 }
1664 if (!mr->allocated_from_cache)
1665 kfree(mr);
1666}
1667
1668int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
1669{
1670 dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr));
1671 return 0;
1672}
1673
1674struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
1675 enum ib_mr_type mr_type,
1676 u32 max_num_sg)
1677{
1678 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1679 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1680 int ndescs = ALIGN(max_num_sg, 4);
1681 struct mlx5_ib_mr *mr;
1682 void *mkc;
1683 u32 *in;
1684 int err;
1685
1686 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1687 if (!mr)
1688 return ERR_PTR(-ENOMEM);
1689
1690 in = kzalloc(inlen, GFP_KERNEL);
1691 if (!in) {
1692 err = -ENOMEM;
1693 goto err_free;
1694 }
1695
1696 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1697 MLX5_SET(mkc, mkc, free, 1);
1698 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1699 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1700 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1701
1702 if (mr_type == IB_MR_TYPE_MEM_REG) {
1703 mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
1704 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
1705 err = mlx5_alloc_priv_descs(pd->device, mr,
1706 ndescs, sizeof(struct mlx5_mtt));
1707 if (err)
1708 goto err_free_in;
1709
1710 mr->desc_size = sizeof(struct mlx5_mtt);
1711 mr->max_descs = ndescs;
1712 } else if (mr_type == IB_MR_TYPE_SG_GAPS) {
1713 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
1714
1715 err = mlx5_alloc_priv_descs(pd->device, mr,
1716 ndescs, sizeof(struct mlx5_klm));
1717 if (err)
1718 goto err_free_in;
1719 mr->desc_size = sizeof(struct mlx5_klm);
1720 mr->max_descs = ndescs;
1721 } else if (mr_type == IB_MR_TYPE_SIGNATURE) {
1722 u32 psv_index[2];
1723
1724 MLX5_SET(mkc, mkc, bsf_en, 1);
1725 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
1726 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
1727 if (!mr->sig) {
1728 err = -ENOMEM;
1729 goto err_free_in;
1730 }
1731
1732
1733 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn,
1734 2, psv_index);
1735 if (err)
1736 goto err_free_sig;
1737
1738 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
1739 mr->sig->psv_memory.psv_idx = psv_index[0];
1740 mr->sig->psv_wire.psv_idx = psv_index[1];
1741
1742 mr->sig->sig_status_checked = true;
1743 mr->sig->sig_err_exists = false;
1744
1745 ++mr->sig->sigerr_count;
1746 } else {
1747 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
1748 err = -EINVAL;
1749 goto err_free_in;
1750 }
1751
1752 MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3);
1753 MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7);
1754 MLX5_SET(mkc, mkc, umr_en, 1);
1755
1756 mr->ibmr.device = pd->device;
1757 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
1758 if (err)
1759 goto err_destroy_psv;
1760
1761 mr->mmkey.type = MLX5_MKEY_MR;
1762 mr->ibmr.lkey = mr->mmkey.key;
1763 mr->ibmr.rkey = mr->mmkey.key;
1764 mr->umem = NULL;
1765 kfree(in);
1766
1767 return &mr->ibmr;
1768
1769err_destroy_psv:
1770 if (mr->sig) {
1771 if (mlx5_core_destroy_psv(dev->mdev,
1772 mr->sig->psv_memory.psv_idx))
1773 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1774 mr->sig->psv_memory.psv_idx);
1775 if (mlx5_core_destroy_psv(dev->mdev,
1776 mr->sig->psv_wire.psv_idx))
1777 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1778 mr->sig->psv_wire.psv_idx);
1779 }
1780 mlx5_free_priv_descs(mr);
1781err_free_sig:
1782 kfree(mr->sig);
1783err_free_in:
1784 kfree(in);
1785err_free:
1786 kfree(mr);
1787 return ERR_PTR(err);
1788}
1789
1790struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
1791 struct ib_udata *udata)
1792{
1793 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1794 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1795 struct mlx5_ib_mw *mw = NULL;
1796 u32 *in = NULL;
1797 void *mkc;
1798 int ndescs;
1799 int err;
1800 struct mlx5_ib_alloc_mw req = {};
1801 struct {
1802 __u32 comp_mask;
1803 __u32 response_length;
1804 } resp = {};
1805
1806 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1807 if (err)
1808 return ERR_PTR(err);
1809
1810 if (req.comp_mask || req.reserved1 || req.reserved2)
1811 return ERR_PTR(-EOPNOTSUPP);
1812
1813 if (udata->inlen > sizeof(req) &&
1814 !ib_is_udata_cleared(udata, sizeof(req),
1815 udata->inlen - sizeof(req)))
1816 return ERR_PTR(-EOPNOTSUPP);
1817
1818 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
1819
1820 mw = kzalloc(sizeof(*mw), GFP_KERNEL);
1821 in = kzalloc(inlen, GFP_KERNEL);
1822 if (!mw || !in) {
1823 err = -ENOMEM;
1824 goto free;
1825 }
1826
1827 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1828
1829 MLX5_SET(mkc, mkc, free, 1);
1830 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1831 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1832 MLX5_SET(mkc, mkc, umr_en, 1);
1833 MLX5_SET(mkc, mkc, lr, 1);
1834 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
1835 MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2)));
1836 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1837
1838 err = mlx5_core_create_mkey(dev->mdev, &mw->mmkey, in, inlen);
1839 if (err)
1840 goto free;
1841
1842 mw->mmkey.type = MLX5_MKEY_MW;
1843 mw->ibmw.rkey = mw->mmkey.key;
1844 mw->ndescs = ndescs;
1845
1846 resp.response_length = min(offsetof(typeof(resp), response_length) +
1847 sizeof(resp.response_length), udata->outlen);
1848 if (resp.response_length) {
1849 err = ib_copy_to_udata(udata, &resp, resp.response_length);
1850 if (err) {
1851 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
1852 goto free;
1853 }
1854 }
1855
1856 kfree(in);
1857 return &mw->ibmw;
1858
1859free:
1860 kfree(mw);
1861 kfree(in);
1862 return ERR_PTR(err);
1863}
1864
1865int mlx5_ib_dealloc_mw(struct ib_mw *mw)
1866{
1867 struct mlx5_ib_mw *mmw = to_mmw(mw);
1868 int err;
1869
1870 err = mlx5_core_destroy_mkey((to_mdev(mw->device))->mdev,
1871 &mmw->mmkey);
1872 if (!err)
1873 kfree(mmw);
1874 return err;
1875}
1876
1877int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1878 struct ib_mr_status *mr_status)
1879{
1880 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
1881 int ret = 0;
1882
1883 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
1884 pr_err("Invalid status check mask\n");
1885 ret = -EINVAL;
1886 goto done;
1887 }
1888
1889 mr_status->fail_status = 0;
1890 if (check_mask & IB_MR_CHECK_SIG_STATUS) {
1891 if (!mmr->sig) {
1892 ret = -EINVAL;
1893 pr_err("signature status check requested on a non-signature enabled MR\n");
1894 goto done;
1895 }
1896
1897 mmr->sig->sig_status_checked = true;
1898 if (!mmr->sig->sig_err_exists)
1899 goto done;
1900
1901 if (ibmr->lkey == mmr->sig->err_item.key)
1902 memcpy(&mr_status->sig_err, &mmr->sig->err_item,
1903 sizeof(mr_status->sig_err));
1904 else {
1905 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
1906 mr_status->sig_err.sig_err_offset = 0;
1907 mr_status->sig_err.key = mmr->sig->err_item.key;
1908 }
1909
1910 mmr->sig->sig_err_exists = false;
1911 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
1912 }
1913
1914done:
1915 return ret;
1916}
1917
1918static int
1919mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
1920 struct scatterlist *sgl,
1921 unsigned short sg_nents,
1922 unsigned int *sg_offset_p)
1923{
1924 struct scatterlist *sg = sgl;
1925 struct mlx5_klm *klms = mr->descs;
1926 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
1927 u32 lkey = mr->ibmr.pd->local_dma_lkey;
1928 int i;
1929
1930 mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
1931 mr->ibmr.length = 0;
1932
1933 for_each_sg(sgl, sg, sg_nents, i) {
1934 if (unlikely(i >= mr->max_descs))
1935 break;
1936 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset);
1937 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset);
1938 klms[i].key = cpu_to_be32(lkey);
1939 mr->ibmr.length += sg_dma_len(sg) - sg_offset;
1940
1941 sg_offset = 0;
1942 }
1943 mr->ndescs = i;
1944
1945 if (sg_offset_p)
1946 *sg_offset_p = sg_offset;
1947
1948 return i;
1949}
1950
1951static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
1952{
1953 struct mlx5_ib_mr *mr = to_mmr(ibmr);
1954 __be64 *descs;
1955
1956 if (unlikely(mr->ndescs == mr->max_descs))
1957 return -ENOMEM;
1958
1959 descs = mr->descs;
1960 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
1961
1962 return 0;
1963}
1964
1965int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1966 unsigned int *sg_offset)
1967{
1968 struct mlx5_ib_mr *mr = to_mmr(ibmr);
1969 int n;
1970
1971 mr->ndescs = 0;
1972
1973 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map,
1974 mr->desc_size * mr->max_descs,
1975 DMA_TO_DEVICE);
1976
1977 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
1978 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset);
1979 else
1980 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
1981 mlx5_set_page);
1982
1983 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map,
1984 mr->desc_size * mr->max_descs,
1985 DMA_TO_DEVICE);
1986
1987 return n;
1988}
1989
