1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/kref.h>
35#include <linux/random.h>
36#include <linux/debugfs.h>
37#include <linux/export.h>
38#include <linux/delay.h>
39#include <rdma/ib_umem.h>
40#include "mlx5_ib.h"
41
42enum {
43 MAX_PENDING_REG_MR = 8,
44};
45
46enum {
47 MLX5_UMR_ALIGN = 2048
48};
49
50static __be64 *mr_align(__be64 *ptr, int align)
51{
52 unsigned long mask = align - 1;
53
54 return (__be64 *)(((unsigned long)ptr + mask) & ~mask);
55}
56
57static int order2idx(struct mlx5_ib_dev *dev, int order)
58{
59 struct mlx5_mr_cache *cache = &dev->cache;
60
61 if (order < cache->ent[0].order)
62 return 0;
63 else
64 return order - cache->ent[0].order;
65}
66
67static void reg_mr_callback(int status, void *context)
68{
69 struct mlx5_ib_mr *mr = context;
70 struct mlx5_ib_dev *dev = mr->dev;
71 struct mlx5_mr_cache *cache = &dev->cache;
72 int c = order2idx(dev, mr->order);
73 struct mlx5_cache_ent *ent = &cache->ent[c];
74 u8 key;
75 unsigned long flags;
76 struct mlx5_mr_table *table = &dev->mdev->priv.mr_table;
77 int err;
78
79 spin_lock_irqsave(&ent->lock, flags);
80 ent->pending--;
81 spin_unlock_irqrestore(&ent->lock, flags);
82 if (status) {
83 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
84 kfree(mr);
85 dev->fill_delay = 1;
86 mod_timer(&dev->delay_timer, jiffies + HZ);
87 return;
88 }
89
90 if (mr->out.hdr.status) {
91 mlx5_ib_warn(dev, "failed - status %d, syndorme 0x%x\n",
92 mr->out.hdr.status,
93 be32_to_cpu(mr->out.hdr.syndrome));
94 kfree(mr);
95 dev->fill_delay = 1;
96 mod_timer(&dev->delay_timer, jiffies + HZ);
97 return;
98 }
99
100 spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags);
101 key = dev->mdev->priv.mkey_key++;
102 spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags);
103 mr->mmr.key = mlx5_idx_to_mkey(be32_to_cpu(mr->out.mkey) & 0xffffff) | key;
104
105 cache->last_add = jiffies;
106
107 spin_lock_irqsave(&ent->lock, flags);
108 list_add_tail(&mr->list, &ent->head);
109 ent->cur++;
110 ent->size++;
111 spin_unlock_irqrestore(&ent->lock, flags);
112
113 write_lock_irqsave(&table->lock, flags);
114 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmr.key),
115 &mr->mmr);
116 if (err)
117 pr_err("Error inserting to mr tree. 0x%x\n", -err);
118 write_unlock_irqrestore(&table->lock, flags);
119}
120
121static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
122{
123 struct mlx5_mr_cache *cache = &dev->cache;
124 struct mlx5_cache_ent *ent = &cache->ent[c];
125 struct mlx5_create_mkey_mbox_in *in;
126 struct mlx5_ib_mr *mr;
127 int npages = 1 << ent->order;
128 int err = 0;
129 int i;
130
131 in = kzalloc(sizeof(*in), GFP_KERNEL);
132 if (!in)
133 return -ENOMEM;
134
135 for (i = 0; i < num; i++) {
136 if (ent->pending >= MAX_PENDING_REG_MR) {
137 err = -EAGAIN;
138 break;
139 }
140
141 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
142 if (!mr) {
143 err = -ENOMEM;
144 break;
145 }
146 mr->order = ent->order;
147 mr->umred = 1;
148 mr->dev = dev;
149 in->seg.status = 1 << 6;
150 in->seg.xlt_oct_size = cpu_to_be32((npages + 1) / 2);
151 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
152 in->seg.flags = MLX5_ACCESS_MODE_MTT | MLX5_PERM_UMR_EN;
153 in->seg.log2_page_size = 12;
154
155 spin_lock_irq(&ent->lock);
156 ent->pending++;
157 spin_unlock_irq(&ent->lock);
158 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in,
159 sizeof(*in), reg_mr_callback,
160 mr, &mr->out);
161 if (err) {
162 mlx5_ib_warn(dev, "create mkey failed %d\n", err);
163 kfree(mr);
164 break;
165 }
166 }
167
168 kfree(in);
169 return err;
170}
171
172static void remove_keys(struct mlx5_ib_dev *dev, int c, int num)
173{
174 struct mlx5_mr_cache *cache = &dev->cache;
175 struct mlx5_cache_ent *ent = &cache->ent[c];
176 struct mlx5_ib_mr *mr;
177 int err;
178 int i;
179
180 for (i = 0; i < num; i++) {
181 spin_lock_irq(&ent->lock);
182 if (list_empty(&ent->head)) {
183 spin_unlock_irq(&ent->lock);
184 return;
185 }
186 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
187 list_del(&mr->list);
188 ent->cur--;
189 ent->size--;
190 spin_unlock_irq(&ent->lock);
191 err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
192 if (err)
193 mlx5_ib_warn(dev, "failed destroy mkey\n");
194 else
195 kfree(mr);
196 }
197}
198
199static ssize_t size_write(struct file *filp, const char __user *buf,
200 size_t count, loff_t *pos)
201{
202 struct mlx5_cache_ent *ent = filp->private_data;
203 struct mlx5_ib_dev *dev = ent->dev;
204 char lbuf[20];
205 u32 var;
206 int err;
207 int c;
208
209 if (copy_from_user(lbuf, buf, sizeof(lbuf)))
210 return -EFAULT;
211
212 c = order2idx(dev, ent->order);
213 lbuf[sizeof(lbuf) - 1] = 0;
214
215 if (sscanf(lbuf, "%u", &var) != 1)
216 return -EINVAL;
217
218 if (var < ent->limit)
219 return -EINVAL;
220
221 if (var > ent->size) {
222 do {
223 err = add_keys(dev, c, var - ent->size);
224 if (err && err != -EAGAIN)
225 return err;
226
227 usleep_range(3000, 5000);
228 } while (err);
229 } else if (var < ent->size) {
230 remove_keys(dev, c, ent->size - var);
231 }
232
233 return count;
234}
235
236static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
237 loff_t *pos)
238{
239 struct mlx5_cache_ent *ent = filp->private_data;
240 char lbuf[20];
241 int err;
242
243 if (*pos)
244 return 0;
245
246 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size);
247 if (err < 0)
248 return err;
249
250 if (copy_to_user(buf, lbuf, err))
251 return -EFAULT;
252
253 *pos += err;
254
255 return err;
256}
257
258static const struct file_operations size_fops = {
259 .owner = THIS_MODULE,
260 .open = simple_open,
261 .write = size_write,
262 .read = size_read,
263};
264
265static ssize_t limit_write(struct file *filp, const char __user *buf,
266 size_t count, loff_t *pos)
267{
268 struct mlx5_cache_ent *ent = filp->private_data;
269 struct mlx5_ib_dev *dev = ent->dev;
270 char lbuf[20];
271 u32 var;
272 int err;
273 int c;
274
275 if (copy_from_user(lbuf, buf, sizeof(lbuf)))
276 return -EFAULT;
277
278 c = order2idx(dev, ent->order);
279 lbuf[sizeof(lbuf) - 1] = 0;
280
281 if (sscanf(lbuf, "%u", &var) != 1)
282 return -EINVAL;
283
284 if (var > ent->size)
285 return -EINVAL;
286
287 ent->limit = var;
288
289 if (ent->cur < ent->limit) {
290 err = add_keys(dev, c, 2 * ent->limit - ent->cur);
291 if (err)
292 return err;
293 }
294
295 return count;
296}
297
298static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
299 loff_t *pos)
300{
301 struct mlx5_cache_ent *ent = filp->private_data;
302 char lbuf[20];
303 int err;
304
305 if (*pos)
306 return 0;
307
308 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
309 if (err < 0)
310 return err;
311
312 if (copy_to_user(buf, lbuf, err))
313 return -EFAULT;
314
315 *pos += err;
316
317 return err;
318}
319
320static const struct file_operations limit_fops = {
321 .owner = THIS_MODULE,
322 .open = simple_open,
323 .write = limit_write,
324 .read = limit_read,
325};
326
327static int someone_adding(struct mlx5_mr_cache *cache)
328{
329 int i;
330
331 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
332 if (cache->ent[i].cur < cache->ent[i].limit)
333 return 1;
334 }
335
336 return 0;
337}
338
339static void __cache_work_func(struct mlx5_cache_ent *ent)
340{
341 struct mlx5_ib_dev *dev = ent->dev;
342 struct mlx5_mr_cache *cache = &dev->cache;
343 int i = order2idx(dev, ent->order);
344 int err;
345
346 if (cache->stopped)
347 return;
348
349 ent = &dev->cache.ent[i];
350 if (ent->cur < 2 * ent->limit && !dev->fill_delay) {
351 err = add_keys(dev, i, 1);
352 if (ent->cur < 2 * ent->limit) {
353 if (err == -EAGAIN) {
354 mlx5_ib_dbg(dev, "returned eagain, order %d\n",
355 i + 2);
356 queue_delayed_work(cache->wq, &ent->dwork,
357 msecs_to_jiffies(3));
358 } else if (err) {
359 mlx5_ib_warn(dev, "command failed order %d, err %d\n",
360 i + 2, err);
361 queue_delayed_work(cache->wq, &ent->dwork,
362 msecs_to_jiffies(1000));
363 } else {
364 queue_work(cache->wq, &ent->work);
365 }
366 }
367 } else if (ent->cur > 2 * ent->limit) {
368 if (!someone_adding(cache) &&
369 time_after(jiffies, cache->last_add + 300 * HZ)) {
370 remove_keys(dev, i, 1);
371 if (ent->cur > ent->limit)
372 queue_work(cache->wq, &ent->work);
373 } else {
374 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
375 }
376 }
377}
378
379static void delayed_cache_work_func(struct work_struct *work)
380{
381 struct mlx5_cache_ent *ent;
382
383 ent = container_of(work, struct mlx5_cache_ent, dwork.work);
384 __cache_work_func(ent);
385}
386
387static void cache_work_func(struct work_struct *work)
388{
389 struct mlx5_cache_ent *ent;
390
391 ent = container_of(work, struct mlx5_cache_ent, work);
392 __cache_work_func(ent);
393}
394
395static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
396{
397 struct mlx5_mr_cache *cache = &dev->cache;
398 struct mlx5_ib_mr *mr = NULL;
399 struct mlx5_cache_ent *ent;
400 int c;
401 int i;
402
403 c = order2idx(dev, order);
404 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
405 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
406 return NULL;
407 }
408
409 for (i = c; i < MAX_MR_CACHE_ENTRIES; i++) {
410 ent = &cache->ent[i];
411
412 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
413
414 spin_lock_irq(&ent->lock);
415 if (!list_empty(&ent->head)) {
416 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
417 list);
418 list_del(&mr->list);
419 ent->cur--;
420 spin_unlock_irq(&ent->lock);
421 if (ent->cur < ent->limit)
422 queue_work(cache->wq, &ent->work);
423 break;
424 }
425 spin_unlock_irq(&ent->lock);
426
427 queue_work(cache->wq, &ent->work);
428
429 if (mr)
430 break;
431 }
432
433 if (!mr)
434 cache->ent[c].miss++;
435
436 return mr;
437}
438
439static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
440{
441 struct mlx5_mr_cache *cache = &dev->cache;
442 struct mlx5_cache_ent *ent;
443 int shrink = 0;
444 int c;
445
446 c = order2idx(dev, mr->order);
447 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
448 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c);
449 return;
450 }
451 ent = &cache->ent[c];
452 spin_lock_irq(&ent->lock);
453 list_add_tail(&mr->list, &ent->head);
454 ent->cur++;
455 if (ent->cur > 2 * ent->limit)
456 shrink = 1;
457 spin_unlock_irq(&ent->lock);
458
459 if (shrink)
460 queue_work(cache->wq, &ent->work);
461}
462
463static void clean_keys(struct mlx5_ib_dev *dev, int c)
464{
465 struct mlx5_mr_cache *cache = &dev->cache;
466 struct mlx5_cache_ent *ent = &cache->ent[c];
467 struct mlx5_ib_mr *mr;
468 int err;
469
470 cancel_delayed_work(&ent->dwork);
471 while (1) {
472 spin_lock_irq(&ent->lock);
473 if (list_empty(&ent->head)) {
474 spin_unlock_irq(&ent->lock);
475 return;
476 }
477 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
478 list_del(&mr->list);
479 ent->cur--;
480 ent->size--;
481 spin_unlock_irq(&ent->lock);
482 err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
483 if (err)
484 mlx5_ib_warn(dev, "failed destroy mkey\n");
485 else
486 kfree(mr);
487 }
488}
489
490static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
491{
492 struct mlx5_mr_cache *cache = &dev->cache;
493 struct mlx5_cache_ent *ent;
494 int i;
495
496 if (!mlx5_debugfs_root)
497 return 0;
498
499 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
500 if (!cache->root)
501 return -ENOMEM;
502
503 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
504 ent = &cache->ent[i];
505 sprintf(ent->name, "%d", ent->order);
506 ent->dir = debugfs_create_dir(ent->name, cache->root);
507 if (!ent->dir)
508 return -ENOMEM;
509
510 ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent,
511 &size_fops);
512 if (!ent->fsize)
513 return -ENOMEM;
514
515 ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent,
516 &limit_fops);
517 if (!ent->flimit)
518 return -ENOMEM;
519
520 ent->fcur = debugfs_create_u32("cur", 0400, ent->dir,
521 &ent->cur);
522 if (!ent->fcur)
523 return -ENOMEM;
524
525 ent->fmiss = debugfs_create_u32("miss", 0600, ent->dir,
526 &ent->miss);
527 if (!ent->fmiss)
528 return -ENOMEM;
529 }
530
531 return 0;
532}
533
534static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
535{
536 if (!mlx5_debugfs_root)
537 return;
538
539 debugfs_remove_recursive(dev->cache.root);
540}
541
542static void delay_time_func(unsigned long ctx)
543{
544 struct mlx5_ib_dev *dev = (struct mlx5_ib_dev *)ctx;
545
546 dev->fill_delay = 0;
547}
548
549int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
550{
551 struct mlx5_mr_cache *cache = &dev->cache;
552 struct mlx5_cache_ent *ent;
553 int limit;
554 int err;
555 int i;
556
557 cache->wq = create_singlethread_workqueue("mkey_cache");
558 if (!cache->wq) {
559 mlx5_ib_warn(dev, "failed to create work queue\n");
560 return -ENOMEM;
561 }
562
563 setup_timer(&dev->delay_timer, delay_time_func, (unsigned long)dev);
564 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
565 INIT_LIST_HEAD(&cache->ent[i].head);
566 spin_lock_init(&cache->ent[i].lock);
567
568 ent = &cache->ent[i];
569 INIT_LIST_HEAD(&ent->head);
570 spin_lock_init(&ent->lock);
571 ent->order = i + 2;
572 ent->dev = dev;
573
574 if (dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE)
575 limit = dev->mdev->profile->mr_cache[i].limit;
576 else
577 limit = 0;
578
579 INIT_WORK(&ent->work, cache_work_func);
580 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
581 ent->limit = limit;
582 queue_work(cache->wq, &ent->work);
583 }
584
585 err = mlx5_mr_cache_debugfs_init(dev);
586 if (err)
587 mlx5_ib_warn(dev, "cache debugfs failure\n");
588
589 return 0;
590}
591
592int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
593{
594 int i;
595
596 dev->cache.stopped = 1;
597 flush_workqueue(dev->cache.wq);
598
599 mlx5_mr_cache_debugfs_cleanup(dev);
600
601 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
602 clean_keys(dev, i);
603
604 destroy_workqueue(dev->cache.wq);
605 del_timer_sync(&dev->delay_timer);
606
607 return 0;
608}
609
610struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
611{
612 struct mlx5_ib_dev *dev = to_mdev(pd->device);
613 struct mlx5_core_dev *mdev = dev->mdev;
614 struct mlx5_create_mkey_mbox_in *in;
615 struct mlx5_mkey_seg *seg;
616 struct mlx5_ib_mr *mr;
617 int err;
618
619 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
620 if (!mr)
621 return ERR_PTR(-ENOMEM);
622
623 in = kzalloc(sizeof(*in), GFP_KERNEL);
624 if (!in) {
625 err = -ENOMEM;
626 goto err_free;
627 }
628
629 seg = &in->seg;
630 seg->flags = convert_access(acc) | MLX5_ACCESS_MODE_PA;
631 seg->flags_pd = cpu_to_be32(to_mpd(pd)->pdn | MLX5_MKEY_LEN64);
632 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
633 seg->start_addr = 0;
634
635 err = mlx5_core_create_mkey(mdev, &mr->mmr, in, sizeof(*in), NULL, NULL,
636 NULL);
637 if (err)
638 goto err_in;
639
640 kfree(in);
641 mr->ibmr.lkey = mr->mmr.key;
642 mr->ibmr.rkey = mr->mmr.key;
643 mr->umem = NULL;
644
645 return &mr->ibmr;
646
647err_in:
648 kfree(in);
649
650err_free:
651 kfree(mr);
652
653 return ERR_PTR(err);
654}
655
656static int get_octo_len(u64 addr, u64 len, int page_size)
657{
658 u64 offset;
659 int npages;
660
661 offset = addr & (page_size - 1);
662 npages = ALIGN(len + offset, page_size) >> ilog2(page_size);
663 return (npages + 1) / 2;
664}
665
666static int use_umr(int order)
667{
668 return order <= 17;
669}
670
671static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
672 struct ib_sge *sg, u64 dma, int n, u32 key,
673 int page_shift, u64 virt_addr, u64 len,
674 int access_flags)
675{
676 struct mlx5_ib_dev *dev = to_mdev(pd->device);
677 struct ib_mr *mr = dev->umrc.mr;
678
679 sg->addr = dma;
680 sg->length = ALIGN(sizeof(u64) * n, 64);
681 sg->lkey = mr->lkey;
682
683 wr->next = NULL;
684 wr->send_flags = 0;
685 wr->sg_list = sg;
686 if (n)
687 wr->num_sge = 1;
688 else
689 wr->num_sge = 0;
690
691 wr->opcode = MLX5_IB_WR_UMR;
692 wr->wr.fast_reg.page_list_len = n;
693 wr->wr.fast_reg.page_shift = page_shift;
694 wr->wr.fast_reg.rkey = key;
695 wr->wr.fast_reg.iova_start = virt_addr;
696 wr->wr.fast_reg.length = len;
697 wr->wr.fast_reg.access_flags = access_flags;
698 wr->wr.fast_reg.page_list = (struct ib_fast_reg_page_list *)pd;
699}
700
701static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
702 struct ib_send_wr *wr, u32 key)
703{
704 wr->send_flags = MLX5_IB_SEND_UMR_UNREG;
705 wr->opcode = MLX5_IB_WR_UMR;
706 wr->wr.fast_reg.rkey = key;
707}
708
709void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context)
710{
711 struct mlx5_ib_umr_context *context;
712 struct ib_wc wc;
713 int err;
714
715 while (1) {
716 err = ib_poll_cq(cq, 1, &wc);
717 if (err < 0) {
718 pr_warn("poll cq error %d\n", err);
719 return;
720 }
721 if (err == 0)
722 break;
723
724 context = (struct mlx5_ib_umr_context *) (unsigned long) wc.wr_id;
725 context->status = wc.status;
726 complete(&context->done);
727 }
728 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
729}
730
731static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
732 u64 virt_addr, u64 len, int npages,
733 int page_shift, int order, int access_flags)
734{
735 struct mlx5_ib_dev *dev = to_mdev(pd->device);
736 struct device *ddev = dev->ib_dev.dma_device;
737 struct umr_common *umrc = &dev->umrc;
738 struct mlx5_ib_umr_context umr_context;
739 struct ib_send_wr wr, *bad;
740 struct mlx5_ib_mr *mr;
741 struct ib_sge sg;
742 int size = sizeof(u64) * npages;
743 int err = 0;
744 int i;
745
746 for (i = 0; i < 1; i++) {
747 mr = alloc_cached_mr(dev, order);
748 if (mr)
749 break;
750
751 err = add_keys(dev, order2idx(dev, order), 1);
752 if (err && err != -EAGAIN) {
753 mlx5_ib_warn(dev, "add_keys failed, err %d\n", err);
754 break;
755 }
756 }
757
758 if (!mr)
759 return ERR_PTR(-EAGAIN);
760
761 mr->pas = kmalloc(size + MLX5_UMR_ALIGN - 1, GFP_KERNEL);
762 if (!mr->pas) {
763 err = -ENOMEM;
764 goto free_mr;
765 }
766
767 mlx5_ib_populate_pas(dev, umem, page_shift,
768 mr_align(mr->pas, MLX5_UMR_ALIGN), 1);
769
770 mr->dma = dma_map_single(ddev, mr_align(mr->pas, MLX5_UMR_ALIGN), size,
771 DMA_TO_DEVICE);
772 if (dma_mapping_error(ddev, mr->dma)) {
773 err = -ENOMEM;
774 goto free_pas;
775 }
776
777 memset(&wr, 0, sizeof(wr));
778 wr.wr_id = (u64)(unsigned long)&umr_context;
779 prep_umr_reg_wqe(pd, &wr, &sg, mr->dma, npages, mr->mmr.key, page_shift, virt_addr, len, access_flags);
780
781 mlx5_ib_init_umr_context(&umr_context);
782 down(&umrc->sem);
783 err = ib_post_send(umrc->qp, &wr, &bad);
784 if (err) {
785 mlx5_ib_warn(dev, "post send failed, err %d\n", err);
786 goto unmap_dma;
787 } else {
788 wait_for_completion(&umr_context.done);
789 if (umr_context.status != IB_WC_SUCCESS) {
790 mlx5_ib_warn(dev, "reg umr failed\n");
791 err = -EFAULT;
792 }
793 }
794
795 mr->mmr.iova = virt_addr;
796 mr->mmr.size = len;
797 mr->mmr.pd = to_mpd(pd)->pdn;
798
799unmap_dma:
800 up(&umrc->sem);
801 dma_unmap_single(ddev, mr->dma, size, DMA_TO_DEVICE);
802
803free_pas:
804 kfree(mr->pas);
805
806free_mr:
807 if (err) {
808 free_cached_mr(dev, mr);
809 return ERR_PTR(err);
810 }
811
812 return mr;
813}
814
815static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr,
816 u64 length, struct ib_umem *umem,
817 int npages, int page_shift,
818 int access_flags)
819{
820 struct mlx5_ib_dev *dev = to_mdev(pd->device);
821 struct mlx5_create_mkey_mbox_in *in;
822 struct mlx5_ib_mr *mr;
823 int inlen;
824 int err;
825
826 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
827 if (!mr)
828 return ERR_PTR(-ENOMEM);
829
830 inlen = sizeof(*in) + sizeof(*in->pas) * ((npages + 1) / 2) * 2;
831 in = mlx5_vzalloc(inlen);
832 if (!in) {
833 err = -ENOMEM;
834 goto err_1;
835 }
836 mlx5_ib_populate_pas(dev, umem, page_shift, in->pas, 0);
837
838 in->seg.flags = convert_access(access_flags) |
839 MLX5_ACCESS_MODE_MTT;
840 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
841 in->seg.start_addr = cpu_to_be64(virt_addr);
842 in->seg.len = cpu_to_be64(length);
843 in->seg.bsfs_octo_size = 0;
844 in->seg.xlt_oct_size = cpu_to_be32(get_octo_len(virt_addr, length, 1 << page_shift));
845 in->seg.log2_page_size = page_shift;
846 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
847 in->xlat_oct_act_size = cpu_to_be32(get_octo_len(virt_addr, length,
848 1 << page_shift));
849 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, inlen, NULL,
850 NULL, NULL);
851 if (err) {
852 mlx5_ib_warn(dev, "create mkey failed\n");
853 goto err_2;
854 }
855 mr->umem = umem;
856 mlx5_vfree(in);
857
858 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmr.key);
859
860 return mr;
861
862err_2:
863 mlx5_vfree(in);
864
865err_1:
866 kfree(mr);
867
868 return ERR_PTR(err);
869}
870
871struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
872 u64 virt_addr, int access_flags,
873 struct ib_udata *udata)
874{
875 struct mlx5_ib_dev *dev = to_mdev(pd->device);
876 struct mlx5_ib_mr *mr = NULL;
877 struct ib_umem *umem;
878 int page_shift;
879 int npages;
880 int ncont;
881 int order;
882 int err;
883
884 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
885 start, virt_addr, length, access_flags);
886 umem = ib_umem_get(pd->uobject->context, start, length, access_flags,
887 0);
888 if (IS_ERR(umem)) {
889 mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(umem));
890 return (void *)umem;
891 }
892
893 mlx5_ib_cont_pages(umem, start, &npages, &page_shift, &ncont, &order);
894 if (!npages) {
895 mlx5_ib_warn(dev, "avoid zero region\n");
896 err = -EINVAL;
897 goto error;
898 }
899
900 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
901 npages, ncont, order, page_shift);
902
903 if (use_umr(order)) {
904 mr = reg_umr(pd, umem, virt_addr, length, ncont, page_shift,
905 order, access_flags);
906 if (PTR_ERR(mr) == -EAGAIN) {
907 mlx5_ib_dbg(dev, "cache empty for order %d", order);
908 mr = NULL;
909 }
910 }
911
912 if (!mr)
913 mr = reg_create(pd, virt_addr, length, umem, ncont, page_shift,
914 access_flags);
915
916 if (IS_ERR(mr)) {
917 err = PTR_ERR(mr);
918 goto error;
919 }
920
921 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmr.key);
922
923 mr->umem = umem;
924 mr->npages = npages;
925 spin_lock(&dev->mr_lock);
926 dev->mdev->priv.reg_pages += npages;
927 spin_unlock(&dev->mr_lock);
928 mr->ibmr.lkey = mr->mmr.key;
929 mr->ibmr.rkey = mr->mmr.key;
930
931 return &mr->ibmr;
932
933error:
934 ib_umem_release(umem);
935 return ERR_PTR(err);
936}
937
938static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
939{
940 struct umr_common *umrc = &dev->umrc;
941 struct mlx5_ib_umr_context umr_context;
942 struct ib_send_wr wr, *bad;
943 int err;
944
945 memset(&wr, 0, sizeof(wr));
946 wr.wr_id = (u64)(unsigned long)&umr_context;
947 prep_umr_unreg_wqe(dev, &wr, mr->mmr.key);
948
949 mlx5_ib_init_umr_context(&umr_context);
950 down(&umrc->sem);
951 err = ib_post_send(umrc->qp, &wr, &bad);
952 if (err) {
953 up(&umrc->sem);
954 mlx5_ib_dbg(dev, "err %d\n", err);
955 goto error;
956 } else {
957 wait_for_completion(&umr_context.done);
958 up(&umrc->sem);
959 }
960 if (umr_context.status != IB_WC_SUCCESS) {
961 mlx5_ib_warn(dev, "unreg umr failed\n");
962 err = -EFAULT;
963 goto error;
964 }
965 return 0;
966
967error:
968 return err;
969}
970
971int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
972{
973 struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
974 struct mlx5_ib_mr *mr = to_mmr(ibmr);
975 struct ib_umem *umem = mr->umem;
976 int npages = mr->npages;
977 int umred = mr->umred;
978 int err;
979
980 if (!umred) {
981 err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
982 if (err) {
983 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
984 mr->mmr.key, err);
985 return err;
986 }
987 } else {
988 err = unreg_umr(dev, mr);
989 if (err) {
990 mlx5_ib_warn(dev, "failed unregister\n");
991 return err;
992 }
993 free_cached_mr(dev, mr);
994 }
995
996 if (umem) {
997 ib_umem_release(umem);
998 spin_lock(&dev->mr_lock);
999 dev->mdev->priv.reg_pages -= npages;
1000 spin_unlock(&dev->mr_lock);
1001 }
1002
1003 if (!umred)
1004 kfree(mr);
1005
1006 return 0;
1007}
1008
1009struct ib_mr *mlx5_ib_create_mr(struct ib_pd *pd,
1010 struct ib_mr_init_attr *mr_init_attr)
1011{
1012 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1013 struct mlx5_create_mkey_mbox_in *in;
1014 struct mlx5_ib_mr *mr;
1015 int access_mode, err;
1016 int ndescs = roundup(mr_init_attr->max_reg_descriptors, 4);
1017
1018 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1019 if (!mr)
1020 return ERR_PTR(-ENOMEM);
1021
1022 in = kzalloc(sizeof(*in), GFP_KERNEL);
1023 if (!in) {
1024 err = -ENOMEM;
1025 goto err_free;
1026 }
1027
1028 in->seg.status = 1 << 6;
1029 in->seg.xlt_oct_size = cpu_to_be32(ndescs);
1030 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
1031 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
1032 access_mode = MLX5_ACCESS_MODE_MTT;
1033
1034 if (mr_init_attr->flags & IB_MR_SIGNATURE_EN) {
1035 u32 psv_index[2];
1036
1037 in->seg.flags_pd = cpu_to_be32(be32_to_cpu(in->seg.flags_pd) |
1038 MLX5_MKEY_BSF_EN);
1039 in->seg.bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
1040 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
1041 if (!mr->sig) {
1042 err = -ENOMEM;
1043 goto err_free_in;
1044 }
1045
1046
1047 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn,
1048 2, psv_index);
1049 if (err)
1050 goto err_free_sig;
1051
1052 access_mode = MLX5_ACCESS_MODE_KLM;
1053 mr->sig->psv_memory.psv_idx = psv_index[0];
1054 mr->sig->psv_wire.psv_idx = psv_index[1];
1055
1056 mr->sig->sig_status_checked = true;
1057 mr->sig->sig_err_exists = false;
1058
1059 ++mr->sig->sigerr_count;
1060 }
1061
1062 in->seg.flags = MLX5_PERM_UMR_EN | access_mode;
1063 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, sizeof(*in),
1064 NULL, NULL, NULL);
1065 if (err)
1066 goto err_destroy_psv;
1067
1068 mr->ibmr.lkey = mr->mmr.key;
1069 mr->ibmr.rkey = mr->mmr.key;
1070 mr->umem = NULL;
1071 kfree(in);
1072
1073 return &mr->ibmr;
1074
1075err_destroy_psv:
1076 if (mr->sig) {
1077 if (mlx5_core_destroy_psv(dev->mdev,
1078 mr->sig->psv_memory.psv_idx))
1079 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1080 mr->sig->psv_memory.psv_idx);
1081 if (mlx5_core_destroy_psv(dev->mdev,
1082 mr->sig->psv_wire.psv_idx))
1083 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1084 mr->sig->psv_wire.psv_idx);
1085 }
1086err_free_sig:
1087 kfree(mr->sig);
1088err_free_in:
1089 kfree(in);
1090err_free:
1091 kfree(mr);
1092 return ERR_PTR(err);
1093}
1094
1095int mlx5_ib_destroy_mr(struct ib_mr *ibmr)
1096{
1097 struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
1098 struct mlx5_ib_mr *mr = to_mmr(ibmr);
1099 int err;
1100
1101 if (mr->sig) {
1102 if (mlx5_core_destroy_psv(dev->mdev,
1103 mr->sig->psv_memory.psv_idx))
1104 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1105 mr->sig->psv_memory.psv_idx);
1106 if (mlx5_core_destroy_psv(dev->mdev,
1107 mr->sig->psv_wire.psv_idx))
1108 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1109 mr->sig->psv_wire.psv_idx);
1110 kfree(mr->sig);
1111 }
1112
1113 err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
1114 if (err) {
1115 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
1116 mr->mmr.key, err);
1117 return err;
1118 }
1119
1120 kfree(mr);
1121
1122 return err;
1123}
1124
1125struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd,
1126 int max_page_list_len)
1127{
1128 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1129 struct mlx5_create_mkey_mbox_in *in;
1130 struct mlx5_ib_mr *mr;
1131 int err;
1132
1133 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1134 if (!mr)
1135 return ERR_PTR(-ENOMEM);
1136
1137 in = kzalloc(sizeof(*in), GFP_KERNEL);
1138 if (!in) {
1139 err = -ENOMEM;
1140 goto err_free;
1141 }
1142
1143 in->seg.status = 1 << 6;
1144 in->seg.xlt_oct_size = cpu_to_be32((max_page_list_len + 1) / 2);
1145 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
1146 in->seg.flags = MLX5_PERM_UMR_EN | MLX5_ACCESS_MODE_MTT;
1147 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
1148
1149
1150 in->seg.log2_page_size = PAGE_SHIFT;
1151
1152 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, sizeof(*in), NULL,
1153 NULL, NULL);
1154 kfree(in);
1155 if (err)
1156 goto err_free;
1157
1158 mr->ibmr.lkey = mr->mmr.key;
1159 mr->ibmr.rkey = mr->mmr.key;
1160 mr->umem = NULL;
1161
1162 return &mr->ibmr;
1163
1164err_free:
1165 kfree(mr);
1166 return ERR_PTR(err);
1167}
1168
1169struct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,
1170 int page_list_len)
1171{
1172 struct mlx5_ib_fast_reg_page_list *mfrpl;
1173 int size = page_list_len * sizeof(u64);
1174
1175 mfrpl = kmalloc(sizeof(*mfrpl), GFP_KERNEL);
1176 if (!mfrpl)
1177 return ERR_PTR(-ENOMEM);
1178
1179 mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL);
1180 if (!mfrpl->ibfrpl.page_list)
1181 goto err_free;
1182
1183 mfrpl->mapped_page_list = dma_alloc_coherent(ibdev->dma_device,
1184 size, &mfrpl->map,
1185 GFP_KERNEL);
1186 if (!mfrpl->mapped_page_list)
1187 goto err_free;
1188
1189 WARN_ON(mfrpl->map & 0x3f);
1190
1191 return &mfrpl->ibfrpl;
1192
1193err_free:
1194 kfree(mfrpl->ibfrpl.page_list);
1195 kfree(mfrpl);
1196 return ERR_PTR(-ENOMEM);
1197}
1198
1199void mlx5_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1200{
1201 struct mlx5_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list);
1202 struct mlx5_ib_dev *dev = to_mdev(page_list->device);
1203 int size = page_list->max_page_list_len * sizeof(u64);
1204
1205 dma_free_coherent(&dev->mdev->pdev->dev, size, mfrpl->mapped_page_list,
1206 mfrpl->map);
1207 kfree(mfrpl->ibfrpl.page_list);
1208 kfree(mfrpl);
1209}
1210
1211int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1212 struct ib_mr_status *mr_status)
1213{
1214 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
1215 int ret = 0;
1216
1217 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
1218 pr_err("Invalid status check mask\n");
1219 ret = -EINVAL;
1220 goto done;
1221 }
1222
1223 mr_status->fail_status = 0;
1224 if (check_mask & IB_MR_CHECK_SIG_STATUS) {
1225 if (!mmr->sig) {
1226 ret = -EINVAL;
1227 pr_err("signature status check requested on a non-signature enabled MR\n");
1228 goto done;
1229 }
1230
1231 mmr->sig->sig_status_checked = true;
1232 if (!mmr->sig->sig_err_exists)
1233 goto done;
1234
1235 if (ibmr->lkey == mmr->sig->err_item.key)
1236 memcpy(&mr_status->sig_err, &mmr->sig->err_item,
1237 sizeof(mr_status->sig_err));
1238 else {
1239 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
1240 mr_status->sig_err.sig_err_offset = 0;
1241 mr_status->sig_err.key = mmr->sig->err_item.key;
1242 }
1243
1244 mmr->sig->sig_err_exists = false;
1245 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
1246 }
1247
1248done:
1249 return ret;
1250}
1251
