1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/iova.h>
21#include <linux/module.h>
22#include <linux/slab.h>
23#include <linux/smp.h>
24#include <linux/bitops.h>
25#include <linux/cpu.h>
26
27
28#define IOVA_ANCHOR ~0UL
29
30static bool iova_rcache_insert(struct iova_domain *iovad,
31 unsigned long pfn,
32 unsigned long size);
33static unsigned long iova_rcache_get(struct iova_domain *iovad,
34 unsigned long size,
35 unsigned long limit_pfn);
36static void init_iova_rcaches(struct iova_domain *iovad);
37static void free_iova_rcaches(struct iova_domain *iovad);
38static void fq_destroy_all_entries(struct iova_domain *iovad);
39static void fq_flush_timeout(struct timer_list *t);
40
41void
42init_iova_domain(struct iova_domain *iovad, unsigned long granule,
43 unsigned long start_pfn)
44{
45
46
47
48
49
50 BUG_ON((granule > PAGE_SIZE) || !is_power_of_2(granule));
51
52 spin_lock_init(&iovad->iova_rbtree_lock);
53 iovad->rbroot = RB_ROOT;
54 iovad->cached_node = &iovad->anchor.node;
55 iovad->cached32_node = &iovad->anchor.node;
56 iovad->granule = granule;
57 iovad->start_pfn = start_pfn;
58 iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad));
59 iovad->flush_cb = NULL;
60 iovad->fq = NULL;
61 iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
62 rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
63 rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
64 init_iova_rcaches(iovad);
65}
66EXPORT_SYMBOL_GPL(init_iova_domain);
67
68static void free_iova_flush_queue(struct iova_domain *iovad)
69{
70 if (!iovad->fq)
71 return;
72
73 if (timer_pending(&iovad->fq_timer))
74 del_timer(&iovad->fq_timer);
75
76 fq_destroy_all_entries(iovad);
77
78 free_percpu(iovad->fq);
79
80 iovad->fq = NULL;
81 iovad->flush_cb = NULL;
82 iovad->entry_dtor = NULL;
83}
84
85int init_iova_flush_queue(struct iova_domain *iovad,
86 iova_flush_cb flush_cb, iova_entry_dtor entry_dtor)
87{
88 int cpu;
89
90 atomic64_set(&iovad->fq_flush_start_cnt, 0);
91 atomic64_set(&iovad->fq_flush_finish_cnt, 0);
92
93 iovad->fq = alloc_percpu(struct iova_fq);
94 if (!iovad->fq)
95 return -ENOMEM;
96
97 iovad->flush_cb = flush_cb;
98 iovad->entry_dtor = entry_dtor;
99
100 for_each_possible_cpu(cpu) {
101 struct iova_fq *fq;
102
103 fq = per_cpu_ptr(iovad->fq, cpu);
104 fq->head = 0;
105 fq->tail = 0;
106
107 spin_lock_init(&fq->lock);
108 }
109
110 timer_setup(&iovad->fq_timer, fq_flush_timeout, 0);
111 atomic_set(&iovad->fq_timer_on, 0);
112
113 return 0;
114}
115EXPORT_SYMBOL_GPL(init_iova_flush_queue);
116
117static struct rb_node *
118__get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn)
119{
120 if (limit_pfn <= iovad->dma_32bit_pfn)
121 return iovad->cached32_node;
122
123 return iovad->cached_node;
124}
125
126static void
127__cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new)
128{
129 if (new->pfn_hi < iovad->dma_32bit_pfn)
130 iovad->cached32_node = &new->node;
131 else
132 iovad->cached_node = &new->node;
133}
134
135static void
136__cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)
137{
138 struct iova *cached_iova;
139
140 cached_iova = rb_entry(iovad->cached32_node, struct iova, node);
141 if (free->pfn_hi < iovad->dma_32bit_pfn &&
142 free->pfn_lo >= cached_iova->pfn_lo)
143 iovad->cached32_node = rb_next(&free->node);
144
145 cached_iova = rb_entry(iovad->cached_node, struct iova, node);
146 if (free->pfn_lo >= cached_iova->pfn_lo)
147 iovad->cached_node = rb_next(&free->node);
148}
149
150
151static void
152iova_insert_rbtree(struct rb_root *root, struct iova *iova,
153 struct rb_node *start)
154{
155 struct rb_node **new, *parent = NULL;
156
157 new = (start) ? &start : &(root->rb_node);
158
159 while (*new) {
160 struct iova *this = rb_entry(*new, struct iova, node);
161
162 parent = *new;
163
164 if (iova->pfn_lo < this->pfn_lo)
165 new = &((*new)->rb_left);
166 else if (iova->pfn_lo > this->pfn_lo)
167 new = &((*new)->rb_right);
168 else {
169 WARN_ON(1);
170 return;
171 }
172 }
173
174 rb_link_node(&iova->node, parent, new);
175 rb_insert_color(&iova->node, root);
176}
177
178static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
179 unsigned long size, unsigned long limit_pfn,
180 struct iova *new, bool size_aligned)
181{
182 struct rb_node *curr, *prev;
183 struct iova *curr_iova;
184 unsigned long flags;
185 unsigned long new_pfn;
186 unsigned long align_mask = ~0UL;
187
188 if (size_aligned)
189 align_mask <<= fls_long(size - 1);
190
191
192 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
193 curr = __get_cached_rbnode(iovad, limit_pfn);
194 curr_iova = rb_entry(curr, struct iova, node);
195 do {
196 limit_pfn = min(limit_pfn, curr_iova->pfn_lo);
197 new_pfn = (limit_pfn - size) & align_mask;
198 prev = curr;
199 curr = rb_prev(curr);
200 curr_iova = rb_entry(curr, struct iova, node);
201 } while (curr && new_pfn <= curr_iova->pfn_hi);
202
203 if (limit_pfn < size || new_pfn < iovad->start_pfn) {
204 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
205 return -ENOMEM;
206 }
207
208
209 new->pfn_lo = new_pfn;
210 new->pfn_hi = new->pfn_lo + size - 1;
211
212
213 iova_insert_rbtree(&iovad->rbroot, new, prev);
214 __cached_rbnode_insert_update(iovad, new);
215
216 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
217
218
219 return 0;
220}
221
222static struct kmem_cache *iova_cache;
223static unsigned int iova_cache_users;
224static DEFINE_MUTEX(iova_cache_mutex);
225
226struct iova *alloc_iova_mem(void)
227{
228 return kmem_cache_alloc(iova_cache, GFP_ATOMIC);
229}
230EXPORT_SYMBOL(alloc_iova_mem);
231
232void free_iova_mem(struct iova *iova)
233{
234 if (iova->pfn_lo != IOVA_ANCHOR)
235 kmem_cache_free(iova_cache, iova);
236}
237EXPORT_SYMBOL(free_iova_mem);
238
239int iova_cache_get(void)
240{
241 mutex_lock(&iova_cache_mutex);
242 if (!iova_cache_users) {
243 iova_cache = kmem_cache_create(
244 "iommu_iova", sizeof(struct iova), 0,
245 SLAB_HWCACHE_ALIGN, NULL);
246 if (!iova_cache) {
247 mutex_unlock(&iova_cache_mutex);
248 printk(KERN_ERR "Couldn't create iova cache\n");
249 return -ENOMEM;
250 }
251 }
252
253 iova_cache_users++;
254 mutex_unlock(&iova_cache_mutex);
255
256 return 0;
257}
258EXPORT_SYMBOL_GPL(iova_cache_get);
259
260void iova_cache_put(void)
261{
262 mutex_lock(&iova_cache_mutex);
263 if (WARN_ON(!iova_cache_users)) {
264 mutex_unlock(&iova_cache_mutex);
265 return;
266 }
267 iova_cache_users--;
268 if (!iova_cache_users)
269 kmem_cache_destroy(iova_cache);
270 mutex_unlock(&iova_cache_mutex);
271}
272EXPORT_SYMBOL_GPL(iova_cache_put);
273
274
275
276
277
278
279
280
281
282
283
284
285struct iova *
286alloc_iova(struct iova_domain *iovad, unsigned long size,
287 unsigned long limit_pfn,
288 bool size_aligned)
289{
290 struct iova *new_iova;
291 int ret;
292
293 new_iova = alloc_iova_mem();
294 if (!new_iova)
295 return NULL;
296
297 ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn + 1,
298 new_iova, size_aligned);
299
300 if (ret) {
301 free_iova_mem(new_iova);
302 return NULL;
303 }
304
305 return new_iova;
306}
307EXPORT_SYMBOL_GPL(alloc_iova);
308
309static struct iova *
310private_find_iova(struct iova_domain *iovad, unsigned long pfn)
311{
312 struct rb_node *node = iovad->rbroot.rb_node;
313
314 assert_spin_locked(&iovad->iova_rbtree_lock);
315
316 while (node) {
317 struct iova *iova = rb_entry(node, struct iova, node);
318
319 if (pfn < iova->pfn_lo)
320 node = node->rb_left;
321 else if (pfn > iova->pfn_hi)
322 node = node->rb_right;
323 else
324 return iova;
325 }
326
327 return NULL;
328}
329
330static void private_free_iova(struct iova_domain *iovad, struct iova *iova)
331{
332 assert_spin_locked(&iovad->iova_rbtree_lock);
333 __cached_rbnode_delete_update(iovad, iova);
334 rb_erase(&iova->node, &iovad->rbroot);
335 free_iova_mem(iova);
336}
337
338
339
340
341
342
343
344
345struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn)
346{
347 unsigned long flags;
348 struct iova *iova;
349
350
351 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
352 iova = private_find_iova(iovad, pfn);
353 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
354 return iova;
355}
356EXPORT_SYMBOL_GPL(find_iova);
357
358
359
360
361
362
363
364void
365__free_iova(struct iova_domain *iovad, struct iova *iova)
366{
367 unsigned long flags;
368
369 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
370 private_free_iova(iovad, iova);
371 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
372}
373EXPORT_SYMBOL_GPL(__free_iova);
374
375
376
377
378
379
380
381
382void
383free_iova(struct iova_domain *iovad, unsigned long pfn)
384{
385 struct iova *iova = find_iova(iovad, pfn);
386
387 if (iova)
388 __free_iova(iovad, iova);
389
390}
391EXPORT_SYMBOL_GPL(free_iova);
392
393
394
395
396
397
398
399
400
401
402
403unsigned long
404alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
405 unsigned long limit_pfn, bool flush_rcache)
406{
407 unsigned long iova_pfn;
408 struct iova *new_iova;
409
410 iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1);
411 if (iova_pfn)
412 return iova_pfn;
413
414retry:
415 new_iova = alloc_iova(iovad, size, limit_pfn, true);
416 if (!new_iova) {
417 unsigned int cpu;
418
419 if (!flush_rcache)
420 return 0;
421
422
423 flush_rcache = false;
424 for_each_online_cpu(cpu)
425 free_cpu_cached_iovas(cpu, iovad);
426 goto retry;
427 }
428
429 return new_iova->pfn_lo;
430}
431EXPORT_SYMBOL_GPL(alloc_iova_fast);
432
433
434
435
436
437
438
439
440
441void
442free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
443{
444 if (iova_rcache_insert(iovad, pfn, size))
445 return;
446
447 free_iova(iovad, pfn);
448}
449EXPORT_SYMBOL_GPL(free_iova_fast);
450
451#define fq_ring_for_each(i, fq) \
452 for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
453
454static inline bool fq_full(struct iova_fq *fq)
455{
456 assert_spin_locked(&fq->lock);
457 return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
458}
459
460static inline unsigned fq_ring_add(struct iova_fq *fq)
461{
462 unsigned idx = fq->tail;
463
464 assert_spin_locked(&fq->lock);
465
466 fq->tail = (idx + 1) % IOVA_FQ_SIZE;
467
468 return idx;
469}
470
471static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)
472{
473 u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt);
474 unsigned idx;
475
476 assert_spin_locked(&fq->lock);
477
478 fq_ring_for_each(idx, fq) {
479
480 if (fq->entries[idx].counter >= counter)
481 break;
482
483 if (iovad->entry_dtor)
484 iovad->entry_dtor(fq->entries[idx].data);
485
486 free_iova_fast(iovad,
487 fq->entries[idx].iova_pfn,
488 fq->entries[idx].pages);
489
490 fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
491 }
492}
493
494static void iova_domain_flush(struct iova_domain *iovad)
495{
496 atomic64_inc(&iovad->fq_flush_start_cnt);
497 iovad->flush_cb(iovad);
498 atomic64_inc(&iovad->fq_flush_finish_cnt);
499}
500
501static void fq_destroy_all_entries(struct iova_domain *iovad)
502{
503 int cpu;
504
505
506
507
508
509
510 if (!iovad->entry_dtor)
511 return;
512
513 for_each_possible_cpu(cpu) {
514 struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
515 int idx;
516
517 fq_ring_for_each(idx, fq)
518 iovad->entry_dtor(fq->entries[idx].data);
519 }
520}
521
522static void fq_flush_timeout(struct timer_list *t)
523{
524 struct iova_domain *iovad = from_timer(iovad, t, fq_timer);
525 int cpu;
526
527 atomic_set(&iovad->fq_timer_on, 0);
528 iova_domain_flush(iovad);
529
530 for_each_possible_cpu(cpu) {
531 unsigned long flags;
532 struct iova_fq *fq;
533
534 fq = per_cpu_ptr(iovad->fq, cpu);
535 spin_lock_irqsave(&fq->lock, flags);
536 fq_ring_free(iovad, fq);
537 spin_unlock_irqrestore(&fq->lock, flags);
538 }
539}
540
541void queue_iova(struct iova_domain *iovad,
542 unsigned long pfn, unsigned long pages,
543 unsigned long data)
544{
545 struct iova_fq *fq = raw_cpu_ptr(iovad->fq);
546 unsigned long flags;
547 unsigned idx;
548
549 spin_lock_irqsave(&fq->lock, flags);
550
551
552
553
554
555
556 fq_ring_free(iovad, fq);
557
558 if (fq_full(fq)) {
559 iova_domain_flush(iovad);
560 fq_ring_free(iovad, fq);
561 }
562
563 idx = fq_ring_add(fq);
564
565 fq->entries[idx].iova_pfn = pfn;
566 fq->entries[idx].pages = pages;
567 fq->entries[idx].data = data;
568 fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);
569
570 spin_unlock_irqrestore(&fq->lock, flags);
571
572 if (atomic_cmpxchg(&iovad->fq_timer_on, 0, 1) == 0)
573 mod_timer(&iovad->fq_timer,
574 jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
575}
576EXPORT_SYMBOL_GPL(queue_iova);
577
578
579
580
581
582
583void put_iova_domain(struct iova_domain *iovad)
584{
585 struct iova *iova, *tmp;
586
587 free_iova_flush_queue(iovad);
588 free_iova_rcaches(iovad);
589 rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
590 free_iova_mem(iova);
591}
592EXPORT_SYMBOL_GPL(put_iova_domain);
593
594static int
595__is_range_overlap(struct rb_node *node,
596 unsigned long pfn_lo, unsigned long pfn_hi)
597{
598 struct iova *iova = rb_entry(node, struct iova, node);
599
600 if ((pfn_lo <= iova->pfn_hi) && (pfn_hi >= iova->pfn_lo))
601 return 1;
602 return 0;
603}
604
605static inline struct iova *
606alloc_and_init_iova(unsigned long pfn_lo, unsigned long pfn_hi)
607{
608 struct iova *iova;
609
610 iova = alloc_iova_mem();
611 if (iova) {
612 iova->pfn_lo = pfn_lo;
613 iova->pfn_hi = pfn_hi;
614 }
615
616 return iova;
617}
618
619static struct iova *
620__insert_new_range(struct iova_domain *iovad,
621 unsigned long pfn_lo, unsigned long pfn_hi)
622{
623 struct iova *iova;
624
625 iova = alloc_and_init_iova(pfn_lo, pfn_hi);
626 if (iova)
627 iova_insert_rbtree(&iovad->rbroot, iova, NULL);
628
629 return iova;
630}
631
632static void
633__adjust_overlap_range(struct iova *iova,
634 unsigned long *pfn_lo, unsigned long *pfn_hi)
635{
636 if (*pfn_lo < iova->pfn_lo)
637 iova->pfn_lo = *pfn_lo;
638 if (*pfn_hi > iova->pfn_hi)
639 *pfn_lo = iova->pfn_hi + 1;
640}
641
642
643
644
645
646
647
648
649
650struct iova *
651reserve_iova(struct iova_domain *iovad,
652 unsigned long pfn_lo, unsigned long pfn_hi)
653{
654 struct rb_node *node;
655 unsigned long flags;
656 struct iova *iova;
657 unsigned int overlap = 0;
658
659
660 if (WARN_ON((pfn_hi | pfn_lo) > (ULLONG_MAX >> iova_shift(iovad))))
661 return NULL;
662
663 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
664 for (node = rb_first(&iovad->rbroot); node; node = rb_next(node)) {
665 if (__is_range_overlap(node, pfn_lo, pfn_hi)) {
666 iova = rb_entry(node, struct iova, node);
667 __adjust_overlap_range(iova, &pfn_lo, &pfn_hi);
668 if ((pfn_lo >= iova->pfn_lo) &&
669 (pfn_hi <= iova->pfn_hi))
670 goto finish;
671 overlap = 1;
672
673 } else if (overlap)
674 break;
675 }
676
677
678
679
680 iova = __insert_new_range(iovad, pfn_lo, pfn_hi);
681finish:
682
683 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
684 return iova;
685}
686EXPORT_SYMBOL_GPL(reserve_iova);
687
688
689
690
691
692
693
694
695void
696copy_reserved_iova(struct iova_domain *from, struct iova_domain *to)
697{
698 unsigned long flags;
699 struct rb_node *node;
700
701 spin_lock_irqsave(&from->iova_rbtree_lock, flags);
702 for (node = rb_first(&from->rbroot); node; node = rb_next(node)) {
703 struct iova *iova = rb_entry(node, struct iova, node);
704 struct iova *new_iova;
705
706 if (iova->pfn_lo == IOVA_ANCHOR)
707 continue;
708
709 new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi);
710 if (!new_iova)
711 printk(KERN_ERR "Reserve iova range %lx@%lx failed\n",
712 iova->pfn_lo, iova->pfn_lo);
713 }
714 spin_unlock_irqrestore(&from->iova_rbtree_lock, flags);
715}
716EXPORT_SYMBOL_GPL(copy_reserved_iova);
717
718struct iova *
719split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,
720 unsigned long pfn_lo, unsigned long pfn_hi)
721{
722 unsigned long flags;
723 struct iova *prev = NULL, *next = NULL;
724
725 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
726 if (iova->pfn_lo < pfn_lo) {
727 prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1);
728 if (prev == NULL)
729 goto error;
730 }
731 if (iova->pfn_hi > pfn_hi) {
732 next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi);
733 if (next == NULL)
734 goto error;
735 }
736
737 __cached_rbnode_delete_update(iovad, iova);
738 rb_erase(&iova->node, &iovad->rbroot);
739
740 if (prev) {
741 iova_insert_rbtree(&iovad->rbroot, prev, NULL);
742 iova->pfn_lo = pfn_lo;
743 }
744 if (next) {
745 iova_insert_rbtree(&iovad->rbroot, next, NULL);
746 iova->pfn_hi = pfn_hi;
747 }
748 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
749
750 return iova;
751
752error:
753 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
754 if (prev)
755 free_iova_mem(prev);
756 return NULL;
757}
758
759
760
761
762
763
764
765
766
767#define IOVA_MAG_SIZE 128
768
769struct iova_magazine {
770 unsigned long size;
771 unsigned long pfns[IOVA_MAG_SIZE];
772};
773
774struct iova_cpu_rcache {
775 spinlock_t lock;
776 struct iova_magazine *loaded;
777 struct iova_magazine *prev;
778};
779
780static struct iova_magazine *iova_magazine_alloc(gfp_t flags)
781{
782 return kzalloc(sizeof(struct iova_magazine), flags);
783}
784
785static void iova_magazine_free(struct iova_magazine *mag)
786{
787 kfree(mag);
788}
789
790static void
791iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
792{
793 unsigned long flags;
794 int i;
795
796 if (!mag)
797 return;
798
799 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
800
801 for (i = 0 ; i < mag->size; ++i) {
802 struct iova *iova = private_find_iova(iovad, mag->pfns[i]);
803
804 BUG_ON(!iova);
805 private_free_iova(iovad, iova);
806 }
807
808 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
809
810 mag->size = 0;
811}
812
813static bool iova_magazine_full(struct iova_magazine *mag)
814{
815 return (mag && mag->size == IOVA_MAG_SIZE);
816}
817
818static bool iova_magazine_empty(struct iova_magazine *mag)
819{
820 return (!mag || mag->size == 0);
821}
822
823static unsigned long iova_magazine_pop(struct iova_magazine *mag,
824 unsigned long limit_pfn)
825{
826 int i;
827 unsigned long pfn;
828
829 BUG_ON(iova_magazine_empty(mag));
830
831
832 for (i = mag->size - 1; mag->pfns[i] > limit_pfn; i--)
833 if (i == 0)
834 return 0;
835
836
837 pfn = mag->pfns[i];
838 mag->pfns[i] = mag->pfns[--mag->size];
839
840 return pfn;
841}
842
843static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn)
844{
845 BUG_ON(iova_magazine_full(mag));
846
847 mag->pfns[mag->size++] = pfn;
848}
849
850static void init_iova_rcaches(struct iova_domain *iovad)
851{
852 struct iova_cpu_rcache *cpu_rcache;
853 struct iova_rcache *rcache;
854 unsigned int cpu;
855 int i;
856
857 for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
858 rcache = &iovad->rcaches[i];
859 spin_lock_init(&rcache->lock);
860 rcache->depot_size = 0;
861 rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
862 if (WARN_ON(!rcache->cpu_rcaches))
863 continue;
864 for_each_possible_cpu(cpu) {
865 cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
866 spin_lock_init(&cpu_rcache->lock);
867 cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);
868 cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
869 }
870 }
871}
872
873
874
875
876
877
878
879static bool __iova_rcache_insert(struct iova_domain *iovad,
880 struct iova_rcache *rcache,
881 unsigned long iova_pfn)
882{
883 struct iova_magazine *mag_to_free = NULL;
884 struct iova_cpu_rcache *cpu_rcache;
885 bool can_insert = false;
886 unsigned long flags;
887
888 cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
889 spin_lock_irqsave(&cpu_rcache->lock, flags);
890
891 if (!iova_magazine_full(cpu_rcache->loaded)) {
892 can_insert = true;
893 } else if (!iova_magazine_full(cpu_rcache->prev)) {
894 swap(cpu_rcache->prev, cpu_rcache->loaded);
895 can_insert = true;
896 } else {
897 struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
898
899 if (new_mag) {
900 spin_lock(&rcache->lock);
901 if (rcache->depot_size < MAX_GLOBAL_MAGS) {
902 rcache->depot[rcache->depot_size++] =
903 cpu_rcache->loaded;
904 } else {
905 mag_to_free = cpu_rcache->loaded;
906 }
907 spin_unlock(&rcache->lock);
908
909 cpu_rcache->loaded = new_mag;
910 can_insert = true;
911 }
912 }
913
914 if (can_insert)
915 iova_magazine_push(cpu_rcache->loaded, iova_pfn);
916
917 spin_unlock_irqrestore(&cpu_rcache->lock, flags);
918
919 if (mag_to_free) {
920 iova_magazine_free_pfns(mag_to_free, iovad);
921 iova_magazine_free(mag_to_free);
922 }
923
924 return can_insert;
925}
926
927static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn,
928 unsigned long size)
929{
930 unsigned int log_size = order_base_2(size);
931
932 if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
933 return false;
934
935 return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn);
936}
937
938
939
940
941
942
943static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
944 unsigned long limit_pfn)
945{
946 struct iova_cpu_rcache *cpu_rcache;
947 unsigned long iova_pfn = 0;
948 bool has_pfn = false;
949 unsigned long flags;
950
951 cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
952 spin_lock_irqsave(&cpu_rcache->lock, flags);
953
954 if (!iova_magazine_empty(cpu_rcache->loaded)) {
955 has_pfn = true;
956 } else if (!iova_magazine_empty(cpu_rcache->prev)) {
957 swap(cpu_rcache->prev, cpu_rcache->loaded);
958 has_pfn = true;
959 } else {
960 spin_lock(&rcache->lock);
961 if (rcache->depot_size > 0) {
962 iova_magazine_free(cpu_rcache->loaded);
963 cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
964 has_pfn = true;
965 }
966 spin_unlock(&rcache->lock);
967 }
968
969 if (has_pfn)
970 iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn);
971
972 spin_unlock_irqrestore(&cpu_rcache->lock, flags);
973
974 return iova_pfn;
975}
976
977
978
979
980
981
982static unsigned long iova_rcache_get(struct iova_domain *iovad,
983 unsigned long size,
984 unsigned long limit_pfn)
985{
986 unsigned int log_size = order_base_2(size);
987
988 if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
989 return 0;
990
991 return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);
992}
993
994
995
996
997static void free_iova_rcaches(struct iova_domain *iovad)
998{
999 struct iova_rcache *rcache;
1000 struct iova_cpu_rcache *cpu_rcache;
1001 unsigned int cpu;
1002 int i, j;
1003
1004 for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
1005 rcache = &iovad->rcaches[i];
1006 for_each_possible_cpu(cpu) {
1007 cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
1008 iova_magazine_free(cpu_rcache->loaded);
1009 iova_magazine_free(cpu_rcache->prev);
1010 }
1011 free_percpu(rcache->cpu_rcaches);
1012 for (j = 0; j < rcache->depot_size; ++j)
1013 iova_magazine_free(rcache->depot[j]);
1014 }
1015}
1016
1017
1018
1019
1020void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
1021{
1022 struct iova_cpu_rcache *cpu_rcache;
1023 struct iova_rcache *rcache;
1024 unsigned long flags;
1025 int i;
1026
1027 for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
1028 rcache = &iovad->rcaches[i];
1029 cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
1030 spin_lock_irqsave(&cpu_rcache->lock, flags);
1031 iova_magazine_free_pfns(cpu_rcache->loaded, iovad);
1032 iova_magazine_free_pfns(cpu_rcache->prev, iovad);
1033 spin_unlock_irqrestore(&cpu_rcache->lock, flags);
1034 }
1035}
1036
1037MODULE_AUTHOR("Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>");
1038MODULE_LICENSE("GPL");
1039