1
2
3#include <linux/sched/mm.h>
4#include <linux/mutex.h>
5#include <linux/mm.h>
6#include <linux/mm_types.h>
7#include <linux/mmu_context.h>
8#include <linux/mmu_notifier.h>
9#include <asm/copro.h>
10#include <asm/pnv-ocxl.h>
11#include <misc/ocxl.h>
12#include "ocxl_internal.h"
13#include "trace.h"
14
15
16#define SPA_PASID_BITS 15
17#define SPA_PASID_MAX ((1 << SPA_PASID_BITS) - 1)
18#define SPA_PE_MASK SPA_PASID_MAX
19#define SPA_SPA_SIZE_LOG 22
20
21#define SPA_CFG_SF (1ull << (63-0))
22#define SPA_CFG_TA (1ull << (63-1))
23#define SPA_CFG_HV (1ull << (63-3))
24#define SPA_CFG_UV (1ull << (63-4))
25#define SPA_CFG_XLAT_hpt (0ull << (63-6))
26#define SPA_CFG_XLAT_roh (2ull << (63-6))
27#define SPA_CFG_XLAT_ror (3ull << (63-6))
28#define SPA_CFG_PR (1ull << (63-49))
29#define SPA_CFG_TC (1ull << (63-54))
30#define SPA_CFG_DR (1ull << (63-59))
31
32#define SPA_XSL_TF (1ull << (63-3))
33#define SPA_XSL_S (1ull << (63-38))
34
35#define SPA_PE_VALID 0x80000000
36
37struct ocxl_link;
38
39struct pe_data {
40 struct mm_struct *mm;
41
42 void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr);
43
44 void *xsl_err_data;
45 struct rcu_head rcu;
46 struct ocxl_link *link;
47 struct mmu_notifier mmu_notifier;
48};
49
50struct spa {
51 struct ocxl_process_element *spa_mem;
52 int spa_order;
53 struct mutex spa_lock;
54 struct radix_tree_root pe_tree;
55 char *irq_name;
56 int virq;
57 void __iomem *reg_dsisr;
58 void __iomem *reg_dar;
59 void __iomem *reg_tfc;
60 void __iomem *reg_pe_handle;
61
62
63
64
65
66
67 struct xsl_fault {
68 struct work_struct fault_work;
69 u64 pe;
70 u64 dsisr;
71 u64 dar;
72 struct pe_data pe_data;
73 } xsl_fault;
74};
75
76
77
78
79
80
81
82
83
84struct ocxl_link {
85 struct list_head list;
86 struct kref ref;
87 int domain;
88 int bus;
89 int dev;
90 void __iomem *arva;
91 spinlock_t atsd_lock;
92 atomic_t irq_available;
93 struct spa *spa;
94 void *platform_data;
95};
96static struct list_head links_list = LIST_HEAD_INIT(links_list);
97static DEFINE_MUTEX(links_list_lock);
98
99enum xsl_response {
100 CONTINUE,
101 ADDRESS_ERROR,
102 RESTART,
103};
104
105
106static void read_irq(struct spa *spa, u64 *dsisr, u64 *dar, u64 *pe)
107{
108 u64 reg;
109
110 *dsisr = in_be64(spa->reg_dsisr);
111 *dar = in_be64(spa->reg_dar);
112 reg = in_be64(spa->reg_pe_handle);
113 *pe = reg & SPA_PE_MASK;
114}
115
116static void ack_irq(struct spa *spa, enum xsl_response r)
117{
118 u64 reg = 0;
119
120
121 if (r == RESTART)
122 reg = PPC_BIT(31);
123 else if (r == ADDRESS_ERROR)
124 reg = PPC_BIT(30);
125 else
126 WARN(1, "Invalid irq response %d\n", r);
127
128 if (reg) {
129 trace_ocxl_fault_ack(spa->spa_mem, spa->xsl_fault.pe,
130 spa->xsl_fault.dsisr, spa->xsl_fault.dar, reg);
131 out_be64(spa->reg_tfc, reg);
132 }
133}
134
135static void xsl_fault_handler_bh(struct work_struct *fault_work)
136{
137 vm_fault_t flt = 0;
138 unsigned long access, flags, inv_flags = 0;
139 enum xsl_response r;
140 struct xsl_fault *fault = container_of(fault_work, struct xsl_fault,
141 fault_work);
142 struct spa *spa = container_of(fault, struct spa, xsl_fault);
143
144 int rc;
145
146
147
148
149
150 rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr,
151 &flt);
152 if (rc) {
153 pr_debug("copro_handle_mm_fault failed: %d\n", rc);
154 if (fault->pe_data.xsl_err_cb) {
155 fault->pe_data.xsl_err_cb(
156 fault->pe_data.xsl_err_data,
157 fault->dar, fault->dsisr);
158 }
159 r = ADDRESS_ERROR;
160 goto ack;
161 }
162
163 if (!radix_enabled()) {
164
165
166
167
168
169 access = _PAGE_PRESENT | _PAGE_READ;
170 if (fault->dsisr & SPA_XSL_S)
171 access |= _PAGE_WRITE;
172
173 if (REGION_ID(fault->dar) != USER_REGION_ID)
174 access |= _PAGE_PRIVILEGED;
175
176 local_irq_save(flags);
177 hash_page_mm(fault->pe_data.mm, fault->dar, access, 0x300,
178 inv_flags);
179 local_irq_restore(flags);
180 }
181 r = RESTART;
182ack:
183 mmput(fault->pe_data.mm);
184 ack_irq(spa, r);
185}
186
187static irqreturn_t xsl_fault_handler(int irq, void *data)
188{
189 struct ocxl_link *link = (struct ocxl_link *) data;
190 struct spa *spa = link->spa;
191 u64 dsisr, dar, pe_handle;
192 struct pe_data *pe_data;
193 struct ocxl_process_element *pe;
194 int pid;
195 bool schedule = false;
196
197 read_irq(spa, &dsisr, &dar, &pe_handle);
198 trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1);
199
200 WARN_ON(pe_handle > SPA_PE_MASK);
201 pe = spa->spa_mem + pe_handle;
202 pid = be32_to_cpu(pe->pid);
203
204
205
206
207
208
209
210 if (!(dsisr & SPA_XSL_TF)) {
211 WARN(1, "Invalid xsl interrupt fault register %#llx\n", dsisr);
212 ack_irq(spa, ADDRESS_ERROR);
213 return IRQ_HANDLED;
214 }
215
216 rcu_read_lock();
217 pe_data = radix_tree_lookup(&spa->pe_tree, pe_handle);
218 if (!pe_data) {
219
220
221
222
223
224
225
226
227
228
229 rcu_read_unlock();
230 pr_debug("Unknown mm context for xsl interrupt\n");
231 ack_irq(spa, ADDRESS_ERROR);
232 return IRQ_HANDLED;
233 }
234
235 if (!pe_data->mm) {
236
237
238
239
240 rcu_read_unlock();
241 pr_warn("Unresolved OpenCAPI xsl fault in kernel context\n");
242 ack_irq(spa, ADDRESS_ERROR);
243 return IRQ_HANDLED;
244 }
245 WARN_ON(pe_data->mm->context.id != pid);
246
247 if (mmget_not_zero(pe_data->mm)) {
248 spa->xsl_fault.pe = pe_handle;
249 spa->xsl_fault.dar = dar;
250 spa->xsl_fault.dsisr = dsisr;
251 spa->xsl_fault.pe_data = *pe_data;
252 schedule = true;
253
254 }
255 rcu_read_unlock();
256 if (schedule)
257 schedule_work(&spa->xsl_fault.fault_work);
258 else
259 ack_irq(spa, ADDRESS_ERROR);
260 return IRQ_HANDLED;
261}
262
263static void unmap_irq_registers(struct spa *spa)
264{
265 pnv_ocxl_unmap_xsl_regs(spa->reg_dsisr, spa->reg_dar, spa->reg_tfc,
266 spa->reg_pe_handle);
267}
268
269static int map_irq_registers(struct pci_dev *dev, struct spa *spa)
270{
271 return pnv_ocxl_map_xsl_regs(dev, &spa->reg_dsisr, &spa->reg_dar,
272 &spa->reg_tfc, &spa->reg_pe_handle);
273}
274
275static int setup_xsl_irq(struct pci_dev *dev, struct ocxl_link *link)
276{
277 struct spa *spa = link->spa;
278 int rc;
279 int hwirq;
280
281 rc = pnv_ocxl_get_xsl_irq(dev, &hwirq);
282 if (rc)
283 return rc;
284
285 rc = map_irq_registers(dev, spa);
286 if (rc)
287 return rc;
288
289 spa->irq_name = kasprintf(GFP_KERNEL, "ocxl-xsl-%x-%x-%x",
290 link->domain, link->bus, link->dev);
291 if (!spa->irq_name) {
292 dev_err(&dev->dev, "Can't allocate name for xsl interrupt\n");
293 rc = -ENOMEM;
294 goto err_xsl;
295 }
296
297
298
299
300 spa->virq = irq_create_mapping(NULL, hwirq);
301 if (!spa->virq) {
302 dev_err(&dev->dev,
303 "irq_create_mapping failed for translation interrupt\n");
304 rc = -EINVAL;
305 goto err_name;
306 }
307
308 dev_dbg(&dev->dev, "hwirq %d mapped to virq %d\n", hwirq, spa->virq);
309
310 rc = request_irq(spa->virq, xsl_fault_handler, 0, spa->irq_name,
311 link);
312 if (rc) {
313 dev_err(&dev->dev,
314 "request_irq failed for translation interrupt: %d\n",
315 rc);
316 rc = -EINVAL;
317 goto err_mapping;
318 }
319 return 0;
320
321err_mapping:
322 irq_dispose_mapping(spa->virq);
323err_name:
324 kfree(spa->irq_name);
325err_xsl:
326 unmap_irq_registers(spa);
327 return rc;
328}
329
330static void release_xsl_irq(struct ocxl_link *link)
331{
332 struct spa *spa = link->spa;
333
334 if (spa->virq) {
335 free_irq(spa->virq, link);
336 irq_dispose_mapping(spa->virq);
337 }
338 kfree(spa->irq_name);
339 unmap_irq_registers(spa);
340}
341
342static int alloc_spa(struct pci_dev *dev, struct ocxl_link *link)
343{
344 struct spa *spa;
345
346 spa = kzalloc(sizeof(struct spa), GFP_KERNEL);
347 if (!spa)
348 return -ENOMEM;
349
350 mutex_init(&spa->spa_lock);
351 INIT_RADIX_TREE(&spa->pe_tree, GFP_KERNEL);
352 INIT_WORK(&spa->xsl_fault.fault_work, xsl_fault_handler_bh);
353
354 spa->spa_order = SPA_SPA_SIZE_LOG - PAGE_SHIFT;
355 spa->spa_mem = (struct ocxl_process_element *)
356 __get_free_pages(GFP_KERNEL | __GFP_ZERO, spa->spa_order);
357 if (!spa->spa_mem) {
358 dev_err(&dev->dev, "Can't allocate Shared Process Area\n");
359 kfree(spa);
360 return -ENOMEM;
361 }
362 pr_debug("Allocated SPA for %x:%x:%x at %p\n", link->domain, link->bus,
363 link->dev, spa->spa_mem);
364
365 link->spa = spa;
366 return 0;
367}
368
369static void free_spa(struct ocxl_link *link)
370{
371 struct spa *spa = link->spa;
372
373 pr_debug("Freeing SPA for %x:%x:%x\n", link->domain, link->bus,
374 link->dev);
375
376 if (spa && spa->spa_mem) {
377 free_pages((unsigned long) spa->spa_mem, spa->spa_order);
378 kfree(spa);
379 link->spa = NULL;
380 }
381}
382
383static int alloc_link(struct pci_dev *dev, int PE_mask, struct ocxl_link **out_link)
384{
385 struct ocxl_link *link;
386 int rc;
387
388 link = kzalloc(sizeof(struct ocxl_link), GFP_KERNEL);
389 if (!link)
390 return -ENOMEM;
391
392 kref_init(&link->ref);
393 link->domain = pci_domain_nr(dev->bus);
394 link->bus = dev->bus->number;
395 link->dev = PCI_SLOT(dev->devfn);
396 atomic_set(&link->irq_available, MAX_IRQ_PER_LINK);
397 spin_lock_init(&link->atsd_lock);
398
399 rc = alloc_spa(dev, link);
400 if (rc)
401 goto err_free;
402
403 rc = setup_xsl_irq(dev, link);
404 if (rc)
405 goto err_spa;
406
407
408 rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
409 &link->platform_data);
410 if (rc)
411 goto err_xsl_irq;
412
413
414
415
416
417
418 pnv_ocxl_map_lpar(dev, mfspr(SPRN_LPID), 0, &link->arva);
419
420 *out_link = link;
421 return 0;
422
423err_xsl_irq:
424 release_xsl_irq(link);
425err_spa:
426 free_spa(link);
427err_free:
428 kfree(link);
429 return rc;
430}
431
432static void free_link(struct ocxl_link *link)
433{
434 release_xsl_irq(link);
435 free_spa(link);
436 kfree(link);
437}
438
439int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle)
440{
441 int rc = 0;
442 struct ocxl_link *link;
443
444 mutex_lock(&links_list_lock);
445 list_for_each_entry(link, &links_list, list) {
446
447 if (link->domain == pci_domain_nr(dev->bus) &&
448 link->bus == dev->bus->number &&
449 link->dev == PCI_SLOT(dev->devfn)) {
450 kref_get(&link->ref);
451 *link_handle = link;
452 goto unlock;
453 }
454 }
455 rc = alloc_link(dev, PE_mask, &link);
456 if (rc)
457 goto unlock;
458
459 list_add(&link->list, &links_list);
460 *link_handle = link;
461unlock:
462 mutex_unlock(&links_list_lock);
463 return rc;
464}
465EXPORT_SYMBOL_GPL(ocxl_link_setup);
466
467static void release_xsl(struct kref *ref)
468{
469 struct ocxl_link *link = container_of(ref, struct ocxl_link, ref);
470
471 if (link->arva) {
472 pnv_ocxl_unmap_lpar(link->arva);
473 link->arva = NULL;
474 }
475
476 list_del(&link->list);
477
478 pnv_ocxl_spa_release(link->platform_data);
479 free_link(link);
480}
481
482void ocxl_link_release(struct pci_dev *dev, void *link_handle)
483{
484 struct ocxl_link *link = (struct ocxl_link *) link_handle;
485
486 mutex_lock(&links_list_lock);
487 kref_put(&link->ref, release_xsl);
488 mutex_unlock(&links_list_lock);
489}
490EXPORT_SYMBOL_GPL(ocxl_link_release);
491
492static void invalidate_range(struct mmu_notifier *mn,
493 struct mm_struct *mm,
494 unsigned long start, unsigned long end)
495{
496 struct pe_data *pe_data = container_of(mn, struct pe_data, mmu_notifier);
497 struct ocxl_link *link = pe_data->link;
498 unsigned long addr, pid, page_size = PAGE_SIZE;
499
500 pid = mm->context.id;
501 trace_ocxl_mmu_notifier_range(start, end, pid);
502
503 spin_lock(&link->atsd_lock);
504 for (addr = start; addr < end; addr += page_size)
505 pnv_ocxl_tlb_invalidate(link->arva, pid, addr, page_size);
506 spin_unlock(&link->atsd_lock);
507}
508
509static const struct mmu_notifier_ops ocxl_mmu_notifier_ops = {
510 .invalidate_range = invalidate_range,
511};
512
513static u64 calculate_cfg_state(bool kernel)
514{
515 u64 state;
516
517 state = SPA_CFG_DR;
518 if (mfspr(SPRN_LPCR) & LPCR_TC)
519 state |= SPA_CFG_TC;
520 if (radix_enabled())
521 state |= SPA_CFG_XLAT_ror;
522 else
523 state |= SPA_CFG_XLAT_hpt;
524 state |= SPA_CFG_HV;
525 if (kernel) {
526 if (mfmsr() & MSR_SF)
527 state |= SPA_CFG_SF;
528 } else {
529 state |= SPA_CFG_PR;
530 if (!test_tsk_thread_flag(current, TIF_32BIT))
531 state |= SPA_CFG_SF;
532 }
533 return state;
534}
535
536int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr,
537 u64 amr, u16 bdf, struct mm_struct *mm,
538 void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr),
539 void *xsl_err_data)
540{
541 struct ocxl_link *link = (struct ocxl_link *) link_handle;
542 struct spa *spa = link->spa;
543 struct ocxl_process_element *pe;
544 int pe_handle, rc = 0;
545 struct pe_data *pe_data;
546
547 BUILD_BUG_ON(sizeof(struct ocxl_process_element) != 128);
548 if (pasid > SPA_PASID_MAX)
549 return -EINVAL;
550
551 mutex_lock(&spa->spa_lock);
552 pe_handle = pasid & SPA_PE_MASK;
553 pe = spa->spa_mem + pe_handle;
554
555 if (pe->software_state) {
556 rc = -EBUSY;
557 goto unlock;
558 }
559
560 pe_data = kmalloc(sizeof(*pe_data), GFP_KERNEL);
561 if (!pe_data) {
562 rc = -ENOMEM;
563 goto unlock;
564 }
565
566 pe_data->mm = mm;
567 pe_data->xsl_err_cb = xsl_err_cb;
568 pe_data->xsl_err_data = xsl_err_data;
569 pe_data->link = link;
570 pe_data->mmu_notifier.ops = &ocxl_mmu_notifier_ops;
571
572 memset(pe, 0, sizeof(struct ocxl_process_element));
573 pe->config_state = cpu_to_be64(calculate_cfg_state(pidr == 0));
574 pe->pasid = cpu_to_be32(pasid << (31 - 19));
575 pe->bdf = cpu_to_be16(bdf);
576 pe->lpid = cpu_to_be32(mfspr(SPRN_LPID));
577 pe->pid = cpu_to_be32(pidr);
578 pe->tid = cpu_to_be32(tidr);
579 pe->amr = cpu_to_be64(amr);
580 pe->software_state = cpu_to_be32(SPA_PE_VALID);
581
582
583
584
585
586
587 if (mm) {
588 mm_context_add_copro(mm);
589 if (link->arva) {
590
591
592
593 trace_ocxl_init_mmu_notifier(pasid, mm->context.id);
594 mmu_notifier_register(&pe_data->mmu_notifier, mm);
595 }
596 }
597
598
599
600
601
602
603 mb();
604 radix_tree_insert(&spa->pe_tree, pe_handle, pe_data);
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620 if (mm)
621 mmgrab(mm);
622 trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr);
623unlock:
624 mutex_unlock(&spa->spa_lock);
625 return rc;
626}
627EXPORT_SYMBOL_GPL(ocxl_link_add_pe);
628
629int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid)
630{
631 struct ocxl_link *link = (struct ocxl_link *) link_handle;
632 struct spa *spa = link->spa;
633 struct ocxl_process_element *pe;
634 int pe_handle, rc;
635
636 if (pasid > SPA_PASID_MAX)
637 return -EINVAL;
638
639 pe_handle = pasid & SPA_PE_MASK;
640 pe = spa->spa_mem + pe_handle;
641
642 mutex_lock(&spa->spa_lock);
643
644 pe->tid = cpu_to_be32(tid);
645
646
647
648
649
650
651 mb();
652
653
654
655
656
657
658 rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
659 WARN_ON(rc);
660
661 mutex_unlock(&spa->spa_lock);
662 return rc;
663}
664
665int ocxl_link_remove_pe(void *link_handle, int pasid)
666{
667 struct ocxl_link *link = (struct ocxl_link *) link_handle;
668 struct spa *spa = link->spa;
669 struct ocxl_process_element *pe;
670 struct pe_data *pe_data;
671 int pe_handle, rc;
672
673 if (pasid > SPA_PASID_MAX)
674 return -EINVAL;
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694 pe_handle = pasid & SPA_PE_MASK;
695 pe = spa->spa_mem + pe_handle;
696
697 mutex_lock(&spa->spa_lock);
698
699 if (!(be32_to_cpu(pe->software_state) & SPA_PE_VALID)) {
700 rc = -EINVAL;
701 goto unlock;
702 }
703
704 trace_ocxl_context_remove(current->pid, spa->spa_mem, pasid,
705 be32_to_cpu(pe->pid), be32_to_cpu(pe->tid));
706
707 memset(pe, 0, sizeof(struct ocxl_process_element));
708
709
710
711
712
713 mb();
714
715
716
717
718
719
720 rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
721 WARN_ON(rc);
722
723 pe_data = radix_tree_delete(&spa->pe_tree, pe_handle);
724 if (!pe_data) {
725 WARN(1, "Couldn't find pe data when removing PE\n");
726 } else {
727 if (pe_data->mm) {
728 if (link->arva) {
729 trace_ocxl_release_mmu_notifier(pasid,
730 pe_data->mm->context.id);
731 mmu_notifier_unregister(&pe_data->mmu_notifier,
732 pe_data->mm);
733 spin_lock(&link->atsd_lock);
734 pnv_ocxl_tlb_invalidate(link->arva,
735 pe_data->mm->context.id,
736 0ull,
737 PAGE_SIZE);
738 spin_unlock(&link->atsd_lock);
739 }
740 mm_context_remove_copro(pe_data->mm);
741 mmdrop(pe_data->mm);
742 }
743 kfree_rcu(pe_data, rcu);
744 }
745unlock:
746 mutex_unlock(&spa->spa_lock);
747 return rc;
748}
749EXPORT_SYMBOL_GPL(ocxl_link_remove_pe);
750
751int ocxl_link_irq_alloc(void *link_handle, int *hw_irq, u64 *trigger_addr)
752{
753 struct ocxl_link *link = (struct ocxl_link *) link_handle;
754 int rc, irq;
755 u64 addr;
756
757 if (atomic_dec_if_positive(&link->irq_available) < 0)
758 return -ENOSPC;
759
760 rc = pnv_ocxl_alloc_xive_irq(&irq, &addr);
761 if (rc) {
762 atomic_inc(&link->irq_available);
763 return rc;
764 }
765
766 *hw_irq = irq;
767 *trigger_addr = addr;
768 return 0;
769}
770EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc);
771
772void ocxl_link_free_irq(void *link_handle, int hw_irq)
773{
774 struct ocxl_link *link = (struct ocxl_link *) link_handle;
775
776 pnv_ocxl_free_xive_irq(hw_irq);
777 atomic_inc(&link->irq_available);
778}
779EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
780