1
2
3
4
5
6#define pr_fmt(fmt) "vas: " fmt
7
8#include <linux/types.h>
9#include <linux/mutex.h>
10#include <linux/slab.h>
11#include <linux/io.h>
12#include <linux/log2.h>
13#include <linux/rcupdate.h>
14#include <linux/cred.h>
15#include <linux/sched/mm.h>
16#include <linux/mmu_context.h>
17#include <asm/switch_to.h>
18#include <asm/ppc-opcode.h>
19#include <asm/vas.h>
20#include "vas.h"
21#include "copy-paste.h"
22
23#define CREATE_TRACE_POINTS
24#include "vas-trace.h"
25
26
27
28
29
30void vas_win_paste_addr(struct pnv_vas_window *window, u64 *addr, int *len)
31{
32 int winid;
33 u64 base, shift;
34
35 base = window->vinst->paste_base_addr;
36 shift = window->vinst->paste_win_id_shift;
37 winid = window->vas_win.winid;
38
39 *addr = base + (winid << shift);
40 if (len)
41 *len = PAGE_SIZE;
42
43 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
44}
45
46static inline void get_hvwc_mmio_bar(struct pnv_vas_window *window,
47 u64 *start, int *len)
48{
49 u64 pbaddr;
50
51 pbaddr = window->vinst->hvwc_bar_start;
52 *start = pbaddr + window->vas_win.winid * VAS_HVWC_SIZE;
53 *len = VAS_HVWC_SIZE;
54}
55
56static inline void get_uwc_mmio_bar(struct pnv_vas_window *window,
57 u64 *start, int *len)
58{
59 u64 pbaddr;
60
61 pbaddr = window->vinst->uwc_bar_start;
62 *start = pbaddr + window->vas_win.winid * VAS_UWC_SIZE;
63 *len = VAS_UWC_SIZE;
64}
65
66
67
68
69
70
71static void *map_paste_region(struct pnv_vas_window *txwin)
72{
73 int len;
74 void *map;
75 char *name;
76 u64 start;
77
78 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
79 txwin->vas_win.winid);
80 if (!name)
81 goto free_name;
82
83 txwin->paste_addr_name = name;
84 vas_win_paste_addr(txwin, &start, &len);
85
86 if (!request_mem_region(start, len, name)) {
87 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
88 __func__, start, len);
89 goto free_name;
90 }
91
92 map = ioremap_cache(start, len);
93 if (!map) {
94 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
95 start, len);
96 goto free_name;
97 }
98
99 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
100 return map;
101
102free_name:
103 kfree(name);
104 return ERR_PTR(-ENOMEM);
105}
106
107static void *map_mmio_region(char *name, u64 start, int len)
108{
109 void *map;
110
111 if (!request_mem_region(start, len, name)) {
112 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
113 __func__, start, len);
114 return NULL;
115 }
116
117 map = ioremap(start, len);
118 if (!map) {
119 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
120 len);
121 return NULL;
122 }
123
124 return map;
125}
126
127static void unmap_region(void *addr, u64 start, int len)
128{
129 iounmap(addr);
130 release_mem_region((phys_addr_t)start, len);
131}
132
133
134
135
136static void unmap_paste_region(struct pnv_vas_window *window)
137{
138 int len;
139 u64 busaddr_start;
140
141 if (window->paste_kaddr) {
142 vas_win_paste_addr(window, &busaddr_start, &len);
143 unmap_region(window->paste_kaddr, busaddr_start, len);
144 window->paste_kaddr = NULL;
145 kfree(window->paste_addr_name);
146 window->paste_addr_name = NULL;
147 }
148}
149
150
151
152
153
154
155
156
157static void unmap_winctx_mmio_bars(struct pnv_vas_window *window)
158{
159 int len;
160 void *uwc_map;
161 void *hvwc_map;
162 u64 busaddr_start;
163
164 mutex_lock(&vas_mutex);
165
166 hvwc_map = window->hvwc_map;
167 window->hvwc_map = NULL;
168
169 uwc_map = window->uwc_map;
170 window->uwc_map = NULL;
171
172 mutex_unlock(&vas_mutex);
173
174 if (hvwc_map) {
175 get_hvwc_mmio_bar(window, &busaddr_start, &len);
176 unmap_region(hvwc_map, busaddr_start, len);
177 }
178
179 if (uwc_map) {
180 get_uwc_mmio_bar(window, &busaddr_start, &len);
181 unmap_region(uwc_map, busaddr_start, len);
182 }
183}
184
185
186
187
188
189
190static int map_winctx_mmio_bars(struct pnv_vas_window *window)
191{
192 int len;
193 u64 start;
194
195 get_hvwc_mmio_bar(window, &start, &len);
196 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
197
198 get_uwc_mmio_bar(window, &start, &len);
199 window->uwc_map = map_mmio_region("UWCM_Window", start, len);
200
201 if (!window->hvwc_map || !window->uwc_map) {
202 unmap_winctx_mmio_bars(window);
203 return -1;
204 }
205
206 return 0;
207}
208
209
210
211
212
213
214
215
216
217
218static void reset_window_regs(struct pnv_vas_window *window)
219{
220 write_hvwc_reg(window, VREG(LPID), 0ULL);
221 write_hvwc_reg(window, VREG(PID), 0ULL);
222 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
223 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
224 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
225 write_hvwc_reg(window, VREG(AMR), 0ULL);
226 write_hvwc_reg(window, VREG(SEIDR), 0ULL);
227 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
228 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
229 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
230 write_hvwc_reg(window, VREG(PSWID), 0ULL);
231 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
232 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
233 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
234 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
235 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
236 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
237 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
238 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
239 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
240 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
241 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
242 write_hvwc_reg(window, VREG(WINCTL), 0ULL);
243 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
244 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
245 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
246 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
247 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
248 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
249 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
250 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
251 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
252 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
253
254
255
256
257
258
259
260
261
262
263
264
265}
266
267
268
269
270
271
272
273
274static void init_xlate_regs(struct pnv_vas_window *window, bool user_win)
275{
276 u64 lpcr, val;
277
278
279
280
281
282 val = 0ULL;
283 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
284 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
285 if (user_win) {
286 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
287 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
288 }
289 write_hvwc_reg(window, VREG(XLATE_MSR), val);
290
291 lpcr = mfspr(SPRN_LPCR);
292 val = 0ULL;
293
294
295
296
297
298
299
300 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
301 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
302 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
303 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
304 write_hvwc_reg(window, VREG(XLATE_LPCR), val);
305
306
307
308
309
310
311
312
313 val = 0ULL;
314 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
315 write_hvwc_reg(window, VREG(XLATE_CTL), val);
316
317
318
319
320 val = 0ULL;
321 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
322 write_hvwc_reg(window, VREG(AMR), val);
323
324 val = 0ULL;
325 val = SET_FIELD(VAS_SEIDR, val, 0);
326 write_hvwc_reg(window, VREG(SEIDR), val);
327}
328
329
330
331
332
333
334
335
336
337
338
339static void init_rsvd_tx_buf_count(struct pnv_vas_window *txwin,
340 struct vas_winctx *winctx)
341{
342 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
343}
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361static void init_winctx_regs(struct pnv_vas_window *window,
362 struct vas_winctx *winctx)
363{
364 u64 val;
365 int fifo_size;
366
367 reset_window_regs(window);
368
369 val = 0ULL;
370 val = SET_FIELD(VAS_LPID, val, winctx->lpid);
371 write_hvwc_reg(window, VREG(LPID), val);
372
373 val = 0ULL;
374 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
375 write_hvwc_reg(window, VREG(PID), val);
376
377 init_xlate_regs(window, winctx->user_win);
378
379 val = 0ULL;
380 val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id);
381 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
382
383
384 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
385
386 val = 0ULL;
387 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
388 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
389
390 val = 0ULL;
391 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
392 write_hvwc_reg(window, VREG(PSWID), val);
393
394 write_hvwc_reg(window, VREG(SPARE1), 0ULL);
395 write_hvwc_reg(window, VREG(SPARE2), 0ULL);
396 write_hvwc_reg(window, VREG(SPARE3), 0ULL);
397
398
399
400
401
402
403
404
405
406
407 val = __pa(winctx->rx_fifo);
408 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
409 write_hvwc_reg(window, VREG(LFIFO_BAR), val);
410
411 val = 0ULL;
412 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
413 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
414
415 val = 0ULL;
416 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
417 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
418 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
419
420 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
421 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
422 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
423
424 val = 0ULL;
425 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
426 write_hvwc_reg(window, VREG(LRX_WCRED), val);
427
428 val = 0ULL;
429 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
430 write_hvwc_reg(window, VREG(TX_WCRED), val);
431
432 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
433 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
434
435 fifo_size = winctx->rx_fifo_size / 1024;
436
437 val = 0ULL;
438 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
439 write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
440
441
442
443
444
445
446 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
447
448 init_rsvd_tx_buf_count(window, winctx);
449
450
451 val = 0ULL;
452 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
453 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
454
455 write_hvwc_reg(window, VREG(SPARE4), 0ULL);
456
457 val = 0ULL;
458 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
459 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
460 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
461 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
462 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
463
464 val = 0ULL;
465 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
466 write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
467
468 val = 0ULL;
469 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
470 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
471
472 val = 0ULL;
473 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
474 write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
475
476 val = 0ULL;
477 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
478 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
479 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
480
481
482
483 write_hvwc_reg(window, VREG(SPARE5), 0ULL);
484 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
485 write_hvwc_reg(window, VREG(SPARE6), 0ULL);
486
487
488 val = 0ULL;
489 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
490 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
491
492
493 val = 0ULL;
494 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
495 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
496 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
497 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
498 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
499 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
500 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
501 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
502 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
503 write_hvwc_reg(window, VREG(WINCTL), val);
504}
505
506static void vas_release_window_id(struct ida *ida, int winid)
507{
508 ida_free(ida, winid);
509}
510
511static int vas_assign_window_id(struct ida *ida)
512{
513 int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
514
515 if (winid == -ENOSPC) {
516 pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
517 return -EAGAIN;
518 }
519
520 return winid;
521}
522
523static void vas_window_free(struct pnv_vas_window *window)
524{
525 struct vas_instance *vinst = window->vinst;
526 int winid = window->vas_win.winid;
527
528 unmap_winctx_mmio_bars(window);
529
530 vas_window_free_dbgdir(window);
531
532 kfree(window);
533
534 vas_release_window_id(&vinst->ida, winid);
535}
536
537static struct pnv_vas_window *vas_window_alloc(struct vas_instance *vinst)
538{
539 int winid;
540 struct pnv_vas_window *window;
541
542 winid = vas_assign_window_id(&vinst->ida);
543 if (winid < 0)
544 return ERR_PTR(winid);
545
546 window = kzalloc(sizeof(*window), GFP_KERNEL);
547 if (!window)
548 goto out_free;
549
550 window->vinst = vinst;
551 window->vas_win.winid = winid;
552
553 if (map_winctx_mmio_bars(window))
554 goto out_free;
555
556 vas_window_init_dbgdir(window);
557
558 return window;
559
560out_free:
561 kfree(window);
562 vas_release_window_id(&vinst->ida, winid);
563 return ERR_PTR(-ENOMEM);
564}
565
566static void put_rx_win(struct pnv_vas_window *rxwin)
567{
568
569 WARN_ON_ONCE(rxwin->tx_win);
570
571 atomic_dec(&rxwin->num_txwins);
572}
573
574
575
576
577
578
579
580
581
582static struct pnv_vas_window *get_user_rxwin(struct vas_instance *vinst,
583 u32 pswid)
584{
585 int vasid, winid;
586 struct pnv_vas_window *rxwin;
587
588 decode_pswid(pswid, &vasid, &winid);
589
590 if (vinst->vas_id != vasid)
591 return ERR_PTR(-EINVAL);
592
593 rxwin = vinst->windows[winid];
594
595 if (!rxwin || rxwin->tx_win || rxwin->vas_win.cop != VAS_COP_TYPE_FTW)
596 return ERR_PTR(-EINVAL);
597
598 return rxwin;
599}
600
601
602
603
604
605
606
607static struct pnv_vas_window *get_vinst_rxwin(struct vas_instance *vinst,
608 enum vas_cop_type cop, u32 pswid)
609{
610 struct pnv_vas_window *rxwin;
611
612 mutex_lock(&vinst->mutex);
613
614 if (cop == VAS_COP_TYPE_FTW)
615 rxwin = get_user_rxwin(vinst, pswid);
616 else
617 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
618
619 if (!IS_ERR(rxwin))
620 atomic_inc(&rxwin->num_txwins);
621
622 mutex_unlock(&vinst->mutex);
623
624 return rxwin;
625}
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642static void set_vinst_win(struct vas_instance *vinst,
643 struct pnv_vas_window *window)
644{
645 int id = window->vas_win.winid;
646
647 mutex_lock(&vinst->mutex);
648
649
650
651
652
653 if (!window->user_win && !window->tx_win) {
654 WARN_ON_ONCE(vinst->rxwin[window->vas_win.cop]);
655 vinst->rxwin[window->vas_win.cop] = window;
656 }
657
658 WARN_ON_ONCE(vinst->windows[id] != NULL);
659 vinst->windows[id] = window;
660
661 mutex_unlock(&vinst->mutex);
662}
663
664
665
666
667
668static void clear_vinst_win(struct pnv_vas_window *window)
669{
670 int id = window->vas_win.winid;
671 struct vas_instance *vinst = window->vinst;
672
673 mutex_lock(&vinst->mutex);
674
675 if (!window->user_win && !window->tx_win) {
676 WARN_ON_ONCE(!vinst->rxwin[window->vas_win.cop]);
677 vinst->rxwin[window->vas_win.cop] = NULL;
678 }
679
680 WARN_ON_ONCE(vinst->windows[id] != window);
681 vinst->windows[id] = NULL;
682
683 mutex_unlock(&vinst->mutex);
684}
685
686static void init_winctx_for_rxwin(struct pnv_vas_window *rxwin,
687 struct vas_rx_win_attr *rxattr,
688 struct vas_winctx *winctx)
689{
690
691
692
693
694
695
696
697
698
699
700
701
702
703 memset(winctx, 0, sizeof(struct vas_winctx));
704
705 winctx->rx_fifo = rxattr->rx_fifo;
706 winctx->rx_fifo_size = rxattr->rx_fifo_size;
707 winctx->wcreds_max = rxwin->vas_win.wcreds_max;
708 winctx->pin_win = rxattr->pin_win;
709
710 winctx->nx_win = rxattr->nx_win;
711 winctx->fault_win = rxattr->fault_win;
712 winctx->user_win = rxattr->user_win;
713 winctx->rej_no_credit = rxattr->rej_no_credit;
714 winctx->rx_word_mode = rxattr->rx_win_ord_mode;
715 winctx->tx_word_mode = rxattr->tx_win_ord_mode;
716 winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
717 winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
718 winctx->notify_early = rxattr->notify_early;
719
720 if (winctx->nx_win) {
721 winctx->data_stamp = true;
722 winctx->intr_disable = true;
723 winctx->pin_win = true;
724
725 WARN_ON_ONCE(winctx->fault_win);
726 WARN_ON_ONCE(!winctx->rx_word_mode);
727 WARN_ON_ONCE(!winctx->tx_word_mode);
728 WARN_ON_ONCE(winctx->notify_after_count);
729 } else if (winctx->fault_win) {
730 winctx->notify_disable = true;
731 } else if (winctx->user_win) {
732
733
734
735
736
737
738
739
740 winctx->fifo_disable = true;
741 winctx->intr_disable = true;
742 winctx->rx_fifo = NULL;
743 }
744
745 winctx->lnotify_lpid = rxattr->lnotify_lpid;
746 winctx->lnotify_pid = rxattr->lnotify_pid;
747 winctx->lnotify_tid = rxattr->lnotify_tid;
748 winctx->pswid = rxattr->pswid;
749 winctx->dma_type = VAS_DMA_TYPE_INJECT;
750 winctx->tc_mode = rxattr->tc_mode;
751
752 winctx->min_scope = VAS_SCOPE_LOCAL;
753 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
754 if (rxwin->vinst->virq)
755 winctx->irq_port = rxwin->vinst->irq_port;
756}
757
758static bool rx_win_args_valid(enum vas_cop_type cop,
759 struct vas_rx_win_attr *attr)
760{
761 pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
762 attr->fault_win, attr->notify_disable,
763 attr->intr_disable, attr->notify_early,
764 attr->rx_fifo_size);
765
766 if (cop >= VAS_COP_TYPE_MAX)
767 return false;
768
769 if (cop != VAS_COP_TYPE_FTW &&
770 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
771 return false;
772
773 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
774 return false;
775
776 if (!attr->wcreds_max)
777 return false;
778
779 if (attr->nx_win) {
780
781 if (attr->fault_win || attr->user_win)
782 return false;
783
784
785
786
787 if (attr->notify_disable || !attr->intr_disable ||
788 attr->notify_early)
789 return false;
790 } else if (attr->fault_win) {
791
792 if (attr->user_win)
793 return false;
794
795
796
797
798
799 if (!attr->notify_disable || attr->intr_disable)
800 return false;
801
802 } else if (attr->user_win) {
803
804
805
806
807 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
808 return false;
809 } else {
810
811 return false;
812 }
813
814 return true;
815}
816
817void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
818{
819 memset(rxattr, 0, sizeof(*rxattr));
820
821 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
822 cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
823 rxattr->pin_win = true;
824 rxattr->nx_win = true;
825 rxattr->fault_win = false;
826 rxattr->intr_disable = true;
827 rxattr->rx_wcred_mode = true;
828 rxattr->tx_wcred_mode = true;
829 rxattr->rx_win_ord_mode = true;
830 rxattr->tx_win_ord_mode = true;
831 } else if (cop == VAS_COP_TYPE_FAULT) {
832 rxattr->pin_win = true;
833 rxattr->fault_win = true;
834 rxattr->notify_disable = true;
835 rxattr->rx_wcred_mode = true;
836 rxattr->rx_win_ord_mode = true;
837 rxattr->rej_no_credit = true;
838 rxattr->tc_mode = VAS_THRESH_DISABLED;
839 } else if (cop == VAS_COP_TYPE_FTW) {
840 rxattr->user_win = true;
841 rxattr->intr_disable = true;
842
843
844
845
846
847
848
849 }
850}
851EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
852
853struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
854 struct vas_rx_win_attr *rxattr)
855{
856 struct pnv_vas_window *rxwin;
857 struct vas_winctx winctx;
858 struct vas_instance *vinst;
859
860 trace_vas_rx_win_open(current, vasid, cop, rxattr);
861
862 if (!rx_win_args_valid(cop, rxattr))
863 return ERR_PTR(-EINVAL);
864
865 vinst = find_vas_instance(vasid);
866 if (!vinst) {
867 pr_devel("vasid %d not found!\n", vasid);
868 return ERR_PTR(-EINVAL);
869 }
870 pr_devel("Found instance %d\n", vasid);
871
872 rxwin = vas_window_alloc(vinst);
873 if (IS_ERR(rxwin)) {
874 pr_devel("Unable to allocate memory for Rx window\n");
875 return (struct vas_window *)rxwin;
876 }
877
878 rxwin->tx_win = false;
879 rxwin->nx_win = rxattr->nx_win;
880 rxwin->user_win = rxattr->user_win;
881 rxwin->vas_win.cop = cop;
882 rxwin->vas_win.wcreds_max = rxattr->wcreds_max;
883
884 init_winctx_for_rxwin(rxwin, rxattr, &winctx);
885 init_winctx_regs(rxwin, &winctx);
886
887 set_vinst_win(vinst, rxwin);
888
889 return &rxwin->vas_win;
890}
891EXPORT_SYMBOL_GPL(vas_rx_win_open);
892
893void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
894{
895 memset(txattr, 0, sizeof(*txattr));
896
897 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
898 cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
899 txattr->rej_no_credit = false;
900 txattr->rx_wcred_mode = true;
901 txattr->tx_wcred_mode = true;
902 txattr->rx_win_ord_mode = true;
903 txattr->tx_win_ord_mode = true;
904 } else if (cop == VAS_COP_TYPE_FTW) {
905 txattr->user_win = true;
906 }
907}
908EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
909
910static void init_winctx_for_txwin(struct pnv_vas_window *txwin,
911 struct vas_tx_win_attr *txattr,
912 struct vas_winctx *winctx)
913{
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929 memset(winctx, 0, sizeof(struct vas_winctx));
930
931 winctx->wcreds_max = txwin->vas_win.wcreds_max;
932
933 winctx->user_win = txattr->user_win;
934 winctx->nx_win = txwin->rxwin->nx_win;
935 winctx->pin_win = txattr->pin_win;
936 winctx->rej_no_credit = txattr->rej_no_credit;
937 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
938
939 winctx->rx_wcred_mode = txattr->rx_wcred_mode;
940 winctx->tx_wcred_mode = txattr->tx_wcred_mode;
941 winctx->rx_word_mode = txattr->rx_win_ord_mode;
942 winctx->tx_word_mode = txattr->tx_win_ord_mode;
943 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
944
945 winctx->intr_disable = true;
946 if (winctx->nx_win)
947 winctx->data_stamp = true;
948
949 winctx->lpid = txattr->lpid;
950 winctx->pidr = txattr->pidr;
951 winctx->rx_win_id = txwin->rxwin->vas_win.winid;
952
953
954
955
956 if (txwin->vinst->virq)
957 winctx->fault_win_id = txwin->vinst->fault_win->vas_win.winid;
958
959 winctx->dma_type = VAS_DMA_TYPE_INJECT;
960 winctx->tc_mode = txattr->tc_mode;
961 winctx->min_scope = VAS_SCOPE_LOCAL;
962 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
963 if (txwin->vinst->virq)
964 winctx->irq_port = txwin->vinst->irq_port;
965
966 winctx->pswid = txattr->pswid ? txattr->pswid :
967 encode_pswid(txwin->vinst->vas_id,
968 txwin->vas_win.winid);
969}
970
971static bool tx_win_args_valid(enum vas_cop_type cop,
972 struct vas_tx_win_attr *attr)
973{
974 if (attr->tc_mode != VAS_THRESH_DISABLED)
975 return false;
976
977 if (cop > VAS_COP_TYPE_MAX)
978 return false;
979
980 if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
981 return false;
982
983 if (attr->user_win) {
984 if (attr->rsvd_txbuf_count)
985 return false;
986
987 if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
988 cop != VAS_COP_TYPE_GZIP_HIPRI)
989 return false;
990 }
991
992 return true;
993}
994
995struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
996 struct vas_tx_win_attr *attr)
997{
998 int rc;
999 struct pnv_vas_window *txwin;
1000 struct pnv_vas_window *rxwin;
1001 struct vas_winctx winctx;
1002 struct vas_instance *vinst;
1003
1004 trace_vas_tx_win_open(current, vasid, cop, attr);
1005
1006 if (!tx_win_args_valid(cop, attr))
1007 return ERR_PTR(-EINVAL);
1008
1009
1010
1011
1012
1013
1014 if (vasid == -1 && attr->pswid)
1015 decode_pswid(attr->pswid, &vasid, NULL);
1016
1017 vinst = find_vas_instance(vasid);
1018 if (!vinst) {
1019 pr_devel("vasid %d not found!\n", vasid);
1020 return ERR_PTR(-EINVAL);
1021 }
1022
1023 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
1024 if (IS_ERR(rxwin)) {
1025 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1026 return (struct vas_window *)rxwin;
1027 }
1028
1029 txwin = vas_window_alloc(vinst);
1030 if (IS_ERR(txwin)) {
1031 rc = PTR_ERR(txwin);
1032 goto put_rxwin;
1033 }
1034
1035 txwin->vas_win.cop = cop;
1036 txwin->tx_win = 1;
1037 txwin->rxwin = rxwin;
1038 txwin->nx_win = txwin->rxwin->nx_win;
1039 txwin->user_win = attr->user_win;
1040 txwin->vas_win.wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1041
1042 init_winctx_for_txwin(txwin, attr, &winctx);
1043
1044 init_winctx_regs(txwin, &winctx);
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054 if (!txwin->user_win) {
1055 txwin->paste_kaddr = map_paste_region(txwin);
1056 if (IS_ERR(txwin->paste_kaddr)) {
1057 rc = PTR_ERR(txwin->paste_kaddr);
1058 goto free_window;
1059 }
1060 } else {
1061
1062
1063
1064
1065
1066 if (!vinst->virq) {
1067 rc = -ENODEV;
1068 goto free_window;
1069 }
1070 rc = get_vas_user_win_ref(&txwin->vas_win.task_ref);
1071 if (rc)
1072 goto free_window;
1073
1074 vas_user_win_add_mm_context(&txwin->vas_win.task_ref);
1075 }
1076
1077 set_vinst_win(vinst, txwin);
1078
1079 return &txwin->vas_win;
1080
1081free_window:
1082 vas_window_free(txwin);
1083
1084put_rxwin:
1085 put_rx_win(rxwin);
1086 return ERR_PTR(rc);
1087
1088}
1089EXPORT_SYMBOL_GPL(vas_tx_win_open);
1090
1091int vas_copy_crb(void *crb, int offset)
1092{
1093 return vas_copy(crb, offset);
1094}
1095EXPORT_SYMBOL_GPL(vas_copy_crb);
1096
1097#define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1098int vas_paste_crb(struct vas_window *vwin, int offset, bool re)
1099{
1100 struct pnv_vas_window *txwin;
1101 int rc;
1102 void *addr;
1103 uint64_t val;
1104
1105 txwin = container_of(vwin, struct pnv_vas_window, vas_win);
1106 trace_vas_paste_crb(current, txwin);
1107
1108
1109
1110
1111
1112
1113 WARN_ON_ONCE(txwin->nx_win && !re);
1114
1115 addr = txwin->paste_kaddr;
1116 if (re) {
1117
1118
1119
1120
1121 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1122 addr += val;
1123 }
1124
1125
1126
1127
1128
1129 rc = vas_paste(addr, offset);
1130 if (rc == 2)
1131 rc = 0;
1132 else
1133 rc = -EINVAL;
1134
1135 pr_debug("Txwin #%d: Msg count %llu\n", txwin->vas_win.winid,
1136 read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1137
1138 return rc;
1139}
1140EXPORT_SYMBOL_GPL(vas_paste_crb);
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155static void poll_window_credits(struct pnv_vas_window *window)
1156{
1157 u64 val;
1158 int creds, mode;
1159 int count = 0;
1160
1161 val = read_hvwc_reg(window, VREG(WINCTL));
1162 if (window->tx_win)
1163 mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1164 else
1165 mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1166
1167 if (!mode)
1168 return;
1169retry:
1170 if (window->tx_win) {
1171 val = read_hvwc_reg(window, VREG(TX_WCRED));
1172 creds = GET_FIELD(VAS_TX_WCRED, val);
1173 } else {
1174 val = read_hvwc_reg(window, VREG(LRX_WCRED));
1175 creds = GET_FIELD(VAS_LRX_WCRED, val);
1176 }
1177
1178
1179
1180
1181
1182
1183
1184
1185 if (creds < window->vas_win.wcreds_max) {
1186 val = 0;
1187 set_current_state(TASK_UNINTERRUPTIBLE);
1188 schedule_timeout(msecs_to_jiffies(10));
1189 count++;
1190
1191
1192
1193
1194 if (!(count % 1000))
1195 pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Retries %d\n",
1196 vas_window_pid(&window->vas_win),
1197 window->vas_win.winid,
1198 creds, count);
1199
1200 goto retry;
1201 }
1202}
1203
1204
1205
1206
1207
1208
1209static void poll_window_busy_state(struct pnv_vas_window *window)
1210{
1211 int busy;
1212 u64 val;
1213 int count = 0;
1214
1215retry:
1216 val = read_hvwc_reg(window, VREG(WIN_STATUS));
1217 busy = GET_FIELD(VAS_WIN_BUSY, val);
1218 if (busy) {
1219 val = 0;
1220 set_current_state(TASK_UNINTERRUPTIBLE);
1221 schedule_timeout(msecs_to_jiffies(10));
1222 count++;
1223
1224
1225
1226
1227 if (!(count % 1000))
1228 pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n",
1229 vas_window_pid(&window->vas_win),
1230 window->vas_win.winid, count);
1231
1232 goto retry;
1233 }
1234}
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252static void poll_window_castout(struct pnv_vas_window *window)
1253{
1254
1255}
1256
1257
1258
1259
1260
1261static void unpin_close_window(struct pnv_vas_window *window)
1262{
1263 u64 val;
1264
1265 val = read_hvwc_reg(window, VREG(WINCTL));
1266 val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1267 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1268 write_hvwc_reg(window, VREG(WINCTL), val);
1269}
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283int vas_win_close(struct vas_window *vwin)
1284{
1285 struct pnv_vas_window *window;
1286
1287 if (!vwin)
1288 return 0;
1289
1290 window = container_of(vwin, struct pnv_vas_window, vas_win);
1291
1292 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1293 pr_devel("Attempting to close an active Rx window!\n");
1294 WARN_ON_ONCE(1);
1295 return -EBUSY;
1296 }
1297
1298 unmap_paste_region(window);
1299
1300 poll_window_busy_state(window);
1301
1302 unpin_close_window(window);
1303
1304 poll_window_credits(window);
1305
1306 clear_vinst_win(window);
1307
1308 poll_window_castout(window);
1309
1310
1311 if (window->tx_win) {
1312 if (window->user_win) {
1313 put_vas_user_win_ref(&vwin->task_ref);
1314 mm_context_remove_vas_window(vwin->task_ref.mm);
1315 }
1316 put_rx_win(window->rxwin);
1317 }
1318
1319 vas_window_free(window);
1320
1321 return 0;
1322}
1323EXPORT_SYMBOL_GPL(vas_win_close);
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347void vas_return_credit(struct pnv_vas_window *window, bool tx)
1348{
1349 uint64_t val;
1350
1351 val = 0ULL;
1352 if (tx) {
1353 val = SET_FIELD(VAS_TX_WCRED, val, 1);
1354 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), val);
1355 } else {
1356 val = SET_FIELD(VAS_LRX_WCRED, val, 1);
1357 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), val);
1358 }
1359}
1360
1361struct pnv_vas_window *vas_pswid_to_window(struct vas_instance *vinst,
1362 uint32_t pswid)
1363{
1364 struct pnv_vas_window *window;
1365 int winid;
1366
1367 if (!pswid) {
1368 pr_devel("%s: called for pswid 0!\n", __func__);
1369 return ERR_PTR(-ESRCH);
1370 }
1371
1372 decode_pswid(pswid, NULL, &winid);
1373
1374 if (winid >= VAS_WINDOWS_PER_CHIP)
1375 return ERR_PTR(-ESRCH);
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386 window = vinst->windows[winid];
1387
1388 if (!window) {
1389 pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n",
1390 winid, pswid, vinst);
1391 return NULL;
1392 }
1393
1394
1395
1396
1397
1398
1399
1400 if (!window->tx_win || !window->user_win || !window->nx_win ||
1401 window->vas_win.cop == VAS_COP_TYPE_FAULT ||
1402 window->vas_win.cop == VAS_COP_TYPE_FTW) {
1403 pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n",
1404 winid, window->tx_win, window->user_win,
1405 window->nx_win, window->vas_win.cop);
1406 WARN_ON(1);
1407 }
1408
1409 return window;
1410}
1411
1412static struct vas_window *vas_user_win_open(int vas_id, u64 flags,
1413 enum vas_cop_type cop_type)
1414{
1415 struct vas_tx_win_attr txattr = {};
1416
1417 vas_init_tx_win_attr(&txattr, cop_type);
1418
1419 txattr.lpid = mfspr(SPRN_LPID);
1420 txattr.pidr = mfspr(SPRN_PID);
1421 txattr.user_win = true;
1422 txattr.rsvd_txbuf_count = false;
1423 txattr.pswid = false;
1424
1425 pr_devel("Pid %d: Opening txwin, PIDR %ld\n", txattr.pidr,
1426 mfspr(SPRN_PID));
1427
1428 return vas_tx_win_open(vas_id, cop_type, &txattr);
1429}
1430
1431static u64 vas_user_win_paste_addr(struct vas_window *txwin)
1432{
1433 struct pnv_vas_window *win;
1434 u64 paste_addr;
1435
1436 win = container_of(txwin, struct pnv_vas_window, vas_win);
1437 vas_win_paste_addr(win, &paste_addr, NULL);
1438
1439 return paste_addr;
1440}
1441
1442static int vas_user_win_close(struct vas_window *txwin)
1443{
1444 vas_win_close(txwin);
1445
1446 return 0;
1447}
1448
1449static const struct vas_user_win_ops vops = {
1450 .open_win = vas_user_win_open,
1451 .paste_addr = vas_user_win_paste_addr,
1452 .close_win = vas_user_win_close,
1453};
1454
1455
1456
1457
1458
1459int vas_register_api_powernv(struct module *mod, enum vas_cop_type cop_type,
1460 const char *name)
1461{
1462
1463 return vas_register_coproc_api(mod, cop_type, name, &vops);
1464}
1465EXPORT_SYMBOL_GPL(vas_register_api_powernv);
1466
1467void vas_unregister_api_powernv(void)
1468{
1469 vas_unregister_coproc_api();
1470}
1471EXPORT_SYMBOL_GPL(vas_unregister_api_powernv);
1472