1
2
3
4
5
6#define pr_fmt(fmt) "vas: " fmt
7
8#include <linux/types.h>
9#include <linux/mutex.h>
10#include <linux/slab.h>
11#include <linux/io.h>
12#include <linux/log2.h>
13#include <linux/rcupdate.h>
14#include <linux/cred.h>
15#include <linux/sched/mm.h>
16#include <linux/mmu_context.h>
17#include <asm/switch_to.h>
18#include <asm/ppc-opcode.h>
19#include "vas.h"
20#include "copy-paste.h"
21
22#define CREATE_TRACE_POINTS
23#include "vas-trace.h"
24
25
26
27
28
29void vas_win_paste_addr(struct vas_window *window, u64 *addr, int *len)
30{
31 int winid;
32 u64 base, shift;
33
34 base = window->vinst->paste_base_addr;
35 shift = window->vinst->paste_win_id_shift;
36 winid = window->winid;
37
38 *addr = base + (winid << shift);
39 if (len)
40 *len = PAGE_SIZE;
41
42 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
43}
44
45static inline void get_hvwc_mmio_bar(struct vas_window *window,
46 u64 *start, int *len)
47{
48 u64 pbaddr;
49
50 pbaddr = window->vinst->hvwc_bar_start;
51 *start = pbaddr + window->winid * VAS_HVWC_SIZE;
52 *len = VAS_HVWC_SIZE;
53}
54
55static inline void get_uwc_mmio_bar(struct vas_window *window,
56 u64 *start, int *len)
57{
58 u64 pbaddr;
59
60 pbaddr = window->vinst->uwc_bar_start;
61 *start = pbaddr + window->winid * VAS_UWC_SIZE;
62 *len = VAS_UWC_SIZE;
63}
64
65
66
67
68
69
70static void *map_paste_region(struct vas_window *txwin)
71{
72 int len;
73 void *map;
74 char *name;
75 u64 start;
76
77 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
78 txwin->winid);
79 if (!name)
80 goto free_name;
81
82 txwin->paste_addr_name = name;
83 vas_win_paste_addr(txwin, &start, &len);
84
85 if (!request_mem_region(start, len, name)) {
86 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
87 __func__, start, len);
88 goto free_name;
89 }
90
91 map = ioremap_cache(start, len);
92 if (!map) {
93 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
94 start, len);
95 goto free_name;
96 }
97
98 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
99 return map;
100
101free_name:
102 kfree(name);
103 return ERR_PTR(-ENOMEM);
104}
105
106static void *map_mmio_region(char *name, u64 start, int len)
107{
108 void *map;
109
110 if (!request_mem_region(start, len, name)) {
111 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
112 __func__, start, len);
113 return NULL;
114 }
115
116 map = ioremap(start, len);
117 if (!map) {
118 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
119 len);
120 return NULL;
121 }
122
123 return map;
124}
125
126static void unmap_region(void *addr, u64 start, int len)
127{
128 iounmap(addr);
129 release_mem_region((phys_addr_t)start, len);
130}
131
132
133
134
135static void unmap_paste_region(struct vas_window *window)
136{
137 int len;
138 u64 busaddr_start;
139
140 if (window->paste_kaddr) {
141 vas_win_paste_addr(window, &busaddr_start, &len);
142 unmap_region(window->paste_kaddr, busaddr_start, len);
143 window->paste_kaddr = NULL;
144 kfree(window->paste_addr_name);
145 window->paste_addr_name = NULL;
146 }
147}
148
149
150
151
152
153
154
155
156static void unmap_winctx_mmio_bars(struct vas_window *window)
157{
158 int len;
159 void *uwc_map;
160 void *hvwc_map;
161 u64 busaddr_start;
162
163 mutex_lock(&vas_mutex);
164
165 hvwc_map = window->hvwc_map;
166 window->hvwc_map = NULL;
167
168 uwc_map = window->uwc_map;
169 window->uwc_map = NULL;
170
171 mutex_unlock(&vas_mutex);
172
173 if (hvwc_map) {
174 get_hvwc_mmio_bar(window, &busaddr_start, &len);
175 unmap_region(hvwc_map, busaddr_start, len);
176 }
177
178 if (uwc_map) {
179 get_uwc_mmio_bar(window, &busaddr_start, &len);
180 unmap_region(uwc_map, busaddr_start, len);
181 }
182}
183
184
185
186
187
188
189static int map_winctx_mmio_bars(struct vas_window *window)
190{
191 int len;
192 u64 start;
193
194 get_hvwc_mmio_bar(window, &start, &len);
195 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
196
197 get_uwc_mmio_bar(window, &start, &len);
198 window->uwc_map = map_mmio_region("UWCM_Window", start, len);
199
200 if (!window->hvwc_map || !window->uwc_map) {
201 unmap_winctx_mmio_bars(window);
202 return -1;
203 }
204
205 return 0;
206}
207
208
209
210
211
212
213
214
215
216
217static void reset_window_regs(struct vas_window *window)
218{
219 write_hvwc_reg(window, VREG(LPID), 0ULL);
220 write_hvwc_reg(window, VREG(PID), 0ULL);
221 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
222 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
223 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
224 write_hvwc_reg(window, VREG(AMR), 0ULL);
225 write_hvwc_reg(window, VREG(SEIDR), 0ULL);
226 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
227 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
228 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
229 write_hvwc_reg(window, VREG(PSWID), 0ULL);
230 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
231 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
232 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
233 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
234 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
235 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
236 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
237 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
238 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
239 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
240 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
241 write_hvwc_reg(window, VREG(WINCTL), 0ULL);
242 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
243 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
244 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
245 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
246 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
247 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
248 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
249 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
250 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
251 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
252
253
254
255
256
257
258
259
260
261
262
263
264}
265
266
267
268
269
270
271
272
273static void init_xlate_regs(struct vas_window *window, bool user_win)
274{
275 u64 lpcr, val;
276
277
278
279
280
281 val = 0ULL;
282 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
283 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
284 if (user_win) {
285 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
286 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
287 }
288 write_hvwc_reg(window, VREG(XLATE_MSR), val);
289
290 lpcr = mfspr(SPRN_LPCR);
291 val = 0ULL;
292
293
294
295
296
297
298
299 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
300 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
301 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
302 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
303 write_hvwc_reg(window, VREG(XLATE_LPCR), val);
304
305
306
307
308
309
310
311
312 val = 0ULL;
313 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
314 write_hvwc_reg(window, VREG(XLATE_CTL), val);
315
316
317
318
319 val = 0ULL;
320 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
321 write_hvwc_reg(window, VREG(AMR), val);
322
323 val = 0ULL;
324 val = SET_FIELD(VAS_SEIDR, val, 0);
325 write_hvwc_reg(window, VREG(SEIDR), val);
326}
327
328
329
330
331
332
333
334
335
336
337
338static void init_rsvd_tx_buf_count(struct vas_window *txwin,
339 struct vas_winctx *winctx)
340{
341 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
342}
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360static void init_winctx_regs(struct vas_window *window,
361 struct vas_winctx *winctx)
362{
363 u64 val;
364 int fifo_size;
365
366 reset_window_regs(window);
367
368 val = 0ULL;
369 val = SET_FIELD(VAS_LPID, val, winctx->lpid);
370 write_hvwc_reg(window, VREG(LPID), val);
371
372 val = 0ULL;
373 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
374 write_hvwc_reg(window, VREG(PID), val);
375
376 init_xlate_regs(window, winctx->user_win);
377
378 val = 0ULL;
379 val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id);
380 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
381
382
383 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
384
385 val = 0ULL;
386 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
387 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
388
389 val = 0ULL;
390 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
391 write_hvwc_reg(window, VREG(PSWID), val);
392
393 write_hvwc_reg(window, VREG(SPARE1), 0ULL);
394 write_hvwc_reg(window, VREG(SPARE2), 0ULL);
395 write_hvwc_reg(window, VREG(SPARE3), 0ULL);
396
397
398
399
400
401
402
403
404
405
406 val = __pa(winctx->rx_fifo);
407 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
408 write_hvwc_reg(window, VREG(LFIFO_BAR), val);
409
410 val = 0ULL;
411 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
412 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
413
414 val = 0ULL;
415 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
416 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
417 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
418
419 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
420 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
421 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
422
423 val = 0ULL;
424 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
425 write_hvwc_reg(window, VREG(LRX_WCRED), val);
426
427 val = 0ULL;
428 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
429 write_hvwc_reg(window, VREG(TX_WCRED), val);
430
431 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
432 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
433
434 fifo_size = winctx->rx_fifo_size / 1024;
435
436 val = 0ULL;
437 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
438 write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
439
440
441
442
443
444
445 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
446
447 init_rsvd_tx_buf_count(window, winctx);
448
449
450 val = 0ULL;
451 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
452 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
453
454 write_hvwc_reg(window, VREG(SPARE4), 0ULL);
455
456 val = 0ULL;
457 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
458 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
459 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
460 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
461 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
462
463 val = 0ULL;
464 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
465 write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
466
467 val = 0ULL;
468 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
469 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
470
471 val = 0ULL;
472 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
473 write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
474
475 val = 0ULL;
476 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
477 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
478 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
479
480
481
482 write_hvwc_reg(window, VREG(SPARE5), 0ULL);
483 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
484 write_hvwc_reg(window, VREG(SPARE6), 0ULL);
485
486
487 val = 0ULL;
488 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
489 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
490
491
492 val = 0ULL;
493 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
494 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
495 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
496 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
497 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
498 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
499 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
500 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
501 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
502 write_hvwc_reg(window, VREG(WINCTL), val);
503}
504
505static void vas_release_window_id(struct ida *ida, int winid)
506{
507 ida_free(ida, winid);
508}
509
510static int vas_assign_window_id(struct ida *ida)
511{
512 int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
513
514 if (winid == -ENOSPC) {
515 pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
516 return -EAGAIN;
517 }
518
519 return winid;
520}
521
522static void vas_window_free(struct vas_window *window)
523{
524 int winid = window->winid;
525 struct vas_instance *vinst = window->vinst;
526
527 unmap_winctx_mmio_bars(window);
528
529 vas_window_free_dbgdir(window);
530
531 kfree(window);
532
533 vas_release_window_id(&vinst->ida, winid);
534}
535
536static struct vas_window *vas_window_alloc(struct vas_instance *vinst)
537{
538 int winid;
539 struct vas_window *window;
540
541 winid = vas_assign_window_id(&vinst->ida);
542 if (winid < 0)
543 return ERR_PTR(winid);
544
545 window = kzalloc(sizeof(*window), GFP_KERNEL);
546 if (!window)
547 goto out_free;
548
549 window->vinst = vinst;
550 window->winid = winid;
551
552 if (map_winctx_mmio_bars(window))
553 goto out_free;
554
555 vas_window_init_dbgdir(window);
556
557 return window;
558
559out_free:
560 kfree(window);
561 vas_release_window_id(&vinst->ida, winid);
562 return ERR_PTR(-ENOMEM);
563}
564
565static void put_rx_win(struct vas_window *rxwin)
566{
567
568 WARN_ON_ONCE(rxwin->tx_win);
569
570 atomic_dec(&rxwin->num_txwins);
571}
572
573
574
575
576
577
578
579
580
581static struct vas_window *get_user_rxwin(struct vas_instance *vinst, u32 pswid)
582{
583 int vasid, winid;
584 struct vas_window *rxwin;
585
586 decode_pswid(pswid, &vasid, &winid);
587
588 if (vinst->vas_id != vasid)
589 return ERR_PTR(-EINVAL);
590
591 rxwin = vinst->windows[winid];
592
593 if (!rxwin || rxwin->tx_win || rxwin->cop != VAS_COP_TYPE_FTW)
594 return ERR_PTR(-EINVAL);
595
596 return rxwin;
597}
598
599
600
601
602
603
604
605static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst,
606 enum vas_cop_type cop, u32 pswid)
607{
608 struct vas_window *rxwin;
609
610 mutex_lock(&vinst->mutex);
611
612 if (cop == VAS_COP_TYPE_FTW)
613 rxwin = get_user_rxwin(vinst, pswid);
614 else
615 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
616
617 if (!IS_ERR(rxwin))
618 atomic_inc(&rxwin->num_txwins);
619
620 mutex_unlock(&vinst->mutex);
621
622 return rxwin;
623}
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640static void set_vinst_win(struct vas_instance *vinst,
641 struct vas_window *window)
642{
643 int id = window->winid;
644
645 mutex_lock(&vinst->mutex);
646
647
648
649
650
651 if (!window->user_win && !window->tx_win) {
652 WARN_ON_ONCE(vinst->rxwin[window->cop]);
653 vinst->rxwin[window->cop] = window;
654 }
655
656 WARN_ON_ONCE(vinst->windows[id] != NULL);
657 vinst->windows[id] = window;
658
659 mutex_unlock(&vinst->mutex);
660}
661
662
663
664
665
666static void clear_vinst_win(struct vas_window *window)
667{
668 int id = window->winid;
669 struct vas_instance *vinst = window->vinst;
670
671 mutex_lock(&vinst->mutex);
672
673 if (!window->user_win && !window->tx_win) {
674 WARN_ON_ONCE(!vinst->rxwin[window->cop]);
675 vinst->rxwin[window->cop] = NULL;
676 }
677
678 WARN_ON_ONCE(vinst->windows[id] != window);
679 vinst->windows[id] = NULL;
680
681 mutex_unlock(&vinst->mutex);
682}
683
684static void init_winctx_for_rxwin(struct vas_window *rxwin,
685 struct vas_rx_win_attr *rxattr,
686 struct vas_winctx *winctx)
687{
688
689
690
691
692
693
694
695
696
697
698
699
700
701 memset(winctx, 0, sizeof(struct vas_winctx));
702
703 winctx->rx_fifo = rxattr->rx_fifo;
704 winctx->rx_fifo_size = rxattr->rx_fifo_size;
705 winctx->wcreds_max = rxwin->wcreds_max;
706 winctx->pin_win = rxattr->pin_win;
707
708 winctx->nx_win = rxattr->nx_win;
709 winctx->fault_win = rxattr->fault_win;
710 winctx->user_win = rxattr->user_win;
711 winctx->rej_no_credit = rxattr->rej_no_credit;
712 winctx->rx_word_mode = rxattr->rx_win_ord_mode;
713 winctx->tx_word_mode = rxattr->tx_win_ord_mode;
714 winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
715 winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
716 winctx->notify_early = rxattr->notify_early;
717
718 if (winctx->nx_win) {
719 winctx->data_stamp = true;
720 winctx->intr_disable = true;
721 winctx->pin_win = true;
722
723 WARN_ON_ONCE(winctx->fault_win);
724 WARN_ON_ONCE(!winctx->rx_word_mode);
725 WARN_ON_ONCE(!winctx->tx_word_mode);
726 WARN_ON_ONCE(winctx->notify_after_count);
727 } else if (winctx->fault_win) {
728 winctx->notify_disable = true;
729 } else if (winctx->user_win) {
730
731
732
733
734
735
736
737
738 winctx->fifo_disable = true;
739 winctx->intr_disable = true;
740 winctx->rx_fifo = NULL;
741 }
742
743 winctx->lnotify_lpid = rxattr->lnotify_lpid;
744 winctx->lnotify_pid = rxattr->lnotify_pid;
745 winctx->lnotify_tid = rxattr->lnotify_tid;
746 winctx->pswid = rxattr->pswid;
747 winctx->dma_type = VAS_DMA_TYPE_INJECT;
748 winctx->tc_mode = rxattr->tc_mode;
749
750 winctx->min_scope = VAS_SCOPE_LOCAL;
751 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
752 if (rxwin->vinst->virq)
753 winctx->irq_port = rxwin->vinst->irq_port;
754}
755
756static bool rx_win_args_valid(enum vas_cop_type cop,
757 struct vas_rx_win_attr *attr)
758{
759 pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
760 attr->fault_win, attr->notify_disable,
761 attr->intr_disable, attr->notify_early,
762 attr->rx_fifo_size);
763
764 if (cop >= VAS_COP_TYPE_MAX)
765 return false;
766
767 if (cop != VAS_COP_TYPE_FTW &&
768 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
769 return false;
770
771 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
772 return false;
773
774 if (!attr->wcreds_max)
775 return false;
776
777 if (attr->nx_win) {
778
779 if (attr->fault_win || attr->user_win)
780 return false;
781
782
783
784
785 if (attr->notify_disable || !attr->intr_disable ||
786 attr->notify_early)
787 return false;
788 } else if (attr->fault_win) {
789
790 if (attr->user_win)
791 return false;
792
793
794
795
796
797 if (!attr->notify_disable || attr->intr_disable)
798 return false;
799
800 } else if (attr->user_win) {
801
802
803
804
805 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
806 return false;
807 } else {
808
809 return false;
810 }
811
812 return true;
813}
814
815void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
816{
817 memset(rxattr, 0, sizeof(*rxattr));
818
819 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
820 cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
821 rxattr->pin_win = true;
822 rxattr->nx_win = true;
823 rxattr->fault_win = false;
824 rxattr->intr_disable = true;
825 rxattr->rx_wcred_mode = true;
826 rxattr->tx_wcred_mode = true;
827 rxattr->rx_win_ord_mode = true;
828 rxattr->tx_win_ord_mode = true;
829 } else if (cop == VAS_COP_TYPE_FAULT) {
830 rxattr->pin_win = true;
831 rxattr->fault_win = true;
832 rxattr->notify_disable = true;
833 rxattr->rx_wcred_mode = true;
834 rxattr->rx_win_ord_mode = true;
835 rxattr->rej_no_credit = true;
836 rxattr->tc_mode = VAS_THRESH_DISABLED;
837 } else if (cop == VAS_COP_TYPE_FTW) {
838 rxattr->user_win = true;
839 rxattr->intr_disable = true;
840
841
842
843
844
845
846
847 }
848}
849EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
850
851struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
852 struct vas_rx_win_attr *rxattr)
853{
854 struct vas_window *rxwin;
855 struct vas_winctx winctx;
856 struct vas_instance *vinst;
857
858 trace_vas_rx_win_open(current, vasid, cop, rxattr);
859
860 if (!rx_win_args_valid(cop, rxattr))
861 return ERR_PTR(-EINVAL);
862
863 vinst = find_vas_instance(vasid);
864 if (!vinst) {
865 pr_devel("vasid %d not found!\n", vasid);
866 return ERR_PTR(-EINVAL);
867 }
868 pr_devel("Found instance %d\n", vasid);
869
870 rxwin = vas_window_alloc(vinst);
871 if (IS_ERR(rxwin)) {
872 pr_devel("Unable to allocate memory for Rx window\n");
873 return rxwin;
874 }
875
876 rxwin->tx_win = false;
877 rxwin->nx_win = rxattr->nx_win;
878 rxwin->user_win = rxattr->user_win;
879 rxwin->cop = cop;
880 rxwin->wcreds_max = rxattr->wcreds_max;
881
882 init_winctx_for_rxwin(rxwin, rxattr, &winctx);
883 init_winctx_regs(rxwin, &winctx);
884
885 set_vinst_win(vinst, rxwin);
886
887 return rxwin;
888}
889EXPORT_SYMBOL_GPL(vas_rx_win_open);
890
891void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
892{
893 memset(txattr, 0, sizeof(*txattr));
894
895 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
896 cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
897 txattr->rej_no_credit = false;
898 txattr->rx_wcred_mode = true;
899 txattr->tx_wcred_mode = true;
900 txattr->rx_win_ord_mode = true;
901 txattr->tx_win_ord_mode = true;
902 } else if (cop == VAS_COP_TYPE_FTW) {
903 txattr->user_win = true;
904 }
905}
906EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
907
908static void init_winctx_for_txwin(struct vas_window *txwin,
909 struct vas_tx_win_attr *txattr,
910 struct vas_winctx *winctx)
911{
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927 memset(winctx, 0, sizeof(struct vas_winctx));
928
929 winctx->wcreds_max = txwin->wcreds_max;
930
931 winctx->user_win = txattr->user_win;
932 winctx->nx_win = txwin->rxwin->nx_win;
933 winctx->pin_win = txattr->pin_win;
934 winctx->rej_no_credit = txattr->rej_no_credit;
935 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
936
937 winctx->rx_wcred_mode = txattr->rx_wcred_mode;
938 winctx->tx_wcred_mode = txattr->tx_wcred_mode;
939 winctx->rx_word_mode = txattr->rx_win_ord_mode;
940 winctx->tx_word_mode = txattr->tx_win_ord_mode;
941 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
942
943 winctx->intr_disable = true;
944 if (winctx->nx_win)
945 winctx->data_stamp = true;
946
947 winctx->lpid = txattr->lpid;
948 winctx->pidr = txattr->pidr;
949 winctx->rx_win_id = txwin->rxwin->winid;
950
951
952
953
954 if (txwin->vinst->virq)
955 winctx->fault_win_id = txwin->vinst->fault_win->winid;
956
957 winctx->dma_type = VAS_DMA_TYPE_INJECT;
958 winctx->tc_mode = txattr->tc_mode;
959 winctx->min_scope = VAS_SCOPE_LOCAL;
960 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
961 if (txwin->vinst->virq)
962 winctx->irq_port = txwin->vinst->irq_port;
963
964 winctx->pswid = txattr->pswid ? txattr->pswid :
965 encode_pswid(txwin->vinst->vas_id, txwin->winid);
966}
967
968static bool tx_win_args_valid(enum vas_cop_type cop,
969 struct vas_tx_win_attr *attr)
970{
971 if (attr->tc_mode != VAS_THRESH_DISABLED)
972 return false;
973
974 if (cop > VAS_COP_TYPE_MAX)
975 return false;
976
977 if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
978 return false;
979
980 if (attr->user_win) {
981 if (attr->rsvd_txbuf_count)
982 return false;
983
984 if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
985 cop != VAS_COP_TYPE_GZIP_HIPRI)
986 return false;
987 }
988
989 return true;
990}
991
992struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
993 struct vas_tx_win_attr *attr)
994{
995 int rc;
996 struct vas_window *txwin;
997 struct vas_window *rxwin;
998 struct vas_winctx winctx;
999 struct vas_instance *vinst;
1000
1001 trace_vas_tx_win_open(current, vasid, cop, attr);
1002
1003 if (!tx_win_args_valid(cop, attr))
1004 return ERR_PTR(-EINVAL);
1005
1006
1007
1008
1009
1010
1011 if (vasid == -1 && attr->pswid)
1012 decode_pswid(attr->pswid, &vasid, NULL);
1013
1014 vinst = find_vas_instance(vasid);
1015 if (!vinst) {
1016 pr_devel("vasid %d not found!\n", vasid);
1017 return ERR_PTR(-EINVAL);
1018 }
1019
1020 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
1021 if (IS_ERR(rxwin)) {
1022 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1023 return rxwin;
1024 }
1025
1026 txwin = vas_window_alloc(vinst);
1027 if (IS_ERR(txwin)) {
1028 rc = PTR_ERR(txwin);
1029 goto put_rxwin;
1030 }
1031
1032 txwin->cop = cop;
1033 txwin->tx_win = 1;
1034 txwin->rxwin = rxwin;
1035 txwin->nx_win = txwin->rxwin->nx_win;
1036 txwin->user_win = attr->user_win;
1037 txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1038
1039 init_winctx_for_txwin(txwin, attr, &winctx);
1040
1041 init_winctx_regs(txwin, &winctx);
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051 if (!txwin->user_win) {
1052 txwin->paste_kaddr = map_paste_region(txwin);
1053 if (IS_ERR(txwin->paste_kaddr)) {
1054 rc = PTR_ERR(txwin->paste_kaddr);
1055 goto free_window;
1056 }
1057 } else {
1058
1059
1060
1061
1062
1063 if (!vinst->virq) {
1064 rc = -ENODEV;
1065 goto free_window;
1066 }
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076 txwin->pid = get_task_pid(current, PIDTYPE_PID);
1077
1078
1079
1080 txwin->mm = get_task_mm(current);
1081
1082 if (!txwin->mm) {
1083 put_pid(txwin->pid);
1084 pr_err("VAS: pid(%d): mm_struct is not found\n",
1085 current->pid);
1086 rc = -EPERM;
1087 goto free_window;
1088 }
1089
1090 mmgrab(txwin->mm);
1091 mmput(txwin->mm);
1092 mm_context_add_vas_window(txwin->mm);
1093
1094
1095
1096
1097
1098
1099
1100 txwin->tgid = find_get_pid(task_tgid_vnr(current));
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111 asm volatile(PPC_CP_ABORT);
1112 }
1113
1114 set_vinst_win(vinst, txwin);
1115
1116 return txwin;
1117
1118free_window:
1119 vas_window_free(txwin);
1120
1121put_rxwin:
1122 put_rx_win(rxwin);
1123 return ERR_PTR(rc);
1124
1125}
1126EXPORT_SYMBOL_GPL(vas_tx_win_open);
1127
1128int vas_copy_crb(void *crb, int offset)
1129{
1130 return vas_copy(crb, offset);
1131}
1132EXPORT_SYMBOL_GPL(vas_copy_crb);
1133
1134#define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1135int vas_paste_crb(struct vas_window *txwin, int offset, bool re)
1136{
1137 int rc;
1138 void *addr;
1139 uint64_t val;
1140
1141 trace_vas_paste_crb(current, txwin);
1142
1143
1144
1145
1146
1147
1148 WARN_ON_ONCE(txwin->nx_win && !re);
1149
1150 addr = txwin->paste_kaddr;
1151 if (re) {
1152
1153
1154
1155
1156 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1157 addr += val;
1158 }
1159
1160
1161
1162
1163
1164 rc = vas_paste(addr, offset);
1165 if (rc == 2)
1166 rc = 0;
1167 else
1168 rc = -EINVAL;
1169
1170 pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid,
1171 read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1172
1173 return rc;
1174}
1175EXPORT_SYMBOL_GPL(vas_paste_crb);
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190static void poll_window_credits(struct vas_window *window)
1191{
1192 u64 val;
1193 int creds, mode;
1194 int count = 0;
1195
1196 val = read_hvwc_reg(window, VREG(WINCTL));
1197 if (window->tx_win)
1198 mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1199 else
1200 mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1201
1202 if (!mode)
1203 return;
1204retry:
1205 if (window->tx_win) {
1206 val = read_hvwc_reg(window, VREG(TX_WCRED));
1207 creds = GET_FIELD(VAS_TX_WCRED, val);
1208 } else {
1209 val = read_hvwc_reg(window, VREG(LRX_WCRED));
1210 creds = GET_FIELD(VAS_LRX_WCRED, val);
1211 }
1212
1213
1214
1215
1216
1217
1218
1219
1220 if (creds < window->wcreds_max) {
1221 val = 0;
1222 set_current_state(TASK_UNINTERRUPTIBLE);
1223 schedule_timeout(msecs_to_jiffies(10));
1224 count++;
1225
1226
1227
1228
1229 if (!(count % 1000))
1230 pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Retries %d\n",
1231 vas_window_pid(window), window->winid,
1232 creds, count);
1233
1234 goto retry;
1235 }
1236}
1237
1238
1239
1240
1241
1242
1243static void poll_window_busy_state(struct vas_window *window)
1244{
1245 int busy;
1246 u64 val;
1247 int count = 0;
1248
1249retry:
1250 val = read_hvwc_reg(window, VREG(WIN_STATUS));
1251 busy = GET_FIELD(VAS_WIN_BUSY, val);
1252 if (busy) {
1253 val = 0;
1254 set_current_state(TASK_UNINTERRUPTIBLE);
1255 schedule_timeout(msecs_to_jiffies(10));
1256 count++;
1257
1258
1259
1260
1261 if (!(count % 1000))
1262 pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n",
1263 vas_window_pid(window), window->winid, count);
1264
1265 goto retry;
1266 }
1267}
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285static void poll_window_castout(struct vas_window *window)
1286{
1287
1288}
1289
1290
1291
1292
1293
1294static void unpin_close_window(struct vas_window *window)
1295{
1296 u64 val;
1297
1298 val = read_hvwc_reg(window, VREG(WINCTL));
1299 val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1300 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1301 write_hvwc_reg(window, VREG(WINCTL), val);
1302}
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316int vas_win_close(struct vas_window *window)
1317{
1318 if (!window)
1319 return 0;
1320
1321 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1322 pr_devel("Attempting to close an active Rx window!\n");
1323 WARN_ON_ONCE(1);
1324 return -EBUSY;
1325 }
1326
1327 unmap_paste_region(window);
1328
1329 poll_window_busy_state(window);
1330
1331 unpin_close_window(window);
1332
1333 poll_window_credits(window);
1334
1335 clear_vinst_win(window);
1336
1337 poll_window_castout(window);
1338
1339
1340 if (window->tx_win) {
1341 if (window->user_win) {
1342
1343 put_pid(window->pid);
1344 if (window->mm) {
1345 mm_context_remove_vas_window(window->mm);
1346 mmdrop(window->mm);
1347 }
1348 }
1349 put_rx_win(window->rxwin);
1350 }
1351
1352 vas_window_free(window);
1353
1354 return 0;
1355}
1356EXPORT_SYMBOL_GPL(vas_win_close);
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380void vas_return_credit(struct vas_window *window, bool tx)
1381{
1382 uint64_t val;
1383
1384 val = 0ULL;
1385 if (tx) {
1386 val = SET_FIELD(VAS_TX_WCRED, val, 1);
1387 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), val);
1388 } else {
1389 val = SET_FIELD(VAS_LRX_WCRED, val, 1);
1390 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), val);
1391 }
1392}
1393
1394struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
1395 uint32_t pswid)
1396{
1397 struct vas_window *window;
1398 int winid;
1399
1400 if (!pswid) {
1401 pr_devel("%s: called for pswid 0!\n", __func__);
1402 return ERR_PTR(-ESRCH);
1403 }
1404
1405 decode_pswid(pswid, NULL, &winid);
1406
1407 if (winid >= VAS_WINDOWS_PER_CHIP)
1408 return ERR_PTR(-ESRCH);
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419 window = vinst->windows[winid];
1420
1421 if (!window) {
1422 pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n",
1423 winid, pswid, vinst);
1424 return NULL;
1425 }
1426
1427
1428
1429
1430
1431
1432
1433 if (!window->tx_win || !window->user_win || !window->nx_win ||
1434 window->cop == VAS_COP_TYPE_FAULT ||
1435 window->cop == VAS_COP_TYPE_FTW) {
1436 pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n",
1437 winid, window->tx_win, window->user_win,
1438 window->nx_win, window->cop);
1439 WARN_ON(1);
1440 }
1441
1442 return window;
1443}
1444