1
2
3
4
5
6#define pr_fmt(fmt) "vas: " fmt
7
8#include <linux/types.h>
9#include <linux/mutex.h>
10#include <linux/slab.h>
11#include <linux/io.h>
12#include <linux/log2.h>
13#include <linux/rcupdate.h>
14#include <linux/cred.h>
15#include <asm/switch_to.h>
16#include <asm/ppc-opcode.h>
17#include "vas.h"
18#include "copy-paste.h"
19
20#define CREATE_TRACE_POINTS
21#include "vas-trace.h"
22
23
24
25
26
27static void compute_paste_address(struct vas_window *window, u64 *addr, int *len)
28{
29 int winid;
30 u64 base, shift;
31
32 base = window->vinst->paste_base_addr;
33 shift = window->vinst->paste_win_id_shift;
34 winid = window->winid;
35
36 *addr = base + (winid << shift);
37 if (len)
38 *len = PAGE_SIZE;
39
40 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
41}
42
43u64 vas_win_paste_addr(struct vas_window *win)
44{
45 u64 addr;
46
47 compute_paste_address(win, &addr, NULL);
48
49 return addr;
50}
51EXPORT_SYMBOL(vas_win_paste_addr);
52
53static inline void get_hvwc_mmio_bar(struct vas_window *window,
54 u64 *start, int *len)
55{
56 u64 pbaddr;
57
58 pbaddr = window->vinst->hvwc_bar_start;
59 *start = pbaddr + window->winid * VAS_HVWC_SIZE;
60 *len = VAS_HVWC_SIZE;
61}
62
63static inline void get_uwc_mmio_bar(struct vas_window *window,
64 u64 *start, int *len)
65{
66 u64 pbaddr;
67
68 pbaddr = window->vinst->uwc_bar_start;
69 *start = pbaddr + window->winid * VAS_UWC_SIZE;
70 *len = VAS_UWC_SIZE;
71}
72
73
74
75
76
77
78static void *map_paste_region(struct vas_window *txwin)
79{
80 int len;
81 void *map;
82 char *name;
83 u64 start;
84
85 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
86 txwin->winid);
87 if (!name)
88 goto free_name;
89
90 txwin->paste_addr_name = name;
91 compute_paste_address(txwin, &start, &len);
92
93 if (!request_mem_region(start, len, name)) {
94 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
95 __func__, start, len);
96 goto free_name;
97 }
98
99 map = ioremap_cache(start, len);
100 if (!map) {
101 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
102 start, len);
103 goto free_name;
104 }
105
106 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
107 return map;
108
109free_name:
110 kfree(name);
111 return ERR_PTR(-ENOMEM);
112}
113
114static void *map_mmio_region(char *name, u64 start, int len)
115{
116 void *map;
117
118 if (!request_mem_region(start, len, name)) {
119 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
120 __func__, start, len);
121 return NULL;
122 }
123
124 map = ioremap(start, len);
125 if (!map) {
126 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
127 len);
128 return NULL;
129 }
130
131 return map;
132}
133
134static void unmap_region(void *addr, u64 start, int len)
135{
136 iounmap(addr);
137 release_mem_region((phys_addr_t)start, len);
138}
139
140
141
142
143static void unmap_paste_region(struct vas_window *window)
144{
145 int len;
146 u64 busaddr_start;
147
148 if (window->paste_kaddr) {
149 compute_paste_address(window, &busaddr_start, &len);
150 unmap_region(window->paste_kaddr, busaddr_start, len);
151 window->paste_kaddr = NULL;
152 kfree(window->paste_addr_name);
153 window->paste_addr_name = NULL;
154 }
155}
156
157
158
159
160
161
162
163
164static void unmap_winctx_mmio_bars(struct vas_window *window)
165{
166 int len;
167 void *uwc_map;
168 void *hvwc_map;
169 u64 busaddr_start;
170
171 mutex_lock(&vas_mutex);
172
173 hvwc_map = window->hvwc_map;
174 window->hvwc_map = NULL;
175
176 uwc_map = window->uwc_map;
177 window->uwc_map = NULL;
178
179 mutex_unlock(&vas_mutex);
180
181 if (hvwc_map) {
182 get_hvwc_mmio_bar(window, &busaddr_start, &len);
183 unmap_region(hvwc_map, busaddr_start, len);
184 }
185
186 if (uwc_map) {
187 get_uwc_mmio_bar(window, &busaddr_start, &len);
188 unmap_region(uwc_map, busaddr_start, len);
189 }
190}
191
192
193
194
195
196
197int map_winctx_mmio_bars(struct vas_window *window)
198{
199 int len;
200 u64 start;
201
202 get_hvwc_mmio_bar(window, &start, &len);
203 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
204
205 get_uwc_mmio_bar(window, &start, &len);
206 window->uwc_map = map_mmio_region("UWCM_Window", start, len);
207
208 if (!window->hvwc_map || !window->uwc_map) {
209 unmap_winctx_mmio_bars(window);
210 return -1;
211 }
212
213 return 0;
214}
215
216
217
218
219
220
221
222
223
224
225void reset_window_regs(struct vas_window *window)
226{
227 write_hvwc_reg(window, VREG(LPID), 0ULL);
228 write_hvwc_reg(window, VREG(PID), 0ULL);
229 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
230 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
231 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
232 write_hvwc_reg(window, VREG(AMR), 0ULL);
233 write_hvwc_reg(window, VREG(SEIDR), 0ULL);
234 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
235 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
236 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
237 write_hvwc_reg(window, VREG(PSWID), 0ULL);
238 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
239 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
240 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
241 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
242 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
243 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
244 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
245 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
246 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
247 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
248 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
249 write_hvwc_reg(window, VREG(WINCTL), 0ULL);
250 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
251 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
252 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
253 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
254 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
255 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
256 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
257 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
258 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
259 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
260
261
262
263
264
265
266
267
268
269
270
271
272}
273
274
275
276
277
278
279
280
281static void init_xlate_regs(struct vas_window *window, bool user_win)
282{
283 u64 lpcr, val;
284
285
286
287
288
289 val = 0ULL;
290 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
291 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
292 if (user_win) {
293 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
294 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
295 }
296 write_hvwc_reg(window, VREG(XLATE_MSR), val);
297
298 lpcr = mfspr(SPRN_LPCR);
299 val = 0ULL;
300
301
302
303
304
305
306
307 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
308 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
309 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
310 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
311 write_hvwc_reg(window, VREG(XLATE_LPCR), val);
312
313
314
315
316
317
318
319
320 val = 0ULL;
321 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
322 write_hvwc_reg(window, VREG(XLATE_CTL), val);
323
324
325
326
327 val = 0ULL;
328 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
329 write_hvwc_reg(window, VREG(AMR), val);
330
331 val = 0ULL;
332 val = SET_FIELD(VAS_SEIDR, val, 0);
333 write_hvwc_reg(window, VREG(SEIDR), val);
334}
335
336
337
338
339
340
341
342
343
344
345
346static void init_rsvd_tx_buf_count(struct vas_window *txwin,
347 struct vas_winctx *winctx)
348{
349 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
350}
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368int init_winctx_regs(struct vas_window *window, struct vas_winctx *winctx)
369{
370 u64 val;
371 int fifo_size;
372
373 reset_window_regs(window);
374
375 val = 0ULL;
376 val = SET_FIELD(VAS_LPID, val, winctx->lpid);
377 write_hvwc_reg(window, VREG(LPID), val);
378
379 val = 0ULL;
380 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
381 write_hvwc_reg(window, VREG(PID), val);
382
383 init_xlate_regs(window, winctx->user_win);
384
385 val = 0ULL;
386 val = SET_FIELD(VAS_FAULT_TX_WIN, val, 0);
387 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
388
389
390 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
391
392 val = 0ULL;
393 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
394 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
395
396 val = 0ULL;
397 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
398 write_hvwc_reg(window, VREG(PSWID), val);
399
400 write_hvwc_reg(window, VREG(SPARE1), 0ULL);
401 write_hvwc_reg(window, VREG(SPARE2), 0ULL);
402 write_hvwc_reg(window, VREG(SPARE3), 0ULL);
403
404
405
406
407
408
409
410
411
412
413 val = __pa(winctx->rx_fifo);
414 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
415 write_hvwc_reg(window, VREG(LFIFO_BAR), val);
416
417 val = 0ULL;
418 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
419 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
420
421 val = 0ULL;
422 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
423 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
424 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
425
426 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
427 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
428 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
429
430 val = 0ULL;
431 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
432 write_hvwc_reg(window, VREG(LRX_WCRED), val);
433
434 val = 0ULL;
435 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
436 write_hvwc_reg(window, VREG(TX_WCRED), val);
437
438 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
439 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
440
441 fifo_size = winctx->rx_fifo_size / 1024;
442
443 val = 0ULL;
444 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
445 write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
446
447
448
449
450
451
452 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
453
454 init_rsvd_tx_buf_count(window, winctx);
455
456
457 val = 0ULL;
458 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
459 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
460
461 write_hvwc_reg(window, VREG(SPARE4), 0ULL);
462
463 val = 0ULL;
464 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
465 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
466 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
467 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
468 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
469
470 val = 0ULL;
471 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
472 write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
473
474 val = 0ULL;
475 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
476 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
477
478 val = 0ULL;
479 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
480 write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
481
482 val = 0ULL;
483 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
484 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
485 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
486
487
488
489 write_hvwc_reg(window, VREG(SPARE5), 0ULL);
490 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
491 write_hvwc_reg(window, VREG(SPARE6), 0ULL);
492
493
494 val = 0ULL;
495 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
496 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
497
498
499 val = 0ULL;
500 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
501 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
502 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
503 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
504 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
505 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
506 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
507 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
508 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
509 write_hvwc_reg(window, VREG(WINCTL), val);
510
511 return 0;
512}
513
514static void vas_release_window_id(struct ida *ida, int winid)
515{
516 ida_free(ida, winid);
517}
518
519static int vas_assign_window_id(struct ida *ida)
520{
521 int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
522
523 if (winid == -ENOSPC) {
524 pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
525 return -EAGAIN;
526 }
527
528 return winid;
529}
530
531static void vas_window_free(struct vas_window *window)
532{
533 int winid = window->winid;
534 struct vas_instance *vinst = window->vinst;
535
536 unmap_winctx_mmio_bars(window);
537
538 vas_window_free_dbgdir(window);
539
540 kfree(window);
541
542 vas_release_window_id(&vinst->ida, winid);
543}
544
545static struct vas_window *vas_window_alloc(struct vas_instance *vinst)
546{
547 int winid;
548 struct vas_window *window;
549
550 winid = vas_assign_window_id(&vinst->ida);
551 if (winid < 0)
552 return ERR_PTR(winid);
553
554 window = kzalloc(sizeof(*window), GFP_KERNEL);
555 if (!window)
556 goto out_free;
557
558 window->vinst = vinst;
559 window->winid = winid;
560
561 if (map_winctx_mmio_bars(window))
562 goto out_free;
563
564 vas_window_init_dbgdir(window);
565
566 return window;
567
568out_free:
569 kfree(window);
570 vas_release_window_id(&vinst->ida, winid);
571 return ERR_PTR(-ENOMEM);
572}
573
574static void put_rx_win(struct vas_window *rxwin)
575{
576
577 WARN_ON_ONCE(rxwin->tx_win);
578
579 atomic_dec(&rxwin->num_txwins);
580}
581
582
583
584
585
586
587
588
589
590static struct vas_window *get_user_rxwin(struct vas_instance *vinst, u32 pswid)
591{
592 int vasid, winid;
593 struct vas_window *rxwin;
594
595 decode_pswid(pswid, &vasid, &winid);
596
597 if (vinst->vas_id != vasid)
598 return ERR_PTR(-EINVAL);
599
600 rxwin = vinst->windows[winid];
601
602 if (!rxwin || rxwin->tx_win || rxwin->cop != VAS_COP_TYPE_FTW)
603 return ERR_PTR(-EINVAL);
604
605 return rxwin;
606}
607
608
609
610
611
612
613
614static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst,
615 enum vas_cop_type cop, u32 pswid)
616{
617 struct vas_window *rxwin;
618
619 mutex_lock(&vinst->mutex);
620
621 if (cop == VAS_COP_TYPE_FTW)
622 rxwin = get_user_rxwin(vinst, pswid);
623 else
624 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
625
626 if (!IS_ERR(rxwin))
627 atomic_inc(&rxwin->num_txwins);
628
629 mutex_unlock(&vinst->mutex);
630
631 return rxwin;
632}
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649static void set_vinst_win(struct vas_instance *vinst,
650 struct vas_window *window)
651{
652 int id = window->winid;
653
654 mutex_lock(&vinst->mutex);
655
656
657
658
659
660 if (!window->user_win && !window->tx_win) {
661 WARN_ON_ONCE(vinst->rxwin[window->cop]);
662 vinst->rxwin[window->cop] = window;
663 }
664
665 WARN_ON_ONCE(vinst->windows[id] != NULL);
666 vinst->windows[id] = window;
667
668 mutex_unlock(&vinst->mutex);
669}
670
671
672
673
674
675static void clear_vinst_win(struct vas_window *window)
676{
677 int id = window->winid;
678 struct vas_instance *vinst = window->vinst;
679
680 mutex_lock(&vinst->mutex);
681
682 if (!window->user_win && !window->tx_win) {
683 WARN_ON_ONCE(!vinst->rxwin[window->cop]);
684 vinst->rxwin[window->cop] = NULL;
685 }
686
687 WARN_ON_ONCE(vinst->windows[id] != window);
688 vinst->windows[id] = NULL;
689
690 mutex_unlock(&vinst->mutex);
691}
692
693static void init_winctx_for_rxwin(struct vas_window *rxwin,
694 struct vas_rx_win_attr *rxattr,
695 struct vas_winctx *winctx)
696{
697
698
699
700
701
702
703
704
705
706
707
708
709
710 memset(winctx, 0, sizeof(struct vas_winctx));
711
712 winctx->rx_fifo = rxattr->rx_fifo;
713 winctx->rx_fifo_size = rxattr->rx_fifo_size;
714 winctx->wcreds_max = rxwin->wcreds_max;
715 winctx->pin_win = rxattr->pin_win;
716
717 winctx->nx_win = rxattr->nx_win;
718 winctx->fault_win = rxattr->fault_win;
719 winctx->user_win = rxattr->user_win;
720 winctx->rej_no_credit = rxattr->rej_no_credit;
721 winctx->rx_word_mode = rxattr->rx_win_ord_mode;
722 winctx->tx_word_mode = rxattr->tx_win_ord_mode;
723 winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
724 winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
725 winctx->notify_early = rxattr->notify_early;
726
727 if (winctx->nx_win) {
728 winctx->data_stamp = true;
729 winctx->intr_disable = true;
730 winctx->pin_win = true;
731
732 WARN_ON_ONCE(winctx->fault_win);
733 WARN_ON_ONCE(!winctx->rx_word_mode);
734 WARN_ON_ONCE(!winctx->tx_word_mode);
735 WARN_ON_ONCE(winctx->notify_after_count);
736 } else if (winctx->fault_win) {
737 winctx->notify_disable = true;
738 } else if (winctx->user_win) {
739
740
741
742
743
744
745
746
747 winctx->fifo_disable = true;
748 winctx->intr_disable = true;
749 winctx->rx_fifo = NULL;
750 }
751
752 winctx->lnotify_lpid = rxattr->lnotify_lpid;
753 winctx->lnotify_pid = rxattr->lnotify_pid;
754 winctx->lnotify_tid = rxattr->lnotify_tid;
755 winctx->pswid = rxattr->pswid;
756 winctx->dma_type = VAS_DMA_TYPE_INJECT;
757 winctx->tc_mode = rxattr->tc_mode;
758
759 winctx->min_scope = VAS_SCOPE_LOCAL;
760 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
761}
762
763static bool rx_win_args_valid(enum vas_cop_type cop,
764 struct vas_rx_win_attr *attr)
765{
766 pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
767 attr->fault_win, attr->notify_disable,
768 attr->intr_disable, attr->notify_early,
769 attr->rx_fifo_size);
770
771 if (cop >= VAS_COP_TYPE_MAX)
772 return false;
773
774 if (cop != VAS_COP_TYPE_FTW &&
775 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
776 return false;
777
778 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
779 return false;
780
781 if (attr->wcreds_max > VAS_RX_WCREDS_MAX)
782 return false;
783
784 if (attr->nx_win) {
785
786 if (attr->fault_win || attr->user_win)
787 return false;
788
789
790
791
792 if (attr->notify_disable || !attr->intr_disable ||
793 attr->notify_early)
794 return false;
795 } else if (attr->fault_win) {
796
797 if (attr->user_win)
798 return false;
799
800
801
802
803
804 if (!attr->notify_disable || attr->intr_disable)
805 return false;
806
807 } else if (attr->user_win) {
808
809
810
811
812 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
813 return false;
814 } else {
815
816 return false;
817 }
818
819 return true;
820}
821
822void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
823{
824 memset(rxattr, 0, sizeof(*rxattr));
825
826 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
827 rxattr->pin_win = true;
828 rxattr->nx_win = true;
829 rxattr->fault_win = false;
830 rxattr->intr_disable = true;
831 rxattr->rx_wcred_mode = true;
832 rxattr->tx_wcred_mode = true;
833 rxattr->rx_win_ord_mode = true;
834 rxattr->tx_win_ord_mode = true;
835 } else if (cop == VAS_COP_TYPE_FAULT) {
836 rxattr->pin_win = true;
837 rxattr->fault_win = true;
838 rxattr->notify_disable = true;
839 rxattr->rx_wcred_mode = true;
840 rxattr->tx_wcred_mode = true;
841 rxattr->rx_win_ord_mode = true;
842 rxattr->tx_win_ord_mode = true;
843 } else if (cop == VAS_COP_TYPE_FTW) {
844 rxattr->user_win = true;
845 rxattr->intr_disable = true;
846
847
848
849
850
851
852
853 }
854}
855EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
856
857struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
858 struct vas_rx_win_attr *rxattr)
859{
860 struct vas_window *rxwin;
861 struct vas_winctx winctx;
862 struct vas_instance *vinst;
863
864 trace_vas_rx_win_open(current, vasid, cop, rxattr);
865
866 if (!rx_win_args_valid(cop, rxattr))
867 return ERR_PTR(-EINVAL);
868
869 vinst = find_vas_instance(vasid);
870 if (!vinst) {
871 pr_devel("vasid %d not found!\n", vasid);
872 return ERR_PTR(-EINVAL);
873 }
874 pr_devel("Found instance %d\n", vasid);
875
876 rxwin = vas_window_alloc(vinst);
877 if (IS_ERR(rxwin)) {
878 pr_devel("Unable to allocate memory for Rx window\n");
879 return rxwin;
880 }
881
882 rxwin->tx_win = false;
883 rxwin->nx_win = rxattr->nx_win;
884 rxwin->user_win = rxattr->user_win;
885 rxwin->cop = cop;
886 rxwin->wcreds_max = rxattr->wcreds_max ?: VAS_WCREDS_DEFAULT;
887 if (rxattr->user_win)
888 rxwin->pid = task_pid_vnr(current);
889
890 init_winctx_for_rxwin(rxwin, rxattr, &winctx);
891 init_winctx_regs(rxwin, &winctx);
892
893 set_vinst_win(vinst, rxwin);
894
895 return rxwin;
896}
897EXPORT_SYMBOL_GPL(vas_rx_win_open);
898
899void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
900{
901 memset(txattr, 0, sizeof(*txattr));
902
903 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
904 txattr->rej_no_credit = false;
905 txattr->rx_wcred_mode = true;
906 txattr->tx_wcred_mode = true;
907 txattr->rx_win_ord_mode = true;
908 txattr->tx_win_ord_mode = true;
909 } else if (cop == VAS_COP_TYPE_FTW) {
910 txattr->user_win = true;
911 }
912}
913EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
914
915static void init_winctx_for_txwin(struct vas_window *txwin,
916 struct vas_tx_win_attr *txattr,
917 struct vas_winctx *winctx)
918{
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934 memset(winctx, 0, sizeof(struct vas_winctx));
935
936 winctx->wcreds_max = txwin->wcreds_max;
937
938 winctx->user_win = txattr->user_win;
939 winctx->nx_win = txwin->rxwin->nx_win;
940 winctx->pin_win = txattr->pin_win;
941 winctx->rej_no_credit = txattr->rej_no_credit;
942 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
943
944 winctx->rx_wcred_mode = txattr->rx_wcred_mode;
945 winctx->tx_wcred_mode = txattr->tx_wcred_mode;
946 winctx->rx_word_mode = txattr->rx_win_ord_mode;
947 winctx->tx_word_mode = txattr->tx_win_ord_mode;
948 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
949
950 winctx->intr_disable = true;
951 if (winctx->nx_win)
952 winctx->data_stamp = true;
953
954 winctx->lpid = txattr->lpid;
955 winctx->pidr = txattr->pidr;
956 winctx->rx_win_id = txwin->rxwin->winid;
957
958 winctx->dma_type = VAS_DMA_TYPE_INJECT;
959 winctx->tc_mode = txattr->tc_mode;
960 winctx->min_scope = VAS_SCOPE_LOCAL;
961 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
962
963 winctx->pswid = 0;
964}
965
966static bool tx_win_args_valid(enum vas_cop_type cop,
967 struct vas_tx_win_attr *attr)
968{
969 if (attr->tc_mode != VAS_THRESH_DISABLED)
970 return false;
971
972 if (cop > VAS_COP_TYPE_MAX)
973 return false;
974
975 if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
976 return false;
977
978 if (attr->user_win &&
979 (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count))
980 return false;
981
982 return true;
983}
984
985struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
986 struct vas_tx_win_attr *attr)
987{
988 int rc;
989 struct vas_window *txwin;
990 struct vas_window *rxwin;
991 struct vas_winctx winctx;
992 struct vas_instance *vinst;
993
994 trace_vas_tx_win_open(current, vasid, cop, attr);
995
996 if (!tx_win_args_valid(cop, attr))
997 return ERR_PTR(-EINVAL);
998
999
1000
1001
1002
1003
1004 if (vasid == -1 && attr->pswid)
1005 decode_pswid(attr->pswid, &vasid, NULL);
1006
1007 vinst = find_vas_instance(vasid);
1008 if (!vinst) {
1009 pr_devel("vasid %d not found!\n", vasid);
1010 return ERR_PTR(-EINVAL);
1011 }
1012
1013 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
1014 if (IS_ERR(rxwin)) {
1015 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1016 return rxwin;
1017 }
1018
1019 txwin = vas_window_alloc(vinst);
1020 if (IS_ERR(txwin)) {
1021 rc = PTR_ERR(txwin);
1022 goto put_rxwin;
1023 }
1024
1025 txwin->cop = cop;
1026 txwin->tx_win = 1;
1027 txwin->rxwin = rxwin;
1028 txwin->nx_win = txwin->rxwin->nx_win;
1029 txwin->pid = attr->pid;
1030 txwin->user_win = attr->user_win;
1031 txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1032
1033 init_winctx_for_txwin(txwin, attr, &winctx);
1034
1035 init_winctx_regs(txwin, &winctx);
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045 if (!txwin->user_win) {
1046 txwin->paste_kaddr = map_paste_region(txwin);
1047 if (IS_ERR(txwin->paste_kaddr)) {
1048 rc = PTR_ERR(txwin->paste_kaddr);
1049 goto free_window;
1050 }
1051 } else {
1052
1053
1054
1055
1056 rc = set_thread_uses_vas();
1057 if (rc)
1058 goto free_window;
1059 }
1060
1061 set_vinst_win(vinst, txwin);
1062
1063 return txwin;
1064
1065free_window:
1066 vas_window_free(txwin);
1067
1068put_rxwin:
1069 put_rx_win(rxwin);
1070 return ERR_PTR(rc);
1071
1072}
1073EXPORT_SYMBOL_GPL(vas_tx_win_open);
1074
1075int vas_copy_crb(void *crb, int offset)
1076{
1077 return vas_copy(crb, offset);
1078}
1079EXPORT_SYMBOL_GPL(vas_copy_crb);
1080
1081#define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1082int vas_paste_crb(struct vas_window *txwin, int offset, bool re)
1083{
1084 int rc;
1085 void *addr;
1086 uint64_t val;
1087
1088 trace_vas_paste_crb(current, txwin);
1089
1090
1091
1092
1093
1094
1095 WARN_ON_ONCE(txwin->nx_win && !re);
1096
1097 addr = txwin->paste_kaddr;
1098 if (re) {
1099
1100
1101
1102
1103 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1104 addr += val;
1105 }
1106
1107
1108
1109
1110
1111 rc = vas_paste(addr, offset);
1112 if (rc == 2)
1113 rc = 0;
1114 else
1115 rc = -EINVAL;
1116
1117 pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid,
1118 read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1119
1120 return rc;
1121}
1122EXPORT_SYMBOL_GPL(vas_paste_crb);
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137static void poll_window_credits(struct vas_window *window)
1138{
1139 u64 val;
1140 int creds, mode;
1141
1142 val = read_hvwc_reg(window, VREG(WINCTL));
1143 if (window->tx_win)
1144 mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1145 else
1146 mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1147
1148 if (!mode)
1149 return;
1150retry:
1151 if (window->tx_win) {
1152 val = read_hvwc_reg(window, VREG(TX_WCRED));
1153 creds = GET_FIELD(VAS_TX_WCRED, val);
1154 } else {
1155 val = read_hvwc_reg(window, VREG(LRX_WCRED));
1156 creds = GET_FIELD(VAS_LRX_WCRED, val);
1157 }
1158
1159 if (creds < window->wcreds_max) {
1160 val = 0;
1161 set_current_state(TASK_UNINTERRUPTIBLE);
1162 schedule_timeout(msecs_to_jiffies(10));
1163 goto retry;
1164 }
1165}
1166
1167
1168
1169
1170
1171
1172static void poll_window_busy_state(struct vas_window *window)
1173{
1174 int busy;
1175 u64 val;
1176
1177retry:
1178 val = read_hvwc_reg(window, VREG(WIN_STATUS));
1179 busy = GET_FIELD(VAS_WIN_BUSY, val);
1180 if (busy) {
1181 val = 0;
1182 set_current_state(TASK_UNINTERRUPTIBLE);
1183 schedule_timeout(msecs_to_jiffies(5));
1184 goto retry;
1185 }
1186}
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204static void poll_window_castout(struct vas_window *window)
1205{
1206
1207}
1208
1209
1210
1211
1212
1213static void unpin_close_window(struct vas_window *window)
1214{
1215 u64 val;
1216
1217 val = read_hvwc_reg(window, VREG(WINCTL));
1218 val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1219 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1220 write_hvwc_reg(window, VREG(WINCTL), val);
1221}
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235int vas_win_close(struct vas_window *window)
1236{
1237 if (!window)
1238 return 0;
1239
1240 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1241 pr_devel("Attempting to close an active Rx window!\n");
1242 WARN_ON_ONCE(1);
1243 return -EBUSY;
1244 }
1245
1246 unmap_paste_region(window);
1247
1248 clear_vinst_win(window);
1249
1250 poll_window_busy_state(window);
1251
1252 unpin_close_window(window);
1253
1254 poll_window_credits(window);
1255
1256 poll_window_castout(window);
1257
1258
1259 if (window->tx_win)
1260 put_rx_win(window->rxwin);
1261
1262 vas_window_free(window);
1263
1264 return 0;
1265}
1266EXPORT_SYMBOL_GPL(vas_win_close);
1267
1268
1269
1270
1271u32 vas_win_id(struct vas_window *win)
1272{
1273 return encode_pswid(win->vinst->vas_id, win->winid);
1274}
1275EXPORT_SYMBOL_GPL(vas_win_id);
1276