1
2
3
4
5
6
7
8
9
10
11#include <linux/ratelimit.h>
12#include <linux/mm.h>
13#include <linux/slab.h>
14#include <linux/iommu.h>
15#include <linux/vfio.h>
16#include <asm/idals.h>
17
18#include "vfio_ccw_cp.h"
19
20struct pfn_array {
21
22 unsigned long pa_iova;
23
24 unsigned long *pa_iova_pfn;
25
26 unsigned long *pa_pfn;
27
28 int pa_nr;
29};
30
31struct ccwchain {
32 struct list_head next;
33 struct ccw1 *ch_ccw;
34
35 u64 ch_iova;
36
37 int ch_len;
38
39 struct pfn_array *ch_pa;
40};
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59static int pfn_array_alloc(struct pfn_array *pa, u64 iova, unsigned int len)
60{
61 int i;
62
63 if (pa->pa_nr || pa->pa_iova_pfn)
64 return -EINVAL;
65
66 pa->pa_iova = iova;
67
68 pa->pa_nr = ((iova & ~PAGE_MASK) + len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
69 if (!pa->pa_nr)
70 return -EINVAL;
71
72 pa->pa_iova_pfn = kcalloc(pa->pa_nr,
73 sizeof(*pa->pa_iova_pfn) +
74 sizeof(*pa->pa_pfn),
75 GFP_KERNEL);
76 if (unlikely(!pa->pa_iova_pfn)) {
77 pa->pa_nr = 0;
78 return -ENOMEM;
79 }
80 pa->pa_pfn = pa->pa_iova_pfn + pa->pa_nr;
81
82 pa->pa_iova_pfn[0] = pa->pa_iova >> PAGE_SHIFT;
83 pa->pa_pfn[0] = -1ULL;
84 for (i = 1; i < pa->pa_nr; i++) {
85 pa->pa_iova_pfn[i] = pa->pa_iova_pfn[i - 1] + 1;
86 pa->pa_pfn[i] = -1ULL;
87 }
88
89 return 0;
90}
91
92
93
94
95
96
97
98
99
100
101static int pfn_array_pin(struct pfn_array *pa, struct device *mdev)
102{
103 int ret = 0;
104
105 ret = vfio_pin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr,
106 IOMMU_READ | IOMMU_WRITE, pa->pa_pfn);
107
108 if (ret < 0) {
109 goto err_out;
110 } else if (ret > 0 && ret != pa->pa_nr) {
111 vfio_unpin_pages(mdev, pa->pa_iova_pfn, ret);
112 ret = -EINVAL;
113 goto err_out;
114 }
115
116 return ret;
117
118err_out:
119 pa->pa_nr = 0;
120
121 return ret;
122}
123
124
125static void pfn_array_unpin_free(struct pfn_array *pa, struct device *mdev)
126{
127
128 if (pa->pa_nr)
129 vfio_unpin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr);
130 pa->pa_nr = 0;
131 kfree(pa->pa_iova_pfn);
132}
133
134static bool pfn_array_iova_pinned(struct pfn_array *pa, unsigned long iova)
135{
136 unsigned long iova_pfn = iova >> PAGE_SHIFT;
137 int i;
138
139 for (i = 0; i < pa->pa_nr; i++)
140 if (pa->pa_iova_pfn[i] == iova_pfn)
141 return true;
142
143 return false;
144}
145
146static inline void pfn_array_idal_create_words(
147 struct pfn_array *pa,
148 unsigned long *idaws)
149{
150 int i;
151
152
153
154
155
156
157
158
159
160 for (i = 0; i < pa->pa_nr; i++)
161 idaws[i] = pa->pa_pfn[i] << PAGE_SHIFT;
162
163
164 idaws[0] += pa->pa_iova & (PAGE_SIZE - 1);
165}
166
167static void convert_ccw0_to_ccw1(struct ccw1 *source, unsigned long len)
168{
169 struct ccw0 ccw0;
170 struct ccw1 *pccw1 = source;
171 int i;
172
173 for (i = 0; i < len; i++) {
174 ccw0 = *(struct ccw0 *)pccw1;
175 if ((pccw1->cmd_code & 0x0f) == CCW_CMD_TIC) {
176 pccw1->cmd_code = CCW_CMD_TIC;
177 pccw1->flags = 0;
178 pccw1->count = 0;
179 } else {
180 pccw1->cmd_code = ccw0.cmd_code;
181 pccw1->flags = ccw0.flags;
182 pccw1->count = ccw0.count;
183 }
184 pccw1->cda = ccw0.cda;
185 pccw1++;
186 }
187}
188
189
190
191
192
193static long copy_from_iova(struct device *mdev,
194 void *to, u64 iova,
195 unsigned long n)
196{
197 struct pfn_array pa = {0};
198 u64 from;
199 int i, ret;
200 unsigned long l, m;
201
202 ret = pfn_array_alloc(&pa, iova, n);
203 if (ret < 0)
204 return ret;
205
206 ret = pfn_array_pin(&pa, mdev);
207 if (ret < 0) {
208 pfn_array_unpin_free(&pa, mdev);
209 return ret;
210 }
211
212 l = n;
213 for (i = 0; i < pa.pa_nr; i++) {
214 from = pa.pa_pfn[i] << PAGE_SHIFT;
215 m = PAGE_SIZE;
216 if (i == 0) {
217 from += iova & (PAGE_SIZE - 1);
218 m -= iova & (PAGE_SIZE - 1);
219 }
220
221 m = min(l, m);
222 memcpy(to + (n - l), (void *)from, m);
223
224 l -= m;
225 if (l == 0)
226 break;
227 }
228
229 pfn_array_unpin_free(&pa, mdev);
230
231 return l;
232}
233
234
235
236
237#define ccw_is_read(_ccw) (((_ccw)->cmd_code & 0x03) == 0x02)
238#define ccw_is_read_backward(_ccw) (((_ccw)->cmd_code & 0x0F) == 0x0C)
239#define ccw_is_sense(_ccw) (((_ccw)->cmd_code & 0x0F) == CCW_CMD_BASIC_SENSE)
240
241#define ccw_is_noop(_ccw) ((_ccw)->cmd_code == CCW_CMD_NOOP)
242
243#define ccw_is_tic(_ccw) ((_ccw)->cmd_code == CCW_CMD_TIC)
244
245#define ccw_is_idal(_ccw) ((_ccw)->flags & CCW_FLAG_IDA)
246#define ccw_is_skip(_ccw) ((_ccw)->flags & CCW_FLAG_SKIP)
247
248#define ccw_is_chain(_ccw) ((_ccw)->flags & (CCW_FLAG_CC | CCW_FLAG_DC))
249
250
251
252
253
254
255
256
257
258static inline int ccw_does_data_transfer(struct ccw1 *ccw)
259{
260
261 if (ccw->count == 0)
262 return 0;
263
264
265 if (ccw_is_noop(ccw))
266 return 0;
267
268
269 if (!ccw_is_skip(ccw))
270 return 1;
271
272
273
274
275
276
277 if (ccw_is_read(ccw) || ccw_is_read_backward(ccw))
278 return 0;
279
280 if (ccw_is_sense(ccw))
281 return 0;
282
283
284 return 1;
285}
286
287
288
289
290
291
292
293
294
295
296
297
298
299static inline int is_cpa_within_range(u32 cpa, u32 head, int len)
300{
301 u32 tail = head + (len - 1) * sizeof(struct ccw1);
302
303 return (head <= cpa && cpa <= tail);
304}
305
306static inline int is_tic_within_range(struct ccw1 *ccw, u32 head, int len)
307{
308 if (!ccw_is_tic(ccw))
309 return 0;
310
311 return is_cpa_within_range(ccw->cda, head, len);
312}
313
314static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len)
315{
316 struct ccwchain *chain;
317 void *data;
318 size_t size;
319
320
321 size = ((sizeof(*chain) + 7L) & -8L) +
322 sizeof(*chain->ch_ccw) * len +
323 sizeof(*chain->ch_pa) * len;
324 chain = kzalloc(size, GFP_DMA | GFP_KERNEL);
325 if (!chain)
326 return NULL;
327
328 data = (u8 *)chain + ((sizeof(*chain) + 7L) & -8L);
329 chain->ch_ccw = (struct ccw1 *)data;
330
331 data = (u8 *)(chain->ch_ccw) + sizeof(*chain->ch_ccw) * len;
332 chain->ch_pa = (struct pfn_array *)data;
333
334 chain->ch_len = len;
335
336 list_add_tail(&chain->next, &cp->ccwchain_list);
337
338 return chain;
339}
340
341static void ccwchain_free(struct ccwchain *chain)
342{
343 list_del(&chain->next);
344 kfree(chain);
345}
346
347
348static void ccwchain_cda_free(struct ccwchain *chain, int idx)
349{
350 struct ccw1 *ccw = chain->ch_ccw + idx;
351
352 if (ccw_is_tic(ccw))
353 return;
354
355 kfree((void *)(u64)ccw->cda);
356}
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371static int ccwchain_calc_length(u64 iova, struct channel_program *cp)
372{
373 struct ccw1 *ccw = cp->guest_cp;
374 int cnt = 0;
375
376 do {
377 cnt++;
378
379
380
381
382
383
384 if ((!cp->orb.cmd.c64 || cp->orb.cmd.i2k) && ccw_is_idal(ccw))
385 return -EOPNOTSUPP;
386
387
388
389
390
391
392
393
394
395 if (!ccw_is_chain(ccw) && !is_tic_within_range(ccw, iova, cnt))
396 break;
397
398 ccw++;
399 } while (cnt < CCWCHAIN_LEN_MAX + 1);
400
401 if (cnt == CCWCHAIN_LEN_MAX + 1)
402 cnt = -EINVAL;
403
404 return cnt;
405}
406
407static int tic_target_chain_exists(struct ccw1 *tic, struct channel_program *cp)
408{
409 struct ccwchain *chain;
410 u32 ccw_head;
411
412 list_for_each_entry(chain, &cp->ccwchain_list, next) {
413 ccw_head = chain->ch_iova;
414 if (is_cpa_within_range(tic->cda, ccw_head, chain->ch_len))
415 return 1;
416 }
417
418 return 0;
419}
420
421static int ccwchain_loop_tic(struct ccwchain *chain,
422 struct channel_program *cp);
423
424static int ccwchain_handle_ccw(u32 cda, struct channel_program *cp)
425{
426 struct ccwchain *chain;
427 int len, ret;
428
429
430 len = copy_from_iova(cp->mdev, cp->guest_cp, cda,
431 CCWCHAIN_LEN_MAX * sizeof(struct ccw1));
432 if (len)
433 return len;
434
435
436 if (!cp->orb.cmd.fmt)
437 convert_ccw0_to_ccw1(cp->guest_cp, CCWCHAIN_LEN_MAX);
438
439
440 len = ccwchain_calc_length(cda, cp);
441 if (len < 0)
442 return len;
443
444
445 chain = ccwchain_alloc(cp, len);
446 if (!chain)
447 return -ENOMEM;
448 chain->ch_iova = cda;
449
450
451 memcpy(chain->ch_ccw, cp->guest_cp, len * sizeof(struct ccw1));
452
453
454 ret = ccwchain_loop_tic(chain, cp);
455
456 if (ret)
457 ccwchain_free(chain);
458
459 return ret;
460}
461
462
463static int ccwchain_loop_tic(struct ccwchain *chain, struct channel_program *cp)
464{
465 struct ccw1 *tic;
466 int i, ret;
467
468 for (i = 0; i < chain->ch_len; i++) {
469 tic = chain->ch_ccw + i;
470
471 if (!ccw_is_tic(tic))
472 continue;
473
474
475 if (tic_target_chain_exists(tic, cp))
476 continue;
477
478
479 ret = ccwchain_handle_ccw(tic->cda, cp);
480 if (ret)
481 return ret;
482 }
483
484 return 0;
485}
486
487static int ccwchain_fetch_tic(struct ccwchain *chain,
488 int idx,
489 struct channel_program *cp)
490{
491 struct ccw1 *ccw = chain->ch_ccw + idx;
492 struct ccwchain *iter;
493 u32 ccw_head;
494
495 list_for_each_entry(iter, &cp->ccwchain_list, next) {
496 ccw_head = iter->ch_iova;
497 if (is_cpa_within_range(ccw->cda, ccw_head, iter->ch_len)) {
498 ccw->cda = (__u32) (addr_t) (((char *)iter->ch_ccw) +
499 (ccw->cda - ccw_head));
500 return 0;
501 }
502 }
503
504 return -EFAULT;
505}
506
507static int ccwchain_fetch_direct(struct ccwchain *chain,
508 int idx,
509 struct channel_program *cp)
510{
511 struct ccw1 *ccw;
512 struct pfn_array *pa;
513 u64 iova;
514 unsigned long *idaws;
515 int ret;
516 int bytes = 1;
517 int idaw_nr, idal_len;
518 int i;
519
520 ccw = chain->ch_ccw + idx;
521
522 if (ccw->count)
523 bytes = ccw->count;
524
525
526 if (ccw_is_idal(ccw)) {
527
528
529 ret = copy_from_iova(cp->mdev, &iova, ccw->cda, sizeof(iova));
530 if (ret)
531 return ret;
532 } else {
533 iova = ccw->cda;
534 }
535 idaw_nr = idal_nr_words((void *)iova, bytes);
536 idal_len = idaw_nr * sizeof(*idaws);
537
538
539 idaws = kcalloc(idaw_nr, sizeof(*idaws), GFP_DMA | GFP_KERNEL);
540 if (!idaws) {
541 ret = -ENOMEM;
542 goto out_init;
543 }
544
545
546
547
548
549
550
551 pa = chain->ch_pa + idx;
552 ret = pfn_array_alloc(pa, iova, bytes);
553 if (ret < 0)
554 goto out_free_idaws;
555
556 if (ccw_is_idal(ccw)) {
557
558 ret = copy_from_iova(cp->mdev, idaws, ccw->cda, idal_len);
559 if (ret)
560 goto out_unpin;
561
562
563
564
565
566 for (i = 0; i < idaw_nr; i++)
567 pa->pa_iova_pfn[i] = idaws[i] >> PAGE_SHIFT;
568 } else {
569
570
571
572
573
574 }
575
576 if (ccw_does_data_transfer(ccw)) {
577 ret = pfn_array_pin(pa, cp->mdev);
578 if (ret < 0)
579 goto out_unpin;
580 } else {
581 pa->pa_nr = 0;
582 }
583
584 ccw->cda = (__u32) virt_to_phys(idaws);
585 ccw->flags |= CCW_FLAG_IDA;
586
587
588 pfn_array_idal_create_words(pa, idaws);
589
590 return 0;
591
592out_unpin:
593 pfn_array_unpin_free(pa, cp->mdev);
594out_free_idaws:
595 kfree(idaws);
596out_init:
597 ccw->cda = 0;
598 return ret;
599}
600
601
602
603
604
605
606
607static int ccwchain_fetch_one(struct ccwchain *chain,
608 int idx,
609 struct channel_program *cp)
610{
611 struct ccw1 *ccw = chain->ch_ccw + idx;
612
613 if (ccw_is_tic(ccw))
614 return ccwchain_fetch_tic(chain, idx, cp);
615
616 return ccwchain_fetch_direct(chain, idx, cp);
617}
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb)
636{
637
638 static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 1);
639 int ret;
640
641
642
643
644
645
646
647
648 if (!orb->cmd.pfch && __ratelimit(&ratelimit_state))
649 dev_warn(mdev, "Prefetching channel program even though prefetch not specified in ORB");
650
651 INIT_LIST_HEAD(&cp->ccwchain_list);
652 memcpy(&cp->orb, orb, sizeof(*orb));
653 cp->mdev = mdev;
654
655
656 ret = ccwchain_handle_ccw(orb->cmd.cpa, cp);
657
658 if (!ret) {
659 cp->initialized = true;
660
661
662
663
664 cp->orb.cmd.c64 = 1;
665 }
666
667 return ret;
668}
669
670
671
672
673
674
675
676
677
678
679void cp_free(struct channel_program *cp)
680{
681 struct ccwchain *chain, *temp;
682 int i;
683
684 if (!cp->initialized)
685 return;
686
687 cp->initialized = false;
688 list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {
689 for (i = 0; i < chain->ch_len; i++) {
690 pfn_array_unpin_free(chain->ch_pa + i, cp->mdev);
691 ccwchain_cda_free(chain, i);
692 }
693 ccwchain_free(chain);
694 }
695}
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734int cp_prefetch(struct channel_program *cp)
735{
736 struct ccwchain *chain;
737 int len, idx, ret;
738
739
740 if (!cp->initialized)
741 return -EINVAL;
742
743 list_for_each_entry(chain, &cp->ccwchain_list, next) {
744 len = chain->ch_len;
745 for (idx = 0; idx < len; idx++) {
746 ret = ccwchain_fetch_one(chain, idx, cp);
747 if (ret)
748 goto out_err;
749 }
750 }
751
752 return 0;
753out_err:
754
755 chain->ch_len = idx;
756 list_for_each_entry_continue(chain, &cp->ccwchain_list, next) {
757 chain->ch_len = 0;
758 }
759 return ret;
760}
761
762
763
764
765
766
767
768
769
770
771
772union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8 lpm)
773{
774 union orb *orb;
775 struct ccwchain *chain;
776 struct ccw1 *cpa;
777
778
779 if (!cp->initialized)
780 return NULL;
781
782 orb = &cp->orb;
783
784 orb->cmd.intparm = intparm;
785 orb->cmd.fmt = 1;
786 orb->cmd.key = PAGE_DEFAULT_KEY >> 4;
787
788 if (orb->cmd.lpm == 0)
789 orb->cmd.lpm = lpm;
790
791 chain = list_first_entry(&cp->ccwchain_list, struct ccwchain, next);
792 cpa = chain->ch_ccw;
793 orb->cmd.cpa = (__u32) __pa(cpa);
794
795 return orb;
796}
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812void cp_update_scsw(struct channel_program *cp, union scsw *scsw)
813{
814 struct ccwchain *chain;
815 u32 cpa = scsw->cmd.cpa;
816 u32 ccw_head;
817
818 if (!cp->initialized)
819 return;
820
821
822
823
824
825
826
827 list_for_each_entry(chain, &cp->ccwchain_list, next) {
828 ccw_head = (u32)(u64)chain->ch_ccw;
829
830
831
832
833 if (is_cpa_within_range(cpa, ccw_head, chain->ch_len + 1)) {
834
835
836
837
838
839
840 cpa = chain->ch_iova + (cpa - ccw_head);
841 break;
842 }
843 }
844
845 scsw->cmd.cpa = cpa;
846}
847
848
849
850
851
852
853
854
855
856bool cp_iova_pinned(struct channel_program *cp, u64 iova)
857{
858 struct ccwchain *chain;
859 int i;
860
861 if (!cp->initialized)
862 return false;
863
864 list_for_each_entry(chain, &cp->ccwchain_list, next) {
865 for (i = 0; i < chain->ch_len; i++)
866 if (pfn_array_iova_pinned(chain->ch_pa + i, iova))
867 return true;
868 }
869
870 return false;
871}
872