1
2
3
4
5
6
7
8
9
10
11#include <linux/ratelimit.h>
12#include <linux/mm.h>
13#include <linux/slab.h>
14#include <linux/iommu.h>
15#include <linux/vfio.h>
16#include <asm/idals.h>
17
18#include "vfio_ccw_cp.h"
19
20struct pfn_array {
21
22 unsigned long pa_iova;
23
24 unsigned long *pa_iova_pfn;
25
26 unsigned long *pa_pfn;
27
28 int pa_nr;
29};
30
31struct ccwchain {
32 struct list_head next;
33 struct ccw1 *ch_ccw;
34
35 u64 ch_iova;
36
37 int ch_len;
38
39 struct pfn_array *ch_pa;
40};
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59static int pfn_array_alloc(struct pfn_array *pa, u64 iova, unsigned int len)
60{
61 int i;
62
63 if (pa->pa_nr || pa->pa_iova_pfn)
64 return -EINVAL;
65
66 pa->pa_iova = iova;
67
68 pa->pa_nr = ((iova & ~PAGE_MASK) + len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
69 if (!pa->pa_nr)
70 return -EINVAL;
71
72 pa->pa_iova_pfn = kcalloc(pa->pa_nr,
73 sizeof(*pa->pa_iova_pfn) +
74 sizeof(*pa->pa_pfn),
75 GFP_KERNEL);
76 if (unlikely(!pa->pa_iova_pfn)) {
77 pa->pa_nr = 0;
78 return -ENOMEM;
79 }
80 pa->pa_pfn = pa->pa_iova_pfn + pa->pa_nr;
81
82 pa->pa_iova_pfn[0] = pa->pa_iova >> PAGE_SHIFT;
83 pa->pa_pfn[0] = -1ULL;
84 for (i = 1; i < pa->pa_nr; i++) {
85 pa->pa_iova_pfn[i] = pa->pa_iova_pfn[i - 1] + 1;
86 pa->pa_pfn[i] = -1ULL;
87 }
88
89 return 0;
90}
91
92
93
94
95
96
97
98
99
100
101static int pfn_array_pin(struct pfn_array *pa, struct device *mdev)
102{
103 int ret = 0;
104
105 ret = vfio_pin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr,
106 IOMMU_READ | IOMMU_WRITE, pa->pa_pfn);
107
108 if (ret < 0) {
109 goto err_out;
110 } else if (ret > 0 && ret != pa->pa_nr) {
111 vfio_unpin_pages(mdev, pa->pa_iova_pfn, ret);
112 ret = -EINVAL;
113 goto err_out;
114 }
115
116 return ret;
117
118err_out:
119 pa->pa_nr = 0;
120
121 return ret;
122}
123
124
125static void pfn_array_unpin_free(struct pfn_array *pa, struct device *mdev)
126{
127
128 if (pa->pa_nr)
129 vfio_unpin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr);
130 pa->pa_nr = 0;
131 kfree(pa->pa_iova_pfn);
132}
133
134static bool pfn_array_iova_pinned(struct pfn_array *pa, unsigned long iova)
135{
136 unsigned long iova_pfn = iova >> PAGE_SHIFT;
137 int i;
138
139 for (i = 0; i < pa->pa_nr; i++)
140 if (pa->pa_iova_pfn[i] == iova_pfn)
141 return true;
142
143 return false;
144}
145
146static inline void pfn_array_idal_create_words(
147 struct pfn_array *pa,
148 unsigned long *idaws)
149{
150 int i;
151
152
153
154
155
156
157
158
159
160 for (i = 0; i < pa->pa_nr; i++)
161 idaws[i] = pa->pa_pfn[i] << PAGE_SHIFT;
162
163
164 idaws[0] += pa->pa_iova & (PAGE_SIZE - 1);
165}
166
167static void convert_ccw0_to_ccw1(struct ccw1 *source, unsigned long len)
168{
169 struct ccw0 ccw0;
170 struct ccw1 *pccw1 = source;
171 int i;
172
173 for (i = 0; i < len; i++) {
174 ccw0 = *(struct ccw0 *)pccw1;
175 if ((pccw1->cmd_code & 0x0f) == CCW_CMD_TIC) {
176 pccw1->cmd_code = CCW_CMD_TIC;
177 pccw1->flags = 0;
178 pccw1->count = 0;
179 } else {
180 pccw1->cmd_code = ccw0.cmd_code;
181 pccw1->flags = ccw0.flags;
182 pccw1->count = ccw0.count;
183 }
184 pccw1->cda = ccw0.cda;
185 pccw1++;
186 }
187}
188
189
190
191
192
193static long copy_from_iova(struct device *mdev,
194 void *to, u64 iova,
195 unsigned long n)
196{
197 struct pfn_array pa = {0};
198 u64 from;
199 int i, ret;
200 unsigned long l, m;
201
202 ret = pfn_array_alloc(&pa, iova, n);
203 if (ret < 0)
204 return ret;
205
206 ret = pfn_array_pin(&pa, mdev);
207 if (ret < 0) {
208 pfn_array_unpin_free(&pa, mdev);
209 return ret;
210 }
211
212 l = n;
213 for (i = 0; i < pa.pa_nr; i++) {
214 from = pa.pa_pfn[i] << PAGE_SHIFT;
215 m = PAGE_SIZE;
216 if (i == 0) {
217 from += iova & (PAGE_SIZE - 1);
218 m -= iova & (PAGE_SIZE - 1);
219 }
220
221 m = min(l, m);
222 memcpy(to + (n - l), (void *)from, m);
223
224 l -= m;
225 if (l == 0)
226 break;
227 }
228
229 pfn_array_unpin_free(&pa, mdev);
230
231 return l;
232}
233
234
235
236
237#define ccw_is_read(_ccw) (((_ccw)->cmd_code & 0x03) == 0x02)
238#define ccw_is_read_backward(_ccw) (((_ccw)->cmd_code & 0x0F) == 0x0C)
239#define ccw_is_sense(_ccw) (((_ccw)->cmd_code & 0x0F) == CCW_CMD_BASIC_SENSE)
240
241#define ccw_is_noop(_ccw) ((_ccw)->cmd_code == CCW_CMD_NOOP)
242
243#define ccw_is_tic(_ccw) ((_ccw)->cmd_code == CCW_CMD_TIC)
244
245#define ccw_is_idal(_ccw) ((_ccw)->flags & CCW_FLAG_IDA)
246#define ccw_is_skip(_ccw) ((_ccw)->flags & CCW_FLAG_SKIP)
247
248#define ccw_is_chain(_ccw) ((_ccw)->flags & (CCW_FLAG_CC | CCW_FLAG_DC))
249
250
251
252
253
254
255
256
257
258static inline int ccw_does_data_transfer(struct ccw1 *ccw)
259{
260
261 if (ccw->count == 0)
262 return 0;
263
264
265 if (ccw_is_noop(ccw))
266 return 0;
267
268
269 if (!ccw_is_skip(ccw))
270 return 1;
271
272
273
274
275
276
277 if (ccw_is_read(ccw) || ccw_is_read_backward(ccw))
278 return 0;
279
280 if (ccw_is_sense(ccw))
281 return 0;
282
283
284 return 1;
285}
286
287
288
289
290
291
292
293
294
295
296
297
298
299static inline int is_cpa_within_range(u32 cpa, u32 head, int len)
300{
301 u32 tail = head + (len - 1) * sizeof(struct ccw1);
302
303 return (head <= cpa && cpa <= tail);
304}
305
306static inline int is_tic_within_range(struct ccw1 *ccw, u32 head, int len)
307{
308 if (!ccw_is_tic(ccw))
309 return 0;
310
311 return is_cpa_within_range(ccw->cda, head, len);
312}
313
314static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len)
315{
316 struct ccwchain *chain;
317 void *data;
318 size_t size;
319
320
321 size = ((sizeof(*chain) + 7L) & -8L) +
322 sizeof(*chain->ch_ccw) * len +
323 sizeof(*chain->ch_pa) * len;
324 chain = kzalloc(size, GFP_DMA | GFP_KERNEL);
325 if (!chain)
326 return NULL;
327
328 data = (u8 *)chain + ((sizeof(*chain) + 7L) & -8L);
329 chain->ch_ccw = (struct ccw1 *)data;
330
331 data = (u8 *)(chain->ch_ccw) + sizeof(*chain->ch_ccw) * len;
332 chain->ch_pa = (struct pfn_array *)data;
333
334 chain->ch_len = len;
335
336 list_add_tail(&chain->next, &cp->ccwchain_list);
337
338 return chain;
339}
340
341static void ccwchain_free(struct ccwchain *chain)
342{
343 list_del(&chain->next);
344 kfree(chain);
345}
346
347
348static void ccwchain_cda_free(struct ccwchain *chain, int idx)
349{
350 struct ccw1 *ccw = chain->ch_ccw + idx;
351
352 if (ccw_is_tic(ccw))
353 return;
354
355 kfree((void *)(u64)ccw->cda);
356}
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371static int ccwchain_calc_length(u64 iova, struct channel_program *cp)
372{
373 struct ccw1 *ccw = cp->guest_cp;
374 int cnt = 0;
375
376 do {
377 cnt++;
378
379
380
381
382
383
384 if ((!cp->orb.cmd.c64 || cp->orb.cmd.i2k) && ccw_is_idal(ccw))
385 return -EOPNOTSUPP;
386
387
388
389
390
391
392
393
394
395 if (!ccw_is_chain(ccw) && !is_tic_within_range(ccw, iova, cnt))
396 break;
397
398 ccw++;
399 } while (cnt < CCWCHAIN_LEN_MAX + 1);
400
401 if (cnt == CCWCHAIN_LEN_MAX + 1)
402 cnt = -EINVAL;
403
404 return cnt;
405}
406
407static int tic_target_chain_exists(struct ccw1 *tic, struct channel_program *cp)
408{
409 struct ccwchain *chain;
410 u32 ccw_head;
411
412 list_for_each_entry(chain, &cp->ccwchain_list, next) {
413 ccw_head = chain->ch_iova;
414 if (is_cpa_within_range(tic->cda, ccw_head, chain->ch_len))
415 return 1;
416 }
417
418 return 0;
419}
420
421static int ccwchain_loop_tic(struct ccwchain *chain,
422 struct channel_program *cp);
423
424static int ccwchain_handle_ccw(u32 cda, struct channel_program *cp)
425{
426 struct ccwchain *chain;
427 int len, ret;
428
429
430 len = copy_from_iova(cp->mdev, cp->guest_cp, cda,
431 CCWCHAIN_LEN_MAX * sizeof(struct ccw1));
432 if (len)
433 return len;
434
435
436 if (!cp->orb.cmd.fmt)
437 convert_ccw0_to_ccw1(cp->guest_cp, CCWCHAIN_LEN_MAX);
438
439
440 len = ccwchain_calc_length(cda, cp);
441 if (len < 0)
442 return len;
443
444
445 chain = ccwchain_alloc(cp, len);
446 if (!chain)
447 return -ENOMEM;
448 chain->ch_iova = cda;
449
450
451 memcpy(chain->ch_ccw, cp->guest_cp, len * sizeof(struct ccw1));
452
453
454 ret = ccwchain_loop_tic(chain, cp);
455
456 if (ret)
457 ccwchain_free(chain);
458
459 return ret;
460}
461
462
463static int ccwchain_loop_tic(struct ccwchain *chain, struct channel_program *cp)
464{
465 struct ccw1 *tic;
466 int i, ret;
467
468 for (i = 0; i < chain->ch_len; i++) {
469 tic = chain->ch_ccw + i;
470
471 if (!ccw_is_tic(tic))
472 continue;
473
474
475 if (tic_target_chain_exists(tic, cp))
476 continue;
477
478
479 ret = ccwchain_handle_ccw(tic->cda, cp);
480 if (ret)
481 return ret;
482 }
483
484 return 0;
485}
486
487static int ccwchain_fetch_tic(struct ccwchain *chain,
488 int idx,
489 struct channel_program *cp)
490{
491 struct ccw1 *ccw = chain->ch_ccw + idx;
492 struct ccwchain *iter;
493 u32 ccw_head;
494
495 list_for_each_entry(iter, &cp->ccwchain_list, next) {
496 ccw_head = iter->ch_iova;
497 if (is_cpa_within_range(ccw->cda, ccw_head, iter->ch_len)) {
498 ccw->cda = (__u32) (addr_t) (((char *)iter->ch_ccw) +
499 (ccw->cda - ccw_head));
500 return 0;
501 }
502 }
503
504 return -EFAULT;
505}
506
507static int ccwchain_fetch_direct(struct ccwchain *chain,
508 int idx,
509 struct channel_program *cp)
510{
511 struct ccw1 *ccw;
512 struct pfn_array *pa;
513 u64 iova;
514 unsigned long *idaws;
515 int ret;
516 int bytes = 1;
517 int idaw_nr, idal_len;
518 int i;
519
520 ccw = chain->ch_ccw + idx;
521
522 if (ccw->count)
523 bytes = ccw->count;
524
525
526 if (ccw_is_idal(ccw)) {
527
528
529 ret = copy_from_iova(cp->mdev, &iova, ccw->cda, sizeof(iova));
530 if (ret)
531 return ret;
532 } else {
533 iova = ccw->cda;
534 }
535 idaw_nr = idal_nr_words((void *)iova, bytes);
536 idal_len = idaw_nr * sizeof(*idaws);
537
538
539 idaws = kcalloc(idaw_nr, sizeof(*idaws), GFP_DMA | GFP_KERNEL);
540 if (!idaws) {
541 ret = -ENOMEM;
542 goto out_init;
543 }
544
545
546
547
548
549
550
551 pa = chain->ch_pa + idx;
552 ret = pfn_array_alloc(pa, iova, bytes);
553 if (ret < 0)
554 goto out_free_idaws;
555
556 if (ccw_is_idal(ccw)) {
557
558 ret = copy_from_iova(cp->mdev, idaws, ccw->cda, idal_len);
559 if (ret)
560 goto out_unpin;
561
562
563
564
565
566 for (i = 0; i < idaw_nr; i++)
567 pa->pa_iova_pfn[i] = idaws[i] >> PAGE_SHIFT;
568 } else {
569
570
571
572
573
574 }
575
576 if (ccw_does_data_transfer(ccw)) {
577 ret = pfn_array_pin(pa, cp->mdev);
578 if (ret < 0)
579 goto out_unpin;
580 } else {
581 pa->pa_nr = 0;
582 }
583
584 ccw->cda = (__u32) virt_to_phys(idaws);
585 ccw->flags |= CCW_FLAG_IDA;
586
587
588 pfn_array_idal_create_words(pa, idaws);
589
590 return 0;
591
592out_unpin:
593 pfn_array_unpin_free(pa, cp->mdev);
594out_free_idaws:
595 kfree(idaws);
596out_init:
597 ccw->cda = 0;
598 return ret;
599}
600
601
602
603
604
605
606
607static int ccwchain_fetch_one(struct ccwchain *chain,
608 int idx,
609 struct channel_program *cp)
610{
611 struct ccw1 *ccw = chain->ch_ccw + idx;
612
613 if (ccw_is_tic(ccw))
614 return ccwchain_fetch_tic(chain, idx, cp);
615
616 return ccwchain_fetch_direct(chain, idx, cp);
617}
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb)
636{
637
638 static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 1);
639 int ret;
640
641
642 if (cp->initialized)
643 return -EBUSY;
644
645
646
647
648
649
650
651
652 if (!orb->cmd.pfch && __ratelimit(&ratelimit_state))
653 dev_warn(mdev, "Prefetching channel program even though prefetch not specified in ORB");
654
655 INIT_LIST_HEAD(&cp->ccwchain_list);
656 memcpy(&cp->orb, orb, sizeof(*orb));
657 cp->mdev = mdev;
658
659
660 ret = ccwchain_handle_ccw(orb->cmd.cpa, cp);
661
662 if (!ret) {
663 cp->initialized = true;
664
665
666
667
668 cp->orb.cmd.c64 = 1;
669 }
670
671 return ret;
672}
673
674
675
676
677
678
679
680
681
682
683void cp_free(struct channel_program *cp)
684{
685 struct ccwchain *chain, *temp;
686 int i;
687
688 if (!cp->initialized)
689 return;
690
691 cp->initialized = false;
692 list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {
693 for (i = 0; i < chain->ch_len; i++) {
694 pfn_array_unpin_free(chain->ch_pa + i, cp->mdev);
695 ccwchain_cda_free(chain, i);
696 }
697 ccwchain_free(chain);
698 }
699}
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738int cp_prefetch(struct channel_program *cp)
739{
740 struct ccwchain *chain;
741 int len, idx, ret;
742
743
744 if (!cp->initialized)
745 return -EINVAL;
746
747 list_for_each_entry(chain, &cp->ccwchain_list, next) {
748 len = chain->ch_len;
749 for (idx = 0; idx < len; idx++) {
750 ret = ccwchain_fetch_one(chain, idx, cp);
751 if (ret)
752 goto out_err;
753 }
754 }
755
756 return 0;
757out_err:
758
759 chain->ch_len = idx;
760 list_for_each_entry_continue(chain, &cp->ccwchain_list, next) {
761 chain->ch_len = 0;
762 }
763 return ret;
764}
765
766
767
768
769
770
771
772
773
774
775
776union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8 lpm)
777{
778 union orb *orb;
779 struct ccwchain *chain;
780 struct ccw1 *cpa;
781
782
783 if (!cp->initialized)
784 return NULL;
785
786 orb = &cp->orb;
787
788 orb->cmd.intparm = intparm;
789 orb->cmd.fmt = 1;
790 orb->cmd.key = PAGE_DEFAULT_KEY >> 4;
791
792 if (orb->cmd.lpm == 0)
793 orb->cmd.lpm = lpm;
794
795 chain = list_first_entry(&cp->ccwchain_list, struct ccwchain, next);
796 cpa = chain->ch_ccw;
797 orb->cmd.cpa = (__u32) __pa(cpa);
798
799 return orb;
800}
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816void cp_update_scsw(struct channel_program *cp, union scsw *scsw)
817{
818 struct ccwchain *chain;
819 u32 cpa = scsw->cmd.cpa;
820 u32 ccw_head;
821
822 if (!cp->initialized)
823 return;
824
825
826
827
828
829
830
831 list_for_each_entry(chain, &cp->ccwchain_list, next) {
832 ccw_head = (u32)(u64)chain->ch_ccw;
833
834
835
836
837 if (is_cpa_within_range(cpa, ccw_head, chain->ch_len + 1)) {
838
839
840
841
842
843
844 cpa = chain->ch_iova + (cpa - ccw_head);
845 break;
846 }
847 }
848
849 scsw->cmd.cpa = cpa;
850}
851
852
853
854
855
856
857
858
859
860bool cp_iova_pinned(struct channel_program *cp, u64 iova)
861{
862 struct ccwchain *chain;
863 int i;
864
865 if (!cp->initialized)
866 return false;
867
868 list_for_each_entry(chain, &cp->ccwchain_list, next) {
869 for (i = 0; i < chain->ch_len; i++)
870 if (pfn_array_iova_pinned(chain->ch_pa + i, iova))
871 return true;
872 }
873
874 return false;
875}
876