1
2
3
4
5
6
7
8#include <linux/vmalloc.h>
9#include <linux/err.h>
10#include <asm/pgtable.h>
11#include "kvm-s390.h"
12#include "gaccess.h"
13
14union asce {
15 unsigned long val;
16 struct {
17 unsigned long origin : 52;
18 unsigned long : 2;
19 unsigned long g : 1;
20 unsigned long p : 1;
21 unsigned long s : 1;
22 unsigned long x : 1;
23 unsigned long r : 1;
24 unsigned long : 1;
25 unsigned long dt : 2;
26 unsigned long tl : 2;
27 };
28};
29
30enum {
31 ASCE_TYPE_SEGMENT = 0,
32 ASCE_TYPE_REGION3 = 1,
33 ASCE_TYPE_REGION2 = 2,
34 ASCE_TYPE_REGION1 = 3
35};
36
37union region1_table_entry {
38 unsigned long val;
39 struct {
40 unsigned long rto: 52;
41 unsigned long : 2;
42 unsigned long p : 1;
43 unsigned long : 1;
44 unsigned long tf : 2;
45 unsigned long i : 1;
46 unsigned long : 1;
47 unsigned long tt : 2;
48 unsigned long tl : 2;
49 };
50};
51
52union region2_table_entry {
53 unsigned long val;
54 struct {
55 unsigned long rto: 52;
56 unsigned long : 2;
57 unsigned long p : 1;
58 unsigned long : 1;
59 unsigned long tf : 2;
60 unsigned long i : 1;
61 unsigned long : 1;
62 unsigned long tt : 2;
63 unsigned long tl : 2;
64 };
65};
66
67struct region3_table_entry_fc0 {
68 unsigned long sto: 52;
69 unsigned long : 1;
70 unsigned long fc : 1;
71 unsigned long p : 1;
72 unsigned long : 1;
73 unsigned long tf : 2;
74 unsigned long i : 1;
75 unsigned long cr : 1;
76 unsigned long tt : 2;
77 unsigned long tl : 2;
78};
79
80struct region3_table_entry_fc1 {
81 unsigned long rfaa : 33;
82 unsigned long : 14;
83 unsigned long av : 1;
84 unsigned long acc: 4;
85 unsigned long f : 1;
86 unsigned long fc : 1;
87 unsigned long p : 1;
88 unsigned long co : 1;
89 unsigned long : 2;
90 unsigned long i : 1;
91 unsigned long cr : 1;
92 unsigned long tt : 2;
93 unsigned long : 2;
94};
95
96union region3_table_entry {
97 unsigned long val;
98 struct region3_table_entry_fc0 fc0;
99 struct region3_table_entry_fc1 fc1;
100 struct {
101 unsigned long : 53;
102 unsigned long fc : 1;
103 unsigned long : 4;
104 unsigned long i : 1;
105 unsigned long cr : 1;
106 unsigned long tt : 2;
107 unsigned long : 2;
108 };
109};
110
111struct segment_entry_fc0 {
112 unsigned long pto: 53;
113 unsigned long fc : 1;
114 unsigned long p : 1;
115 unsigned long : 3;
116 unsigned long i : 1;
117 unsigned long cs : 1;
118 unsigned long tt : 2;
119 unsigned long : 2;
120};
121
122struct segment_entry_fc1 {
123 unsigned long sfaa : 44;
124 unsigned long : 3;
125 unsigned long av : 1;
126 unsigned long acc: 4;
127 unsigned long f : 1;
128 unsigned long fc : 1;
129 unsigned long p : 1;
130 unsigned long co : 1;
131 unsigned long : 2;
132 unsigned long i : 1;
133 unsigned long cs : 1;
134 unsigned long tt : 2;
135 unsigned long : 2;
136};
137
138union segment_table_entry {
139 unsigned long val;
140 struct segment_entry_fc0 fc0;
141 struct segment_entry_fc1 fc1;
142 struct {
143 unsigned long : 53;
144 unsigned long fc : 1;
145 unsigned long : 4;
146 unsigned long i : 1;
147 unsigned long cs : 1;
148 unsigned long tt : 2;
149 unsigned long : 2;
150 };
151};
152
153enum {
154 TABLE_TYPE_SEGMENT = 0,
155 TABLE_TYPE_REGION3 = 1,
156 TABLE_TYPE_REGION2 = 2,
157 TABLE_TYPE_REGION1 = 3
158};
159
160union page_table_entry {
161 unsigned long val;
162 struct {
163 unsigned long pfra : 52;
164 unsigned long z : 1;
165 unsigned long i : 1;
166 unsigned long p : 1;
167 unsigned long co : 1;
168 unsigned long : 8;
169 };
170};
171
172
173
174
175
176union vaddress {
177 unsigned long addr;
178 struct {
179 unsigned long rfx : 11;
180 unsigned long rsx : 11;
181 unsigned long rtx : 11;
182 unsigned long sx : 11;
183 unsigned long px : 8;
184 unsigned long bx : 12;
185 };
186 struct {
187 unsigned long rfx01 : 2;
188 unsigned long : 9;
189 unsigned long rsx01 : 2;
190 unsigned long : 9;
191 unsigned long rtx01 : 2;
192 unsigned long : 9;
193 unsigned long sx01 : 2;
194 unsigned long : 29;
195 };
196};
197
198
199
200
201
202
203union raddress {
204 unsigned long addr;
205 unsigned long rfaa : 33;
206 unsigned long sfaa : 44;
207 unsigned long pfra : 52;
208};
209
210
211int ipte_lock_held(struct kvm_vcpu *vcpu)
212{
213 union ipte_control *ic = &vcpu->kvm->arch.sca->ipte_control;
214
215 if (vcpu->arch.sie_block->eca & 1)
216 return ic->kh != 0;
217 return vcpu->kvm->arch.ipte_lock_count != 0;
218}
219
220static void ipte_lock_simple(struct kvm_vcpu *vcpu)
221{
222 union ipte_control old, new, *ic;
223
224 mutex_lock(&vcpu->kvm->arch.ipte_mutex);
225 vcpu->kvm->arch.ipte_lock_count++;
226 if (vcpu->kvm->arch.ipte_lock_count > 1)
227 goto out;
228 ic = &vcpu->kvm->arch.sca->ipte_control;
229 do {
230 old = READ_ONCE(*ic);
231 while (old.k) {
232 cond_resched();
233 old = READ_ONCE(*ic);
234 }
235 new = old;
236 new.k = 1;
237 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
238out:
239 mutex_unlock(&vcpu->kvm->arch.ipte_mutex);
240}
241
242static void ipte_unlock_simple(struct kvm_vcpu *vcpu)
243{
244 union ipte_control old, new, *ic;
245
246 mutex_lock(&vcpu->kvm->arch.ipte_mutex);
247 vcpu->kvm->arch.ipte_lock_count--;
248 if (vcpu->kvm->arch.ipte_lock_count)
249 goto out;
250 ic = &vcpu->kvm->arch.sca->ipte_control;
251 do {
252 old = READ_ONCE(*ic);
253 new = old;
254 new.k = 0;
255 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
256 wake_up(&vcpu->kvm->arch.ipte_wq);
257out:
258 mutex_unlock(&vcpu->kvm->arch.ipte_mutex);
259}
260
261static void ipte_lock_siif(struct kvm_vcpu *vcpu)
262{
263 union ipte_control old, new, *ic;
264
265 ic = &vcpu->kvm->arch.sca->ipte_control;
266 do {
267 old = READ_ONCE(*ic);
268 while (old.kg) {
269 cond_resched();
270 old = READ_ONCE(*ic);
271 }
272 new = old;
273 new.k = 1;
274 new.kh++;
275 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
276}
277
278static void ipte_unlock_siif(struct kvm_vcpu *vcpu)
279{
280 union ipte_control old, new, *ic;
281
282 ic = &vcpu->kvm->arch.sca->ipte_control;
283 do {
284 old = READ_ONCE(*ic);
285 new = old;
286 new.kh--;
287 if (!new.kh)
288 new.k = 0;
289 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
290 if (!new.kh)
291 wake_up(&vcpu->kvm->arch.ipte_wq);
292}
293
294void ipte_lock(struct kvm_vcpu *vcpu)
295{
296 if (vcpu->arch.sie_block->eca & 1)
297 ipte_lock_siif(vcpu);
298 else
299 ipte_lock_simple(vcpu);
300}
301
302void ipte_unlock(struct kvm_vcpu *vcpu)
303{
304 if (vcpu->arch.sie_block->eca & 1)
305 ipte_unlock_siif(vcpu);
306 else
307 ipte_unlock_simple(vcpu);
308}
309
310static unsigned long get_vcpu_asce(struct kvm_vcpu *vcpu)
311{
312 switch (psw_bits(vcpu->arch.sie_block->gpsw).as) {
313 case PSW_AS_PRIMARY:
314 return vcpu->arch.sie_block->gcr[1];
315 case PSW_AS_SECONDARY:
316 return vcpu->arch.sie_block->gcr[7];
317 case PSW_AS_HOME:
318 return vcpu->arch.sie_block->gcr[13];
319 }
320 return 0;
321}
322
323static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
324{
325 return kvm_read_guest(kvm, gpa, val, sizeof(*val));
326}
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
348 unsigned long *gpa, int write)
349{
350 union vaddress vaddr = {.addr = gva};
351 union raddress raddr = {.addr = gva};
352 union page_table_entry pte;
353 int dat_protection = 0;
354 union ctlreg0 ctlreg0;
355 unsigned long ptr;
356 int edat1, edat2;
357 union asce asce;
358
359 ctlreg0.val = vcpu->arch.sie_block->gcr[0];
360 edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
361 edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
362 asce.val = get_vcpu_asce(vcpu);
363 if (asce.r)
364 goto real_address;
365 ptr = asce.origin * 4096;
366 switch (asce.dt) {
367 case ASCE_TYPE_REGION1:
368 if (vaddr.rfx01 > asce.tl)
369 return PGM_REGION_FIRST_TRANS;
370 ptr += vaddr.rfx * 8;
371 break;
372 case ASCE_TYPE_REGION2:
373 if (vaddr.rfx)
374 return PGM_ASCE_TYPE;
375 if (vaddr.rsx01 > asce.tl)
376 return PGM_REGION_SECOND_TRANS;
377 ptr += vaddr.rsx * 8;
378 break;
379 case ASCE_TYPE_REGION3:
380 if (vaddr.rfx || vaddr.rsx)
381 return PGM_ASCE_TYPE;
382 if (vaddr.rtx01 > asce.tl)
383 return PGM_REGION_THIRD_TRANS;
384 ptr += vaddr.rtx * 8;
385 break;
386 case ASCE_TYPE_SEGMENT:
387 if (vaddr.rfx || vaddr.rsx || vaddr.rtx)
388 return PGM_ASCE_TYPE;
389 if (vaddr.sx01 > asce.tl)
390 return PGM_SEGMENT_TRANSLATION;
391 ptr += vaddr.sx * 8;
392 break;
393 }
394 switch (asce.dt) {
395 case ASCE_TYPE_REGION1: {
396 union region1_table_entry rfte;
397
398 if (kvm_is_error_gpa(vcpu->kvm, ptr))
399 return PGM_ADDRESSING;
400 if (deref_table(vcpu->kvm, ptr, &rfte.val))
401 return -EFAULT;
402 if (rfte.i)
403 return PGM_REGION_FIRST_TRANS;
404 if (rfte.tt != TABLE_TYPE_REGION1)
405 return PGM_TRANSLATION_SPEC;
406 if (vaddr.rsx01 < rfte.tf || vaddr.rsx01 > rfte.tl)
407 return PGM_REGION_SECOND_TRANS;
408 if (edat1)
409 dat_protection |= rfte.p;
410 ptr = rfte.rto * 4096 + vaddr.rsx * 8;
411 }
412
413 case ASCE_TYPE_REGION2: {
414 union region2_table_entry rste;
415
416 if (kvm_is_error_gpa(vcpu->kvm, ptr))
417 return PGM_ADDRESSING;
418 if (deref_table(vcpu->kvm, ptr, &rste.val))
419 return -EFAULT;
420 if (rste.i)
421 return PGM_REGION_SECOND_TRANS;
422 if (rste.tt != TABLE_TYPE_REGION2)
423 return PGM_TRANSLATION_SPEC;
424 if (vaddr.rtx01 < rste.tf || vaddr.rtx01 > rste.tl)
425 return PGM_REGION_THIRD_TRANS;
426 if (edat1)
427 dat_protection |= rste.p;
428 ptr = rste.rto * 4096 + vaddr.rtx * 8;
429 }
430
431 case ASCE_TYPE_REGION3: {
432 union region3_table_entry rtte;
433
434 if (kvm_is_error_gpa(vcpu->kvm, ptr))
435 return PGM_ADDRESSING;
436 if (deref_table(vcpu->kvm, ptr, &rtte.val))
437 return -EFAULT;
438 if (rtte.i)
439 return PGM_REGION_THIRD_TRANS;
440 if (rtte.tt != TABLE_TYPE_REGION3)
441 return PGM_TRANSLATION_SPEC;
442 if (rtte.cr && asce.p && edat2)
443 return PGM_TRANSLATION_SPEC;
444 if (rtte.fc && edat2) {
445 dat_protection |= rtte.fc1.p;
446 raddr.rfaa = rtte.fc1.rfaa;
447 goto absolute_address;
448 }
449 if (vaddr.sx01 < rtte.fc0.tf)
450 return PGM_SEGMENT_TRANSLATION;
451 if (vaddr.sx01 > rtte.fc0.tl)
452 return PGM_SEGMENT_TRANSLATION;
453 if (edat1)
454 dat_protection |= rtte.fc0.p;
455 ptr = rtte.fc0.sto * 4096 + vaddr.sx * 8;
456 }
457
458 case ASCE_TYPE_SEGMENT: {
459 union segment_table_entry ste;
460
461 if (kvm_is_error_gpa(vcpu->kvm, ptr))
462 return PGM_ADDRESSING;
463 if (deref_table(vcpu->kvm, ptr, &ste.val))
464 return -EFAULT;
465 if (ste.i)
466 return PGM_SEGMENT_TRANSLATION;
467 if (ste.tt != TABLE_TYPE_SEGMENT)
468 return PGM_TRANSLATION_SPEC;
469 if (ste.cs && asce.p)
470 return PGM_TRANSLATION_SPEC;
471 if (ste.fc && edat1) {
472 dat_protection |= ste.fc1.p;
473 raddr.sfaa = ste.fc1.sfaa;
474 goto absolute_address;
475 }
476 dat_protection |= ste.fc0.p;
477 ptr = ste.fc0.pto * 2048 + vaddr.px * 8;
478 }
479 }
480 if (kvm_is_error_gpa(vcpu->kvm, ptr))
481 return PGM_ADDRESSING;
482 if (deref_table(vcpu->kvm, ptr, &pte.val))
483 return -EFAULT;
484 if (pte.i)
485 return PGM_PAGE_TRANSLATION;
486 if (pte.z)
487 return PGM_TRANSLATION_SPEC;
488 if (pte.co && !edat1)
489 return PGM_TRANSLATION_SPEC;
490 dat_protection |= pte.p;
491 raddr.pfra = pte.pfra;
492real_address:
493 raddr.addr = kvm_s390_real_to_abs(vcpu, raddr.addr);
494absolute_address:
495 if (write && dat_protection)
496 return PGM_PROTECTION;
497 if (kvm_is_error_gpa(vcpu->kvm, raddr.addr))
498 return PGM_ADDRESSING;
499 *gpa = raddr.addr;
500 return 0;
501}
502
503static inline int is_low_address(unsigned long ga)
504{
505
506 return (ga & ~0x11fful) == 0;
507}
508
509static int low_address_protection_enabled(struct kvm_vcpu *vcpu)
510{
511 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
512 psw_t *psw = &vcpu->arch.sie_block->gpsw;
513 union asce asce;
514
515 if (!ctlreg0.lap)
516 return 0;
517 asce.val = get_vcpu_asce(vcpu);
518 if (psw_bits(*psw).t && asce.p)
519 return 0;
520 return 1;
521}
522
523struct trans_exc_code_bits {
524 unsigned long addr : 52;
525 unsigned long fsi : 2;
526 unsigned long : 7;
527 unsigned long b61 : 1;
528 unsigned long as : 2;
529};
530
531enum {
532 FSI_UNKNOWN = 0,
533 FSI_STORE = 1,
534 FSI_FETCH = 2
535};
536
537static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
538 unsigned long *pages, unsigned long nr_pages,
539 int write)
540{
541 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
542 psw_t *psw = &vcpu->arch.sie_block->gpsw;
543 struct trans_exc_code_bits *tec_bits;
544 int lap_enabled, rc;
545
546 memset(pgm, 0, sizeof(*pgm));
547 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
548 tec_bits->fsi = write ? FSI_STORE : FSI_FETCH;
549 tec_bits->as = psw_bits(*psw).as;
550 lap_enabled = low_address_protection_enabled(vcpu);
551 while (nr_pages) {
552 ga = kvm_s390_logical_to_effective(vcpu, ga);
553 tec_bits->addr = ga >> PAGE_SHIFT;
554 if (write && lap_enabled && is_low_address(ga)) {
555 pgm->code = PGM_PROTECTION;
556 return pgm->code;
557 }
558 ga &= PAGE_MASK;
559 if (psw_bits(*psw).t) {
560 rc = guest_translate(vcpu, ga, pages, write);
561 if (rc < 0)
562 return rc;
563 if (rc == PGM_PROTECTION)
564 tec_bits->b61 = 1;
565 if (rc)
566 pgm->code = rc;
567 } else {
568 *pages = kvm_s390_real_to_abs(vcpu, ga);
569 if (kvm_is_error_gpa(vcpu->kvm, *pages))
570 pgm->code = PGM_ADDRESSING;
571 }
572 if (pgm->code)
573 return pgm->code;
574 ga += PAGE_SIZE;
575 pages++;
576 nr_pages--;
577 }
578 return 0;
579}
580
581int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
582 unsigned long len, int write)
583{
584 psw_t *psw = &vcpu->arch.sie_block->gpsw;
585 unsigned long _len, nr_pages, gpa, idx;
586 unsigned long pages_array[2];
587 unsigned long *pages;
588 int need_ipte_lock;
589 union asce asce;
590 int rc;
591
592 if (!len)
593 return 0;
594
595 if (psw_bits(*psw).t && psw_bits(*psw).as == PSW_AS_ACCREG)
596 return -EOPNOTSUPP;
597 nr_pages = (((ga & ~PAGE_MASK) + len - 1) >> PAGE_SHIFT) + 1;
598 pages = pages_array;
599 if (nr_pages > ARRAY_SIZE(pages_array))
600 pages = vmalloc(nr_pages * sizeof(unsigned long));
601 if (!pages)
602 return -ENOMEM;
603 asce.val = get_vcpu_asce(vcpu);
604 need_ipte_lock = psw_bits(*psw).t && !asce.r;
605 if (need_ipte_lock)
606 ipte_lock(vcpu);
607 rc = guest_page_range(vcpu, ga, pages, nr_pages, write);
608 for (idx = 0; idx < nr_pages && !rc; idx++) {
609 gpa = *(pages + idx) + (ga & ~PAGE_MASK);
610 _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len);
611 if (write)
612 rc = kvm_write_guest(vcpu->kvm, gpa, data, _len);
613 else
614 rc = kvm_read_guest(vcpu->kvm, gpa, data, _len);
615 len -= _len;
616 ga += _len;
617 data += _len;
618 }
619 if (need_ipte_lock)
620 ipte_unlock(vcpu);
621 if (nr_pages > ARRAY_SIZE(pages_array))
622 vfree(pages);
623 return rc;
624}
625
626int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
627 void *data, unsigned long len, int write)
628{
629 unsigned long _len, gpa;
630 int rc = 0;
631
632 while (len && !rc) {
633 gpa = kvm_s390_real_to_abs(vcpu, gra);
634 _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len);
635 if (write)
636 rc = write_guest_abs(vcpu, gpa, data, _len);
637 else
638 rc = read_guest_abs(vcpu, gpa, data, _len);
639 len -= _len;
640 gra += _len;
641 data += _len;
642 }
643 return rc;
644}
645
646
647
648
649
650
651
652
653
654
655int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
656 unsigned long *gpa, int write)
657{
658 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
659 psw_t *psw = &vcpu->arch.sie_block->gpsw;
660 struct trans_exc_code_bits *tec;
661 union asce asce;
662 int rc;
663
664
665 if (psw_bits(*psw).t && psw_bits(*psw).as == PSW_AS_ACCREG)
666 return -EOPNOTSUPP;
667
668 gva = kvm_s390_logical_to_effective(vcpu, gva);
669 memset(pgm, 0, sizeof(*pgm));
670 tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
671 tec->as = psw_bits(*psw).as;
672 tec->fsi = write ? FSI_STORE : FSI_FETCH;
673 tec->addr = gva >> PAGE_SHIFT;
674 if (is_low_address(gva) && low_address_protection_enabled(vcpu)) {
675 if (write) {
676 rc = pgm->code = PGM_PROTECTION;
677 return rc;
678 }
679 }
680
681 asce.val = get_vcpu_asce(vcpu);
682 if (psw_bits(*psw).t && !asce.r) {
683 rc = guest_translate(vcpu, gva, gpa, write);
684 if (rc > 0) {
685 if (rc == PGM_PROTECTION)
686 tec->b61 = 1;
687 pgm->code = rc;
688 }
689 } else {
690 rc = 0;
691 *gpa = kvm_s390_real_to_abs(vcpu, gva);
692 if (kvm_is_error_gpa(vcpu->kvm, *gpa))
693 rc = pgm->code = PGM_ADDRESSING;
694 }
695
696 return rc;
697}
698
699
700
701
702
703
704
705
706
707
708int kvm_s390_check_low_addr_protection(struct kvm_vcpu *vcpu, unsigned long ga)
709{
710 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
711 psw_t *psw = &vcpu->arch.sie_block->gpsw;
712 struct trans_exc_code_bits *tec_bits;
713
714 if (!is_low_address(ga) || !low_address_protection_enabled(vcpu))
715 return 0;
716
717 memset(pgm, 0, sizeof(*pgm));
718 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
719 tec_bits->fsi = FSI_STORE;
720 tec_bits->as = psw_bits(*psw).as;
721 tec_bits->addr = ga >> PAGE_SHIFT;
722 pgm->code = PGM_PROTECTION;
723
724 return pgm->code;
725}
726