1
2
3
4
5
6#include <linux/kernel.h>
7#include <linux/kprobes.h>
8#include <linux/vmalloc.h>
9#include <linux/init.h>
10#include <linux/mm.h>
11#include <linux/cpuhotplug.h>
12#include <linux/slab.h>
13#include <linux/uaccess.h>
14
15#include <asm/tlbflush.h>
16#include <asm/page.h>
17#include <asm/code-patching.h>
18#include <asm/setup.h>
19#include <asm/inst.h>
20
21static int __patch_instruction(struct ppc_inst *exec_addr, struct ppc_inst instr,
22 struct ppc_inst *patch_addr)
23{
24 int err = 0;
25
26 if (!ppc_inst_prefixed(instr)) {
27 __put_user_asm(ppc_inst_val(instr), patch_addr, err, "stw");
28 } else {
29 __put_user_asm(ppc_inst_as_u64(instr), patch_addr, err, "std");
30 }
31
32 if (err)
33 return err;
34
35 asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
36 "r" (exec_addr));
37
38 return 0;
39}
40
41int raw_patch_instruction(struct ppc_inst *addr, struct ppc_inst instr)
42{
43 return __patch_instruction(addr, instr, addr);
44}
45
46#ifdef CONFIG_STRICT_KERNEL_RWX
47static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
48
49static int text_area_cpu_up(unsigned int cpu)
50{
51 struct vm_struct *area;
52
53 area = get_vm_area(PAGE_SIZE, VM_ALLOC);
54 if (!area) {
55 WARN_ONCE(1, "Failed to create text area for cpu %d\n",
56 cpu);
57 return -1;
58 }
59 this_cpu_write(text_poke_area, area);
60
61 return 0;
62}
63
64static int text_area_cpu_down(unsigned int cpu)
65{
66 free_vm_area(this_cpu_read(text_poke_area));
67 return 0;
68}
69
70
71
72
73
74
75
76
77
78static int __init setup_text_poke_area(void)
79{
80 BUG_ON(!cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
81 "powerpc/text_poke:online", text_area_cpu_up,
82 text_area_cpu_down));
83
84 return 0;
85}
86late_initcall(setup_text_poke_area);
87
88
89
90
91static int map_patch_area(void *addr, unsigned long text_poke_addr)
92{
93 unsigned long pfn;
94 int err;
95
96 if (is_vmalloc_or_module_addr(addr))
97 pfn = vmalloc_to_pfn(addr);
98 else
99 pfn = __pa_symbol(addr) >> PAGE_SHIFT;
100
101 err = map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
102
103 pr_devel("Mapped addr %lx with pfn %lx:%d\n", text_poke_addr, pfn, err);
104 if (err)
105 return -1;
106
107 return 0;
108}
109
110static inline int unmap_patch_area(unsigned long addr)
111{
112 pte_t *ptep;
113 pmd_t *pmdp;
114 pud_t *pudp;
115 p4d_t *p4dp;
116 pgd_t *pgdp;
117
118 pgdp = pgd_offset_k(addr);
119 if (unlikely(!pgdp))
120 return -EINVAL;
121
122 p4dp = p4d_offset(pgdp, addr);
123 if (unlikely(!p4dp))
124 return -EINVAL;
125
126 pudp = pud_offset(p4dp, addr);
127 if (unlikely(!pudp))
128 return -EINVAL;
129
130 pmdp = pmd_offset(pudp, addr);
131 if (unlikely(!pmdp))
132 return -EINVAL;
133
134 ptep = pte_offset_kernel(pmdp, addr);
135 if (unlikely(!ptep))
136 return -EINVAL;
137
138 pr_devel("clearing mm %p, pte %p, addr %lx\n", &init_mm, ptep, addr);
139
140
141
142
143 pte_clear(&init_mm, addr, ptep);
144 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
145
146 return 0;
147}
148
149static int do_patch_instruction(struct ppc_inst *addr, struct ppc_inst instr)
150{
151 int err;
152 struct ppc_inst *patch_addr = NULL;
153 unsigned long flags;
154 unsigned long text_poke_addr;
155 unsigned long kaddr = (unsigned long)addr;
156
157
158
159
160
161
162 if (!this_cpu_read(text_poke_area))
163 return raw_patch_instruction(addr, instr);
164
165 local_irq_save(flags);
166
167 text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr;
168 if (map_patch_area(addr, text_poke_addr)) {
169 err = -1;
170 goto out;
171 }
172
173 patch_addr = (struct ppc_inst *)(text_poke_addr + (kaddr & ~PAGE_MASK));
174
175 __patch_instruction(addr, instr, patch_addr);
176
177 err = unmap_patch_area(text_poke_addr);
178 if (err)
179 pr_warn("failed to unmap %lx\n", text_poke_addr);
180
181out:
182 local_irq_restore(flags);
183
184 return err;
185}
186#else
187
188static int do_patch_instruction(struct ppc_inst *addr, struct ppc_inst instr)
189{
190 return raw_patch_instruction(addr, instr);
191}
192
193#endif
194
195int patch_instruction(struct ppc_inst *addr, struct ppc_inst instr)
196{
197
198 if (init_mem_is_free && init_section_contains(addr, 4)) {
199 pr_debug("Skipping init section patching addr: 0x%px\n", addr);
200 return 0;
201 }
202 return do_patch_instruction(addr, instr);
203}
204NOKPROBE_SYMBOL(patch_instruction);
205
206int patch_branch(struct ppc_inst *addr, unsigned long target, int flags)
207{
208 struct ppc_inst instr;
209
210 create_branch(&instr, addr, target, flags);
211 return patch_instruction(addr, instr);
212}
213
214bool is_offset_in_branch_range(long offset)
215{
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233 return (offset >= -0x2000000 && offset <= 0x1fffffc && !(offset & 0x3));
234}
235
236
237
238
239
240bool is_conditional_branch(struct ppc_inst instr)
241{
242 unsigned int opcode = ppc_inst_primary_opcode(instr);
243
244 if (opcode == 16)
245 return true;
246 if (opcode == 19) {
247 switch ((ppc_inst_val(instr) >> 1) & 0x3ff) {
248 case 16:
249 case 528:
250 case 560:
251 return true;
252 }
253 }
254 return false;
255}
256NOKPROBE_SYMBOL(is_conditional_branch);
257
258int create_branch(struct ppc_inst *instr,
259 const struct ppc_inst *addr,
260 unsigned long target, int flags)
261{
262 long offset;
263
264 *instr = ppc_inst(0);
265 offset = target;
266 if (! (flags & BRANCH_ABSOLUTE))
267 offset = offset - (unsigned long)addr;
268
269
270 if (!is_offset_in_branch_range(offset))
271 return 1;
272
273
274 *instr = ppc_inst(0x48000000 | (flags & 0x3) | (offset & 0x03FFFFFC));
275
276 return 0;
277}
278
279int create_cond_branch(struct ppc_inst *instr, const struct ppc_inst *addr,
280 unsigned long target, int flags)
281{
282 long offset;
283
284 offset = target;
285 if (! (flags & BRANCH_ABSOLUTE))
286 offset = offset - (unsigned long)addr;
287
288
289 if (offset < -0x8000 || offset > 0x7FFF || offset & 0x3)
290 return 1;
291
292
293 *instr = ppc_inst(0x40000000 | (flags & 0x3FF0003) | (offset & 0xFFFC));
294
295 return 0;
296}
297
298static unsigned int branch_opcode(struct ppc_inst instr)
299{
300 return ppc_inst_primary_opcode(instr) & 0x3F;
301}
302
303static int instr_is_branch_iform(struct ppc_inst instr)
304{
305 return branch_opcode(instr) == 18;
306}
307
308static int instr_is_branch_bform(struct ppc_inst instr)
309{
310 return branch_opcode(instr) == 16;
311}
312
313int instr_is_relative_branch(struct ppc_inst instr)
314{
315 if (ppc_inst_val(instr) & BRANCH_ABSOLUTE)
316 return 0;
317
318 return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
319}
320
321int instr_is_relative_link_branch(struct ppc_inst instr)
322{
323 return instr_is_relative_branch(instr) && (ppc_inst_val(instr) & BRANCH_SET_LINK);
324}
325
326static unsigned long branch_iform_target(const struct ppc_inst *instr)
327{
328 signed long imm;
329
330 imm = ppc_inst_val(*instr) & 0x3FFFFFC;
331
332
333 if (imm & 0x2000000)
334 imm -= 0x4000000;
335
336 if ((ppc_inst_val(*instr) & BRANCH_ABSOLUTE) == 0)
337 imm += (unsigned long)instr;
338
339 return (unsigned long)imm;
340}
341
342static unsigned long branch_bform_target(const struct ppc_inst *instr)
343{
344 signed long imm;
345
346 imm = ppc_inst_val(*instr) & 0xFFFC;
347
348
349 if (imm & 0x8000)
350 imm -= 0x10000;
351
352 if ((ppc_inst_val(*instr) & BRANCH_ABSOLUTE) == 0)
353 imm += (unsigned long)instr;
354
355 return (unsigned long)imm;
356}
357
358unsigned long branch_target(const struct ppc_inst *instr)
359{
360 if (instr_is_branch_iform(ppc_inst_read(instr)))
361 return branch_iform_target(instr);
362 else if (instr_is_branch_bform(ppc_inst_read(instr)))
363 return branch_bform_target(instr);
364
365 return 0;
366}
367
368int instr_is_branch_to_addr(const struct ppc_inst *instr, unsigned long addr)
369{
370 if (instr_is_branch_iform(ppc_inst_read(instr)) ||
371 instr_is_branch_bform(ppc_inst_read(instr)))
372 return branch_target(instr) == addr;
373
374 return 0;
375}
376
377int translate_branch(struct ppc_inst *instr, const struct ppc_inst *dest,
378 const struct ppc_inst *src)
379{
380 unsigned long target;
381 target = branch_target(src);
382
383 if (instr_is_branch_iform(ppc_inst_read(src)))
384 return create_branch(instr, dest, target,
385 ppc_inst_val(ppc_inst_read(src)));
386 else if (instr_is_branch_bform(ppc_inst_read(src)))
387 return create_cond_branch(instr, dest, target,
388 ppc_inst_val(ppc_inst_read(src)));
389
390 return 1;
391}
392
393#ifdef CONFIG_PPC_BOOK3E_64
394void __patch_exception(int exc, unsigned long addr)
395{
396 extern unsigned int interrupt_base_book3e;
397 unsigned int *ibase = &interrupt_base_book3e;
398
399
400
401
402
403
404
405 patch_branch((struct ppc_inst *)(ibase + (exc / 4) + 1), addr, 0);
406}
407#endif
408
409#ifdef CONFIG_CODE_PATCHING_SELFTEST
410
411static void __init test_trampoline(void)
412{
413 asm ("nop;\n");
414}
415
416#define check(x) \
417 if (!(x)) printk("code-patching: test failed at line %d\n", __LINE__);
418
419static void __init test_branch_iform(void)
420{
421 int err;
422 struct ppc_inst instr;
423 unsigned long addr;
424
425 addr = (unsigned long)&instr;
426
427
428 check(instr_is_branch_iform(ppc_inst(0x48000000)));
429
430 check(instr_is_branch_iform(ppc_inst(0x4bffffff)));
431
432 check(!instr_is_branch_iform(ppc_inst(0xcbffffff)));
433
434 check(!instr_is_branch_iform(ppc_inst(0x7bffffff)));
435
436
437 check(instr_is_branch_iform(ppc_inst(0x48000001)));
438
439 check(instr_is_branch_iform(ppc_inst(0x4bfffffd)));
440
441 check(instr_is_branch_iform(ppc_inst(0x4bff00fd)));
442
443 check(!instr_is_branch_iform(ppc_inst(0x7bfffffd)));
444
445
446 instr = ppc_inst(0x48000103);
447 check(instr_is_branch_to_addr(&instr, 0x100));
448
449 instr = ppc_inst(0x480420ff);
450 check(instr_is_branch_to_addr(&instr, 0x420fc));
451
452 instr = ppc_inst(0x49fffffc);
453 check(instr_is_branch_to_addr(&instr, addr + 0x1FFFFFC));
454
455 instr = ppc_inst(0x4bfffffc);
456 check(instr_is_branch_to_addr(&instr, addr - 4));
457
458 instr = ppc_inst(0x4a000000);
459 check(instr_is_branch_to_addr(&instr, addr - 0x2000000));
460
461
462 err = create_branch(&instr, &instr, addr, BRANCH_SET_LINK);
463 check(instr_is_branch_to_addr(&instr, addr));
464
465
466 err = create_branch(&instr, &instr, addr - 0x100, BRANCH_SET_LINK);
467 check(instr_is_branch_to_addr(&instr, addr - 0x100));
468
469
470 err = create_branch(&instr, &instr, addr + 0x100, 0);
471 check(instr_is_branch_to_addr(&instr, addr + 0x100));
472
473
474 err = create_branch(&instr, &instr, addr - 0x2000000, BRANCH_SET_LINK);
475 check(instr_is_branch_to_addr(&instr, addr - 0x2000000));
476
477
478 err = create_branch(&instr, &instr, addr - 0x2000004, BRANCH_SET_LINK);
479 check(err);
480
481
482 err = create_branch(&instr, &instr, addr + 0x2000000, BRANCH_SET_LINK);
483 check(err);
484
485
486 err = create_branch(&instr, &instr, addr + 3, BRANCH_SET_LINK);
487 check(err);
488
489
490 err = create_branch(&instr, &instr, addr, 0xFFFFFFFC);
491 check(instr_is_branch_to_addr(&instr, addr));
492 check(ppc_inst_equal(instr, ppc_inst(0x48000000)));
493}
494
495static void __init test_create_function_call(void)
496{
497 struct ppc_inst *iptr;
498 unsigned long dest;
499 struct ppc_inst instr;
500
501
502 iptr = (struct ppc_inst *)ppc_function_entry(test_trampoline);
503 dest = ppc_function_entry(test_create_function_call);
504 create_branch(&instr, iptr, dest, BRANCH_SET_LINK);
505 patch_instruction(iptr, instr);
506 check(instr_is_branch_to_addr(iptr, dest));
507}
508
509static void __init test_branch_bform(void)
510{
511 int err;
512 unsigned long addr;
513 struct ppc_inst *iptr, instr;
514 unsigned int flags;
515
516 iptr = &instr;
517 addr = (unsigned long)iptr;
518
519
520 check(instr_is_branch_bform(ppc_inst(0x40000000)));
521
522 check(instr_is_branch_bform(ppc_inst(0x43ffffff)));
523
524 check(!instr_is_branch_bform(ppc_inst(0xc3ffffff)));
525
526 check(!instr_is_branch_bform(ppc_inst(0x7bffffff)));
527
528
529 instr = ppc_inst(0x43ff0103);
530 check(instr_is_branch_to_addr(&instr, 0x100));
531
532 instr = ppc_inst(0x43ff20ff);
533 check(instr_is_branch_to_addr(&instr, 0x20fc));
534
535 instr = ppc_inst(0x43ff7ffc);
536 check(instr_is_branch_to_addr(&instr, addr + 0x7FFC));
537
538 instr = ppc_inst(0x43fffffc);
539 check(instr_is_branch_to_addr(&instr, addr - 4));
540
541 instr = ppc_inst(0x43ff8000);
542 check(instr_is_branch_to_addr(&instr, addr - 0x8000));
543
544
545 flags = 0x3ff000 | BRANCH_SET_LINK;
546
547
548 err = create_cond_branch(&instr, iptr, addr, flags);
549 check(instr_is_branch_to_addr(&instr, addr));
550
551
552 err = create_cond_branch(&instr, iptr, addr - 0x100, flags);
553 check(instr_is_branch_to_addr(&instr, addr - 0x100));
554
555
556 err = create_cond_branch(&instr, iptr, addr + 0x100, flags);
557 check(instr_is_branch_to_addr(&instr, addr + 0x100));
558
559
560 err = create_cond_branch(&instr, iptr, addr - 0x8000, flags);
561 check(instr_is_branch_to_addr(&instr, addr - 0x8000));
562
563
564 err = create_cond_branch(&instr, iptr, addr - 0x8004, flags);
565 check(err);
566
567
568 err = create_cond_branch(&instr, iptr, addr + 0x8000, flags);
569 check(err);
570
571
572 err = create_cond_branch(&instr, iptr, addr + 3, flags);
573 check(err);
574
575
576 err = create_cond_branch(&instr, iptr, addr, 0xFFFFFFFC);
577 check(instr_is_branch_to_addr(&instr, addr));
578 check(ppc_inst_equal(instr, ppc_inst(0x43FF0000)));
579}
580
581static void __init test_translate_branch(void)
582{
583 unsigned long addr;
584 void *p, *q;
585 struct ppc_inst instr;
586 void *buf;
587
588 buf = vmalloc(PAGE_ALIGN(0x2000000 + 1));
589 check(buf);
590 if (!buf)
591 return;
592
593
594 p = buf;
595 addr = (unsigned long)p;
596 patch_branch(p, addr, 0);
597 check(instr_is_branch_to_addr(p, addr));
598 q = p + 4;
599 translate_branch(&instr, q, p);
600 patch_instruction(q, instr);
601 check(instr_is_branch_to_addr(q, addr));
602
603
604 p = buf;
605 addr = (unsigned long)p;
606 patch_branch(p, addr, 0);
607 q = buf + 0x2000000;
608 translate_branch(&instr, q, p);
609 patch_instruction(q, instr);
610 check(instr_is_branch_to_addr(p, addr));
611 check(instr_is_branch_to_addr(q, addr));
612 check(ppc_inst_equal(ppc_inst_read(q), ppc_inst(0x4a000000)));
613
614
615 p = buf + 0x2000000;
616 addr = (unsigned long)p;
617 patch_branch(p, addr, 0);
618 q = buf + 4;
619 translate_branch(&instr, q, p);
620 patch_instruction(q, instr);
621 check(instr_is_branch_to_addr(p, addr));
622 check(instr_is_branch_to_addr(q, addr));
623 check(ppc_inst_equal(ppc_inst_read(q), ppc_inst(0x49fffffc)));
624
625
626 p = buf;
627 addr = 0x1000000 + (unsigned long)buf;
628 patch_branch(p, addr, BRANCH_SET_LINK);
629 q = buf + 0x1400000;
630 translate_branch(&instr, q, p);
631 patch_instruction(q, instr);
632 check(instr_is_branch_to_addr(p, addr));
633 check(instr_is_branch_to_addr(q, addr));
634
635
636 p = buf + 0x1000000;
637 addr = 0x2000000 + (unsigned long)buf;
638 patch_branch(p, addr, 0);
639 q = buf + 4;
640 translate_branch(&instr, q, p);
641 patch_instruction(q, instr);
642 check(instr_is_branch_to_addr(p, addr));
643 check(instr_is_branch_to_addr(q, addr));
644
645
646
647
648
649 p = buf;
650 addr = (unsigned long)p;
651 create_cond_branch(&instr, p, addr, 0);
652 patch_instruction(p, instr);
653 check(instr_is_branch_to_addr(p, addr));
654 q = buf + 4;
655 translate_branch(&instr, q, p);
656 patch_instruction(q, instr);
657 check(instr_is_branch_to_addr(q, addr));
658
659
660 p = buf;
661 addr = (unsigned long)p;
662 create_cond_branch(&instr, p, addr, 0xFFFFFFFC);
663 patch_instruction(p, instr);
664 q = buf + 0x8000;
665 translate_branch(&instr, q, p);
666 patch_instruction(q, instr);
667 check(instr_is_branch_to_addr(p, addr));
668 check(instr_is_branch_to_addr(q, addr));
669 check(ppc_inst_equal(ppc_inst_read(q), ppc_inst(0x43ff8000)));
670
671
672 p = buf + 0x8000;
673 addr = (unsigned long)p;
674 create_cond_branch(&instr, p, addr, 0xFFFFFFFC);
675 patch_instruction(p, instr);
676 q = buf + 4;
677 translate_branch(&instr, q, p);
678 patch_instruction(q, instr);
679 check(instr_is_branch_to_addr(p, addr));
680 check(instr_is_branch_to_addr(q, addr));
681 check(ppc_inst_equal(ppc_inst_read(q), ppc_inst(0x43ff7ffc)));
682
683
684 p = buf;
685 addr = 0x3000 + (unsigned long)buf;
686 create_cond_branch(&instr, p, addr, BRANCH_SET_LINK);
687 patch_instruction(p, instr);
688 q = buf + 0x5000;
689 translate_branch(&instr, q, p);
690 patch_instruction(q, instr);
691 check(instr_is_branch_to_addr(p, addr));
692 check(instr_is_branch_to_addr(q, addr));
693
694
695 p = buf + 0x2000;
696 addr = 0x4000 + (unsigned long)buf;
697 create_cond_branch(&instr, p, addr, 0);
698 patch_instruction(p, instr);
699 q = buf + 4;
700 translate_branch(&instr, q, p);
701 patch_instruction(q, instr);
702 check(instr_is_branch_to_addr(p, addr));
703 check(instr_is_branch_to_addr(q, addr));
704
705
706 vfree(buf);
707}
708
709#ifdef CONFIG_PPC64
710static void __init test_prefixed_patching(void)
711{
712 extern unsigned int code_patching_test1[];
713 extern unsigned int code_patching_test1_expected[];
714 extern unsigned int end_code_patching_test1[];
715
716 __patch_instruction((struct ppc_inst *)code_patching_test1,
717 ppc_inst_prefix(OP_PREFIX << 26, 0x00000000),
718 (struct ppc_inst *)code_patching_test1);
719
720 check(!memcmp(code_patching_test1,
721 code_patching_test1_expected,
722 sizeof(unsigned int) *
723 (end_code_patching_test1 - code_patching_test1)));
724}
725#else
726static inline void test_prefixed_patching(void) {}
727#endif
728
729static int __init test_code_patching(void)
730{
731 printk(KERN_DEBUG "Running code patching self-tests ...\n");
732
733 test_branch_iform();
734 test_branch_bform();
735 test_create_function_call();
736 test_translate_branch();
737 test_prefixed_patching();
738
739 return 0;
740}
741late_initcall(test_code_patching);
742
743#endif
744