1#define pr_fmt(fmt) "SMP alternatives: " fmt
2
3#include <linux/module.h>
4#include <linux/sched.h>
5#include <linux/mutex.h>
6#include <linux/list.h>
7#include <linux/stringify.h>
8#include <linux/kprobes.h>
9#include <linux/mm.h>
10#include <linux/vmalloc.h>
11#include <linux/memory.h>
12#include <linux/stop_machine.h>
13#include <linux/slab.h>
14#include <linux/kdebug.h>
15#include <asm/alternative.h>
16#include <asm/sections.h>
17#include <asm/pgtable.h>
18#include <asm/mce.h>
19#include <asm/nmi.h>
20#include <asm/cacheflush.h>
21#include <asm/tlbflush.h>
22#include <asm/io.h>
23#include <asm/fixmap.h>
24
25#define MAX_PATCH_LEN (255-1)
26
27static int __initdata_or_module debug_alternative;
28
29static int __init debug_alt(char *str)
30{
31 debug_alternative = 1;
32 return 1;
33}
34__setup("debug-alternative", debug_alt);
35
36static int noreplace_smp;
37
38static int __init setup_noreplace_smp(char *str)
39{
40 noreplace_smp = 1;
41 return 1;
42}
43__setup("noreplace-smp", setup_noreplace_smp);
44
45#ifdef CONFIG_PARAVIRT
46static int __initdata_or_module noreplace_paravirt = 0;
47
48static int __init setup_noreplace_paravirt(char *str)
49{
50 noreplace_paravirt = 1;
51 return 1;
52}
53__setup("noreplace-paravirt", setup_noreplace_paravirt);
54#endif
55
56#define DPRINTK(fmt, ...) \
57do { \
58 if (debug_alternative) \
59 printk(KERN_DEBUG fmt, ##__VA_ARGS__); \
60} while (0)
61
62
63
64
65
66
67
68
69
70
71#if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
72static const unsigned char intelnops[] =
73{
74 GENERIC_NOP1,
75 GENERIC_NOP2,
76 GENERIC_NOP3,
77 GENERIC_NOP4,
78 GENERIC_NOP5,
79 GENERIC_NOP6,
80 GENERIC_NOP7,
81 GENERIC_NOP8,
82 GENERIC_NOP5_ATOMIC
83};
84static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
85{
86 NULL,
87 intelnops,
88 intelnops + 1,
89 intelnops + 1 + 2,
90 intelnops + 1 + 2 + 3,
91 intelnops + 1 + 2 + 3 + 4,
92 intelnops + 1 + 2 + 3 + 4 + 5,
93 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
94 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
95 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
96};
97#endif
98
99#ifdef K8_NOP1
100static const unsigned char k8nops[] =
101{
102 K8_NOP1,
103 K8_NOP2,
104 K8_NOP3,
105 K8_NOP4,
106 K8_NOP5,
107 K8_NOP6,
108 K8_NOP7,
109 K8_NOP8,
110 K8_NOP5_ATOMIC
111};
112static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
113{
114 NULL,
115 k8nops,
116 k8nops + 1,
117 k8nops + 1 + 2,
118 k8nops + 1 + 2 + 3,
119 k8nops + 1 + 2 + 3 + 4,
120 k8nops + 1 + 2 + 3 + 4 + 5,
121 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
122 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
123 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
124};
125#endif
126
127#if defined(K7_NOP1) && !defined(CONFIG_X86_64)
128static const unsigned char k7nops[] =
129{
130 K7_NOP1,
131 K7_NOP2,
132 K7_NOP3,
133 K7_NOP4,
134 K7_NOP5,
135 K7_NOP6,
136 K7_NOP7,
137 K7_NOP8,
138 K7_NOP5_ATOMIC
139};
140static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
141{
142 NULL,
143 k7nops,
144 k7nops + 1,
145 k7nops + 1 + 2,
146 k7nops + 1 + 2 + 3,
147 k7nops + 1 + 2 + 3 + 4,
148 k7nops + 1 + 2 + 3 + 4 + 5,
149 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
150 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
151 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
152};
153#endif
154
155#ifdef P6_NOP1
156static const unsigned char p6nops[] =
157{
158 P6_NOP1,
159 P6_NOP2,
160 P6_NOP3,
161 P6_NOP4,
162 P6_NOP5,
163 P6_NOP6,
164 P6_NOP7,
165 P6_NOP8,
166 P6_NOP5_ATOMIC
167};
168static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
169{
170 NULL,
171 p6nops,
172 p6nops + 1,
173 p6nops + 1 + 2,
174 p6nops + 1 + 2 + 3,
175 p6nops + 1 + 2 + 3 + 4,
176 p6nops + 1 + 2 + 3 + 4 + 5,
177 p6nops + 1 + 2 + 3 + 4 + 5 + 6,
178 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
179 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
180};
181#endif
182
183
184#ifdef CONFIG_X86_64
185const unsigned char * const *ideal_nops = p6_nops;
186#else
187const unsigned char * const *ideal_nops = intel_nops;
188#endif
189
190void __init arch_init_ideal_nops(void)
191{
192 switch (boot_cpu_data.x86_vendor) {
193 case X86_VENDOR_INTEL:
194
195
196
197
198
199 if (boot_cpu_data.x86 == 6 &&
200 boot_cpu_data.x86_model >= 0x0f &&
201 boot_cpu_data.x86_model != 0x1c &&
202 boot_cpu_data.x86_model != 0x26 &&
203 boot_cpu_data.x86_model != 0x27 &&
204 boot_cpu_data.x86_model < 0x30) {
205 ideal_nops = k8_nops;
206 } else if (boot_cpu_has(X86_FEATURE_NOPL)) {
207 ideal_nops = p6_nops;
208 } else {
209#ifdef CONFIG_X86_64
210 ideal_nops = k8_nops;
211#else
212 ideal_nops = intel_nops;
213#endif
214 }
215 break;
216 default:
217#ifdef CONFIG_X86_64
218 ideal_nops = k8_nops;
219#else
220 if (boot_cpu_has(X86_FEATURE_K8))
221 ideal_nops = k8_nops;
222 else if (boot_cpu_has(X86_FEATURE_K7))
223 ideal_nops = k7_nops;
224 else
225 ideal_nops = intel_nops;
226#endif
227 }
228}
229
230
231static void __init_or_module add_nops(void *insns, unsigned int len)
232{
233 while (len > 0) {
234 unsigned int noplen = len;
235 if (noplen > ASM_NOP_MAX)
236 noplen = ASM_NOP_MAX;
237 memcpy(insns, ideal_nops[noplen], noplen);
238 insns += noplen;
239 len -= noplen;
240 }
241}
242
243extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
244extern s32 __smp_locks[], __smp_locks_end[];
245void *text_poke_early(void *addr, const void *opcode, size_t len);
246
247
248
249
250
251
252
253void __init_or_module apply_alternatives(struct alt_instr *start,
254 struct alt_instr *end)
255{
256 struct alt_instr *a;
257 u8 *instr, *replacement;
258 u8 insnbuf[MAX_PATCH_LEN];
259
260 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
261
262
263
264
265
266
267
268
269
270 for (a = start; a < end; a++) {
271 instr = (u8 *)&a->instr_offset + a->instr_offset;
272 replacement = (u8 *)&a->repl_offset + a->repl_offset;
273 BUG_ON(a->replacementlen > a->instrlen);
274 BUG_ON(a->instrlen > sizeof(insnbuf));
275 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
276 if (!boot_cpu_has(a->cpuid))
277 continue;
278
279 memcpy(insnbuf, replacement, a->replacementlen);
280
281
282 if (*insnbuf == 0xe8 && a->replacementlen == 5)
283 *(s32 *)(insnbuf + 1) += replacement - instr;
284
285 add_nops(insnbuf + a->replacementlen,
286 a->instrlen - a->replacementlen);
287
288 text_poke_early(instr, insnbuf, a->instrlen);
289 }
290}
291
292#ifdef CONFIG_SMP
293
294static void alternatives_smp_lock(const s32 *start, const s32 *end,
295 u8 *text, u8 *text_end)
296{
297 const s32 *poff;
298
299 mutex_lock(&text_mutex);
300 for (poff = start; poff < end; poff++) {
301 u8 *ptr = (u8 *)poff + *poff;
302
303 if (!*poff || ptr < text || ptr >= text_end)
304 continue;
305
306 if (*ptr == 0x3e)
307 text_poke(ptr, ((unsigned char []){0xf0}), 1);
308 }
309 mutex_unlock(&text_mutex);
310}
311
312static void alternatives_smp_unlock(const s32 *start, const s32 *end,
313 u8 *text, u8 *text_end)
314{
315 const s32 *poff;
316
317 mutex_lock(&text_mutex);
318 for (poff = start; poff < end; poff++) {
319 u8 *ptr = (u8 *)poff + *poff;
320
321 if (!*poff || ptr < text || ptr >= text_end)
322 continue;
323
324 if (*ptr == 0xf0)
325 text_poke(ptr, ((unsigned char []){0x3E}), 1);
326 }
327 mutex_unlock(&text_mutex);
328}
329
330struct smp_alt_module {
331
332 struct module *mod;
333 char *name;
334
335
336 const s32 *locks;
337 const s32 *locks_end;
338
339
340 u8 *text;
341 u8 *text_end;
342
343 struct list_head next;
344};
345static LIST_HEAD(smp_alt_modules);
346static DEFINE_MUTEX(smp_alt);
347static bool uniproc_patched = false;
348
349void __init_or_module alternatives_smp_module_add(struct module *mod,
350 char *name,
351 void *locks, void *locks_end,
352 void *text, void *text_end)
353{
354 struct smp_alt_module *smp;
355
356 mutex_lock(&smp_alt);
357 if (!uniproc_patched)
358 goto unlock;
359
360 if (num_possible_cpus() == 1)
361
362 goto smp_unlock;
363
364 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
365 if (NULL == smp)
366
367 goto unlock;
368
369 smp->mod = mod;
370 smp->name = name;
371 smp->locks = locks;
372 smp->locks_end = locks_end;
373 smp->text = text;
374 smp->text_end = text_end;
375 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
376 __func__, smp->locks, smp->locks_end,
377 smp->text, smp->text_end, smp->name);
378
379 list_add_tail(&smp->next, &smp_alt_modules);
380smp_unlock:
381 alternatives_smp_unlock(locks, locks_end, text, text_end);
382unlock:
383 mutex_unlock(&smp_alt);
384}
385
386void __init_or_module alternatives_smp_module_del(struct module *mod)
387{
388 struct smp_alt_module *item;
389
390 mutex_lock(&smp_alt);
391 list_for_each_entry(item, &smp_alt_modules, next) {
392 if (mod != item->mod)
393 continue;
394 list_del(&item->next);
395 kfree(item);
396 break;
397 }
398 mutex_unlock(&smp_alt);
399}
400
401void alternatives_enable_smp(void)
402{
403 struct smp_alt_module *mod;
404
405#ifdef CONFIG_LOCKDEP
406
407
408
409
410
411
412
413 pr_info("lockdep: fixing up alternatives\n");
414#endif
415
416
417 BUG_ON(num_possible_cpus() == 1);
418
419 mutex_lock(&smp_alt);
420
421 if (uniproc_patched) {
422 pr_info("switching to SMP code\n");
423 BUG_ON(num_online_cpus() != 1);
424 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
425 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
426 list_for_each_entry(mod, &smp_alt_modules, next)
427 alternatives_smp_lock(mod->locks, mod->locks_end,
428 mod->text, mod->text_end);
429 uniproc_patched = false;
430 }
431 mutex_unlock(&smp_alt);
432}
433
434
435int alternatives_text_reserved(void *start, void *end)
436{
437 struct smp_alt_module *mod;
438 const s32 *poff;
439 u8 *text_start = start;
440 u8 *text_end = end;
441
442 list_for_each_entry(mod, &smp_alt_modules, next) {
443 if (mod->text > text_end || mod->text_end < text_start)
444 continue;
445 for (poff = mod->locks; poff < mod->locks_end; poff++) {
446 const u8 *ptr = (const u8 *)poff + *poff;
447
448 if (text_start <= ptr && text_end > ptr)
449 return 1;
450 }
451 }
452
453 return 0;
454}
455#endif
456
457#ifdef CONFIG_PARAVIRT
458void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
459 struct paravirt_patch_site *end)
460{
461 struct paravirt_patch_site *p;
462 char insnbuf[MAX_PATCH_LEN];
463
464 if (noreplace_paravirt)
465 return;
466
467 for (p = start; p < end; p++) {
468 unsigned int used;
469
470 BUG_ON(p->len > MAX_PATCH_LEN);
471
472 memcpy(insnbuf, p->instr, p->len);
473 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf,
474 (unsigned long)p->instr, p->len);
475
476 BUG_ON(used > p->len);
477
478
479 add_nops(insnbuf + used, p->len - used);
480 text_poke_early(p->instr, insnbuf, p->len);
481 }
482}
483extern struct paravirt_patch_site __start_parainstructions[],
484 __stop_parainstructions[];
485#endif
486
487void __init alternative_instructions(void)
488{
489
490
491
492 stop_nmi();
493
494
495
496
497
498
499
500
501
502
503
504
505 apply_alternatives(__alt_instructions, __alt_instructions_end);
506
507#ifdef CONFIG_SMP
508
509 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
510 uniproc_patched = true;
511 alternatives_smp_module_add(NULL, "core kernel",
512 __smp_locks, __smp_locks_end,
513 _text, _etext);
514 }
515
516 if (!uniproc_patched || num_possible_cpus() == 1)
517 free_init_pages("SMP alternatives",
518 (unsigned long)__smp_locks,
519 (unsigned long)__smp_locks_end);
520#endif
521
522 apply_paravirt(__parainstructions, __parainstructions_end);
523
524 restart_nmi();
525}
526
527
528
529
530
531
532
533
534
535
536
537
538
539void *__init_or_module text_poke_early(void *addr, const void *opcode,
540 size_t len)
541{
542 unsigned long flags;
543 local_irq_save(flags);
544 memcpy(addr, opcode, len);
545 sync_core();
546 local_irq_restore(flags);
547
548
549 return addr;
550}
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
566{
567 unsigned long flags;
568 char *vaddr;
569 struct page *pages[2];
570 int i;
571
572 if (!core_kernel_text((unsigned long)addr)) {
573 pages[0] = vmalloc_to_page(addr);
574 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
575 } else {
576 pages[0] = virt_to_page(addr);
577 WARN_ON(!PageReserved(pages[0]));
578 pages[1] = virt_to_page(addr + PAGE_SIZE);
579 }
580 BUG_ON(!pages[0]);
581 local_irq_save(flags);
582 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0]));
583 if (pages[1])
584 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1]));
585 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0);
586 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
587 clear_fixmap(FIX_TEXT_POKE0);
588 if (pages[1])
589 clear_fixmap(FIX_TEXT_POKE1);
590 local_flush_tlb();
591 sync_core();
592
593
594 for (i = 0; i < len; i++)
595 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
596 local_irq_restore(flags);
597 return addr;
598}
599
600static void do_sync_core(void *info)
601{
602 sync_core();
603}
604
605static bool bp_patching_in_progress;
606static void *bp_int3_handler, *bp_int3_addr;
607
608int poke_int3_handler(struct pt_regs *regs)
609{
610
611 smp_rmb();
612
613 if (likely(!bp_patching_in_progress))
614 return 0;
615
616 if (user_mode_vm(regs) || regs->ip != (unsigned long)bp_int3_addr)
617 return 0;
618
619
620 regs->ip = (unsigned long) bp_int3_handler;
621
622 return 1;
623
624}
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
649{
650 unsigned char int3 = 0xcc;
651
652 bp_int3_handler = handler;
653 bp_int3_addr = (u8 *)addr + sizeof(int3);
654 bp_patching_in_progress = true;
655
656
657
658
659
660 smp_wmb();
661
662 text_poke(addr, &int3, sizeof(int3));
663
664 on_each_cpu(do_sync_core, NULL, 1);
665
666 if (len - sizeof(int3) > 0) {
667
668 text_poke((char *)addr + sizeof(int3),
669 (const char *) opcode + sizeof(int3),
670 len - sizeof(int3));
671
672
673
674
675
676 on_each_cpu(do_sync_core, NULL, 1);
677 }
678
679
680 text_poke(addr, opcode, sizeof(int3));
681
682 on_each_cpu(do_sync_core, NULL, 1);
683
684 bp_patching_in_progress = false;
685 smp_wmb();
686
687 return addr;
688}
689
690