1
2
3
4
5
6
7
8
9
10
11
12#include <asm/asm-prototypes.h>
13#include <asm/mmu.h>
14#include <asm/mmu_context.h>
15#include <asm/paca.h>
16#include <asm/ppc-opcode.h>
17#include <asm/cputable.h>
18#include <asm/cacheflush.h>
19#include <asm/smp.h>
20#include <linux/compiler.h>
21#include <linux/context_tracking.h>
22#include <linux/mm_types.h>
23#include <linux/pgtable.h>
24
25#include <asm/udbg.h>
26#include <asm/code-patching.h>
27
28#include "internal.h"
29
30
31enum slb_index {
32 LINEAR_INDEX = 0,
33 KSTACK_INDEX = 1,
34};
35
36static long slb_allocate_user(struct mm_struct *mm, unsigned long ea);
37
38#define slb_esid_mask(ssize) \
39 (((ssize) == MMU_SEGSIZE_256M)? ESID_MASK: ESID_MASK_1T)
40
41static inline unsigned long mk_esid_data(unsigned long ea, int ssize,
42 enum slb_index index)
43{
44 return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | index;
45}
46
47static inline unsigned long __mk_vsid_data(unsigned long vsid, int ssize,
48 unsigned long flags)
49{
50 return (vsid << slb_vsid_shift(ssize)) | flags |
51 ((unsigned long) ssize << SLB_VSID_SSIZE_SHIFT);
52}
53
54static inline unsigned long mk_vsid_data(unsigned long ea, int ssize,
55 unsigned long flags)
56{
57 return __mk_vsid_data(get_kernel_vsid(ea, ssize), ssize, flags);
58}
59
60bool stress_slb_enabled __initdata;
61
62static int __init parse_stress_slb(char *p)
63{
64 stress_slb_enabled = true;
65 return 0;
66}
67early_param("stress_slb", parse_stress_slb);
68
69__ro_after_init DEFINE_STATIC_KEY_FALSE(stress_slb_key);
70
71static void assert_slb_presence(bool present, unsigned long ea)
72{
73#ifdef CONFIG_DEBUG_VM
74 unsigned long tmp;
75
76 WARN_ON_ONCE(mfmsr() & MSR_EE);
77
78 if (!cpu_has_feature(CPU_FTR_ARCH_206))
79 return;
80
81
82
83
84
85 ea &= ~((1UL << SID_SHIFT) - 1);
86 asm volatile(__PPC_SLBFEE_DOT(%0, %1) : "=r"(tmp) : "r"(ea) : "cr0");
87
88 WARN_ON(present == (tmp == 0));
89#endif
90}
91
92static inline void slb_shadow_update(unsigned long ea, int ssize,
93 unsigned long flags,
94 enum slb_index index)
95{
96 struct slb_shadow *p = get_slb_shadow();
97
98
99
100
101
102
103 WRITE_ONCE(p->save_area[index].esid, 0);
104 WRITE_ONCE(p->save_area[index].vsid, cpu_to_be64(mk_vsid_data(ea, ssize, flags)));
105 WRITE_ONCE(p->save_area[index].esid, cpu_to_be64(mk_esid_data(ea, ssize, index)));
106}
107
108static inline void slb_shadow_clear(enum slb_index index)
109{
110 WRITE_ONCE(get_slb_shadow()->save_area[index].esid, cpu_to_be64(index));
111}
112
113static inline void create_shadowed_slbe(unsigned long ea, int ssize,
114 unsigned long flags,
115 enum slb_index index)
116{
117
118
119
120
121
122 slb_shadow_update(ea, ssize, flags, index);
123
124 assert_slb_presence(false, ea);
125 asm volatile("slbmte %0,%1" :
126 : "r" (mk_vsid_data(ea, ssize, flags)),
127 "r" (mk_esid_data(ea, ssize, index))
128 : "memory" );
129}
130
131
132
133
134
135void __slb_restore_bolted_realmode(void)
136{
137 struct slb_shadow *p = get_slb_shadow();
138 enum slb_index index;
139
140
141 for (index = 0; index < SLB_NUM_BOLTED; index++) {
142 asm volatile("slbmte %0,%1" :
143 : "r" (be64_to_cpu(p->save_area[index].vsid)),
144 "r" (be64_to_cpu(p->save_area[index].esid)));
145 }
146
147 assert_slb_presence(true, local_paca->kstack);
148}
149
150
151
152
153void slb_restore_bolted_realmode(void)
154{
155 __slb_restore_bolted_realmode();
156 get_paca()->slb_cache_ptr = 0;
157
158 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
159 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
160}
161
162
163
164
165void slb_flush_all_realmode(void)
166{
167 asm volatile("slbmte %0,%0; slbia" : : "r" (0));
168}
169
170static __always_inline void __slb_flush_and_restore_bolted(bool preserve_kernel_lookaside)
171{
172 struct slb_shadow *p = get_slb_shadow();
173 unsigned long ksp_esid_data, ksp_vsid_data;
174 u32 ih;
175
176
177
178
179
180
181
182
183
184 if (preserve_kernel_lookaside)
185 ih = 1;
186 else
187 ih = 0;
188
189 ksp_esid_data = be64_to_cpu(p->save_area[KSTACK_INDEX].esid);
190 ksp_vsid_data = be64_to_cpu(p->save_area[KSTACK_INDEX].vsid);
191
192 asm volatile(PPC_SLBIA(%0)" \n"
193 "slbmte %1, %2 \n"
194 :: "i" (ih),
195 "r" (ksp_vsid_data),
196 "r" (ksp_esid_data)
197 : "memory");
198}
199
200
201
202
203
204void slb_flush_and_restore_bolted(void)
205{
206 BUILD_BUG_ON(SLB_NUM_BOLTED != 2);
207
208 WARN_ON(!irqs_disabled());
209
210
211
212
213
214 hard_irq_disable();
215
216 isync();
217 __slb_flush_and_restore_bolted(false);
218 isync();
219
220 assert_slb_presence(true, get_paca()->kstack);
221
222 get_paca()->slb_cache_ptr = 0;
223
224 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
225 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
226}
227
228void slb_save_contents(struct slb_entry *slb_ptr)
229{
230 int i;
231 unsigned long e, v;
232
233
234 get_paca()->slb_save_cache_ptr = get_paca()->slb_cache_ptr;
235
236 if (!slb_ptr)
237 return;
238
239 for (i = 0; i < mmu_slb_size; i++) {
240 asm volatile("slbmfee %0,%1" : "=r" (e) : "r" (i));
241 asm volatile("slbmfev %0,%1" : "=r" (v) : "r" (i));
242 slb_ptr->esid = e;
243 slb_ptr->vsid = v;
244 slb_ptr++;
245 }
246}
247
248void slb_dump_contents(struct slb_entry *slb_ptr)
249{
250 int i, n;
251 unsigned long e, v;
252 unsigned long llp;
253
254 if (!slb_ptr)
255 return;
256
257 pr_err("SLB contents of cpu 0x%x\n", smp_processor_id());
258 pr_err("Last SLB entry inserted at slot %d\n", get_paca()->stab_rr);
259
260 for (i = 0; i < mmu_slb_size; i++) {
261 e = slb_ptr->esid;
262 v = slb_ptr->vsid;
263 slb_ptr++;
264
265 if (!e && !v)
266 continue;
267
268 pr_err("%02d %016lx %016lx\n", i, e, v);
269
270 if (!(e & SLB_ESID_V)) {
271 pr_err("\n");
272 continue;
273 }
274 llp = v & SLB_VSID_LLP;
275 if (v & SLB_VSID_B_1T) {
276 pr_err(" 1T ESID=%9lx VSID=%13lx LLP:%3lx\n",
277 GET_ESID_1T(e),
278 (v & ~SLB_VSID_B) >> SLB_VSID_SHIFT_1T, llp);
279 } else {
280 pr_err(" 256M ESID=%9lx VSID=%13lx LLP:%3lx\n",
281 GET_ESID(e),
282 (v & ~SLB_VSID_B) >> SLB_VSID_SHIFT, llp);
283 }
284 }
285 pr_err("----------------------------------\n");
286
287
288 pr_err("SLB cache ptr value = %d\n", get_paca()->slb_save_cache_ptr);
289 pr_err("Valid SLB cache entries:\n");
290 n = min_t(int, get_paca()->slb_save_cache_ptr, SLB_CACHE_ENTRIES);
291 for (i = 0; i < n; i++)
292 pr_err("%02d EA[0-35]=%9x\n", i, get_paca()->slb_cache[i]);
293 pr_err("Rest of SLB cache entries:\n");
294 for (i = n; i < SLB_CACHE_ENTRIES; i++)
295 pr_err("%02d EA[0-35]=%9x\n", i, get_paca()->slb_cache[i]);
296}
297
298void slb_vmalloc_update(void)
299{
300
301
302
303 slb_flush_and_restore_bolted();
304}
305
306static bool preload_hit(struct thread_info *ti, unsigned long esid)
307{
308 unsigned char i;
309
310 for (i = 0; i < ti->slb_preload_nr; i++) {
311 unsigned char idx;
312
313 idx = (ti->slb_preload_tail + i) % SLB_PRELOAD_NR;
314 if (esid == ti->slb_preload_esid[idx])
315 return true;
316 }
317 return false;
318}
319
320static bool preload_add(struct thread_info *ti, unsigned long ea)
321{
322 unsigned char idx;
323 unsigned long esid;
324
325 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) {
326
327 if (ea & ESID_MASK_1T)
328 ea &= ESID_MASK_1T;
329 }
330
331 esid = ea >> SID_SHIFT;
332
333 if (preload_hit(ti, esid))
334 return false;
335
336 idx = (ti->slb_preload_tail + ti->slb_preload_nr) % SLB_PRELOAD_NR;
337 ti->slb_preload_esid[idx] = esid;
338 if (ti->slb_preload_nr == SLB_PRELOAD_NR)
339 ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR;
340 else
341 ti->slb_preload_nr++;
342
343 return true;
344}
345
346static void preload_age(struct thread_info *ti)
347{
348 if (!ti->slb_preload_nr)
349 return;
350 ti->slb_preload_nr--;
351 ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR;
352}
353
354void slb_setup_new_exec(void)
355{
356 struct thread_info *ti = current_thread_info();
357 struct mm_struct *mm = current->mm;
358 unsigned long exec = 0x10000000;
359
360 WARN_ON(irqs_disabled());
361
362
363
364
365
366 if (ti->slb_preload_nr + 2 > SLB_PRELOAD_NR)
367 return;
368
369 hard_irq_disable();
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387 if (!is_kernel_addr(exec)) {
388 if (preload_add(ti, exec))
389 slb_allocate_user(mm, exec);
390 }
391
392
393 if (!is_kernel_addr(mm->mmap_base)) {
394 if (preload_add(ti, mm->mmap_base))
395 slb_allocate_user(mm, mm->mmap_base);
396 }
397
398
399 asm volatile("isync" : : : "memory");
400
401 local_irq_enable();
402}
403
404void preload_new_slb_context(unsigned long start, unsigned long sp)
405{
406 struct thread_info *ti = current_thread_info();
407 struct mm_struct *mm = current->mm;
408 unsigned long heap = mm->start_brk;
409
410 WARN_ON(irqs_disabled());
411
412
413 if (ti->slb_preload_nr + 3 > SLB_PRELOAD_NR)
414 return;
415
416 hard_irq_disable();
417
418
419 if (!is_kernel_addr(start)) {
420 if (preload_add(ti, start))
421 slb_allocate_user(mm, start);
422 }
423
424
425 if (!is_kernel_addr(sp)) {
426 if (preload_add(ti, sp))
427 slb_allocate_user(mm, sp);
428 }
429
430
431 if (heap && !is_kernel_addr(heap)) {
432 if (preload_add(ti, heap))
433 slb_allocate_user(mm, heap);
434 }
435
436
437 asm volatile("isync" : : : "memory");
438
439 local_irq_enable();
440}
441
442static void slb_cache_slbie_kernel(unsigned int index)
443{
444 unsigned long slbie_data = get_paca()->slb_cache[index];
445 unsigned long ksp = get_paca()->kstack;
446
447 slbie_data <<= SID_SHIFT;
448 slbie_data |= 0xc000000000000000ULL;
449 if ((ksp & slb_esid_mask(mmu_kernel_ssize)) == slbie_data)
450 return;
451 slbie_data |= mmu_kernel_ssize << SLBIE_SSIZE_SHIFT;
452
453 asm volatile("slbie %0" : : "r" (slbie_data));
454}
455
456static void slb_cache_slbie_user(unsigned int index)
457{
458 unsigned long slbie_data = get_paca()->slb_cache[index];
459
460 slbie_data <<= SID_SHIFT;
461 slbie_data |= user_segment_size(slbie_data) << SLBIE_SSIZE_SHIFT;
462 slbie_data |= SLBIE_C;
463
464 asm volatile("slbie %0" : : "r" (slbie_data));
465}
466
467
468void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
469{
470 struct thread_info *ti = task_thread_info(tsk);
471 unsigned char i;
472
473
474
475
476
477
478
479 hard_irq_disable();
480 isync();
481 if (stress_slb()) {
482 __slb_flush_and_restore_bolted(false);
483 isync();
484 get_paca()->slb_cache_ptr = 0;
485 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
486
487 } else if (cpu_has_feature(CPU_FTR_ARCH_300)) {
488
489
490
491
492
493
494 asm volatile(PPC_SLBIA(3));
495
496 } else {
497 unsigned long offset = get_paca()->slb_cache_ptr;
498
499 if (!mmu_has_feature(MMU_FTR_NO_SLBIE_B) &&
500 offset <= SLB_CACHE_ENTRIES) {
501
502
503
504
505
506
507 for (i = 0; i < offset; i++)
508 slb_cache_slbie_user(i);
509
510
511 if (!cpu_has_feature(CPU_FTR_ARCH_207S) && offset == 1)
512 slb_cache_slbie_user(0);
513
514 } else {
515
516 __slb_flush_and_restore_bolted(true);
517 isync();
518
519 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
520 }
521
522 get_paca()->slb_cache_ptr = 0;
523 }
524 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
525
526 copy_mm_to_paca(mm);
527
528
529
530
531
532
533
534 tsk->thread.load_slb++;
535 if (!tsk->thread.load_slb) {
536 unsigned long pc = KSTK_EIP(tsk);
537
538 preload_age(ti);
539 preload_add(ti, pc);
540 }
541
542 for (i = 0; i < ti->slb_preload_nr; i++) {
543 unsigned char idx;
544 unsigned long ea;
545
546 idx = (ti->slb_preload_tail + i) % SLB_PRELOAD_NR;
547 ea = (unsigned long)ti->slb_preload_esid[idx] << SID_SHIFT;
548
549 slb_allocate_user(mm, ea);
550 }
551
552
553
554
555
556
557 isync();
558}
559
560void slb_set_size(u16 size)
561{
562 mmu_slb_size = size;
563}
564
565void slb_initialize(void)
566{
567 unsigned long linear_llp, vmalloc_llp, io_llp;
568 unsigned long lflags;
569 static int slb_encoding_inited;
570#ifdef CONFIG_SPARSEMEM_VMEMMAP
571 unsigned long vmemmap_llp;
572#endif
573
574
575 linear_llp = mmu_psize_defs[mmu_linear_psize].sllp;
576 io_llp = mmu_psize_defs[mmu_io_psize].sllp;
577 vmalloc_llp = mmu_psize_defs[mmu_vmalloc_psize].sllp;
578 get_paca()->vmalloc_sllp = SLB_VSID_KERNEL | vmalloc_llp;
579#ifdef CONFIG_SPARSEMEM_VMEMMAP
580 vmemmap_llp = mmu_psize_defs[mmu_vmemmap_psize].sllp;
581#endif
582 if (!slb_encoding_inited) {
583 slb_encoding_inited = 1;
584 pr_devel("SLB: linear LLP = %04lx\n", linear_llp);
585 pr_devel("SLB: io LLP = %04lx\n", io_llp);
586#ifdef CONFIG_SPARSEMEM_VMEMMAP
587 pr_devel("SLB: vmemmap LLP = %04lx\n", vmemmap_llp);
588#endif
589 }
590
591 get_paca()->stab_rr = SLB_NUM_BOLTED - 1;
592 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
593 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
594
595 lflags = SLB_VSID_KERNEL | linear_llp;
596
597
598 asm volatile("isync":::"memory");
599 asm volatile("slbmte %0,%0"::"r" (0) : "memory");
600 asm volatile("isync; slbia; isync":::"memory");
601 create_shadowed_slbe(PAGE_OFFSET, mmu_kernel_ssize, lflags, LINEAR_INDEX);
602
603
604
605
606
607
608
609 slb_shadow_clear(KSTACK_INDEX);
610 if (raw_smp_processor_id() != boot_cpuid &&
611 (get_paca()->kstack & slb_esid_mask(mmu_kernel_ssize)) > PAGE_OFFSET)
612 create_shadowed_slbe(get_paca()->kstack,
613 mmu_kernel_ssize, lflags, KSTACK_INDEX);
614
615 asm volatile("isync":::"memory");
616}
617
618static void slb_cache_update(unsigned long esid_data)
619{
620 int slb_cache_index;
621
622 if (cpu_has_feature(CPU_FTR_ARCH_300))
623 return;
624
625 if (stress_slb())
626 return;
627
628
629
630
631 slb_cache_index = local_paca->slb_cache_ptr;
632 if (slb_cache_index < SLB_CACHE_ENTRIES) {
633
634
635
636
637 local_paca->slb_cache[slb_cache_index++] = esid_data >> SID_SHIFT;
638 local_paca->slb_cache_ptr++;
639 } else {
640
641
642
643
644
645 local_paca->slb_cache_ptr = SLB_CACHE_ENTRIES + 1;
646 }
647}
648
649static enum slb_index alloc_slb_index(bool kernel)
650{
651 enum slb_index index;
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669 if (local_paca->slb_used_bitmap != U32_MAX) {
670 index = ffz(local_paca->slb_used_bitmap);
671 local_paca->slb_used_bitmap |= 1U << index;
672 if (kernel)
673 local_paca->slb_kern_bitmap |= 1U << index;
674 } else {
675
676 index = local_paca->stab_rr;
677 if (index < (mmu_slb_size - 1))
678 index++;
679 else
680 index = SLB_NUM_BOLTED;
681 local_paca->stab_rr = index;
682 if (index < 32) {
683 if (kernel)
684 local_paca->slb_kern_bitmap |= 1U << index;
685 else
686 local_paca->slb_kern_bitmap &= ~(1U << index);
687 }
688 }
689 BUG_ON(index < SLB_NUM_BOLTED);
690
691 return index;
692}
693
694static long slb_insert_entry(unsigned long ea, unsigned long context,
695 unsigned long flags, int ssize, bool kernel)
696{
697 unsigned long vsid;
698 unsigned long vsid_data, esid_data;
699 enum slb_index index;
700
701 vsid = get_vsid(context, ea, ssize);
702 if (!vsid)
703 return -EFAULT;
704
705
706
707
708
709
710
711
712
713
714 barrier();
715
716 index = alloc_slb_index(kernel);
717
718 vsid_data = __mk_vsid_data(vsid, ssize, flags);
719 esid_data = mk_esid_data(ea, ssize, index);
720
721
722
723
724
725
726
727 assert_slb_presence(false, ea);
728 if (stress_slb()) {
729 int slb_cache_index = local_paca->slb_cache_ptr;
730
731
732
733
734
735
736
737 BUILD_BUG_ON(SLB_CACHE_ENTRIES < 3);
738 if (!kernel || slb_cache_index == 3) {
739 int i;
740
741 for (i = 0; i < slb_cache_index; i++)
742 slb_cache_slbie_kernel(i);
743 slb_cache_index = 0;
744 }
745
746 if (kernel)
747 local_paca->slb_cache[slb_cache_index++] = esid_data >> SID_SHIFT;
748 local_paca->slb_cache_ptr = slb_cache_index;
749 }
750 asm volatile("slbmte %0, %1" : : "r" (vsid_data), "r" (esid_data));
751
752 barrier();
753
754 if (!kernel)
755 slb_cache_update(esid_data);
756
757 return 0;
758}
759
760static long slb_allocate_kernel(unsigned long ea, unsigned long id)
761{
762 unsigned long context;
763 unsigned long flags;
764 int ssize;
765
766 if (id == LINEAR_MAP_REGION_ID) {
767
768
769 if ((ea & EA_MASK) > (1UL << H_MAX_PHYSMEM_BITS))
770 return -EFAULT;
771
772 flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_linear_psize].sllp;
773
774#ifdef CONFIG_SPARSEMEM_VMEMMAP
775 } else if (id == VMEMMAP_REGION_ID) {
776
777 if (ea >= H_VMEMMAP_END)
778 return -EFAULT;
779
780 flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_vmemmap_psize].sllp;
781#endif
782 } else if (id == VMALLOC_REGION_ID) {
783
784 if (ea >= H_VMALLOC_END)
785 return -EFAULT;
786
787 flags = local_paca->vmalloc_sllp;
788
789 } else if (id == IO_REGION_ID) {
790
791 if (ea >= H_KERN_IO_END)
792 return -EFAULT;
793
794 flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_io_psize].sllp;
795
796 } else {
797 return -EFAULT;
798 }
799
800 ssize = MMU_SEGSIZE_1T;
801 if (!mmu_has_feature(MMU_FTR_1T_SEGMENT))
802 ssize = MMU_SEGSIZE_256M;
803
804 context = get_kernel_context(ea);
805
806 return slb_insert_entry(ea, context, flags, ssize, true);
807}
808
809static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)
810{
811 unsigned long context;
812 unsigned long flags;
813 int bpsize;
814 int ssize;
815
816
817
818
819
820 if (ea >= mm_ctx_slb_addr_limit(&mm->context))
821 return -EFAULT;
822
823 context = get_user_context(&mm->context, ea);
824 if (!context)
825 return -EFAULT;
826
827 if (unlikely(ea >= H_PGTABLE_RANGE)) {
828 WARN_ON(1);
829 return -EFAULT;
830 }
831
832 ssize = user_segment_size(ea);
833
834 bpsize = get_slice_psize(mm, ea);
835 flags = SLB_VSID_USER | mmu_psize_defs[bpsize].sllp;
836
837 return slb_insert_entry(ea, context, flags, ssize, false);
838}
839
840long do_slb_fault(struct pt_regs *regs, unsigned long ea)
841{
842 unsigned long id = get_region_id(ea);
843
844
845 VM_WARN_ON(mfmsr() & MSR_EE);
846
847 if (unlikely(!(regs->msr & MSR_RI)))
848 return -EINVAL;
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865 if (id >= LINEAR_MAP_REGION_ID) {
866 long err;
867#ifdef CONFIG_DEBUG_VM
868
869 BUG_ON(local_paca->in_kernel_slb_handler);
870 local_paca->in_kernel_slb_handler = 1;
871#endif
872 err = slb_allocate_kernel(ea, id);
873#ifdef CONFIG_DEBUG_VM
874 local_paca->in_kernel_slb_handler = 0;
875#endif
876 return err;
877 } else {
878 struct mm_struct *mm = current->mm;
879 long err;
880
881 if (unlikely(!mm))
882 return -EFAULT;
883
884 err = slb_allocate_user(mm, ea);
885 if (!err)
886 preload_add(current_thread_info(), ea);
887
888 return err;
889 }
890}
891
892void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err)
893{
894 if (err == -EFAULT) {
895 if (user_mode(regs))
896 _exception(SIGSEGV, regs, SEGV_BNDERR, ea);
897 else
898 bad_page_fault(regs, ea, SIGSEGV);
899 } else if (err == -EINVAL) {
900 unrecoverable_exception(regs);
901 } else {
902 BUG();
903 }
904}
905