1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/module.h>
21#include <linux/signal.h>
22#include <linux/sched.h>
23#include <linux/kernel.h>
24#include <linux/errno.h>
25#include <linux/string.h>
26#include <linux/types.h>
27#include <linux/ptrace.h>
28#include <linux/mman.h>
29#include <linux/mm.h>
30#include <linux/interrupt.h>
31
32#include <asm/page.h>
33#include <asm/pgtable.h>
34#include <asm/mmu.h>
35#include <linux/mmu_context.h>
36#include <linux/uaccess.h>
37#include <asm/exceptions.h>
38
39static unsigned long pte_misses;
40static unsigned long pte_errors;
41
42
43
44
45
46static int store_updates_sp(struct pt_regs *regs)
47{
48 unsigned int inst;
49
50 if (get_user(inst, (unsigned int __user *)regs->pc))
51 return 0;
52
53 if (((inst >> 21) & 0x1f) != 1)
54 return 0;
55
56 if ((inst & 0xd0000000) == 0xd0000000)
57 return 1;
58 return 0;
59}
60
61
62
63
64
65
66
67void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
68{
69 const struct exception_table_entry *fixup;
70
71
72 fixup = search_exception_tables(regs->pc);
73 if (fixup) {
74 regs->pc = fixup->fixup;
75 return;
76 }
77
78
79 die("kernel access of bad area", regs, sig);
80}
81
82
83
84
85
86void do_page_fault(struct pt_regs *regs, unsigned long address,
87 unsigned long error_code)
88{
89 struct vm_area_struct *vma;
90 struct mm_struct *mm = current->mm;
91 siginfo_t info;
92 int code = SEGV_MAPERR;
93 int is_write = error_code & ESR_S;
94 int fault;
95 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
96
97 regs->ear = address;
98 regs->esr = error_code;
99
100
101 if (unlikely(kernel_mode(regs) && (address >= TASK_SIZE))) {
102 pr_warn("kernel task_size exceed");
103 _exception(SIGSEGV, regs, code, address);
104 }
105
106
107 if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11)
108 is_write = 0;
109
110 if (unlikely(faulthandler_disabled() || !mm)) {
111 if (kernel_mode(regs))
112 goto bad_area_nosemaphore;
113
114
115
116 pr_emerg("Page fault in user mode with faulthandler_disabled(), mm = %p\n",
117 mm);
118 pr_emerg("r15 = %lx MSR = %lx\n",
119 regs->r15, regs->msr);
120 die("Weird page fault", regs, SIGSEGV);
121 }
122
123 if (user_mode(regs))
124 flags |= FAULT_FLAG_USER;
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
142 if (kernel_mode(regs) && !search_exception_tables(regs->pc))
143 goto bad_area_nosemaphore;
144
145retry:
146 down_read(&mm->mmap_sem);
147 }
148
149 vma = find_vma(mm, address);
150 if (unlikely(!vma))
151 goto bad_area;
152
153 if (vma->vm_start <= address)
154 goto good_area;
155
156 if (unlikely(!(vma->vm_flags & VM_GROWSDOWN)))
157 goto bad_area;
158
159 if (unlikely(!is_write))
160 goto bad_area;
161
162
163
164
165
166
167
168
169
170
171 if (unlikely(address + 0x100000 < vma->vm_end)) {
172
173
174 struct pt_regs *uregs = current->thread.regs;
175 if (uregs == NULL)
176 goto bad_area;
177
178
179
180
181
182
183
184
185
186
187
188
189
190 if (address + 2048 < uregs->r1
191 && (kernel_mode(regs) || !store_updates_sp(regs)))
192 goto bad_area;
193 }
194 if (expand_stack(vma, address))
195 goto bad_area;
196
197good_area:
198 code = SEGV_ACCERR;
199
200
201 if (unlikely(is_write)) {
202 if (unlikely(!(vma->vm_flags & VM_WRITE)))
203 goto bad_area;
204 flags |= FAULT_FLAG_WRITE;
205
206 } else {
207
208 if (unlikely(error_code & 0x08000000))
209 goto bad_area;
210 if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC))))
211 goto bad_area;
212 }
213
214
215
216
217
218
219 fault = handle_mm_fault(mm, vma, address, flags);
220
221 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
222 return;
223
224 if (unlikely(fault & VM_FAULT_ERROR)) {
225 if (fault & VM_FAULT_OOM)
226 goto out_of_memory;
227 else if (fault & VM_FAULT_SIGSEGV)
228 goto bad_area;
229 else if (fault & VM_FAULT_SIGBUS)
230 goto do_sigbus;
231 BUG();
232 }
233
234 if (flags & FAULT_FLAG_ALLOW_RETRY) {
235 if (unlikely(fault & VM_FAULT_MAJOR))
236 current->maj_flt++;
237 else
238 current->min_flt++;
239 if (fault & VM_FAULT_RETRY) {
240 flags &= ~FAULT_FLAG_ALLOW_RETRY;
241 flags |= FAULT_FLAG_TRIED;
242
243
244
245
246
247
248
249 goto retry;
250 }
251 }
252
253 up_read(&mm->mmap_sem);
254
255
256
257
258
259
260 pte_misses++;
261 return;
262
263bad_area:
264 up_read(&mm->mmap_sem);
265
266bad_area_nosemaphore:
267 pte_errors++;
268
269
270 if (user_mode(regs)) {
271 _exception(SIGSEGV, regs, code, address);
272
273
274
275
276
277 return;
278 }
279
280 bad_page_fault(regs, address, SIGSEGV);
281 return;
282
283
284
285
286
287out_of_memory:
288 up_read(&mm->mmap_sem);
289 if (!user_mode(regs))
290 bad_page_fault(regs, address, SIGKILL);
291 else
292 pagefault_out_of_memory();
293 return;
294
295do_sigbus:
296 up_read(&mm->mmap_sem);
297 if (user_mode(regs)) {
298 info.si_signo = SIGBUS;
299 info.si_errno = 0;
300 info.si_code = BUS_ADRERR;
301 info.si_addr = (void __user *)address;
302 force_sig_info(SIGBUS, &info, current);
303 return;
304 }
305 bad_page_fault(regs, address, SIGBUS);
306}
307