1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75#include <linux/mm.h>
76#include <linux/module.h>
77#include <linux/signal.h>
78#include <linux/smp.h>
79#include <linux/sched.h>
80#include <linux/debugfs.h>
81#include <linux/perf_event.h>
82
83#include <asm/asm.h>
84#include <asm/branch.h>
85#include <asm/byteorder.h>
86#include <asm/cop2.h>
87#include <asm/inst.h>
88#include <asm/uaccess.h>
89#include <asm/system.h>
90
91#define STR(x) __STR(x)
92#define __STR(x) #x
93
94enum {
95 UNALIGNED_ACTION_QUIET,
96 UNALIGNED_ACTION_SIGNAL,
97 UNALIGNED_ACTION_SHOW,
98};
99#ifdef CONFIG_DEBUG_FS
100static u32 unaligned_instructions;
101static u32 unaligned_action;
102#else
103#define unaligned_action UNALIGNED_ACTION_QUIET
104#endif
105extern void show_registers(struct pt_regs *regs);
106
107static void emulate_load_store_insn(struct pt_regs *regs,
108 void __user *addr, unsigned int __user *pc)
109{
110 union mips_instruction insn;
111 unsigned long value;
112 unsigned int res;
113
114 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS,
115 1, 0, regs, 0);
116
117
118
119
120 __get_user(insn.word, pc);
121
122 switch (insn.i_format.opcode) {
123
124
125
126
127
128
129 case ll_op:
130 case lld_op:
131 case sc_op:
132 case scd_op:
133
134
135
136
137
138
139 case ldl_op:
140 case ldr_op:
141 case lwl_op:
142 case lwr_op:
143 case sdl_op:
144 case sdr_op:
145 case swl_op:
146 case swr_op:
147 case lb_op:
148 case lbu_op:
149 case sb_op:
150 goto sigbus;
151
152
153
154
155 case lh_op:
156 if (!access_ok(VERIFY_READ, addr, 2))
157 goto sigbus;
158
159 __asm__ __volatile__ (".set\tnoat\n"
160#ifdef __BIG_ENDIAN
161 "1:\tlb\t%0, 0(%2)\n"
162 "2:\tlbu\t$1, 1(%2)\n\t"
163#endif
164#ifdef __LITTLE_ENDIAN
165 "1:\tlb\t%0, 1(%2)\n"
166 "2:\tlbu\t$1, 0(%2)\n\t"
167#endif
168 "sll\t%0, 0x8\n\t"
169 "or\t%0, $1\n\t"
170 "li\t%1, 0\n"
171 "3:\t.set\tat\n\t"
172 ".section\t.fixup,\"ax\"\n\t"
173 "4:\tli\t%1, %3\n\t"
174 "j\t3b\n\t"
175 ".previous\n\t"
176 ".section\t__ex_table,\"a\"\n\t"
177 STR(PTR)"\t1b, 4b\n\t"
178 STR(PTR)"\t2b, 4b\n\t"
179 ".previous"
180 : "=&r" (value), "=r" (res)
181 : "r" (addr), "i" (-EFAULT));
182 if (res)
183 goto fault;
184 compute_return_epc(regs);
185 regs->regs[insn.i_format.rt] = value;
186 break;
187
188 case lw_op:
189 if (!access_ok(VERIFY_READ, addr, 4))
190 goto sigbus;
191
192 __asm__ __volatile__ (
193#ifdef __BIG_ENDIAN
194 "1:\tlwl\t%0, (%2)\n"
195 "2:\tlwr\t%0, 3(%2)\n\t"
196#endif
197#ifdef __LITTLE_ENDIAN
198 "1:\tlwl\t%0, 3(%2)\n"
199 "2:\tlwr\t%0, (%2)\n\t"
200#endif
201 "li\t%1, 0\n"
202 "3:\t.section\t.fixup,\"ax\"\n\t"
203 "4:\tli\t%1, %3\n\t"
204 "j\t3b\n\t"
205 ".previous\n\t"
206 ".section\t__ex_table,\"a\"\n\t"
207 STR(PTR)"\t1b, 4b\n\t"
208 STR(PTR)"\t2b, 4b\n\t"
209 ".previous"
210 : "=&r" (value), "=r" (res)
211 : "r" (addr), "i" (-EFAULT));
212 if (res)
213 goto fault;
214 compute_return_epc(regs);
215 regs->regs[insn.i_format.rt] = value;
216 break;
217
218 case lhu_op:
219 if (!access_ok(VERIFY_READ, addr, 2))
220 goto sigbus;
221
222 __asm__ __volatile__ (
223 ".set\tnoat\n"
224#ifdef __BIG_ENDIAN
225 "1:\tlbu\t%0, 0(%2)\n"
226 "2:\tlbu\t$1, 1(%2)\n\t"
227#endif
228#ifdef __LITTLE_ENDIAN
229 "1:\tlbu\t%0, 1(%2)\n"
230 "2:\tlbu\t$1, 0(%2)\n\t"
231#endif
232 "sll\t%0, 0x8\n\t"
233 "or\t%0, $1\n\t"
234 "li\t%1, 0\n"
235 "3:\t.set\tat\n\t"
236 ".section\t.fixup,\"ax\"\n\t"
237 "4:\tli\t%1, %3\n\t"
238 "j\t3b\n\t"
239 ".previous\n\t"
240 ".section\t__ex_table,\"a\"\n\t"
241 STR(PTR)"\t1b, 4b\n\t"
242 STR(PTR)"\t2b, 4b\n\t"
243 ".previous"
244 : "=&r" (value), "=r" (res)
245 : "r" (addr), "i" (-EFAULT));
246 if (res)
247 goto fault;
248 compute_return_epc(regs);
249 regs->regs[insn.i_format.rt] = value;
250 break;
251
252 case lwu_op:
253#ifdef CONFIG_64BIT
254
255
256
257
258
259
260
261 if (!access_ok(VERIFY_READ, addr, 4))
262 goto sigbus;
263
264 __asm__ __volatile__ (
265#ifdef __BIG_ENDIAN
266 "1:\tlwl\t%0, (%2)\n"
267 "2:\tlwr\t%0, 3(%2)\n\t"
268#endif
269#ifdef __LITTLE_ENDIAN
270 "1:\tlwl\t%0, 3(%2)\n"
271 "2:\tlwr\t%0, (%2)\n\t"
272#endif
273 "dsll\t%0, %0, 32\n\t"
274 "dsrl\t%0, %0, 32\n\t"
275 "li\t%1, 0\n"
276 "3:\t.section\t.fixup,\"ax\"\n\t"
277 "4:\tli\t%1, %3\n\t"
278 "j\t3b\n\t"
279 ".previous\n\t"
280 ".section\t__ex_table,\"a\"\n\t"
281 STR(PTR)"\t1b, 4b\n\t"
282 STR(PTR)"\t2b, 4b\n\t"
283 ".previous"
284 : "=&r" (value), "=r" (res)
285 : "r" (addr), "i" (-EFAULT));
286 if (res)
287 goto fault;
288 compute_return_epc(regs);
289 regs->regs[insn.i_format.rt] = value;
290 break;
291#endif
292
293
294 goto sigill;
295
296 case ld_op:
297#ifdef CONFIG_64BIT
298
299
300
301
302
303
304
305 if (!access_ok(VERIFY_READ, addr, 8))
306 goto sigbus;
307
308 __asm__ __volatile__ (
309#ifdef __BIG_ENDIAN
310 "1:\tldl\t%0, (%2)\n"
311 "2:\tldr\t%0, 7(%2)\n\t"
312#endif
313#ifdef __LITTLE_ENDIAN
314 "1:\tldl\t%0, 7(%2)\n"
315 "2:\tldr\t%0, (%2)\n\t"
316#endif
317 "li\t%1, 0\n"
318 "3:\t.section\t.fixup,\"ax\"\n\t"
319 "4:\tli\t%1, %3\n\t"
320 "j\t3b\n\t"
321 ".previous\n\t"
322 ".section\t__ex_table,\"a\"\n\t"
323 STR(PTR)"\t1b, 4b\n\t"
324 STR(PTR)"\t2b, 4b\n\t"
325 ".previous"
326 : "=&r" (value), "=r" (res)
327 : "r" (addr), "i" (-EFAULT));
328 if (res)
329 goto fault;
330 compute_return_epc(regs);
331 regs->regs[insn.i_format.rt] = value;
332 break;
333#endif
334
335
336 goto sigill;
337
338 case sh_op:
339 if (!access_ok(VERIFY_WRITE, addr, 2))
340 goto sigbus;
341
342 value = regs->regs[insn.i_format.rt];
343 __asm__ __volatile__ (
344#ifdef __BIG_ENDIAN
345 ".set\tnoat\n"
346 "1:\tsb\t%1, 1(%2)\n\t"
347 "srl\t$1, %1, 0x8\n"
348 "2:\tsb\t$1, 0(%2)\n\t"
349 ".set\tat\n\t"
350#endif
351#ifdef __LITTLE_ENDIAN
352 ".set\tnoat\n"
353 "1:\tsb\t%1, 0(%2)\n\t"
354 "srl\t$1,%1, 0x8\n"
355 "2:\tsb\t$1, 1(%2)\n\t"
356 ".set\tat\n\t"
357#endif
358 "li\t%0, 0\n"
359 "3:\n\t"
360 ".section\t.fixup,\"ax\"\n\t"
361 "4:\tli\t%0, %3\n\t"
362 "j\t3b\n\t"
363 ".previous\n\t"
364 ".section\t__ex_table,\"a\"\n\t"
365 STR(PTR)"\t1b, 4b\n\t"
366 STR(PTR)"\t2b, 4b\n\t"
367 ".previous"
368 : "=r" (res)
369 : "r" (value), "r" (addr), "i" (-EFAULT));
370 if (res)
371 goto fault;
372 compute_return_epc(regs);
373 break;
374
375 case sw_op:
376 if (!access_ok(VERIFY_WRITE, addr, 4))
377 goto sigbus;
378
379 value = regs->regs[insn.i_format.rt];
380 __asm__ __volatile__ (
381#ifdef __BIG_ENDIAN
382 "1:\tswl\t%1,(%2)\n"
383 "2:\tswr\t%1, 3(%2)\n\t"
384#endif
385#ifdef __LITTLE_ENDIAN
386 "1:\tswl\t%1, 3(%2)\n"
387 "2:\tswr\t%1, (%2)\n\t"
388#endif
389 "li\t%0, 0\n"
390 "3:\n\t"
391 ".section\t.fixup,\"ax\"\n\t"
392 "4:\tli\t%0, %3\n\t"
393 "j\t3b\n\t"
394 ".previous\n\t"
395 ".section\t__ex_table,\"a\"\n\t"
396 STR(PTR)"\t1b, 4b\n\t"
397 STR(PTR)"\t2b, 4b\n\t"
398 ".previous"
399 : "=r" (res)
400 : "r" (value), "r" (addr), "i" (-EFAULT));
401 if (res)
402 goto fault;
403 compute_return_epc(regs);
404 break;
405
406 case sd_op:
407#ifdef CONFIG_64BIT
408
409
410
411
412
413
414
415 if (!access_ok(VERIFY_WRITE, addr, 8))
416 goto sigbus;
417
418 value = regs->regs[insn.i_format.rt];
419 __asm__ __volatile__ (
420#ifdef __BIG_ENDIAN
421 "1:\tsdl\t%1,(%2)\n"
422 "2:\tsdr\t%1, 7(%2)\n\t"
423#endif
424#ifdef __LITTLE_ENDIAN
425 "1:\tsdl\t%1, 7(%2)\n"
426 "2:\tsdr\t%1, (%2)\n\t"
427#endif
428 "li\t%0, 0\n"
429 "3:\n\t"
430 ".section\t.fixup,\"ax\"\n\t"
431 "4:\tli\t%0, %3\n\t"
432 "j\t3b\n\t"
433 ".previous\n\t"
434 ".section\t__ex_table,\"a\"\n\t"
435 STR(PTR)"\t1b, 4b\n\t"
436 STR(PTR)"\t2b, 4b\n\t"
437 ".previous"
438 : "=r" (res)
439 : "r" (value), "r" (addr), "i" (-EFAULT));
440 if (res)
441 goto fault;
442 compute_return_epc(regs);
443 break;
444#endif
445
446
447 goto sigill;
448
449 case lwc1_op:
450 case ldc1_op:
451 case swc1_op:
452 case sdc1_op:
453
454
455
456 goto sigbus;
457
458
459
460
461
462
463 case lwc2_op:
464 cu2_notifier_call_chain(CU2_LWC2_OP, regs);
465 break;
466
467 case ldc2_op:
468 cu2_notifier_call_chain(CU2_LDC2_OP, regs);
469 break;
470
471 case swc2_op:
472 cu2_notifier_call_chain(CU2_SWC2_OP, regs);
473 break;
474
475 case sdc2_op:
476 cu2_notifier_call_chain(CU2_SDC2_OP, regs);
477 break;
478
479 default:
480
481
482
483
484 goto sigill;
485 }
486
487#ifdef CONFIG_DEBUG_FS
488 unaligned_instructions++;
489#endif
490
491 return;
492
493fault:
494
495 if (fixup_exception(regs))
496 return;
497
498 die_if_kernel("Unhandled kernel unaligned access", regs);
499 force_sig(SIGSEGV, current);
500
501 return;
502
503sigbus:
504 die_if_kernel("Unhandled kernel unaligned access", regs);
505 force_sig(SIGBUS, current);
506
507 return;
508
509sigill:
510 die_if_kernel("Unhandled kernel unaligned access or invalid instruction", regs);
511 force_sig(SIGILL, current);
512}
513
514asmlinkage void do_ade(struct pt_regs *regs)
515{
516 unsigned int __user *pc;
517 mm_segment_t seg;
518
519 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
520 1, 0, regs, regs->cp0_badvaddr);
521
522
523
524
525 if ((regs->cp0_badvaddr == regs->cp0_epc) || (regs->cp0_epc & 0x1))
526 goto sigbus;
527
528 pc = (unsigned int __user *) exception_epc(regs);
529 if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
530 goto sigbus;
531 if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
532 goto sigbus;
533 else if (unaligned_action == UNALIGNED_ACTION_SHOW)
534 show_registers(regs);
535
536
537
538
539
540 seg = get_fs();
541 if (!user_mode(regs))
542 set_fs(KERNEL_DS);
543 emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
544 set_fs(seg);
545
546 return;
547
548sigbus:
549 die_if_kernel("Kernel unaligned instruction access", regs);
550 force_sig(SIGBUS, current);
551
552
553
554
555}
556
557#ifdef CONFIG_DEBUG_FS
558extern struct dentry *mips_debugfs_dir;
559static int __init debugfs_unaligned(void)
560{
561 struct dentry *d;
562
563 if (!mips_debugfs_dir)
564 return -ENODEV;
565 d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
566 mips_debugfs_dir, &unaligned_instructions);
567 if (!d)
568 return -ENOMEM;
569 d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
570 mips_debugfs_dir, &unaligned_action);
571 if (!d)
572 return -ENOMEM;
573 return 0;
574}
575__initcall(debugfs_unaligned);
576#endif
577