linux/arch/ia64/kernel/unaligned.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Architecture-specific unaligned trap handling.
   4 *
   5 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
   6 *      Stephane Eranian <eranian@hpl.hp.com>
   7 *      David Mosberger-Tang <davidm@hpl.hp.com>
   8 *
   9 * 2002/12/09   Fix rotating register handling (off-by-1 error, missing fr-rotation).  Fix
  10 *              get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
  11 *              stacked register returns an undefined value; it does NOT trigger a
  12 *              "rsvd register fault").
  13 * 2001/10/11   Fix unaligned access to rotating registers in s/w pipelined loops.
  14 * 2001/08/13   Correct size of extended floats (float_fsz) from 16 to 10 bytes.
  15 * 2001/01/17   Add support emulation of unaligned kernel accesses.
  16 */
  17#include <linux/jiffies.h>
  18#include <linux/kernel.h>
  19#include <linux/sched/signal.h>
  20#include <linux/tty.h>
  21#include <linux/extable.h>
  22#include <linux/ratelimit.h>
  23#include <linux/uaccess.h>
  24
  25#include <asm/intrinsics.h>
  26#include <asm/processor.h>
  27#include <asm/rse.h>
  28#include <asm/exception.h>
  29#include <asm/unaligned.h>
  30
  31extern int die_if_kernel(char *str, struct pt_regs *regs, long err);
  32
  33#undef DEBUG_UNALIGNED_TRAP
  34
  35#ifdef DEBUG_UNALIGNED_TRAP
  36# define DPRINT(a...)   do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0)
  37# define DDUMP(str,vp,len)      dump(str, vp, len)
  38
  39static void
  40dump (const char *str, void *vp, size_t len)
  41{
  42        unsigned char *cp = vp;
  43        int i;
  44
  45        printk("%s", str);
  46        for (i = 0; i < len; ++i)
  47                printk (" %02x", *cp++);
  48        printk("\n");
  49}
  50#else
  51# define DPRINT(a...)
  52# define DDUMP(str,vp,len)
  53#endif
  54
  55#define IA64_FIRST_STACKED_GR   32
  56#define IA64_FIRST_ROTATING_FR  32
  57#define SIGN_EXT9               0xffffffffffffff00ul
  58
  59/*
  60 *  sysctl settable hook which tells the kernel whether to honor the
  61 *  IA64_THREAD_UAC_NOPRINT prctl.  Because this is user settable, we want
  62 *  to allow the super user to enable/disable this for security reasons
  63 *  (i.e. don't allow attacker to fill up logs with unaligned accesses).
  64 */
  65int no_unaligned_warning;
  66int unaligned_dump_stack;
  67
  68/*
  69 * For M-unit:
  70 *
  71 *  opcode |   m  |   x6    |
  72 * --------|------|---------|
  73 * [40-37] | [36] | [35:30] |
  74 * --------|------|---------|
  75 *     4   |   1  |    6    | = 11 bits
  76 * --------------------------
  77 * However bits [31:30] are not directly useful to distinguish between
  78 * load/store so we can use [35:32] instead, which gives the following
  79 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
  80 * checking the m-bit until later in the load/store emulation.
  81 */
  82#define IA64_OPCODE_MASK        0x1ef
  83#define IA64_OPCODE_SHIFT       32
  84
  85/*
  86 * Table C-28 Integer Load/Store
  87 *
  88 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
  89 *
  90 * ld8.fill, st8.fill  MUST be aligned because the RNATs are based on
  91 * the address (bits [8:3]), so we must failed.
  92 */
  93#define LD_OP            0x080
  94#define LDS_OP           0x081
  95#define LDA_OP           0x082
  96#define LDSA_OP          0x083
  97#define LDBIAS_OP        0x084
  98#define LDACQ_OP         0x085
  99/* 0x086, 0x087 are not relevant */
 100#define LDCCLR_OP        0x088
 101#define LDCNC_OP         0x089
 102#define LDCCLRACQ_OP     0x08a
 103#define ST_OP            0x08c
 104#define STREL_OP         0x08d
 105/* 0x08e,0x8f are not relevant */
 106
 107/*
 108 * Table C-29 Integer Load +Reg
 109 *
 110 * we use the ld->m (bit [36:36]) field to determine whether or not we have
 111 * a load/store of this form.
 112 */
 113
 114/*
 115 * Table C-30 Integer Load/Store +Imm
 116 *
 117 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
 118 *
 119 * ld8.fill, st8.fill  must be aligned because the Nat register are based on
 120 * the address, so we must fail and the program must be fixed.
 121 */
 122#define LD_IMM_OP            0x0a0
 123#define LDS_IMM_OP           0x0a1
 124#define LDA_IMM_OP           0x0a2
 125#define LDSA_IMM_OP          0x0a3
 126#define LDBIAS_IMM_OP        0x0a4
 127#define LDACQ_IMM_OP         0x0a5
 128/* 0x0a6, 0xa7 are not relevant */
 129#define LDCCLR_IMM_OP        0x0a8
 130#define LDCNC_IMM_OP         0x0a9
 131#define LDCCLRACQ_IMM_OP     0x0aa
 132#define ST_IMM_OP            0x0ac
 133#define STREL_IMM_OP         0x0ad
 134/* 0x0ae,0xaf are not relevant */
 135
 136/*
 137 * Table C-32 Floating-point Load/Store
 138 */
 139#define LDF_OP           0x0c0
 140#define LDFS_OP          0x0c1
 141#define LDFA_OP          0x0c2
 142#define LDFSA_OP         0x0c3
 143/* 0x0c6 is irrelevant */
 144#define LDFCCLR_OP       0x0c8
 145#define LDFCNC_OP        0x0c9
 146/* 0x0cb is irrelevant  */
 147#define STF_OP           0x0cc
 148
 149/*
 150 * Table C-33 Floating-point Load +Reg
 151 *
 152 * we use the ld->m (bit [36:36]) field to determine whether or not we have
 153 * a load/store of this form.
 154 */
 155
 156/*
 157 * Table C-34 Floating-point Load/Store +Imm
 158 */
 159#define LDF_IMM_OP       0x0e0
 160#define LDFS_IMM_OP      0x0e1
 161#define LDFA_IMM_OP      0x0e2
 162#define LDFSA_IMM_OP     0x0e3
 163/* 0x0e6 is irrelevant */
 164#define LDFCCLR_IMM_OP   0x0e8
 165#define LDFCNC_IMM_OP    0x0e9
 166#define STF_IMM_OP       0x0ec
 167
 168typedef struct {
 169        unsigned long    qp:6;  /* [0:5]   */
 170        unsigned long    r1:7;  /* [6:12]  */
 171        unsigned long   imm:7;  /* [13:19] */
 172        unsigned long    r3:7;  /* [20:26] */
 173        unsigned long     x:1;  /* [27:27] */
 174        unsigned long  hint:2;  /* [28:29] */
 175        unsigned long x6_sz:2;  /* [30:31] */
 176        unsigned long x6_op:4;  /* [32:35], x6 = x6_sz|x6_op */
 177        unsigned long     m:1;  /* [36:36] */
 178        unsigned long    op:4;  /* [37:40] */
 179        unsigned long   pad:23; /* [41:63] */
 180} load_store_t;
 181
 182
 183typedef enum {
 184        UPD_IMMEDIATE,  /* ldXZ r1=[r3],imm(9) */
 185        UPD_REG         /* ldXZ r1=[r3],r2     */
 186} update_t;
 187
 188/*
 189 * We use tables to keep track of the offsets of registers in the saved state.
 190 * This way we save having big switch/case statements.
 191 *
 192 * We use bit 0 to indicate switch_stack or pt_regs.
 193 * The offset is simply shifted by 1 bit.
 194 * A 2-byte value should be enough to hold any kind of offset
 195 *
 196 * In case the calling convention changes (and thus pt_regs/switch_stack)
 197 * simply use RSW instead of RPT or vice-versa.
 198 */
 199
 200#define RPO(x)  ((size_t) &((struct pt_regs *)0)->x)
 201#define RSO(x)  ((size_t) &((struct switch_stack *)0)->x)
 202
 203#define RPT(x)          (RPO(x) << 1)
 204#define RSW(x)          (1| RSO(x)<<1)
 205
 206#define GR_OFFS(x)      (gr_info[x]>>1)
 207#define GR_IN_SW(x)     (gr_info[x] & 0x1)
 208
 209#define FR_OFFS(x)      (fr_info[x]>>1)
 210#define FR_IN_SW(x)     (fr_info[x] & 0x1)
 211
 212static u16 gr_info[32]={
 213        0,                      /* r0 is read-only : WE SHOULD NEVER GET THIS */
 214
 215        RPT(r1), RPT(r2), RPT(r3),
 216
 217        RSW(r4), RSW(r5), RSW(r6), RSW(r7),
 218
 219        RPT(r8), RPT(r9), RPT(r10), RPT(r11),
 220        RPT(r12), RPT(r13), RPT(r14), RPT(r15),
 221
 222        RPT(r16), RPT(r17), RPT(r18), RPT(r19),
 223        RPT(r20), RPT(r21), RPT(r22), RPT(r23),
 224        RPT(r24), RPT(r25), RPT(r26), RPT(r27),
 225        RPT(r28), RPT(r29), RPT(r30), RPT(r31)
 226};
 227
 228static u16 fr_info[32]={
 229        0,                      /* constant : WE SHOULD NEVER GET THIS */
 230        0,                      /* constant : WE SHOULD NEVER GET THIS */
 231
 232        RSW(f2), RSW(f3), RSW(f4), RSW(f5),
 233
 234        RPT(f6), RPT(f7), RPT(f8), RPT(f9),
 235        RPT(f10), RPT(f11),
 236
 237        RSW(f12), RSW(f13), RSW(f14),
 238        RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19),
 239        RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24),
 240        RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29),
 241        RSW(f30), RSW(f31)
 242};
 243
 244/* Invalidate ALAT entry for integer register REGNO.  */
 245static void
 246invala_gr (int regno)
 247{
 248#       define F(reg)   case reg: ia64_invala_gr(reg); break
 249
 250        switch (regno) {
 251                F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
 252                F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
 253                F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
 254                F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
 255                F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
 256                F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
 257                F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
 258                F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
 259                F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
 260                F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
 261                F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
 262                F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
 263                F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
 264                F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
 265                F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
 266                F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
 267        }
 268#       undef F
 269}
 270
 271/* Invalidate ALAT entry for floating-point register REGNO.  */
 272static void
 273invala_fr (int regno)
 274{
 275#       define F(reg)   case reg: ia64_invala_fr(reg); break
 276
 277        switch (regno) {
 278                F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
 279                F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
 280                F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
 281                F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
 282                F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
 283                F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
 284                F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
 285                F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
 286                F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
 287                F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
 288                F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
 289                F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
 290                F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
 291                F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
 292                F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
 293                F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
 294        }
 295#       undef F
 296}
 297
 298static inline unsigned long
 299rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg)
 300{
 301        reg += rrb;
 302        if (reg >= sor)
 303                reg -= sor;
 304        return reg;
 305}
 306
 307static void
 308set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat)
 309{
 310        struct switch_stack *sw = (struct switch_stack *) regs - 1;
 311        unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end;
 312        unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
 313        unsigned long rnats, nat_mask;
 314        unsigned long on_kbs;
 315        long sof = (regs->cr_ifs) & 0x7f;
 316        long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
 317        long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
 318        long ridx = r1 - 32;
 319
 320        if (ridx >= sof) {
 321                /* this should never happen, as the "rsvd register fault" has higher priority */
 322                DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof);
 323                return;
 324        }
 325
 326        if (ridx < sor)
 327                ridx = rotate_reg(sor, rrb_gr, ridx);
 328
 329        DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
 330               r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
 331
 332        on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
 333        addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
 334        if (addr >= kbs) {
 335                /* the register is on the kernel backing store: easy... */
 336                rnat_addr = ia64_rse_rnat_addr(addr);
 337                if ((unsigned long) rnat_addr >= sw->ar_bspstore)
 338                        rnat_addr = &sw->ar_rnat;
 339                nat_mask = 1UL << ia64_rse_slot_num(addr);
 340
 341                *addr = val;
 342                if (nat)
 343                        *rnat_addr |=  nat_mask;
 344                else
 345                        *rnat_addr &= ~nat_mask;
 346                return;
 347        }
 348
 349        if (!user_stack(current, regs)) {
 350                DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1);
 351                return;
 352        }
 353
 354        bspstore = (unsigned long *)regs->ar_bspstore;
 355        ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
 356        bsp     = ia64_rse_skip_regs(ubs_end, -sof);
 357        addr    = ia64_rse_skip_regs(bsp, ridx);
 358
 359        DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
 360
 361        ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
 362
 363        rnat_addr = ia64_rse_rnat_addr(addr);
 364
 365        ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
 366        DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
 367               (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1);
 368
 369        nat_mask = 1UL << ia64_rse_slot_num(addr);
 370        if (nat)
 371                rnats |=  nat_mask;
 372        else
 373                rnats &= ~nat_mask;
 374        ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats);
 375
 376        DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats);
 377}
 378
 379
 380static void
 381get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat)
 382{
 383        struct switch_stack *sw = (struct switch_stack *) regs - 1;
 384        unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore;
 385        unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
 386        unsigned long rnats, nat_mask;
 387        unsigned long on_kbs;
 388        long sof = (regs->cr_ifs) & 0x7f;
 389        long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
 390        long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
 391        long ridx = r1 - 32;
 392
 393        if (ridx >= sof) {
 394                /* read of out-of-frame register returns an undefined value; 0 in our case.  */
 395                DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof);
 396                goto fail;
 397        }
 398
 399        if (ridx < sor)
 400                ridx = rotate_reg(sor, rrb_gr, ridx);
 401
 402        DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
 403               r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
 404
 405        on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
 406        addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
 407        if (addr >= kbs) {
 408                /* the register is on the kernel backing store: easy... */
 409                *val = *addr;
 410                if (nat) {
 411                        rnat_addr = ia64_rse_rnat_addr(addr);
 412                        if ((unsigned long) rnat_addr >= sw->ar_bspstore)
 413                                rnat_addr = &sw->ar_rnat;
 414                        nat_mask = 1UL << ia64_rse_slot_num(addr);
 415                        *nat = (*rnat_addr & nat_mask) != 0;
 416                }
 417                return;
 418        }
 419
 420        if (!user_stack(current, regs)) {
 421                DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1);
 422                goto fail;
 423        }
 424
 425        bspstore = (unsigned long *)regs->ar_bspstore;
 426        ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
 427        bsp     = ia64_rse_skip_regs(ubs_end, -sof);
 428        addr    = ia64_rse_skip_regs(bsp, ridx);
 429
 430        DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
 431
 432        ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
 433
 434        if (nat) {
 435                rnat_addr = ia64_rse_rnat_addr(addr);
 436                nat_mask = 1UL << ia64_rse_slot_num(addr);
 437
 438                DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats);
 439
 440                ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
 441                *nat = (rnats & nat_mask) != 0;
 442        }
 443        return;
 444
 445  fail:
 446        *val = 0;
 447        if (nat)
 448                *nat = 0;
 449        return;
 450}
 451
 452
 453static void
 454setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs)
 455{
 456        struct switch_stack *sw = (struct switch_stack *) regs - 1;
 457        unsigned long addr;
 458        unsigned long bitmask;
 459        unsigned long *unat;
 460
 461        /*
 462         * First takes care of stacked registers
 463         */
 464        if (regnum >= IA64_FIRST_STACKED_GR) {
 465                set_rse_reg(regs, regnum, val, nat);
 466                return;
 467        }
 468
 469        /*
 470         * Using r0 as a target raises a General Exception fault which has higher priority
 471         * than the Unaligned Reference fault.
 472         */
 473
 474        /*
 475         * Now look at registers in [0-31] range and init correct UNAT
 476         */
 477        if (GR_IN_SW(regnum)) {
 478                addr = (unsigned long)sw;
 479                unat = &sw->ar_unat;
 480        } else {
 481                addr = (unsigned long)regs;
 482                unat = &sw->caller_unat;
 483        }
 484        DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
 485               addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum));
 486        /*
 487         * add offset from base of struct
 488         * and do it !
 489         */
 490        addr += GR_OFFS(regnum);
 491
 492        *(unsigned long *)addr = val;
 493
 494        /*
 495         * We need to clear the corresponding UNAT bit to fully emulate the load
 496         * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
 497         */
 498        bitmask   = 1UL << (addr >> 3 & 0x3f);
 499        DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat);
 500        if (nat) {
 501                *unat |= bitmask;
 502        } else {
 503                *unat &= ~bitmask;
 504        }
 505        DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat);
 506}
 507
 508/*
 509 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
 510 * range from 32-127, result is in the range from 0-95.
 511 */
 512static inline unsigned long
 513fph_index (struct pt_regs *regs, long regnum)
 514{
 515        unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
 516        return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
 517}
 518
 519static void
 520setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
 521{
 522        struct switch_stack *sw = (struct switch_stack *)regs - 1;
 523        unsigned long addr;
 524
 525        /*
 526         * From EAS-2.5: FPDisableFault has higher priority than Unaligned
 527         * Fault. Thus, when we get here, we know the partition is enabled.
 528         * To update f32-f127, there are three choices:
 529         *
 530         *      (1) save f32-f127 to thread.fph and update the values there
 531         *      (2) use a gigantic switch statement to directly access the registers
 532         *      (3) generate code on the fly to update the desired register
 533         *
 534         * For now, we are using approach (1).
 535         */
 536        if (regnum >= IA64_FIRST_ROTATING_FR) {
 537                ia64_sync_fph(current);
 538                current->thread.fph[fph_index(regs, regnum)] = *fpval;
 539        } else {
 540                /*
 541                 * pt_regs or switch_stack ?
 542                 */
 543                if (FR_IN_SW(regnum)) {
 544                        addr = (unsigned long)sw;
 545                } else {
 546                        addr = (unsigned long)regs;
 547                }
 548
 549                DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum));
 550
 551                addr += FR_OFFS(regnum);
 552                *(struct ia64_fpreg *)addr = *fpval;
 553
 554                /*
 555                 * mark the low partition as being used now
 556                 *
 557                 * It is highly unlikely that this bit is not already set, but
 558                 * let's do it for safety.
 559                 */
 560                regs->cr_ipsr |= IA64_PSR_MFL;
 561        }
 562}
 563
 564/*
 565 * Those 2 inline functions generate the spilled versions of the constant floating point
 566 * registers which can be used with stfX
 567 */
 568static inline void
 569float_spill_f0 (struct ia64_fpreg *final)
 570{
 571        ia64_stf_spill(final, 0);
 572}
 573
 574static inline void
 575float_spill_f1 (struct ia64_fpreg *final)
 576{
 577        ia64_stf_spill(final, 1);
 578}
 579
 580static void
 581getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
 582{
 583        struct switch_stack *sw = (struct switch_stack *) regs - 1;
 584        unsigned long addr;
 585
 586        /*
 587         * From EAS-2.5: FPDisableFault has higher priority than
 588         * Unaligned Fault. Thus, when we get here, we know the partition is
 589         * enabled.
 590         *
 591         * When regnum > 31, the register is still live and we need to force a save
 592         * to current->thread.fph to get access to it.  See discussion in setfpreg()
 593         * for reasons and other ways of doing this.
 594         */
 595        if (regnum >= IA64_FIRST_ROTATING_FR) {
 596                ia64_flush_fph(current);
 597                *fpval = current->thread.fph[fph_index(regs, regnum)];
 598        } else {
 599                /*
 600                 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
 601                 * not saved, we must generate their spilled form on the fly
 602                 */
 603                switch(regnum) {
 604                case 0:
 605                        float_spill_f0(fpval);
 606                        break;
 607                case 1:
 608                        float_spill_f1(fpval);
 609                        break;
 610                default:
 611                        /*
 612                         * pt_regs or switch_stack ?
 613                         */
 614                        addr =  FR_IN_SW(regnum) ? (unsigned long)sw
 615                                                 : (unsigned long)regs;
 616
 617                        DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
 618                               FR_IN_SW(regnum), addr, FR_OFFS(regnum));
 619
 620                        addr  += FR_OFFS(regnum);
 621                        *fpval = *(struct ia64_fpreg *)addr;
 622                }
 623        }
 624}
 625
 626
 627static void
 628getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs)
 629{
 630        struct switch_stack *sw = (struct switch_stack *) regs - 1;
 631        unsigned long addr, *unat;
 632
 633        if (regnum >= IA64_FIRST_STACKED_GR) {
 634                get_rse_reg(regs, regnum, val, nat);
 635                return;
 636        }
 637
 638        /*
 639         * take care of r0 (read-only always evaluate to 0)
 640         */
 641        if (regnum == 0) {
 642                *val = 0;
 643                if (nat)
 644                        *nat = 0;
 645                return;
 646        }
 647
 648        /*
 649         * Now look at registers in [0-31] range and init correct UNAT
 650         */
 651        if (GR_IN_SW(regnum)) {
 652                addr = (unsigned long)sw;
 653                unat = &sw->ar_unat;
 654        } else {
 655                addr = (unsigned long)regs;
 656                unat = &sw->caller_unat;
 657        }
 658
 659        DPRINT("addr_base=%lx offset=0x%x\n", addr,  GR_OFFS(regnum));
 660
 661        addr += GR_OFFS(regnum);
 662
 663        *val  = *(unsigned long *)addr;
 664
 665        /*
 666         * do it only when requested
 667         */
 668        if (nat)
 669                *nat  = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL;
 670}
 671
 672static void
 673emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa)
 674{
 675        /*
 676         * IMPORTANT:
 677         * Given the way we handle unaligned speculative loads, we should
 678         * not get to this point in the code but we keep this sanity check,
 679         * just in case.
 680         */
 681        if (ld.x6_op == 1 || ld.x6_op == 3) {
 682                printk(KERN_ERR "%s: register update on speculative load, error\n", __func__);
 683                if (die_if_kernel("unaligned reference on speculative load with register update\n",
 684                                  regs, 30))
 685                        return;
 686        }
 687
 688
 689        /*
 690         * at this point, we know that the base register to update is valid i.e.,
 691         * it's not r0
 692         */
 693        if (type == UPD_IMMEDIATE) {
 694                unsigned long imm;
 695
 696                /*
 697                 * Load +Imm: ldXZ r1=[r3],imm(9)
 698                 *
 699                 *
 700                 * form imm9: [13:19] contain the first 7 bits
 701                 */
 702                imm = ld.x << 7 | ld.imm;
 703
 704                /*
 705                 * sign extend (1+8bits) if m set
 706                 */
 707                if (ld.m) imm |= SIGN_EXT9;
 708
 709                /*
 710                 * ifa == r3 and we know that the NaT bit on r3 was clear so
 711                 * we can directly use ifa.
 712                 */
 713                ifa += imm;
 714
 715                setreg(ld.r3, ifa, 0, regs);
 716
 717                DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa);
 718
 719        } else if (ld.m) {
 720                unsigned long r2;
 721                int nat_r2;
 722
 723                /*
 724                 * Load +Reg Opcode: ldXZ r1=[r3],r2
 725                 *
 726                 * Note: that we update r3 even in the case of ldfX.a
 727                 * (where the load does not happen)
 728                 *
 729                 * The way the load algorithm works, we know that r3 does not
 730                 * have its NaT bit set (would have gotten NaT consumption
 731                 * before getting the unaligned fault). So we can use ifa
 732                 * which equals r3 at this point.
 733                 *
 734                 * IMPORTANT:
 735                 * The above statement holds ONLY because we know that we
 736                 * never reach this code when trying to do a ldX.s.
 737                 * If we ever make it to here on an ldfX.s then
 738                 */
 739                getreg(ld.imm, &r2, &nat_r2, regs);
 740
 741                ifa += r2;
 742
 743                /*
 744                 * propagate Nat r2 -> r3
 745                 */
 746                setreg(ld.r3, ifa, nat_r2, regs);
 747
 748                DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2);
 749        }
 750}
 751
 752
 753static int
 754emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
 755{
 756        unsigned int len = 1 << ld.x6_sz;
 757        unsigned long val = 0;
 758
 759        /*
 760         * r0, as target, doesn't need to be checked because Illegal Instruction
 761         * faults have higher priority than unaligned faults.
 762         *
 763         * r0 cannot be found as the base as it would never generate an
 764         * unaligned reference.
 765         */
 766
 767        /*
 768         * ldX.a we will emulate load and also invalidate the ALAT entry.
 769         * See comment below for explanation on how we handle ldX.a
 770         */
 771
 772        if (len != 2 && len != 4 && len != 8) {
 773                DPRINT("unknown size: x6=%d\n", ld.x6_sz);
 774                return -1;
 775        }
 776        /* this assumes little-endian byte-order: */
 777        if (copy_from_user(&val, (void __user *) ifa, len))
 778                return -1;
 779        setreg(ld.r1, val, 0, regs);
 780
 781        /*
 782         * check for updates on any kind of loads
 783         */
 784        if (ld.op == 0x5 || ld.m)
 785                emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
 786
 787        /*
 788         * handling of various loads (based on EAS2.4):
 789         *
 790         * ldX.acq (ordered load):
 791         *      - acquire semantics would have been used, so force fence instead.
 792         *
 793         * ldX.c.clr (check load and clear):
 794         *      - if we get to this handler, it's because the entry was not in the ALAT.
 795         *        Therefore the operation reverts to a normal load
 796         *
 797         * ldX.c.nc (check load no clear):
 798         *      - same as previous one
 799         *
 800         * ldX.c.clr.acq (ordered check load and clear):
 801         *      - same as above for c.clr part. The load needs to have acquire semantics. So
 802         *        we use the fence semantics which is stronger and thus ensures correctness.
 803         *
 804         * ldX.a (advanced load):
 805         *      - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
 806         *        address doesn't match requested size alignment. This means that we would
 807         *        possibly need more than one load to get the result.
 808         *
 809         *        The load part can be handled just like a normal load, however the difficult
 810         *        part is to get the right thing into the ALAT. The critical piece of information
 811         *        in the base address of the load & size. To do that, a ld.a must be executed,
 812         *        clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
 813         *        if we use the same target register, we will be okay for the check.a instruction.
 814         *        If we look at the store, basically a stX [r3]=r1 checks the ALAT  for any entry
 815         *        which would overlap within [r3,r3+X] (the size of the load was store in the
 816         *        ALAT). If such an entry is found the entry is invalidated. But this is not good
 817         *        enough, take the following example:
 818         *              r3=3
 819         *              ld4.a r1=[r3]
 820         *
 821         *        Could be emulated by doing:
 822         *              ld1.a r1=[r3],1
 823         *              store to temporary;
 824         *              ld1.a r1=[r3],1
 825         *              store & shift to temporary;
 826         *              ld1.a r1=[r3],1
 827         *              store & shift to temporary;
 828         *              ld1.a r1=[r3]
 829         *              store & shift to temporary;
 830         *              r1=temporary
 831         *
 832         *        So in this case, you would get the right value is r1 but the wrong info in
 833         *        the ALAT.  Notice that you could do it in reverse to finish with address 3
 834         *        but you would still get the size wrong.  To get the size right, one needs to
 835         *        execute exactly the same kind of load. You could do it from a aligned
 836         *        temporary location, but you would get the address wrong.
 837         *
 838         *        So no matter what, it is not possible to emulate an advanced load
 839         *        correctly. But is that really critical ?
 840         *
 841         *        We will always convert ld.a into a normal load with ALAT invalidated.  This
 842         *        will enable compiler to do optimization where certain code path after ld.a
 843         *        is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
 844         *
 845         *        If there is a store after the advanced load, one must either do a ld.c.* or
 846         *        chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
 847         *        entry found in ALAT), and that's perfectly ok because:
 848         *
 849         *              - ld.c.*, if the entry is not present a  normal load is executed
 850         *              - chk.a.*, if the entry is not present, execution jumps to recovery code
 851         *
 852         *        In either case, the load can be potentially retried in another form.
 853         *
 854         *        ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
 855         *        up a stale entry later). The register base update MUST also be performed.
 856         */
 857
 858        /*
 859         * when the load has the .acq completer then
 860         * use ordering fence.
 861         */
 862        if (ld.x6_op == 0x5 || ld.x6_op == 0xa)
 863                mb();
 864
 865        /*
 866         * invalidate ALAT entry in case of advanced load
 867         */
 868        if (ld.x6_op == 0x2)
 869                invala_gr(ld.r1);
 870
 871        return 0;
 872}
 873
 874static int
 875emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
 876{
 877        unsigned long r2;
 878        unsigned int len = 1 << ld.x6_sz;
 879
 880        /*
 881         * if we get to this handler, Nat bits on both r3 and r2 have already
 882         * been checked. so we don't need to do it
 883         *
 884         * extract the value to be stored
 885         */
 886        getreg(ld.imm, &r2, NULL, regs);
 887
 888        /*
 889         * we rely on the macros in unaligned.h for now i.e.,
 890         * we let the compiler figure out how to read memory gracefully.
 891         *
 892         * We need this switch/case because the way the inline function
 893         * works. The code is optimized by the compiler and looks like
 894         * a single switch/case.
 895         */
 896        DPRINT("st%d [%lx]=%lx\n", len, ifa, r2);
 897
 898        if (len != 2 && len != 4 && len != 8) {
 899                DPRINT("unknown size: x6=%d\n", ld.x6_sz);
 900                return -1;
 901        }
 902
 903        /* this assumes little-endian byte-order: */
 904        if (copy_to_user((void __user *) ifa, &r2, len))
 905                return -1;
 906
 907        /*
 908         * stX [r3]=r2,imm(9)
 909         *
 910         * NOTE:
 911         * ld.r3 can never be r0, because r0 would not generate an
 912         * unaligned access.
 913         */
 914        if (ld.op == 0x5) {
 915                unsigned long imm;
 916
 917                /*
 918                 * form imm9: [12:6] contain first 7bits
 919                 */
 920                imm = ld.x << 7 | ld.r1;
 921                /*
 922                 * sign extend (8bits) if m set
 923                 */
 924                if (ld.m) imm |= SIGN_EXT9;
 925                /*
 926                 * ifa == r3 (NaT is necessarily cleared)
 927                 */
 928                ifa += imm;
 929
 930                DPRINT("imm=%lx r3=%lx\n", imm, ifa);
 931
 932                setreg(ld.r3, ifa, 0, regs);
 933        }
 934        /*
 935         * we don't have alat_invalidate_multiple() so we need
 936         * to do the complete flush :-<<
 937         */
 938        ia64_invala();
 939
 940        /*
 941         * stX.rel: use fence instead of release
 942         */
 943        if (ld.x6_op == 0xd)
 944                mb();
 945
 946        return 0;
 947}
 948
 949/*
 950 * floating point operations sizes in bytes
 951 */
 952static const unsigned char float_fsz[4]={
 953        10, /* extended precision (e) */
 954        8,  /* integer (8)            */
 955        4,  /* single precision (s)   */
 956        8   /* double precision (d)   */
 957};
 958
 959static inline void
 960mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
 961{
 962        ia64_ldfe(6, init);
 963        ia64_stop();
 964        ia64_stf_spill(final, 6);
 965}
 966
 967static inline void
 968mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
 969{
 970        ia64_ldf8(6, init);
 971        ia64_stop();
 972        ia64_stf_spill(final, 6);
 973}
 974
 975static inline void
 976mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
 977{
 978        ia64_ldfs(6, init);
 979        ia64_stop();
 980        ia64_stf_spill(final, 6);
 981}
 982
 983static inline void
 984mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
 985{
 986        ia64_ldfd(6, init);
 987        ia64_stop();
 988        ia64_stf_spill(final, 6);
 989}
 990
 991static inline void
 992float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
 993{
 994        ia64_ldf_fill(6, init);
 995        ia64_stop();
 996        ia64_stfe(final, 6);
 997}
 998
 999static inline void
1000float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
1001{
1002        ia64_ldf_fill(6, init);
1003        ia64_stop();
1004        ia64_stf8(final, 6);
1005}
1006
1007static inline void
1008float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
1009{
1010        ia64_ldf_fill(6, init);
1011        ia64_stop();
1012        ia64_stfs(final, 6);
1013}
1014
1015static inline void
1016float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1017{
1018        ia64_ldf_fill(6, init);
1019        ia64_stop();
1020        ia64_stfd(final, 6);
1021}
1022
1023static int
1024emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1025{
1026        struct ia64_fpreg fpr_init[2];
1027        struct ia64_fpreg fpr_final[2];
1028        unsigned long len = float_fsz[ld.x6_sz];
1029
1030        /*
1031         * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1032         * higher priority than unaligned faults.
1033         *
1034         * r0 cannot be found as the base as it would never generate an unaligned
1035         * reference.
1036         */
1037
1038        /*
1039         * make sure we get clean buffers
1040         */
1041        memset(&fpr_init, 0, sizeof(fpr_init));
1042        memset(&fpr_final, 0, sizeof(fpr_final));
1043
1044        /*
1045         * ldfpX.a: we don't try to emulate anything but we must
1046         * invalidate the ALAT entry and execute updates, if any.
1047         */
1048        if (ld.x6_op != 0x2) {
1049                /*
1050                 * This assumes little-endian byte-order.  Note that there is no "ldfpe"
1051                 * instruction:
1052                 */
1053                if (copy_from_user(&fpr_init[0], (void __user *) ifa, len)
1054                    || copy_from_user(&fpr_init[1], (void __user *) (ifa + len), len))
1055                        return -1;
1056
1057                DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz);
1058                DDUMP("frp_init =", &fpr_init, 2*len);
1059                /*
1060                 * XXX fixme
1061                 * Could optimize inlines by using ldfpX & 2 spills
1062                 */
1063                switch( ld.x6_sz ) {
1064                        case 0:
1065                                mem2float_extended(&fpr_init[0], &fpr_final[0]);
1066                                mem2float_extended(&fpr_init[1], &fpr_final[1]);
1067                                break;
1068                        case 1:
1069                                mem2float_integer(&fpr_init[0], &fpr_final[0]);
1070                                mem2float_integer(&fpr_init[1], &fpr_final[1]);
1071                                break;
1072                        case 2:
1073                                mem2float_single(&fpr_init[0], &fpr_final[0]);
1074                                mem2float_single(&fpr_init[1], &fpr_final[1]);
1075                                break;
1076                        case 3:
1077                                mem2float_double(&fpr_init[0], &fpr_final[0]);
1078                                mem2float_double(&fpr_init[1], &fpr_final[1]);
1079                                break;
1080                }
1081                DDUMP("fpr_final =", &fpr_final, 2*len);
1082                /*
1083                 * XXX fixme
1084                 *
1085                 * A possible optimization would be to drop fpr_final and directly
1086                 * use the storage from the saved context i.e., the actual final
1087                 * destination (pt_regs, switch_stack or thread structure).
1088                 */
1089                setfpreg(ld.r1, &fpr_final[0], regs);
1090                setfpreg(ld.imm, &fpr_final[1], regs);
1091        }
1092
1093        /*
1094         * Check for updates: only immediate updates are available for this
1095         * instruction.
1096         */
1097        if (ld.m) {
1098                /*
1099                 * the immediate is implicit given the ldsz of the operation:
1100                 * single: 8 (2x4) and for  all others it's 16 (2x8)
1101                 */
1102                ifa += len<<1;
1103
1104                /*
1105                 * IMPORTANT:
1106                 * the fact that we force the NaT of r3 to zero is ONLY valid
1107                 * as long as we don't come here with a ldfpX.s.
1108                 * For this reason we keep this sanity check
1109                 */
1110                if (ld.x6_op == 1 || ld.x6_op == 3)
1111                        printk(KERN_ERR "%s: register update on speculative load pair, error\n",
1112                               __func__);
1113
1114                setreg(ld.r3, ifa, 0, regs);
1115        }
1116
1117        /*
1118         * Invalidate ALAT entries, if any, for both registers.
1119         */
1120        if (ld.x6_op == 0x2) {
1121                invala_fr(ld.r1);
1122                invala_fr(ld.imm);
1123        }
1124        return 0;
1125}
1126
1127
1128static int
1129emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1130{
1131        struct ia64_fpreg fpr_init;
1132        struct ia64_fpreg fpr_final;
1133        unsigned long len = float_fsz[ld.x6_sz];
1134
1135        /*
1136         * fr0 & fr1 don't need to be checked because Illegal Instruction
1137         * faults have higher priority than unaligned faults.
1138         *
1139         * r0 cannot be found as the base as it would never generate an
1140         * unaligned reference.
1141         */
1142
1143        /*
1144         * make sure we get clean buffers
1145         */
1146        memset(&fpr_init,0, sizeof(fpr_init));
1147        memset(&fpr_final,0, sizeof(fpr_final));
1148
1149        /*
1150         * ldfX.a we don't try to emulate anything but we must
1151         * invalidate the ALAT entry.
1152         * See comments in ldX for descriptions on how the various loads are handled.
1153         */
1154        if (ld.x6_op != 0x2) {
1155                if (copy_from_user(&fpr_init, (void __user *) ifa, len))
1156                        return -1;
1157
1158                DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1159                DDUMP("fpr_init =", &fpr_init, len);
1160                /*
1161                 * we only do something for x6_op={0,8,9}
1162                 */
1163                switch( ld.x6_sz ) {
1164                        case 0:
1165                                mem2float_extended(&fpr_init, &fpr_final);
1166                                break;
1167                        case 1:
1168                                mem2float_integer(&fpr_init, &fpr_final);
1169                                break;
1170                        case 2:
1171                                mem2float_single(&fpr_init, &fpr_final);
1172                                break;
1173                        case 3:
1174                                mem2float_double(&fpr_init, &fpr_final);
1175                                break;
1176                }
1177                DDUMP("fpr_final =", &fpr_final, len);
1178                /*
1179                 * XXX fixme
1180                 *
1181                 * A possible optimization would be to drop fpr_final and directly
1182                 * use the storage from the saved context i.e., the actual final
1183                 * destination (pt_regs, switch_stack or thread structure).
1184                 */
1185                setfpreg(ld.r1, &fpr_final, regs);
1186        }
1187
1188        /*
1189         * check for updates on any loads
1190         */
1191        if (ld.op == 0x7 || ld.m)
1192                emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
1193
1194        /*
1195         * invalidate ALAT entry in case of advanced floating point loads
1196         */
1197        if (ld.x6_op == 0x2)
1198                invala_fr(ld.r1);
1199
1200        return 0;
1201}
1202
1203
1204static int
1205emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1206{
1207        struct ia64_fpreg fpr_init;
1208        struct ia64_fpreg fpr_final;
1209        unsigned long len = float_fsz[ld.x6_sz];
1210
1211        /*
1212         * make sure we get clean buffers
1213         */
1214        memset(&fpr_init,0, sizeof(fpr_init));
1215        memset(&fpr_final,0, sizeof(fpr_final));
1216
1217        /*
1218         * if we get to this handler, Nat bits on both r3 and r2 have already
1219         * been checked. so we don't need to do it
1220         *
1221         * extract the value to be stored
1222         */
1223        getfpreg(ld.imm, &fpr_init, regs);
1224        /*
1225         * during this step, we extract the spilled registers from the saved
1226         * context i.e., we refill. Then we store (no spill) to temporary
1227         * aligned location
1228         */
1229        switch( ld.x6_sz ) {
1230                case 0:
1231                        float2mem_extended(&fpr_init, &fpr_final);
1232                        break;
1233                case 1:
1234                        float2mem_integer(&fpr_init, &fpr_final);
1235                        break;
1236                case 2:
1237                        float2mem_single(&fpr_init, &fpr_final);
1238                        break;
1239                case 3:
1240                        float2mem_double(&fpr_init, &fpr_final);
1241                        break;
1242        }
1243        DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1244        DDUMP("fpr_init =", &fpr_init, len);
1245        DDUMP("fpr_final =", &fpr_final, len);
1246
1247        if (copy_to_user((void __user *) ifa, &fpr_final, len))
1248                return -1;
1249
1250        /*
1251         * stfX [r3]=r2,imm(9)
1252         *
1253         * NOTE:
1254         * ld.r3 can never be r0, because r0 would not generate an
1255         * unaligned access.
1256         */
1257        if (ld.op == 0x7) {
1258                unsigned long imm;
1259
1260                /*
1261                 * form imm9: [12:6] contain first 7bits
1262                 */
1263                imm = ld.x << 7 | ld.r1;
1264                /*
1265                 * sign extend (8bits) if m set
1266                 */
1267                if (ld.m)
1268                        imm |= SIGN_EXT9;
1269                /*
1270                 * ifa == r3 (NaT is necessarily cleared)
1271                 */
1272                ifa += imm;
1273
1274                DPRINT("imm=%lx r3=%lx\n", imm, ifa);
1275
1276                setreg(ld.r3, ifa, 0, regs);
1277        }
1278        /*
1279         * we don't have alat_invalidate_multiple() so we need
1280         * to do the complete flush :-<<
1281         */
1282        ia64_invala();
1283
1284        return 0;
1285}
1286
1287/*
1288 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1289 * eventually fix the program.  However, we don't want to do that for every access so we
1290 * pace it with jiffies.
1291 */
1292static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5);
1293
1294void
1295ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
1296{
1297        struct ia64_psr *ipsr = ia64_psr(regs);
1298        mm_segment_t old_fs = get_fs();
1299        unsigned long bundle[2];
1300        unsigned long opcode;
1301        const struct exception_table_entry *eh = NULL;
1302        union {
1303                unsigned long l;
1304                load_store_t insn;
1305        } u;
1306        int ret = -1;
1307
1308        if (ia64_psr(regs)->be) {
1309                /* we don't support big-endian accesses */
1310                if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0))
1311                        return;
1312                goto force_sigbus;
1313        }
1314
1315        /*
1316         * Treat kernel accesses for which there is an exception handler entry the same as
1317         * user-level unaligned accesses.  Otherwise, a clever program could trick this
1318         * handler into reading an arbitrary kernel addresses...
1319         */
1320        if (!user_mode(regs))
1321                eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri);
1322        if (user_mode(regs) || eh) {
1323                if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0)
1324                        goto force_sigbus;
1325
1326                if (!no_unaligned_warning &&
1327                    !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) &&
1328                    __ratelimit(&logging_rate_limit))
1329                {
1330                        char buf[200];  /* comm[] is at most 16 bytes... */
1331                        size_t len;
1332
1333                        len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, "
1334                                      "ip=0x%016lx\n\r", current->comm,
1335                                      task_pid_nr(current),
1336                                      ifa, regs->cr_iip + ipsr->ri);
1337                        /*
1338                         * Don't call tty_write_message() if we're in the kernel; we might
1339                         * be holding locks...
1340                         */
1341                        if (user_mode(regs)) {
1342                                struct tty_struct *tty = get_current_tty();
1343                                tty_write_message(tty, buf);
1344                                tty_kref_put(tty);
1345                        }
1346                        buf[len-1] = '\0';      /* drop '\r' */
1347                        /* watch for command names containing %s */
1348                        printk(KERN_WARNING "%s", buf);
1349                } else {
1350                        if (no_unaligned_warning) {
1351                                printk_once(KERN_WARNING "%s(%d) encountered an "
1352                                       "unaligned exception which required\n"
1353                                       "kernel assistance, which degrades "
1354                                       "the performance of the application.\n"
1355                                       "Unaligned exception warnings have "
1356                                       "been disabled by the system "
1357                                       "administrator\n"
1358                                       "echo 0 > /proc/sys/kernel/ignore-"
1359                                       "unaligned-usertrap to re-enable\n",
1360                                       current->comm, task_pid_nr(current));
1361                        }
1362                }
1363        } else {
1364                if (__ratelimit(&logging_rate_limit)) {
1365                        printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1366                               ifa, regs->cr_iip + ipsr->ri);
1367                        if (unaligned_dump_stack)
1368                                dump_stack();
1369                }
1370                set_fs(KERNEL_DS);
1371        }
1372
1373        DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1374               regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it);
1375
1376        if (__copy_from_user(bundle, (void __user *) regs->cr_iip, 16))
1377                goto failure;
1378
1379        /*
1380         * extract the instruction from the bundle given the slot number
1381         */
1382        switch (ipsr->ri) {
1383              default:
1384              case 0: u.l = (bundle[0] >>  5); break;
1385              case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break;
1386              case 2: u.l = (bundle[1] >> 23); break;
1387        }
1388        opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK;
1389
1390        DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1391               "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm,
1392               u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op);
1393
1394        /*
1395         * IMPORTANT:
1396         * Notice that the switch statement DOES not cover all possible instructions
1397         * that DO generate unaligned references. This is made on purpose because for some
1398         * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1399         * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1400         * the program will get a signal and die:
1401         *
1402         *      load/store:
1403         *              - ldX.spill
1404         *              - stX.spill
1405         *      Reason: RNATs are based on addresses
1406         *              - ld16
1407         *              - st16
1408         *      Reason: ld16 and st16 are supposed to occur in a single
1409         *              memory op
1410         *
1411         *      synchronization:
1412         *              - cmpxchg
1413         *              - fetchadd
1414         *              - xchg
1415         *      Reason: ATOMIC operations cannot be emulated properly using multiple
1416         *              instructions.
1417         *
1418         *      speculative loads:
1419         *              - ldX.sZ
1420         *      Reason: side effects, code must be ready to deal with failure so simpler
1421         *              to let the load fail.
1422         * ---------------------------------------------------------------------------------
1423         * XXX fixme
1424         *
1425         * I would like to get rid of this switch case and do something
1426         * more elegant.
1427         */
1428        switch (opcode) {
1429              case LDS_OP:
1430              case LDSA_OP:
1431                if (u.insn.x)
1432                        /* oops, really a semaphore op (cmpxchg, etc) */
1433                        goto failure;
1434                fallthrough;
1435              case LDS_IMM_OP:
1436              case LDSA_IMM_OP:
1437              case LDFS_OP:
1438              case LDFSA_OP:
1439              case LDFS_IMM_OP:
1440                /*
1441                 * The instruction will be retried with deferred exceptions turned on, and
1442                 * we should get Nat bit installed
1443                 *
1444                 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1445                 * are actually executed even though the operation failed. So we don't
1446                 * need to take care of this.
1447                 */
1448                DPRINT("forcing PSR_ED\n");
1449                regs->cr_ipsr |= IA64_PSR_ED;
1450                goto done;
1451
1452              case LD_OP:
1453              case LDA_OP:
1454              case LDBIAS_OP:
1455              case LDACQ_OP:
1456              case LDCCLR_OP:
1457              case LDCNC_OP:
1458              case LDCCLRACQ_OP:
1459                if (u.insn.x)
1460                        /* oops, really a semaphore op (cmpxchg, etc) */
1461                        goto failure;
1462                fallthrough;
1463              case LD_IMM_OP:
1464              case LDA_IMM_OP:
1465              case LDBIAS_IMM_OP:
1466              case LDACQ_IMM_OP:
1467              case LDCCLR_IMM_OP:
1468              case LDCNC_IMM_OP:
1469              case LDCCLRACQ_IMM_OP:
1470                ret = emulate_load_int(ifa, u.insn, regs);
1471                break;
1472
1473              case ST_OP:
1474              case STREL_OP:
1475                if (u.insn.x)
1476                        /* oops, really a semaphore op (cmpxchg, etc) */
1477                        goto failure;
1478                fallthrough;
1479              case ST_IMM_OP:
1480              case STREL_IMM_OP:
1481                ret = emulate_store_int(ifa, u.insn, regs);
1482                break;
1483
1484              case LDF_OP:
1485              case LDFA_OP:
1486              case LDFCCLR_OP:
1487              case LDFCNC_OP:
1488                if (u.insn.x)
1489                        ret = emulate_load_floatpair(ifa, u.insn, regs);
1490                else
1491                        ret = emulate_load_float(ifa, u.insn, regs);
1492                break;
1493
1494              case LDF_IMM_OP:
1495              case LDFA_IMM_OP:
1496              case LDFCCLR_IMM_OP:
1497              case LDFCNC_IMM_OP:
1498                ret = emulate_load_float(ifa, u.insn, regs);
1499                break;
1500
1501              case STF_OP:
1502              case STF_IMM_OP:
1503                ret = emulate_store_float(ifa, u.insn, regs);
1504                break;
1505
1506              default:
1507                goto failure;
1508        }
1509        DPRINT("ret=%d\n", ret);
1510        if (ret)
1511                goto failure;
1512
1513        if (ipsr->ri == 2)
1514                /*
1515                 * given today's architecture this case is not likely to happen because a
1516                 * memory access instruction (M) can never be in the last slot of a
1517                 * bundle. But let's keep it for now.
1518                 */
1519                regs->cr_iip += 16;
1520        ipsr->ri = (ipsr->ri + 1) & 0x3;
1521
1522        DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip);
1523  done:
1524        set_fs(old_fs);         /* restore original address limit */
1525        return;
1526
1527  failure:
1528        /* something went wrong... */
1529        if (!user_mode(regs)) {
1530                if (eh) {
1531                        ia64_handle_exception(regs, eh);
1532                        goto done;
1533                }
1534                if (die_if_kernel("error during unaligned kernel access\n", regs, ret))
1535                        return;
1536                /* NOT_REACHED */
1537        }
1538  force_sigbus:
1539        force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) ifa,
1540                        0, 0, 0);
1541        goto done;
1542}
1543