qemu/target/arm/pauth_helper.c
<<
>>
Prefs
   1/*
   2 * ARM v8.3-PAuth Operations
   3 *
   4 * Copyright (c) 2019 Linaro, Ltd.
   5 *
   6 * This library is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU Lesser General Public
   8 * License as published by the Free Software Foundation; either
   9 * version 2.1 of the License, or (at your option) any later version.
  10 *
  11 * This library is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 * Lesser General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU Lesser General Public
  17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20#include "qemu/osdep.h"
  21#include "cpu.h"
  22#include "internals.h"
  23#include "exec/exec-all.h"
  24#include "exec/cpu_ldst.h"
  25#include "exec/helper-proto.h"
  26#include "tcg/tcg-gvec-desc.h"
  27#include "qemu/xxhash.h"
  28
  29
  30static uint64_t pac_cell_shuffle(uint64_t i)
  31{
  32    uint64_t o = 0;
  33
  34    o |= extract64(i, 52, 4);
  35    o |= extract64(i, 24, 4) << 4;
  36    o |= extract64(i, 44, 4) << 8;
  37    o |= extract64(i,  0, 4) << 12;
  38
  39    o |= extract64(i, 28, 4) << 16;
  40    o |= extract64(i, 48, 4) << 20;
  41    o |= extract64(i,  4, 4) << 24;
  42    o |= extract64(i, 40, 4) << 28;
  43
  44    o |= extract64(i, 32, 4) << 32;
  45    o |= extract64(i, 12, 4) << 36;
  46    o |= extract64(i, 56, 4) << 40;
  47    o |= extract64(i, 20, 4) << 44;
  48
  49    o |= extract64(i,  8, 4) << 48;
  50    o |= extract64(i, 36, 4) << 52;
  51    o |= extract64(i, 16, 4) << 56;
  52    o |= extract64(i, 60, 4) << 60;
  53
  54    return o;
  55}
  56
  57static uint64_t pac_cell_inv_shuffle(uint64_t i)
  58{
  59    uint64_t o = 0;
  60
  61    o |= extract64(i, 12, 4);
  62    o |= extract64(i, 24, 4) << 4;
  63    o |= extract64(i, 48, 4) << 8;
  64    o |= extract64(i, 36, 4) << 12;
  65
  66    o |= extract64(i, 56, 4) << 16;
  67    o |= extract64(i, 44, 4) << 20;
  68    o |= extract64(i,  4, 4) << 24;
  69    o |= extract64(i, 16, 4) << 28;
  70
  71    o |= i & MAKE_64BIT_MASK(32, 4);
  72    o |= extract64(i, 52, 4) << 36;
  73    o |= extract64(i, 28, 4) << 40;
  74    o |= extract64(i,  8, 4) << 44;
  75
  76    o |= extract64(i, 20, 4) << 48;
  77    o |= extract64(i,  0, 4) << 52;
  78    o |= extract64(i, 40, 4) << 56;
  79    o |= i & MAKE_64BIT_MASK(60, 4);
  80
  81    return o;
  82}
  83
  84static uint64_t pac_sub(uint64_t i)
  85{
  86    static const uint8_t sub[16] = {
  87        0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe,
  88        0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa,
  89    };
  90    uint64_t o = 0;
  91    int b;
  92
  93    for (b = 0; b < 64; b += 4) {
  94        o |= (uint64_t)sub[(i >> b) & 0xf] << b;
  95    }
  96    return o;
  97}
  98
  99static uint64_t pac_inv_sub(uint64_t i)
 100{
 101    static const uint8_t inv_sub[16] = {
 102        0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9,
 103        0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3,
 104    };
 105    uint64_t o = 0;
 106    int b;
 107
 108    for (b = 0; b < 64; b += 4) {
 109        o |= (uint64_t)inv_sub[(i >> b) & 0xf] << b;
 110    }
 111    return o;
 112}
 113
 114static int rot_cell(int cell, int n)
 115{
 116    /* 4-bit rotate left by n.  */
 117    cell |= cell << 4;
 118    return extract32(cell, 4 - n, 4);
 119}
 120
 121static uint64_t pac_mult(uint64_t i)
 122{
 123    uint64_t o = 0;
 124    int b;
 125
 126    for (b = 0; b < 4 * 4; b += 4) {
 127        int i0, i4, i8, ic, t0, t1, t2, t3;
 128
 129        i0 = extract64(i, b, 4);
 130        i4 = extract64(i, b + 4 * 4, 4);
 131        i8 = extract64(i, b + 8 * 4, 4);
 132        ic = extract64(i, b + 12 * 4, 4);
 133
 134        t0 = rot_cell(i8, 1) ^ rot_cell(i4, 2) ^ rot_cell(i0, 1);
 135        t1 = rot_cell(ic, 1) ^ rot_cell(i4, 1) ^ rot_cell(i0, 2);
 136        t2 = rot_cell(ic, 2) ^ rot_cell(i8, 1) ^ rot_cell(i0, 1);
 137        t3 = rot_cell(ic, 1) ^ rot_cell(i8, 2) ^ rot_cell(i4, 1);
 138
 139        o |= (uint64_t)t3 << b;
 140        o |= (uint64_t)t2 << (b + 4 * 4);
 141        o |= (uint64_t)t1 << (b + 8 * 4);
 142        o |= (uint64_t)t0 << (b + 12 * 4);
 143    }
 144    return o;
 145}
 146
 147static uint64_t tweak_cell_rot(uint64_t cell)
 148{
 149    return (cell >> 1) | (((cell ^ (cell >> 1)) & 1) << 3);
 150}
 151
 152static uint64_t tweak_shuffle(uint64_t i)
 153{
 154    uint64_t o = 0;
 155
 156    o |= extract64(i, 16, 4) << 0;
 157    o |= extract64(i, 20, 4) << 4;
 158    o |= tweak_cell_rot(extract64(i, 24, 4)) << 8;
 159    o |= extract64(i, 28, 4) << 12;
 160
 161    o |= tweak_cell_rot(extract64(i, 44, 4)) << 16;
 162    o |= extract64(i,  8, 4) << 20;
 163    o |= extract64(i, 12, 4) << 24;
 164    o |= tweak_cell_rot(extract64(i, 32, 4)) << 28;
 165
 166    o |= extract64(i, 48, 4) << 32;
 167    o |= extract64(i, 52, 4) << 36;
 168    o |= extract64(i, 56, 4) << 40;
 169    o |= tweak_cell_rot(extract64(i, 60, 4)) << 44;
 170
 171    o |= tweak_cell_rot(extract64(i,  0, 4)) << 48;
 172    o |= extract64(i,  4, 4) << 52;
 173    o |= tweak_cell_rot(extract64(i, 40, 4)) << 56;
 174    o |= tweak_cell_rot(extract64(i, 36, 4)) << 60;
 175
 176    return o;
 177}
 178
 179static uint64_t tweak_cell_inv_rot(uint64_t cell)
 180{
 181    return ((cell << 1) & 0xf) | ((cell & 1) ^ (cell >> 3));
 182}
 183
 184static uint64_t tweak_inv_shuffle(uint64_t i)
 185{
 186    uint64_t o = 0;
 187
 188    o |= tweak_cell_inv_rot(extract64(i, 48, 4));
 189    o |= extract64(i, 52, 4) << 4;
 190    o |= extract64(i, 20, 4) << 8;
 191    o |= extract64(i, 24, 4) << 12;
 192
 193    o |= extract64(i,  0, 4) << 16;
 194    o |= extract64(i,  4, 4) << 20;
 195    o |= tweak_cell_inv_rot(extract64(i,  8, 4)) << 24;
 196    o |= extract64(i, 12, 4) << 28;
 197
 198    o |= tweak_cell_inv_rot(extract64(i, 28, 4)) << 32;
 199    o |= tweak_cell_inv_rot(extract64(i, 60, 4)) << 36;
 200    o |= tweak_cell_inv_rot(extract64(i, 56, 4)) << 40;
 201    o |= tweak_cell_inv_rot(extract64(i, 16, 4)) << 44;
 202
 203    o |= extract64(i, 32, 4) << 48;
 204    o |= extract64(i, 36, 4) << 52;
 205    o |= extract64(i, 40, 4) << 56;
 206    o |= tweak_cell_inv_rot(extract64(i, 44, 4)) << 60;
 207
 208    return o;
 209}
 210
 211static uint64_t pauth_computepac_architected(uint64_t data, uint64_t modifier,
 212                                             ARMPACKey key)
 213{
 214    static const uint64_t RC[5] = {
 215        0x0000000000000000ull,
 216        0x13198A2E03707344ull,
 217        0xA4093822299F31D0ull,
 218        0x082EFA98EC4E6C89ull,
 219        0x452821E638D01377ull,
 220    };
 221    const uint64_t alpha = 0xC0AC29B7C97C50DDull;
 222    /*
 223     * Note that in the ARM pseudocode, key0 contains bits <127:64>
 224     * and key1 contains bits <63:0> of the 128-bit key.
 225     */
 226    uint64_t key0 = key.hi, key1 = key.lo;
 227    uint64_t workingval, runningmod, roundkey, modk0;
 228    int i;
 229
 230    modk0 = (key0 << 63) | ((key0 >> 1) ^ (key0 >> 63));
 231    runningmod = modifier;
 232    workingval = data ^ key0;
 233
 234    for (i = 0; i <= 4; ++i) {
 235        roundkey = key1 ^ runningmod;
 236        workingval ^= roundkey;
 237        workingval ^= RC[i];
 238        if (i > 0) {
 239            workingval = pac_cell_shuffle(workingval);
 240            workingval = pac_mult(workingval);
 241        }
 242        workingval = pac_sub(workingval);
 243        runningmod = tweak_shuffle(runningmod);
 244    }
 245    roundkey = modk0 ^ runningmod;
 246    workingval ^= roundkey;
 247    workingval = pac_cell_shuffle(workingval);
 248    workingval = pac_mult(workingval);
 249    workingval = pac_sub(workingval);
 250    workingval = pac_cell_shuffle(workingval);
 251    workingval = pac_mult(workingval);
 252    workingval ^= key1;
 253    workingval = pac_cell_inv_shuffle(workingval);
 254    workingval = pac_inv_sub(workingval);
 255    workingval = pac_mult(workingval);
 256    workingval = pac_cell_inv_shuffle(workingval);
 257    workingval ^= key0;
 258    workingval ^= runningmod;
 259    for (i = 0; i <= 4; ++i) {
 260        workingval = pac_inv_sub(workingval);
 261        if (i < 4) {
 262            workingval = pac_mult(workingval);
 263            workingval = pac_cell_inv_shuffle(workingval);
 264        }
 265        runningmod = tweak_inv_shuffle(runningmod);
 266        roundkey = key1 ^ runningmod;
 267        workingval ^= RC[4 - i];
 268        workingval ^= roundkey;
 269        workingval ^= alpha;
 270    }
 271    workingval ^= modk0;
 272
 273    return workingval;
 274}
 275
 276static uint64_t pauth_computepac_impdef(uint64_t data, uint64_t modifier,
 277                                        ARMPACKey key)
 278{
 279    return qemu_xxhash64_4(data, modifier, key.lo, key.hi);
 280}
 281
 282static uint64_t pauth_computepac(CPUARMState *env, uint64_t data,
 283                                 uint64_t modifier, ARMPACKey key)
 284{
 285    if (cpu_isar_feature(aa64_pauth_arch, env_archcpu(env))) {
 286        return pauth_computepac_architected(data, modifier, key);
 287    } else {
 288        return pauth_computepac_impdef(data, modifier, key);
 289    }
 290}
 291
 292static uint64_t pauth_addpac(CPUARMState *env, uint64_t ptr, uint64_t modifier,
 293                             ARMPACKey *key, bool data)
 294{
 295    ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env);
 296    ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data);
 297    uint64_t pac, ext_ptr, ext, test;
 298    int bot_bit, top_bit;
 299
 300    /* If tagged pointers are in use, use ptr<55>, otherwise ptr<63>.  */
 301    if (param.tbi) {
 302        ext = sextract64(ptr, 55, 1);
 303    } else {
 304        ext = sextract64(ptr, 63, 1);
 305    }
 306
 307    /* Build a pointer with known good extension bits.  */
 308    top_bit = 64 - 8 * param.tbi;
 309    bot_bit = 64 - param.tsz;
 310    ext_ptr = deposit64(ptr, bot_bit, top_bit - bot_bit, ext);
 311
 312    pac = pauth_computepac(env, ext_ptr, modifier, *key);
 313
 314    /*
 315     * Check if the ptr has good extension bits and corrupt the
 316     * pointer authentication code if not.
 317     */
 318    test = sextract64(ptr, bot_bit, top_bit - bot_bit);
 319    if (test != 0 && test != -1) {
 320        /*
 321         * Note that our top_bit is one greater than the pseudocode's
 322         * version, hence "- 2" here.
 323         */
 324        pac ^= MAKE_64BIT_MASK(top_bit - 2, 1);
 325    }
 326
 327    /*
 328     * Preserve the determination between upper and lower at bit 55,
 329     * and insert pointer authentication code.
 330     */
 331    if (param.tbi) {
 332        ptr &= ~MAKE_64BIT_MASK(bot_bit, 55 - bot_bit + 1);
 333        pac &= MAKE_64BIT_MASK(bot_bit, 54 - bot_bit + 1);
 334    } else {
 335        ptr &= MAKE_64BIT_MASK(0, bot_bit);
 336        pac &= ~(MAKE_64BIT_MASK(55, 1) | MAKE_64BIT_MASK(0, bot_bit));
 337    }
 338    ext &= MAKE_64BIT_MASK(55, 1);
 339    return pac | ext | ptr;
 340}
 341
 342static uint64_t pauth_original_ptr(uint64_t ptr, ARMVAParameters param)
 343{
 344    /* Note that bit 55 is used whether or not the regime has 2 ranges. */
 345    uint64_t extfield = sextract64(ptr, 55, 1);
 346    int bot_pac_bit = 64 - param.tsz;
 347    int top_pac_bit = 64 - 8 * param.tbi;
 348
 349    return deposit64(ptr, bot_pac_bit, top_pac_bit - bot_pac_bit, extfield);
 350}
 351
 352static uint64_t pauth_auth(CPUARMState *env, uint64_t ptr, uint64_t modifier,
 353                           ARMPACKey *key, bool data, int keynumber)
 354{
 355    ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env);
 356    ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data);
 357    int bot_bit, top_bit;
 358    uint64_t pac, orig_ptr, test;
 359
 360    orig_ptr = pauth_original_ptr(ptr, param);
 361    pac = pauth_computepac(env, orig_ptr, modifier, *key);
 362    bot_bit = 64 - param.tsz;
 363    top_bit = 64 - 8 * param.tbi;
 364
 365    test = (pac ^ ptr) & ~MAKE_64BIT_MASK(55, 1);
 366    if (unlikely(extract64(test, bot_bit, top_bit - bot_bit))) {
 367        int error_code = (keynumber << 1) | (keynumber ^ 1);
 368        if (param.tbi) {
 369            return deposit64(orig_ptr, 53, 2, error_code);
 370        } else {
 371            return deposit64(orig_ptr, 61, 2, error_code);
 372        }
 373    }
 374    return orig_ptr;
 375}
 376
 377static uint64_t pauth_strip(CPUARMState *env, uint64_t ptr, bool data)
 378{
 379    ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env);
 380    ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data);
 381
 382    return pauth_original_ptr(ptr, param);
 383}
 384
 385static G_NORETURN
 386void pauth_trap(CPUARMState *env, int target_el, uintptr_t ra)
 387{
 388    raise_exception_ra(env, EXCP_UDEF, syn_pactrap(), target_el, ra);
 389}
 390
 391static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra)
 392{
 393    if (el < 2 && arm_is_el2_enabled(env)) {
 394        uint64_t hcr = arm_hcr_el2_eff(env);
 395        bool trap = !(hcr & HCR_API);
 396        if (el == 0) {
 397            /* Trap only applies to EL1&0 regime.  */
 398            trap &= (hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE);
 399        }
 400        /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB].  */
 401        if (trap) {
 402            pauth_trap(env, 2, ra);
 403        }
 404    }
 405    if (el < 3 && arm_feature(env, ARM_FEATURE_EL3)) {
 406        if (!(env->cp15.scr_el3 & SCR_API)) {
 407            pauth_trap(env, 3, ra);
 408        }
 409    }
 410}
 411
 412static bool pauth_key_enabled(CPUARMState *env, int el, uint32_t bit)
 413{
 414    return (arm_sctlr(env, el) & bit) != 0;
 415}
 416
 417uint64_t HELPER(pacia)(CPUARMState *env, uint64_t x, uint64_t y)
 418{
 419    int el = arm_current_el(env);
 420    if (!pauth_key_enabled(env, el, SCTLR_EnIA)) {
 421        return x;
 422    }
 423    pauth_check_trap(env, el, GETPC());
 424    return pauth_addpac(env, x, y, &env->keys.apia, false);
 425}
 426
 427uint64_t HELPER(pacib)(CPUARMState *env, uint64_t x, uint64_t y)
 428{
 429    int el = arm_current_el(env);
 430    if (!pauth_key_enabled(env, el, SCTLR_EnIB)) {
 431        return x;
 432    }
 433    pauth_check_trap(env, el, GETPC());
 434    return pauth_addpac(env, x, y, &env->keys.apib, false);
 435}
 436
 437uint64_t HELPER(pacda)(CPUARMState *env, uint64_t x, uint64_t y)
 438{
 439    int el = arm_current_el(env);
 440    if (!pauth_key_enabled(env, el, SCTLR_EnDA)) {
 441        return x;
 442    }
 443    pauth_check_trap(env, el, GETPC());
 444    return pauth_addpac(env, x, y, &env->keys.apda, true);
 445}
 446
 447uint64_t HELPER(pacdb)(CPUARMState *env, uint64_t x, uint64_t y)
 448{
 449    int el = arm_current_el(env);
 450    if (!pauth_key_enabled(env, el, SCTLR_EnDB)) {
 451        return x;
 452    }
 453    pauth_check_trap(env, el, GETPC());
 454    return pauth_addpac(env, x, y, &env->keys.apdb, true);
 455}
 456
 457uint64_t HELPER(pacga)(CPUARMState *env, uint64_t x, uint64_t y)
 458{
 459    uint64_t pac;
 460
 461    pauth_check_trap(env, arm_current_el(env), GETPC());
 462    pac = pauth_computepac(env, x, y, env->keys.apga);
 463
 464    return pac & 0xffffffff00000000ull;
 465}
 466
 467uint64_t HELPER(autia)(CPUARMState *env, uint64_t x, uint64_t y)
 468{
 469    int el = arm_current_el(env);
 470    if (!pauth_key_enabled(env, el, SCTLR_EnIA)) {
 471        return x;
 472    }
 473    pauth_check_trap(env, el, GETPC());
 474    return pauth_auth(env, x, y, &env->keys.apia, false, 0);
 475}
 476
 477uint64_t HELPER(autib)(CPUARMState *env, uint64_t x, uint64_t y)
 478{
 479    int el = arm_current_el(env);
 480    if (!pauth_key_enabled(env, el, SCTLR_EnIB)) {
 481        return x;
 482    }
 483    pauth_check_trap(env, el, GETPC());
 484    return pauth_auth(env, x, y, &env->keys.apib, false, 1);
 485}
 486
 487uint64_t HELPER(autda)(CPUARMState *env, uint64_t x, uint64_t y)
 488{
 489    int el = arm_current_el(env);
 490    if (!pauth_key_enabled(env, el, SCTLR_EnDA)) {
 491        return x;
 492    }
 493    pauth_check_trap(env, el, GETPC());
 494    return pauth_auth(env, x, y, &env->keys.apda, true, 0);
 495}
 496
 497uint64_t HELPER(autdb)(CPUARMState *env, uint64_t x, uint64_t y)
 498{
 499    int el = arm_current_el(env);
 500    if (!pauth_key_enabled(env, el, SCTLR_EnDB)) {
 501        return x;
 502    }
 503    pauth_check_trap(env, el, GETPC());
 504    return pauth_auth(env, x, y, &env->keys.apdb, true, 1);
 505}
 506
 507uint64_t HELPER(xpaci)(CPUARMState *env, uint64_t a)
 508{
 509    return pauth_strip(env, a, false);
 510}
 511
 512uint64_t HELPER(xpacd)(CPUARMState *env, uint64_t a)
 513{
 514    return pauth_strip(env, a, true);
 515}
 516