LXR qemu/target/i386/tcg/sysemu/excp

   1/*
   2 *  x86 exception helpers - sysemu code
   3 *
   4 *  Copyright (c) 2003 Fabrice Bellard
   5 *
   6 * This library is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU Lesser General Public
   8 * License as published by the Free Software Foundation; either
   9 * version 2.1 of the License, or (at your option) any later version.
  10 *
  11 * This library is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 * Lesser General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU Lesser General Public
  17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20#include "qemu/osdep.h"
  21#include "cpu.h"
  22#include "exec/exec-all.h"
  23#include "tcg/helper-tcg.h"
  24
  25#define PG_ERROR_OK (-1)
  26
  27typedef hwaddr (*MMUTranslateFunc)(CPUState *cs, hwaddr gphys, MMUAccessType access_type,
  28                                int *prot);
  29
  30#define GET_HPHYS(cs, gpa, access_type, prot)  \
  31        (get_hphys_func ? get_hphys_func(cs, gpa, access_type, prot) : gpa)
  32
  33static int mmu_translate(CPUState *cs, hwaddr addr, MMUTranslateFunc get_hphys_func,
  34                         uint64_t cr3, int is_write1, int mmu_idx, int pg_mode,
  35                         hwaddr *xlat, int *page_size, int *prot)
  36{
  37    X86CPU *cpu = X86_CPU(cs);
  38    CPUX86State *env = &cpu->env;
  39    uint64_t ptep, pte;
  40    int32_t a20_mask;
  41    target_ulong pde_addr, pte_addr;
  42    int error_code = 0;
  43    int is_dirty, is_write, is_user;
  44    uint64_t rsvd_mask = PG_ADDRESS_MASK & ~MAKE_64BIT_MASK(0, cpu->phys_bits);
  45    uint32_t page_offset;
  46    uint32_t pkr;
  47
  48    is_user = (mmu_idx == MMU_USER_IDX);
  49    is_write = is_write1 & 1;
  50    a20_mask = x86_get_a20_mask(env);
  51
  52    if (!(pg_mode & PG_MODE_NXE)) {
  53        rsvd_mask |= PG_NX_MASK;
  54    }
  55
  56    if (pg_mode & PG_MODE_PAE) {
  57        uint64_t pde, pdpe;
  58        target_ulong pdpe_addr;
  59
  60#ifdef TARGET_X86_64
  61        if (pg_mode & PG_MODE_LMA) {
  62            bool la57 = pg_mode & PG_MODE_LA57;
  63            uint64_t pml5e_addr, pml5e;
  64            uint64_t pml4e_addr, pml4e;
  65
  66            if (la57) {
  67                pml5e_addr = ((cr3 & ~0xfff) +
  68                        (((addr >> 48) & 0x1ff) << 3)) & a20_mask;
  69                pml5e_addr = GET_HPHYS(cs, pml5e_addr, MMU_DATA_STORE, NULL);
  70                pml5e = x86_ldq_phys(cs, pml5e_addr);
  71                if (!(pml5e & PG_PRESENT_MASK)) {
  72                    goto do_fault;
  73                }
  74                if (pml5e & (rsvd_mask | PG_PSE_MASK)) {
  75                    goto do_fault_rsvd;
  76                }
  77                if (!(pml5e & PG_ACCESSED_MASK)) {
  78                    pml5e |= PG_ACCESSED_MASK;
  79                    x86_stl_phys_notdirty(cs, pml5e_addr, pml5e);
  80                }
  81                ptep = pml5e ^ PG_NX_MASK;
  82            } else {
  83                pml5e = cr3;
  84                ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK;
  85            }
  86
  87            pml4e_addr = ((pml5e & PG_ADDRESS_MASK) +
  88                    (((addr >> 39) & 0x1ff) << 3)) & a20_mask;
  89            pml4e_addr = GET_HPHYS(cs, pml4e_addr, MMU_DATA_STORE, NULL);
  90            pml4e = x86_ldq_phys(cs, pml4e_addr);
  91            if (!(pml4e & PG_PRESENT_MASK)) {
  92                goto do_fault;
  93            }
  94            if (pml4e & (rsvd_mask | PG_PSE_MASK)) {
  95                goto do_fault_rsvd;
  96            }
  97            if (!(pml4e & PG_ACCESSED_MASK)) {
  98                pml4e |= PG_ACCESSED_MASK;
  99                x86_stl_phys_notdirty(cs, pml4e_addr, pml4e);
 100            }
 101            ptep &= pml4e ^ PG_NX_MASK;
 102            pdpe_addr = ((pml4e & PG_ADDRESS_MASK) + (((addr >> 30) & 0x1ff) << 3)) &
 103                a20_mask;
 104            pdpe_addr = GET_HPHYS(cs, pdpe_addr, MMU_DATA_STORE, NULL);
 105            pdpe = x86_ldq_phys(cs, pdpe_addr);
 106            if (!(pdpe & PG_PRESENT_MASK)) {
 107                goto do_fault;
 108            }
 109            if (pdpe & rsvd_mask) {
 110                goto do_fault_rsvd;
 111            }
 112            ptep &= pdpe ^ PG_NX_MASK;
 113            if (!(pdpe & PG_ACCESSED_MASK)) {
 114                pdpe |= PG_ACCESSED_MASK;
 115                x86_stl_phys_notdirty(cs, pdpe_addr, pdpe);
 116            }
 117            if (pdpe & PG_PSE_MASK) {
 118                /* 1 GB page */
 119                *page_size = 1024 * 1024 * 1024;
 120                pte_addr = pdpe_addr;
 121                pte = pdpe;
 122                goto do_check_protect;
 123            }
 124        } else
 125#endif
 126        {
 127            /* XXX: load them when cr3 is loaded ? */
 128            pdpe_addr = ((cr3 & ~0x1f) + ((addr >> 27) & 0x18)) &
 129                a20_mask;
 130            pdpe_addr = GET_HPHYS(cs, pdpe_addr, MMU_DATA_STORE, NULL);
 131            pdpe = x86_ldq_phys(cs, pdpe_addr);
 132            if (!(pdpe & PG_PRESENT_MASK)) {
 133                goto do_fault;
 134            }
 135            rsvd_mask |= PG_HI_USER_MASK;
 136            if (pdpe & (rsvd_mask | PG_NX_MASK)) {
 137                goto do_fault_rsvd;
 138            }
 139            ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK;
 140        }
 141
 142        pde_addr = ((pdpe & PG_ADDRESS_MASK) + (((addr >> 21) & 0x1ff) << 3)) &
 143            a20_mask;
 144        pde_addr = GET_HPHYS(cs, pde_addr, MMU_DATA_STORE, NULL);
 145        pde = x86_ldq_phys(cs, pde_addr);
 146        if (!(pde & PG_PRESENT_MASK)) {
 147            goto do_fault;
 148        }
 149        if (pde & rsvd_mask) {
 150            goto do_fault_rsvd;
 151        }
 152        ptep &= pde ^ PG_NX_MASK;
 153        if (pde & PG_PSE_MASK) {
 154            /* 2 MB page */
 155            *page_size = 2048 * 1024;
 156            pte_addr = pde_addr;
 157            pte = pde;
 158            goto do_check_protect;
 159        }
 160        /* 4 KB page */
 161        if (!(pde & PG_ACCESSED_MASK)) {
 162            pde |= PG_ACCESSED_MASK;
 163            x86_stl_phys_notdirty(cs, pde_addr, pde);
 164        }
 165        pte_addr = ((pde & PG_ADDRESS_MASK) + (((addr >> 12) & 0x1ff) << 3)) &
 166            a20_mask;
 167        pte_addr = GET_HPHYS(cs, pte_addr, MMU_DATA_STORE, NULL);
 168        pte = x86_ldq_phys(cs, pte_addr);
 169        if (!(pte & PG_PRESENT_MASK)) {
 170            goto do_fault;
 171        }
 172        if (pte & rsvd_mask) {
 173            goto do_fault_rsvd;
 174        }
 175        /* combine pde and pte nx, user and rw protections */
 176        ptep &= pte ^ PG_NX_MASK;
 177        *page_size = 4096;
 178    } else {
 179        uint32_t pde;
 180
 181        /* page directory entry */
 182        pde_addr = ((cr3 & ~0xfff) + ((addr >> 20) & 0xffc)) &
 183            a20_mask;
 184        pde_addr = GET_HPHYS(cs, pde_addr, MMU_DATA_STORE, NULL);
 185        pde = x86_ldl_phys(cs, pde_addr);
 186        if (!(pde & PG_PRESENT_MASK)) {
 187            goto do_fault;
 188        }
 189        ptep = pde | PG_NX_MASK;
 190
 191        /* if PSE bit is set, then we use a 4MB page */
 192        if ((pde & PG_PSE_MASK) && (pg_mode & PG_MODE_PSE)) {
 193            *page_size = 4096 * 1024;
 194            pte_addr = pde_addr;
 195
 196            /* Bits 20-13 provide bits 39-32 of the address, bit 21 is reserved.
 197             * Leave bits 20-13 in place for setting accessed/dirty bits below.
 198             */
 199            pte = pde | ((pde & 0x1fe000LL) << (32 - 13));
 200            rsvd_mask = 0x200000;
 201            goto do_check_protect_pse36;
 202        }
 203
 204        if (!(pde & PG_ACCESSED_MASK)) {
 205            pde |= PG_ACCESSED_MASK;
 206            x86_stl_phys_notdirty(cs, pde_addr, pde);
 207        }
 208
 209        /* page directory entry */
 210        pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) &
 211            a20_mask;
 212        pte_addr = GET_HPHYS(cs, pte_addr, MMU_DATA_STORE, NULL);
 213        pte = x86_ldl_phys(cs, pte_addr);
 214        if (!(pte & PG_PRESENT_MASK)) {
 215            goto do_fault;
 216        }
 217        /* combine pde and pte user and rw protections */
 218        ptep &= pte | PG_NX_MASK;
 219        *page_size = 4096;
 220        rsvd_mask = 0;
 221    }
 222
 223do_check_protect:
 224    rsvd_mask |= (*page_size - 1) & PG_ADDRESS_MASK & ~PG_PSE_PAT_MASK;
 225do_check_protect_pse36:
 226    if (pte & rsvd_mask) {
 227        goto do_fault_rsvd;
 228    }
 229    ptep ^= PG_NX_MASK;
 230
 231    /* can the page can be put in the TLB?  prot will tell us */
 232    if (is_user && !(ptep & PG_USER_MASK)) {
 233        goto do_fault_protect;
 234    }
 235
 236    *prot = 0;
 237    if (mmu_idx != MMU_KSMAP_IDX || !(ptep & PG_USER_MASK)) {
 238        *prot |= PAGE_READ;
 239        if ((ptep & PG_RW_MASK) || !(is_user || (pg_mode & PG_MODE_WP))) {
 240            *prot |= PAGE_WRITE;
 241        }
 242    }
 243    if (!(ptep & PG_NX_MASK) &&
 244        (mmu_idx == MMU_USER_IDX ||
 245         !((pg_mode & PG_MODE_SMEP) && (ptep & PG_USER_MASK)))) {
 246        *prot |= PAGE_EXEC;
 247    }
 248
 249    if (ptep & PG_USER_MASK) {
 250        pkr = pg_mode & PG_MODE_PKE ? env->pkru : 0;
 251    } else {
 252        pkr = pg_mode & PG_MODE_PKS ? env->pkrs : 0;
 253    }
 254    if (pkr) {
 255        uint32_t pk = (pte & PG_PKRU_MASK) >> PG_PKRU_BIT;
 256        uint32_t pkr_ad = (pkr >> pk * 2) & 1;
 257        uint32_t pkr_wd = (pkr >> pk * 2) & 2;
 258        uint32_t pkr_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 259
 260        if (pkr_ad) {
 261            pkr_prot &= ~(PAGE_READ | PAGE_WRITE);
 262        } else if (pkr_wd && (is_user || (pg_mode & PG_MODE_WP))) {
 263            pkr_prot &= ~PAGE_WRITE;
 264        }
 265
 266        *prot &= pkr_prot;
 267        if ((pkr_prot & (1 << is_write1)) == 0) {
 268            assert(is_write1 != 2);
 269            error_code |= PG_ERROR_PK_MASK;
 270            goto do_fault_protect;
 271        }
 272    }
 273
 274    if ((*prot & (1 << is_write1)) == 0) {
 275        goto do_fault_protect;
 276    }
 277
 278    /* yes, it can! */
 279    is_dirty = is_write && !(pte & PG_DIRTY_MASK);
 280    if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
 281        pte |= PG_ACCESSED_MASK;
 282        if (is_dirty) {
 283            pte |= PG_DIRTY_MASK;
 284        }
 285        x86_stl_phys_notdirty(cs, pte_addr, pte);
 286    }
 287
 288    if (!(pte & PG_DIRTY_MASK)) {
 289        /* only set write access if already dirty... otherwise wait
 290           for dirty access */
 291        assert(!is_write);
 292        *prot &= ~PAGE_WRITE;
 293    }
 294
 295    pte = pte & a20_mask;
 296
 297    /* align to page_size */
 298    pte &= PG_ADDRESS_MASK & ~(*page_size - 1);
 299    page_offset = addr & (*page_size - 1);
 300    *xlat = GET_HPHYS(cs, pte + page_offset, is_write1, prot);
 301    return PG_ERROR_OK;
 302
 303 do_fault_rsvd:
 304    error_code |= PG_ERROR_RSVD_MASK;
 305 do_fault_protect:
 306    error_code |= PG_ERROR_P_MASK;
 307 do_fault:
 308    error_code |= (is_write << PG_ERROR_W_BIT);
 309    if (is_user)
 310        error_code |= PG_ERROR_U_MASK;
 311    if (is_write1 == 2 &&
 312        ((pg_mode & PG_MODE_NXE) || (pg_mode & PG_MODE_SMEP)))
 313        error_code |= PG_ERROR_I_D_MASK;
 314    return error_code;
 315}
 316
 317hwaddr get_hphys(CPUState *cs, hwaddr gphys, MMUAccessType access_type,
 318                        int *prot)
 319{
 320    CPUX86State *env = &X86_CPU(cs)->env;
 321    uint64_t exit_info_1;
 322    int page_size;
 323    int next_prot;
 324    hwaddr hphys;
 325
 326    if (likely(!(env->hflags2 & HF2_NPT_MASK))) {
 327        return gphys;
 328    }
 329
 330    exit_info_1 = mmu_translate(cs, gphys, NULL, env->nested_cr3,
 331                               access_type, MMU_USER_IDX, env->nested_pg_mode,
 332                               &hphys, &page_size, &next_prot);
 333    if (exit_info_1 == PG_ERROR_OK) {
 334        if (prot) {
 335            *prot &= next_prot;
 336        }
 337        return hphys;
 338    }
 339
 340    x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2),
 341                 gphys);
 342    if (prot) {
 343        exit_info_1 |= SVM_NPTEXIT_GPA;
 344    } else { /* page table access */
 345        exit_info_1 |= SVM_NPTEXIT_GPT;
 346    }
 347    cpu_vmexit(env, SVM_EXIT_NPF, exit_info_1, env->retaddr);
 348}
 349
 350/* return value:
 351 * -1 = cannot handle fault
 352 * 0  = nothing more to do
 353 * 1  = generate PF fault
 354 */
 355static int handle_mmu_fault(CPUState *cs, vaddr addr, int size,
 356                            int is_write1, int mmu_idx)
 357{
 358    X86CPU *cpu = X86_CPU(cs);
 359    CPUX86State *env = &cpu->env;
 360    int error_code = PG_ERROR_OK;
 361    int pg_mode, prot, page_size;
 362    hwaddr paddr;
 363    hwaddr vaddr;
 364
 365#if defined(DEBUG_MMU)
 366    printf("MMU fault: addr=%" VADDR_PRIx " w=%d mmu=%d eip=" TARGET_FMT_lx "\n",
 367           addr, is_write1, mmu_idx, env->eip);
 368#endif
 369
 370    if (!(env->cr[0] & CR0_PG_MASK)) {
 371        paddr = addr;
 372#ifdef TARGET_X86_64
 373        if (!(env->hflags & HF_LMA_MASK)) {
 374            /* Without long mode we can only address 32bits in real mode */
 375            paddr = (uint32_t)paddr;
 376        }
 377#endif
 378        prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 379        page_size = 4096;
 380    } else {
 381        pg_mode = get_pg_mode(env);
 382        if (pg_mode & PG_MODE_LMA) {
 383            int32_t sext;
 384
 385            /* test virtual address sign extension */
 386            sext = (int64_t)addr >> (pg_mode & PG_MODE_LA57 ? 56 : 47);
 387            if (sext != 0 && sext != -1) {
 388                env->error_code = 0;
 389                cs->exception_index = EXCP0D_GPF;
 390                return 1;
 391            }
 392        }
 393
 394        error_code = mmu_translate(cs, addr, get_hphys, env->cr[3], is_write1,
 395                                   mmu_idx, pg_mode,
 396                                   &paddr, &page_size, &prot);
 397    }
 398
 399    if (error_code == PG_ERROR_OK) {
 400        /* Even if 4MB pages, we map only one 4KB page in the cache to
 401           avoid filling it too fast */
 402        vaddr = addr & TARGET_PAGE_MASK;
 403        paddr &= TARGET_PAGE_MASK;
 404
 405        assert(prot & (1 << is_write1));
 406        tlb_set_page_with_attrs(cs, vaddr, paddr, cpu_get_mem_attrs(env),
 407                                prot, mmu_idx, page_size);
 408        return 0;
 409    } else {
 410        if (env->intercept_exceptions & (1 << EXCP0E_PAGE)) {
 411            /* cr2 is not modified in case of exceptions */
 412            x86_stq_phys(cs,
 413                     env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2),
 414                     addr);
 415        } else {
 416            env->cr[2] = addr;
 417        }
 418        env->error_code = error_code;
 419        cs->exception_index = EXCP0E_PAGE;
 420        return 1;
 421    }
 422}
 423
 424bool x86_cpu_tlb_fill(CPUState *cs, vaddr addr, int size,
 425                      MMUAccessType access_type, int mmu_idx,
 426                      bool probe, uintptr_t retaddr)
 427{
 428    X86CPU *cpu = X86_CPU(cs);
 429    CPUX86State *env = &cpu->env;
 430
 431    env->retaddr = retaddr;
 432    if (handle_mmu_fault(cs, addr, size, access_type, mmu_idx)) {
 433        /* FIXME: On error in get_hphys we have already jumped out.  */
 434        g_assert(!probe);
 435        raise_exception_err_ra(env, cs->exception_index,
 436                               env->error_code, retaddr);
 437    }
 438    return true;
 439}
 440