1/* 2 * arch/ia64/kernel/ivt.S 3 * 4 * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co 5 * Stephane Eranian <eranian@hpl.hp.com> 6 * David Mosberger <davidm@hpl.hp.com> 7 * Copyright (C) 2000, 2002-2003 Intel Co 8 * Asit Mallick <asit.k.mallick@intel.com> 9 * Suresh Siddha <suresh.b.siddha@intel.com> 10 * Kenneth Chen <kenneth.w.chen@intel.com> 11 * Fenghua Yu <fenghua.yu@intel.com> 12 * 13 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> TLB handling for SMP 14 * 00/12/20 David Mosberger-Tang <davidm@hpl.hp.com> DTLB/ITLB handler now uses virtual PT. 15 * 16 * Copyright (C) 2005 Hewlett-Packard Co 17 * Dan Magenheimer <dan.magenheimer@hp.com> 18 * Xen paravirtualization 19 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp> 20 * VA Linux Systems Japan K.K. 21 * pv_ops. 22 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 23 */ 24/* 25 * This file defines the interruption vector table used by the CPU. 26 * It does not include one entry per possible cause of interruption. 27 * 28 * The first 20 entries of the table contain 64 bundles each while the 29 * remaining 48 entries contain only 16 bundles each. 30 * 31 * The 64 bundles are used to allow inlining the whole handler for critical 32 * interruptions like TLB misses. 33 * 34 * For each entry, the comment is as follows: 35 * 36 * // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51) 37 * entry offset ----/ / / / / 38 * entry number ---------/ / / / 39 * size of the entry -------------/ / / 40 * vector name -------------------------------------/ / 41 * interruptions triggering this vector ----------------------/ 42 * 43 * The table is 32KB in size and must be aligned on 32KB boundary. 44 * (The CPU ignores the 15 lower bits of the address) 45 * 46 * Table is based upon EAS2.6 (Oct 1999) 47 */ 48 49 50#include <asm/asmmacro.h> 51#include <asm/break.h> 52#include <asm/kregs.h> 53#include <asm/asm-offsets.h> 54#include <asm/pgtable.h> 55#include <asm/processor.h> 56#include <asm/ptrace.h> 57#include <asm/thread_info.h> 58#include <asm/unistd.h> 59#include <asm/errno.h> 60 61#if 1 62# define PSR_DEFAULT_BITS psr.ac 63#else 64# define PSR_DEFAULT_BITS 0 65#endif 66 67#if 0 68 /* 69 * This lets you track the last eight faults that occurred on the CPU. Make sure ar.k2 isn't 70 * needed for something else before enabling this... 71 */ 72# define DBG_FAULT(i) mov r16=ar.k2;; shl r16=r16,8;; add r16=(i),r16;;mov ar.k2=r16 73#else 74# define DBG_FAULT(i) 75#endif 76 77#include "minstate.h" 78 79#define FAULT(n) \ 80 mov r31=pr; \ 81 mov r19=n;; /* prepare to save predicates */ \ 82 br.sptk.many dispatch_to_fault_handler 83 84 .section .text..ivt,"ax" 85 86 .align 32768 // align on 32KB boundary 87 .global ia64_ivt 88ia64_ivt: 89///////////////////////////////////////////////////////////////////////////////////////// 90// 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47) 91ENTRY(vhpt_miss) 92 DBG_FAULT(0) 93 /* 94 * The VHPT vector is invoked when the TLB entry for the virtual page table 95 * is missing. This happens only as a result of a previous 96 * (the "original") TLB miss, which may either be caused by an instruction 97 * fetch or a data access (or non-access). 98 * 99 * What we do here is normal TLB miss handing for the _original_ miss, 100 * followed by inserting the TLB entry for the virtual page table page 101 * that the VHPT walker was attempting to access. The latter gets 102 * inserted as long as page table entry above pte level have valid 103 * mappings for the faulting address. The TLB entry for the original 104 * miss gets inserted only if the pte entry indicates that the page is 105 * present. 106 * 107 * do_page_fault gets invoked in the following cases: 108 * - the faulting virtual address uses unimplemented address bits 109 * - the faulting virtual address has no valid page table mapping 110 */ 111 MOV_FROM_IFA(r16) // get address that caused the TLB miss 112#ifdef CONFIG_HUGETLB_PAGE 113 movl r18=PAGE_SHIFT 114 MOV_FROM_ITIR(r25) 115#endif 116 ;; 117 RSM_PSR_DT // use physical addressing for data 118 mov r31=pr // save the predicate registers 119 mov r19=IA64_KR(PT_BASE) // get page table base address 120 shl r21=r16,3 // shift bit 60 into sign bit 121 shr.u r17=r16,61 // get the region number into r17 122 ;; 123 shr.u r22=r21,3 124#ifdef CONFIG_HUGETLB_PAGE 125 extr.u r26=r25,2,6 126 ;; 127 cmp.ne p8,p0=r18,r26 128 sub r27=r26,r18 129 ;; 130(p8) dep r25=r18,r25,2,6 131(p8) shr r22=r22,r27 132#endif 133 ;; 134 cmp.eq p6,p7=5,r17 // is IFA pointing into to region 5? 135 shr.u r18=r22,PGDIR_SHIFT // get bottom portion of pgd index bit 136 ;; 137(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place 138 139 srlz.d 140 LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir 141 142 .pred.rel "mutex", p6, p7 143(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT 144(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3 145 ;; 146(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5 147(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4] 148 cmp.eq p7,p6=0,r21 // unused address bits all zeroes? 149#ifdef CONFIG_PGTABLE_4 150 shr.u r28=r22,PUD_SHIFT // shift pud index into position 151#else 152 shr.u r18=r22,PMD_SHIFT // shift pmd index into position 153#endif 154 ;; 155 ld8 r17=[r17] // get *pgd (may be 0) 156 ;; 157(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL? 158#ifdef CONFIG_PGTABLE_4 159 dep r28=r28,r17,3,(PAGE_SHIFT-3) // r28=pud_offset(pgd,addr) 160 ;; 161 shr.u r18=r22,PMD_SHIFT // shift pmd index into position 162(p7) ld8 r29=[r28] // get *pud (may be 0) 163 ;; 164(p7) cmp.eq.or.andcm p6,p7=r29,r0 // was pud_present(*pud) == NULL? 165 dep r17=r18,r29,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr) 166#else 167 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pgd,addr) 168#endif 169 ;; 170(p7) ld8 r20=[r17] // get *pmd (may be 0) 171 shr.u r19=r22,PAGE_SHIFT // shift pte index into position 172 ;; 173(p7) cmp.eq.or.andcm p6,p7=r20,r0 // was pmd_present(*pmd) == NULL? 174 dep r21=r19,r20,3,(PAGE_SHIFT-3) // r21=pte_offset(pmd,addr) 175 ;; 176(p7) ld8 r18=[r21] // read *pte 177 MOV_FROM_ISR(r19) // cr.isr bit 32 tells us if this is an insn miss 178 ;; 179(p7) tbit.z p6,p7=r18,_PAGE_P_BIT // page present bit cleared? 180 MOV_FROM_IHA(r22) // get the VHPT address that caused the TLB miss 181 ;; // avoid RAW on p7 182(p7) tbit.nz.unc p10,p11=r19,32 // is it an instruction TLB miss? 183 dep r23=0,r20,0,PAGE_SHIFT // clear low bits to get page address 184 ;; 185 ITC_I_AND_D(p10, p11, r18, r24) // insert the instruction TLB entry and 186 // insert the data TLB entry 187(p6) br.cond.spnt.many page_fault // handle bad address/page not present (page fault) 188 MOV_TO_IFA(r22, r24) 189 190#ifdef CONFIG_HUGETLB_PAGE 191 MOV_TO_ITIR(p8, r25, r24) // change to default page-size for VHPT 192#endif 193 194 /* 195 * Now compute and insert the TLB entry for the virtual page table. We never 196 * execute in a page table page so there is no need to set the exception deferral 197 * bit. 198 */ 199 adds r24=__DIRTY_BITS_NO_ED|_PAGE_PL_0|_PAGE_AR_RW,r23 200 ;; 201 ITC_D(p7, r24, r25) 202 ;; 203#ifdef CONFIG_SMP 204 /* 205 * Tell the assemblers dependency-violation checker that the above "itc" instructions 206 * cannot possibly affect the following loads: 207 */ 208 dv_serialize_data 209 210 /* 211 * Re-check pagetable entry. If they changed, we may have received a ptc.g 212 * between reading the pagetable and the "itc". If so, flush the entry we 213 * inserted and retry. At this point, we have: 214 * 215 * r28 = equivalent of pud_offset(pgd, ifa) 216 * r17 = equivalent of pmd_offset(pud, ifa) 217 * r21 = equivalent of pte_offset(pmd, ifa) 218 * 219 * r29 = *pud 220 * r20 = *pmd 221 * r18 = *pte 222 */ 223 ld8 r25=[r21] // read *pte again 224 ld8 r26=[r17] // read *pmd again 225#ifdef CONFIG_PGTABLE_4 226 ld8 r19=[r28] // read *pud again 227#endif 228 cmp.ne p6,p7=r0,r0 229 ;; 230 cmp.ne.or.andcm p6,p7=r26,r20 // did *pmd change 231#ifdef CONFIG_PGTABLE_4 232 cmp.ne.or.andcm p6,p7=r19,r29 // did *pud change 233#endif 234 mov r27=PAGE_SHIFT<<2 235 ;; 236(p6) ptc.l r22,r27 // purge PTE page translation 237(p7) cmp.ne.or.andcm p6,p7=r25,r18 // did *pte change 238 ;; 239(p6) ptc.l r16,r27 // purge translation 240#endif 241 242 mov pr=r31,-1 // restore predicate registers 243 RFI 244END(vhpt_miss) 245 246 .org ia64_ivt+0x400 247///////////////////////////////////////////////////////////////////////////////////////// 248// 0x0400 Entry 1 (size 64 bundles) ITLB (21) 249ENTRY(itlb_miss) 250 DBG_FAULT(1) 251 /* 252 * The ITLB handler accesses the PTE via the virtually mapped linear 253 * page table. If a nested TLB miss occurs, we switch into physical 254 * mode, walk the page table, and then re-execute the PTE read and 255 * go on normally after that. 256 */ 257 MOV_FROM_IFA(r16) // get virtual address 258 mov r29=b0 // save b0 259 mov r31=pr // save predicates 260.itlb_fault: 261 MOV_FROM_IHA(r17) // get virtual address of PTE 262 movl r30=1f // load nested fault continuation point 263 ;; 2641: ld8 r18=[r17] // read *pte 265 ;; 266 mov b0=r29 267 tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared? 268(p6) br.cond.spnt page_fault 269 ;; 270 ITC_I(p0, r18, r19) 271 ;; 272#ifdef CONFIG_SMP 273 /* 274 * Tell the assemblers dependency-violation checker that the above "itc" instructions 275 * cannot possibly affect the following loads: 276 */ 277 dv_serialize_data 278 279 ld8 r19=[r17] // read *pte again and see if same 280 mov r20=PAGE_SHIFT<<2 // setup page size for purge 281 ;; 282 cmp.ne p7,p0=r18,r19 283 ;; 284(p7) ptc.l r16,r20 285#endif 286 mov pr=r31,-1 287 RFI 288END(itlb_miss) 289 290 .org ia64_ivt+0x0800 291///////////////////////////////////////////////////////////////////////////////////////// 292// 0x0800 Entry 2 (size 64 bundles) DTLB (9,48) 293ENTRY(dtlb_miss) 294 DBG_FAULT(2) 295 /* 296 * The DTLB handler accesses the PTE via the virtually mapped linear 297 * page table. If a nested TLB miss occurs, we switch into physical 298 * mode, walk the page table, and then re-execute the PTE read and 299 * go on normally after that. 300 */ 301 MOV_FROM_IFA(r16) // get virtual address 302 mov r29=b0 // save b0 303 mov r31=pr // save predicates 304dtlb_fault: 305 MOV_FROM_IHA(r17) // get virtual address of PTE 306 movl r30=1f // load nested fault continuation point 307 ;; 3081: ld8 r18=[r17] // read *pte 309 ;; 310 mov b0=r29 311 tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared? 312(p6) br.cond.spnt page_fault 313 ;; 314 ITC_D(p0, r18, r19) 315 ;; 316#ifdef CONFIG_SMP 317 /* 318 * Tell the assemblers dependency-violation checker that the above "itc" instructions 319 * cannot possibly affect the following loads: 320 */ 321 dv_serialize_data 322 323 ld8 r19=[r17] // read *pte again and see if same 324 mov r20=PAGE_SHIFT<<2 // setup page size for purge 325 ;; 326 cmp.ne p7,p0=r18,r19 327 ;; 328(p7) ptc.l r16,r20 329#endif 330 mov pr=r31,-1 331 RFI 332END(dtlb_miss) 333 334 .org ia64_ivt+0x0c00 335///////////////////////////////////////////////////////////////////////////////////////// 336// 0x0c00 Entry 3 (size 64 bundles) Alt ITLB (19) 337ENTRY(alt_itlb_miss) 338 DBG_FAULT(3) 339 MOV_FROM_IFA(r16) // get address that caused the TLB miss 340 movl r17=PAGE_KERNEL 341 MOV_FROM_IPSR(p0, r21) 342 movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff) 343 mov r31=pr 344 ;; 345#ifdef CONFIG_DISABLE_VHPT 346 shr.u r22=r16,61 // get the region number into r21 347 ;; 348 cmp.gt p8,p0=6,r22 // user mode 349 ;; 350 THASH(p8, r17, r16, r23) 351 ;; 352 MOV_TO_IHA(p8, r17, r23) 353(p8) mov r29=b0 // save b0 354(p8) br.cond.dptk .itlb_fault 355#endif 356 extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl 357 and r19=r19,r16 // clear ed, reserved bits, and PTE control bits 358 shr.u r18=r16,57 // move address bit 61 to bit 4 359 ;; 360 andcm r18=0x10,r18 // bit 4=~address-bit(61) 361 cmp.ne p8,p0=r0,r23 // psr.cpl != 0? 362 or r19=r17,r19 // insert PTE control bits into r19 363 ;; 364 or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6 365(p8) br.cond.spnt page_fault 366 ;; 367 ITC_I(p0, r19, r18) // insert the TLB entry 368 mov pr=r31,-1 369 RFI 370END(alt_itlb_miss) 371 372 .org ia64_ivt+0x1000 373///////////////////////////////////////////////////////////////////////////////////////// 374// 0x1000 Entry 4 (size 64 bundles) Alt DTLB (7,46) 375ENTRY(alt_dtlb_miss) 376 DBG_FAULT(4) 377 MOV_FROM_IFA(r16) // get address that caused the TLB miss 378 movl r17=PAGE_KERNEL 379 MOV_FROM_ISR(r20) 380 movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff) 381 MOV_FROM_IPSR(p0, r21) 382 mov r31=pr 383 mov r24=PERCPU_ADDR 384 ;; 385#ifdef CONFIG_DISABLE_VHPT 386 shr.u r22=r16,61 // get the region number into r21 387 ;; 388 cmp.gt p8,p0=6,r22 // access to region 0-5 389 ;; 390 THASH(p8, r17, r16, r25) 391 ;; 392 MOV_TO_IHA(p8, r17, r25) 393(p8) mov r29=b0 // save b0 394(p8) br.cond.dptk dtlb_fault 395#endif 396 cmp.ge p10,p11=r16,r24 // access to per_cpu_data? 397 tbit.z p12,p0=r16,61 // access to region 6? 398 mov r25=PERCPU_PAGE_SHIFT << 2 399 mov r26=PERCPU_PAGE_SIZE 400 nop.m 0 401 nop.b 0 402 ;; 403(p10) mov r19=IA64_KR(PER_CPU_DATA) 404(p11) and r19=r19,r16 // clear non-ppn fields 405 extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl 406 and r22=IA64_ISR_CODE_MASK,r20 // get the isr.code field 407 tbit.nz p6,p7=r20,IA64_ISR_SP_BIT // is speculation bit on? 408 tbit.nz p9,p0=r20,IA64_ISR_NA_BIT // is non-access bit on? 409 ;; 410(p10) sub r19=r19,r26 411 MOV_TO_ITIR(p10, r25, r24) 412 cmp.ne p8,p0=r0,r23 413(p9) cmp.eq.or.andcm p6,p7=IA64_ISR_CODE_LFETCH,r22 // check isr.code field 414(p12) dep r17=-1,r17,4,1 // set ma=UC for region 6 addr 415(p8) br.cond.spnt page_fault 416 417 dep r21=-1,r21,IA64_PSR_ED_BIT,1 418 ;; 419 or r19=r19,r17 // insert PTE control bits into r19 420 MOV_TO_IPSR(p6, r21, r24) 421 ;; 422 ITC_D(p7, r19, r18) // insert the TLB entry 423 mov pr=r31,-1 424 RFI 425END(alt_dtlb_miss) 426 427 .org ia64_ivt+0x1400 428///////////////////////////////////////////////////////////////////////////////////////// 429// 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45) 430ENTRY(nested_dtlb_miss) 431 /* 432 * In the absence of kernel bugs, we get here when the virtually mapped linear 433 * page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction 434 * Access-bit, or Data Access-bit faults). If the DTLB entry for the virtual page 435 * table is missing, a nested TLB miss fault is triggered and control is 436 * transferred to this point. When this happens, we lookup the pte for the 437 * faulting address by walking the page table in physical mode and return to the 438 * continuation point passed in register r30 (or call page_fault if the address is 439 * not mapped). 440 * 441 * Input: r16: faulting address 442 * r29: saved b0 443 * r30: continuation address 444 * r31: saved pr 445 * 446 * Output: r17: physical address of PTE of faulting address 447 * r29: saved b0 448 * r30: continuation address 449 * r31: saved pr 450 * 451 * Clobbered: b0, r18, r19, r21, r22, psr.dt (cleared) 452 */ 453 RSM_PSR_DT // switch to using physical data addressing 454 mov r19=IA64_KR(PT_BASE) // get the page table base address 455 shl r21=r16,3 // shift bit 60 into sign bit 456 MOV_FROM_ITIR(r18) 457 ;; 458 shr.u r17=r16,61 // get the region number into r17 459 extr.u r18=r18,2,6 // get the faulting page size 460 ;; 461 cmp.eq p6,p7=5,r17 // is faulting address in region 5? 462 add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address 463 add r18=PGDIR_SHIFT-PAGE_SHIFT,r18 464 ;; 465 shr.u r22=r16,r22 466 shr.u r18=r16,r18 467(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place 468 469 srlz.d 470 LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir 471 472 .pred.rel "mutex", p6, p7 473(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT 474(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3 475 ;; 476(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5 477(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4] 478 cmp.eq p7,p6=0,r21 // unused address bits all zeroes? 479#ifdef CONFIG_PGTABLE_4 480 shr.u r18=r22,PUD_SHIFT // shift pud index into position 481#else 482 shr.u r18=r22,PMD_SHIFT // shift pmd index into position 483#endif 484 ;; 485 ld8 r17=[r17] // get *pgd (may be 0) 486 ;; 487(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL? 488 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=p[u|m]d_offset(pgd,addr) 489 ;; 490#ifdef CONFIG_PGTABLE_4 491(p7) ld8 r17=[r17] // get *pud (may be 0) 492 shr.u r18=r22,PMD_SHIFT // shift pmd index into position 493 ;; 494(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL? 495 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr) 496 ;; 497#endif 498(p7) ld8 r17=[r17] // get *pmd (may be 0) 499 shr.u r19=r22,PAGE_SHIFT // shift pte index into position 500 ;; 501(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL? 502 dep r17=r19,r17,3,(PAGE_SHIFT-3) // r17=pte_offset(pmd,addr); 503(p6) br.cond.spnt page_fault 504 mov b0=r30 505 br.sptk.many b0 // return to continuation point 506END(nested_dtlb_miss) 507 508 .org ia64_ivt+0x1800 509///////////////////////////////////////////////////////////////////////////////////////// 510// 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24) 511ENTRY(ikey_miss) 512 DBG_FAULT(6) 513 FAULT(6) 514END(ikey_miss) 515 516 .org ia64_ivt+0x1c00 517///////////////////////////////////////////////////////////////////////////////////////// 518// 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51) 519ENTRY(dkey_miss) 520 DBG_FAULT(7) 521 FAULT(7) 522END(dkey_miss) 523 524 .org ia64_ivt+0x2000 525///////////////////////////////////////////////////////////////////////////////////////// 526// 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54) 527ENTRY(dirty_bit) 528 DBG_FAULT(8) 529 /* 530 * What we do here is to simply turn on the dirty bit in the PTE. We need to 531 * update both the page-table and the TLB entry. To efficiently access the PTE, 532 * we address it through the virtual page table. Most likely, the TLB entry for 533 * the relevant virtual page table page is still present in the TLB so we can 534 * normally do this without additional TLB misses. In case the necessary virtual 535 * page table TLB entry isn't present, we take a nested TLB miss hit where we look 536 * up the physical address of the L3 PTE and then continue at label 1 below. 537 */ 538 MOV_FROM_IFA(r16) // get the address that caused the fault 539 movl r30=1f // load continuation point in case of nested fault 540 ;; 541 THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE 542 mov r29=b0 // save b0 in case of nested fault 543 mov r31=pr // save pr 544#ifdef CONFIG_SMP 545 mov r28=ar.ccv // save ar.ccv 546 ;; 5471: ld8 r18=[r17] 548 ;; // avoid RAW on r18 549 mov ar.ccv=r18 // set compare value for cmpxchg 550 or r25=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits 551 tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit 552 ;; 553(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only update if page is present 554 mov r24=PAGE_SHIFT<<2 555 ;; 556(p6) cmp.eq p6,p7=r26,r18 // Only compare if page is present 557 ;; 558 ITC_D(p6, r25, r18) // install updated PTE 559 ;; 560 /* 561 * Tell the assemblers dependency-violation checker that the above "itc" instructions 562 * cannot possibly affect the following loads: 563 */ 564 dv_serialize_data 565 566 ld8 r18=[r17] // read PTE again 567 ;; 568 cmp.eq p6,p7=r18,r25 // is it same as the newly installed 569 ;; 570(p7) ptc.l r16,r24 571 mov b0=r29 // restore b0 572 mov ar.ccv=r28 573#else 574 ;; 5751: ld8 r18=[r17] 576 ;; // avoid RAW on r18 577 or r18=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits 578 mov b0=r29 // restore b0 579 ;; 580 st8 [r17]=r18 // store back updated PTE 581 ITC_D(p0, r18, r16) // install updated PTE 582#endif 583 mov pr=r31,-1 // restore pr 584 RFI 585END(dirty_bit) 586 587 .org ia64_ivt+0x2400 588///////////////////////////////////////////////////////////////////////////////////////// 589// 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27) 590ENTRY(iaccess_bit) 591 DBG_FAULT(9) 592 // Like Entry 8, except for instruction access 593 MOV_FROM_IFA(r16) // get the address that caused the fault 594 movl r30=1f // load continuation point in case of nested fault 595 mov r31=pr // save predicates 596#ifdef CONFIG_ITANIUM 597 /* 598 * Erratum 10 (IFA may contain incorrect address) has "NoFix" status. 599 */ 600 MOV_FROM_IPSR(p0, r17) 601 ;; 602 MOV_FROM_IIP(r18) 603 tbit.z p6,p0=r17,IA64_PSR_IS_BIT // IA64 instruction set? 604 ;; 605(p6) mov r16=r18 // if so, use cr.iip instead of cr.ifa 606#endif /* CONFIG_ITANIUM */ 607 ;; 608 THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE 609 mov r29=b0 // save b0 in case of nested fault) 610#ifdef CONFIG_SMP 611 mov r28=ar.ccv // save ar.ccv 612 ;; 6131: ld8 r18=[r17] 614 ;; 615 mov ar.ccv=r18 // set compare value for cmpxchg 616 or r25=_PAGE_A,r18 // set the accessed bit 617 tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit 618 ;; 619(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present 620 mov r24=PAGE_SHIFT<<2 621 ;; 622(p6) cmp.eq p6,p7=r26,r18 // Only if page present 623 ;; 624 ITC_I(p6, r25, r26) // install updated PTE 625 ;; 626 /* 627 * Tell the assemblers dependency-violation checker that the above "itc" instructions 628 * cannot possibly affect the following loads: 629 */ 630 dv_serialize_data 631 632 ld8 r18=[r17] // read PTE again 633 ;; 634 cmp.eq p6,p7=r18,r25 // is it same as the newly installed 635 ;; 636(p7) ptc.l r16,r24 637 mov b0=r29 // restore b0 638 mov ar.ccv=r28 639#else /* !CONFIG_SMP */ 640 ;; 6411: ld8 r18=[r17] 642 ;; 643 or r18=_PAGE_A,r18 // set the accessed bit 644 mov b0=r29 // restore b0 645 ;; 646 st8 [r17]=r18 // store back updated PTE 647 ITC_I(p0, r18, r16) // install updated PTE 648#endif /* !CONFIG_SMP */ 649 mov pr=r31,-1 650 RFI 651END(iaccess_bit) 652 653 .org ia64_ivt+0x2800 654///////////////////////////////////////////////////////////////////////////////////////// 655// 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55) 656ENTRY(daccess_bit) 657 DBG_FAULT(10) 658 // Like Entry 8, except for data access 659 MOV_FROM_IFA(r16) // get the address that caused the fault 660 movl r30=1f // load continuation point in case of nested fault 661 ;; 662 THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE 663 mov r31=pr 664 mov r29=b0 // save b0 in case of nested fault) 665#ifdef CONFIG_SMP 666 mov r28=ar.ccv // save ar.ccv 667 ;; 6681: ld8 r18=[r17] 669 ;; // avoid RAW on r18 670 mov ar.ccv=r18 // set compare value for cmpxchg 671 or r25=_PAGE_A,r18 // set the dirty bit 672 tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit 673 ;; 674(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page is present 675 mov r24=PAGE_SHIFT<<2 676 ;; 677(p6) cmp.eq p6,p7=r26,r18 // Only if page is present 678 ;; 679 ITC_D(p6, r25, r26) // install updated PTE 680 /* 681 * Tell the assemblers dependency-violation checker that the above "itc" instructions 682 * cannot possibly affect the following loads: 683 */ 684 dv_serialize_data 685 ;; 686 ld8 r18=[r17] // read PTE again 687 ;; 688 cmp.eq p6,p7=r18,r25 // is it same as the newly installed 689 ;; 690(p7) ptc.l r16,r24 691 mov ar.ccv=r28 692#else 693 ;; 6941: ld8 r18=[r17] 695 ;; // avoid RAW on r18 696 or r18=_PAGE_A,r18 // set the accessed bit 697 ;; 698 st8 [r17]=r18 // store back updated PTE 699 ITC_D(p0, r18, r16) // install updated PTE 700#endif 701 mov b0=r29 // restore b0 702 mov pr=r31,-1 703 RFI 704END(daccess_bit) 705 706 .org ia64_ivt+0x2c00 707///////////////////////////////////////////////////////////////////////////////////////// 708// 0x2c00 Entry 11 (size 64 bundles) Break instruction (33) 709ENTRY(break_fault) 710 /* 711 * The streamlined system call entry/exit paths only save/restore the initial part 712 * of pt_regs. This implies that the callers of system-calls must adhere to the 713 * normal procedure calling conventions. 714 * 715 * Registers to be saved & restored: 716 * CR registers: cr.ipsr, cr.iip, cr.ifs 717 * AR registers: ar.unat, ar.pfs, ar.rsc, ar.rnat, ar.bspstore, ar.fpsr 718 * others: pr, b0, b6, loadrs, r1, r11, r12, r13, r15 719 * Registers to be restored only: 720 * r8-r11: output value from the system call. 721 * 722 * During system call exit, scratch registers (including r15) are modified/cleared 723 * to prevent leaking bits from kernel to user level. 724 */ 725 DBG_FAULT(11) 726 mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc) 727 MOV_FROM_IPSR(p0, r29) // M2 (12 cyc) 728 mov r31=pr // I0 (2 cyc) 729 730 MOV_FROM_IIM(r17) // M2 (2 cyc) 731 mov.m r27=ar.rsc // M2 (12 cyc) 732 mov r18=__IA64_BREAK_SYSCALL // A 733 734 mov.m ar.rsc=0 // M2 735 mov.m r21=ar.fpsr // M2 (12 cyc) 736 mov r19=b6 // I0 (2 cyc) 737 ;; 738 mov.m r23=ar.bspstore // M2 (12 cyc) 739 mov.m r24=ar.rnat // M2 (5 cyc) 740 mov.i r26=ar.pfs // I0 (2 cyc) 741 742 invala // M0|1 743 nop.m 0 // M 744 mov r20=r1 // A save r1 745 746 nop.m 0 747 movl r30=sys_call_table // X 748 749 MOV_FROM_IIP(r28) // M2 (2 cyc) 750 cmp.eq p0,p7=r18,r17 // I0 is this a system call? 751(p7) br.cond.spnt non_syscall // B no -> 752 // 753 // From this point on, we are definitely on the syscall-path 754 // and we can use (non-banked) scratch registers. 755 // 756/////////////////////////////////////////////////////////////////////// 757 mov r1=r16 // A move task-pointer to "addl"-addressable reg 758 mov r2=r16 // A setup r2 for ia64_syscall_setup 759 add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = ¤t_thread_info()->flags 760 761 adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 762 adds r15=-1024,r15 // A subtract 1024 from syscall number 763 mov r3=NR_syscalls - 1 764 ;; 765 ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag 766 ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags 767 extr.u r8=r29,41,2 // I0 extract ei field from cr.ipsr 768 769 shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024) 770 addl r22=IA64_RBS_OFFSET,r1 // A compute base of RBS 771 cmp.leu p6,p7=r15,r3 // A syscall number in range? 772 ;; 773 774 lfetch.fault.excl.nt1 [r22] // M0|1 prefetch RBS 775(p6) ld8 r30=[r30] // M0|1 load address of syscall entry point 776 tnat.nz.or p7,p0=r15 // I0 is syscall nr a NaT? 777 778 mov.m ar.bspstore=r22 // M2 switch to kernel RBS 779 cmp.eq p8,p9=2,r8 // A isr.ei==2? 780 ;; 781 782(p8) mov r8=0 // A clear ei to 0 783(p7) movl r30=sys_ni_syscall // X 784 785(p8) adds r28=16,r28 // A switch cr.iip to next bundle 786(p9) adds r8=1,r8 // A increment ei to next slot 787#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 788 ;; 789 mov b6=r30 // I0 setup syscall handler branch reg early 790#else 791 nop.i 0 792 ;; 793#endif 794 795 mov.m r25=ar.unat // M2 (5 cyc) 796 dep r29=r8,r29,41,2 // I0 insert new ei into cr.ipsr 797 adds r15=1024,r15 // A restore original syscall number 798 // 799 // If any of the above loads miss in L1D, we'll stall here until 800 // the data arrives. 801 // 802/////////////////////////////////////////////////////////////////////// 803 st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag 804#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 805 MOV_FROM_ITC(p0, p14, r30, r18) // M get cycle for accounting 806#else 807 mov b6=r30 // I0 setup syscall handler branch reg early 808#endif 809 cmp.eq pKStk,pUStk=r0,r17 // A were we on kernel stacks already? 810 811 and r9=_TIF_SYSCALL_TRACEAUDIT,r9 // A mask trace or audit 812 mov r18=ar.bsp // M2 (12 cyc) 813(pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS 814 ;; 815.back_from_break_fixup: 816(pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack 817 cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited? 818 br.call.sptk.many b7=ia64_syscall_setup // B 8191: 820#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 821 // mov.m r30=ar.itc is called in advance, and r13 is current 822 add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13 // A 823 add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13 // A 824(pKStk) br.cond.spnt .skip_accounting // B unlikely skip 825 ;; 826 ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // M get last stamp 827 ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // M time at leave 828 ;; 829 ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // M cumulated stime 830 ld8 r21=[r17] // M cumulated utime 831 sub r22=r19,r18 // A stime before leave 832 ;; 833 st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // M update stamp 834 sub r18=r30,r19 // A elapsed time in user 835 ;; 836 add r20=r20,r22 // A sum stime 837 add r21=r21,r18 // A sum utime 838 ;; 839 st8 [r16]=r20 // M update stime 840 st8 [r17]=r21 // M update utime 841 ;; 842.skip_accounting: 843#endif 844 mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0 845 nop 0 846 BSW_1(r2, r14) // B (6 cyc) regs are saved, switch to bank 1 847 ;; 848 849 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r16) // M2 now it's safe to re-enable intr.-collection 850 // M0 ensure interruption collection is on 851 movl r3=ia64_ret_from_syscall // X 852 ;; 853 mov rp=r3 // I0 set the real return addr 854(p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT 855 856 SSM_PSR_I(p15, p15, r16) // M2 restore psr.i 857(p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr) 858 br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic 859 // NOT REACHED 860/////////////////////////////////////////////////////////////////////// 861 // On entry, we optimistically assumed that we're coming from user-space. 862 // For the rare cases where a system-call is done from within the kernel, 863 // we fix things up at this point: 864.break_fixup: 865 add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure 866 mov ar.rnat=r24 // M2 restore kernel's AR.RNAT 867 ;; 868 mov ar.bspstore=r23 // M2 restore kernel's AR.BSPSTORE 869 br.cond.sptk .back_from_break_fixup 870END(break_fault) 871 872 .org ia64_ivt+0x3000 873///////////////////////////////////////////////////////////////////////////////////////// 874// 0x3000 Entry 12 (size 64 bundles) External Interrupt (4) 875ENTRY(interrupt) 876 /* interrupt handler has become too big to fit this area. */ 877 br.sptk.many __interrupt 878END(interrupt) 879 880 .org ia64_ivt+0x3400 881///////////////////////////////////////////////////////////////////////////////////////// 882// 0x3400 Entry 13 (size 64 bundles) Reserved 883 DBG_FAULT(13) 884 FAULT(13) 885 886 .org ia64_ivt+0x3800 887///////////////////////////////////////////////////////////////////////////////////////// 888// 0x3800 Entry 14 (size 64 bundles) Reserved 889 DBG_FAULT(14) 890 FAULT(14) 891 892 /* 893 * There is no particular reason for this code to be here, other than that 894 * there happens to be space here that would go unused otherwise. If this 895 * fault ever gets "unreserved", simply moved the following code to a more 896 * suitable spot... 897 * 898 * ia64_syscall_setup() is a separate subroutine so that it can 899 * allocate stacked registers so it can safely demine any 900 * potential NaT values from the input registers. 901 * 902 * On entry: 903 * - executing on bank 0 or bank 1 register set (doesn't matter) 904 * - r1: stack pointer 905 * - r2: current task pointer 906 * - r3: preserved 907 * - r11: original contents (saved ar.pfs to be saved) 908 * - r12: original contents (sp to be saved) 909 * - r13: original contents (tp to be saved) 910 * - r15: original contents (syscall # to be saved) 911 * - r18: saved bsp (after switching to kernel stack) 912 * - r19: saved b6 913 * - r20: saved r1 (gp) 914 * - r21: saved ar.fpsr 915 * - r22: kernel's register backing store base (krbs_base) 916 * - r23: saved ar.bspstore 917 * - r24: saved ar.rnat 918 * - r25: saved ar.unat 919 * - r26: saved ar.pfs 920 * - r27: saved ar.rsc 921 * - r28: saved cr.iip 922 * - r29: saved cr.ipsr 923 * - r30: ar.itc for accounting (don't touch) 924 * - r31: saved pr 925 * - b0: original contents (to be saved) 926 * On exit: 927 * - p10: TRUE if syscall is invoked with more than 8 out 928 * registers or r15's Nat is true 929 * - r1: kernel's gp 930 * - r3: preserved (same as on entry) 931 * - r8: -EINVAL if p10 is true 932 * - r12: points to kernel stack 933 * - r13: points to current task 934 * - r14: preserved (same as on entry) 935 * - p13: preserved 936 * - p15: TRUE if interrupts need to be re-enabled 937 * - ar.fpsr: set to kernel settings 938 * - b6: preserved (same as on entry) 939 */ 940#ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE 941GLOBAL_ENTRY(ia64_syscall_setup) 942#if PT(B6) != 0 943# error This code assumes that b6 is the first field in pt_regs. 944#endif 945 st8 [r1]=r19 // save b6 946 add r16=PT(CR_IPSR),r1 // initialize first base pointer 947 add r17=PT(R11),r1 // initialize second base pointer 948 ;; 949 alloc r19=ar.pfs,8,0,0,0 // ensure in0-in7 are writable 950 st8 [r16]=r29,PT(AR_PFS)-PT(CR_IPSR) // save cr.ipsr 951 tnat.nz p8,p0=in0 952 953 st8.spill [r17]=r11,PT(CR_IIP)-PT(R11) // save r11 954 tnat.nz p9,p0=in1 955(pKStk) mov r18=r0 // make sure r18 isn't NaT 956 ;; 957 958 st8 [r16]=r26,PT(CR_IFS)-PT(AR_PFS) // save ar.pfs 959 st8 [r17]=r28,PT(AR_UNAT)-PT(CR_IIP) // save cr.iip 960 mov r28=b0 // save b0 (2 cyc) 961 ;; 962 963 st8 [r17]=r25,PT(AR_RSC)-PT(AR_UNAT) // save ar.unat 964 dep r19=0,r19,38,26 // clear all bits but 0..37 [I0] 965(p8) mov in0=-1 966 ;; 967 968 st8 [r16]=r19,PT(AR_RNAT)-PT(CR_IFS) // store ar.pfs.pfm in cr.ifs 969 extr.u r11=r19,7,7 // I0 // get sol of ar.pfs 970 and r8=0x7f,r19 // A // get sof of ar.pfs 971 972 st8 [r17]=r27,PT(AR_BSPSTORE)-PT(AR_RSC)// save ar.rsc 973 tbit.nz p15,p0=r29,IA64_PSR_I_BIT // I0 974(p9) mov in1=-1 975 ;; 976 977(pUStk) sub r18=r18,r22 // r18=RSE.ndirty*8 978 tnat.nz p10,p0=in2 979 add r11=8,r11 980 ;; 981(pKStk) adds r16=PT(PR)-PT(AR_RNAT),r16 // skip over ar_rnat field 982(pKStk) adds r17=PT(B0)-PT(AR_BSPSTORE),r17 // skip over ar_bspstore field 983 tnat.nz p11,p0=in3 984 ;; 985(p10) mov in2=-1 986 tnat.nz p12,p0=in4 // [I0] 987(p11) mov in3=-1 988 ;; 989(pUStk) st8 [r16]=r24,PT(PR)-PT(AR_RNAT) // save ar.rnat 990(pUStk) st8 [r17]=r23,PT(B0)-PT(AR_BSPSTORE) // save ar.bspstore 991 shl r18=r18,16 // compute ar.rsc to be used for "loadrs" 992 ;; 993 st8 [r16]=r31,PT(LOADRS)-PT(PR) // save predicates 994 st8 [r17]=r28,PT(R1)-PT(B0) // save b0 995 tnat.nz p13,p0=in5 // [I0] 996 ;; 997 st8 [r16]=r18,PT(R12)-PT(LOADRS) // save ar.rsc value for "loadrs" 998 st8.spill [r17]=r20,PT(R13)-PT(R1) // save original r1 999(p12) mov in4=-1 1000 ;;
1001 1002.mem.offset 0,0; st8.spill [r16]=r12,PT(AR_FPSR)-PT(R12) // save r12 1003.mem.offset 8,0; st8.spill [r17]=r13,PT(R15)-PT(R13) // save r13 1004(p13) mov in5=-1 1005 ;; 1006 st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr 1007 tnat.nz p13,p0=in6 1008 cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8 1009 ;; 1010 mov r8=1 1011(p9) tnat.nz p10,p0=r15 1012 adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch) 1013 1014 st8.spill [r17]=r15 // save r15 1015 tnat.nz p8,p0=in7 1016 nop.i 0 1017 1018 mov r13=r2 // establish `current' 1019 movl r1=__gp // establish kernel global pointer 1020 ;; 1021 st8 [r16]=r8 // ensure pt_regs.r8 != 0 (see handle_syscall_error) 1022(p13) mov in6=-1 1023(p8) mov in7=-1 1024 1025 cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0 1026 movl r17=FPSR_DEFAULT 1027 ;; 1028 mov.m ar.fpsr=r17 // set ar.fpsr to kernel default value 1029(p10) mov r8=-EINVAL 1030 br.ret.sptk.many b7 1031END(ia64_syscall_setup) 1032#endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */ 1033 1034 .org ia64_ivt+0x3c00 1035///////////////////////////////////////////////////////////////////////////////////////// 1036// 0x3c00 Entry 15 (size 64 bundles) Reserved 1037 DBG_FAULT(15) 1038 FAULT(15) 1039 1040 .org ia64_ivt+0x4000 1041///////////////////////////////////////////////////////////////////////////////////////// 1042// 0x4000 Entry 16 (size 64 bundles) Reserved 1043 DBG_FAULT(16) 1044 FAULT(16) 1045 1046#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(__IA64_ASM_PARAVIRTUALIZED_NATIVE) 1047 /* 1048 * There is no particular reason for this code to be here, other than 1049 * that there happens to be space here that would go unused otherwise. 1050 * If this fault ever gets "unreserved", simply moved the following 1051 * code to a more suitable spot... 1052 * 1053 * account_sys_enter is called from SAVE_MIN* macros if accounting is 1054 * enabled and if the macro is entered from user mode. 1055 */ 1056GLOBAL_ENTRY(account_sys_enter) 1057 // mov.m r20=ar.itc is called in advance, and r13 is current 1058 add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13 1059 add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13 1060 ;; 1061 ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel 1062 ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at left from kernel 1063 ;; 1064 ld8 r23=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime 1065 ld8 r21=[r17] // cumulated utime 1066 sub r22=r19,r18 // stime before leave kernel 1067 ;; 1068 st8 [r16]=r20,TI_AC_STIME-TI_AC_STAMP // update stamp 1069 sub r18=r20,r19 // elapsed time in user mode 1070 ;; 1071 add r23=r23,r22 // sum stime 1072 add r21=r21,r18 // sum utime 1073 ;; 1074 st8 [r16]=r23 // update stime 1075 st8 [r17]=r21 // update utime 1076 ;; 1077 br.ret.sptk.many rp 1078END(account_sys_enter) 1079#endif 1080 1081 .org ia64_ivt+0x4400 1082///////////////////////////////////////////////////////////////////////////////////////// 1083// 0x4400 Entry 17 (size 64 bundles) Reserved 1084 DBG_FAULT(17) 1085 FAULT(17) 1086 1087 .org ia64_ivt+0x4800 1088///////////////////////////////////////////////////////////////////////////////////////// 1089// 0x4800 Entry 18 (size 64 bundles) Reserved 1090 DBG_FAULT(18) 1091 FAULT(18) 1092 1093 .org ia64_ivt+0x4c00 1094///////////////////////////////////////////////////////////////////////////////////////// 1095// 0x4c00 Entry 19 (size 64 bundles) Reserved 1096 DBG_FAULT(19) 1097 FAULT(19) 1098 1099// 1100// --- End of long entries, Beginning of short entries 1101// 1102 1103 .org ia64_ivt+0x5000 1104///////////////////////////////////////////////////////////////////////////////////////// 1105// 0x5000 Entry 20 (size 16 bundles) Page Not Present (10,22,49) 1106ENTRY(page_not_present) 1107 DBG_FAULT(20) 1108 MOV_FROM_IFA(r16) 1109 RSM_PSR_DT 1110 /* 1111 * The Linux page fault handler doesn't expect non-present pages to be in 1112 * the TLB. Flush the existing entry now, so we meet that expectation. 1113 */ 1114 mov r17=PAGE_SHIFT<<2 1115 ;; 1116 ptc.l r16,r17 1117 ;; 1118 mov r31=pr 1119 srlz.d 1120 br.sptk.many page_fault 1121END(page_not_present) 1122 1123 .org ia64_ivt+0x5100 1124///////////////////////////////////////////////////////////////////////////////////////// 1125// 0x5100 Entry 21 (size 16 bundles) Key Permission (13,25,52) 1126ENTRY(key_permission) 1127 DBG_FAULT(21) 1128 MOV_FROM_IFA(r16) 1129 RSM_PSR_DT 1130 mov r31=pr 1131 ;; 1132 srlz.d 1133 br.sptk.many page_fault 1134END(key_permission) 1135 1136 .org ia64_ivt+0x5200 1137///////////////////////////////////////////////////////////////////////////////////////// 1138// 0x5200 Entry 22 (size 16 bundles) Instruction Access Rights (26) 1139ENTRY(iaccess_rights) 1140 DBG_FAULT(22) 1141 MOV_FROM_IFA(r16) 1142 RSM_PSR_DT 1143 mov r31=pr 1144 ;; 1145 srlz.d 1146 br.sptk.many page_fault 1147END(iaccess_rights) 1148 1149 .org ia64_ivt+0x5300 1150///////////////////////////////////////////////////////////////////////////////////////// 1151// 0x5300 Entry 23 (size 16 bundles) Data Access Rights (14,53) 1152ENTRY(daccess_rights) 1153 DBG_FAULT(23) 1154 MOV_FROM_IFA(r16) 1155 RSM_PSR_DT 1156 mov r31=pr 1157 ;; 1158 srlz.d 1159 br.sptk.many page_fault 1160END(daccess_rights) 1161 1162 .org ia64_ivt+0x5400 1163///////////////////////////////////////////////////////////////////////////////////////// 1164// 0x5400 Entry 24 (size 16 bundles) General Exception (5,32,34,36,38,39) 1165ENTRY(general_exception) 1166 DBG_FAULT(24) 1167 MOV_FROM_ISR(r16) 1168 mov r31=pr 1169 ;; 1170 cmp4.eq p6,p0=0,r16 1171(p6) br.sptk.many dispatch_illegal_op_fault 1172 ;; 1173 mov r19=24 // fault number 1174 br.sptk.many dispatch_to_fault_handler 1175END(general_exception) 1176 1177 .org ia64_ivt+0x5500 1178///////////////////////////////////////////////////////////////////////////////////////// 1179// 0x5500 Entry 25 (size 16 bundles) Disabled FP-Register (35) 1180ENTRY(disabled_fp_reg) 1181 DBG_FAULT(25) 1182 rsm psr.dfh // ensure we can access fph 1183 ;; 1184 srlz.d 1185 mov r31=pr 1186 mov r19=25 1187 br.sptk.many dispatch_to_fault_handler 1188END(disabled_fp_reg) 1189 1190 .org ia64_ivt+0x5600 1191///////////////////////////////////////////////////////////////////////////////////////// 1192// 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50) 1193ENTRY(nat_consumption) 1194 DBG_FAULT(26) 1195 1196 MOV_FROM_IPSR(p0, r16) 1197 MOV_FROM_ISR(r17) 1198 mov r31=pr // save PR 1199 ;; 1200 and r18=0xf,r17 // r18 = cr.ipsr.code{3:0} 1201 tbit.z p6,p0=r17,IA64_ISR_NA_BIT 1202 ;; 1203 cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18 1204 dep r16=-1,r16,IA64_PSR_ED_BIT,1 1205(p6) br.cond.spnt 1f // branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH) 1206 ;; 1207 MOV_TO_IPSR(p0, r16, r18) 1208 mov pr=r31,-1 1209 ;; 1210 RFI 1211 12121: mov pr=r31,-1 1213 ;; 1214 FAULT(26) 1215END(nat_consumption) 1216 1217 .org ia64_ivt+0x5700 1218///////////////////////////////////////////////////////////////////////////////////////// 1219// 0x5700 Entry 27 (size 16 bundles) Speculation (40) 1220ENTRY(speculation_vector) 1221 DBG_FAULT(27) 1222 /* 1223 * A [f]chk.[as] instruction needs to take the branch to the recovery code but 1224 * this part of the architecture is not implemented in hardware on some CPUs, such 1225 * as Itanium. Thus, in general we need to emulate the behavior. IIM contains 1226 * the relative target (not yet sign extended). So after sign extending it we 1227 * simply add it to IIP. We also need to reset the EI field of the IPSR to zero, 1228 * i.e., the slot to restart into. 1229 * 1230 * cr.imm contains zero_ext(imm21) 1231 */ 1232 MOV_FROM_IIM(r18) 1233 ;; 1234 MOV_FROM_IIP(r17) 1235 shl r18=r18,43 // put sign bit in position (43=64-21) 1236 ;; 1237 1238 MOV_FROM_IPSR(p0, r16) 1239 shr r18=r18,39 // sign extend (39=43-4) 1240 ;; 1241 1242 add r17=r17,r18 // now add the offset 1243 ;; 1244 MOV_TO_IIP(r17, r19) 1245 dep r16=0,r16,41,2 // clear EI 1246 ;; 1247 1248 MOV_TO_IPSR(p0, r16, r19) 1249 ;; 1250 1251 RFI 1252END(speculation_vector) 1253 1254 .org ia64_ivt+0x5800 1255///////////////////////////////////////////////////////////////////////////////////////// 1256// 0x5800 Entry 28 (size 16 bundles) Reserved 1257 DBG_FAULT(28) 1258 FAULT(28) 1259 1260 .org ia64_ivt+0x5900 1261///////////////////////////////////////////////////////////////////////////////////////// 1262// 0x5900 Entry 29 (size 16 bundles) Debug (16,28,56) 1263ENTRY(debug_vector) 1264 DBG_FAULT(29) 1265 FAULT(29) 1266END(debug_vector) 1267 1268 .org ia64_ivt+0x5a00 1269///////////////////////////////////////////////////////////////////////////////////////// 1270// 0x5a00 Entry 30 (size 16 bundles) Unaligned Reference (57) 1271ENTRY(unaligned_access) 1272 DBG_FAULT(30) 1273 mov r31=pr // prepare to save predicates 1274 ;; 1275 br.sptk.many dispatch_unaligned_handler 1276END(unaligned_access) 1277 1278 .org ia64_ivt+0x5b00 1279///////////////////////////////////////////////////////////////////////////////////////// 1280// 0x5b00 Entry 31 (size 16 bundles) Unsupported Data Reference (57) 1281ENTRY(unsupported_data_reference) 1282 DBG_FAULT(31) 1283 FAULT(31) 1284END(unsupported_data_reference) 1285 1286 .org ia64_ivt+0x5c00 1287///////////////////////////////////////////////////////////////////////////////////////// 1288// 0x5c00 Entry 32 (size 16 bundles) Floating-Point Fault (64) 1289ENTRY(floating_point_fault) 1290 DBG_FAULT(32) 1291 FAULT(32) 1292END(floating_point_fault) 1293 1294 .org ia64_ivt+0x5d00 1295///////////////////////////////////////////////////////////////////////////////////////// 1296// 0x5d00 Entry 33 (size 16 bundles) Floating Point Trap (66) 1297ENTRY(floating_point_trap) 1298 DBG_FAULT(33) 1299 FAULT(33) 1300END(floating_point_trap) 1301 1302 .org ia64_ivt+0x5e00 1303///////////////////////////////////////////////////////////////////////////////////////// 1304// 0x5e00 Entry 34 (size 16 bundles) Lower Privilege Transfer Trap (66) 1305ENTRY(lower_privilege_trap) 1306 DBG_FAULT(34) 1307 FAULT(34) 1308END(lower_privilege_trap) 1309 1310 .org ia64_ivt+0x5f00 1311///////////////////////////////////////////////////////////////////////////////////////// 1312// 0x5f00 Entry 35 (size 16 bundles) Taken Branch Trap (68) 1313ENTRY(taken_branch_trap) 1314 DBG_FAULT(35) 1315 FAULT(35) 1316END(taken_branch_trap) 1317 1318 .org ia64_ivt+0x6000 1319///////////////////////////////////////////////////////////////////////////////////////// 1320// 0x6000 Entry 36 (size 16 bundles) Single Step Trap (69) 1321ENTRY(single_step_trap) 1322 DBG_FAULT(36) 1323 FAULT(36) 1324END(single_step_trap) 1325 1326 .org ia64_ivt+0x6100 1327///////////////////////////////////////////////////////////////////////////////////////// 1328// 0x6100 Entry 37 (size 16 bundles) Reserved 1329 DBG_FAULT(37) 1330 FAULT(37) 1331 1332 .org ia64_ivt+0x6200 1333///////////////////////////////////////////////////////////////////////////////////////// 1334// 0x6200 Entry 38 (size 16 bundles) Reserved 1335 DBG_FAULT(38) 1336 FAULT(38) 1337 1338 .org ia64_ivt+0x6300 1339///////////////////////////////////////////////////////////////////////////////////////// 1340// 0x6300 Entry 39 (size 16 bundles) Reserved 1341 DBG_FAULT(39) 1342 FAULT(39) 1343 1344 .org ia64_ivt+0x6400 1345///////////////////////////////////////////////////////////////////////////////////////// 1346// 0x6400 Entry 40 (size 16 bundles) Reserved 1347 DBG_FAULT(40) 1348 FAULT(40) 1349 1350 .org ia64_ivt+0x6500 1351///////////////////////////////////////////////////////////////////////////////////////// 1352// 0x6500 Entry 41 (size 16 bundles) Reserved 1353 DBG_FAULT(41) 1354 FAULT(41) 1355 1356 .org ia64_ivt+0x6600 1357///////////////////////////////////////////////////////////////////////////////////////// 1358// 0x6600 Entry 42 (size 16 bundles) Reserved 1359 DBG_FAULT(42) 1360 FAULT(42) 1361 1362 .org ia64_ivt+0x6700 1363///////////////////////////////////////////////////////////////////////////////////////// 1364// 0x6700 Entry 43 (size 16 bundles) Reserved 1365 DBG_FAULT(43) 1366 FAULT(43) 1367 1368 .org ia64_ivt+0x6800 1369///////////////////////////////////////////////////////////////////////////////////////// 1370// 0x6800 Entry 44 (size 16 bundles) Reserved 1371 DBG_FAULT(44) 1372 FAULT(44) 1373 1374 .org ia64_ivt+0x6900 1375///////////////////////////////////////////////////////////////////////////////////////// 1376// 0x6900 Entry 45 (size 16 bundles) IA-32 Exeception (17,18,29,41,42,43,44,58,60,61,62,72,73,75,76,77) 1377ENTRY(ia32_exception) 1378 DBG_FAULT(45) 1379 FAULT(45) 1380END(ia32_exception) 1381 1382 .org ia64_ivt+0x6a00 1383///////////////////////////////////////////////////////////////////////////////////////// 1384// 0x6a00 Entry 46 (size 16 bundles) IA-32 Intercept (30,31,59,70,71) 1385ENTRY(ia32_intercept) 1386 DBG_FAULT(46) 1387 FAULT(46) 1388END(ia32_intercept) 1389 1390 .org ia64_ivt+0x6b00 1391///////////////////////////////////////////////////////////////////////////////////////// 1392// 0x6b00 Entry 47 (size 16 bundles) IA-32 Interrupt (74) 1393ENTRY(ia32_interrupt) 1394 DBG_FAULT(47) 1395 FAULT(47) 1396END(ia32_interrupt) 1397 1398 .org ia64_ivt+0x6c00 1399///////////////////////////////////////////////////////////////////////////////////////// 1400// 0x6c00 Entry 48 (size 16 bundles) Reserved 1401 DBG_FAULT(48) 1402 FAULT(48) 1403 1404 .org ia64_ivt+0x6d00 1405///////////////////////////////////////////////////////////////////////////////////////// 1406// 0x6d00 Entry 49 (size 16 bundles) Reserved 1407 DBG_FAULT(49) 1408 FAULT(49) 1409 1410 .org ia64_ivt+0x6e00 1411///////////////////////////////////////////////////////////////////////////////////////// 1412// 0x6e00 Entry 50 (size 16 bundles) Reserved 1413 DBG_FAULT(50) 1414 FAULT(50) 1415 1416 .org ia64_ivt+0x6f00 1417///////////////////////////////////////////////////////////////////////////////////////// 1418// 0x6f00 Entry 51 (size 16 bundles) Reserved 1419 DBG_FAULT(51) 1420 FAULT(51) 1421 1422 .org ia64_ivt+0x7000 1423///////////////////////////////////////////////////////////////////////////////////////// 1424// 0x7000 Entry 52 (size 16 bundles) Reserved 1425 DBG_FAULT(52) 1426 FAULT(52) 1427 1428 .org ia64_ivt+0x7100 1429///////////////////////////////////////////////////////////////////////////////////////// 1430// 0x7100 Entry 53 (size 16 bundles) Reserved 1431 DBG_FAULT(53) 1432 FAULT(53) 1433 1434 .org ia64_ivt+0x7200 1435///////////////////////////////////////////////////////////////////////////////////////// 1436// 0x7200 Entry 54 (size 16 bundles) Reserved 1437 DBG_FAULT(54) 1438 FAULT(54) 1439 1440 .org ia64_ivt+0x7300 1441///////////////////////////////////////////////////////////////////////////////////////// 1442// 0x7300 Entry 55 (size 16 bundles) Reserved 1443 DBG_FAULT(55) 1444 FAULT(55) 1445 1446 .org ia64_ivt+0x7400 1447///////////////////////////////////////////////////////////////////////////////////////// 1448// 0x7400 Entry 56 (size 16 bundles) Reserved 1449 DBG_FAULT(56) 1450 FAULT(56) 1451 1452 .org ia64_ivt+0x7500 1453///////////////////////////////////////////////////////////////////////////////////////// 1454// 0x7500 Entry 57 (size 16 bundles) Reserved 1455 DBG_FAULT(57) 1456 FAULT(57) 1457 1458 .org ia64_ivt+0x7600 1459///////////////////////////////////////////////////////////////////////////////////////// 1460// 0x7600 Entry 58 (size 16 bundles) Reserved 1461 DBG_FAULT(58) 1462 FAULT(58) 1463 1464 .org ia64_ivt+0x7700 1465///////////////////////////////////////////////////////////////////////////////////////// 1466// 0x7700 Entry 59 (size 16 bundles) Reserved 1467 DBG_FAULT(59) 1468 FAULT(59) 1469 1470 .org ia64_ivt+0x7800 1471///////////////////////////////////////////////////////////////////////////////////////// 1472// 0x7800 Entry 60 (size 16 bundles) Reserved 1473 DBG_FAULT(60) 1474 FAULT(60) 1475 1476 .org ia64_ivt+0x7900 1477///////////////////////////////////////////////////////////////////////////////////////// 1478// 0x7900 Entry 61 (size 16 bundles) Reserved 1479 DBG_FAULT(61) 1480 FAULT(61) 1481 1482 .org ia64_ivt+0x7a00 1483///////////////////////////////////////////////////////////////////////////////////////// 1484// 0x7a00 Entry 62 (size 16 bundles) Reserved 1485 DBG_FAULT(62) 1486 FAULT(62) 1487 1488 .org ia64_ivt+0x7b00 1489///////////////////////////////////////////////////////////////////////////////////////// 1490// 0x7b00 Entry 63 (size 16 bundles) Reserved 1491 DBG_FAULT(63) 1492 FAULT(63) 1493 1494 .org ia64_ivt+0x7c00 1495///////////////////////////////////////////////////////////////////////////////////////// 1496// 0x7c00 Entry 64 (size 16 bundles) Reserved 1497 DBG_FAULT(64) 1498 FAULT(64) 1499 1500 .org ia64_ivt+0x7d00 1501///////////////////////////////////////////////////////////////////////////////////////// 1502// 0x7d00 Entry 65 (size 16 bundles) Reserved 1503 DBG_FAULT(65) 1504 FAULT(65) 1505 1506 .org ia64_ivt+0x7e00 1507///////////////////////////////////////////////////////////////////////////////////////// 1508// 0x7e00 Entry 66 (size 16 bundles) Reserved 1509 DBG_FAULT(66) 1510 FAULT(66) 1511 1512 .org ia64_ivt+0x7f00 1513///////////////////////////////////////////////////////////////////////////////////////// 1514// 0x7f00 Entry 67 (size 16 bundles) Reserved 1515 DBG_FAULT(67) 1516 FAULT(67) 1517 1518 //----------------------------------------------------------------------------------- 1519 // call do_page_fault (predicates are in r31, psr.dt may be off, r16 is faulting address) 1520ENTRY(page_fault) 1521 SSM_PSR_DT_AND_SRLZ_I 1522 ;; 1523 SAVE_MIN_WITH_COVER 1524 alloc r15=ar.pfs,0,0,3,0 1525 MOV_FROM_IFA(out0) 1526 MOV_FROM_ISR(out1) 1527 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r14, r3) 1528 adds r3=8,r2 // set up second base pointer 1529 SSM_PSR_I(p15, p15, r14) // restore psr.i 1530 movl r14=ia64_leave_kernel 1531 ;; 1532 SAVE_REST 1533 mov rp=r14 1534 ;; 1535 adds out2=16,r12 // out2 = pointer to pt_regs 1536 br.call.sptk.many b6=ia64_do_page_fault // ignore return address 1537END(page_fault) 1538 1539ENTRY(non_syscall) 1540 mov ar.rsc=r27 // restore ar.rsc before SAVE_MIN_WITH_COVER 1541 ;; 1542 SAVE_MIN_WITH_COVER 1543 1544 // There is no particular reason for this code to be here, other than that 1545 // there happens to be space here that would go unused otherwise. If this 1546 // fault ever gets "unreserved", simply moved the following code to a more 1547 // suitable spot... 1548 1549 alloc r14=ar.pfs,0,0,2,0 1550 MOV_FROM_IIM(out0) 1551 add out1=16,sp 1552 adds r3=8,r2 // set up second base pointer for SAVE_REST 1553 1554 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r15, r24) 1555 // guarantee that interruption collection is on 1556 SSM_PSR_I(p15, p15, r15) // restore psr.i 1557 movl r15=ia64_leave_kernel 1558 ;; 1559 SAVE_REST 1560 mov rp=r15 1561 ;; 1562 br.call.sptk.many b6=ia64_bad_break // avoid WAW on CFM and ignore return addr 1563END(non_syscall) 1564 1565ENTRY(__interrupt) 1566 DBG_FAULT(12) 1567 mov r31=pr // prepare to save predicates 1568 ;; 1569 SAVE_MIN_WITH_COVER // uses r31; defines r2 and r3 1570 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r14) 1571 // ensure everybody knows psr.ic is back on 1572 adds r3=8,r2 // set up second base pointer for SAVE_REST 1573 ;; 1574 SAVE_REST 1575 ;; 1576 MCA_RECOVER_RANGE(interrupt) 1577 alloc r14=ar.pfs,0,0,2,0 // must be first in an insn group 1578 MOV_FROM_IVR(out0, r8) // pass cr.ivr as first arg 1579 add out1=16,sp // pass pointer to pt_regs as second arg 1580 ;; 1581 srlz.d // make sure we see the effect of cr.ivr 1582 movl r14=ia64_leave_kernel 1583 ;; 1584 mov rp=r14 1585 br.call.sptk.many b6=ia64_handle_irq 1586END(__interrupt) 1587 1588 /* 1589 * There is no particular reason for this code to be here, other than that 1590 * there happens to be space here that would go unused otherwise. If this 1591 * fault ever gets "unreserved", simply moved the following code to a more 1592 * suitable spot... 1593 */ 1594 1595ENTRY(dispatch_unaligned_handler) 1596 SAVE_MIN_WITH_COVER 1597 ;; 1598 alloc r14=ar.pfs,0,0,2,0 // now it's safe (must be first in insn group!) 1599 MOV_FROM_IFA(out0) 1600 adds out1=16,sp 1601 1602 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r24) 1603 // guarantee that interruption collection is on 1604 SSM_PSR_I(p15, p15, r3) // restore psr.i 1605 adds r3=8,r2 // set up second base pointer 1606 ;; 1607 SAVE_REST 1608 movl r14=ia64_leave_kernel 1609 ;; 1610 mov rp=r14 1611 br.sptk.many ia64_prepare_handle_unaligned 1612END(dispatch_unaligned_handler) 1613 1614 /* 1615 * There is no particular reason for this code to be here, other than that 1616 * there happens to be space here that would go unused otherwise. If this 1617 * fault ever gets "unreserved", simply moved the following code to a more 1618 * suitable spot... 1619 */ 1620 1621ENTRY(dispatch_to_fault_handler) 1622 /* 1623 * Input: 1624 * psr.ic: off 1625 * r19: fault vector number (e.g., 24 for General Exception) 1626 * r31: contains saved predicates (pr) 1627 */ 1628 SAVE_MIN_WITH_COVER_R19 1629 alloc r14=ar.pfs,0,0,5,0 1630 MOV_FROM_ISR(out1) 1631 MOV_FROM_IFA(out2) 1632 MOV_FROM_IIM(out3) 1633 MOV_FROM_ITIR(out4) 1634 ;; 1635 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, out0) 1636 // guarantee that interruption collection is on 1637 mov out0=r15 1638 ;; 1639 SSM_PSR_I(p15, p15, r3) // restore psr.i 1640 adds r3=8,r2 // set up second base pointer for SAVE_REST 1641 ;; 1642 SAVE_REST 1643 movl r14=ia64_leave_kernel 1644 ;; 1645 mov rp=r14 1646 br.call.sptk.many b6=ia64_fault 1647END(dispatch_to_fault_handler) 1648 1649 /* 1650 * Squatting in this space ... 1651 * 1652 * This special case dispatcher for illegal operation faults allows preserved 1653 * registers to be modified through a callback function (asm only) that is handed 1654 * back from the fault handler in r8. Up to three arguments can be passed to the 1655 * callback function by returning an aggregate with the callback as its first 1656 * element, followed by the arguments. 1657 */ 1658ENTRY(dispatch_illegal_op_fault) 1659 .prologue 1660 .body 1661 SAVE_MIN_WITH_COVER 1662 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r24) 1663 // guarantee that interruption collection is on 1664 ;; 1665 SSM_PSR_I(p15, p15, r3) // restore psr.i 1666 adds r3=8,r2 // set up second base pointer for SAVE_REST 1667 ;; 1668 alloc r14=ar.pfs,0,0,1,0 // must be first in insn group 1669 mov out0=ar.ec 1670 ;; 1671 SAVE_REST 1672 PT_REGS_UNWIND_INFO(0) 1673 ;; 1674 br.call.sptk.many rp=ia64_illegal_op_fault 1675.ret0: ;; 1676 alloc r14=ar.pfs,0,0,3,0 // must be first in insn group 1677 mov out0=r9 1678 mov out1=r10 1679 mov out2=r11 1680 movl r15=ia64_leave_kernel 1681 ;; 1682 mov rp=r15 1683 mov b6=r8 1684 ;; 1685 cmp.ne p6,p0=0,r8 1686(p6) br.call.dpnt.many b6=b6 // call returns to ia64_leave_kernel 1687 br.sptk.many ia64_leave_kernel 1688END(dispatch_illegal_op_fault) 1689