LXR linux/arch/sparc/include/asm/pgtable

   1/*
   2 * pgtable.h: SpitFire page table operations.
   3 *
   4 * Copyright 1996,1997 David S. Miller (davem@caip.rutgers.edu)
   5 * Copyright 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   6 */
   7
   8#ifndef _SPARC64_PGTABLE_H
   9#define _SPARC64_PGTABLE_H
  10
  11/* This file contains the functions and defines necessary to modify and use
  12 * the SpitFire page tables.
  13 */
  14
  15#include <linux/compiler.h>
  16#include <linux/const.h>
  17#include <asm/types.h>
  18#include <asm/spitfire.h>
  19#include <asm/asi.h>
  20#include <asm/page.h>
  21#include <asm/processor.h>
  22
  23/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).
  24 * The page copy blockops can use 0x6000000 to 0x8000000.
  25 * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range.
  26 * The 4M TSB is mapped in the 0x8400000 to 0x8800000 range.
  27 * The PROM resides in an area spanning 0xf0000000 to 0x100000000.
  28 * The vmalloc area spans 0x100000000 to 0x200000000.
  29 * Since modules need to be in the lowest 32-bits of the address space,
  30 * we place them right before the OBP area from 0x10000000 to 0xf0000000.
  31 * There is a single static kernel PMD which maps from 0x0 to address
  32 * 0x400000000.
  33 */
  34#define TLBTEMP_BASE            _AC(0x0000000006000000,UL)
  35#define TSBMAP_8K_BASE          _AC(0x0000000008000000,UL)
  36#define TSBMAP_4M_BASE          _AC(0x0000000008400000,UL)
  37#define MODULES_VADDR           _AC(0x0000000010000000,UL)
  38#define MODULES_LEN             _AC(0x00000000e0000000,UL)
  39#define MODULES_END             _AC(0x00000000f0000000,UL)
  40#define LOW_OBP_ADDRESS         _AC(0x00000000f0000000,UL)
  41#define HI_OBP_ADDRESS          _AC(0x0000000100000000,UL)
  42#define VMALLOC_START           _AC(0x0000000100000000,UL)
  43#define VMEMMAP_BASE            VMALLOC_END
  44
  45/* PMD_SHIFT determines the size of the area a second-level page
  46 * table can map
  47 */
  48#define PMD_SHIFT       (PAGE_SHIFT + (PAGE_SHIFT-3))
  49#define PMD_SIZE        (_AC(1,UL) << PMD_SHIFT)
  50#define PMD_MASK        (~(PMD_SIZE-1))
  51#define PMD_BITS        (PAGE_SHIFT - 3)
  52
  53/* PUD_SHIFT determines the size of the area a third-level page
  54 * table can map
  55 */
  56#define PUD_SHIFT       (PMD_SHIFT + PMD_BITS)
  57#define PUD_SIZE        (_AC(1,UL) << PUD_SHIFT)
  58#define PUD_MASK        (~(PUD_SIZE-1))
  59#define PUD_BITS        (PAGE_SHIFT - 3)
  60
  61/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
  62#define PGDIR_SHIFT     (PUD_SHIFT + PUD_BITS)
  63#define PGDIR_SIZE      (_AC(1,UL) << PGDIR_SHIFT)
  64#define PGDIR_MASK      (~(PGDIR_SIZE-1))
  65#define PGDIR_BITS      (PAGE_SHIFT - 3)
  66
  67#if (MAX_PHYS_ADDRESS_BITS > PGDIR_SHIFT + PGDIR_BITS)
  68#error MAX_PHYS_ADDRESS_BITS exceeds what kernel page tables can support
  69#endif
  70
  71#if (PGDIR_SHIFT + PGDIR_BITS) != 53
  72#error Page table parameters do not cover virtual address space properly.
  73#endif
  74
  75#if (PMD_SHIFT != HPAGE_SHIFT)
  76#error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages.
  77#endif
  78
  79#ifndef __ASSEMBLY__
  80
  81extern unsigned long VMALLOC_END;
  82
  83#define vmemmap                 ((struct page *)VMEMMAP_BASE)
  84
  85#include <linux/sched.h>
  86
  87bool kern_addr_valid(unsigned long addr);
  88
  89/* Entries per page directory level. */
  90#define PTRS_PER_PTE    (1UL << (PAGE_SHIFT-3))
  91#define PTRS_PER_PMD    (1UL << PMD_BITS)
  92#define PTRS_PER_PUD    (1UL << PUD_BITS)
  93#define PTRS_PER_PGD    (1UL << PGDIR_BITS)
  94
  95/* Kernel has a separate 44bit address space. */
  96#define FIRST_USER_ADDRESS      0UL
  97
  98#define pmd_ERROR(e)                                                    \
  99        pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n",             \
 100               __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0))
 101#define pud_ERROR(e)                                                    \
 102        pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n",             \
 103               __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))
 104#define pgd_ERROR(e)                                                    \
 105        pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n",             \
 106               __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
 107
 108#endif /* !(__ASSEMBLY__) */
 109
 110/* PTE bits which are the same in SUN4U and SUN4V format.  */
 111#define _PAGE_VALID       _AC(0x8000000000000000,UL) /* Valid TTE            */
 112#define _PAGE_R           _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
 113#define _PAGE_SPECIAL     _AC(0x0200000000000000,UL) /* Special page         */
 114#define _PAGE_PMD_HUGE    _AC(0x0100000000000000,UL) /* Huge page            */
 115#define _PAGE_PUD_HUGE    _PAGE_PMD_HUGE
 116
 117/* Advertise support for _PAGE_SPECIAL */
 118#define __HAVE_ARCH_PTE_SPECIAL
 119
 120/* SUN4U pte bits... */
 121#define _PAGE_SZ4MB_4U    _AC(0x6000000000000000,UL) /* 4MB Page             */
 122#define _PAGE_SZ512K_4U   _AC(0x4000000000000000,UL) /* 512K Page            */
 123#define _PAGE_SZ64K_4U    _AC(0x2000000000000000,UL) /* 64K Page             */
 124#define _PAGE_SZ8K_4U     _AC(0x0000000000000000,UL) /* 8K Page              */
 125#define _PAGE_NFO_4U      _AC(0x1000000000000000,UL) /* No Fault Only        */
 126#define _PAGE_IE_4U       _AC(0x0800000000000000,UL) /* Invert Endianness    */
 127#define _PAGE_SOFT2_4U    _AC(0x07FC000000000000,UL) /* Software bits, set 2 */
 128#define _PAGE_SPECIAL_4U  _AC(0x0200000000000000,UL) /* Special page         */
 129#define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page            */
 130#define _PAGE_RES1_4U     _AC(0x0002000000000000,UL) /* Reserved             */
 131#define _PAGE_SZ32MB_4U   _AC(0x0001000000000000,UL) /* (Panther) 32MB page  */
 132#define _PAGE_SZ256MB_4U  _AC(0x2001000000000000,UL) /* (Panther) 256MB page */
 133#define _PAGE_SZALL_4U    _AC(0x6001000000000000,UL) /* All pgsz bits        */
 134#define _PAGE_SN_4U       _AC(0x0000800000000000,UL) /* (Cheetah) Snoop      */
 135#define _PAGE_RES2_4U     _AC(0x0000780000000000,UL) /* Reserved             */
 136#define _PAGE_PADDR_4U    _AC(0x000007FFFFFFE000,UL) /* (Cheetah) pa[42:13]  */
 137#define _PAGE_SOFT_4U     _AC(0x0000000000001F80,UL) /* Software bits:       */
 138#define _PAGE_EXEC_4U     _AC(0x0000000000001000,UL) /* Executable SW bit    */
 139#define _PAGE_MODIFIED_4U _AC(0x0000000000000800,UL) /* Modified (dirty)     */
 140#define _PAGE_ACCESSED_4U _AC(0x0000000000000400,UL) /* Accessed (ref'd)     */
 141#define _PAGE_READ_4U     _AC(0x0000000000000200,UL) /* Readable SW Bit      */
 142#define _PAGE_WRITE_4U    _AC(0x0000000000000100,UL) /* Writable SW Bit      */
 143#define _PAGE_PRESENT_4U  _AC(0x0000000000000080,UL) /* Present              */
 144#define _PAGE_L_4U        _AC(0x0000000000000040,UL) /* Locked TTE           */
 145#define _PAGE_CP_4U       _AC(0x0000000000000020,UL) /* Cacheable in P-Cache */
 146#define _PAGE_CV_4U       _AC(0x0000000000000010,UL) /* Cacheable in V-Cache */
 147#define _PAGE_E_4U        _AC(0x0000000000000008,UL) /* side-Effect          */
 148#define _PAGE_P_4U        _AC(0x0000000000000004,UL) /* Privileged Page      */
 149#define _PAGE_W_4U        _AC(0x0000000000000002,UL) /* Writable             */
 150
 151/* SUN4V pte bits... */
 152#define _PAGE_NFO_4V      _AC(0x4000000000000000,UL) /* No Fault Only        */
 153#define _PAGE_SOFT2_4V    _AC(0x3F00000000000000,UL) /* Software bits, set 2 */
 154#define _PAGE_MODIFIED_4V _AC(0x2000000000000000,UL) /* Modified (dirty)     */
 155#define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd)     */
 156#define _PAGE_READ_4V     _AC(0x0800000000000000,UL) /* Readable SW Bit      */
 157#define _PAGE_WRITE_4V    _AC(0x0400000000000000,UL) /* Writable SW Bit      */
 158#define _PAGE_SPECIAL_4V  _AC(0x0200000000000000,UL) /* Special page         */
 159#define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page            */
 160#define _PAGE_PADDR_4V    _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13]         */
 161#define _PAGE_IE_4V       _AC(0x0000000000001000,UL) /* Invert Endianness    */
 162#define _PAGE_E_4V        _AC(0x0000000000000800,UL) /* side-Effect          */
 163#define _PAGE_CP_4V       _AC(0x0000000000000400,UL) /* Cacheable in P-Cache */
 164#define _PAGE_CV_4V       _AC(0x0000000000000200,UL) /* Cacheable in V-Cache */
 165#define _PAGE_P_4V        _AC(0x0000000000000100,UL) /* Privileged Page      */
 166#define _PAGE_EXEC_4V     _AC(0x0000000000000080,UL) /* Executable Page      */
 167#define _PAGE_W_4V        _AC(0x0000000000000040,UL) /* Writable             */
 168#define _PAGE_SOFT_4V     _AC(0x0000000000000030,UL) /* Software bits        */
 169#define _PAGE_PRESENT_4V  _AC(0x0000000000000010,UL) /* Present              */
 170#define _PAGE_RESV_4V     _AC(0x0000000000000008,UL) /* Reserved             */
 171#define _PAGE_SZ16GB_4V   _AC(0x0000000000000007,UL) /* 16GB Page            */
 172#define _PAGE_SZ2GB_4V    _AC(0x0000000000000006,UL) /* 2GB Page             */
 173#define _PAGE_SZ256MB_4V  _AC(0x0000000000000005,UL) /* 256MB Page           */
 174#define _PAGE_SZ32MB_4V   _AC(0x0000000000000004,UL) /* 32MB Page            */
 175#define _PAGE_SZ4MB_4V    _AC(0x0000000000000003,UL) /* 4MB Page             */
 176#define _PAGE_SZ512K_4V   _AC(0x0000000000000002,UL) /* 512K Page            */
 177#define _PAGE_SZ64K_4V    _AC(0x0000000000000001,UL) /* 64K Page             */
 178#define _PAGE_SZ8K_4V     _AC(0x0000000000000000,UL) /* 8K Page              */
 179#define _PAGE_SZALL_4V    _AC(0x0000000000000007,UL) /* All pgsz bits        */
 180
 181#define _PAGE_SZBITS_4U _PAGE_SZ8K_4U
 182#define _PAGE_SZBITS_4V _PAGE_SZ8K_4V
 183
 184#if REAL_HPAGE_SHIFT != 22
 185#error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up
 186#endif
 187
 188#define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U
 189#define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V
 190
 191/* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */
 192#define __P000  __pgprot(0)
 193#define __P001  __pgprot(0)
 194#define __P010  __pgprot(0)
 195#define __P011  __pgprot(0)
 196#define __P100  __pgprot(0)
 197#define __P101  __pgprot(0)
 198#define __P110  __pgprot(0)
 199#define __P111  __pgprot(0)
 200
 201#define __S000  __pgprot(0)
 202#define __S001  __pgprot(0)
 203#define __S010  __pgprot(0)
 204#define __S011  __pgprot(0)
 205#define __S100  __pgprot(0)
 206#define __S101  __pgprot(0)
 207#define __S110  __pgprot(0)
 208#define __S111  __pgprot(0)
 209
 210#ifndef __ASSEMBLY__
 211
 212pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long);
 213
 214unsigned long pte_sz_bits(unsigned long size);
 215
 216extern pgprot_t PAGE_KERNEL;
 217extern pgprot_t PAGE_KERNEL_LOCKED;
 218extern pgprot_t PAGE_COPY;
 219extern pgprot_t PAGE_SHARED;
 220
 221/* XXX This ugliness is for the atyfb driver's sparc mmap() support. XXX */
 222extern unsigned long _PAGE_IE;
 223extern unsigned long _PAGE_E;
 224extern unsigned long _PAGE_CACHE;
 225
 226extern unsigned long pg_iobits;
 227extern unsigned long _PAGE_ALL_SZ_BITS;
 228
 229extern struct page *mem_map_zero;
 230#define ZERO_PAGE(vaddr)        (mem_map_zero)
 231
 232/* PFNs are real physical page numbers.  However, mem_map only begins to record
 233 * per-page information starting at pfn_base.  This is to handle systems where
 234 * the first physical page in the machine is at some huge physical address,
 235 * such as 4GB.   This is common on a partitioned E10000, for example.
 236 */
 237static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
 238{
 239        unsigned long paddr = pfn << PAGE_SHIFT;
 240
 241        BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL);
 242        return __pte(paddr | pgprot_val(prot));
 243}
 244#define mk_pte(page, pgprot)    pfn_pte(page_to_pfn(page), (pgprot))
 245
 246#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 247static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
 248{
 249        pte_t pte = pfn_pte(page_nr, pgprot);
 250
 251        return __pmd(pte_val(pte));
 252}
 253#define mk_pmd(page, pgprot)    pfn_pmd(page_to_pfn(page), (pgprot))
 254#endif
 255
 256/* This one can be done with two shifts.  */
 257static inline unsigned long pte_pfn(pte_t pte)
 258{
 259        unsigned long ret;
 260
 261        __asm__ __volatile__(
 262        "\n661: sllx            %1, %2, %0\n"
 263        "       srlx            %0, %3, %0\n"
 264        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 265        "       .word           661b\n"
 266        "       sllx            %1, %4, %0\n"
 267        "       srlx            %0, %5, %0\n"
 268        "       .previous\n"
 269        : "=r" (ret)
 270        : "r" (pte_val(pte)),
 271          "i" (21), "i" (21 + PAGE_SHIFT),
 272          "i" (8), "i" (8 + PAGE_SHIFT));
 273
 274        return ret;
 275}
 276#define pte_page(x) pfn_to_page(pte_pfn(x))
 277
 278static inline pte_t pte_modify(pte_t pte, pgprot_t prot)
 279{
 280        unsigned long mask, tmp;
 281
 282        /* SUN4U: 0x630107ffffffec38 (negated == 0x9cfef800000013c7)
 283         * SUN4V: 0x33ffffffffffee07 (negated == 0xcc000000000011f8)
 284         *
 285         * Even if we use negation tricks the result is still a 6
 286         * instruction sequence, so don't try to play fancy and just
 287         * do the most straightforward implementation.
 288         *
 289         * Note: We encode this into 3 sun4v 2-insn patch sequences.
 290         */
 291
 292        BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL);
 293        __asm__ __volatile__(
 294        "\n661: sethi           %%uhi(%2), %1\n"
 295        "       sethi           %%hi(%2), %0\n"
 296        "\n662: or              %1, %%ulo(%2), %1\n"
 297        "       or              %0, %%lo(%2), %0\n"
 298        "\n663: sllx            %1, 32, %1\n"
 299        "       or              %0, %1, %0\n"
 300        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 301        "       .word           661b\n"
 302        "       sethi           %%uhi(%3), %1\n"
 303        "       sethi           %%hi(%3), %0\n"
 304        "       .word           662b\n"
 305        "       or              %1, %%ulo(%3), %1\n"
 306        "       or              %0, %%lo(%3), %0\n"
 307        "       .word           663b\n"
 308        "       sllx            %1, 32, %1\n"
 309        "       or              %0, %1, %0\n"
 310        "       .previous\n"
 311        "       .section        .sun_m7_2insn_patch, \"ax\"\n"
 312        "       .word           661b\n"
 313        "       sethi           %%uhi(%4), %1\n"
 314        "       sethi           %%hi(%4), %0\n"
 315        "       .word           662b\n"
 316        "       or              %1, %%ulo(%4), %1\n"
 317        "       or              %0, %%lo(%4), %0\n"
 318        "       .word           663b\n"
 319        "       sllx            %1, 32, %1\n"
 320        "       or              %0, %1, %0\n"
 321        "       .previous\n"
 322        : "=r" (mask), "=r" (tmp)
 323        : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U |
 324               _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U |
 325               _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U),
 326          "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
 327               _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V |
 328               _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V),
 329          "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
 330               _PAGE_CP_4V | _PAGE_E_4V |
 331               _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V));
 332
 333        return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask));
 334}
 335
 336#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 337static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
 338{
 339        pte_t pte = __pte(pmd_val(pmd));
 340
 341        pte = pte_modify(pte, newprot);
 342
 343        return __pmd(pte_val(pte));
 344}
 345#endif
 346
 347static inline pgprot_t pgprot_noncached(pgprot_t prot)
 348{
 349        unsigned long val = pgprot_val(prot);
 350
 351        __asm__ __volatile__(
 352        "\n661: andn            %0, %2, %0\n"
 353        "       or              %0, %3, %0\n"
 354        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 355        "       .word           661b\n"
 356        "       andn            %0, %4, %0\n"
 357        "       or              %0, %5, %0\n"
 358        "       .previous\n"
 359        "       .section        .sun_m7_2insn_patch, \"ax\"\n"
 360        "       .word           661b\n"
 361        "       andn            %0, %6, %0\n"
 362        "       or              %0, %5, %0\n"
 363        "       .previous\n"
 364        : "=r" (val)
 365        : "0" (val), "i" (_PAGE_CP_4U | _PAGE_CV_4U), "i" (_PAGE_E_4U),
 366                     "i" (_PAGE_CP_4V | _PAGE_CV_4V), "i" (_PAGE_E_4V),
 367                     "i" (_PAGE_CP_4V));
 368
 369        return __pgprot(val);
 370}
 371/* Various pieces of code check for platform support by ifdef testing
 372 * on "pgprot_noncached".  That's broken and should be fixed, but for
 373 * now...
 374 */
 375#define pgprot_noncached pgprot_noncached
 376
 377#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 378static inline pte_t pte_mkhuge(pte_t pte)
 379{
 380        unsigned long mask;
 381
 382        __asm__ __volatile__(
 383        "\n661: sethi           %%uhi(%1), %0\n"
 384        "       sllx            %0, 32, %0\n"
 385        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 386        "       .word           661b\n"
 387        "       mov             %2, %0\n"
 388        "       nop\n"
 389        "       .previous\n"
 390        : "=r" (mask)
 391        : "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V));
 392
 393        return __pte(pte_val(pte) | mask);
 394}
 395#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 396static inline pmd_t pmd_mkhuge(pmd_t pmd)
 397{
 398        pte_t pte = __pte(pmd_val(pmd));
 399
 400        pte = pte_mkhuge(pte);
 401        pte_val(pte) |= _PAGE_PMD_HUGE;
 402
 403        return __pmd(pte_val(pte));
 404}
 405#endif
 406#endif
 407
 408static inline pte_t pte_mkdirty(pte_t pte)
 409{
 410        unsigned long val = pte_val(pte), tmp;
 411
 412        __asm__ __volatile__(
 413        "\n661: or              %0, %3, %0\n"
 414        "       nop\n"
 415        "\n662: nop\n"
 416        "       nop\n"
 417        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 418        "       .word           661b\n"
 419        "       sethi           %%uhi(%4), %1\n"
 420        "       sllx            %1, 32, %1\n"
 421        "       .word           662b\n"
 422        "       or              %1, %%lo(%4), %1\n"
 423        "       or              %0, %1, %0\n"
 424        "       .previous\n"
 425        : "=r" (val), "=r" (tmp)
 426        : "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U),
 427          "i" (_PAGE_MODIFIED_4V | _PAGE_W_4V));
 428
 429        return __pte(val);
 430}
 431
 432static inline pte_t pte_mkclean(pte_t pte)
 433{
 434        unsigned long val = pte_val(pte), tmp;
 435
 436        __asm__ __volatile__(
 437        "\n661: andn            %0, %3, %0\n"
 438        "       nop\n"
 439        "\n662: nop\n"
 440        "       nop\n"
 441        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 442        "       .word           661b\n"
 443        "       sethi           %%uhi(%4), %1\n"
 444        "       sllx            %1, 32, %1\n"
 445        "       .word           662b\n"
 446        "       or              %1, %%lo(%4), %1\n"
 447        "       andn            %0, %1, %0\n"
 448        "       .previous\n"
 449        : "=r" (val), "=r" (tmp)
 450        : "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U),
 451          "i" (_PAGE_MODIFIED_4V | _PAGE_W_4V));
 452
 453        return __pte(val);
 454}
 455
 456static inline pte_t pte_mkwrite(pte_t pte)
 457{
 458        unsigned long val = pte_val(pte), mask;
 459
 460        __asm__ __volatile__(
 461        "\n661: mov             %1, %0\n"
 462        "       nop\n"
 463        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 464        "       .word           661b\n"
 465        "       sethi           %%uhi(%2), %0\n"
 466        "       sllx            %0, 32, %0\n"
 467        "       .previous\n"
 468        : "=r" (mask)
 469        : "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V));
 470
 471        return __pte(val | mask);
 472}
 473
 474static inline pte_t pte_wrprotect(pte_t pte)
 475{
 476        unsigned long val = pte_val(pte), tmp;
 477
 478        __asm__ __volatile__(
 479        "\n661: andn            %0, %3, %0\n"
 480        "       nop\n"
 481        "\n662: nop\n"
 482        "       nop\n"
 483        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 484        "       .word           661b\n"
 485        "       sethi           %%uhi(%4), %1\n"
 486        "       sllx            %1, 32, %1\n"
 487        "       .word           662b\n"
 488        "       or              %1, %%lo(%4), %1\n"
 489        "       andn            %0, %1, %0\n"
 490        "       .previous\n"
 491        : "=r" (val), "=r" (tmp)
 492        : "0" (val), "i" (_PAGE_WRITE_4U | _PAGE_W_4U),
 493          "i" (_PAGE_WRITE_4V | _PAGE_W_4V));
 494
 495        return __pte(val);
 496}
 497
 498static inline pte_t pte_mkold(pte_t pte)
 499{
 500        unsigned long mask;
 501
 502        __asm__ __volatile__(
 503        "\n661: mov             %1, %0\n"
 504        "       nop\n"
 505        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 506        "       .word           661b\n"
 507        "       sethi           %%uhi(%2), %0\n"
 508        "       sllx            %0, 32, %0\n"
 509        "       .previous\n"
 510        : "=r" (mask)
 511        : "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
 512
 513        mask |= _PAGE_R;
 514
 515        return __pte(pte_val(pte) & ~mask);
 516}
 517
 518static inline pte_t pte_mkyoung(pte_t pte)
 519{
 520        unsigned long mask;
 521
 522        __asm__ __volatile__(
 523        "\n661: mov             %1, %0\n"
 524        "       nop\n"
 525        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 526        "       .word           661b\n"
 527        "       sethi           %%uhi(%2), %0\n"
 528        "       sllx            %0, 32, %0\n"
 529        "       .previous\n"
 530        : "=r" (mask)
 531        : "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
 532
 533        mask |= _PAGE_R;
 534
 535        return __pte(pte_val(pte) | mask);
 536}
 537
 538static inline pte_t pte_mkspecial(pte_t pte)
 539{
 540        pte_val(pte) |= _PAGE_SPECIAL;
 541        return pte;
 542}
 543
 544static inline unsigned long pte_young(pte_t pte)
 545{
 546        unsigned long mask;
 547
 548        __asm__ __volatile__(
 549        "\n661: mov             %1, %0\n"
 550        "       nop\n"
 551        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 552        "       .word           661b\n"
 553        "       sethi           %%uhi(%2), %0\n"
 554        "       sllx            %0, 32, %0\n"
 555        "       .previous\n"
 556        : "=r" (mask)
 557        : "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
 558
 559        return (pte_val(pte) & mask);
 560}
 561
 562static inline unsigned long pte_dirty(pte_t pte)
 563{
 564        unsigned long mask;
 565
 566        __asm__ __volatile__(
 567        "\n661: mov             %1, %0\n"
 568        "       nop\n"
 569        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 570        "       .word           661b\n"
 571        "       sethi           %%uhi(%2), %0\n"
 572        "       sllx            %0, 32, %0\n"
 573        "       .previous\n"
 574        : "=r" (mask)
 575        : "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V));
 576
 577        return (pte_val(pte) & mask);
 578}
 579
 580static inline unsigned long pte_write(pte_t pte)
 581{
 582        unsigned long mask;
 583
 584        __asm__ __volatile__(
 585        "\n661: mov             %1, %0\n"
 586        "       nop\n"
 587        "       .section        .sun4v_2insn_patch, \"ax\"\n"
 588        "       .word           661b\n"
 589        "       sethi           %%uhi(%2), %0\n"
 590        "       sllx            %0, 32, %0\n"
 591        "       .previous\n"
 592        : "=r" (mask)
 593        : "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V));
 594
 595        return (pte_val(pte) & mask);
 596}
 597
 598static inline unsigned long pte_exec(pte_t pte)
 599{
 600        unsigned long mask;
 601
 602        __asm__ __volatile__(
 603        "\n661: sethi           %%hi(%1), %0\n"
 604        "       .section        .sun4v_1insn_patch, \"ax\"\n"
 605        "       .word           661b\n"
 606        "       mov             %2, %0\n"
 607        "       .previous\n"
 608        : "=r" (mask)
 609        : "i" (_PAGE_EXEC_4U), "i" (_PAGE_EXEC_4V));
 610
 611        return (pte_val(pte) & mask);
 612}
 613
 614static inline unsigned long pte_present(pte_t pte)
 615{
 616        unsigned long val = pte_val(pte);
 617
 618        __asm__ __volatile__(
 619        "\n661: and             %0, %2, %0\n"
 620        "       .section        .sun4v_1insn_patch, \"ax\"\n"
 621        "       .word           661b\n"
 622        "       and             %0, %3, %0\n"
 623        "       .previous\n"
 624        : "=r" (val)
 625        : "0" (val), "i" (_PAGE_PRESENT_4U), "i" (_PAGE_PRESENT_4V));
 626
 627        return val;
 628}
 629
 630#define pte_accessible pte_accessible
 631static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a)
 632{
 633        return pte_val(a) & _PAGE_VALID;
 634}
 635
 636static inline unsigned long pte_special(pte_t pte)
 637{
 638        return pte_val(pte) & _PAGE_SPECIAL;
 639}
 640
 641static inline unsigned long pmd_large(pmd_t pmd)
 642{
 643        pte_t pte = __pte(pmd_val(pmd));
 644
 645        return pte_val(pte) & _PAGE_PMD_HUGE;
 646}
 647
 648static inline unsigned long pmd_pfn(pmd_t pmd)
 649{
 650        pte_t pte = __pte(pmd_val(pmd));
 651
 652        return pte_pfn(pte);
 653}
 654
 655#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 656static inline unsigned long pmd_dirty(pmd_t pmd)
 657{
 658        pte_t pte = __pte(pmd_val(pmd));
 659
 660        return pte_dirty(pte);
 661}
 662
 663static inline unsigned long pmd_young(pmd_t pmd)
 664{
 665        pte_t pte = __pte(pmd_val(pmd));
 666
 667        return pte_young(pte);
 668}
 669
 670static inline unsigned long pmd_write(pmd_t pmd)
 671{
 672        pte_t pte = __pte(pmd_val(pmd));
 673
 674        return pte_write(pte);
 675}
 676
 677static inline unsigned long pmd_trans_huge(pmd_t pmd)
 678{
 679        pte_t pte = __pte(pmd_val(pmd));
 680
 681        return pte_val(pte) & _PAGE_PMD_HUGE;
 682}
 683
 684#define has_transparent_hugepage() 1
 685
 686static inline pmd_t pmd_mkold(pmd_t pmd)
 687{
 688        pte_t pte = __pte(pmd_val(pmd));
 689
 690        pte = pte_mkold(pte);
 691
 692        return __pmd(pte_val(pte));
 693}
 694
 695static inline pmd_t pmd_wrprotect(pmd_t pmd)
 696{
 697        pte_t pte = __pte(pmd_val(pmd));
 698
 699        pte = pte_wrprotect(pte);
 700
 701        return __pmd(pte_val(pte));
 702}
 703
 704static inline pmd_t pmd_mkdirty(pmd_t pmd)
 705{
 706        pte_t pte = __pte(pmd_val(pmd));
 707
 708        pte = pte_mkdirty(pte);
 709
 710        return __pmd(pte_val(pte));
 711}
 712
 713static inline pmd_t pmd_mkclean(pmd_t pmd)
 714{
 715        pte_t pte = __pte(pmd_val(pmd));
 716
 717        pte = pte_mkclean(pte);
 718
 719        return __pmd(pte_val(pte));
 720}
 721
 722static inline pmd_t pmd_mkyoung(pmd_t pmd)
 723{
 724        pte_t pte = __pte(pmd_val(pmd));
 725
 726        pte = pte_mkyoung(pte);
 727
 728        return __pmd(pte_val(pte));
 729}
 730
 731static inline pmd_t pmd_mkwrite(pmd_t pmd)
 732{
 733        pte_t pte = __pte(pmd_val(pmd));
 734
 735        pte = pte_mkwrite(pte);
 736
 737        return __pmd(pte_val(pte));
 738}
 739
 740static inline pgprot_t pmd_pgprot(pmd_t entry)
 741{
 742        unsigned long val = pmd_val(entry);
 743
 744        return __pgprot(val);
 745}
 746#endif
 747
 748static inline int pmd_present(pmd_t pmd)
 749{
 750        return pmd_val(pmd) != 0UL;
 751}
 752
 753#define pmd_none(pmd)                   (!pmd_val(pmd))
 754
 755/* pmd_bad() is only called on non-trans-huge PMDs.  Our encoding is
 756 * very simple, it's just the physical address.  PTE tables are of
 757 * size PAGE_SIZE so make sure the sub-PAGE_SIZE bits are clear and
 758 * the top bits outside of the range of any physical address size we
 759 * support are clear as well.  We also validate the physical itself.
 760 */
 761#define pmd_bad(pmd)                    (pmd_val(pmd) & ~PAGE_MASK)
 762
 763#define pud_none(pud)                   (!pud_val(pud))
 764
 765#define pud_bad(pud)                    (pud_val(pud) & ~PAGE_MASK)
 766
 767#define pgd_none(pgd)                   (!pgd_val(pgd))
 768
 769#define pgd_bad(pgd)                    (pgd_val(pgd) & ~PAGE_MASK)
 770
 771#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 772void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 773                pmd_t *pmdp, pmd_t pmd);
 774#else
 775static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 776                              pmd_t *pmdp, pmd_t pmd)
 777{
 778        *pmdp = pmd;
 779}
 780#endif
 781
 782static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
 783{
 784        unsigned long val = __pa((unsigned long) (ptep));
 785
 786        pmd_val(*pmdp) = val;
 787}
 788
 789#define pud_set(pudp, pmdp)     \
 790        (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp))))
 791static inline unsigned long __pmd_page(pmd_t pmd)
 792{
 793        pte_t pte = __pte(pmd_val(pmd));
 794        unsigned long pfn;
 795
 796        pfn = pte_pfn(pte);
 797
 798        return ((unsigned long) __va(pfn << PAGE_SHIFT));
 799}
 800#define pmd_page(pmd)                   virt_to_page((void *)__pmd_page(pmd))
 801#define pud_page_vaddr(pud)             \
 802        ((unsigned long) __va(pud_val(pud)))
 803#define pud_page(pud)                   virt_to_page((void *)pud_page_vaddr(pud))
 804#define pmd_clear(pmdp)                 (pmd_val(*(pmdp)) = 0UL)
 805#define pud_present(pud)                (pud_val(pud) != 0U)
 806#define pud_clear(pudp)                 (pud_val(*(pudp)) = 0UL)
 807#define pgd_page_vaddr(pgd)             \
 808        ((unsigned long) __va(pgd_val(pgd)))
 809#define pgd_present(pgd)                (pgd_val(pgd) != 0U)
 810#define pgd_clear(pgdp)                 (pgd_val(*(pgd)) = 0UL)
 811
 812static inline unsigned long pud_large(pud_t pud)
 813{
 814        pte_t pte = __pte(pud_val(pud));
 815
 816        return pte_val(pte) & _PAGE_PMD_HUGE;
 817}
 818
 819static inline unsigned long pud_pfn(pud_t pud)
 820{
 821        pte_t pte = __pte(pud_val(pud));
 822
 823        return pte_pfn(pte);
 824}
 825
 826/* Same in both SUN4V and SUN4U.  */
 827#define pte_none(pte)                   (!pte_val(pte))
 828
 829#define pgd_set(pgdp, pudp)     \
 830        (pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))
 831
 832/* to find an entry in a page-table-directory. */
 833#define pgd_index(address)      (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
 834#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
 835
 836/* to find an entry in a kernel page-table-directory */
 837#define pgd_offset_k(address) pgd_offset(&init_mm, address)
 838
 839/* Find an entry in the third-level page table.. */
 840#define pud_index(address)      (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
 841#define pud_offset(pgdp, address)       \
 842        ((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))
 843
 844/* Find an entry in the second-level page table.. */
 845#define pmd_offset(pudp, address)       \
 846        ((pmd_t *) pud_page_vaddr(*(pudp)) + \
 847         (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)))
 848
 849/* Find an entry in the third-level page table.. */
 850#define pte_index(dir, address) \
 851        ((pte_t *) __pmd_page(*(dir)) + \
 852         ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)))
 853#define pte_offset_kernel               pte_index
 854#define pte_offset_map                  pte_index
 855#define pte_unmap(pte)                  do { } while (0)
 856
 857/* Actual page table PTE updates.  */
 858void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
 859                   pte_t *ptep, pte_t orig, int fullmm);
 860
 861#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
 862static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
 863                                            unsigned long addr,
 864                                            pmd_t *pmdp)
 865{
 866        pmd_t pmd = *pmdp;
 867        set_pmd_at(mm, addr, pmdp, __pmd(0UL));
 868        return pmd;
 869}
 870
 871static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
 872                             pte_t *ptep, pte_t pte, int fullmm)
 873{
 874        pte_t orig = *ptep;
 875
 876        *ptep = pte;
 877
 878        /* It is more efficient to let flush_tlb_kernel_range()
 879         * handle init_mm tlb flushes.
 880         *
 881         * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
 882         *             and SUN4V pte layout, so this inline test is fine.
 883         */
 884        if (likely(mm != &init_mm) && pte_accessible(mm, orig))
 885                tlb_batch_add(mm, addr, ptep, orig, fullmm);
 886}
 887
 888#define set_pte_at(mm,addr,ptep,pte)    \
 889        __set_pte_at((mm), (addr), (ptep), (pte), 0)
 890
 891#define pte_clear(mm,addr,ptep)         \
 892        set_pte_at((mm), (addr), (ptep), __pte(0UL))
 893
 894#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
 895#define pte_clear_not_present_full(mm,addr,ptep,fullmm) \
 896        __set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm))
 897
 898#ifdef DCACHE_ALIASING_POSSIBLE
 899#define __HAVE_ARCH_MOVE_PTE
 900#define move_pte(pte, prot, old_addr, new_addr)                         \
 901({                                                                      \
 902        pte_t newpte = (pte);                                           \
 903        if (tlb_type != hypervisor && pte_present(pte)) {               \
 904                unsigned long this_pfn = pte_pfn(pte);                  \
 905                                                                        \
 906                if (pfn_valid(this_pfn) &&                              \
 907                    (((old_addr) ^ (new_addr)) & (1 << 13)))            \
 908                        flush_dcache_page_all(current->mm,              \
 909                                              pfn_to_page(this_pfn));   \
 910        }                                                               \
 911        newpte;                                                         \
 912})
 913#endif
 914
 915extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 916
 917void paging_init(void);
 918unsigned long find_ecache_flush_span(unsigned long size);
 919
 920struct seq_file;
 921void mmu_info(struct seq_file *);
 922
 923struct vm_area_struct;
 924void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *);
 925#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 926void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
 927                          pmd_t *pmd);
 928
 929#define __HAVE_ARCH_PMDP_INVALIDATE
 930extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
 931                            pmd_t *pmdp);
 932
 933#define __HAVE_ARCH_PGTABLE_DEPOSIT
 934void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 935                                pgtable_t pgtable);
 936
 937#define __HAVE_ARCH_PGTABLE_WITHDRAW
 938pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
 939#endif
 940
 941/* Encode and de-code a swap entry */
 942#define __swp_type(entry)       (((entry).val >> PAGE_SHIFT) & 0xffUL)
 943#define __swp_offset(entry)     ((entry).val >> (PAGE_SHIFT + 8UL))
 944#define __swp_entry(type, offset)       \
 945        ( (swp_entry_t) \
 946          { \
 947                (((long)(type) << PAGE_SHIFT) | \
 948                 ((long)(offset) << (PAGE_SHIFT + 8UL))) \
 949          } )
 950#define __pte_to_swp_entry(pte)         ((swp_entry_t) { pte_val(pte) })
 951#define __swp_entry_to_pte(x)           ((pte_t) { (x).val })
 952
 953int page_in_phys_avail(unsigned long paddr);
 954
 955/*
 956 * For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in
 957 * its high 4 bits.  These macros/functions put it there or get it from there.
 958 */
 959#define MK_IOSPACE_PFN(space, pfn)      (pfn | (space << (BITS_PER_LONG - 4)))
 960#define GET_IOSPACE(pfn)                (pfn >> (BITS_PER_LONG - 4))
 961#define GET_PFN(pfn)                    (pfn & 0x0fffffffffffffffUL)
 962
 963int remap_pfn_range(struct vm_area_struct *, unsigned long, unsigned long,
 964                    unsigned long, pgprot_t);
 965
 966static inline int io_remap_pfn_range(struct vm_area_struct *vma,
 967                                     unsigned long from, unsigned long pfn,
 968                                     unsigned long size, pgprot_t prot)
 969{
 970        unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT;
 971        int space = GET_IOSPACE(pfn);
 972        unsigned long phys_base;
 973
 974        phys_base = offset | (((unsigned long) space) << 32UL);
 975
 976        return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot);
 977}
 978#define io_remap_pfn_range io_remap_pfn_range 
 979
 980#include <asm/tlbflush.h>
 981#include <asm-generic/pgtable.h>
 982
 983/* We provide our own get_unmapped_area to cope with VA holes and
 984 * SHM area cache aliasing for userland.
 985 */
 986#define HAVE_ARCH_UNMAPPED_AREA
 987#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
 988
 989/* We provide a special get_unmapped_area for framebuffer mmaps to try and use
 990 * the largest alignment possible such that larget PTEs can be used.
 991 */
 992unsigned long get_fb_unmapped_area(struct file *filp, unsigned long,
 993                                   unsigned long, unsigned long,
 994                                   unsigned long);
 995#define HAVE_ARCH_FB_UNMAPPED_AREA
 996
 997void pgtable_cache_init(void);
 998void sun4v_register_fault_status(void);
 999void sun4v_ktsb_register(void);
1000void __init cheetah_ecache_flush_init(void);

1001void sun4v_patch_tlb_handlers(void);
1002
1003extern unsigned long cmdline_memory_size;
1004
1005asmlinkage void do_sparc64_fault(struct pt_regs *regs);
1006
1007#endif /* !(__ASSEMBLY__) */
1008
1009#endif /* !(_SPARC64_PGTABLE_H) */
1010