linux/arch/sparc/mm/srmmu.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * srmmu.c:  SRMMU specific routines for memory management.
   4 *
   5 * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
   6 * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
   7 * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
   8 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   9 * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
  10 */
  11
  12#include <linux/seq_file.h>
  13#include <linux/spinlock.h>
  14#include <linux/memblock.h>
  15#include <linux/pagemap.h>
  16#include <linux/vmalloc.h>
  17#include <linux/kdebug.h>
  18#include <linux/export.h>
  19#include <linux/kernel.h>
  20#include <linux/init.h>
  21#include <linux/log2.h>
  22#include <linux/gfp.h>
  23#include <linux/fs.h>
  24#include <linux/mm.h>
  25
  26#include <asm/mmu_context.h>
  27#include <asm/cacheflush.h>
  28#include <asm/tlbflush.h>
  29#include <asm/io-unit.h>
  30#include <asm/pgalloc.h>
  31#include <asm/pgtable.h>
  32#include <asm/bitext.h>
  33#include <asm/vaddrs.h>
  34#include <asm/cache.h>
  35#include <asm/traps.h>
  36#include <asm/oplib.h>
  37#include <asm/mbus.h>
  38#include <asm/page.h>
  39#include <asm/asi.h>
  40#include <asm/smp.h>
  41#include <asm/io.h>
  42
  43/* Now the cpu specific definitions. */
  44#include <asm/turbosparc.h>
  45#include <asm/tsunami.h>
  46#include <asm/viking.h>
  47#include <asm/swift.h>
  48#include <asm/leon.h>
  49#include <asm/mxcc.h>
  50#include <asm/ross.h>
  51
  52#include "mm_32.h"
  53
  54enum mbus_module srmmu_modtype;
  55static unsigned int hwbug_bitmask;
  56int vac_cache_size;
  57EXPORT_SYMBOL(vac_cache_size);
  58int vac_line_size;
  59
  60extern struct resource sparc_iomap;
  61
  62extern unsigned long last_valid_pfn;
  63
  64static pgd_t *srmmu_swapper_pg_dir;
  65
  66const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops;
  67EXPORT_SYMBOL(sparc32_cachetlb_ops);
  68
  69#ifdef CONFIG_SMP
  70const struct sparc32_cachetlb_ops *local_ops;
  71
  72#define FLUSH_BEGIN(mm)
  73#define FLUSH_END
  74#else
  75#define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) {
  76#define FLUSH_END       }
  77#endif
  78
  79int flush_page_for_dma_global = 1;
  80
  81char *srmmu_name;
  82
  83ctxd_t *srmmu_ctx_table_phys;
  84static ctxd_t *srmmu_context_table;
  85
  86int viking_mxcc_present;
  87static DEFINE_SPINLOCK(srmmu_context_spinlock);
  88
  89static int is_hypersparc;
  90
  91static int srmmu_cache_pagetables;
  92
  93/* these will be initialized in srmmu_nocache_calcsize() */
  94static unsigned long srmmu_nocache_size;
  95static unsigned long srmmu_nocache_end;
  96
  97/* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */
  98#define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)
  99
 100/* The context table is a nocache user with the biggest alignment needs. */
 101#define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS)
 102
 103void *srmmu_nocache_pool;
 104static struct bit_map srmmu_nocache_map;
 105
 106static inline int srmmu_pmd_none(pmd_t pmd)
 107{ return !(pmd_val(pmd) & 0xFFFFFFF); }
 108
 109/* XXX should we hyper_flush_whole_icache here - Anton */
 110static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
 111{
 112        pte_t pte;
 113
 114        pte = __pte((SRMMU_ET_PTD | (__nocache_pa(pgdp) >> 4)));
 115        set_pte((pte_t *)ctxp, pte);
 116}
 117
 118/*
 119 * Locations of MSI Registers.
 120 */
 121#define MSI_MBUS_ARBEN  0xe0001008      /* MBus Arbiter Enable register */
 122
 123/*
 124 * Useful bits in the MSI Registers.
 125 */
 126#define MSI_ASYNC_MODE  0x80000000      /* Operate the MSI asynchronously */
 127
 128static void msi_set_sync(void)
 129{
 130        __asm__ __volatile__ ("lda [%0] %1, %%g3\n\t"
 131                              "andn %%g3, %2, %%g3\n\t"
 132                              "sta %%g3, [%0] %1\n\t" : :
 133                              "r" (MSI_MBUS_ARBEN),
 134                              "i" (ASI_M_CTL), "r" (MSI_ASYNC_MODE) : "g3");
 135}
 136
 137void pmd_set(pmd_t *pmdp, pte_t *ptep)
 138{
 139        unsigned long ptp;      /* Physical address, shifted right by 4 */
 140        int i;
 141
 142        ptp = __nocache_pa(ptep) >> 4;
 143        for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
 144                set_pte((pte_t *)&pmdp->pmdv[i], __pte(SRMMU_ET_PTD | ptp));
 145                ptp += (SRMMU_REAL_PTRS_PER_PTE * sizeof(pte_t) >> 4);
 146        }
 147}
 148
 149void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
 150{
 151        unsigned long ptp;      /* Physical address, shifted right by 4 */
 152        int i;
 153
 154        ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4);      /* watch for overflow */
 155        for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
 156                set_pte((pte_t *)&pmdp->pmdv[i], __pte(SRMMU_ET_PTD | ptp));
 157                ptp += (SRMMU_REAL_PTRS_PER_PTE * sizeof(pte_t) >> 4);
 158        }
 159}
 160
 161/* Find an entry in the third-level page table.. */
 162pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address)
 163{
 164        void *pte;
 165
 166        pte = __nocache_va((dir->pmdv[0] & SRMMU_PTD_PMASK) << 4);
 167        return (pte_t *) pte +
 168            ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1));
 169}
 170
 171/*
 172 * size: bytes to allocate in the nocache area.
 173 * align: bytes, number to align at.
 174 * Returns the virtual address of the allocated area.
 175 */
 176static void *__srmmu_get_nocache(int size, int align)
 177{
 178        int offset;
 179        unsigned long addr;
 180
 181        if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
 182                printk(KERN_ERR "Size 0x%x too small for nocache request\n",
 183                       size);
 184                size = SRMMU_NOCACHE_BITMAP_SHIFT;
 185        }
 186        if (size & (SRMMU_NOCACHE_BITMAP_SHIFT - 1)) {
 187                printk(KERN_ERR "Size 0x%x unaligned int nocache request\n",
 188                       size);
 189                size += SRMMU_NOCACHE_BITMAP_SHIFT - 1;
 190        }
 191        BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
 192
 193        offset = bit_map_string_get(&srmmu_nocache_map,
 194                                    size >> SRMMU_NOCACHE_BITMAP_SHIFT,
 195                                    align >> SRMMU_NOCACHE_BITMAP_SHIFT);
 196        if (offset == -1) {
 197                printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n",
 198                       size, (int) srmmu_nocache_size,
 199                       srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
 200                return NULL;
 201        }
 202
 203        addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT);
 204        return (void *)addr;
 205}
 206
 207void *srmmu_get_nocache(int size, int align)
 208{
 209        void *tmp;
 210
 211        tmp = __srmmu_get_nocache(size, align);
 212
 213        if (tmp)
 214                memset(tmp, 0, size);
 215
 216        return tmp;
 217}
 218
 219void srmmu_free_nocache(void *addr, int size)
 220{
 221        unsigned long vaddr;
 222        int offset;
 223
 224        vaddr = (unsigned long)addr;
 225        if (vaddr < SRMMU_NOCACHE_VADDR) {
 226                printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
 227                    vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
 228                BUG();
 229        }
 230        if (vaddr + size > srmmu_nocache_end) {
 231                printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
 232                    vaddr, srmmu_nocache_end);
 233                BUG();
 234        }
 235        if (!is_power_of_2(size)) {
 236                printk("Size 0x%x is not a power of 2\n", size);
 237                BUG();
 238        }
 239        if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
 240                printk("Size 0x%x is too small\n", size);
 241                BUG();
 242        }
 243        if (vaddr & (size - 1)) {
 244                printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
 245                BUG();
 246        }
 247
 248        offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
 249        size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
 250
 251        bit_map_clear(&srmmu_nocache_map, offset, size);
 252}
 253
 254static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
 255                                                 unsigned long end);
 256
 257/* Return how much physical memory we have.  */
 258static unsigned long __init probe_memory(void)
 259{
 260        unsigned long total = 0;
 261        int i;
 262
 263        for (i = 0; sp_banks[i].num_bytes; i++)
 264                total += sp_banks[i].num_bytes;
 265
 266        return total;
 267}
 268
 269/*
 270 * Reserve nocache dynamically proportionally to the amount of
 271 * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
 272 */
 273static void __init srmmu_nocache_calcsize(void)
 274{
 275        unsigned long sysmemavail = probe_memory() / 1024;
 276        int srmmu_nocache_npages;
 277
 278        srmmu_nocache_npages =
 279                sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256;
 280
 281 /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */
 282        // if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256;
 283        if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES)
 284                srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES;
 285
 286        /* anything above 1280 blows up */
 287        if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES)
 288                srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES;
 289
 290        srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE;
 291        srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size;
 292}
 293
 294static void __init srmmu_nocache_init(void)
 295{
 296        void *srmmu_nocache_bitmap;
 297        unsigned int bitmap_bits;
 298        pgd_t *pgd;
 299        pmd_t *pmd;
 300        pte_t *pte;
 301        unsigned long paddr, vaddr;
 302        unsigned long pteval;
 303
 304        bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT;
 305
 306        srmmu_nocache_pool = memblock_alloc(srmmu_nocache_size,
 307                                            SRMMU_NOCACHE_ALIGN_MAX);
 308        if (!srmmu_nocache_pool)
 309                panic("%s: Failed to allocate %lu bytes align=0x%x\n",
 310                      __func__, srmmu_nocache_size, SRMMU_NOCACHE_ALIGN_MAX);
 311        memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
 312
 313        srmmu_nocache_bitmap =
 314                memblock_alloc(BITS_TO_LONGS(bitmap_bits) * sizeof(long),
 315                               SMP_CACHE_BYTES);
 316        if (!srmmu_nocache_bitmap)
 317                panic("%s: Failed to allocate %zu bytes\n", __func__,
 318                      BITS_TO_LONGS(bitmap_bits) * sizeof(long));
 319        bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
 320
 321        srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
 322        memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
 323        init_mm.pgd = srmmu_swapper_pg_dir;
 324
 325        srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end);
 326
 327        paddr = __pa((unsigned long)srmmu_nocache_pool);
 328        vaddr = SRMMU_NOCACHE_VADDR;
 329
 330        while (vaddr < srmmu_nocache_end) {
 331                pgd = pgd_offset_k(vaddr);
 332                pmd = pmd_offset(__nocache_fix(pgd), vaddr);
 333                pte = pte_offset_kernel(__nocache_fix(pmd), vaddr);
 334
 335                pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);
 336
 337                if (srmmu_cache_pagetables)
 338                        pteval |= SRMMU_CACHE;
 339
 340                set_pte(__nocache_fix(pte), __pte(pteval));
 341
 342                vaddr += PAGE_SIZE;
 343                paddr += PAGE_SIZE;
 344        }
 345
 346        flush_cache_all();
 347        flush_tlb_all();
 348}
 349
 350pgd_t *get_pgd_fast(void)
 351{
 352        pgd_t *pgd = NULL;
 353
 354        pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
 355        if (pgd) {
 356                pgd_t *init = pgd_offset_k(0);
 357                memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
 358                memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
 359                                                (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
 360        }
 361
 362        return pgd;
 363}
 364
 365/*
 366 * Hardware needs alignment to 256 only, but we align to whole page size
 367 * to reduce fragmentation problems due to the buddy principle.
 368 * XXX Provide actual fragmentation statistics in /proc.
 369 *
 370 * Alignments up to the page size are the same for physical and virtual
 371 * addresses of the nocache area.
 372 */
 373pgtable_t pte_alloc_one(struct mm_struct *mm)
 374{
 375        unsigned long pte;
 376        struct page *page;
 377
 378        if ((pte = (unsigned long)pte_alloc_one_kernel(mm)) == 0)
 379                return NULL;
 380        page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
 381        if (!pgtable_pte_page_ctor(page)) {
 382                __free_page(page);
 383                return NULL;
 384        }
 385        return page;
 386}
 387
 388void pte_free(struct mm_struct *mm, pgtable_t pte)
 389{
 390        unsigned long p;
 391
 392        pgtable_pte_page_dtor(pte);
 393        p = (unsigned long)page_address(pte);   /* Cached address (for test) */
 394        if (p == 0)
 395                BUG();
 396        p = page_to_pfn(pte) << PAGE_SHIFT;     /* Physical address */
 397
 398        /* free non cached virtual address*/
 399        srmmu_free_nocache(__nocache_va(p), PTE_SIZE);
 400}
 401
 402/* context handling - a dynamically sized pool is used */
 403#define NO_CONTEXT      -1
 404
 405struct ctx_list {
 406        struct ctx_list *next;
 407        struct ctx_list *prev;
 408        unsigned int ctx_number;
 409        struct mm_struct *ctx_mm;
 410};
 411
 412static struct ctx_list *ctx_list_pool;
 413static struct ctx_list ctx_free;
 414static struct ctx_list ctx_used;
 415
 416/* At boot time we determine the number of contexts */
 417static int num_contexts;
 418
 419static inline void remove_from_ctx_list(struct ctx_list *entry)
 420{
 421        entry->next->prev = entry->prev;
 422        entry->prev->next = entry->next;
 423}
 424
 425static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry)
 426{
 427        entry->next = head;
 428        (entry->prev = head->prev)->next = entry;
 429        head->prev = entry;
 430}
 431#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry)
 432#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry)
 433
 434
 435static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
 436{
 437        struct ctx_list *ctxp;
 438
 439        ctxp = ctx_free.next;
 440        if (ctxp != &ctx_free) {
 441                remove_from_ctx_list(ctxp);
 442                add_to_used_ctxlist(ctxp);
 443                mm->context = ctxp->ctx_number;
 444                ctxp->ctx_mm = mm;
 445                return;
 446        }
 447        ctxp = ctx_used.next;
 448        if (ctxp->ctx_mm == old_mm)
 449                ctxp = ctxp->next;
 450        if (ctxp == &ctx_used)
 451                panic("out of mmu contexts");
 452        flush_cache_mm(ctxp->ctx_mm);
 453        flush_tlb_mm(ctxp->ctx_mm);
 454        remove_from_ctx_list(ctxp);
 455        add_to_used_ctxlist(ctxp);
 456        ctxp->ctx_mm->context = NO_CONTEXT;
 457        ctxp->ctx_mm = mm;
 458        mm->context = ctxp->ctx_number;
 459}
 460
 461static inline void free_context(int context)
 462{
 463        struct ctx_list *ctx_old;
 464
 465        ctx_old = ctx_list_pool + context;
 466        remove_from_ctx_list(ctx_old);
 467        add_to_free_ctxlist(ctx_old);
 468}
 469
 470static void __init sparc_context_init(int numctx)
 471{
 472        int ctx;
 473        unsigned long size;
 474
 475        size = numctx * sizeof(struct ctx_list);
 476        ctx_list_pool = memblock_alloc(size, SMP_CACHE_BYTES);
 477        if (!ctx_list_pool)
 478                panic("%s: Failed to allocate %lu bytes\n", __func__, size);
 479
 480        for (ctx = 0; ctx < numctx; ctx++) {
 481                struct ctx_list *clist;
 482
 483                clist = (ctx_list_pool + ctx);
 484                clist->ctx_number = ctx;
 485                clist->ctx_mm = NULL;
 486        }
 487        ctx_free.next = ctx_free.prev = &ctx_free;
 488        ctx_used.next = ctx_used.prev = &ctx_used;
 489        for (ctx = 0; ctx < numctx; ctx++)
 490                add_to_free_ctxlist(ctx_list_pool + ctx);
 491}
 492
 493void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
 494               struct task_struct *tsk)
 495{
 496        unsigned long flags;
 497
 498        if (mm->context == NO_CONTEXT) {
 499                spin_lock_irqsave(&srmmu_context_spinlock, flags);
 500                alloc_context(old_mm, mm);
 501                spin_unlock_irqrestore(&srmmu_context_spinlock, flags);
 502                srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd);
 503        }
 504
 505        if (sparc_cpu_model == sparc_leon)
 506                leon_switch_mm();
 507
 508        if (is_hypersparc)
 509                hyper_flush_whole_icache();
 510
 511        srmmu_set_context(mm->context);
 512}
 513
 514/* Low level IO area allocation on the SRMMU. */
 515static inline void srmmu_mapioaddr(unsigned long physaddr,
 516                                   unsigned long virt_addr, int bus_type)
 517{
 518        pgd_t *pgdp;
 519        pmd_t *pmdp;
 520        pte_t *ptep;
 521        unsigned long tmp;
 522
 523        physaddr &= PAGE_MASK;
 524        pgdp = pgd_offset_k(virt_addr);
 525        pmdp = pmd_offset(pgdp, virt_addr);
 526        ptep = pte_offset_kernel(pmdp, virt_addr);
 527        tmp = (physaddr >> 4) | SRMMU_ET_PTE;
 528
 529        /* I need to test whether this is consistent over all
 530         * sun4m's.  The bus_type represents the upper 4 bits of
 531         * 36-bit physical address on the I/O space lines...
 532         */
 533        tmp |= (bus_type << 28);
 534        tmp |= SRMMU_PRIV;
 535        __flush_page_to_ram(virt_addr);
 536        set_pte(ptep, __pte(tmp));
 537}
 538
 539void srmmu_mapiorange(unsigned int bus, unsigned long xpa,
 540                      unsigned long xva, unsigned int len)
 541{
 542        while (len != 0) {
 543                len -= PAGE_SIZE;
 544                srmmu_mapioaddr(xpa, xva, bus);
 545                xva += PAGE_SIZE;
 546                xpa += PAGE_SIZE;
 547        }
 548        flush_tlb_all();
 549}
 550
 551static inline void srmmu_unmapioaddr(unsigned long virt_addr)
 552{
 553        pgd_t *pgdp;
 554        pmd_t *pmdp;
 555        pte_t *ptep;
 556
 557        pgdp = pgd_offset_k(virt_addr);
 558        pmdp = pmd_offset(pgdp, virt_addr);
 559        ptep = pte_offset_kernel(pmdp, virt_addr);
 560
 561        /* No need to flush uncacheable page. */
 562        __pte_clear(ptep);
 563}
 564
 565void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len)
 566{
 567        while (len != 0) {
 568                len -= PAGE_SIZE;
 569                srmmu_unmapioaddr(virt_addr);
 570                virt_addr += PAGE_SIZE;
 571        }
 572        flush_tlb_all();
 573}
 574
 575/* tsunami.S */
 576extern void tsunami_flush_cache_all(void);
 577extern void tsunami_flush_cache_mm(struct mm_struct *mm);
 578extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 579extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
 580extern void tsunami_flush_page_to_ram(unsigned long page);
 581extern void tsunami_flush_page_for_dma(unsigned long page);
 582extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
 583extern void tsunami_flush_tlb_all(void);
 584extern void tsunami_flush_tlb_mm(struct mm_struct *mm);
 585extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 586extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
 587extern void tsunami_setup_blockops(void);
 588
 589/* swift.S */
 590extern void swift_flush_cache_all(void);
 591extern void swift_flush_cache_mm(struct mm_struct *mm);
 592extern void swift_flush_cache_range(struct vm_area_struct *vma,
 593                                    unsigned long start, unsigned long end);
 594extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
 595extern void swift_flush_page_to_ram(unsigned long page);
 596extern void swift_flush_page_for_dma(unsigned long page);
 597extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
 598extern void swift_flush_tlb_all(void);
 599extern void swift_flush_tlb_mm(struct mm_struct *mm);
 600extern void swift_flush_tlb_range(struct vm_area_struct *vma,
 601                                  unsigned long start, unsigned long end);
 602extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
 603
 604#if 0  /* P3: deadwood to debug precise flushes on Swift. */
 605void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 606{
 607        int cctx, ctx1;
 608
 609        page &= PAGE_MASK;
 610        if ((ctx1 = vma->vm_mm->context) != -1) {
 611                cctx = srmmu_get_context();
 612/* Is context # ever different from current context? P3 */
 613                if (cctx != ctx1) {
 614                        printk("flush ctx %02x curr %02x\n", ctx1, cctx);
 615                        srmmu_set_context(ctx1);
 616                        swift_flush_page(page);
 617                        __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
 618                                        "r" (page), "i" (ASI_M_FLUSH_PROBE));
 619                        srmmu_set_context(cctx);
 620                } else {
 621                         /* Rm. prot. bits from virt. c. */
 622                        /* swift_flush_cache_all(); */
 623                        /* swift_flush_cache_page(vma, page); */
 624                        swift_flush_page(page);
 625
 626                        __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
 627                                "r" (page), "i" (ASI_M_FLUSH_PROBE));
 628                        /* same as above: srmmu_flush_tlb_page() */
 629                }
 630        }
 631}
 632#endif
 633
 634/*
 635 * The following are all MBUS based SRMMU modules, and therefore could
 636 * be found in a multiprocessor configuration.  On the whole, these
 637 * chips seems to be much more touchy about DVMA and page tables
 638 * with respect to cache coherency.
 639 */
 640
 641/* viking.S */
 642extern void viking_flush_cache_all(void);
 643extern void viking_flush_cache_mm(struct mm_struct *mm);
 644extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start,
 645                                     unsigned long end);
 646extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
 647extern void viking_flush_page_to_ram(unsigned long page);
 648extern void viking_flush_page_for_dma(unsigned long page);
 649extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr);
 650extern void viking_flush_page(unsigned long page);
 651extern void viking_mxcc_flush_page(unsigned long page);
 652extern void viking_flush_tlb_all(void);
 653extern void viking_flush_tlb_mm(struct mm_struct *mm);
 654extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 655                                   unsigned long end);
 656extern void viking_flush_tlb_page(struct vm_area_struct *vma,
 657                                  unsigned long page);
 658extern void sun4dsmp_flush_tlb_all(void);
 659extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm);
 660extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 661                                   unsigned long end);
 662extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma,
 663                                  unsigned long page);
 664
 665/* hypersparc.S */
 666extern void hypersparc_flush_cache_all(void);
 667extern void hypersparc_flush_cache_mm(struct mm_struct *mm);
 668extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 669extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
 670extern void hypersparc_flush_page_to_ram(unsigned long page);
 671extern void hypersparc_flush_page_for_dma(unsigned long page);
 672extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
 673extern void hypersparc_flush_tlb_all(void);
 674extern void hypersparc_flush_tlb_mm(struct mm_struct *mm);
 675extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 676extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
 677extern void hypersparc_setup_blockops(void);
 678
 679/*
 680 * NOTE: All of this startup code assumes the low 16mb (approx.) of
 681 *       kernel mappings are done with one single contiguous chunk of
 682 *       ram.  On small ram machines (classics mainly) we only get
 683 *       around 8mb mapped for us.
 684 */
 685
 686static void __init early_pgtable_allocfail(char *type)
 687{
 688        prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
 689        prom_halt();
 690}
 691
 692static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
 693                                                        unsigned long end)
 694{
 695        pgd_t *pgdp;
 696        pmd_t *pmdp;
 697        pte_t *ptep;
 698
 699        while (start < end) {
 700                pgdp = pgd_offset_k(start);
 701                if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
 702                        pmdp = __srmmu_get_nocache(
 703                            SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
 704                        if (pmdp == NULL)
 705                                early_pgtable_allocfail("pmd");
 706                        memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
 707                        pgd_set(__nocache_fix(pgdp), pmdp);
 708                }
 709                pmdp = pmd_offset(__nocache_fix(pgdp), start);
 710                if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
 711                        ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
 712                        if (ptep == NULL)
 713                                early_pgtable_allocfail("pte");
 714                        memset(__nocache_fix(ptep), 0, PTE_SIZE);
 715                        pmd_set(__nocache_fix(pmdp), ptep);
 716                }
 717                if (start > (0xffffffffUL - PMD_SIZE))
 718                        break;
 719                start = (start + PMD_SIZE) & PMD_MASK;
 720        }
 721}
 722
 723static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
 724                                                  unsigned long end)
 725{
 726        pgd_t *pgdp;
 727        pmd_t *pmdp;
 728        pte_t *ptep;
 729
 730        while (start < end) {
 731                pgdp = pgd_offset_k(start);
 732                if (pgd_none(*pgdp)) {
 733                        pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
 734                        if (pmdp == NULL)
 735                                early_pgtable_allocfail("pmd");
 736                        memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
 737                        pgd_set(pgdp, pmdp);
 738                }
 739                pmdp = pmd_offset(pgdp, start);
 740                if (srmmu_pmd_none(*pmdp)) {
 741                        ptep = __srmmu_get_nocache(PTE_SIZE,
 742                                                             PTE_SIZE);
 743                        if (ptep == NULL)
 744                                early_pgtable_allocfail("pte");
 745                        memset(ptep, 0, PTE_SIZE);
 746                        pmd_set(pmdp, ptep);
 747                }
 748                if (start > (0xffffffffUL - PMD_SIZE))
 749                        break;
 750                start = (start + PMD_SIZE) & PMD_MASK;
 751        }
 752}
 753
 754/* These flush types are not available on all chips... */
 755static inline unsigned long srmmu_probe(unsigned long vaddr)
 756{
 757        unsigned long retval;
 758
 759        if (sparc_cpu_model != sparc_leon) {
 760
 761                vaddr &= PAGE_MASK;
 762                __asm__ __volatile__("lda [%1] %2, %0\n\t" :
 763                                     "=r" (retval) :
 764                                     "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE));
 765        } else {
 766                retval = leon_swprobe(vaddr, NULL);
 767        }
 768        return retval;
 769}
 770
 771/*
 772 * This is much cleaner than poking around physical address space
 773 * looking at the prom's page table directly which is what most
 774 * other OS's do.  Yuck... this is much better.
 775 */
 776static void __init srmmu_inherit_prom_mappings(unsigned long start,
 777                                               unsigned long end)
 778{
 779        unsigned long probed;
 780        unsigned long addr;
 781        pgd_t *pgdp;
 782        pmd_t *pmdp;
 783        pte_t *ptep;
 784        int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
 785
 786        while (start <= end) {
 787                if (start == 0)
 788                        break; /* probably wrap around */
 789                if (start == 0xfef00000)
 790                        start = KADB_DEBUGGER_BEGVM;
 791                probed = srmmu_probe(start);
 792                if (!probed) {
 793                        /* continue probing until we find an entry */
 794                        start += PAGE_SIZE;
 795                        continue;
 796                }
 797
 798                /* A red snapper, see what it really is. */
 799                what = 0;
 800                addr = start - PAGE_SIZE;
 801
 802                if (!(start & ~(SRMMU_REAL_PMD_MASK))) {
 803                        if (srmmu_probe(addr + SRMMU_REAL_PMD_SIZE) == probed)
 804                                what = 1;
 805                }
 806
 807                if (!(start & ~(SRMMU_PGDIR_MASK))) {
 808                        if (srmmu_probe(addr + SRMMU_PGDIR_SIZE) == probed)
 809                                what = 2;
 810                }
 811
 812                pgdp = pgd_offset_k(start);
 813                if (what == 2) {
 814                        *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed);
 815                        start += SRMMU_PGDIR_SIZE;
 816                        continue;
 817                }
 818                if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
 819                        pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
 820                                                   SRMMU_PMD_TABLE_SIZE);
 821                        if (pmdp == NULL)
 822                                early_pgtable_allocfail("pmd");
 823                        memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
 824                        pgd_set(__nocache_fix(pgdp), pmdp);
 825                }
 826                pmdp = pmd_offset(__nocache_fix(pgdp), start);
 827                if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
 828                        ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
 829                        if (ptep == NULL)
 830                                early_pgtable_allocfail("pte");
 831                        memset(__nocache_fix(ptep), 0, PTE_SIZE);
 832                        pmd_set(__nocache_fix(pmdp), ptep);
 833                }
 834                if (what == 1) {
 835                        /* We bend the rule where all 16 PTPs in a pmd_t point
 836                         * inside the same PTE page, and we leak a perfectly
 837                         * good hardware PTE piece. Alternatives seem worse.
 838                         */
 839                        unsigned int x; /* Index of HW PMD in soft cluster */
 840                        unsigned long *val;
 841                        x = (start >> PMD_SHIFT) & 15;
 842                        val = &pmdp->pmdv[x];
 843                        *(unsigned long *)__nocache_fix(val) = probed;
 844                        start += SRMMU_REAL_PMD_SIZE;
 845                        continue;
 846                }
 847                ptep = pte_offset_kernel(__nocache_fix(pmdp), start);
 848                *(pte_t *)__nocache_fix(ptep) = __pte(probed);
 849                start += PAGE_SIZE;
 850        }
 851}
 852
 853#define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID)
 854
 855/* Create a third-level SRMMU 16MB page mapping. */
 856static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base)
 857{
 858        pgd_t *pgdp = pgd_offset_k(vaddr);
 859        unsigned long big_pte;
 860
 861        big_pte = KERNEL_PTE(phys_base >> 4);
 862        *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte);
 863}
 864
 865/* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */
 866static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
 867{
 868        unsigned long pstart = (sp_banks[sp_entry].base_addr & SRMMU_PGDIR_MASK);
 869        unsigned long vstart = (vbase & SRMMU_PGDIR_MASK);
 870        unsigned long vend = SRMMU_PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes);
 871        /* Map "low" memory only */
 872        const unsigned long min_vaddr = PAGE_OFFSET;
 873        const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM;
 874
 875        if (vstart < min_vaddr || vstart >= max_vaddr)
 876                return vstart;
 877
 878        if (vend > max_vaddr || vend < min_vaddr)
 879                vend = max_vaddr;
 880
 881        while (vstart < vend) {
 882                do_large_mapping(vstart, pstart);
 883                vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE;
 884        }
 885        return vstart;
 886}
 887
 888static void __init map_kernel(void)
 889{
 890        int i;
 891
 892        if (phys_base > 0) {
 893                do_large_mapping(PAGE_OFFSET, phys_base);
 894        }
 895
 896        for (i = 0; sp_banks[i].num_bytes != 0; i++) {
 897                map_spbank((unsigned long)__va(sp_banks[i].base_addr), i);
 898        }
 899}
 900
 901void (*poke_srmmu)(void) = NULL;
 902
 903void __init srmmu_paging_init(void)
 904{
 905        int i;
 906        phandle cpunode;
 907        char node_str[128];
 908        pgd_t *pgd;
 909        pmd_t *pmd;
 910        pte_t *pte;
 911        unsigned long pages_avail;
 912
 913        init_mm.context = (unsigned long) NO_CONTEXT;
 914        sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
 915
 916        if (sparc_cpu_model == sun4d)
 917                num_contexts = 65536; /* We know it is Viking */
 918        else {
 919                /* Find the number of contexts on the srmmu. */
 920                cpunode = prom_getchild(prom_root_node);
 921                num_contexts = 0;
 922                while (cpunode != 0) {
 923                        prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
 924                        if (!strcmp(node_str, "cpu")) {
 925                                num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
 926                                break;
 927                        }
 928                        cpunode = prom_getsibling(cpunode);
 929                }
 930        }
 931
 932        if (!num_contexts) {
 933                prom_printf("Something wrong, can't find cpu node in paging_init.\n");
 934                prom_halt();
 935        }
 936
 937        pages_avail = 0;
 938        last_valid_pfn = bootmem_init(&pages_avail);
 939
 940        srmmu_nocache_calcsize();
 941        srmmu_nocache_init();
 942        srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE));
 943        map_kernel();
 944
 945        /* ctx table has to be physically aligned to its size */
 946        srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t));
 947        srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa(srmmu_context_table);
 948
 949        for (i = 0; i < num_contexts; i++)
 950                srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
 951
 952        flush_cache_all();
 953        srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys);
 954#ifdef CONFIG_SMP
 955        /* Stop from hanging here... */
 956        local_ops->tlb_all();
 957#else
 958        flush_tlb_all();
 959#endif
 960        poke_srmmu();
 961
 962        srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END);
 963        srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END);
 964
 965        srmmu_allocate_ptable_skeleton(
 966                __fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP);
 967        srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END);
 968
 969        pgd = pgd_offset_k(PKMAP_BASE);
 970        pmd = pmd_offset(pgd, PKMAP_BASE);
 971        pte = pte_offset_kernel(pmd, PKMAP_BASE);
 972        pkmap_page_table = pte;
 973
 974        flush_cache_all();
 975        flush_tlb_all();
 976
 977        sparc_context_init(num_contexts);
 978
 979        kmap_init();
 980
 981        {
 982                unsigned long zones_size[MAX_NR_ZONES];
 983                unsigned long zholes_size[MAX_NR_ZONES];
 984                unsigned long npages;
 985                int znum;
 986
 987                for (znum = 0; znum < MAX_NR_ZONES; znum++)
 988                        zones_size[znum] = zholes_size[znum] = 0;
 989
 990                npages = max_low_pfn - pfn_base;
 991
 992                zones_size[ZONE_DMA] = npages;
 993                zholes_size[ZONE_DMA] = npages - pages_avail;
 994
 995                npages = highend_pfn - max_low_pfn;
 996                zones_size[ZONE_HIGHMEM] = npages;
 997                zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
 998
 999                free_area_init_node(0, zones_size, pfn_base, zholes_size);
1000        }
1001}
1002
1003void mmu_info(struct seq_file *m)
1004{
1005        seq_printf(m,
1006                   "MMU type\t: %s\n"
1007                   "contexts\t: %d\n"
1008                   "nocache total\t: %ld\n"
1009                   "nocache used\t: %d\n",
1010                   srmmu_name,
1011                   num_contexts,
1012                   srmmu_nocache_size,
1013                   srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
1014}
1015
1016int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
1017{
1018        mm->context = NO_CONTEXT;
1019        return 0;
1020}
1021
1022void destroy_context(struct mm_struct *mm)
1023{
1024        unsigned long flags;
1025
1026        if (mm->context != NO_CONTEXT) {
1027                flush_cache_mm(mm);
1028                srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
1029                flush_tlb_mm(mm);
1030                spin_lock_irqsave(&srmmu_context_spinlock, flags);
1031                free_context(mm->context);
1032                spin_unlock_irqrestore(&srmmu_context_spinlock, flags);
1033                mm->context = NO_CONTEXT;
1034        }
1035}
1036
1037/* Init various srmmu chip types. */
1038static void __init srmmu_is_bad(void)
1039{
1040        prom_printf("Could not determine SRMMU chip type.\n");
1041        prom_halt();
1042}
1043
1044static void __init init_vac_layout(void)
1045{
1046        phandle nd;
1047        int cache_lines;
1048        char node_str[128];
1049#ifdef CONFIG_SMP
1050        int cpu = 0;
1051        unsigned long max_size = 0;
1052        unsigned long min_line_size = 0x10000000;
1053#endif
1054
1055        nd = prom_getchild(prom_root_node);
1056        while ((nd = prom_getsibling(nd)) != 0) {
1057                prom_getstring(nd, "device_type", node_str, sizeof(node_str));
1058                if (!strcmp(node_str, "cpu")) {
1059                        vac_line_size = prom_getint(nd, "cache-line-size");
1060                        if (vac_line_size == -1) {
1061                                prom_printf("can't determine cache-line-size, halting.\n");
1062                                prom_halt();
1063                        }
1064                        cache_lines = prom_getint(nd, "cache-nlines");
1065                        if (cache_lines == -1) {
1066                                prom_printf("can't determine cache-nlines, halting.\n");
1067                                prom_halt();
1068                        }
1069
1070                        vac_cache_size = cache_lines * vac_line_size;
1071#ifdef CONFIG_SMP
1072                        if (vac_cache_size > max_size)
1073                                max_size = vac_cache_size;
1074                        if (vac_line_size < min_line_size)
1075                                min_line_size = vac_line_size;
1076                        //FIXME: cpus not contiguous!!
1077                        cpu++;
1078                        if (cpu >= nr_cpu_ids || !cpu_online(cpu))
1079                                break;
1080#else
1081                        break;
1082#endif
1083                }
1084        }
1085        if (nd == 0) {
1086                prom_printf("No CPU nodes found, halting.\n");
1087                prom_halt();
1088        }
1089#ifdef CONFIG_SMP
1090        vac_cache_size = max_size;
1091        vac_line_size = min_line_size;
1092#endif
1093        printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n",
1094               (int)vac_cache_size, (int)vac_line_size);
1095}
1096
1097static void poke_hypersparc(void)
1098{
1099        volatile unsigned long clear;
1100        unsigned long mreg = srmmu_get_mmureg();
1101
1102        hyper_flush_unconditional_combined();
1103
1104        mreg &= ~(HYPERSPARC_CWENABLE);
1105        mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE);
1106        mreg |= (HYPERSPARC_CMODE);
1107
1108        srmmu_set_mmureg(mreg);
1109
1110#if 0 /* XXX I think this is bad news... -DaveM */
1111        hyper_clear_all_tags();
1112#endif
1113
1114        put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE);
1115        hyper_flush_whole_icache();
1116        clear = srmmu_get_faddr();
1117        clear = srmmu_get_fstatus();
1118}
1119
1120static const struct sparc32_cachetlb_ops hypersparc_ops = {
1121        .cache_all      = hypersparc_flush_cache_all,
1122        .cache_mm       = hypersparc_flush_cache_mm,
1123        .cache_page     = hypersparc_flush_cache_page,
1124        .cache_range    = hypersparc_flush_cache_range,
1125        .tlb_all        = hypersparc_flush_tlb_all,
1126        .tlb_mm         = hypersparc_flush_tlb_mm,
1127        .tlb_page       = hypersparc_flush_tlb_page,
1128        .tlb_range      = hypersparc_flush_tlb_range,
1129        .page_to_ram    = hypersparc_flush_page_to_ram,
1130        .sig_insns      = hypersparc_flush_sig_insns,
1131        .page_for_dma   = hypersparc_flush_page_for_dma,
1132};
1133
1134static void __init init_hypersparc(void)
1135{
1136        srmmu_name = "ROSS HyperSparc";
1137        srmmu_modtype = HyperSparc;
1138
1139        init_vac_layout();
1140
1141        is_hypersparc = 1;
1142        sparc32_cachetlb_ops = &hypersparc_ops;
1143
1144        poke_srmmu = poke_hypersparc;
1145
1146        hypersparc_setup_blockops();
1147}
1148
1149static void poke_swift(void)
1150{
1151        unsigned long mreg;
1152
1153        /* Clear any crap from the cache or else... */
1154        swift_flush_cache_all();
1155
1156        /* Enable I & D caches */
1157        mreg = srmmu_get_mmureg();
1158        mreg |= (SWIFT_IE | SWIFT_DE);
1159        /*
1160         * The Swift branch folding logic is completely broken.  At
1161         * trap time, if things are just right, if can mistakenly
1162         * think that a trap is coming from kernel mode when in fact
1163         * it is coming from user mode (it mis-executes the branch in
1164         * the trap code).  So you see things like crashme completely
1165         * hosing your machine which is completely unacceptable.  Turn
1166         * this shit off... nice job Fujitsu.
1167         */
1168        mreg &= ~(SWIFT_BF);
1169        srmmu_set_mmureg(mreg);
1170}
1171
1172static const struct sparc32_cachetlb_ops swift_ops = {
1173        .cache_all      = swift_flush_cache_all,
1174        .cache_mm       = swift_flush_cache_mm,
1175        .cache_page     = swift_flush_cache_page,
1176        .cache_range    = swift_flush_cache_range,
1177        .tlb_all        = swift_flush_tlb_all,
1178        .tlb_mm         = swift_flush_tlb_mm,
1179        .tlb_page       = swift_flush_tlb_page,
1180        .tlb_range      = swift_flush_tlb_range,
1181        .page_to_ram    = swift_flush_page_to_ram,
1182        .sig_insns      = swift_flush_sig_insns,
1183        .page_for_dma   = swift_flush_page_for_dma,
1184};
1185
1186#define SWIFT_MASKID_ADDR  0x10003018
1187static void __init init_swift(void)
1188{
1189        unsigned long swift_rev;
1190
1191        __asm__ __volatile__("lda [%1] %2, %0\n\t"
1192                             "srl %0, 0x18, %0\n\t" :
1193                             "=r" (swift_rev) :
1194                             "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
1195        srmmu_name = "Fujitsu Swift";
1196        switch (swift_rev) {
1197        case 0x11:
1198        case 0x20:
1199        case 0x23:
1200        case 0x30:
1201                srmmu_modtype = Swift_lots_o_bugs;
1202                hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN);
1203                /*
1204                 * Gee george, I wonder why Sun is so hush hush about
1205                 * this hardware bug... really braindamage stuff going
1206                 * on here.  However I think we can find a way to avoid
1207                 * all of the workaround overhead under Linux.  Basically,
1208                 * any page fault can cause kernel pages to become user
1209                 * accessible (the mmu gets confused and clears some of
1210                 * the ACC bits in kernel ptes).  Aha, sounds pretty
1211                 * horrible eh?  But wait, after extensive testing it appears
1212                 * that if you use pgd_t level large kernel pte's (like the
1213                 * 4MB pages on the Pentium) the bug does not get tripped
1214                 * at all.  This avoids almost all of the major overhead.
1215                 * Welcome to a world where your vendor tells you to,
1216                 * "apply this kernel patch" instead of "sorry for the
1217                 * broken hardware, send it back and we'll give you
1218                 * properly functioning parts"
1219                 */
1220                break;
1221        case 0x25:
1222        case 0x31:
1223                srmmu_modtype = Swift_bad_c;
1224                hwbug_bitmask |= HWBUG_KERN_CBITBROKEN;
1225                /*
1226                 * You see Sun allude to this hardware bug but never
1227                 * admit things directly, they'll say things like,
1228                 * "the Swift chip cache problems" or similar.
1229                 */
1230                break;
1231        default:
1232                srmmu_modtype = Swift_ok;
1233                break;
1234        }
1235
1236        sparc32_cachetlb_ops = &swift_ops;
1237        flush_page_for_dma_global = 0;
1238
1239        /*
1240         * Are you now convinced that the Swift is one of the
1241         * biggest VLSI abortions of all time?  Bravo Fujitsu!
1242         * Fujitsu, the !#?!%$'d up processor people.  I bet if
1243         * you examined the microcode of the Swift you'd find
1244         * XXX's all over the place.
1245         */
1246        poke_srmmu = poke_swift;
1247}
1248
1249static void turbosparc_flush_cache_all(void)
1250{
1251        flush_user_windows();
1252        turbosparc_idflash_clear();
1253}
1254
1255static void turbosparc_flush_cache_mm(struct mm_struct *mm)
1256{
1257        FLUSH_BEGIN(mm)
1258        flush_user_windows();
1259        turbosparc_idflash_clear();
1260        FLUSH_END
1261}
1262
1263static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1264{
1265        FLUSH_BEGIN(vma->vm_mm)
1266        flush_user_windows();
1267        turbosparc_idflash_clear();
1268        FLUSH_END
1269}
1270
1271static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1272{
1273        FLUSH_BEGIN(vma->vm_mm)
1274        flush_user_windows();
1275        if (vma->vm_flags & VM_EXEC)
1276                turbosparc_flush_icache();
1277        turbosparc_flush_dcache();
1278        FLUSH_END
1279}
1280
1281/* TurboSparc is copy-back, if we turn it on, but this does not work. */
1282static void turbosparc_flush_page_to_ram(unsigned long page)
1283{
1284#ifdef TURBOSPARC_WRITEBACK
1285        volatile unsigned long clear;
1286
1287        if (srmmu_probe(page))
1288                turbosparc_flush_page_cache(page);
1289        clear = srmmu_get_fstatus();
1290#endif
1291}
1292
1293static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1294{
1295}
1296
1297static void turbosparc_flush_page_for_dma(unsigned long page)
1298{
1299        turbosparc_flush_dcache();
1300}
1301
1302static void turbosparc_flush_tlb_all(void)
1303{
1304        srmmu_flush_whole_tlb();
1305}
1306
1307static void turbosparc_flush_tlb_mm(struct mm_struct *mm)
1308{
1309        FLUSH_BEGIN(mm)
1310        srmmu_flush_whole_tlb();
1311        FLUSH_END
1312}
1313
1314static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1315{
1316        FLUSH_BEGIN(vma->vm_mm)
1317        srmmu_flush_whole_tlb();
1318        FLUSH_END
1319}
1320
1321static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1322{
1323        FLUSH_BEGIN(vma->vm_mm)
1324        srmmu_flush_whole_tlb();
1325        FLUSH_END
1326}
1327
1328
1329static void poke_turbosparc(void)
1330{
1331        unsigned long mreg = srmmu_get_mmureg();
1332        unsigned long ccreg;
1333
1334        /* Clear any crap from the cache or else... */
1335        turbosparc_flush_cache_all();
1336        /* Temporarily disable I & D caches */
1337        mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE);
1338        mreg &= ~(TURBOSPARC_PCENABLE);         /* Don't check parity */
1339        srmmu_set_mmureg(mreg);
1340
1341        ccreg = turbosparc_get_ccreg();
1342
1343#ifdef TURBOSPARC_WRITEBACK
1344        ccreg |= (TURBOSPARC_SNENABLE);         /* Do DVMA snooping in Dcache */
1345        ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE);
1346                        /* Write-back D-cache, emulate VLSI
1347                         * abortion number three, not number one */
1348#else
1349        /* For now let's play safe, optimize later */
1350        ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE);
1351                        /* Do DVMA snooping in Dcache, Write-thru D-cache */
1352        ccreg &= ~(TURBOSPARC_uS2);
1353                        /* Emulate VLSI abortion number three, not number one */
1354#endif
1355
1356        switch (ccreg & 7) {
1357        case 0: /* No SE cache */
1358        case 7: /* Test mode */
1359                break;
1360        default:
1361                ccreg |= (TURBOSPARC_SCENABLE);
1362        }
1363        turbosparc_set_ccreg(ccreg);
1364
1365        mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
1366        mreg |= (TURBOSPARC_ICSNOOP);           /* Icache snooping on */
1367        srmmu_set_mmureg(mreg);
1368}
1369
1370static const struct sparc32_cachetlb_ops turbosparc_ops = {
1371        .cache_all      = turbosparc_flush_cache_all,
1372        .cache_mm       = turbosparc_flush_cache_mm,
1373        .cache_page     = turbosparc_flush_cache_page,
1374        .cache_range    = turbosparc_flush_cache_range,
1375        .tlb_all        = turbosparc_flush_tlb_all,
1376        .tlb_mm         = turbosparc_flush_tlb_mm,
1377        .tlb_page       = turbosparc_flush_tlb_page,
1378        .tlb_range      = turbosparc_flush_tlb_range,
1379        .page_to_ram    = turbosparc_flush_page_to_ram,
1380        .sig_insns      = turbosparc_flush_sig_insns,
1381        .page_for_dma   = turbosparc_flush_page_for_dma,
1382};
1383
1384static void __init init_turbosparc(void)
1385{
1386        srmmu_name = "Fujitsu TurboSparc";
1387        srmmu_modtype = TurboSparc;
1388        sparc32_cachetlb_ops = &turbosparc_ops;
1389        poke_srmmu = poke_turbosparc;
1390}
1391
1392static void poke_tsunami(void)
1393{
1394        unsigned long mreg = srmmu_get_mmureg();
1395
1396        tsunami_flush_icache();
1397        tsunami_flush_dcache();
1398        mreg &= ~TSUNAMI_ITD;
1399        mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB);
1400        srmmu_set_mmureg(mreg);
1401}
1402
1403static const struct sparc32_cachetlb_ops tsunami_ops = {
1404        .cache_all      = tsunami_flush_cache_all,
1405        .cache_mm       = tsunami_flush_cache_mm,
1406        .cache_page     = tsunami_flush_cache_page,
1407        .cache_range    = tsunami_flush_cache_range,
1408        .tlb_all        = tsunami_flush_tlb_all,
1409        .tlb_mm         = tsunami_flush_tlb_mm,
1410        .tlb_page       = tsunami_flush_tlb_page,
1411        .tlb_range      = tsunami_flush_tlb_range,
1412        .page_to_ram    = tsunami_flush_page_to_ram,
1413        .sig_insns      = tsunami_flush_sig_insns,
1414        .page_for_dma   = tsunami_flush_page_for_dma,
1415};
1416
1417static void __init init_tsunami(void)
1418{
1419        /*
1420         * Tsunami's pretty sane, Sun and TI actually got it
1421         * somewhat right this time.  Fujitsu should have
1422         * taken some lessons from them.
1423         */
1424
1425        srmmu_name = "TI Tsunami";
1426        srmmu_modtype = Tsunami;
1427        sparc32_cachetlb_ops = &tsunami_ops;
1428        poke_srmmu = poke_tsunami;
1429
1430        tsunami_setup_blockops();
1431}
1432
1433static void poke_viking(void)
1434{
1435        unsigned long mreg = srmmu_get_mmureg();
1436        static int smp_catch;
1437
1438        if (viking_mxcc_present) {
1439                unsigned long mxcc_control = mxcc_get_creg();
1440
1441                mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE);
1442                mxcc_control &= ~(MXCC_CTL_RRC);
1443                mxcc_set_creg(mxcc_control);
1444
1445                /*
1446                 * We don't need memory parity checks.
1447                 * XXX This is a mess, have to dig out later. ecd.
1448                viking_mxcc_turn_off_parity(&mreg, &mxcc_control);
1449                 */
1450
1451                /* We do cache ptables on MXCC. */
1452                mreg |= VIKING_TCENABLE;
1453        } else {
1454                unsigned long bpreg;
1455
1456                mreg &= ~(VIKING_TCENABLE);
1457                if (smp_catch++) {
1458                        /* Must disable mixed-cmd mode here for other cpu's. */
1459                        bpreg = viking_get_bpreg();
1460                        bpreg &= ~(VIKING_ACTION_MIX);
1461                        viking_set_bpreg(bpreg);
1462
1463                        /* Just in case PROM does something funny. */
1464                        msi_set_sync();
1465                }
1466        }
1467
1468        mreg |= VIKING_SPENABLE;
1469        mreg |= (VIKING_ICENABLE | VIKING_DCENABLE);
1470        mreg |= VIKING_SBENABLE;
1471        mreg &= ~(VIKING_ACENABLE);
1472        srmmu_set_mmureg(mreg);
1473}
1474
1475static struct sparc32_cachetlb_ops viking_ops __ro_after_init = {
1476        .cache_all      = viking_flush_cache_all,
1477        .cache_mm       = viking_flush_cache_mm,
1478        .cache_page     = viking_flush_cache_page,
1479        .cache_range    = viking_flush_cache_range,
1480        .tlb_all        = viking_flush_tlb_all,
1481        .tlb_mm         = viking_flush_tlb_mm,
1482        .tlb_page       = viking_flush_tlb_page,
1483        .tlb_range      = viking_flush_tlb_range,
1484        .page_to_ram    = viking_flush_page_to_ram,
1485        .sig_insns      = viking_flush_sig_insns,
1486        .page_for_dma   = viking_flush_page_for_dma,
1487};
1488
1489#ifdef CONFIG_SMP
1490/* On sun4d the cpu broadcasts local TLB flushes, so we can just
1491 * perform the local TLB flush and all the other cpus will see it.
1492 * But, unfortunately, there is a bug in the sun4d XBUS backplane
1493 * that requires that we add some synchronization to these flushes.
1494 *
1495 * The bug is that the fifo which keeps track of all the pending TLB
1496 * broadcasts in the system is an entry or two too small, so if we
1497 * have too many going at once we'll overflow that fifo and lose a TLB
1498 * flush resulting in corruption.
1499 *
1500 * Our workaround is to take a global spinlock around the TLB flushes,
1501 * which guarentees we won't ever have too many pending.  It's a big
1502 * hammer, but a semaphore like system to make sure we only have N TLB
1503 * flushes going at once will require SMP locking anyways so there's
1504 * no real value in trying any harder than this.
1505 */
1506static struct sparc32_cachetlb_ops viking_sun4d_smp_ops __ro_after_init = {
1507        .cache_all      = viking_flush_cache_all,
1508        .cache_mm       = viking_flush_cache_mm,
1509        .cache_page     = viking_flush_cache_page,
1510        .cache_range    = viking_flush_cache_range,
1511        .tlb_all        = sun4dsmp_flush_tlb_all,
1512        .tlb_mm         = sun4dsmp_flush_tlb_mm,
1513        .tlb_page       = sun4dsmp_flush_tlb_page,
1514        .tlb_range      = sun4dsmp_flush_tlb_range,
1515        .page_to_ram    = viking_flush_page_to_ram,
1516        .sig_insns      = viking_flush_sig_insns,
1517        .page_for_dma   = viking_flush_page_for_dma,
1518};
1519#endif
1520
1521static void __init init_viking(void)
1522{
1523        unsigned long mreg = srmmu_get_mmureg();
1524
1525        /* Ahhh, the viking.  SRMMU VLSI abortion number two... */
1526        if (mreg & VIKING_MMODE) {
1527                srmmu_name = "TI Viking";
1528                viking_mxcc_present = 0;
1529                msi_set_sync();
1530
1531                /*
1532                 * We need this to make sure old viking takes no hits
1533                 * on it's cache for dma snoops to workaround the
1534                 * "load from non-cacheable memory" interrupt bug.
1535                 * This is only necessary because of the new way in
1536                 * which we use the IOMMU.
1537                 */
1538                viking_ops.page_for_dma = viking_flush_page;
1539#ifdef CONFIG_SMP
1540                viking_sun4d_smp_ops.page_for_dma = viking_flush_page;
1541#endif
1542                flush_page_for_dma_global = 0;
1543        } else {
1544                srmmu_name = "TI Viking/MXCC";
1545                viking_mxcc_present = 1;
1546                srmmu_cache_pagetables = 1;
1547        }
1548
1549        sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1550                &viking_ops;
1551#ifdef CONFIG_SMP
1552        if (sparc_cpu_model == sun4d)
1553                sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1554                        &viking_sun4d_smp_ops;
1555#endif
1556
1557        poke_srmmu = poke_viking;
1558}
1559
1560/* Probe for the srmmu chip version. */
1561static void __init get_srmmu_type(void)
1562{
1563        unsigned long mreg, psr;
1564        unsigned long mod_typ, mod_rev, psr_typ, psr_vers;
1565
1566        srmmu_modtype = SRMMU_INVAL_MOD;
1567        hwbug_bitmask = 0;
1568
1569        mreg = srmmu_get_mmureg(); psr = get_psr();
1570        mod_typ = (mreg & 0xf0000000) >> 28;
1571        mod_rev = (mreg & 0x0f000000) >> 24;
1572        psr_typ = (psr >> 28) & 0xf;
1573        psr_vers = (psr >> 24) & 0xf;
1574
1575        /* First, check for sparc-leon. */
1576        if (sparc_cpu_model == sparc_leon) {
1577                init_leon();
1578                return;
1579        }
1580
1581        /* Second, check for HyperSparc or Cypress. */
1582        if (mod_typ == 1) {
1583                switch (mod_rev) {
1584                case 7:
1585                        /* UP or MP Hypersparc */
1586                        init_hypersparc();
1587                        break;
1588                case 0:
1589                case 2:
1590                case 10:
1591                case 11:
1592                case 12:
1593                case 13:
1594                case 14:
1595                case 15:
1596                default:
1597                        prom_printf("Sparc-Linux Cypress support does not longer exit.\n");
1598                        prom_halt();
1599                        break;
1600                }
1601                return;
1602        }
1603
1604        /* Now Fujitsu TurboSparc. It might happen that it is
1605         * in Swift emulation mode, so we will check later...
1606         */
1607        if (psr_typ == 0 && psr_vers == 5) {
1608                init_turbosparc();
1609                return;
1610        }
1611
1612        /* Next check for Fujitsu Swift. */
1613        if (psr_typ == 0 && psr_vers == 4) {
1614                phandle cpunode;
1615                char node_str[128];
1616
1617                /* Look if it is not a TurboSparc emulating Swift... */
1618                cpunode = prom_getchild(prom_root_node);
1619                while ((cpunode = prom_getsibling(cpunode)) != 0) {
1620                        prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1621                        if (!strcmp(node_str, "cpu")) {
1622                                if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
1623                                    prom_getintdefault(cpunode, "psr-version", 1) == 5) {
1624                                        init_turbosparc();
1625                                        return;
1626                                }
1627                                break;
1628                        }
1629                }
1630
1631                init_swift();
1632                return;
1633        }
1634
1635        /* Now the Viking family of srmmu. */
1636        if (psr_typ == 4 &&
1637           ((psr_vers == 0) ||
1638            ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
1639                init_viking();
1640                return;
1641        }
1642
1643        /* Finally the Tsunami. */
1644        if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
1645                init_tsunami();
1646                return;
1647        }
1648
1649        /* Oh well */
1650        srmmu_is_bad();
1651}
1652
1653#ifdef CONFIG_SMP
1654/* Local cross-calls. */
1655static void smp_flush_page_for_dma(unsigned long page)
1656{
1657        xc1((smpfunc_t) local_ops->page_for_dma, page);
1658        local_ops->page_for_dma(page);
1659}
1660
1661static void smp_flush_cache_all(void)
1662{
1663        xc0((smpfunc_t) local_ops->cache_all);
1664        local_ops->cache_all();
1665}
1666
1667static void smp_flush_tlb_all(void)
1668{
1669        xc0((smpfunc_t) local_ops->tlb_all);
1670        local_ops->tlb_all();
1671}
1672
1673static void smp_flush_cache_mm(struct mm_struct *mm)
1674{
1675        if (mm->context != NO_CONTEXT) {
1676                cpumask_t cpu_mask;
1677                cpumask_copy(&cpu_mask, mm_cpumask(mm));
1678                cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1679                if (!cpumask_empty(&cpu_mask))
1680                        xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm);
1681                local_ops->cache_mm(mm);
1682        }
1683}
1684
1685static void smp_flush_tlb_mm(struct mm_struct *mm)
1686{
1687        if (mm->context != NO_CONTEXT) {
1688                cpumask_t cpu_mask;
1689                cpumask_copy(&cpu_mask, mm_cpumask(mm));
1690                cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1691                if (!cpumask_empty(&cpu_mask)) {
1692                        xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm);
1693                        if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
1694                                cpumask_copy(mm_cpumask(mm),
1695                                             cpumask_of(smp_processor_id()));
1696                }
1697                local_ops->tlb_mm(mm);
1698        }
1699}
1700
1701static void smp_flush_cache_range(struct vm_area_struct *vma,
1702                                  unsigned long start,
1703                                  unsigned long end)
1704{
1705        struct mm_struct *mm = vma->vm_mm;
1706
1707        if (mm->context != NO_CONTEXT) {
1708                cpumask_t cpu_mask;
1709                cpumask_copy(&cpu_mask, mm_cpumask(mm));
1710                cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1711                if (!cpumask_empty(&cpu_mask))
1712                        xc3((smpfunc_t) local_ops->cache_range,
1713                            (unsigned long) vma, start, end);
1714                local_ops->cache_range(vma, start, end);
1715        }
1716}
1717
1718static void smp_flush_tlb_range(struct vm_area_struct *vma,
1719                                unsigned long start,
1720                                unsigned long end)
1721{
1722        struct mm_struct *mm = vma->vm_mm;
1723
1724        if (mm->context != NO_CONTEXT) {
1725                cpumask_t cpu_mask;
1726                cpumask_copy(&cpu_mask, mm_cpumask(mm));
1727                cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1728                if (!cpumask_empty(&cpu_mask))
1729                        xc3((smpfunc_t) local_ops->tlb_range,
1730                            (unsigned long) vma, start, end);
1731                local_ops->tlb_range(vma, start, end);
1732        }
1733}
1734
1735static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1736{
1737        struct mm_struct *mm = vma->vm_mm;
1738
1739        if (mm->context != NO_CONTEXT) {
1740                cpumask_t cpu_mask;
1741                cpumask_copy(&cpu_mask, mm_cpumask(mm));
1742                cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1743                if (!cpumask_empty(&cpu_mask))
1744                        xc2((smpfunc_t) local_ops->cache_page,
1745                            (unsigned long) vma, page);
1746                local_ops->cache_page(vma, page);
1747        }
1748}
1749
1750static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1751{
1752        struct mm_struct *mm = vma->vm_mm;
1753
1754        if (mm->context != NO_CONTEXT) {
1755                cpumask_t cpu_mask;
1756                cpumask_copy(&cpu_mask, mm_cpumask(mm));
1757                cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1758                if (!cpumask_empty(&cpu_mask))
1759                        xc2((smpfunc_t) local_ops->tlb_page,
1760                            (unsigned long) vma, page);
1761                local_ops->tlb_page(vma, page);
1762        }
1763}
1764
1765static void smp_flush_page_to_ram(unsigned long page)
1766{
1767        /* Current theory is that those who call this are the one's
1768         * who have just dirtied their cache with the pages contents
1769         * in kernel space, therefore we only run this on local cpu.
1770         *
1771         * XXX This experiment failed, research further... -DaveM
1772         */
1773#if 1
1774        xc1((smpfunc_t) local_ops->page_to_ram, page);
1775#endif
1776        local_ops->page_to_ram(page);
1777}
1778
1779static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1780{
1781        cpumask_t cpu_mask;
1782        cpumask_copy(&cpu_mask, mm_cpumask(mm));
1783        cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1784        if (!cpumask_empty(&cpu_mask))
1785                xc2((smpfunc_t) local_ops->sig_insns,
1786                    (unsigned long) mm, insn_addr);
1787        local_ops->sig_insns(mm, insn_addr);
1788}
1789
1790static struct sparc32_cachetlb_ops smp_cachetlb_ops __ro_after_init = {
1791        .cache_all      = smp_flush_cache_all,
1792        .cache_mm       = smp_flush_cache_mm,
1793        .cache_page     = smp_flush_cache_page,
1794        .cache_range    = smp_flush_cache_range,
1795        .tlb_all        = smp_flush_tlb_all,
1796        .tlb_mm         = smp_flush_tlb_mm,
1797        .tlb_page       = smp_flush_tlb_page,
1798        .tlb_range      = smp_flush_tlb_range,
1799        .page_to_ram    = smp_flush_page_to_ram,
1800        .sig_insns      = smp_flush_sig_insns,
1801        .page_for_dma   = smp_flush_page_for_dma,
1802};
1803#endif
1804
1805/* Load up routines and constants for sun4m and sun4d mmu */
1806void __init load_mmu(void)
1807{
1808        /* Functions */
1809        get_srmmu_type();
1810
1811#ifdef CONFIG_SMP
1812        /* El switcheroo... */
1813        local_ops = sparc32_cachetlb_ops;
1814
1815        if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) {
1816                smp_cachetlb_ops.tlb_all = local_ops->tlb_all;
1817                smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm;
1818                smp_cachetlb_ops.tlb_range = local_ops->tlb_range;
1819                smp_cachetlb_ops.tlb_page = local_ops->tlb_page;
1820        }
1821
1822        if (poke_srmmu == poke_viking) {
1823                /* Avoid unnecessary cross calls. */
1824                smp_cachetlb_ops.cache_all = local_ops->cache_all;
1825                smp_cachetlb_ops.cache_mm = local_ops->cache_mm;
1826                smp_cachetlb_ops.cache_range = local_ops->cache_range;
1827                smp_cachetlb_ops.cache_page = local_ops->cache_page;
1828
1829                smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram;
1830                smp_cachetlb_ops.sig_insns = local_ops->sig_insns;
1831                smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma;
1832        }
1833
1834        /* It really is const after this point. */
1835        sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1836                &smp_cachetlb_ops;
1837#endif
1838
1839        if (sparc_cpu_model == sun4d)
1840                ld_mmu_iounit();
1841        else
1842                ld_mmu_iommu();
1843#ifdef CONFIG_SMP
1844        if (sparc_cpu_model == sun4d)
1845                sun4d_init_smp();
1846        else if (sparc_cpu_model == sparc_leon)
1847                leon_init_smp();
1848        else
1849                sun4m_init_smp();
1850#endif
1851}
1852