linux/arch/powerpc/mm/tlb_nohash.c
<<
>>
Prefs
   1/*
   2 * This file contains the routines for TLB flushing.
   3 * On machines where the MMU does not use a hash table to store virtual to
   4 * physical translations (ie, SW loaded TLBs or Book3E compilant processors,
   5 * this does -not- include 603 however which shares the implementation with
   6 * hash based processors)
   7 *
   8 *  -- BenH
   9 *
  10 * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
  11 *                     IBM Corp.
  12 *
  13 *  Derived from arch/ppc/mm/init.c:
  14 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  15 *
  16 *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
  17 *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
  18 *    Copyright (C) 1996 Paul Mackerras
  19 *
  20 *  Derived from "arch/i386/mm/init.c"
  21 *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  22 *
  23 *  This program is free software; you can redistribute it and/or
  24 *  modify it under the terms of the GNU General Public License
  25 *  as published by the Free Software Foundation; either version
  26 *  2 of the License, or (at your option) any later version.
  27 *
  28 */
  29
  30#include <linux/kernel.h>
  31#include <linux/export.h>
  32#include <linux/mm.h>
  33#include <linux/init.h>
  34#include <linux/highmem.h>
  35#include <linux/pagemap.h>
  36#include <linux/preempt.h>
  37#include <linux/spinlock.h>
  38#include <linux/memblock.h>
  39#include <linux/of_fdt.h>
  40#include <linux/hugetlb.h>
  41
  42#include <asm/tlbflush.h>
  43#include <asm/tlb.h>
  44#include <asm/code-patching.h>
  45#include <asm/hugetlb.h>
  46
  47#include "mmu_decl.h"
  48
  49/*
  50 * This struct lists the sw-supported page sizes.  The hardawre MMU may support
  51 * other sizes not listed here.   The .ind field is only used on MMUs that have
  52 * indirect page table entries.
  53 */
  54#ifdef CONFIG_PPC_BOOK3E_MMU
  55#ifdef CONFIG_PPC_FSL_BOOK3E
  56struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
  57        [MMU_PAGE_4K] = {
  58                .shift  = 12,
  59                .enc    = BOOK3E_PAGESZ_4K,
  60        },
  61        [MMU_PAGE_4M] = {
  62                .shift  = 22,
  63                .enc    = BOOK3E_PAGESZ_4M,
  64        },
  65        [MMU_PAGE_16M] = {
  66                .shift  = 24,
  67                .enc    = BOOK3E_PAGESZ_16M,
  68        },
  69        [MMU_PAGE_64M] = {
  70                .shift  = 26,
  71                .enc    = BOOK3E_PAGESZ_64M,
  72        },
  73        [MMU_PAGE_256M] = {
  74                .shift  = 28,
  75                .enc    = BOOK3E_PAGESZ_256M,
  76        },
  77        [MMU_PAGE_1G] = {
  78                .shift  = 30,
  79                .enc    = BOOK3E_PAGESZ_1GB,
  80        },
  81};
  82#else
  83struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
  84        [MMU_PAGE_4K] = {
  85                .shift  = 12,
  86                .ind    = 20,
  87                .enc    = BOOK3E_PAGESZ_4K,
  88        },
  89        [MMU_PAGE_16K] = {
  90                .shift  = 14,
  91                .enc    = BOOK3E_PAGESZ_16K,
  92        },
  93        [MMU_PAGE_64K] = {
  94                .shift  = 16,
  95                .ind    = 28,
  96                .enc    = BOOK3E_PAGESZ_64K,
  97        },
  98        [MMU_PAGE_1M] = {
  99                .shift  = 20,
 100                .enc    = BOOK3E_PAGESZ_1M,
 101        },
 102        [MMU_PAGE_16M] = {
 103                .shift  = 24,
 104                .ind    = 36,
 105                .enc    = BOOK3E_PAGESZ_16M,
 106        },
 107        [MMU_PAGE_256M] = {
 108                .shift  = 28,
 109                .enc    = BOOK3E_PAGESZ_256M,
 110        },
 111        [MMU_PAGE_1G] = {
 112                .shift  = 30,
 113                .enc    = BOOK3E_PAGESZ_1GB,
 114        },
 115};
 116#endif /* CONFIG_FSL_BOOKE */
 117
 118static inline int mmu_get_tsize(int psize)
 119{
 120        return mmu_psize_defs[psize].enc;
 121}
 122#else
 123static inline int mmu_get_tsize(int psize)
 124{
 125        /* This isn't used on !Book3E for now */
 126        return 0;
 127}
 128#endif /* CONFIG_PPC_BOOK3E_MMU */
 129
 130/* The variables below are currently only used on 64-bit Book3E
 131 * though this will probably be made common with other nohash
 132 * implementations at some point
 133 */
 134#ifdef CONFIG_PPC64
 135
 136int mmu_linear_psize;           /* Page size used for the linear mapping */
 137int mmu_pte_psize;              /* Page size used for PTE pages */
 138int mmu_vmemmap_psize;          /* Page size used for the virtual mem map */
 139int book3e_htw_enabled;         /* Is HW tablewalk enabled ? */
 140unsigned long linear_map_top;   /* Top of linear mapping */
 141
 142#endif /* CONFIG_PPC64 */
 143
 144#ifdef CONFIG_PPC_FSL_BOOK3E
 145/* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
 146DEFINE_PER_CPU(int, next_tlbcam_idx);
 147EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
 148#endif
 149
 150/*
 151 * Base TLB flushing operations:
 152 *
 153 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 154 *  - flush_tlb_page(vma, vmaddr) flushes one page
 155 *  - flush_tlb_range(vma, start, end) flushes a range of pages
 156 *  - flush_tlb_kernel_range(start, end) flushes kernel pages
 157 *
 158 *  - local_* variants of page and mm only apply to the current
 159 *    processor
 160 */
 161
 162/*
 163 * These are the base non-SMP variants of page and mm flushing
 164 */
 165void local_flush_tlb_mm(struct mm_struct *mm)
 166{
 167        unsigned int pid;
 168
 169        preempt_disable();
 170        pid = mm->context.id;
 171        if (pid != MMU_NO_CONTEXT)
 172                _tlbil_pid(pid);
 173        preempt_enable();
 174}
 175EXPORT_SYMBOL(local_flush_tlb_mm);
 176
 177void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
 178                            int tsize, int ind)
 179{
 180        unsigned int pid;
 181
 182        preempt_disable();
 183        pid = mm ? mm->context.id : 0;
 184        if (pid != MMU_NO_CONTEXT)
 185                _tlbil_va(vmaddr, pid, tsize, ind);
 186        preempt_enable();
 187}
 188
 189void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 190{
 191        __local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
 192                               mmu_get_tsize(mmu_virtual_psize), 0);
 193}
 194EXPORT_SYMBOL(local_flush_tlb_page);
 195
 196/*
 197 * And here are the SMP non-local implementations
 198 */
 199#ifdef CONFIG_SMP
 200
 201static DEFINE_RAW_SPINLOCK(tlbivax_lock);
 202
 203static int mm_is_core_local(struct mm_struct *mm)
 204{
 205        return cpumask_subset(mm_cpumask(mm),
 206                              topology_thread_cpumask(smp_processor_id()));
 207}
 208
 209struct tlb_flush_param {
 210        unsigned long addr;
 211        unsigned int pid;
 212        unsigned int tsize;
 213        unsigned int ind;
 214};
 215
 216static void do_flush_tlb_mm_ipi(void *param)
 217{
 218        struct tlb_flush_param *p = param;
 219
 220        _tlbil_pid(p ? p->pid : 0);
 221}
 222
 223static void do_flush_tlb_page_ipi(void *param)
 224{
 225        struct tlb_flush_param *p = param;
 226
 227        _tlbil_va(p->addr, p->pid, p->tsize, p->ind);
 228}
 229
 230
 231/* Note on invalidations and PID:
 232 *
 233 * We snapshot the PID with preempt disabled. At this point, it can still
 234 * change either because:
 235 * - our context is being stolen (PID -> NO_CONTEXT) on another CPU
 236 * - we are invaliating some target that isn't currently running here
 237 *   and is concurrently acquiring a new PID on another CPU
 238 * - some other CPU is re-acquiring a lost PID for this mm
 239 * etc...
 240 *
 241 * However, this shouldn't be a problem as we only guarantee
 242 * invalidation of TLB entries present prior to this call, so we
 243 * don't care about the PID changing, and invalidating a stale PID
 244 * is generally harmless.
 245 */
 246
 247void flush_tlb_mm(struct mm_struct *mm)
 248{
 249        unsigned int pid;
 250
 251        preempt_disable();
 252        pid = mm->context.id;
 253        if (unlikely(pid == MMU_NO_CONTEXT))
 254                goto no_context;
 255        if (!mm_is_core_local(mm)) {
 256                struct tlb_flush_param p = { .pid = pid };
 257                /* Ignores smp_processor_id() even if set. */
 258                smp_call_function_many(mm_cpumask(mm),
 259                                       do_flush_tlb_mm_ipi, &p, 1);
 260        }
 261        _tlbil_pid(pid);
 262 no_context:
 263        preempt_enable();
 264}
 265EXPORT_SYMBOL(flush_tlb_mm);
 266
 267void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
 268                      int tsize, int ind)
 269{
 270        struct cpumask *cpu_mask;
 271        unsigned int pid;
 272
 273        preempt_disable();
 274        pid = mm ? mm->context.id : 0;
 275        if (unlikely(pid == MMU_NO_CONTEXT))
 276                goto bail;
 277        cpu_mask = mm_cpumask(mm);
 278        if (!mm_is_core_local(mm)) {
 279                /* If broadcast tlbivax is supported, use it */
 280                if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
 281                        int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
 282                        if (lock)
 283                                raw_spin_lock(&tlbivax_lock);
 284                        _tlbivax_bcast(vmaddr, pid, tsize, ind);
 285                        if (lock)
 286                                raw_spin_unlock(&tlbivax_lock);
 287                        goto bail;
 288                } else {
 289                        struct tlb_flush_param p = {
 290                                .pid = pid,
 291                                .addr = vmaddr,
 292                                .tsize = tsize,
 293                                .ind = ind,
 294                        };
 295                        /* Ignores smp_processor_id() even if set in cpu_mask */
 296                        smp_call_function_many(cpu_mask,
 297                                               do_flush_tlb_page_ipi, &p, 1);
 298                }
 299        }
 300        _tlbil_va(vmaddr, pid, tsize, ind);
 301 bail:
 302        preempt_enable();
 303}
 304
 305void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 306{
 307#ifdef CONFIG_HUGETLB_PAGE
 308        if (vma && is_vm_hugetlb_page(vma))
 309                flush_hugetlb_page(vma, vmaddr);
 310#endif
 311
 312        __flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
 313                         mmu_get_tsize(mmu_virtual_psize), 0);
 314}
 315EXPORT_SYMBOL(flush_tlb_page);
 316
 317#endif /* CONFIG_SMP */
 318
 319#ifdef CONFIG_PPC_47x
 320void __init early_init_mmu_47x(void)
 321{
 322#ifdef CONFIG_SMP
 323        unsigned long root = of_get_flat_dt_root();
 324        if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
 325                mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
 326#endif /* CONFIG_SMP */
 327}
 328#endif /* CONFIG_PPC_47x */
 329
 330/*
 331 * Flush kernel TLB entries in the given range
 332 */
 333void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 334{
 335#ifdef CONFIG_SMP
 336        preempt_disable();
 337        smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
 338        _tlbil_pid(0);
 339        preempt_enable();
 340#else
 341        _tlbil_pid(0);
 342#endif
 343}
 344EXPORT_SYMBOL(flush_tlb_kernel_range);
 345
 346/*
 347 * Currently, for range flushing, we just do a full mm flush. This should
 348 * be optimized based on a threshold on the size of the range, since
 349 * some implementation can stack multiple tlbivax before a tlbsync but
 350 * for now, we keep it that way
 351 */
 352void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 353                     unsigned long end)
 354
 355{
 356        flush_tlb_mm(vma->vm_mm);
 357}
 358EXPORT_SYMBOL(flush_tlb_range);
 359
 360void tlb_flush(struct mmu_gather *tlb)
 361{
 362        flush_tlb_mm(tlb->mm);
 363}
 364
 365/*
 366 * Below are functions specific to the 64-bit variant of Book3E though that
 367 * may change in the future
 368 */
 369
 370#ifdef CONFIG_PPC64
 371
 372/*
 373 * Handling of virtual linear page tables or indirect TLB entries
 374 * flushing when PTE pages are freed
 375 */
 376void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
 377{
 378        int tsize = mmu_psize_defs[mmu_pte_psize].enc;
 379
 380        if (book3e_htw_enabled) {
 381                unsigned long start = address & PMD_MASK;
 382                unsigned long end = address + PMD_SIZE;
 383                unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;
 384
 385                /* This isn't the most optimal, ideally we would factor out the
 386                 * while preempt & CPU mask mucking around, or even the IPI but
 387                 * it will do for now
 388                 */
 389                while (start < end) {
 390                        __flush_tlb_page(tlb->mm, start, tsize, 1);
 391                        start += size;
 392                }
 393        } else {
 394                unsigned long rmask = 0xf000000000000000ul;
 395                unsigned long rid = (address & rmask) | 0x1000000000000000ul;
 396                unsigned long vpte = address & ~rmask;
 397
 398#ifdef CONFIG_PPC_64K_PAGES
 399                vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
 400#else
 401                vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
 402#endif
 403                vpte |= rid;
 404                __flush_tlb_page(tlb->mm, vpte, tsize, 0);
 405        }
 406}
 407
 408static void setup_page_sizes(void)
 409{
 410        unsigned int tlb0cfg;
 411        unsigned int tlb0ps;
 412        unsigned int eptcfg;
 413        int i, psize;
 414
 415#ifdef CONFIG_PPC_FSL_BOOK3E
 416        unsigned int mmucfg = mfspr(SPRN_MMUCFG);
 417        int fsl_mmu = mmu_has_feature(MMU_FTR_TYPE_FSL_E);
 418
 419        if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) {
 420                unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
 421                unsigned int min_pg, max_pg;
 422
 423                min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
 424                max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;
 425
 426                for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 427                        struct mmu_psize_def *def;
 428                        unsigned int shift;
 429
 430                        def = &mmu_psize_defs[psize];
 431                        shift = def->shift;
 432
 433                        if (shift == 0)
 434                                continue;
 435
 436                        /* adjust to be in terms of 4^shift Kb */
 437                        shift = (shift - 10) >> 1;
 438
 439                        if ((shift >= min_pg) && (shift <= max_pg))
 440                                def->flags |= MMU_PAGE_SIZE_DIRECT;
 441                }
 442
 443                goto no_indirect;
 444        }
 445
 446        if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V2) {
 447                u32 tlb1ps = mfspr(SPRN_TLB1PS);
 448
 449                for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 450                        struct mmu_psize_def *def = &mmu_psize_defs[psize];
 451
 452                        if (tlb1ps & (1U << (def->shift - 10))) {
 453                                def->flags |= MMU_PAGE_SIZE_DIRECT;
 454                        }
 455                }
 456
 457                goto no_indirect;
 458        }
 459#endif
 460
 461        tlb0cfg = mfspr(SPRN_TLB0CFG);
 462        tlb0ps = mfspr(SPRN_TLB0PS);
 463        eptcfg = mfspr(SPRN_EPTCFG);
 464
 465        /* Look for supported direct sizes */
 466        for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 467                struct mmu_psize_def *def = &mmu_psize_defs[psize];
 468
 469                if (tlb0ps & (1U << (def->shift - 10)))
 470                        def->flags |= MMU_PAGE_SIZE_DIRECT;
 471        }
 472
 473        /* Indirect page sizes supported ? */
 474        if ((tlb0cfg & TLBnCFG_IND) == 0)
 475                goto no_indirect;
 476
 477        /* Now, we only deal with one IND page size for each
 478         * direct size. Hopefully all implementations today are
 479         * unambiguous, but we might want to be careful in the
 480         * future.
 481         */
 482        for (i = 0; i < 3; i++) {
 483                unsigned int ps, sps;
 484
 485                sps = eptcfg & 0x1f;
 486                eptcfg >>= 5;
 487                ps = eptcfg & 0x1f;
 488                eptcfg >>= 5;
 489                if (!ps || !sps)
 490                        continue;
 491                for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
 492                        struct mmu_psize_def *def = &mmu_psize_defs[psize];
 493
 494                        if (ps == (def->shift - 10))
 495                                def->flags |= MMU_PAGE_SIZE_INDIRECT;
 496                        if (sps == (def->shift - 10))
 497                                def->ind = ps + 10;
 498                }
 499        }
 500 no_indirect:
 501
 502        /* Cleanup array and print summary */
 503        pr_info("MMU: Supported page sizes\n");
 504        for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 505                struct mmu_psize_def *def = &mmu_psize_defs[psize];
 506                const char *__page_type_names[] = {
 507                        "unsupported",
 508                        "direct",
 509                        "indirect",
 510                        "direct & indirect"
 511                };
 512                if (def->flags == 0) {
 513                        def->shift = 0; 
 514                        continue;
 515                }
 516                pr_info("  %8ld KB as %s\n", 1ul << (def->shift - 10),
 517                        __page_type_names[def->flags & 0x3]);
 518        }
 519}
 520
 521static void __patch_exception(int exc, unsigned long addr)
 522{
 523        extern unsigned int interrupt_base_book3e;
 524        unsigned int *ibase = &interrupt_base_book3e;
 525 
 526        /* Our exceptions vectors start with a NOP and -then- a branch
 527         * to deal with single stepping from userspace which stops on
 528         * the second instruction. Thus we need to patch the second
 529         * instruction of the exception, not the first one
 530         */
 531
 532        patch_branch(ibase + (exc / 4) + 1, addr, 0);
 533}
 534
 535#define patch_exception(exc, name) do { \
 536        extern unsigned int name; \
 537        __patch_exception((exc), (unsigned long)&name); \
 538} while (0)
 539
 540static void setup_mmu_htw(void)
 541{
 542        /* Check if HW tablewalk is present, and if yes, enable it by:
 543         *
 544         * - patching the TLB miss handlers to branch to the
 545         *   one dedicates to it
 546         *
 547         * - setting the global book3e_htw_enabled
 548         */
 549        unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG);
 550
 551        if ((tlb0cfg & TLBnCFG_IND) &&
 552            (tlb0cfg & TLBnCFG_PT)) {
 553                patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
 554                patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
 555                book3e_htw_enabled = 1;
 556        }
 557        pr_info("MMU: Book3E HW tablewalk %s\n",
 558                book3e_htw_enabled ? "enabled" : "not supported");
 559}
 560
 561/*
 562 * Early initialization of the MMU TLB code
 563 */
 564static void __early_init_mmu(int boot_cpu)
 565{
 566        unsigned int mas4;
 567
 568        /* XXX This will have to be decided at runtime, but right
 569         * now our boot and TLB miss code hard wires it. Ideally
 570         * we should find out a suitable page size and patch the
 571         * TLB miss code (either that or use the PACA to store
 572         * the value we want)
 573         */
 574        mmu_linear_psize = MMU_PAGE_1G;
 575
 576        /* XXX This should be decided at runtime based on supported
 577         * page sizes in the TLB, but for now let's assume 16M is
 578         * always there and a good fit (which it probably is)
 579         */
 580        mmu_vmemmap_psize = MMU_PAGE_16M;
 581
 582        /* XXX This code only checks for TLB 0 capabilities and doesn't
 583         *     check what page size combos are supported by the HW. It
 584         *     also doesn't handle the case where a separate array holds
 585         *     the IND entries from the array loaded by the PT.
 586         */
 587        if (boot_cpu) {
 588                /* Look for supported page sizes */
 589                setup_page_sizes();
 590
 591                /* Look for HW tablewalk support */
 592                setup_mmu_htw();
 593        }
 594
 595        /* Set MAS4 based on page table setting */
 596
 597        mas4 = 0x4 << MAS4_WIMGED_SHIFT;
 598        if (book3e_htw_enabled) {
 599                mas4 |= mas4 | MAS4_INDD;
 600#ifdef CONFIG_PPC_64K_PAGES
 601                mas4 |= BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
 602                mmu_pte_psize = MMU_PAGE_256M;
 603#else
 604                mas4 |= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
 605                mmu_pte_psize = MMU_PAGE_1M;
 606#endif
 607        } else {
 608#ifdef CONFIG_PPC_64K_PAGES
 609                mas4 |= BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
 610#else
 611                mas4 |= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
 612#endif
 613                mmu_pte_psize = mmu_virtual_psize;
 614        }
 615        mtspr(SPRN_MAS4, mas4);
 616
 617        /* Set the global containing the top of the linear mapping
 618         * for use by the TLB miss code
 619         */
 620        linear_map_top = memblock_end_of_DRAM();
 621
 622#ifdef CONFIG_PPC_FSL_BOOK3E
 623        if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
 624                unsigned int num_cams;
 625
 626                /* use a quarter of the TLBCAM for bolted linear map */
 627                num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
 628                linear_map_top = map_mem_in_cams(linear_map_top, num_cams);
 629
 630                /* limit memory so we dont have linear faults */
 631                memblock_enforce_memory_limit(linear_map_top);
 632
 633                patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
 634                patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e);
 635        }
 636#endif
 637
 638        /* A sync won't hurt us after mucking around with
 639         * the MMU configuration
 640         */
 641        mb();
 642
 643        memblock_set_current_limit(linear_map_top);
 644}
 645
 646void __init early_init_mmu(void)
 647{
 648        __early_init_mmu(1);
 649}
 650
 651void early_init_mmu_secondary(void)
 652{
 653        __early_init_mmu(0);
 654}
 655
 656void setup_initial_memory_limit(phys_addr_t first_memblock_base,
 657                                phys_addr_t first_memblock_size)
 658{
 659        /* On non-FSL Embedded 64-bit, we adjust the RMA size to match
 660         * the bolted TLB entry. We know for now that only 1G
 661         * entries are supported though that may eventually
 662         * change.
 663         *
 664         * on FSL Embedded 64-bit, we adjust the RMA size to match the
 665         * first bolted TLB entry size.  We still limit max to 1G even if
 666         * the TLB could cover more.  This is due to what the early init
 667         * code is setup to do.
 668         *
 669         * We crop it to the size of the first MEMBLOCK to
 670         * avoid going over total available memory just in case...
 671         */
 672#ifdef CONFIG_PPC_FSL_BOOK3E
 673        if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
 674                unsigned long linear_sz;
 675                linear_sz = calc_cam_sz(first_memblock_size, PAGE_OFFSET,
 676                                        first_memblock_base);
 677                ppc64_rma_size = min_t(u64, linear_sz, 0x40000000);
 678        } else
 679#endif
 680                ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
 681
 682        /* Finally limit subsequent allocations */
 683        memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
 684}
 685#else /* ! CONFIG_PPC64 */
 686void __init early_init_mmu(void)
 687{
 688#ifdef CONFIG_PPC_47x
 689        early_init_mmu_47x();
 690#endif
 691}
 692#endif /* CONFIG_PPC64 */
 693