linux/arch/powerpc/mm/book3s64/radix_pgtable.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Page table handling routines for radix page table.
   4 *
   5 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
   6 */
   7
   8#define pr_fmt(fmt) "radix-mmu: " fmt
   9
  10#include <linux/io.h>
  11#include <linux/kernel.h>
  12#include <linux/sched/mm.h>
  13#include <linux/memblock.h>
  14#include <linux/of.h>
  15#include <linux/of_fdt.h>
  16#include <linux/mm.h>
  17#include <linux/hugetlb.h>
  18#include <linux/string_helpers.h>
  19#include <linux/memory.h>
  20
  21#include <asm/pgalloc.h>
  22#include <asm/mmu_context.h>
  23#include <asm/dma.h>
  24#include <asm/machdep.h>
  25#include <asm/mmu.h>
  26#include <asm/firmware.h>
  27#include <asm/powernv.h>
  28#include <asm/sections.h>
  29#include <asm/smp.h>
  30#include <asm/trace.h>
  31#include <asm/uaccess.h>
  32#include <asm/ultravisor.h>
  33
  34#include <trace/events/thp.h>
  35
  36unsigned int mmu_base_pid;
  37unsigned long radix_mem_block_size __ro_after_init;
  38
  39static __ref void *early_alloc_pgtable(unsigned long size, int nid,
  40                        unsigned long region_start, unsigned long region_end)
  41{
  42        phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
  43        phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
  44        void *ptr;
  45
  46        if (region_start)
  47                min_addr = region_start;
  48        if (region_end)
  49                max_addr = region_end;
  50
  51        ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
  52
  53        if (!ptr)
  54                panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
  55                      __func__, size, size, nid, &min_addr, &max_addr);
  56
  57        return ptr;
  58}
  59
  60/*
  61 * When allocating pud or pmd pointers, we allocate a complete page
  62 * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
  63 * is to ensure that the page obtained from the memblock allocator
  64 * can be completely used as page table page and can be freed
  65 * correctly when the page table entries are removed.
  66 */
  67static int early_map_kernel_page(unsigned long ea, unsigned long pa,
  68                          pgprot_t flags,
  69                          unsigned int map_page_size,
  70                          int nid,
  71                          unsigned long region_start, unsigned long region_end)
  72{
  73        unsigned long pfn = pa >> PAGE_SHIFT;
  74        pgd_t *pgdp;
  75        p4d_t *p4dp;
  76        pud_t *pudp;
  77        pmd_t *pmdp;
  78        pte_t *ptep;
  79
  80        pgdp = pgd_offset_k(ea);
  81        p4dp = p4d_offset(pgdp, ea);
  82        if (p4d_none(*p4dp)) {
  83                pudp = early_alloc_pgtable(PAGE_SIZE, nid,
  84                                           region_start, region_end);
  85                p4d_populate(&init_mm, p4dp, pudp);
  86        }
  87        pudp = pud_offset(p4dp, ea);
  88        if (map_page_size == PUD_SIZE) {
  89                ptep = (pte_t *)pudp;
  90                goto set_the_pte;
  91        }
  92        if (pud_none(*pudp)) {
  93                pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
  94                                           region_end);
  95                pud_populate(&init_mm, pudp, pmdp);
  96        }
  97        pmdp = pmd_offset(pudp, ea);
  98        if (map_page_size == PMD_SIZE) {
  99                ptep = pmdp_ptep(pmdp);
 100                goto set_the_pte;
 101        }
 102        if (!pmd_present(*pmdp)) {
 103                ptep = early_alloc_pgtable(PAGE_SIZE, nid,
 104                                                region_start, region_end);
 105                pmd_populate_kernel(&init_mm, pmdp, ptep);
 106        }
 107        ptep = pte_offset_kernel(pmdp, ea);
 108
 109set_the_pte:
 110        set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 111        asm volatile("ptesync": : :"memory");
 112        return 0;
 113}
 114
 115/*
 116 * nid, region_start, and region_end are hints to try to place the page
 117 * table memory in the same node or region.
 118 */
 119static int __map_kernel_page(unsigned long ea, unsigned long pa,
 120                          pgprot_t flags,
 121                          unsigned int map_page_size,
 122                          int nid,
 123                          unsigned long region_start, unsigned long region_end)
 124{
 125        unsigned long pfn = pa >> PAGE_SHIFT;
 126        pgd_t *pgdp;
 127        p4d_t *p4dp;
 128        pud_t *pudp;
 129        pmd_t *pmdp;
 130        pte_t *ptep;
 131        /*
 132         * Make sure task size is correct as per the max adddr
 133         */
 134        BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
 135
 136#ifdef CONFIG_PPC_64K_PAGES
 137        BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
 138#endif
 139
 140        if (unlikely(!slab_is_available()))
 141                return early_map_kernel_page(ea, pa, flags, map_page_size,
 142                                                nid, region_start, region_end);
 143
 144        /*
 145         * Should make page table allocation functions be able to take a
 146         * node, so we can place kernel page tables on the right nodes after
 147         * boot.
 148         */
 149        pgdp = pgd_offset_k(ea);
 150        p4dp = p4d_offset(pgdp, ea);
 151        pudp = pud_alloc(&init_mm, p4dp, ea);
 152        if (!pudp)
 153                return -ENOMEM;
 154        if (map_page_size == PUD_SIZE) {
 155                ptep = (pte_t *)pudp;
 156                goto set_the_pte;
 157        }
 158        pmdp = pmd_alloc(&init_mm, pudp, ea);
 159        if (!pmdp)
 160                return -ENOMEM;
 161        if (map_page_size == PMD_SIZE) {
 162                ptep = pmdp_ptep(pmdp);
 163                goto set_the_pte;
 164        }
 165        ptep = pte_alloc_kernel(pmdp, ea);
 166        if (!ptep)
 167                return -ENOMEM;
 168
 169set_the_pte:
 170        set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 171        asm volatile("ptesync": : :"memory");
 172        return 0;
 173}
 174
 175int radix__map_kernel_page(unsigned long ea, unsigned long pa,
 176                          pgprot_t flags,
 177                          unsigned int map_page_size)
 178{
 179        return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
 180}
 181
 182#ifdef CONFIG_STRICT_KERNEL_RWX
 183static void radix__change_memory_range(unsigned long start, unsigned long end,
 184                                       unsigned long clear)
 185{
 186        unsigned long idx;
 187        pgd_t *pgdp;
 188        p4d_t *p4dp;
 189        pud_t *pudp;
 190        pmd_t *pmdp;
 191        pte_t *ptep;
 192
 193        start = ALIGN_DOWN(start, PAGE_SIZE);
 194        end = PAGE_ALIGN(end); // aligns up
 195
 196        pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
 197                 start, end, clear);
 198
 199        for (idx = start; idx < end; idx += PAGE_SIZE) {
 200                pgdp = pgd_offset_k(idx);
 201                p4dp = p4d_offset(pgdp, idx);
 202                pudp = pud_alloc(&init_mm, p4dp, idx);
 203                if (!pudp)
 204                        continue;
 205                if (pud_is_leaf(*pudp)) {
 206                        ptep = (pte_t *)pudp;
 207                        goto update_the_pte;
 208                }
 209                pmdp = pmd_alloc(&init_mm, pudp, idx);
 210                if (!pmdp)
 211                        continue;
 212                if (pmd_is_leaf(*pmdp)) {
 213                        ptep = pmdp_ptep(pmdp);
 214                        goto update_the_pte;
 215                }
 216                ptep = pte_alloc_kernel(pmdp, idx);
 217                if (!ptep)
 218                        continue;
 219update_the_pte:
 220                radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
 221        }
 222
 223        radix__flush_tlb_kernel_range(start, end);
 224}
 225
 226void radix__mark_rodata_ro(void)
 227{
 228        unsigned long start, end;
 229
 230        start = (unsigned long)_stext;
 231        end = (unsigned long)__init_begin;
 232
 233        radix__change_memory_range(start, end, _PAGE_WRITE);
 234}
 235
 236void radix__mark_initmem_nx(void)
 237{
 238        unsigned long start = (unsigned long)__init_begin;
 239        unsigned long end = (unsigned long)__init_end;
 240
 241        radix__change_memory_range(start, end, _PAGE_EXEC);
 242}
 243#endif /* CONFIG_STRICT_KERNEL_RWX */
 244
 245static inline void __meminit
 246print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
 247{
 248        char buf[10];
 249
 250        if (end <= start)
 251                return;
 252
 253        string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
 254
 255        pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
 256                exec ? " (exec)" : "");
 257}
 258
 259static unsigned long next_boundary(unsigned long addr, unsigned long end)
 260{
 261#ifdef CONFIG_STRICT_KERNEL_RWX
 262        if (addr < __pa_symbol(__init_begin))
 263                return __pa_symbol(__init_begin);
 264#endif
 265        return end;
 266}
 267
 268static int __meminit create_physical_mapping(unsigned long start,
 269                                             unsigned long end,
 270                                             unsigned long max_mapping_size,
 271                                             int nid, pgprot_t _prot)
 272{
 273        unsigned long vaddr, addr, mapping_size = 0;
 274        bool prev_exec, exec = false;
 275        pgprot_t prot;
 276        int psize;
 277
 278        start = ALIGN(start, PAGE_SIZE);
 279        end   = ALIGN_DOWN(end, PAGE_SIZE);
 280        for (addr = start; addr < end; addr += mapping_size) {
 281                unsigned long gap, previous_size;
 282                int rc;
 283
 284                gap = next_boundary(addr, end) - addr;
 285                if (gap > max_mapping_size)
 286                        gap = max_mapping_size;
 287                previous_size = mapping_size;
 288                prev_exec = exec;
 289
 290                if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
 291                    mmu_psize_defs[MMU_PAGE_1G].shift) {
 292                        mapping_size = PUD_SIZE;
 293                        psize = MMU_PAGE_1G;
 294                } else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
 295                           mmu_psize_defs[MMU_PAGE_2M].shift) {
 296                        mapping_size = PMD_SIZE;
 297                        psize = MMU_PAGE_2M;
 298                } else {
 299                        mapping_size = PAGE_SIZE;
 300                        psize = mmu_virtual_psize;
 301                }
 302
 303                vaddr = (unsigned long)__va(addr);
 304
 305                if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
 306                    overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
 307                        prot = PAGE_KERNEL_X;
 308                        exec = true;
 309                } else {
 310                        prot = _prot;
 311                        exec = false;
 312                }
 313
 314                if (mapping_size != previous_size || exec != prev_exec) {
 315                        print_mapping(start, addr, previous_size, prev_exec);
 316                        start = addr;
 317                }
 318
 319                rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
 320                if (rc)
 321                        return rc;
 322
 323                update_page_count(psize, 1);
 324        }
 325
 326        print_mapping(start, addr, mapping_size, exec);
 327        return 0;
 328}
 329
 330static void __init radix_init_pgtable(void)
 331{
 332        unsigned long rts_field;
 333        phys_addr_t start, end;
 334        u64 i;
 335
 336        /* We don't support slb for radix */
 337        slb_set_size(0);
 338
 339        /*
 340         * Create the linear mapping
 341         */
 342        for_each_mem_range(i, &start, &end) {
 343                /*
 344                 * The memblock allocator  is up at this point, so the
 345                 * page tables will be allocated within the range. No
 346                 * need or a node (which we don't have yet).
 347                 */
 348
 349                if (end >= RADIX_VMALLOC_START) {
 350                        pr_warn("Outside the supported range\n");
 351                        continue;
 352                }
 353
 354                WARN_ON(create_physical_mapping(start, end,
 355                                                radix_mem_block_size,
 356                                                -1, PAGE_KERNEL));
 357        }
 358
 359        if (!cpu_has_feature(CPU_FTR_HVMODE) &&
 360                        cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
 361                /*
 362                 * Older versions of KVM on these machines perfer if the
 363                 * guest only uses the low 19 PID bits.
 364                 */
 365                mmu_pid_bits = 19;
 366        }
 367        mmu_base_pid = 1;
 368
 369        /*
 370         * Allocate Partition table and process table for the
 371         * host.
 372         */
 373        BUG_ON(PRTB_SIZE_SHIFT > 36);
 374        process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
 375        /*
 376         * Fill in the process table.
 377         */
 378        rts_field = radix__get_tree_size();
 379        process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
 380
 381        /*
 382         * The init_mm context is given the first available (non-zero) PID,
 383         * which is the "guard PID" and contains no page table. PIDR should
 384         * never be set to zero because that duplicates the kernel address
 385         * space at the 0x0... offset (quadrant 0)!
 386         *
 387         * An arbitrary PID that may later be allocated by the PID allocator
 388         * for userspace processes must not be used either, because that
 389         * would cause stale user mappings for that PID on CPUs outside of
 390         * the TLB invalidation scheme (because it won't be in mm_cpumask).
 391         *
 392         * So permanently carve out one PID for the purpose of a guard PID.
 393         */
 394        init_mm.context.id = mmu_base_pid;
 395        mmu_base_pid++;
 396}
 397
 398static void __init radix_init_partition_table(void)
 399{
 400        unsigned long rts_field, dw0, dw1;
 401
 402        mmu_partition_table_init();
 403        rts_field = radix__get_tree_size();
 404        dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
 405        dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
 406        mmu_partition_table_set_entry(0, dw0, dw1, false);
 407
 408        pr_info("Initializing Radix MMU\n");
 409}
 410
 411static int __init get_idx_from_shift(unsigned int shift)
 412{
 413        int idx = -1;
 414
 415        switch (shift) {
 416        case 0xc:
 417                idx = MMU_PAGE_4K;
 418                break;
 419        case 0x10:
 420                idx = MMU_PAGE_64K;
 421                break;
 422        case 0x15:
 423                idx = MMU_PAGE_2M;
 424                break;
 425        case 0x1e:
 426                idx = MMU_PAGE_1G;
 427                break;
 428        }
 429        return idx;
 430}
 431
 432static int __init radix_dt_scan_page_sizes(unsigned long node,
 433                                           const char *uname, int depth,
 434                                           void *data)
 435{
 436        int size = 0;
 437        int shift, idx;
 438        unsigned int ap;
 439        const __be32 *prop;
 440        const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 441
 442        /* We are scanning "cpu" nodes only */
 443        if (type == NULL || strcmp(type, "cpu") != 0)
 444                return 0;
 445
 446        /* Grab page size encodings */
 447        prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
 448        if (!prop)
 449                return 0;
 450
 451        pr_info("Page sizes from device-tree:\n");
 452        for (; size >= 4; size -= 4, ++prop) {
 453
 454                struct mmu_psize_def *def;
 455
 456                /* top 3 bit is AP encoding */
 457                shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
 458                ap = be32_to_cpu(prop[0]) >> 29;
 459                pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
 460
 461                idx = get_idx_from_shift(shift);
 462                if (idx < 0)
 463                        continue;
 464
 465                def = &mmu_psize_defs[idx];
 466                def->shift = shift;
 467                def->ap  = ap;
 468                def->h_rpt_pgsize = psize_to_rpti_pgsize(idx);
 469        }
 470
 471        /* needed ? */
 472        cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
 473        return 1;
 474}
 475
 476#ifdef CONFIG_MEMORY_HOTPLUG
 477static int __init probe_memory_block_size(unsigned long node, const char *uname, int
 478                                          depth, void *data)
 479{
 480        unsigned long *mem_block_size = (unsigned long *)data;
 481        const __be32 *prop;
 482        int len;
 483
 484        if (depth != 1)
 485                return 0;
 486
 487        if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
 488                return 0;
 489
 490        prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
 491
 492        if (!prop || len < dt_root_size_cells * sizeof(__be32))
 493                /*
 494                 * Nothing in the device tree
 495                 */
 496                *mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 497        else
 498                *mem_block_size = of_read_number(prop, dt_root_size_cells);
 499        return 1;
 500}
 501
 502static unsigned long __init radix_memory_block_size(void)
 503{
 504        unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 505
 506        /*
 507         * OPAL firmware feature is set by now. Hence we are ok
 508         * to test OPAL feature.
 509         */
 510        if (firmware_has_feature(FW_FEATURE_OPAL))
 511                mem_block_size = 1UL * 1024 * 1024 * 1024;
 512        else
 513                of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
 514
 515        return mem_block_size;
 516}
 517
 518#else   /* CONFIG_MEMORY_HOTPLUG */
 519
 520static unsigned long __init radix_memory_block_size(void)
 521{
 522        return 1UL * 1024 * 1024 * 1024;
 523}
 524
 525#endif /* CONFIG_MEMORY_HOTPLUG */
 526
 527
 528void __init radix__early_init_devtree(void)
 529{
 530        int rc;
 531
 532        /*
 533         * Try to find the available page sizes in the device-tree
 534         */
 535        rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
 536        if (!rc) {
 537                /*
 538                 * No page size details found in device tree.
 539                 * Let's assume we have page 4k and 64k support
 540                 */
 541                mmu_psize_defs[MMU_PAGE_4K].shift = 12;
 542                mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
 543                mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize =
 544                        psize_to_rpti_pgsize(MMU_PAGE_4K);
 545
 546                mmu_psize_defs[MMU_PAGE_64K].shift = 16;
 547                mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
 548                mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize =
 549                        psize_to_rpti_pgsize(MMU_PAGE_64K);
 550        }
 551
 552        /*
 553         * Max mapping size used when mapping pages. We don't use
 554         * ppc_md.memory_block_size() here because this get called
 555         * early and we don't have machine probe called yet. Also
 556         * the pseries implementation only check for ibm,lmb-size.
 557         * All hypervisor supporting radix do expose that device
 558         * tree node.
 559         */
 560        radix_mem_block_size = radix_memory_block_size();
 561        return;
 562}
 563
 564void __init radix__early_init_mmu(void)
 565{
 566        unsigned long lpcr;
 567
 568#ifdef CONFIG_PPC_64S_HASH_MMU
 569#ifdef CONFIG_PPC_64K_PAGES
 570        /* PAGE_SIZE mappings */
 571        mmu_virtual_psize = MMU_PAGE_64K;
 572#else
 573        mmu_virtual_psize = MMU_PAGE_4K;
 574#endif
 575
 576#ifdef CONFIG_SPARSEMEM_VMEMMAP
 577        /* vmemmap mapping */
 578        if (mmu_psize_defs[MMU_PAGE_2M].shift) {
 579                /*
 580                 * map vmemmap using 2M if available
 581                 */
 582                mmu_vmemmap_psize = MMU_PAGE_2M;
 583        } else
 584                mmu_vmemmap_psize = mmu_virtual_psize;
 585#endif
 586#endif
 587        /*
 588         * initialize page table size
 589         */
 590        __pte_index_size = RADIX_PTE_INDEX_SIZE;
 591        __pmd_index_size = RADIX_PMD_INDEX_SIZE;
 592        __pud_index_size = RADIX_PUD_INDEX_SIZE;
 593        __pgd_index_size = RADIX_PGD_INDEX_SIZE;
 594        __pud_cache_index = RADIX_PUD_INDEX_SIZE;
 595        __pte_table_size = RADIX_PTE_TABLE_SIZE;
 596        __pmd_table_size = RADIX_PMD_TABLE_SIZE;
 597        __pud_table_size = RADIX_PUD_TABLE_SIZE;
 598        __pgd_table_size = RADIX_PGD_TABLE_SIZE;
 599
 600        __pmd_val_bits = RADIX_PMD_VAL_BITS;
 601        __pud_val_bits = RADIX_PUD_VAL_BITS;
 602        __pgd_val_bits = RADIX_PGD_VAL_BITS;
 603
 604        __kernel_virt_start = RADIX_KERN_VIRT_START;
 605        __vmalloc_start = RADIX_VMALLOC_START;
 606        __vmalloc_end = RADIX_VMALLOC_END;
 607        __kernel_io_start = RADIX_KERN_IO_START;
 608        __kernel_io_end = RADIX_KERN_IO_END;
 609        vmemmap = (struct page *)RADIX_VMEMMAP_START;
 610        ioremap_bot = IOREMAP_BASE;
 611
 612#ifdef CONFIG_PCI
 613        pci_io_base = ISA_IO_BASE;
 614#endif
 615        __pte_frag_nr = RADIX_PTE_FRAG_NR;
 616        __pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
 617        __pmd_frag_nr = RADIX_PMD_FRAG_NR;
 618        __pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
 619
 620        radix_init_pgtable();
 621
 622        if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 623                lpcr = mfspr(SPRN_LPCR);
 624                mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 625                radix_init_partition_table();
 626        } else {
 627                radix_init_pseries();
 628        }
 629
 630        memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
 631
 632        /* Switch to the guard PID before turning on MMU */
 633        radix__switch_mmu_context(NULL, &init_mm);
 634        tlbiel_all();
 635}
 636
 637void radix__early_init_mmu_secondary(void)
 638{
 639        unsigned long lpcr;
 640        /*
 641         * update partition table control register and UPRT
 642         */
 643        if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 644                lpcr = mfspr(SPRN_LPCR);
 645                mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 646
 647                set_ptcr_when_no_uv(__pa(partition_tb) |
 648                                    (PATB_SIZE_SHIFT - 12));
 649        }
 650
 651        radix__switch_mmu_context(NULL, &init_mm);
 652        tlbiel_all();
 653
 654        /* Make sure userspace can't change the AMR */
 655        mtspr(SPRN_UAMOR, 0);
 656}
 657
 658/* Called during kexec sequence with MMU off */
 659notrace void radix__mmu_cleanup_all(void)
 660{
 661        unsigned long lpcr;
 662
 663        if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 664                lpcr = mfspr(SPRN_LPCR);
 665                mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
 666                set_ptcr_when_no_uv(0);
 667                powernv_set_nmmu_ptcr(0);
 668                radix__flush_tlb_all();
 669        }
 670}
 671
 672#ifdef CONFIG_MEMORY_HOTPLUG
 673static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
 674{
 675        pte_t *pte;
 676        int i;
 677
 678        for (i = 0; i < PTRS_PER_PTE; i++) {
 679                pte = pte_start + i;
 680                if (!pte_none(*pte))
 681                        return;
 682        }
 683
 684        pte_free_kernel(&init_mm, pte_start);
 685        pmd_clear(pmd);
 686}
 687
 688static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
 689{
 690        pmd_t *pmd;
 691        int i;
 692
 693        for (i = 0; i < PTRS_PER_PMD; i++) {
 694                pmd = pmd_start + i;
 695                if (!pmd_none(*pmd))
 696                        return;
 697        }
 698
 699        pmd_free(&init_mm, pmd_start);
 700        pud_clear(pud);
 701}
 702
 703static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
 704{
 705        pud_t *pud;
 706        int i;
 707
 708        for (i = 0; i < PTRS_PER_PUD; i++) {
 709                pud = pud_start + i;
 710                if (!pud_none(*pud))
 711                        return;
 712        }
 713
 714        pud_free(&init_mm, pud_start);
 715        p4d_clear(p4d);
 716}
 717
 718static void remove_pte_table(pte_t *pte_start, unsigned long addr,
 719                             unsigned long end)
 720{
 721        unsigned long next;
 722        pte_t *pte;
 723
 724        pte = pte_start + pte_index(addr);
 725        for (; addr < end; addr = next, pte++) {
 726                next = (addr + PAGE_SIZE) & PAGE_MASK;
 727                if (next > end)
 728                        next = end;
 729
 730                if (!pte_present(*pte))
 731                        continue;
 732
 733                if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
 734                        /*
 735                         * The vmemmap_free() and remove_section_mapping()
 736                         * codepaths call us with aligned addresses.
 737                         */
 738                        WARN_ONCE(1, "%s: unaligned range\n", __func__);
 739                        continue;
 740                }
 741
 742                pte_clear(&init_mm, addr, pte);
 743        }
 744}
 745
 746static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
 747                             unsigned long end)
 748{
 749        unsigned long next;
 750        pte_t *pte_base;
 751        pmd_t *pmd;
 752
 753        pmd = pmd_start + pmd_index(addr);
 754        for (; addr < end; addr = next, pmd++) {
 755                next = pmd_addr_end(addr, end);
 756
 757                if (!pmd_present(*pmd))
 758                        continue;
 759
 760                if (pmd_is_leaf(*pmd)) {
 761                        if (!IS_ALIGNED(addr, PMD_SIZE) ||
 762                            !IS_ALIGNED(next, PMD_SIZE)) {
 763                                WARN_ONCE(1, "%s: unaligned range\n", __func__);
 764                                continue;
 765                        }
 766                        pte_clear(&init_mm, addr, (pte_t *)pmd);
 767                        continue;
 768                }
 769
 770                pte_base = (pte_t *)pmd_page_vaddr(*pmd);
 771                remove_pte_table(pte_base, addr, next);
 772                free_pte_table(pte_base, pmd);
 773        }
 774}
 775
 776static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
 777                             unsigned long end)
 778{
 779        unsigned long next;
 780        pmd_t *pmd_base;
 781        pud_t *pud;
 782
 783        pud = pud_start + pud_index(addr);
 784        for (; addr < end; addr = next, pud++) {
 785                next = pud_addr_end(addr, end);
 786
 787                if (!pud_present(*pud))
 788                        continue;
 789
 790                if (pud_is_leaf(*pud)) {
 791                        if (!IS_ALIGNED(addr, PUD_SIZE) ||
 792                            !IS_ALIGNED(next, PUD_SIZE)) {
 793                                WARN_ONCE(1, "%s: unaligned range\n", __func__);
 794                                continue;
 795                        }
 796                        pte_clear(&init_mm, addr, (pte_t *)pud);
 797                        continue;
 798                }
 799
 800                pmd_base = pud_pgtable(*pud);
 801                remove_pmd_table(pmd_base, addr, next);
 802                free_pmd_table(pmd_base, pud);
 803        }
 804}
 805
 806static void __meminit remove_pagetable(unsigned long start, unsigned long end)
 807{
 808        unsigned long addr, next;
 809        pud_t *pud_base;
 810        pgd_t *pgd;
 811        p4d_t *p4d;
 812
 813        spin_lock(&init_mm.page_table_lock);
 814
 815        for (addr = start; addr < end; addr = next) {
 816                next = pgd_addr_end(addr, end);
 817
 818                pgd = pgd_offset_k(addr);
 819                p4d = p4d_offset(pgd, addr);
 820                if (!p4d_present(*p4d))
 821                        continue;
 822
 823                if (p4d_is_leaf(*p4d)) {
 824                        if (!IS_ALIGNED(addr, P4D_SIZE) ||
 825                            !IS_ALIGNED(next, P4D_SIZE)) {
 826                                WARN_ONCE(1, "%s: unaligned range\n", __func__);
 827                                continue;
 828                        }
 829
 830                        pte_clear(&init_mm, addr, (pte_t *)pgd);
 831                        continue;
 832                }
 833
 834                pud_base = p4d_pgtable(*p4d);
 835                remove_pud_table(pud_base, addr, next);
 836                free_pud_table(pud_base, p4d);
 837        }
 838
 839        spin_unlock(&init_mm.page_table_lock);
 840        radix__flush_tlb_kernel_range(start, end);
 841}
 842
 843int __meminit radix__create_section_mapping(unsigned long start,
 844                                            unsigned long end, int nid,
 845                                            pgprot_t prot)
 846{
 847        if (end >= RADIX_VMALLOC_START) {
 848                pr_warn("Outside the supported range\n");
 849                return -1;
 850        }
 851
 852        return create_physical_mapping(__pa(start), __pa(end),
 853                                       radix_mem_block_size, nid, prot);
 854}
 855
 856int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
 857{
 858        remove_pagetable(start, end);
 859        return 0;
 860}
 861#endif /* CONFIG_MEMORY_HOTPLUG */
 862
 863#ifdef CONFIG_SPARSEMEM_VMEMMAP
 864static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
 865                                 pgprot_t flags, unsigned int map_page_size,
 866                                 int nid)
 867{
 868        return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
 869}
 870
 871int __meminit radix__vmemmap_create_mapping(unsigned long start,
 872                                      unsigned long page_size,
 873                                      unsigned long phys)
 874{
 875        /* Create a PTE encoding */
 876        unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
 877        int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
 878        int ret;
 879
 880        if ((start + page_size) >= RADIX_VMEMMAP_END) {
 881                pr_warn("Outside the supported range\n");
 882                return -1;
 883        }
 884
 885        ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
 886        BUG_ON(ret);
 887
 888        return 0;
 889}
 890
 891#ifdef CONFIG_MEMORY_HOTPLUG
 892void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
 893{
 894        remove_pagetable(start, start + page_size);
 895}
 896#endif
 897#endif
 898
 899#ifdef CONFIG_DEBUG_PAGEALLOC
 900void radix__kernel_map_pages(struct page *page, int numpages, int enable)
 901{
 902        pr_warn_once("DEBUG_PAGEALLOC not supported in radix mode\n");
 903}
 904#endif
 905
 906#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 907
 908unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
 909                                  pmd_t *pmdp, unsigned long clr,
 910                                  unsigned long set)
 911{
 912        unsigned long old;
 913
 914#ifdef CONFIG_DEBUG_VM
 915        WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
 916        assert_spin_locked(pmd_lockptr(mm, pmdp));
 917#endif
 918
 919        old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
 920        trace_hugepage_update(addr, old, clr, set);
 921
 922        return old;
 923}
 924
 925pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
 926                        pmd_t *pmdp)
 927
 928{
 929        pmd_t pmd;
 930
 931        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 932        VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
 933        VM_BUG_ON(pmd_devmap(*pmdp));
 934        /*
 935         * khugepaged calls this for normal pmd
 936         */
 937        pmd = *pmdp;
 938        pmd_clear(pmdp);
 939
 940        /*
 941         * pmdp collapse_flush need to ensure that there are no parallel gup
 942         * walk after this call. This is needed so that we can have stable
 943         * page ref count when collapsing a page. We don't allow a collapse page
 944         * if we have gup taken on the page. We can ensure that by sending IPI
 945         * because gup walk happens with IRQ disabled.
 946         */
 947        serialize_against_pte_lookup(vma->vm_mm);
 948
 949        radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
 950
 951        return pmd;
 952}
 953
 954/*
 955 * For us pgtable_t is pte_t *. Inorder to save the deposisted
 956 * page table, we consider the allocated page table as a list
 957 * head. On withdraw we need to make sure we zero out the used
 958 * list_head memory area.
 959 */
 960void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 961                                 pgtable_t pgtable)
 962{
 963        struct list_head *lh = (struct list_head *) pgtable;
 964
 965        assert_spin_locked(pmd_lockptr(mm, pmdp));
 966
 967        /* FIFO */
 968        if (!pmd_huge_pte(mm, pmdp))
 969                INIT_LIST_HEAD(lh);
 970        else
 971                list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
 972        pmd_huge_pte(mm, pmdp) = pgtable;
 973}
 974
 975pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 976{
 977        pte_t *ptep;
 978        pgtable_t pgtable;
 979        struct list_head *lh;
 980
 981        assert_spin_locked(pmd_lockptr(mm, pmdp));
 982
 983        /* FIFO */
 984        pgtable = pmd_huge_pte(mm, pmdp);
 985        lh = (struct list_head *) pgtable;
 986        if (list_empty(lh))
 987                pmd_huge_pte(mm, pmdp) = NULL;
 988        else {
 989                pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
 990                list_del(lh);
 991        }
 992        ptep = (pte_t *) pgtable;
 993        *ptep = __pte(0);
 994        ptep++;
 995        *ptep = __pte(0);
 996        return pgtable;
 997}
 998
 999pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1000                                     unsigned long addr, pmd_t *pmdp)
1001{
1002        pmd_t old_pmd;
1003        unsigned long old;
1004
1005        old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1006        old_pmd = __pmd(old);
1007        return old_pmd;
1008}
1009
1010#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1011
1012void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1013                                  pte_t entry, unsigned long address, int psize)
1014{
1015        struct mm_struct *mm = vma->vm_mm;
1016        unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
1017                                              _PAGE_RW | _PAGE_EXEC);
1018
1019        unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1020        /*
1021         * To avoid NMMU hang while relaxing access, we need mark
1022         * the pte invalid in between.
1023         */
1024        if ((change & _PAGE_RW) && atomic_read(&mm->context.copros) > 0) {
1025                unsigned long old_pte, new_pte;
1026
1027                old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1028                /*
1029                 * new value of pte
1030                 */
1031                new_pte = old_pte | set;
1032                radix__flush_tlb_page_psize(mm, address, psize);
1033                __radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1034        } else {
1035                __radix_pte_update(ptep, 0, set);
1036                /*
1037                 * Book3S does not require a TLB flush when relaxing access
1038                 * restrictions when the address space is not attached to a
1039                 * NMMU, because the core MMU will reload the pte after taking
1040                 * an access fault, which is defined by the architecture.
1041                 */
1042        }
1043        /* See ptesync comment in radix__set_pte_at */
1044}
1045
1046void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
1047                                    unsigned long addr, pte_t *ptep,
1048                                    pte_t old_pte, pte_t pte)
1049{
1050        struct mm_struct *mm = vma->vm_mm;
1051
1052        /*
1053         * To avoid NMMU hang while relaxing access we need to flush the tlb before
1054         * we set the new value. We need to do this only for radix, because hash
1055         * translation does flush when updating the linux pte.
1056         */
1057        if (is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
1058            (atomic_read(&mm->context.copros) > 0))
1059                radix__flush_tlb_page(vma, addr);
1060
1061        set_pte_at(mm, addr, ptep, pte);
1062}
1063
1064int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1065{
1066        pte_t *ptep = (pte_t *)pud;
1067        pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
1068
1069        if (!radix_enabled())
1070                return 0;
1071
1072        set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
1073
1074        return 1;
1075}
1076
1077int pud_clear_huge(pud_t *pud)
1078{
1079        if (pud_is_leaf(*pud)) {
1080                pud_clear(pud);
1081                return 1;
1082        }
1083
1084        return 0;
1085}
1086
1087int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1088{
1089        pmd_t *pmd;
1090        int i;
1091
1092        pmd = pud_pgtable(*pud);
1093        pud_clear(pud);
1094
1095        flush_tlb_kernel_range(addr, addr + PUD_SIZE);
1096
1097        for (i = 0; i < PTRS_PER_PMD; i++) {
1098                if (!pmd_none(pmd[i])) {
1099                        pte_t *pte;
1100                        pte = (pte_t *)pmd_page_vaddr(pmd[i]);
1101
1102                        pte_free_kernel(&init_mm, pte);
1103                }
1104        }
1105
1106        pmd_free(&init_mm, pmd);
1107
1108        return 1;
1109}
1110
1111int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1112{
1113        pte_t *ptep = (pte_t *)pmd;
1114        pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
1115
1116        if (!radix_enabled())
1117                return 0;
1118
1119        set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
1120
1121        return 1;
1122}
1123
1124int pmd_clear_huge(pmd_t *pmd)
1125{
1126        if (pmd_is_leaf(*pmd)) {
1127                pmd_clear(pmd);
1128                return 1;
1129        }
1130
1131        return 0;
1132}
1133
1134int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1135{
1136        pte_t *pte;
1137
1138        pte = (pte_t *)pmd_page_vaddr(*pmd);
1139        pmd_clear(pmd);
1140
1141        flush_tlb_kernel_range(addr, addr + PMD_SIZE);
1142
1143        pte_free_kernel(&init_mm, pte);
1144
1145        return 1;
1146}
1147