LXR linux/arch/s390/mm/vmem.c

   1/*
   2 *    Copyright IBM Corp. 2006
   3 *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   4 */
   5
   6#include <linux/bootmem.h>
   7#include <linux/pfn.h>
   8#include <linux/mm.h>
   9#include <linux/module.h>
  10#include <linux/list.h>
  11#include <linux/hugetlb.h>
  12#include <linux/slab.h>
  13#include <linux/memblock.h>
  14#include <asm/pgalloc.h>
  15#include <asm/pgtable.h>
  16#include <asm/setup.h>
  17#include <asm/tlbflush.h>
  18#include <asm/sections.h>
  19
  20static DEFINE_MUTEX(vmem_mutex);
  21
  22struct memory_segment {
  23        struct list_head list;
  24        unsigned long start;
  25        unsigned long size;
  26};
  27
  28static LIST_HEAD(mem_segs);
  29
  30static void __ref *vmem_alloc_pages(unsigned int order)
  31{
  32        if (slab_is_available())
  33                return (void *)__get_free_pages(GFP_KERNEL, order);
  34        return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
  35}
  36
  37static inline pud_t *vmem_pud_alloc(void)
  38{
  39        pud_t *pud = NULL;
  40
  41        pud = vmem_alloc_pages(2);
  42        if (!pud)
  43                return NULL;
  44        clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  45        return pud;
  46}
  47
  48static inline pmd_t *vmem_pmd_alloc(void)
  49{
  50        pmd_t *pmd = NULL;
  51
  52        pmd = vmem_alloc_pages(2);
  53        if (!pmd)
  54                return NULL;
  55        clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  56        return pmd;
  57}
  58
  59static pte_t __ref *vmem_pte_alloc(unsigned long address)
  60{
  61        pte_t *pte;
  62
  63        if (slab_is_available())
  64                pte = (pte_t *) page_table_alloc(&init_mm);
  65        else
  66                pte = alloc_bootmem_align(PTRS_PER_PTE * sizeof(pte_t),
  67                                          PTRS_PER_PTE * sizeof(pte_t));
  68        if (!pte)
  69                return NULL;
  70        clear_table((unsigned long *) pte, _PAGE_INVALID,
  71                    PTRS_PER_PTE * sizeof(pte_t));
  72        return pte;
  73}
  74
  75/*
  76 * Add a physical memory range to the 1:1 mapping.
  77 */
  78static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
  79{
  80        unsigned long end = start + size;
  81        unsigned long address = start;
  82        pgd_t *pg_dir;
  83        pud_t *pu_dir;
  84        pmd_t *pm_dir;
  85        pte_t *pt_dir;
  86        int ret = -ENOMEM;
  87
  88        while (address < end) {
  89                pg_dir = pgd_offset_k(address);
  90                if (pgd_none(*pg_dir)) {
  91                        pu_dir = vmem_pud_alloc();
  92                        if (!pu_dir)
  93                                goto out;
  94                        pgd_populate(&init_mm, pg_dir, pu_dir);
  95                }
  96                pu_dir = pud_offset(pg_dir, address);
  97                if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
  98                    !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) &&
  99                     !debug_pagealloc_enabled()) {
 100                        pud_val(*pu_dir) = __pa(address) |
 101                                _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE |
 102                                (ro ? _REGION_ENTRY_PROTECT : 0);
 103                        address += PUD_SIZE;
 104                        continue;
 105                }
 106                if (pud_none(*pu_dir)) {
 107                        pm_dir = vmem_pmd_alloc();
 108                        if (!pm_dir)
 109                                goto out;
 110                        pud_populate(&init_mm, pu_dir, pm_dir);
 111                }
 112                pm_dir = pmd_offset(pu_dir, address);
 113                if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
 114                    !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) &&
 115                    !debug_pagealloc_enabled()) {
 116                        pmd_val(*pm_dir) = __pa(address) |
 117                                _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
 118                                _SEGMENT_ENTRY_YOUNG |
 119                                (ro ? _SEGMENT_ENTRY_PROTECT : 0);
 120                        address += PMD_SIZE;
 121                        continue;
 122                }
 123                if (pmd_none(*pm_dir)) {
 124                        pt_dir = vmem_pte_alloc(address);
 125                        if (!pt_dir)
 126                                goto out;
 127                        pmd_populate(&init_mm, pm_dir, pt_dir);
 128                }
 129
 130                pt_dir = pte_offset_kernel(pm_dir, address);
 131                pte_val(*pt_dir) = __pa(address) |
 132                        pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
 133                address += PAGE_SIZE;
 134        }
 135        ret = 0;
 136out:
 137        return ret;
 138}
 139
 140/*
 141 * Remove a physical memory range from the 1:1 mapping.
 142 * Currently only invalidates page table entries.
 143 */
 144static void vmem_remove_range(unsigned long start, unsigned long size)
 145{
 146        unsigned long end = start + size;
 147        unsigned long address = start;
 148        pgd_t *pg_dir;
 149        pud_t *pu_dir;
 150        pmd_t *pm_dir;
 151        pte_t *pt_dir;
 152        pte_t  pte;
 153
 154        pte_val(pte) = _PAGE_INVALID;
 155        while (address < end) {
 156                pg_dir = pgd_offset_k(address);
 157                if (pgd_none(*pg_dir)) {
 158                        address += PGDIR_SIZE;
 159                        continue;
 160                }
 161                pu_dir = pud_offset(pg_dir, address);
 162                if (pud_none(*pu_dir)) {
 163                        address += PUD_SIZE;
 164                        continue;
 165                }
 166                if (pud_large(*pu_dir)) {
 167                        pud_clear(pu_dir);
 168                        address += PUD_SIZE;
 169                        continue;
 170                }
 171                pm_dir = pmd_offset(pu_dir, address);
 172                if (pmd_none(*pm_dir)) {
 173                        address += PMD_SIZE;
 174                        continue;
 175                }
 176                if (pmd_large(*pm_dir)) {
 177                        pmd_clear(pm_dir);
 178                        address += PMD_SIZE;
 179                        continue;
 180                }
 181                pt_dir = pte_offset_kernel(pm_dir, address);
 182                *pt_dir = pte;
 183                address += PAGE_SIZE;
 184        }
 185        flush_tlb_kernel_range(start, end);
 186}
 187
 188/*
 189 * Add a backed mem_map array to the virtual mem_map array.
 190 */
 191int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 192{
 193        unsigned long address = start;
 194        pgd_t *pg_dir;
 195        pud_t *pu_dir;
 196        pmd_t *pm_dir;
 197        pte_t *pt_dir;
 198        int ret = -ENOMEM;
 199
 200        for (address = start; address < end;) {
 201                pg_dir = pgd_offset_k(address);
 202                if (pgd_none(*pg_dir)) {
 203                        pu_dir = vmem_pud_alloc();
 204                        if (!pu_dir)
 205                                goto out;
 206                        pgd_populate(&init_mm, pg_dir, pu_dir);
 207                }
 208
 209                pu_dir = pud_offset(pg_dir, address);
 210                if (pud_none(*pu_dir)) {
 211                        pm_dir = vmem_pmd_alloc();
 212                        if (!pm_dir)
 213                                goto out;
 214                        pud_populate(&init_mm, pu_dir, pm_dir);
 215                }
 216
 217                pm_dir = pmd_offset(pu_dir, address);
 218                if (pmd_none(*pm_dir)) {
 219                        /* Use 1MB frames for vmemmap if available. We always
 220                         * use large frames even if they are only partially
 221                         * used.
 222                         * Otherwise we would have also page tables since
 223                         * vmemmap_populate gets called for each section
 224                         * separately. */
 225                        if (MACHINE_HAS_EDAT1) {
 226                                void *new_page;
 227
 228                                new_page = vmemmap_alloc_block(PMD_SIZE, node);
 229                                if (!new_page)
 230                                        goto out;
 231                                pmd_val(*pm_dir) = __pa(new_page) |
 232                                        _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE;
 233                                address = (address + PMD_SIZE) & PMD_MASK;
 234                                continue;
 235                        }
 236                        pt_dir = vmem_pte_alloc(address);
 237                        if (!pt_dir)
 238                                goto out;
 239                        pmd_populate(&init_mm, pm_dir, pt_dir);
 240                } else if (pmd_large(*pm_dir)) {
 241                        address = (address + PMD_SIZE) & PMD_MASK;
 242                        continue;
 243                }
 244
 245                pt_dir = pte_offset_kernel(pm_dir, address);
 246                if (pte_none(*pt_dir)) {
 247                        void *new_page;
 248
 249                        new_page = vmemmap_alloc_block(PAGE_SIZE, node);
 250                        if (!new_page)
 251                                goto out;
 252                        pte_val(*pt_dir) =
 253                                __pa(new_page) | pgprot_val(PAGE_KERNEL);
 254                }
 255                address += PAGE_SIZE;
 256        }
 257        ret = 0;
 258out:
 259        return ret;
 260}
 261
 262void vmemmap_free(unsigned long start, unsigned long end)
 263{
 264}
 265
 266/*
 267 * Add memory segment to the segment list if it doesn't overlap with
 268 * an already present segment.
 269 */
 270static int insert_memory_segment(struct memory_segment *seg)
 271{
 272        struct memory_segment *tmp;
 273
 274        if (seg->start + seg->size > VMEM_MAX_PHYS ||
 275            seg->start + seg->size < seg->start)
 276                return -ERANGE;
 277
 278        list_for_each_entry(tmp, &mem_segs, list) {
 279                if (seg->start >= tmp->start + tmp->size)
 280                        continue;
 281                if (seg->start + seg->size <= tmp->start)
 282                        continue;
 283                return -ENOSPC;
 284        }
 285        list_add(&seg->list, &mem_segs);
 286        return 0;
 287}
 288
 289/*
 290 * Remove memory segment from the segment list.
 291 */
 292static void remove_memory_segment(struct memory_segment *seg)
 293{
 294        list_del(&seg->list);
 295}
 296
 297static void __remove_shared_memory(struct memory_segment *seg)
 298{
 299        remove_memory_segment(seg);
 300        vmem_remove_range(seg->start, seg->size);
 301}
 302
 303int vmem_remove_mapping(unsigned long start, unsigned long size)
 304{
 305        struct memory_segment *seg;
 306        int ret;
 307
 308        mutex_lock(&vmem_mutex);
 309
 310        ret = -ENOENT;
 311        list_for_each_entry(seg, &mem_segs, list) {
 312                if (seg->start == start && seg->size == size)
 313                        break;
 314        }
 315
 316        if (seg->start != start || seg->size != size)
 317                goto out;
 318
 319        ret = 0;
 320        __remove_shared_memory(seg);
 321        kfree(seg);
 322out:
 323        mutex_unlock(&vmem_mutex);
 324        return ret;
 325}
 326
 327int vmem_add_mapping(unsigned long start, unsigned long size)
 328{
 329        struct memory_segment *seg;
 330        int ret;
 331
 332        mutex_lock(&vmem_mutex);
 333        ret = -ENOMEM;
 334        seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 335        if (!seg)
 336                goto out;
 337        seg->start = start;
 338        seg->size = size;
 339
 340        ret = insert_memory_segment(seg);
 341        if (ret)
 342                goto out_free;
 343
 344        ret = vmem_add_mem(start, size, 0);
 345        if (ret)
 346                goto out_remove;
 347        goto out;
 348
 349out_remove:
 350        __remove_shared_memory(seg);
 351out_free:
 352        kfree(seg);
 353out:
 354        mutex_unlock(&vmem_mutex);
 355        return ret;
 356}
 357
 358/*
 359 * map whole physical memory to virtual memory (identity mapping)
 360 * we reserve enough space in the vmalloc area for vmemmap to hotplug
 361 * additional memory segments.
 362 */
 363void __init vmem_map_init(void)
 364{
 365        unsigned long ro_start, ro_end;
 366        struct memblock_region *reg;
 367        phys_addr_t start, end;
 368
 369        ro_start = PFN_ALIGN((unsigned long)&_stext);
 370        ro_end = (unsigned long)&_eshared & PAGE_MASK;
 371        for_each_memblock(memory, reg) {
 372                start = reg->base;
 373                end = reg->base + reg->size - 1;
 374                if (start >= ro_end || end <= ro_start)
 375                        vmem_add_mem(start, end - start, 0);
 376                else if (start >= ro_start && end <= ro_end)
 377                        vmem_add_mem(start, end - start, 1);
 378                else if (start >= ro_start) {
 379                        vmem_add_mem(start, ro_end - start, 1);
 380                        vmem_add_mem(ro_end, end - ro_end, 0);
 381                } else if (end < ro_end) {
 382                        vmem_add_mem(start, ro_start - start, 0);
 383                        vmem_add_mem(ro_start, end - ro_start, 1);
 384                } else {
 385                        vmem_add_mem(start, ro_start - start, 0);
 386                        vmem_add_mem(ro_start, ro_end - ro_start, 1);
 387                        vmem_add_mem(ro_end, end - ro_end, 0);
 388                }
 389        }
 390}
 391
 392/*
 393 * Convert memblock.memory  to a memory segment list so there is a single
 394 * list that contains all memory segments.
 395 */
 396static int __init vmem_convert_memory_chunk(void)
 397{
 398        struct memblock_region *reg;
 399        struct memory_segment *seg;
 400
 401        mutex_lock(&vmem_mutex);
 402        for_each_memblock(memory, reg) {
 403                seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 404                if (!seg)
 405                        panic("Out of memory...\n");
 406                seg->start = reg->base;
 407                seg->size = reg->size;
 408                insert_memory_segment(seg);
 409        }
 410        mutex_unlock(&vmem_mutex);
 411        return 0;
 412}
 413
 414core_initcall(vmem_convert_memory_chunk);
 415