LXR linux/arch/x86/mm/hugetlbpage.c

   1/*
   2 * IA-32 Huge TLB Page Support for Kernel.
   3 *
   4 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
   5 */
   6
   7#include <linux/init.h>
   8#include <linux/fs.h>
   9#include <linux/mm.h>
  10#include <linux/hugetlb.h>
  11#include <linux/pagemap.h>
  12#include <linux/err.h>
  13#include <linux/sysctl.h>
  14#include <asm/mman.h>
  15#include <asm/tlb.h>
  16#include <asm/tlbflush.h>
  17#include <asm/pgalloc.h>
  18
  19#if 0   /* This is just for testing */
  20struct page *
  21follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
  22{
  23        unsigned long start = address;
  24        int length = 1;
  25        int nr;
  26        struct page *page;
  27        struct vm_area_struct *vma;
  28
  29        vma = find_vma(mm, addr);
  30        if (!vma || !is_vm_hugetlb_page(vma))
  31                return ERR_PTR(-EINVAL);
  32
  33        pte = huge_pte_offset(mm, address);
  34
  35        /* hugetlb should be locked, and hence, prefaulted */
  36        WARN_ON(!pte || pte_none(*pte));
  37
  38        page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
  39
  40        WARN_ON(!PageHead(page));
  41
  42        return page;
  43}
  44
  45int pmd_huge(pmd_t pmd)
  46{
  47        return 0;
  48}
  49
  50int pud_huge(pud_t pud)
  51{
  52        return 0;
  53}
  54
  55struct page *
  56follow_huge_pmd(struct mm_struct *mm, unsigned long address,
  57                pmd_t *pmd, int write)
  58{
  59        return NULL;
  60}
  61
  62int pmd_huge_support(void)
  63{
  64        return 0;
  65}
  66#else
  67
  68struct page *
  69follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
  70{
  71        return ERR_PTR(-EINVAL);
  72}
  73
  74int pmd_huge(pmd_t pmd)
  75{
  76        return !!(pmd_val(pmd) & _PAGE_PSE);
  77}
  78
  79int pud_huge(pud_t pud)
  80{
  81        return !!(pud_val(pud) & _PAGE_PSE);
  82}
  83
  84int pmd_huge_support(void)
  85{
  86        return 1;
  87}
  88#endif
  89
  90#ifdef CONFIG_HUGETLB_PAGE
  91static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
  92                unsigned long addr, unsigned long len,
  93                unsigned long pgoff, unsigned long flags)
  94{
  95        struct hstate *h = hstate_file(file);
  96        struct vm_unmapped_area_info info;
  97
  98        info.flags = 0;
  99        info.length = len;
 100        info.low_limit = current->mm->mmap_legacy_base;
 101        info.high_limit = TASK_SIZE;
 102        info.align_mask = PAGE_MASK & ~huge_page_mask(h);
 103        info.align_offset = 0;
 104        return vm_unmapped_area(&info);
 105}
 106
 107static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
 108                unsigned long addr0, unsigned long len,
 109                unsigned long pgoff, unsigned long flags)
 110{
 111        struct hstate *h = hstate_file(file);
 112        struct vm_unmapped_area_info info;
 113        unsigned long addr;
 114
 115        info.flags = VM_UNMAPPED_AREA_TOPDOWN;
 116        info.length = len;
 117        info.low_limit = PAGE_SIZE;
 118        info.high_limit = current->mm->mmap_base;
 119        info.align_mask = PAGE_MASK & ~huge_page_mask(h);
 120        info.align_offset = 0;
 121        addr = vm_unmapped_area(&info);
 122
 123        /*
 124         * A failed mmap() very likely causes application failure,
 125         * so fall back to the bottom-up function here. This scenario
 126         * can happen with large stack limits and large mmap()
 127         * allocations.
 128         */
 129        if (addr & ~PAGE_MASK) {
 130                VM_BUG_ON(addr != -ENOMEM);
 131                info.flags = 0;
 132                info.low_limit = TASK_UNMAPPED_BASE;
 133                info.high_limit = TASK_SIZE;
 134                addr = vm_unmapped_area(&info);
 135        }
 136
 137        return addr;
 138}
 139
 140unsigned long
 141hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 142                unsigned long len, unsigned long pgoff, unsigned long flags)
 143{
 144        struct hstate *h = hstate_file(file);
 145        struct mm_struct *mm = current->mm;
 146        struct vm_area_struct *vma;
 147
 148        if (len & ~huge_page_mask(h))
 149                return -EINVAL;
 150        if (len > TASK_SIZE)
 151                return -ENOMEM;
 152
 153        if (flags & MAP_FIXED) {
 154                if (prepare_hugepage_range(file, addr, len))
 155                        return -EINVAL;
 156                return addr;
 157        }
 158
 159        if (addr) {
 160                addr = ALIGN(addr, huge_page_size(h));
 161                vma = find_vma(mm, addr);
 162                if (TASK_SIZE - len >= addr &&
 163                    (!vma || addr + len <= vma->vm_start))
 164                        return addr;
 165        }
 166        if (mm->get_unmapped_area == arch_get_unmapped_area)
 167                return hugetlb_get_unmapped_area_bottomup(file, addr, len,
 168                                pgoff, flags);
 169        else
 170                return hugetlb_get_unmapped_area_topdown(file, addr, len,
 171                                pgoff, flags);
 172}
 173#endif /* CONFIG_HUGETLB_PAGE */
 174
 175#ifdef CONFIG_X86_64
 176static __init int setup_hugepagesz(char *opt)
 177{
 178        unsigned long ps = memparse(opt, &opt);
 179        if (ps == PMD_SIZE) {
 180                hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
 181        } else if (ps == PUD_SIZE && cpu_has_gbpages) {
 182                hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
 183        } else {
 184                printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n",
 185                        ps >> 20);
 186                return 0;
 187        }
 188        return 1;
 189}
 190__setup("hugepagesz=", setup_hugepagesz);
 191#endif
 192