1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#undef DEBUG
18
19#include <linux/string.h>
20#include <linux/pgtable.h>
21#include <asm/pgalloc.h>
22#include <asm/kup.h>
23
24phys_addr_t memstart_addr __ro_after_init = (phys_addr_t)~0ull;
25EXPORT_SYMBOL_GPL(memstart_addr);
26phys_addr_t kernstart_addr __ro_after_init;
27EXPORT_SYMBOL_GPL(kernstart_addr);
28unsigned long kernstart_virt_addr __ro_after_init = KERNELBASE;
29EXPORT_SYMBOL_GPL(kernstart_virt_addr);
30
31bool disable_kuep = !IS_ENABLED(CONFIG_PPC_KUEP);
32bool disable_kuap = !IS_ENABLED(CONFIG_PPC_KUAP);
33
34static int __init parse_nosmep(char *p)
35{
36 disable_kuep = true;
37 pr_warn("Disabling Kernel Userspace Execution Prevention\n");
38 return 0;
39}
40early_param("nosmep", parse_nosmep);
41
42static int __init parse_nosmap(char *p)
43{
44 disable_kuap = true;
45 pr_warn("Disabling Kernel Userspace Access Protection\n");
46 return 0;
47}
48early_param("nosmap", parse_nosmap);
49
50#define CTOR(shift) static void ctor_##shift(void *addr) \
51{ \
52 memset(addr, 0, sizeof(void *) << (shift)); \
53}
54
55CTOR(0); CTOR(1); CTOR(2); CTOR(3); CTOR(4); CTOR(5); CTOR(6); CTOR(7);
56CTOR(8); CTOR(9); CTOR(10); CTOR(11); CTOR(12); CTOR(13); CTOR(14); CTOR(15);
57
58static inline void (*ctor(int shift))(void *)
59{
60 BUILD_BUG_ON(MAX_PGTABLE_INDEX_SIZE != 15);
61
62 switch (shift) {
63 case 0: return ctor_0;
64 case 1: return ctor_1;
65 case 2: return ctor_2;
66 case 3: return ctor_3;
67 case 4: return ctor_4;
68 case 5: return ctor_5;
69 case 6: return ctor_6;
70 case 7: return ctor_7;
71 case 8: return ctor_8;
72 case 9: return ctor_9;
73 case 10: return ctor_10;
74 case 11: return ctor_11;
75 case 12: return ctor_12;
76 case 13: return ctor_13;
77 case 14: return ctor_14;
78 case 15: return ctor_15;
79 }
80 return NULL;
81}
82
83struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE + 1];
84EXPORT_SYMBOL_GPL(pgtable_cache);
85
86
87
88
89
90
91
92
93void pgtable_cache_add(unsigned int shift)
94{
95 char *name;
96 unsigned long table_size = sizeof(void *) << shift;
97 unsigned long align = table_size;
98
99
100
101
102
103
104
105
106 unsigned long minalign = max(MAX_PGTABLE_INDEX_SIZE + 1,
107 HUGEPD_SHIFT_MASK + 1);
108 struct kmem_cache *new;
109
110
111
112
113 BUG_ON(!is_power_of_2(minalign));
114 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
115
116 if (PGT_CACHE(shift))
117 return;
118
119 align = max_t(unsigned long, align, minalign);
120 name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift);
121 new = kmem_cache_create(name, table_size, align, 0, ctor(shift));
122 if (!new)
123 panic("Could not allocate pgtable cache for order %d", shift);
124
125 kfree(name);
126 pgtable_cache[shift] = new;
127
128 pr_debug("Allocated pgtable cache for order %d\n", shift);
129}
130EXPORT_SYMBOL_GPL(pgtable_cache_add);
131
132void pgtable_cache_init(void)
133{
134 pgtable_cache_add(PGD_INDEX_SIZE);
135
136 if (PMD_CACHE_INDEX)
137 pgtable_cache_add(PMD_CACHE_INDEX);
138
139
140
141
142
143 if (PUD_CACHE_INDEX)
144 pgtable_cache_add(PUD_CACHE_INDEX);
145}
146