1
2
3
4
5
6#include <linux/mm.h>
7#include <linux/page-isolation.h>
8#include <linux/pageblock-flags.h>
9#include <linux/memory.h>
10#include <linux/hugetlb.h>
11#include <linux/page_owner.h>
12#include <linux/migrate.h>
13#include "internal.h"
14
15#define CREATE_TRACE_POINTS
16#include <trace/events/page_isolation.h>
17
18static int set_migratetype_isolate(struct page *page, int migratetype, int isol_flags)
19{
20 struct zone *zone = page_zone(page);
21 struct page *unmovable;
22 unsigned long flags;
23
24 spin_lock_irqsave(&zone->lock, flags);
25
26
27
28
29
30
31 if (is_migrate_isolate_page(page)) {
32 spin_unlock_irqrestore(&zone->lock, flags);
33 return -EBUSY;
34 }
35
36
37
38
39
40 unmovable = has_unmovable_pages(zone, page, migratetype, isol_flags);
41 if (!unmovable) {
42 unsigned long nr_pages;
43 int mt = get_pageblock_migratetype(page);
44
45 set_pageblock_migratetype(page, MIGRATE_ISOLATE);
46 zone->nr_isolate_pageblock++;
47 nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE,
48 NULL);
49
50 __mod_zone_freepage_state(zone, -nr_pages, mt);
51 spin_unlock_irqrestore(&zone->lock, flags);
52 return 0;
53 }
54
55 spin_unlock_irqrestore(&zone->lock, flags);
56 if (isol_flags & REPORT_FAILURE) {
57
58
59
60
61 dump_page(unmovable, "unmovable page");
62 }
63
64 return -EBUSY;
65}
66
67static void unset_migratetype_isolate(struct page *page, unsigned migratetype)
68{
69 struct zone *zone;
70 unsigned long flags, nr_pages;
71 bool isolated_page = false;
72 unsigned int order;
73 unsigned long pfn, buddy_pfn;
74 struct page *buddy;
75
76 zone = page_zone(page);
77 spin_lock_irqsave(&zone->lock, flags);
78 if (!is_migrate_isolate_page(page))
79 goto out;
80
81
82
83
84
85
86
87
88
89 if (PageBuddy(page)) {
90 order = buddy_order(page);
91 if (order >= pageblock_order && order < MAX_ORDER - 1) {
92 pfn = page_to_pfn(page);
93 buddy_pfn = __find_buddy_pfn(pfn, order);
94 buddy = page + (buddy_pfn - pfn);
95
96 if (!is_migrate_isolate_page(buddy)) {
97 __isolate_free_page(page, order);
98 isolated_page = true;
99 }
100 }
101 }
102
103
104
105
106
107
108
109
110
111
112
113 if (!isolated_page) {
114 nr_pages = move_freepages_block(zone, page, migratetype, NULL);
115 __mod_zone_freepage_state(zone, nr_pages, migratetype);
116 }
117 set_pageblock_migratetype(page, migratetype);
118 if (isolated_page)
119 __putback_isolated_page(page, order, migratetype);
120 zone->nr_isolate_pageblock--;
121out:
122 spin_unlock_irqrestore(&zone->lock, flags);
123}
124
125static inline struct page *
126__first_valid_page(unsigned long pfn, unsigned long nr_pages)
127{
128 int i;
129
130 for (i = 0; i < nr_pages; i++) {
131 struct page *page;
132
133 page = pfn_to_online_page(pfn + i);
134 if (!page)
135 continue;
136 return page;
137 }
138 return NULL;
139}
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
183 unsigned migratetype, int flags)
184{
185 unsigned long pfn;
186 unsigned long undo_pfn;
187 struct page *page;
188
189 BUG_ON(!IS_ALIGNED(start_pfn, pageblock_nr_pages));
190 BUG_ON(!IS_ALIGNED(end_pfn, pageblock_nr_pages));
191
192 for (pfn = start_pfn;
193 pfn < end_pfn;
194 pfn += pageblock_nr_pages) {
195 page = __first_valid_page(pfn, pageblock_nr_pages);
196 if (page) {
197 if (set_migratetype_isolate(page, migratetype, flags)) {
198 undo_pfn = pfn;
199 goto undo;
200 }
201 }
202 }
203 return 0;
204undo:
205 for (pfn = start_pfn;
206 pfn < undo_pfn;
207 pfn += pageblock_nr_pages) {
208 struct page *page = pfn_to_online_page(pfn);
209 if (!page)
210 continue;
211 unset_migratetype_isolate(page, migratetype);
212 }
213
214 return -EBUSY;
215}
216
217
218
219
220void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
221 unsigned migratetype)
222{
223 unsigned long pfn;
224 struct page *page;
225
226 BUG_ON(!IS_ALIGNED(start_pfn, pageblock_nr_pages));
227 BUG_ON(!IS_ALIGNED(end_pfn, pageblock_nr_pages));
228
229 for (pfn = start_pfn;
230 pfn < end_pfn;
231 pfn += pageblock_nr_pages) {
232 page = __first_valid_page(pfn, pageblock_nr_pages);
233 if (!page || !is_migrate_isolate_page(page))
234 continue;
235 unset_migratetype_isolate(page, migratetype);
236 }
237}
238
239
240
241
242
243
244
245static unsigned long
246__test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn,
247 int flags)
248{
249 struct page *page;
250
251 while (pfn < end_pfn) {
252 page = pfn_to_page(pfn);
253 if (PageBuddy(page))
254
255
256
257
258
259 pfn += 1 << buddy_order(page);
260 else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
261
262 pfn++;
263 else if ((flags & MEMORY_OFFLINE) && PageOffline(page) &&
264 !page_count(page))
265
266
267
268
269
270 pfn++;
271 else
272 break;
273 }
274
275 return pfn;
276}
277
278
279int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
280 int isol_flags)
281{
282 unsigned long pfn, flags;
283 struct page *page;
284 struct zone *zone;
285 int ret;
286
287
288
289
290
291
292 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
293 page = __first_valid_page(pfn, pageblock_nr_pages);
294 if (page && !is_migrate_isolate_page(page))
295 break;
296 }
297 page = __first_valid_page(start_pfn, end_pfn - start_pfn);
298 if ((pfn < end_pfn) || !page) {
299 ret = -EBUSY;
300 goto out;
301 }
302
303
304 zone = page_zone(page);
305 spin_lock_irqsave(&zone->lock, flags);
306 pfn = __test_page_isolated_in_pageblock(start_pfn, end_pfn, isol_flags);
307 spin_unlock_irqrestore(&zone->lock, flags);
308
309 ret = pfn < end_pfn ? -EBUSY : 0;
310
311out:
312 trace_test_pages_isolated(start_pfn, end_pfn, pfn);
313
314 return ret;
315}
316