1
2#include <linux/mm.h>
3#include <linux/mmzone.h>
4#include <linux/page_reporting.h>
5#include <linux/gfp.h>
6#include <linux/export.h>
7#include <linux/module.h>
8#include <linux/delay.h>
9#include <linux/scatterlist.h>
10
11#include "page_reporting.h"
12#include "internal.h"
13
14unsigned int page_reporting_order = MAX_ORDER;
15module_param(page_reporting_order, uint, 0644);
16MODULE_PARM_DESC(page_reporting_order, "Set page reporting order");
17
18#define PAGE_REPORTING_DELAY (2 * HZ)
19static struct page_reporting_dev_info __rcu *pr_dev_info __read_mostly;
20
21enum {
22 PAGE_REPORTING_IDLE = 0,
23 PAGE_REPORTING_REQUESTED,
24 PAGE_REPORTING_ACTIVE
25};
26
27
28static void
29__page_reporting_request(struct page_reporting_dev_info *prdev)
30{
31 unsigned int state;
32
33
34 state = atomic_read(&prdev->state);
35 if (state == PAGE_REPORTING_REQUESTED)
36 return;
37
38
39
40
41
42 state = atomic_xchg(&prdev->state, PAGE_REPORTING_REQUESTED);
43 if (state != PAGE_REPORTING_IDLE)
44 return;
45
46
47
48
49
50
51 schedule_delayed_work(&prdev->work, PAGE_REPORTING_DELAY);
52}
53
54
55void __page_reporting_notify(void)
56{
57 struct page_reporting_dev_info *prdev;
58
59
60
61
62
63
64 rcu_read_lock();
65 prdev = rcu_dereference(pr_dev_info);
66 if (likely(prdev))
67 __page_reporting_request(prdev);
68
69 rcu_read_unlock();
70}
71
72static void
73page_reporting_drain(struct page_reporting_dev_info *prdev,
74 struct scatterlist *sgl, unsigned int nents, bool reported)
75{
76 struct scatterlist *sg = sgl;
77
78
79
80
81
82 do {
83 struct page *page = sg_page(sg);
84 int mt = get_pageblock_migratetype(page);
85 unsigned int order = get_order(sg->length);
86
87 __putback_isolated_page(page, order, mt);
88
89
90 if (!reported)
91 continue;
92
93
94
95
96
97
98
99
100 if (PageBuddy(page) && buddy_order(page) == order)
101 __SetPageReported(page);
102 } while ((sg = sg_next(sg)));
103
104
105 sg_init_table(sgl, nents);
106}
107
108
109
110
111
112
113static int
114page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone,
115 unsigned int order, unsigned int mt,
116 struct scatterlist *sgl, unsigned int *offset)
117{
118 struct free_area *area = &zone->free_area[order];
119 struct list_head *list = &area->free_list[mt];
120 unsigned int page_len = PAGE_SIZE << order;
121 struct page *page, *next;
122 long budget;
123 int err = 0;
124
125
126
127
128
129 if (list_empty(list))
130 return err;
131
132 spin_lock_irq(&zone->lock);
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148 budget = DIV_ROUND_UP(area->nr_free, PAGE_REPORTING_CAPACITY * 16);
149
150
151 list_for_each_entry_safe(page, next, list, lru) {
152
153 if (PageReported(page))
154 continue;
155
156
157
158
159
160
161 if (budget < 0) {
162 atomic_set(&prdev->state, PAGE_REPORTING_REQUESTED);
163 next = page;
164 break;
165 }
166
167
168 if (*offset) {
169 if (!__isolate_free_page(page, order)) {
170 next = page;
171 break;
172 }
173
174
175 --(*offset);
176 sg_set_page(&sgl[*offset], page, page_len, 0);
177
178 continue;
179 }
180
181
182
183
184
185
186 if (!list_is_first(&page->lru, list))
187 list_rotate_to_front(&page->lru, list);
188
189
190 spin_unlock_irq(&zone->lock);
191
192
193 err = prdev->report(prdev, sgl, PAGE_REPORTING_CAPACITY);
194
195
196 *offset = PAGE_REPORTING_CAPACITY;
197
198
199 budget--;
200
201
202 spin_lock_irq(&zone->lock);
203
204
205 page_reporting_drain(prdev, sgl, PAGE_REPORTING_CAPACITY, !err);
206
207
208
209
210
211 next = list_first_entry(list, struct page, lru);
212
213
214 if (err)
215 break;
216 }
217
218
219 if (!list_entry_is_head(next, list, lru) && !list_is_first(&next->lru, list))
220 list_rotate_to_front(&next->lru, list);
221
222 spin_unlock_irq(&zone->lock);
223
224 return err;
225}
226
227static int
228page_reporting_process_zone(struct page_reporting_dev_info *prdev,
229 struct scatterlist *sgl, struct zone *zone)
230{
231 unsigned int order, mt, leftover, offset = PAGE_REPORTING_CAPACITY;
232 unsigned long watermark;
233 int err = 0;
234
235
236 watermark = low_wmark_pages(zone) +
237 (PAGE_REPORTING_CAPACITY << page_reporting_order);
238
239
240
241
242
243 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
244 return err;
245
246
247 for (order = page_reporting_order; order < MAX_ORDER; order++) {
248 for (mt = 0; mt < MIGRATE_TYPES; mt++) {
249
250 if (is_migrate_isolate(mt))
251 continue;
252
253 err = page_reporting_cycle(prdev, zone, order, mt,
254 sgl, &offset);
255 if (err)
256 return err;
257 }
258 }
259
260
261 leftover = PAGE_REPORTING_CAPACITY - offset;
262 if (leftover) {
263 sgl = &sgl[offset];
264 err = prdev->report(prdev, sgl, leftover);
265
266
267 spin_lock_irq(&zone->lock);
268 page_reporting_drain(prdev, sgl, leftover, !err);
269 spin_unlock_irq(&zone->lock);
270 }
271
272 return err;
273}
274
275static void page_reporting_process(struct work_struct *work)
276{
277 struct delayed_work *d_work = to_delayed_work(work);
278 struct page_reporting_dev_info *prdev =
279 container_of(d_work, struct page_reporting_dev_info, work);
280 int err = 0, state = PAGE_REPORTING_ACTIVE;
281 struct scatterlist *sgl;
282 struct zone *zone;
283
284
285
286
287
288
289
290 atomic_set(&prdev->state, state);
291
292
293 sgl = kmalloc_array(PAGE_REPORTING_CAPACITY, sizeof(*sgl), GFP_KERNEL);
294 if (!sgl)
295 goto err_out;
296
297 sg_init_table(sgl, PAGE_REPORTING_CAPACITY);
298
299 for_each_zone(zone) {
300 err = page_reporting_process_zone(prdev, sgl, zone);
301 if (err)
302 break;
303 }
304
305 kfree(sgl);
306err_out:
307
308
309
310
311
312 state = atomic_cmpxchg(&prdev->state, state, PAGE_REPORTING_IDLE);
313 if (state == PAGE_REPORTING_REQUESTED)
314 schedule_delayed_work(&prdev->work, PAGE_REPORTING_DELAY);
315}
316
317static DEFINE_MUTEX(page_reporting_mutex);
318DEFINE_STATIC_KEY_FALSE(page_reporting_enabled);
319
320int page_reporting_register(struct page_reporting_dev_info *prdev)
321{
322 int err = 0;
323
324 mutex_lock(&page_reporting_mutex);
325
326
327 if (rcu_access_pointer(pr_dev_info)) {
328 err = -EBUSY;
329 goto err_out;
330 }
331
332
333
334
335
336 page_reporting_order = prdev->order ? : pageblock_order;
337
338
339 atomic_set(&prdev->state, PAGE_REPORTING_IDLE);
340 INIT_DELAYED_WORK(&prdev->work, &page_reporting_process);
341
342
343 __page_reporting_request(prdev);
344
345
346 rcu_assign_pointer(pr_dev_info, prdev);
347
348
349 if (!static_key_enabled(&page_reporting_enabled)) {
350 static_branch_enable(&page_reporting_enabled);
351 pr_info("Free page reporting enabled\n");
352 }
353err_out:
354 mutex_unlock(&page_reporting_mutex);
355
356 return err;
357}
358EXPORT_SYMBOL_GPL(page_reporting_register);
359
360void page_reporting_unregister(struct page_reporting_dev_info *prdev)
361{
362 mutex_lock(&page_reporting_mutex);
363
364 if (rcu_access_pointer(pr_dev_info) == prdev) {
365
366 RCU_INIT_POINTER(pr_dev_info, NULL);
367 synchronize_rcu();
368
369
370 cancel_delayed_work_sync(&prdev->work);
371 }
372
373 mutex_unlock(&page_reporting_mutex);
374}
375EXPORT_SYMBOL_GPL(page_reporting_unregister);
376