1
2
3
4
5
6
7
8
9
10
11
12#include <linux/cgroup.h>
13#include <linux/fs.h>
14#include <linux/log2.h>
15#include <linux/sched.h>
16#include <linux/mm.h>
17#include <linux/vmstat.h>
18#include <linux/eventfd.h>
19#include <linux/slab.h>
20#include <linux/swap.h>
21#include <linux/printk.h>
22#include <linux/vmpressure.h>
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16;
39
40
41
42
43
44
45
46static const unsigned int vmpressure_level_med = 60;
47static const unsigned int vmpressure_level_critical = 95;
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10);
69
70static struct vmpressure *work_to_vmpressure(struct work_struct *work)
71{
72 return container_of(work, struct vmpressure, work);
73}
74
75static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr)
76{
77 struct cgroup_subsys_state *css = vmpressure_to_css(vmpr);
78 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
79
80 memcg = parent_mem_cgroup(memcg);
81 if (!memcg)
82 return NULL;
83 return memcg_to_vmpressure(memcg);
84}
85
86enum vmpressure_levels {
87 VMPRESSURE_LOW = 0,
88 VMPRESSURE_MEDIUM,
89 VMPRESSURE_CRITICAL,
90 VMPRESSURE_NUM_LEVELS,
91};
92
93enum vmpressure_modes {
94 VMPRESSURE_NO_PASSTHROUGH = 0,
95 VMPRESSURE_HIERARCHY,
96 VMPRESSURE_LOCAL,
97 VMPRESSURE_NUM_MODES,
98};
99
100static const char * const vmpressure_str_levels[] = {
101 [VMPRESSURE_LOW] = "low",
102 [VMPRESSURE_MEDIUM] = "medium",
103 [VMPRESSURE_CRITICAL] = "critical",
104};
105
106static const char * const vmpressure_str_modes[] = {
107 [VMPRESSURE_NO_PASSTHROUGH] = "default",
108 [VMPRESSURE_HIERARCHY] = "hierarchy",
109 [VMPRESSURE_LOCAL] = "local",
110};
111
112static enum vmpressure_levels vmpressure_level(unsigned long pressure)
113{
114 if (pressure >= vmpressure_level_critical)
115 return VMPRESSURE_CRITICAL;
116 else if (pressure >= vmpressure_level_med)
117 return VMPRESSURE_MEDIUM;
118 return VMPRESSURE_LOW;
119}
120
121static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned,
122 unsigned long reclaimed)
123{
124 unsigned long scale = scanned + reclaimed;
125 unsigned long pressure = 0;
126
127
128
129
130
131
132 if (reclaimed >= scanned)
133 goto out;
134
135
136
137
138
139
140
141 pressure = scale - (reclaimed * scale / scanned);
142 pressure = pressure * 100 / scale;
143
144out:
145 pr_debug("%s: %3lu (s: %lu r: %lu)\n", __func__, pressure,
146 scanned, reclaimed);
147
148 return vmpressure_level(pressure);
149}
150
151struct vmpressure_event {
152 struct eventfd_ctx *efd;
153 enum vmpressure_levels level;
154 enum vmpressure_modes mode;
155 struct list_head node;
156};
157
158static bool vmpressure_event(struct vmpressure *vmpr,
159 const enum vmpressure_levels level,
160 bool ancestor, bool signalled)
161{
162 struct vmpressure_event *ev;
163 bool ret = false;
164
165 mutex_lock(&vmpr->events_lock);
166 list_for_each_entry(ev, &vmpr->events, node) {
167 if (ancestor && ev->mode == VMPRESSURE_LOCAL)
168 continue;
169 if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH)
170 continue;
171 if (level < ev->level)
172 continue;
173 eventfd_signal(ev->efd, 1);
174 ret = true;
175 }
176 mutex_unlock(&vmpr->events_lock);
177
178 return ret;
179}
180
181static void vmpressure_work_fn(struct work_struct *work)
182{
183 struct vmpressure *vmpr = work_to_vmpressure(work);
184 unsigned long scanned;
185 unsigned long reclaimed;
186 enum vmpressure_levels level;
187 bool ancestor = false;
188 bool signalled = false;
189
190 spin_lock(&vmpr->sr_lock);
191
192
193
194
195
196
197
198
199 scanned = vmpr->tree_scanned;
200 if (!scanned) {
201 spin_unlock(&vmpr->sr_lock);
202 return;
203 }
204
205 reclaimed = vmpr->tree_reclaimed;
206 vmpr->tree_scanned = 0;
207 vmpr->tree_reclaimed = 0;
208 spin_unlock(&vmpr->sr_lock);
209
210 level = vmpressure_calc_level(scanned, reclaimed);
211
212 do {
213 if (vmpressure_event(vmpr, level, ancestor, signalled))
214 signalled = true;
215 ancestor = true;
216 } while ((vmpr = vmpressure_parent(vmpr)));
217}
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
241 unsigned long scanned, unsigned long reclaimed)
242{
243 struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
244
245
246
247
248
249
250
251
252
253
254
255
256 if (!(gfp & (__GFP_HIGHMEM | __GFP_MOVABLE | __GFP_IO | __GFP_FS)))
257 return;
258
259
260
261
262
263
264
265
266
267 if (!scanned)
268 return;
269
270 if (tree) {
271 spin_lock(&vmpr->sr_lock);
272 scanned = vmpr->tree_scanned += scanned;
273 vmpr->tree_reclaimed += reclaimed;
274 spin_unlock(&vmpr->sr_lock);
275
276 if (scanned < vmpressure_win)
277 return;
278 schedule_work(&vmpr->work);
279 } else {
280 enum vmpressure_levels level;
281
282
283 if (!memcg || mem_cgroup_is_root(memcg))
284 return;
285
286 spin_lock(&vmpr->sr_lock);
287 scanned = vmpr->scanned += scanned;
288 reclaimed = vmpr->reclaimed += reclaimed;
289 if (scanned < vmpressure_win) {
290 spin_unlock(&vmpr->sr_lock);
291 return;
292 }
293 vmpr->scanned = vmpr->reclaimed = 0;
294 spin_unlock(&vmpr->sr_lock);
295
296 level = vmpressure_calc_level(scanned, reclaimed);
297
298 if (level > VMPRESSURE_LOW) {
299
300
301
302
303
304
305
306
307 memcg->socket_pressure = jiffies + HZ;
308 }
309 }
310}
311
312
313
314
315
316
317
318
319
320
321
322
323void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio)
324{
325
326
327
328
329 if (prio > vmpressure_level_critical_prio)
330 return;
331
332
333
334
335
336
337
338
339 vmpressure(gfp, memcg, true, vmpressure_win, 0);
340}
341
342#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2)
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362int vmpressure_register_event(struct mem_cgroup *memcg,
363 struct eventfd_ctx *eventfd, const char *args)
364{
365 struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
366 struct vmpressure_event *ev;
367 enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH;
368 enum vmpressure_levels level;
369 char *spec, *spec_orig;
370 char *token;
371 int ret = 0;
372
373 spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL);
374 if (!spec)
375 return -ENOMEM;
376
377
378 token = strsep(&spec, ",");
379 ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token);
380 if (ret < 0)
381 goto out;
382 level = ret;
383
384
385 token = strsep(&spec, ",");
386 if (token) {
387 ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token);
388 if (ret < 0)
389 goto out;
390 mode = ret;
391 }
392
393 ev = kzalloc(sizeof(*ev), GFP_KERNEL);
394 if (!ev) {
395 ret = -ENOMEM;
396 goto out;
397 }
398
399 ev->efd = eventfd;
400 ev->level = level;
401 ev->mode = mode;
402
403 mutex_lock(&vmpr->events_lock);
404 list_add(&ev->node, &vmpr->events);
405 mutex_unlock(&vmpr->events_lock);
406 ret = 0;
407out:
408 kfree(spec_orig);
409 return ret;
410}
411
412
413
414
415
416
417
418
419
420
421
422
423void vmpressure_unregister_event(struct mem_cgroup *memcg,
424 struct eventfd_ctx *eventfd)
425{
426 struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
427 struct vmpressure_event *ev;
428
429 mutex_lock(&vmpr->events_lock);
430 list_for_each_entry(ev, &vmpr->events, node) {
431 if (ev->efd != eventfd)
432 continue;
433 list_del(&ev->node);
434 kfree(ev);
435 break;
436 }
437 mutex_unlock(&vmpr->events_lock);
438}
439
440
441
442
443
444
445
446
447void vmpressure_init(struct vmpressure *vmpr)
448{
449 spin_lock_init(&vmpr->sr_lock);
450 mutex_init(&vmpr->events_lock);
451 INIT_LIST_HEAD(&vmpr->events);
452 INIT_WORK(&vmpr->work, vmpressure_work_fn);
453}
454
455
456
457
458
459
460
461
462void vmpressure_cleanup(struct vmpressure *vmpr)
463{
464
465
466
467
468 flush_work(&vmpr->work);
469}
470