1
2
3
4
5
6
7
8#ifndef _LRU_LIST_H
9#define _LRU_LIST_H
10
11#include <linux/list.h>
12#include <linux/nodemask.h>
13#include <linux/shrinker.h>
14
15struct mem_cgroup;
16
17
18enum lru_status {
19 LRU_REMOVED,
20 LRU_REMOVED_RETRY,
21
22 LRU_ROTATE,
23 LRU_SKIP,
24 LRU_RETRY,
25
26};
27
28struct list_lru_one {
29 struct list_head list;
30
31 long nr_items;
32};
33
34struct list_lru_memcg {
35 struct rcu_head rcu;
36
37 struct list_lru_one *lru[];
38};
39
40struct list_lru_node {
41
42 spinlock_t lock;
43
44 struct list_lru_one lru;
45#ifdef CONFIG_MEMCG_KMEM
46
47 struct list_lru_memcg __rcu *memcg_lrus;
48#endif
49 long nr_items;
50} ____cacheline_aligned_in_smp;
51
52struct list_lru {
53 struct list_lru_node *node;
54#ifdef CONFIG_MEMCG_KMEM
55 struct list_head list;
56 int shrinker_id;
57 bool memcg_aware;
58#endif
59};
60
61void list_lru_destroy(struct list_lru *lru);
62int __list_lru_init(struct list_lru *lru, bool memcg_aware,
63 struct lock_class_key *key, struct shrinker *shrinker);
64
65#define list_lru_init(lru) \
66 __list_lru_init((lru), false, NULL, NULL)
67#define list_lru_init_key(lru, key) \
68 __list_lru_init((lru), false, (key), NULL)
69#define list_lru_init_memcg(lru, shrinker) \
70 __list_lru_init((lru), true, NULL, shrinker)
71
72int memcg_update_all_list_lrus(int num_memcgs);
73void memcg_drain_all_list_lrus(int src_idx, struct mem_cgroup *dst_memcg);
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91bool list_lru_add(struct list_lru *lru, struct list_head *item);
92
93
94
95
96
97
98
99
100
101
102
103
104bool list_lru_del(struct list_lru *lru, struct list_head *item);
105
106
107
108
109
110
111
112
113
114
115
116unsigned long list_lru_count_one(struct list_lru *lru,
117 int nid, struct mem_cgroup *memcg);
118unsigned long list_lru_count_node(struct list_lru *lru, int nid);
119
120static inline unsigned long list_lru_shrink_count(struct list_lru *lru,
121 struct shrink_control *sc)
122{
123 return list_lru_count_one(lru, sc->nid, sc->memcg);
124}
125
126static inline unsigned long list_lru_count(struct list_lru *lru)
127{
128 long count = 0;
129 int nid;
130
131 for_each_node_state(nid, N_NORMAL_MEMORY)
132 count += list_lru_count_node(lru, nid);
133
134 return count;
135}
136
137void list_lru_isolate(struct list_lru_one *list, struct list_head *item);
138void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item,
139 struct list_head *head);
140
141typedef enum lru_status (*list_lru_walk_cb)(struct list_head *item,
142 struct list_lru_one *list, spinlock_t *lock, void *cb_arg);
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166unsigned long list_lru_walk_one(struct list_lru *lru,
167 int nid, struct mem_cgroup *memcg,
168 list_lru_walk_cb isolate, void *cb_arg,
169 unsigned long *nr_to_walk);
170
171
172
173
174
175
176
177
178
179
180
181
182
183unsigned long list_lru_walk_one_irq(struct list_lru *lru,
184 int nid, struct mem_cgroup *memcg,
185 list_lru_walk_cb isolate, void *cb_arg,
186 unsigned long *nr_to_walk);
187unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
188 list_lru_walk_cb isolate, void *cb_arg,
189 unsigned long *nr_to_walk);
190
191static inline unsigned long
192list_lru_shrink_walk(struct list_lru *lru, struct shrink_control *sc,
193 list_lru_walk_cb isolate, void *cb_arg)
194{
195 return list_lru_walk_one(lru, sc->nid, sc->memcg, isolate, cb_arg,
196 &sc->nr_to_scan);
197}
198
199static inline unsigned long
200list_lru_shrink_walk_irq(struct list_lru *lru, struct shrink_control *sc,
201 list_lru_walk_cb isolate, void *cb_arg)
202{
203 return list_lru_walk_one_irq(lru, sc->nid, sc->memcg, isolate, cb_arg,
204 &sc->nr_to_scan);
205}
206
207static inline unsigned long
208list_lru_walk(struct list_lru *lru, list_lru_walk_cb isolate,
209 void *cb_arg, unsigned long nr_to_walk)
210{
211 long isolated = 0;
212 int nid;
213
214 for_each_node_state(nid, N_NORMAL_MEMORY) {
215 isolated += list_lru_walk_node(lru, nid, isolate,
216 cb_arg, &nr_to_walk);
217 if (nr_to_walk <= 0)
218 break;
219 }
220 return isolated;
221}
222#endif
223