1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27#include <linux/slab.h>
28#include <asm/cpu_device_id.h>
29#include "intel_rdt.h"
30
31#define MSR_IA32_QM_CTR 0x0c8e
32#define MSR_IA32_QM_EVTSEL 0x0c8d
33
34struct rmid_entry {
35 u32 rmid;
36 int busy;
37 struct list_head list;
38};
39
40
41
42
43
44
45static LIST_HEAD(rmid_free_lru);
46
47
48
49
50
51
52
53
54static unsigned int rmid_limbo_count;
55
56
57
58
59static struct rmid_entry *rmid_ptrs;
60
61
62
63
64
65bool rdt_mon_capable;
66
67
68
69
70unsigned int rdt_mon_features;
71
72
73
74
75
76unsigned int intel_cqm_threshold;
77
78static inline struct rmid_entry *__rmid_entry(u32 rmid)
79{
80 struct rmid_entry *entry;
81
82 entry = &rmid_ptrs[rmid];
83 WARN_ON(entry->rmid != rmid);
84
85 return entry;
86}
87
88static u64 __rmid_read(u32 rmid, u32 eventid)
89{
90 u64 val;
91
92
93
94
95
96
97
98
99
100 wrmsr(MSR_IA32_QM_EVTSEL, eventid, rmid);
101 rdmsrl(MSR_IA32_QM_CTR, val);
102
103 return val;
104}
105
106static bool rmid_dirty(struct rmid_entry *entry)
107{
108 u64 val = __rmid_read(entry->rmid, QOS_L3_OCCUP_EVENT_ID);
109
110 return val >= intel_cqm_threshold;
111}
112
113
114
115
116
117
118
119void __check_limbo(struct rdt_domain *d, bool force_free)
120{
121 struct rmid_entry *entry;
122 struct rdt_resource *r;
123 u32 crmid = 1, nrmid;
124
125 r = &rdt_resources_all[RDT_RESOURCE_L3];
126
127
128
129
130
131
132
133 for (;;) {
134 nrmid = find_next_bit(d->rmid_busy_llc, r->num_rmid, crmid);
135 if (nrmid >= r->num_rmid)
136 break;
137
138 entry = __rmid_entry(nrmid);
139 if (force_free || !rmid_dirty(entry)) {
140 clear_bit(entry->rmid, d->rmid_busy_llc);
141 if (!--entry->busy) {
142 rmid_limbo_count--;
143 list_add_tail(&entry->list, &rmid_free_lru);
144 }
145 }
146 crmid = nrmid + 1;
147 }
148}
149
150bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d)
151{
152 return find_first_bit(d->rmid_busy_llc, r->num_rmid) != r->num_rmid;
153}
154
155
156
157
158
159
160int alloc_rmid(void)
161{
162 struct rmid_entry *entry;
163
164 lockdep_assert_held(&rdtgroup_mutex);
165
166 if (list_empty(&rmid_free_lru))
167 return rmid_limbo_count ? -EBUSY : -ENOSPC;
168
169 entry = list_first_entry(&rmid_free_lru,
170 struct rmid_entry, list);
171 list_del(&entry->list);
172
173 return entry->rmid;
174}
175
176static void add_rmid_to_limbo(struct rmid_entry *entry)
177{
178 struct rdt_resource *r;
179 struct rdt_domain *d;
180 int cpu;
181 u64 val;
182
183 r = &rdt_resources_all[RDT_RESOURCE_L3];
184
185 entry->busy = 0;
186 cpu = get_cpu();
187 list_for_each_entry(d, &r->domains, list) {
188 if (cpumask_test_cpu(cpu, &d->cpu_mask)) {
189 val = __rmid_read(entry->rmid, QOS_L3_OCCUP_EVENT_ID);
190 if (val <= intel_cqm_threshold)
191 continue;
192 }
193
194
195
196
197
198 if (!has_busy_rmid(r, d))
199 cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL);
200 set_bit(entry->rmid, d->rmid_busy_llc);
201 entry->busy++;
202 }
203 put_cpu();
204
205 if (entry->busy)
206 rmid_limbo_count++;
207 else
208 list_add_tail(&entry->list, &rmid_free_lru);
209}
210
211void free_rmid(u32 rmid)
212{
213 struct rmid_entry *entry;
214
215 if (!rmid)
216 return;
217
218 lockdep_assert_held(&rdtgroup_mutex);
219
220 entry = __rmid_entry(rmid);
221
222 if (is_llc_occupancy_enabled())
223 add_rmid_to_limbo(entry);
224 else
225 list_add_tail(&entry->list, &rmid_free_lru);
226}
227
228static int __mon_event_count(u32 rmid, struct rmid_read *rr)
229{
230 u64 chunks, shift, tval;
231 struct mbm_state *m;
232
233 tval = __rmid_read(rmid, rr->evtid);
234 if (tval & (RMID_VAL_ERROR | RMID_VAL_UNAVAIL)) {
235 rr->val = tval;
236 return -EINVAL;
237 }
238 switch (rr->evtid) {
239 case QOS_L3_OCCUP_EVENT_ID:
240 rr->val += tval;
241 return 0;
242 case QOS_L3_MBM_TOTAL_EVENT_ID:
243 m = &rr->d->mbm_total[rmid];
244 break;
245 case QOS_L3_MBM_LOCAL_EVENT_ID:
246 m = &rr->d->mbm_local[rmid];
247 break;
248 default:
249
250
251
252
253 return -EINVAL;
254 }
255
256 if (rr->first) {
257 m->prev_msr = tval;
258 m->chunks = 0;
259 return 0;
260 }
261
262 shift = 64 - MBM_CNTR_WIDTH;
263 chunks = (tval << shift) - (m->prev_msr << shift);
264 chunks >>= shift;
265 m->chunks += chunks;
266 m->prev_msr = tval;
267
268 rr->val += m->chunks;
269 return 0;
270}
271
272
273
274
275
276void mon_event_count(void *info)
277{
278 struct rdtgroup *rdtgrp, *entry;
279 struct rmid_read *rr = info;
280 struct list_head *head;
281
282 rdtgrp = rr->rgrp;
283
284 if (__mon_event_count(rdtgrp->mon.rmid, rr))
285 return;
286
287
288
289
290 head = &rdtgrp->mon.crdtgrp_list;
291
292 if (rdtgrp->type == RDTCTRL_GROUP) {
293 list_for_each_entry(entry, head, mon.crdtgrp_list) {
294 if (__mon_event_count(entry->mon.rmid, rr))
295 return;
296 }
297 }
298}
299
300static void mbm_update(struct rdt_domain *d, int rmid)
301{
302 struct rmid_read rr;
303
304 rr.first = false;
305 rr.d = d;
306
307
308
309
310
311 if (is_mbm_total_enabled()) {
312 rr.evtid = QOS_L3_MBM_TOTAL_EVENT_ID;
313 __mon_event_count(rmid, &rr);
314 }
315 if (is_mbm_local_enabled()) {
316 rr.evtid = QOS_L3_MBM_LOCAL_EVENT_ID;
317 __mon_event_count(rmid, &rr);
318 }
319}
320
321
322
323
324
325void cqm_handle_limbo(struct work_struct *work)
326{
327 unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL);
328 int cpu = smp_processor_id();
329 struct rdt_resource *r;
330 struct rdt_domain *d;
331
332 mutex_lock(&rdtgroup_mutex);
333
334 r = &rdt_resources_all[RDT_RESOURCE_L3];
335 d = get_domain_from_cpu(cpu, r);
336
337 if (!d) {
338 pr_warn_once("Failure to get domain for limbo worker\n");
339 goto out_unlock;
340 }
341
342 __check_limbo(d, false);
343
344 if (has_busy_rmid(r, d))
345 schedule_delayed_work_on(cpu, &d->cqm_limbo, delay);
346
347out_unlock:
348 mutex_unlock(&rdtgroup_mutex);
349}
350
351void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms)
352{
353 unsigned long delay = msecs_to_jiffies(delay_ms);
354 struct rdt_resource *r;
355 int cpu;
356
357 r = &rdt_resources_all[RDT_RESOURCE_L3];
358
359 cpu = cpumask_any(&dom->cpu_mask);
360 dom->cqm_work_cpu = cpu;
361
362 schedule_delayed_work_on(cpu, &dom->cqm_limbo, delay);
363}
364
365void mbm_handle_overflow(struct work_struct *work)
366{
367 unsigned long delay = msecs_to_jiffies(MBM_OVERFLOW_INTERVAL);
368 struct rdtgroup *prgrp, *crgrp;
369 int cpu = smp_processor_id();
370 struct list_head *head;
371 struct rdt_domain *d;
372
373 mutex_lock(&rdtgroup_mutex);
374
375 if (!static_branch_likely(&rdt_enable_key))
376 goto out_unlock;
377
378 d = get_domain_from_cpu(cpu, &rdt_resources_all[RDT_RESOURCE_L3]);
379 if (!d)
380 goto out_unlock;
381
382 list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
383 mbm_update(d, prgrp->mon.rmid);
384
385 head = &prgrp->mon.crdtgrp_list;
386 list_for_each_entry(crgrp, head, mon.crdtgrp_list)
387 mbm_update(d, crgrp->mon.rmid);
388 }
389
390 schedule_delayed_work_on(cpu, &d->mbm_over, delay);
391
392out_unlock:
393 mutex_unlock(&rdtgroup_mutex);
394}
395
396void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms)
397{
398 unsigned long delay = msecs_to_jiffies(delay_ms);
399 int cpu;
400
401 if (!static_branch_likely(&rdt_enable_key))
402 return;
403 cpu = cpumask_any(&dom->cpu_mask);
404 dom->mbm_work_cpu = cpu;
405 schedule_delayed_work_on(cpu, &dom->mbm_over, delay);
406}
407
408static int dom_data_init(struct rdt_resource *r)
409{
410 struct rmid_entry *entry = NULL;
411 int i, nr_rmids;
412
413 nr_rmids = r->num_rmid;
414 rmid_ptrs = kcalloc(nr_rmids, sizeof(struct rmid_entry), GFP_KERNEL);
415 if (!rmid_ptrs)
416 return -ENOMEM;
417
418 for (i = 0; i < nr_rmids; i++) {
419 entry = &rmid_ptrs[i];
420 INIT_LIST_HEAD(&entry->list);
421
422 entry->rmid = i;
423 list_add_tail(&entry->list, &rmid_free_lru);
424 }
425
426
427
428
429
430 entry = __rmid_entry(0);
431 list_del(&entry->list);
432
433 return 0;
434}
435
436static struct mon_evt llc_occupancy_event = {
437 .name = "llc_occupancy",
438 .evtid = QOS_L3_OCCUP_EVENT_ID,
439};
440
441static struct mon_evt mbm_total_event = {
442 .name = "mbm_total_bytes",
443 .evtid = QOS_L3_MBM_TOTAL_EVENT_ID,
444};
445
446static struct mon_evt mbm_local_event = {
447 .name = "mbm_local_bytes",
448 .evtid = QOS_L3_MBM_LOCAL_EVENT_ID,
449};
450
451
452
453
454
455
456
457
458static void l3_mon_evt_init(struct rdt_resource *r)
459{
460 INIT_LIST_HEAD(&r->evt_list);
461
462 if (is_llc_occupancy_enabled())
463 list_add_tail(&llc_occupancy_event.list, &r->evt_list);
464 if (is_mbm_total_enabled())
465 list_add_tail(&mbm_total_event.list, &r->evt_list);
466 if (is_mbm_local_enabled())
467 list_add_tail(&mbm_local_event.list, &r->evt_list);
468}
469
470int rdt_get_mon_l3_config(struct rdt_resource *r)
471{
472 int ret;
473
474 r->mon_scale = boot_cpu_data.x86_cache_occ_scale;
475 r->num_rmid = boot_cpu_data.x86_cache_max_rmid + 1;
476
477
478
479
480
481
482
483
484 intel_cqm_threshold = boot_cpu_data.x86_cache_size * 1024 / r->num_rmid;
485
486
487 intel_cqm_threshold /= r->mon_scale;
488
489 ret = dom_data_init(r);
490 if (ret)
491 return ret;
492
493 l3_mon_evt_init(r);
494
495 r->mon_capable = true;
496 r->mon_enabled = true;
497
498 return 0;
499}
500