1
2#ifndef _ASM_X86_RESCTRL_INTERNAL_H
3#define _ASM_X86_RESCTRL_INTERNAL_H
4
5#include <linux/sched.h>
6#include <linux/kernfs.h>
7#include <linux/fs_context.h>
8#include <linux/jump_label.h>
9
10#define MSR_IA32_L3_QOS_CFG 0xc81
11#define MSR_IA32_L2_QOS_CFG 0xc82
12#define MSR_IA32_L3_CBM_BASE 0xc90
13#define MSR_IA32_L2_CBM_BASE 0xd10
14#define MSR_IA32_MBA_THRTL_BASE 0xd50
15#define MSR_IA32_MBA_BW_BASE 0xc0000200
16
17#define MSR_IA32_QM_CTR 0x0c8e
18#define MSR_IA32_QM_EVTSEL 0x0c8d
19
20#define L3_QOS_CDP_ENABLE 0x01ULL
21
22#define L2_QOS_CDP_ENABLE 0x01ULL
23
24
25
26
27
28#define QOS_L3_OCCUP_EVENT_ID 0x01
29#define QOS_L3_MBM_TOTAL_EVENT_ID 0x02
30#define QOS_L3_MBM_LOCAL_EVENT_ID 0x03
31
32#define CQM_LIMBOCHECK_INTERVAL 1000
33
34#define MBM_CNTR_WIDTH_BASE 24
35#define MBM_OVERFLOW_INTERVAL 1000
36#define MAX_MBA_BW 100u
37#define MBA_IS_LINEAR 0x4
38#define MBA_MAX_MBPS U32_MAX
39#define MAX_MBA_BW_AMD 0x800
40#define MBM_CNTR_WIDTH_OFFSET_AMD 20
41
42#define RMID_VAL_ERROR BIT_ULL(63)
43#define RMID_VAL_UNAVAIL BIT_ULL(62)
44
45
46
47
48
49#define MBM_CNTR_WIDTH_OFFSET_MAX (62 - MBM_CNTR_WIDTH_BASE)
50
51
52struct rdt_fs_context {
53 struct kernfs_fs_context kfc;
54 bool enable_cdpl2;
55 bool enable_cdpl3;
56 bool enable_mba_mbps;
57};
58
59static inline struct rdt_fs_context *rdt_fc2context(struct fs_context *fc)
60{
61 struct kernfs_fs_context *kfc = fc->fs_private;
62
63 return container_of(kfc, struct rdt_fs_context, kfc);
64}
65
66DECLARE_STATIC_KEY_FALSE(rdt_enable_key);
67DECLARE_STATIC_KEY_FALSE(rdt_mon_enable_key);
68
69
70
71
72
73
74
75struct mon_evt {
76 u32 evtid;
77 char *name;
78 struct list_head list;
79};
80
81
82
83
84
85
86
87
88
89
90union mon_data_bits {
91 void *priv;
92 struct {
93 unsigned int rid : 10;
94 unsigned int evtid : 8;
95 unsigned int domid : 14;
96 } u;
97};
98
99struct rmid_read {
100 struct rdtgroup *rgrp;
101 struct rdt_resource *r;
102 struct rdt_domain *d;
103 int evtid;
104 bool first;
105 u64 val;
106};
107
108extern unsigned int resctrl_cqm_threshold;
109extern bool rdt_alloc_capable;
110extern bool rdt_mon_capable;
111extern unsigned int rdt_mon_features;
112
113enum rdt_group_type {
114 RDTCTRL_GROUP = 0,
115 RDTMON_GROUP,
116 RDT_NUM_GROUP,
117};
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138enum rdtgrp_mode {
139 RDT_MODE_SHAREABLE = 0,
140 RDT_MODE_EXCLUSIVE,
141 RDT_MODE_PSEUDO_LOCKSETUP,
142 RDT_MODE_PSEUDO_LOCKED,
143
144
145 RDT_NUM_MODES,
146};
147
148
149
150
151
152
153
154
155struct mongroup {
156 struct kernfs_node *mon_data_kn;
157 struct rdtgroup *parent;
158 struct list_head crdtgrp_list;
159 u32 rmid;
160};
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184struct pseudo_lock_region {
185 struct rdt_resource *r;
186 struct rdt_domain *d;
187 u32 cbm;
188 wait_queue_head_t lock_thread_wq;
189 int thread_done;
190 int cpu;
191 unsigned int line_size;
192 unsigned int size;
193 void *kmem;
194 unsigned int minor;
195 struct dentry *debugfs_dir;
196 struct list_head pm_reqs;
197};
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214struct rdtgroup {
215 struct kernfs_node *kn;
216 struct list_head rdtgroup_list;
217 u32 closid;
218 struct cpumask cpu_mask;
219 int flags;
220 atomic_t waitcount;
221 enum rdt_group_type type;
222 struct mongroup mon;
223 enum rdtgrp_mode mode;
224 struct pseudo_lock_region *plr;
225};
226
227
228#define RDT_DELETED 1
229
230
231#define RFTYPE_FLAGS_CPUS_LIST 1
232
233
234
235
236#define RFTYPE_INFO BIT(0)
237#define RFTYPE_BASE BIT(1)
238#define RF_CTRLSHIFT 4
239#define RF_MONSHIFT 5
240#define RF_TOPSHIFT 6
241#define RFTYPE_CTRL BIT(RF_CTRLSHIFT)
242#define RFTYPE_MON BIT(RF_MONSHIFT)
243#define RFTYPE_TOP BIT(RF_TOPSHIFT)
244#define RFTYPE_RES_CACHE BIT(8)
245#define RFTYPE_RES_MB BIT(9)
246#define RF_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL)
247#define RF_MON_INFO (RFTYPE_INFO | RFTYPE_MON)
248#define RF_TOP_INFO (RFTYPE_INFO | RFTYPE_TOP)
249#define RF_CTRL_BASE (RFTYPE_BASE | RFTYPE_CTRL)
250
251
252extern struct list_head rdt_all_groups;
253
254extern int max_name_width, max_data_width;
255
256int __init rdtgroup_init(void);
257void __exit rdtgroup_exit(void);
258
259
260
261
262
263
264
265
266
267
268
269struct rftype {
270 char *name;
271 umode_t mode;
272 const struct kernfs_ops *kf_ops;
273 unsigned long flags;
274 unsigned long fflags;
275
276 int (*seq_show)(struct kernfs_open_file *of,
277 struct seq_file *sf, void *v);
278
279
280
281
282
283 ssize_t (*write)(struct kernfs_open_file *of,
284 char *buf, size_t nbytes, loff_t off);
285};
286
287
288
289
290
291
292
293
294
295
296struct mbm_state {
297 u64 chunks;
298 u64 prev_msr;
299 u64 prev_bw_msr;
300 u32 prev_bw;
301 u32 delta_bw;
302 bool delta_comp;
303};
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326struct rdt_domain {
327 struct list_head list;
328 int id;
329 struct cpumask cpu_mask;
330 unsigned long *rmid_busy_llc;
331 struct mbm_state *mbm_total;
332 struct mbm_state *mbm_local;
333 struct delayed_work mbm_over;
334 struct delayed_work cqm_limbo;
335 int mbm_work_cpu;
336 int cqm_work_cpu;
337 u32 *ctrl_val;
338 u32 *mbps_val;
339 u32 new_ctrl;
340 bool have_new_ctrl;
341 struct pseudo_lock_region *plr;
342};
343
344
345
346
347
348
349
350struct msr_param {
351 struct rdt_resource *res;
352 int low;
353 int high;
354};
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371struct rdt_cache {
372 unsigned int cbm_len;
373 unsigned int min_cbm_bits;
374 unsigned int cbm_idx_mult;
375 unsigned int cbm_idx_offset;
376 unsigned int shareable_bits;
377 bool arch_has_sparse_bitmaps;
378 bool arch_has_empty_bitmaps;
379 bool arch_has_per_cpu_cfg;
380};
381
382
383
384
385
386
387
388
389
390enum membw_throttle_mode {
391 THREAD_THROTTLE_UNDEFINED = 0,
392 THREAD_THROTTLE_MAX,
393 THREAD_THROTTLE_PER_THREAD,
394};
395
396
397
398
399
400
401
402
403
404
405
406
407struct rdt_membw {
408 u32 min_bw;
409 u32 bw_gran;
410 u32 delay_linear;
411 bool arch_needs_linear;
412 enum membw_throttle_mode throttle_mode;
413 bool mba_sc;
414 u32 *mb_map;
415};
416
417static inline bool is_llc_occupancy_enabled(void)
418{
419 return (rdt_mon_features & (1 << QOS_L3_OCCUP_EVENT_ID));
420}
421
422static inline bool is_mbm_total_enabled(void)
423{
424 return (rdt_mon_features & (1 << QOS_L3_MBM_TOTAL_EVENT_ID));
425}
426
427static inline bool is_mbm_local_enabled(void)
428{
429 return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID));
430}
431
432static inline bool is_mbm_enabled(void)
433{
434 return (is_mbm_total_enabled() || is_mbm_local_enabled());
435}
436
437static inline bool is_mbm_event(int e)
438{
439 return (e >= QOS_L3_MBM_TOTAL_EVENT_ID &&
440 e <= QOS_L3_MBM_LOCAL_EVENT_ID);
441}
442
443struct rdt_parse_data {
444 struct rdtgroup *rdtgrp;
445 char *buf;
446};
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473struct rdt_resource {
474 int rid;
475 bool alloc_enabled;
476 bool mon_enabled;
477 bool alloc_capable;
478 bool mon_capable;
479 char *name;
480 int num_closid;
481 int cache_level;
482 u32 default_ctrl;
483 unsigned int msr_base;
484 void (*msr_update) (struct rdt_domain *d, struct msr_param *m,
485 struct rdt_resource *r);
486 int data_width;
487 struct list_head domains;
488 struct rdt_cache cache;
489 struct rdt_membw membw;
490 const char *format_str;
491 int (*parse_ctrlval)(struct rdt_parse_data *data,
492 struct rdt_resource *r,
493 struct rdt_domain *d);
494 struct list_head evt_list;
495 int num_rmid;
496 unsigned int mon_scale;
497 unsigned int mbm_width;
498 unsigned long fflags;
499};
500
501int parse_cbm(struct rdt_parse_data *data, struct rdt_resource *r,
502 struct rdt_domain *d);
503int parse_bw(struct rdt_parse_data *data, struct rdt_resource *r,
504 struct rdt_domain *d);
505
506extern struct mutex rdtgroup_mutex;
507
508extern struct rdt_resource rdt_resources_all[];
509extern struct rdtgroup rdtgroup_default;
510DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
511
512extern struct dentry *debugfs_resctrl;
513
514enum {
515 RDT_RESOURCE_L3,
516 RDT_RESOURCE_L3DATA,
517 RDT_RESOURCE_L3CODE,
518 RDT_RESOURCE_L2,
519 RDT_RESOURCE_L2DATA,
520 RDT_RESOURCE_L2CODE,
521 RDT_RESOURCE_MBA,
522
523
524 RDT_NUM_RESOURCES,
525};
526
527#define for_each_rdt_resource(r) \
528 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
529 r++)
530
531#define for_each_capable_rdt_resource(r) \
532 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
533 r++) \
534 if (r->alloc_capable || r->mon_capable)
535
536#define for_each_alloc_capable_rdt_resource(r) \
537 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
538 r++) \
539 if (r->alloc_capable)
540
541#define for_each_mon_capable_rdt_resource(r) \
542 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
543 r++) \
544 if (r->mon_capable)
545
546#define for_each_alloc_enabled_rdt_resource(r) \
547 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
548 r++) \
549 if (r->alloc_enabled)
550
551#define for_each_mon_enabled_rdt_resource(r) \
552 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
553 r++) \
554 if (r->mon_enabled)
555
556
557union cpuid_0x10_1_eax {
558 struct {
559 unsigned int cbm_len:5;
560 } split;
561 unsigned int full;
562};
563
564
565union cpuid_0x10_3_eax {
566 struct {
567 unsigned int max_delay:12;
568 } split;
569 unsigned int full;
570};
571
572
573union cpuid_0x10_x_edx {
574 struct {
575 unsigned int cos_max:16;
576 } split;
577 unsigned int full;
578};
579
580void rdt_last_cmd_clear(void);
581void rdt_last_cmd_puts(const char *s);
582__printf(1, 2)
583void rdt_last_cmd_printf(const char *fmt, ...);
584
585void rdt_ctrl_update(void *arg);
586struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
587void rdtgroup_kn_unlock(struct kernfs_node *kn);
588int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name);
589int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name,
590 umode_t mask);
591struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
592 struct list_head **pos);
593ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
594 char *buf, size_t nbytes, loff_t off);
595int rdtgroup_schemata_show(struct kernfs_open_file *of,
596 struct seq_file *s, void *v);
597bool rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d,
598 unsigned long cbm, int closid, bool exclusive);
599unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_domain *d,
600 unsigned long cbm);
601enum rdtgrp_mode rdtgroup_mode_by_closid(int closid);
602int rdtgroup_tasks_assigned(struct rdtgroup *r);
603int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp);
604int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp);
605bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm);
606bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d);
607int rdt_pseudo_lock_init(void);
608void rdt_pseudo_lock_release(void);
609int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp);
610void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp);
611struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
612int update_domains(struct rdt_resource *r, int closid);
613int closids_supported(void);
614void closid_free(int closid);
615int alloc_rmid(void);
616void free_rmid(u32 rmid);
617int rdt_get_mon_l3_config(struct rdt_resource *r);
618void mon_event_count(void *info);
619int rdtgroup_mondata_show(struct seq_file *m, void *arg);
620void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
621 unsigned int dom_id);
622void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
623 struct rdt_domain *d);
624void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
625 struct rdt_domain *d, struct rdtgroup *rdtgrp,
626 int evtid, int first);
627void mbm_setup_overflow_handler(struct rdt_domain *dom,
628 unsigned long delay_ms);
629void mbm_handle_overflow(struct work_struct *work);
630void __init intel_rdt_mbm_apply_quirk(void);
631bool is_mba_sc(struct rdt_resource *r);
632void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm);
633u32 delay_bw_map(unsigned long bw, struct rdt_resource *r);
634void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);
635void cqm_handle_limbo(struct work_struct *work);
636bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d);
637void __check_limbo(struct rdt_domain *d, bool force_free);
638void rdt_domain_reconfigure_cdp(struct rdt_resource *r);
639void __init thread_throttle_mode_init(void);
640
641#endif
642