1
2
3
4
5
6
7
8
9
10
11
12
13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14
15#include <linux/cacheinfo.h>
16#include <linux/cpu.h>
17#include <linux/debugfs.h>
18#include <linux/fs.h>
19#include <linux/fs_parser.h>
20#include <linux/sysfs.h>
21#include <linux/kernfs.h>
22#include <linux/seq_buf.h>
23#include <linux/seq_file.h>
24#include <linux/sched/signal.h>
25#include <linux/sched/task.h>
26#include <linux/slab.h>
27#include <linux/task_work.h>
28#include <linux/user_namespace.h>
29
30#include <uapi/linux/magic.h>
31
32#include <asm/resctrl.h>
33#include "internal.h"
34
35DEFINE_STATIC_KEY_FALSE(rdt_enable_key);
36DEFINE_STATIC_KEY_FALSE(rdt_mon_enable_key);
37DEFINE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
38static struct kernfs_root *rdt_root;
39struct rdtgroup rdtgroup_default;
40LIST_HEAD(rdt_all_groups);
41
42
43static struct kernfs_node *kn_info;
44
45
46static struct kernfs_node *kn_mongrp;
47
48
49static struct kernfs_node *kn_mondata;
50
51static struct seq_buf last_cmd_status;
52static char last_cmd_status_buf[512];
53
54struct dentry *debugfs_resctrl;
55
56void rdt_last_cmd_clear(void)
57{
58 lockdep_assert_held(&rdtgroup_mutex);
59 seq_buf_clear(&last_cmd_status);
60}
61
62void rdt_last_cmd_puts(const char *s)
63{
64 lockdep_assert_held(&rdtgroup_mutex);
65 seq_buf_puts(&last_cmd_status, s);
66}
67
68void rdt_last_cmd_printf(const char *fmt, ...)
69{
70 va_list ap;
71
72 va_start(ap, fmt);
73 lockdep_assert_held(&rdtgroup_mutex);
74 seq_buf_vprintf(&last_cmd_status, fmt, ap);
75 va_end(ap);
76}
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93static int closid_free_map;
94static int closid_free_map_len;
95
96int closids_supported(void)
97{
98 return closid_free_map_len;
99}
100
101static void closid_init(void)
102{
103 struct rdt_resource *r;
104 int rdt_min_closid = 32;
105
106
107 for_each_alloc_enabled_rdt_resource(r)
108 rdt_min_closid = min(rdt_min_closid, r->num_closid);
109
110 closid_free_map = BIT_MASK(rdt_min_closid) - 1;
111
112
113 closid_free_map &= ~1;
114 closid_free_map_len = rdt_min_closid;
115}
116
117static int closid_alloc(void)
118{
119 u32 closid = ffs(closid_free_map);
120
121 if (closid == 0)
122 return -ENOSPC;
123 closid--;
124 closid_free_map &= ~(1 << closid);
125
126 return closid;
127}
128
129void closid_free(int closid)
130{
131 closid_free_map |= 1 << closid;
132}
133
134
135
136
137
138
139
140
141static bool closid_allocated(unsigned int closid)
142{
143 return (closid_free_map & (1 << closid)) == 0;
144}
145
146
147
148
149
150
151
152
153
154
155enum rdtgrp_mode rdtgroup_mode_by_closid(int closid)
156{
157 struct rdtgroup *rdtgrp;
158
159 list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) {
160 if (rdtgrp->closid == closid)
161 return rdtgrp->mode;
162 }
163
164 return RDT_NUM_MODES;
165}
166
167static const char * const rdt_mode_str[] = {
168 [RDT_MODE_SHAREABLE] = "shareable",
169 [RDT_MODE_EXCLUSIVE] = "exclusive",
170 [RDT_MODE_PSEUDO_LOCKSETUP] = "pseudo-locksetup",
171 [RDT_MODE_PSEUDO_LOCKED] = "pseudo-locked",
172};
173
174
175
176
177
178
179
180static const char *rdtgroup_mode_str(enum rdtgrp_mode mode)
181{
182 if (mode < RDT_MODE_SHAREABLE || mode >= RDT_NUM_MODES)
183 return "unknown";
184
185 return rdt_mode_str[mode];
186}
187
188
189static int rdtgroup_kn_set_ugid(struct kernfs_node *kn)
190{
191 struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
192 .ia_uid = current_fsuid(),
193 .ia_gid = current_fsgid(), };
194
195 if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
196 gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
197 return 0;
198
199 return kernfs_setattr(kn, &iattr);
200}
201
202static int rdtgroup_add_file(struct kernfs_node *parent_kn, struct rftype *rft)
203{
204 struct kernfs_node *kn;
205 int ret;
206
207 kn = __kernfs_create_file(parent_kn, rft->name, rft->mode,
208 GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
209 0, rft->kf_ops, rft, NULL, NULL);
210 if (IS_ERR(kn))
211 return PTR_ERR(kn);
212
213 ret = rdtgroup_kn_set_ugid(kn);
214 if (ret) {
215 kernfs_remove(kn);
216 return ret;
217 }
218
219 return 0;
220}
221
222static int rdtgroup_seqfile_show(struct seq_file *m, void *arg)
223{
224 struct kernfs_open_file *of = m->private;
225 struct rftype *rft = of->kn->priv;
226
227 if (rft->seq_show)
228 return rft->seq_show(of, m, arg);
229 return 0;
230}
231
232static ssize_t rdtgroup_file_write(struct kernfs_open_file *of, char *buf,
233 size_t nbytes, loff_t off)
234{
235 struct rftype *rft = of->kn->priv;
236
237 if (rft->write)
238 return rft->write(of, buf, nbytes, off);
239
240 return -EINVAL;
241}
242
243static const struct kernfs_ops rdtgroup_kf_single_ops = {
244 .atomic_write_len = PAGE_SIZE,
245 .write = rdtgroup_file_write,
246 .seq_show = rdtgroup_seqfile_show,
247};
248
249static const struct kernfs_ops kf_mondata_ops = {
250 .atomic_write_len = PAGE_SIZE,
251 .seq_show = rdtgroup_mondata_show,
252};
253
254static bool is_cpu_list(struct kernfs_open_file *of)
255{
256 struct rftype *rft = of->kn->priv;
257
258 return rft->flags & RFTYPE_FLAGS_CPUS_LIST;
259}
260
261static int rdtgroup_cpus_show(struct kernfs_open_file *of,
262 struct seq_file *s, void *v)
263{
264 struct rdtgroup *rdtgrp;
265 struct cpumask *mask;
266 int ret = 0;
267
268 rdtgrp = rdtgroup_kn_lock_live(of->kn);
269
270 if (rdtgrp) {
271 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
272 if (!rdtgrp->plr->d) {
273 rdt_last_cmd_clear();
274 rdt_last_cmd_puts("Cache domain offline\n");
275 ret = -ENODEV;
276 } else {
277 mask = &rdtgrp->plr->d->cpu_mask;
278 seq_printf(s, is_cpu_list(of) ?
279 "%*pbl\n" : "%*pb\n",
280 cpumask_pr_args(mask));
281 }
282 } else {
283 seq_printf(s, is_cpu_list(of) ? "%*pbl\n" : "%*pb\n",
284 cpumask_pr_args(&rdtgrp->cpu_mask));
285 }
286 } else {
287 ret = -ENOENT;
288 }
289 rdtgroup_kn_unlock(of->kn);
290
291 return ret;
292}
293
294
295
296
297
298
299
300static void update_cpu_closid_rmid(void *info)
301{
302 struct rdtgroup *r = info;
303
304 if (r) {
305 this_cpu_write(pqr_state.default_closid, r->closid);
306 this_cpu_write(pqr_state.default_rmid, r->mon.rmid);
307 }
308
309
310
311
312
313
314 resctrl_sched_in();
315}
316
317
318
319
320
321
322static void
323update_closid_rmid(const struct cpumask *cpu_mask, struct rdtgroup *r)
324{
325 int cpu = get_cpu();
326
327 if (cpumask_test_cpu(cpu, cpu_mask))
328 update_cpu_closid_rmid(r);
329 smp_call_function_many(cpu_mask, update_cpu_closid_rmid, r, 1);
330 put_cpu();
331}
332
333static int cpus_mon_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask,
334 cpumask_var_t tmpmask)
335{
336 struct rdtgroup *prgrp = rdtgrp->mon.parent, *crgrp;
337 struct list_head *head;
338
339
340 cpumask_andnot(tmpmask, newmask, &prgrp->cpu_mask);
341 if (cpumask_weight(tmpmask)) {
342 rdt_last_cmd_puts("Can only add CPUs to mongroup that belong to parent\n");
343 return -EINVAL;
344 }
345
346
347 cpumask_andnot(tmpmask, &rdtgrp->cpu_mask, newmask);
348 if (cpumask_weight(tmpmask)) {
349
350 cpumask_or(&prgrp->cpu_mask, &prgrp->cpu_mask, tmpmask);
351 update_closid_rmid(tmpmask, prgrp);
352 }
353
354
355
356
357
358 cpumask_andnot(tmpmask, newmask, &rdtgrp->cpu_mask);
359 if (cpumask_weight(tmpmask)) {
360 head = &prgrp->mon.crdtgrp_list;
361 list_for_each_entry(crgrp, head, mon.crdtgrp_list) {
362 if (crgrp == rdtgrp)
363 continue;
364 cpumask_andnot(&crgrp->cpu_mask, &crgrp->cpu_mask,
365 tmpmask);
366 }
367 update_closid_rmid(tmpmask, rdtgrp);
368 }
369
370
371 cpumask_copy(&rdtgrp->cpu_mask, newmask);
372
373 return 0;
374}
375
376static void cpumask_rdtgrp_clear(struct rdtgroup *r, struct cpumask *m)
377{
378 struct rdtgroup *crgrp;
379
380 cpumask_andnot(&r->cpu_mask, &r->cpu_mask, m);
381
382 list_for_each_entry(crgrp, &r->mon.crdtgrp_list, mon.crdtgrp_list)
383 cpumask_and(&crgrp->cpu_mask, &r->cpu_mask, &crgrp->cpu_mask);
384}
385
386static int cpus_ctrl_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask,
387 cpumask_var_t tmpmask, cpumask_var_t tmpmask1)
388{
389 struct rdtgroup *r, *crgrp;
390 struct list_head *head;
391
392
393 cpumask_andnot(tmpmask, &rdtgrp->cpu_mask, newmask);
394 if (cpumask_weight(tmpmask)) {
395
396 if (rdtgrp == &rdtgroup_default) {
397 rdt_last_cmd_puts("Can't drop CPUs from default group\n");
398 return -EINVAL;
399 }
400
401
402 cpumask_or(&rdtgroup_default.cpu_mask,
403 &rdtgroup_default.cpu_mask, tmpmask);
404 update_closid_rmid(tmpmask, &rdtgroup_default);
405 }
406
407
408
409
410
411
412 cpumask_andnot(tmpmask, newmask, &rdtgrp->cpu_mask);
413 if (cpumask_weight(tmpmask)) {
414 list_for_each_entry(r, &rdt_all_groups, rdtgroup_list) {
415 if (r == rdtgrp)
416 continue;
417 cpumask_and(tmpmask1, &r->cpu_mask, tmpmask);
418 if (cpumask_weight(tmpmask1))
419 cpumask_rdtgrp_clear(r, tmpmask1);
420 }
421 update_closid_rmid(tmpmask, rdtgrp);
422 }
423
424
425 cpumask_copy(&rdtgrp->cpu_mask, newmask);
426
427
428
429
430
431 head = &rdtgrp->mon.crdtgrp_list;
432 list_for_each_entry(crgrp, head, mon.crdtgrp_list) {
433 cpumask_and(tmpmask, &rdtgrp->cpu_mask, &crgrp->cpu_mask);
434 update_closid_rmid(tmpmask, rdtgrp);
435 cpumask_clear(&crgrp->cpu_mask);
436 }
437
438 return 0;
439}
440
441static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of,
442 char *buf, size_t nbytes, loff_t off)
443{
444 cpumask_var_t tmpmask, newmask, tmpmask1;
445 struct rdtgroup *rdtgrp;
446 int ret;
447
448 if (!buf)
449 return -EINVAL;
450
451 if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
452 return -ENOMEM;
453 if (!zalloc_cpumask_var(&newmask, GFP_KERNEL)) {
454 free_cpumask_var(tmpmask);
455 return -ENOMEM;
456 }
457 if (!zalloc_cpumask_var(&tmpmask1, GFP_KERNEL)) {
458 free_cpumask_var(tmpmask);
459 free_cpumask_var(newmask);
460 return -ENOMEM;
461 }
462
463 rdtgrp = rdtgroup_kn_lock_live(of->kn);
464 if (!rdtgrp) {
465 ret = -ENOENT;
466 goto unlock;
467 }
468
469 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED ||
470 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
471 ret = -EINVAL;
472 rdt_last_cmd_puts("Pseudo-locking in progress\n");
473 goto unlock;
474 }
475
476 if (is_cpu_list(of))
477 ret = cpulist_parse(buf, newmask);
478 else
479 ret = cpumask_parse(buf, newmask);
480
481 if (ret) {
482 rdt_last_cmd_puts("Bad CPU list/mask\n");
483 goto unlock;
484 }
485
486
487 cpumask_andnot(tmpmask, newmask, cpu_online_mask);
488 if (cpumask_weight(tmpmask)) {
489 ret = -EINVAL;
490 rdt_last_cmd_puts("Can only assign online CPUs\n");
491 goto unlock;
492 }
493
494 if (rdtgrp->type == RDTCTRL_GROUP)
495 ret = cpus_ctrl_write(rdtgrp, newmask, tmpmask, tmpmask1);
496 else if (rdtgrp->type == RDTMON_GROUP)
497 ret = cpus_mon_write(rdtgrp, newmask, tmpmask);
498 else
499 ret = -EINVAL;
500
501unlock:
502 rdtgroup_kn_unlock(of->kn);
503 free_cpumask_var(tmpmask);
504 free_cpumask_var(newmask);
505 free_cpumask_var(tmpmask1);
506
507 return ret ?: nbytes;
508}
509
510
511
512
513
514
515
516
517
518
519
520
521
522static void rdtgroup_remove(struct rdtgroup *rdtgrp)
523{
524 kernfs_put(rdtgrp->kn);
525 kfree(rdtgrp);
526}
527
528static void _update_task_closid_rmid(void *task)
529{
530
531
532
533
534 if (task == current)
535 resctrl_sched_in();
536}
537
538static void update_task_closid_rmid(struct task_struct *t)
539{
540 if (IS_ENABLED(CONFIG_SMP) && task_curr(t))
541 smp_call_function_single(task_cpu(t), _update_task_closid_rmid, t, 1);
542 else
543 _update_task_closid_rmid(t);
544}
545
546static int __rdtgroup_move_task(struct task_struct *tsk,
547 struct rdtgroup *rdtgrp)
548{
549
550 if ((rdtgrp->type == RDTCTRL_GROUP && tsk->closid == rdtgrp->closid &&
551 tsk->rmid == rdtgrp->mon.rmid) ||
552 (rdtgrp->type == RDTMON_GROUP && tsk->rmid == rdtgrp->mon.rmid &&
553 tsk->closid == rdtgrp->mon.parent->closid))
554 return 0;
555
556
557
558
559
560
561
562
563
564
565 if (rdtgrp->type == RDTCTRL_GROUP) {
566 WRITE_ONCE(tsk->closid, rdtgrp->closid);
567 WRITE_ONCE(tsk->rmid, rdtgrp->mon.rmid);
568 } else if (rdtgrp->type == RDTMON_GROUP) {
569 if (rdtgrp->mon.parent->closid == tsk->closid) {
570 WRITE_ONCE(tsk->rmid, rdtgrp->mon.rmid);
571 } else {
572 rdt_last_cmd_puts("Can't move task to different control group\n");
573 return -EINVAL;
574 }
575 }
576
577
578
579
580
581 barrier();
582
583
584
585
586
587
588
589 update_task_closid_rmid(tsk);
590
591 return 0;
592}
593
594static bool is_closid_match(struct task_struct *t, struct rdtgroup *r)
595{
596 return (rdt_alloc_capable &&
597 (r->type == RDTCTRL_GROUP) && (t->closid == r->closid));
598}
599
600static bool is_rmid_match(struct task_struct *t, struct rdtgroup *r)
601{
602 return (rdt_mon_capable &&
603 (r->type == RDTMON_GROUP) && (t->rmid == r->mon.rmid));
604}
605
606
607
608
609
610
611
612int rdtgroup_tasks_assigned(struct rdtgroup *r)
613{
614 struct task_struct *p, *t;
615 int ret = 0;
616
617 lockdep_assert_held(&rdtgroup_mutex);
618
619 rcu_read_lock();
620 for_each_process_thread(p, t) {
621 if (is_closid_match(t, r) || is_rmid_match(t, r)) {
622 ret = 1;
623 break;
624 }
625 }
626 rcu_read_unlock();
627
628 return ret;
629}
630
631static int rdtgroup_task_write_permission(struct task_struct *task,
632 struct kernfs_open_file *of)
633{
634 const struct cred *tcred = get_task_cred(task);
635 const struct cred *cred = current_cred();
636 int ret = 0;
637
638
639
640
641
642 if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
643 !uid_eq(cred->euid, tcred->uid) &&
644 !uid_eq(cred->euid, tcred->suid)) {
645 rdt_last_cmd_printf("No permission to move task %d\n", task->pid);
646 ret = -EPERM;
647 }
648
649 put_cred(tcred);
650 return ret;
651}
652
653static int rdtgroup_move_task(pid_t pid, struct rdtgroup *rdtgrp,
654 struct kernfs_open_file *of)
655{
656 struct task_struct *tsk;
657 int ret;
658
659 rcu_read_lock();
660 if (pid) {
661 tsk = find_task_by_vpid(pid);
662 if (!tsk) {
663 rcu_read_unlock();
664 rdt_last_cmd_printf("No task %d\n", pid);
665 return -ESRCH;
666 }
667 } else {
668 tsk = current;
669 }
670
671 get_task_struct(tsk);
672 rcu_read_unlock();
673
674 ret = rdtgroup_task_write_permission(tsk, of);
675 if (!ret)
676 ret = __rdtgroup_move_task(tsk, rdtgrp);
677
678 put_task_struct(tsk);
679 return ret;
680}
681
682static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of,
683 char *buf, size_t nbytes, loff_t off)
684{
685 struct rdtgroup *rdtgrp;
686 int ret = 0;
687 pid_t pid;
688
689 if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
690 return -EINVAL;
691 rdtgrp = rdtgroup_kn_lock_live(of->kn);
692 if (!rdtgrp) {
693 rdtgroup_kn_unlock(of->kn);
694 return -ENOENT;
695 }
696 rdt_last_cmd_clear();
697
698 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED ||
699 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
700 ret = -EINVAL;
701 rdt_last_cmd_puts("Pseudo-locking in progress\n");
702 goto unlock;
703 }
704
705 ret = rdtgroup_move_task(pid, rdtgrp, of);
706
707unlock:
708 rdtgroup_kn_unlock(of->kn);
709
710 return ret ?: nbytes;
711}
712
713static void show_rdt_tasks(struct rdtgroup *r, struct seq_file *s)
714{
715 struct task_struct *p, *t;
716
717 rcu_read_lock();
718 for_each_process_thread(p, t) {
719 if (is_closid_match(t, r) || is_rmid_match(t, r))
720 seq_printf(s, "%d\n", t->pid);
721 }
722 rcu_read_unlock();
723}
724
725static int rdtgroup_tasks_show(struct kernfs_open_file *of,
726 struct seq_file *s, void *v)
727{
728 struct rdtgroup *rdtgrp;
729 int ret = 0;
730
731 rdtgrp = rdtgroup_kn_lock_live(of->kn);
732 if (rdtgrp)
733 show_rdt_tasks(rdtgrp, s);
734 else
735 ret = -ENOENT;
736 rdtgroup_kn_unlock(of->kn);
737
738 return ret;
739}
740
741#ifdef CONFIG_PROC_CPU_RESCTRL
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774int proc_resctrl_show(struct seq_file *s, struct pid_namespace *ns,
775 struct pid *pid, struct task_struct *tsk)
776{
777 struct rdtgroup *rdtg;
778 int ret = 0;
779
780 mutex_lock(&rdtgroup_mutex);
781
782
783 if (!static_branch_unlikely(&rdt_enable_key)) {
784 seq_puts(s, "res:\nmon:\n");
785 goto unlock;
786 }
787
788 list_for_each_entry(rdtg, &rdt_all_groups, rdtgroup_list) {
789 struct rdtgroup *crg;
790
791
792
793
794
795 if (rdtg->mode != RDT_MODE_SHAREABLE &&
796 rdtg->mode != RDT_MODE_EXCLUSIVE)
797 continue;
798
799 if (rdtg->closid != tsk->closid)
800 continue;
801
802 seq_printf(s, "res:%s%s\n", (rdtg == &rdtgroup_default) ? "/" : "",
803 rdtg->kn->name);
804 seq_puts(s, "mon:");
805 list_for_each_entry(crg, &rdtg->mon.crdtgrp_list,
806 mon.crdtgrp_list) {
807 if (tsk->rmid != crg->mon.rmid)
808 continue;
809 seq_printf(s, "%s", crg->kn->name);
810 break;
811 }
812 seq_putc(s, '\n');
813 goto unlock;
814 }
815
816
817
818
819 ret = -ENOENT;
820unlock:
821 mutex_unlock(&rdtgroup_mutex);
822
823 return ret;
824}
825#endif
826
827static int rdt_last_cmd_status_show(struct kernfs_open_file *of,
828 struct seq_file *seq, void *v)
829{
830 int len;
831
832 mutex_lock(&rdtgroup_mutex);
833 len = seq_buf_used(&last_cmd_status);
834 if (len)
835 seq_printf(seq, "%.*s", len, last_cmd_status_buf);
836 else
837 seq_puts(seq, "ok\n");
838 mutex_unlock(&rdtgroup_mutex);
839 return 0;
840}
841
842static int rdt_num_closids_show(struct kernfs_open_file *of,
843 struct seq_file *seq, void *v)
844{
845 struct rdt_resource *r = of->kn->parent->priv;
846
847 seq_printf(seq, "%d\n", r->num_closid);
848 return 0;
849}
850
851static int rdt_default_ctrl_show(struct kernfs_open_file *of,
852 struct seq_file *seq, void *v)
853{
854 struct rdt_resource *r = of->kn->parent->priv;
855
856 seq_printf(seq, "%x\n", r->default_ctrl);
857 return 0;
858}
859
860static int rdt_min_cbm_bits_show(struct kernfs_open_file *of,
861 struct seq_file *seq, void *v)
862{
863 struct rdt_resource *r = of->kn->parent->priv;
864
865 seq_printf(seq, "%u\n", r->cache.min_cbm_bits);
866 return 0;
867}
868
869static int rdt_shareable_bits_show(struct kernfs_open_file *of,
870 struct seq_file *seq, void *v)
871{
872 struct rdt_resource *r = of->kn->parent->priv;
873
874 seq_printf(seq, "%x\n", r->cache.shareable_bits);
875 return 0;
876}
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892static int rdt_bit_usage_show(struct kernfs_open_file *of,
893 struct seq_file *seq, void *v)
894{
895 struct rdt_resource *r = of->kn->parent->priv;
896
897
898
899
900 unsigned long sw_shareable = 0, hw_shareable = 0;
901 unsigned long exclusive = 0, pseudo_locked = 0;
902 struct rdt_domain *dom;
903 int i, hwb, swb, excl, psl;
904 enum rdtgrp_mode mode;
905 bool sep = false;
906 u32 *ctrl;
907
908 mutex_lock(&rdtgroup_mutex);
909 hw_shareable = r->cache.shareable_bits;
910 list_for_each_entry(dom, &r->domains, list) {
911 if (sep)
912 seq_putc(seq, ';');
913 ctrl = dom->ctrl_val;
914 sw_shareable = 0;
915 exclusive = 0;
916 seq_printf(seq, "%d=", dom->id);
917 for (i = 0; i < closids_supported(); i++, ctrl++) {
918 if (!closid_allocated(i))
919 continue;
920 mode = rdtgroup_mode_by_closid(i);
921 switch (mode) {
922 case RDT_MODE_SHAREABLE:
923 sw_shareable |= *ctrl;
924 break;
925 case RDT_MODE_EXCLUSIVE:
926 exclusive |= *ctrl;
927 break;
928 case RDT_MODE_PSEUDO_LOCKSETUP:
929
930
931
932
933
934
935
936 break;
937 case RDT_MODE_PSEUDO_LOCKED:
938 case RDT_NUM_MODES:
939 WARN(1,
940 "invalid mode for closid %d\n", i);
941 break;
942 }
943 }
944 for (i = r->cache.cbm_len - 1; i >= 0; i--) {
945 pseudo_locked = dom->plr ? dom->plr->cbm : 0;
946 hwb = test_bit(i, &hw_shareable);
947 swb = test_bit(i, &sw_shareable);
948 excl = test_bit(i, &exclusive);
949 psl = test_bit(i, &pseudo_locked);
950 if (hwb && swb)
951 seq_putc(seq, 'X');
952 else if (hwb && !swb)
953 seq_putc(seq, 'H');
954 else if (!hwb && swb)
955 seq_putc(seq, 'S');
956 else if (excl)
957 seq_putc(seq, 'E');
958 else if (psl)
959 seq_putc(seq, 'P');
960 else
961 seq_putc(seq, '0');
962 }
963 sep = true;
964 }
965 seq_putc(seq, '\n');
966 mutex_unlock(&rdtgroup_mutex);
967 return 0;
968}
969
970static int rdt_min_bw_show(struct kernfs_open_file *of,
971 struct seq_file *seq, void *v)
972{
973 struct rdt_resource *r = of->kn->parent->priv;
974
975 seq_printf(seq, "%u\n", r->membw.min_bw);
976 return 0;
977}
978
979static int rdt_num_rmids_show(struct kernfs_open_file *of,
980 struct seq_file *seq, void *v)
981{
982 struct rdt_resource *r = of->kn->parent->priv;
983
984 seq_printf(seq, "%d\n", r->num_rmid);
985
986 return 0;
987}
988
989static int rdt_mon_features_show(struct kernfs_open_file *of,
990 struct seq_file *seq, void *v)
991{
992 struct rdt_resource *r = of->kn->parent->priv;
993 struct mon_evt *mevt;
994
995 list_for_each_entry(mevt, &r->evt_list, list)
996 seq_printf(seq, "%s\n", mevt->name);
997
998 return 0;
999}
1000
1001static int rdt_bw_gran_show(struct kernfs_open_file *of,
1002 struct seq_file *seq, void *v)
1003{
1004 struct rdt_resource *r = of->kn->parent->priv;
1005
1006 seq_printf(seq, "%u\n", r->membw.bw_gran);
1007 return 0;
1008}
1009
1010static int rdt_delay_linear_show(struct kernfs_open_file *of,
1011 struct seq_file *seq, void *v)
1012{
1013 struct rdt_resource *r = of->kn->parent->priv;
1014
1015 seq_printf(seq, "%u\n", r->membw.delay_linear);
1016 return 0;
1017}
1018
1019static int max_threshold_occ_show(struct kernfs_open_file *of,
1020 struct seq_file *seq, void *v)
1021{
1022 struct rdt_resource *r = of->kn->parent->priv;
1023
1024 seq_printf(seq, "%u\n", resctrl_cqm_threshold * r->mon_scale);
1025
1026 return 0;
1027}
1028
1029static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of,
1030 struct seq_file *seq, void *v)
1031{
1032 struct rdt_resource *r = of->kn->parent->priv;
1033
1034 if (r->membw.throttle_mode == THREAD_THROTTLE_PER_THREAD)
1035 seq_puts(seq, "per-thread\n");
1036 else
1037 seq_puts(seq, "max\n");
1038
1039 return 0;
1040}
1041
1042static ssize_t max_threshold_occ_write(struct kernfs_open_file *of,
1043 char *buf, size_t nbytes, loff_t off)
1044{
1045 struct rdt_resource *r = of->kn->parent->priv;
1046 unsigned int bytes;
1047 int ret;
1048
1049 ret = kstrtouint(buf, 0, &bytes);
1050 if (ret)
1051 return ret;
1052
1053 if (bytes > (boot_cpu_data.x86_cache_size * 1024))
1054 return -EINVAL;
1055
1056 resctrl_cqm_threshold = bytes / r->mon_scale;
1057
1058 return nbytes;
1059}
1060
1061
1062
1063
1064static int rdtgroup_mode_show(struct kernfs_open_file *of,
1065 struct seq_file *s, void *v)
1066{
1067 struct rdtgroup *rdtgrp;
1068
1069 rdtgrp = rdtgroup_kn_lock_live(of->kn);
1070 if (!rdtgrp) {
1071 rdtgroup_kn_unlock(of->kn);
1072 return -ENOENT;
1073 }
1074
1075 seq_printf(s, "%s\n", rdtgroup_mode_str(rdtgrp->mode));
1076
1077 rdtgroup_kn_unlock(of->kn);
1078 return 0;
1079}
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104static int rdt_cdp_peer_get(struct rdt_resource *r, struct rdt_domain *d,
1105 struct rdt_resource **r_cdp,
1106 struct rdt_domain **d_cdp)
1107{
1108 struct rdt_resource *_r_cdp = NULL;
1109 struct rdt_domain *_d_cdp = NULL;
1110 int ret = 0;
1111
1112 switch (r->rid) {
1113 case RDT_RESOURCE_L3DATA:
1114 _r_cdp = &rdt_resources_all[RDT_RESOURCE_L3CODE];
1115 break;
1116 case RDT_RESOURCE_L3CODE:
1117 _r_cdp = &rdt_resources_all[RDT_RESOURCE_L3DATA];
1118 break;
1119 case RDT_RESOURCE_L2DATA:
1120 _r_cdp = &rdt_resources_all[RDT_RESOURCE_L2CODE];
1121 break;
1122 case RDT_RESOURCE_L2CODE:
1123 _r_cdp = &rdt_resources_all[RDT_RESOURCE_L2DATA];
1124 break;
1125 default:
1126 ret = -ENOENT;
1127 goto out;
1128 }
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139 _d_cdp = rdt_find_domain(_r_cdp, d->id, NULL);
1140 if (WARN_ON(IS_ERR_OR_NULL(_d_cdp))) {
1141 _r_cdp = NULL;
1142 _d_cdp = NULL;
1143 ret = -EINVAL;
1144 }
1145
1146out:
1147 *r_cdp = _r_cdp;
1148 *d_cdp = _d_cdp;
1149
1150 return ret;
1151}
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173static bool __rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d,
1174 unsigned long cbm, int closid, bool exclusive)
1175{
1176 enum rdtgrp_mode mode;
1177 unsigned long ctrl_b;
1178 u32 *ctrl;
1179 int i;
1180
1181
1182 if (!exclusive) {
1183 ctrl_b = r->cache.shareable_bits;
1184 if (bitmap_intersects(&cbm, &ctrl_b, r->cache.cbm_len))
1185 return true;
1186 }
1187
1188
1189 ctrl = d->ctrl_val;
1190 for (i = 0; i < closids_supported(); i++, ctrl++) {
1191 ctrl_b = *ctrl;
1192 mode = rdtgroup_mode_by_closid(i);
1193 if (closid_allocated(i) && i != closid &&
1194 mode != RDT_MODE_PSEUDO_LOCKSETUP) {
1195 if (bitmap_intersects(&cbm, &ctrl_b, r->cache.cbm_len)) {
1196 if (exclusive) {
1197 if (mode == RDT_MODE_EXCLUSIVE)
1198 return true;
1199 continue;
1200 }
1201 return true;
1202 }
1203 }
1204 }
1205
1206 return false;
1207}
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229bool rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d,
1230 unsigned long cbm, int closid, bool exclusive)
1231{
1232 struct rdt_resource *r_cdp;
1233 struct rdt_domain *d_cdp;
1234
1235 if (__rdtgroup_cbm_overlaps(r, d, cbm, closid, exclusive))
1236 return true;
1237
1238 if (rdt_cdp_peer_get(r, d, &r_cdp, &d_cdp) < 0)
1239 return false;
1240
1241 return __rdtgroup_cbm_overlaps(r_cdp, d_cdp, cbm, closid, exclusive);
1242}
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp)
1257{
1258 int closid = rdtgrp->closid;
1259 struct rdt_resource *r;
1260 bool has_cache = false;
1261 struct rdt_domain *d;
1262
1263 for_each_alloc_enabled_rdt_resource(r) {
1264 if (r->rid == RDT_RESOURCE_MBA)
1265 continue;
1266 has_cache = true;
1267 list_for_each_entry(d, &r->domains, list) {
1268 if (rdtgroup_cbm_overlaps(r, d, d->ctrl_val[closid],
1269 rdtgrp->closid, false)) {
1270 rdt_last_cmd_puts("Schemata overlaps\n");
1271 return false;
1272 }
1273 }
1274 }
1275
1276 if (!has_cache) {
1277 rdt_last_cmd_puts("Cannot be exclusive without CAT/CDP\n");
1278 return false;
1279 }
1280
1281 return true;
1282}
1283
1284
1285
1286
1287
1288static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of,
1289 char *buf, size_t nbytes, loff_t off)
1290{
1291 struct rdtgroup *rdtgrp;
1292 enum rdtgrp_mode mode;
1293 int ret = 0;
1294
1295
1296 if (nbytes == 0 || buf[nbytes - 1] != '\n')
1297 return -EINVAL;
1298 buf[nbytes - 1] = '\0';
1299
1300 rdtgrp = rdtgroup_kn_lock_live(of->kn);
1301 if (!rdtgrp) {
1302 rdtgroup_kn_unlock(of->kn);
1303 return -ENOENT;
1304 }
1305
1306 rdt_last_cmd_clear();
1307
1308 mode = rdtgrp->mode;
1309
1310 if ((!strcmp(buf, "shareable") && mode == RDT_MODE_SHAREABLE) ||
1311 (!strcmp(buf, "exclusive") && mode == RDT_MODE_EXCLUSIVE) ||
1312 (!strcmp(buf, "pseudo-locksetup") &&
1313 mode == RDT_MODE_PSEUDO_LOCKSETUP) ||
1314 (!strcmp(buf, "pseudo-locked") && mode == RDT_MODE_PSEUDO_LOCKED))
1315 goto out;
1316
1317 if (mode == RDT_MODE_PSEUDO_LOCKED) {
1318 rdt_last_cmd_puts("Cannot change pseudo-locked group\n");
1319 ret = -EINVAL;
1320 goto out;
1321 }
1322
1323 if (!strcmp(buf, "shareable")) {
1324 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
1325 ret = rdtgroup_locksetup_exit(rdtgrp);
1326 if (ret)
1327 goto out;
1328 }
1329 rdtgrp->mode = RDT_MODE_SHAREABLE;
1330 } else if (!strcmp(buf, "exclusive")) {
1331 if (!rdtgroup_mode_test_exclusive(rdtgrp)) {
1332 ret = -EINVAL;
1333 goto out;
1334 }
1335 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
1336 ret = rdtgroup_locksetup_exit(rdtgrp);
1337 if (ret)
1338 goto out;
1339 }
1340 rdtgrp->mode = RDT_MODE_EXCLUSIVE;
1341 } else if (!strcmp(buf, "pseudo-locksetup")) {
1342 ret = rdtgroup_locksetup_enter(rdtgrp);
1343 if (ret)
1344 goto out;
1345 rdtgrp->mode = RDT_MODE_PSEUDO_LOCKSETUP;
1346 } else {
1347 rdt_last_cmd_puts("Unknown or unsupported mode\n");
1348 ret = -EINVAL;
1349 }
1350
1351out:
1352 rdtgroup_kn_unlock(of->kn);
1353 return ret ?: nbytes;
1354}
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r,
1372 struct rdt_domain *d, unsigned long cbm)
1373{
1374 struct cpu_cacheinfo *ci;
1375 unsigned int size = 0;
1376 int num_b, i;
1377
1378 num_b = bitmap_weight(&cbm, r->cache.cbm_len);
1379 ci = get_cpu_cacheinfo(cpumask_any(&d->cpu_mask));
1380 for (i = 0; i < ci->num_leaves; i++) {
1381 if (ci->info_list[i].level == r->cache_level) {
1382 size = ci->info_list[i].size / r->cache.cbm_len * num_b;
1383 break;
1384 }
1385 }
1386
1387 return size;
1388}
1389
1390
1391
1392
1393
1394
1395
1396
1397static int rdtgroup_size_show(struct kernfs_open_file *of,
1398 struct seq_file *s, void *v)
1399{
1400 struct rdtgroup *rdtgrp;
1401 struct rdt_resource *r;
1402 struct rdt_domain *d;
1403 unsigned int size;
1404 int ret = 0;
1405 bool sep;
1406 u32 ctrl;
1407
1408 rdtgrp = rdtgroup_kn_lock_live(of->kn);
1409 if (!rdtgrp) {
1410 rdtgroup_kn_unlock(of->kn);
1411 return -ENOENT;
1412 }
1413
1414 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
1415 if (!rdtgrp->plr->d) {
1416 rdt_last_cmd_clear();
1417 rdt_last_cmd_puts("Cache domain offline\n");
1418 ret = -ENODEV;
1419 } else {
1420 seq_printf(s, "%*s:", max_name_width,
1421 rdtgrp->plr->r->name);
1422 size = rdtgroup_cbm_to_size(rdtgrp->plr->r,
1423 rdtgrp->plr->d,
1424 rdtgrp->plr->cbm);
1425 seq_printf(s, "%d=%u\n", rdtgrp->plr->d->id, size);
1426 }
1427 goto out;
1428 }
1429
1430 for_each_alloc_enabled_rdt_resource(r) {
1431 sep = false;
1432 seq_printf(s, "%*s:", max_name_width, r->name);
1433 list_for_each_entry(d, &r->domains, list) {
1434 if (sep)
1435 seq_putc(s, ';');
1436 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
1437 size = 0;
1438 } else {
1439 ctrl = (!is_mba_sc(r) ?
1440 d->ctrl_val[rdtgrp->closid] :
1441 d->mbps_val[rdtgrp->closid]);
1442 if (r->rid == RDT_RESOURCE_MBA)
1443 size = ctrl;
1444 else
1445 size = rdtgroup_cbm_to_size(r, d, ctrl);
1446 }
1447 seq_printf(s, "%d=%u", d->id, size);
1448 sep = true;
1449 }
1450 seq_putc(s, '\n');
1451 }
1452
1453out:
1454 rdtgroup_kn_unlock(of->kn);
1455
1456 return ret;
1457}
1458
1459
1460static struct rftype res_common_files[] = {
1461 {
1462 .name = "last_cmd_status",
1463 .mode = 0444,
1464 .kf_ops = &rdtgroup_kf_single_ops,
1465 .seq_show = rdt_last_cmd_status_show,
1466 .fflags = RF_TOP_INFO,
1467 },
1468 {
1469 .name = "num_closids",
1470 .mode = 0444,
1471 .kf_ops = &rdtgroup_kf_single_ops,
1472 .seq_show = rdt_num_closids_show,
1473 .fflags = RF_CTRL_INFO,
1474 },
1475 {
1476 .name = "mon_features",
1477 .mode = 0444,
1478 .kf_ops = &rdtgroup_kf_single_ops,
1479 .seq_show = rdt_mon_features_show,
1480 .fflags = RF_MON_INFO,
1481 },
1482 {
1483 .name = "num_rmids",
1484 .mode = 0444,
1485 .kf_ops = &rdtgroup_kf_single_ops,
1486 .seq_show = rdt_num_rmids_show,
1487 .fflags = RF_MON_INFO,
1488 },
1489 {
1490 .name = "cbm_mask",
1491 .mode = 0444,
1492 .kf_ops = &rdtgroup_kf_single_ops,
1493 .seq_show = rdt_default_ctrl_show,
1494 .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE,
1495 },
1496 {
1497 .name = "min_cbm_bits",
1498 .mode = 0444,
1499 .kf_ops = &rdtgroup_kf_single_ops,
1500 .seq_show = rdt_min_cbm_bits_show,
1501 .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE,
1502 },
1503 {
1504 .name = "shareable_bits",
1505 .mode = 0444,
1506 .kf_ops = &rdtgroup_kf_single_ops,
1507 .seq_show = rdt_shareable_bits_show,
1508 .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE,
1509 },
1510 {
1511 .name = "bit_usage",
1512 .mode = 0444,
1513 .kf_ops = &rdtgroup_kf_single_ops,
1514 .seq_show = rdt_bit_usage_show,
1515 .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE,
1516 },
1517 {
1518 .name = "min_bandwidth",
1519 .mode = 0444,
1520 .kf_ops = &rdtgroup_kf_single_ops,
1521 .seq_show = rdt_min_bw_show,
1522 .fflags = RF_CTRL_INFO | RFTYPE_RES_MB,
1523 },
1524 {
1525 .name = "bandwidth_gran",
1526 .mode = 0444,
1527 .kf_ops = &rdtgroup_kf_single_ops,
1528 .seq_show = rdt_bw_gran_show,
1529 .fflags = RF_CTRL_INFO | RFTYPE_RES_MB,
1530 },
1531 {
1532 .name = "delay_linear",
1533 .mode = 0444,
1534 .kf_ops = &rdtgroup_kf_single_ops,
1535 .seq_show = rdt_delay_linear_show,
1536 .fflags = RF_CTRL_INFO | RFTYPE_RES_MB,
1537 },
1538
1539
1540
1541
1542
1543 {
1544 .name = "thread_throttle_mode",
1545 .mode = 0444,
1546 .kf_ops = &rdtgroup_kf_single_ops,
1547 .seq_show = rdt_thread_throttle_mode_show,
1548 },
1549 {
1550 .name = "max_threshold_occupancy",
1551 .mode = 0644,
1552 .kf_ops = &rdtgroup_kf_single_ops,
1553 .write = max_threshold_occ_write,
1554 .seq_show = max_threshold_occ_show,
1555 .fflags = RF_MON_INFO | RFTYPE_RES_CACHE,
1556 },
1557 {
1558 .name = "cpus",
1559 .mode = 0644,
1560 .kf_ops = &rdtgroup_kf_single_ops,
1561 .write = rdtgroup_cpus_write,
1562 .seq_show = rdtgroup_cpus_show,
1563 .fflags = RFTYPE_BASE,
1564 },
1565 {
1566 .name = "cpus_list",
1567 .mode = 0644,
1568 .kf_ops = &rdtgroup_kf_single_ops,
1569 .write = rdtgroup_cpus_write,
1570 .seq_show = rdtgroup_cpus_show,
1571 .flags = RFTYPE_FLAGS_CPUS_LIST,
1572 .fflags = RFTYPE_BASE,
1573 },
1574 {
1575 .name = "tasks",
1576 .mode = 0644,
1577 .kf_ops = &rdtgroup_kf_single_ops,
1578 .write = rdtgroup_tasks_write,
1579 .seq_show = rdtgroup_tasks_show,
1580 .fflags = RFTYPE_BASE,
1581 },
1582 {
1583 .name = "schemata",
1584 .mode = 0644,
1585 .kf_ops = &rdtgroup_kf_single_ops,
1586 .write = rdtgroup_schemata_write,
1587 .seq_show = rdtgroup_schemata_show,
1588 .fflags = RF_CTRL_BASE,
1589 },
1590 {
1591 .name = "mode",
1592 .mode = 0644,
1593 .kf_ops = &rdtgroup_kf_single_ops,
1594 .write = rdtgroup_mode_write,
1595 .seq_show = rdtgroup_mode_show,
1596 .fflags = RF_CTRL_BASE,
1597 },
1598 {
1599 .name = "size",
1600 .mode = 0444,
1601 .kf_ops = &rdtgroup_kf_single_ops,
1602 .seq_show = rdtgroup_size_show,
1603 .fflags = RF_CTRL_BASE,
1604 },
1605
1606};
1607
1608static int rdtgroup_add_files(struct kernfs_node *kn, unsigned long fflags)
1609{
1610 struct rftype *rfts, *rft;
1611 int ret, len;
1612
1613 rfts = res_common_files;
1614 len = ARRAY_SIZE(res_common_files);
1615
1616 lockdep_assert_held(&rdtgroup_mutex);
1617
1618 for (rft = rfts; rft < rfts + len; rft++) {
1619 if (rft->fflags && ((fflags & rft->fflags) == rft->fflags)) {
1620 ret = rdtgroup_add_file(kn, rft);
1621 if (ret)
1622 goto error;
1623 }
1624 }
1625
1626 return 0;
1627error:
1628 pr_warn("Failed to add %s, err=%d\n", rft->name, ret);
1629 while (--rft >= rfts) {
1630 if ((fflags & rft->fflags) == rft->fflags)
1631 kernfs_remove_by_name(kn, rft->name);
1632 }
1633 return ret;
1634}
1635
1636static struct rftype *rdtgroup_get_rftype_by_name(const char *name)
1637{
1638 struct rftype *rfts, *rft;
1639 int len;
1640
1641 rfts = res_common_files;
1642 len = ARRAY_SIZE(res_common_files);
1643
1644 for (rft = rfts; rft < rfts + len; rft++) {
1645 if (!strcmp(rft->name, name))
1646 return rft;
1647 }
1648
1649 return NULL;
1650}
1651
1652void __init thread_throttle_mode_init(void)
1653{
1654 struct rftype *rft;
1655
1656 rft = rdtgroup_get_rftype_by_name("thread_throttle_mode");
1657 if (!rft)
1658 return;
1659
1660 rft->fflags = RF_CTRL_INFO | RFTYPE_RES_MB;
1661}
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name)
1681{
1682 struct iattr iattr = {.ia_valid = ATTR_MODE,};
1683 struct kernfs_node *kn;
1684 int ret = 0;
1685
1686 kn = kernfs_find_and_get_ns(r->kn, name, NULL);
1687 if (!kn)
1688 return -ENOENT;
1689
1690 switch (kernfs_type(kn)) {
1691 case KERNFS_DIR:
1692 iattr.ia_mode = S_IFDIR;
1693 break;
1694 case KERNFS_FILE:
1695 iattr.ia_mode = S_IFREG;
1696 break;
1697 case KERNFS_LINK:
1698 iattr.ia_mode = S_IFLNK;
1699 break;
1700 }
1701
1702 ret = kernfs_setattr(kn, &iattr);
1703 kernfs_put(kn);
1704 return ret;
1705}
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name,
1719 umode_t mask)
1720{
1721 struct iattr iattr = {.ia_valid = ATTR_MODE,};
1722 struct kernfs_node *kn, *parent;
1723 struct rftype *rfts, *rft;
1724 int ret, len;
1725
1726 rfts = res_common_files;
1727 len = ARRAY_SIZE(res_common_files);
1728
1729 for (rft = rfts; rft < rfts + len; rft++) {
1730 if (!strcmp(rft->name, name))
1731 iattr.ia_mode = rft->mode & mask;
1732 }
1733
1734 kn = kernfs_find_and_get_ns(r->kn, name, NULL);
1735 if (!kn)
1736 return -ENOENT;
1737
1738 switch (kernfs_type(kn)) {
1739 case KERNFS_DIR:
1740 parent = kernfs_get_parent(kn);
1741 if (parent) {
1742 iattr.ia_mode |= parent->mode;
1743 kernfs_put(parent);
1744 }
1745 iattr.ia_mode |= S_IFDIR;
1746 break;
1747 case KERNFS_FILE:
1748 iattr.ia_mode |= S_IFREG;
1749 break;
1750 case KERNFS_LINK:
1751 iattr.ia_mode |= S_IFLNK;
1752 break;
1753 }
1754
1755 ret = kernfs_setattr(kn, &iattr);
1756 kernfs_put(kn);
1757 return ret;
1758}
1759
1760static int rdtgroup_mkdir_info_resdir(struct rdt_resource *r, char *name,
1761 unsigned long fflags)
1762{
1763 struct kernfs_node *kn_subdir;
1764 int ret;
1765
1766 kn_subdir = kernfs_create_dir(kn_info, name,
1767 kn_info->mode, r);
1768 if (IS_ERR(kn_subdir))
1769 return PTR_ERR(kn_subdir);
1770
1771 ret = rdtgroup_kn_set_ugid(kn_subdir);
1772 if (ret)
1773 return ret;
1774
1775 ret = rdtgroup_add_files(kn_subdir, fflags);
1776 if (!ret)
1777 kernfs_activate(kn_subdir);
1778
1779 return ret;
1780}
1781
1782static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn)
1783{
1784 struct rdt_resource *r;
1785 unsigned long fflags;
1786 char name[32];
1787 int ret;
1788
1789
1790 kn_info = kernfs_create_dir(parent_kn, "info", parent_kn->mode, NULL);
1791 if (IS_ERR(kn_info))
1792 return PTR_ERR(kn_info);
1793
1794 ret = rdtgroup_add_files(kn_info, RF_TOP_INFO);
1795 if (ret)
1796 goto out_destroy;
1797
1798 for_each_alloc_enabled_rdt_resource(r) {
1799 fflags = r->fflags | RF_CTRL_INFO;
1800 ret = rdtgroup_mkdir_info_resdir(r, r->name, fflags);
1801 if (ret)
1802 goto out_destroy;
1803 }
1804
1805 for_each_mon_enabled_rdt_resource(r) {
1806 fflags = r->fflags | RF_MON_INFO;
1807 sprintf(name, "%s_MON", r->name);
1808 ret = rdtgroup_mkdir_info_resdir(r, name, fflags);
1809 if (ret)
1810 goto out_destroy;
1811 }
1812
1813 ret = rdtgroup_kn_set_ugid(kn_info);
1814 if (ret)
1815 goto out_destroy;
1816
1817 kernfs_activate(kn_info);
1818
1819 return 0;
1820
1821out_destroy:
1822 kernfs_remove(kn_info);
1823 return ret;
1824}
1825
1826static int
1827mongroup_create_dir(struct kernfs_node *parent_kn, struct rdtgroup *prgrp,
1828 char *name, struct kernfs_node **dest_kn)
1829{
1830 struct kernfs_node *kn;
1831 int ret;
1832
1833
1834 kn = kernfs_create_dir(parent_kn, name, parent_kn->mode, prgrp);
1835 if (IS_ERR(kn))
1836 return PTR_ERR(kn);
1837
1838 if (dest_kn)
1839 *dest_kn = kn;
1840
1841 ret = rdtgroup_kn_set_ugid(kn);
1842 if (ret)
1843 goto out_destroy;
1844
1845 kernfs_activate(kn);
1846
1847 return 0;
1848
1849out_destroy:
1850 kernfs_remove(kn);
1851 return ret;
1852}
1853
1854static void l3_qos_cfg_update(void *arg)
1855{
1856 bool *enable = arg;
1857
1858 wrmsrl(MSR_IA32_L3_QOS_CFG, *enable ? L3_QOS_CDP_ENABLE : 0ULL);
1859}
1860
1861static void l2_qos_cfg_update(void *arg)
1862{
1863 bool *enable = arg;
1864
1865 wrmsrl(MSR_IA32_L2_QOS_CFG, *enable ? L2_QOS_CDP_ENABLE : 0ULL);
1866}
1867
1868static inline bool is_mba_linear(void)
1869{
1870 return rdt_resources_all[RDT_RESOURCE_MBA].membw.delay_linear;
1871}
1872
1873static int set_cache_qos_cfg(int level, bool enable)
1874{
1875 void (*update)(void *arg);
1876 struct rdt_resource *r_l;
1877 cpumask_var_t cpu_mask;
1878 struct rdt_domain *d;
1879 int cpu;
1880
1881 if (level == RDT_RESOURCE_L3)
1882 update = l3_qos_cfg_update;
1883 else if (level == RDT_RESOURCE_L2)
1884 update = l2_qos_cfg_update;
1885 else
1886 return -EINVAL;
1887
1888 if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
1889 return -ENOMEM;
1890
1891 r_l = &rdt_resources_all[level];
1892 list_for_each_entry(d, &r_l->domains, list) {
1893 if (r_l->cache.arch_has_per_cpu_cfg)
1894
1895 for_each_cpu(cpu, &d->cpu_mask)
1896 cpumask_set_cpu(cpu, cpu_mask);
1897 else
1898
1899 cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask);
1900 }
1901 cpu = get_cpu();
1902
1903 if (cpumask_test_cpu(cpu, cpu_mask))
1904 update(&enable);
1905
1906 smp_call_function_many(cpu_mask, update, &enable, 1);
1907 put_cpu();
1908
1909 free_cpumask_var(cpu_mask);
1910
1911 return 0;
1912}
1913
1914
1915void rdt_domain_reconfigure_cdp(struct rdt_resource *r)
1916{
1917 if (!r->alloc_capable)
1918 return;
1919
1920 if (r == &rdt_resources_all[RDT_RESOURCE_L2DATA])
1921 l2_qos_cfg_update(&r->alloc_enabled);
1922
1923 if (r == &rdt_resources_all[RDT_RESOURCE_L3DATA])
1924 l3_qos_cfg_update(&r->alloc_enabled);
1925}
1926
1927
1928
1929
1930
1931
1932
1933static int set_mba_sc(bool mba_sc)
1934{
1935 struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_MBA];
1936 struct rdt_domain *d;
1937
1938 if (!is_mbm_enabled() || !is_mba_linear() ||
1939 mba_sc == is_mba_sc(r))
1940 return -EINVAL;
1941
1942 r->membw.mba_sc = mba_sc;
1943 list_for_each_entry(d, &r->domains, list)
1944 setup_default_ctrlval(r, d->ctrl_val, d->mbps_val);
1945
1946 return 0;
1947}
1948
1949static int cdp_enable(int level, int data_type, int code_type)
1950{
1951 struct rdt_resource *r_ldata = &rdt_resources_all[data_type];
1952 struct rdt_resource *r_lcode = &rdt_resources_all[code_type];
1953 struct rdt_resource *r_l = &rdt_resources_all[level];
1954 int ret;
1955
1956 if (!r_l->alloc_capable || !r_ldata->alloc_capable ||
1957 !r_lcode->alloc_capable)
1958 return -EINVAL;
1959
1960 ret = set_cache_qos_cfg(level, true);
1961 if (!ret) {
1962 r_l->alloc_enabled = false;
1963 r_ldata->alloc_enabled = true;
1964 r_lcode->alloc_enabled = true;
1965 }
1966 return ret;
1967}
1968
1969static int cdpl3_enable(void)
1970{
1971 return cdp_enable(RDT_RESOURCE_L3, RDT_RESOURCE_L3DATA,
1972 RDT_RESOURCE_L3CODE);
1973}
1974
1975static int cdpl2_enable(void)
1976{
1977 return cdp_enable(RDT_RESOURCE_L2, RDT_RESOURCE_L2DATA,
1978 RDT_RESOURCE_L2CODE);
1979}
1980
1981static void cdp_disable(int level, int data_type, int code_type)
1982{
1983 struct rdt_resource *r = &rdt_resources_all[level];
1984
1985 r->alloc_enabled = r->alloc_capable;
1986
1987 if (rdt_resources_all[data_type].alloc_enabled) {
1988 rdt_resources_all[data_type].alloc_enabled = false;
1989 rdt_resources_all[code_type].alloc_enabled = false;
1990 set_cache_qos_cfg(level, false);
1991 }
1992}
1993
1994static void cdpl3_disable(void)
1995{
1996 cdp_disable(RDT_RESOURCE_L3, RDT_RESOURCE_L3DATA, RDT_RESOURCE_L3CODE);
1997}
1998
1999static void cdpl2_disable(void)
2000{
2001 cdp_disable(RDT_RESOURCE_L2, RDT_RESOURCE_L2DATA, RDT_RESOURCE_L2CODE);
2002}
2003
2004static void cdp_disable_all(void)
2005{
2006 if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled)
2007 cdpl3_disable();
2008 if (rdt_resources_all[RDT_RESOURCE_L2DATA].alloc_enabled)
2009 cdpl2_disable();
2010}
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020static struct rdtgroup *kernfs_to_rdtgroup(struct kernfs_node *kn)
2021{
2022 if (kernfs_type(kn) == KERNFS_DIR) {
2023
2024
2025
2026
2027
2028
2029 if (kn == kn_info || kn->parent == kn_info)
2030 return NULL;
2031 else
2032 return kn->priv;
2033 } else {
2034 return kn->parent->priv;
2035 }
2036}
2037
2038struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn)
2039{
2040 struct rdtgroup *rdtgrp = kernfs_to_rdtgroup(kn);
2041
2042 if (!rdtgrp)
2043 return NULL;
2044
2045 atomic_inc(&rdtgrp->waitcount);
2046 kernfs_break_active_protection(kn);
2047
2048 mutex_lock(&rdtgroup_mutex);
2049
2050
2051 if (rdtgrp->flags & RDT_DELETED)
2052 return NULL;
2053
2054 return rdtgrp;
2055}
2056
2057void rdtgroup_kn_unlock(struct kernfs_node *kn)
2058{
2059 struct rdtgroup *rdtgrp = kernfs_to_rdtgroup(kn);
2060
2061 if (!rdtgrp)
2062 return;
2063
2064 mutex_unlock(&rdtgroup_mutex);
2065
2066 if (atomic_dec_and_test(&rdtgrp->waitcount) &&
2067 (rdtgrp->flags & RDT_DELETED)) {
2068 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
2069 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)
2070 rdtgroup_pseudo_lock_remove(rdtgrp);
2071 kernfs_unbreak_active_protection(kn);
2072 rdtgroup_remove(rdtgrp);
2073 } else {
2074 kernfs_unbreak_active_protection(kn);
2075 }
2076}
2077
2078static int mkdir_mondata_all(struct kernfs_node *parent_kn,
2079 struct rdtgroup *prgrp,
2080 struct kernfs_node **mon_data_kn);
2081
2082static int rdt_enable_ctx(struct rdt_fs_context *ctx)
2083{
2084 int ret = 0;
2085
2086 if (ctx->enable_cdpl2)
2087 ret = cdpl2_enable();
2088
2089 if (!ret && ctx->enable_cdpl3)
2090 ret = cdpl3_enable();
2091
2092 if (!ret && ctx->enable_mba_mbps)
2093 ret = set_mba_sc(true);
2094
2095 return ret;
2096}
2097
2098static int rdt_get_tree(struct fs_context *fc)
2099{
2100 struct rdt_fs_context *ctx = rdt_fc2context(fc);
2101 struct rdt_domain *dom;
2102 struct rdt_resource *r;
2103 int ret;
2104
2105 cpus_read_lock();
2106 mutex_lock(&rdtgroup_mutex);
2107
2108
2109
2110 if (static_branch_unlikely(&rdt_enable_key)) {
2111 ret = -EBUSY;
2112 goto out;
2113 }
2114
2115 ret = rdt_enable_ctx(ctx);
2116 if (ret < 0)
2117 goto out_cdp;
2118
2119 closid_init();
2120
2121 ret = rdtgroup_create_info_dir(rdtgroup_default.kn);
2122 if (ret < 0)
2123 goto out_mba;
2124
2125 if (rdt_mon_capable) {
2126 ret = mongroup_create_dir(rdtgroup_default.kn,
2127 &rdtgroup_default, "mon_groups",
2128 &kn_mongrp);
2129 if (ret < 0)
2130 goto out_info;
2131
2132 ret = mkdir_mondata_all(rdtgroup_default.kn,
2133 &rdtgroup_default, &kn_mondata);
2134 if (ret < 0)
2135 goto out_mongrp;
2136 rdtgroup_default.mon.mon_data_kn = kn_mondata;
2137 }
2138
2139 ret = rdt_pseudo_lock_init();
2140 if (ret)
2141 goto out_mondata;
2142
2143 ret = kernfs_get_tree(fc);
2144 if (ret < 0)
2145 goto out_psl;
2146
2147 if (rdt_alloc_capable)
2148 static_branch_enable_cpuslocked(&rdt_alloc_enable_key);
2149 if (rdt_mon_capable)
2150 static_branch_enable_cpuslocked(&rdt_mon_enable_key);
2151
2152 if (rdt_alloc_capable || rdt_mon_capable)
2153 static_branch_enable_cpuslocked(&rdt_enable_key);
2154
2155 if (is_mbm_enabled()) {
2156 r = &rdt_resources_all[RDT_RESOURCE_L3];
2157 list_for_each_entry(dom, &r->domains, list)
2158 mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL);
2159 }
2160
2161 goto out;
2162
2163out_psl:
2164 rdt_pseudo_lock_release();
2165out_mondata:
2166 if (rdt_mon_capable)
2167 kernfs_remove(kn_mondata);
2168out_mongrp:
2169 if (rdt_mon_capable)
2170 kernfs_remove(kn_mongrp);
2171out_info:
2172 kernfs_remove(kn_info);
2173out_mba:
2174 if (ctx->enable_mba_mbps)
2175 set_mba_sc(false);
2176out_cdp:
2177 cdp_disable_all();
2178out:
2179 rdt_last_cmd_clear();
2180 mutex_unlock(&rdtgroup_mutex);
2181 cpus_read_unlock();
2182 return ret;
2183}
2184
2185enum rdt_param {
2186 Opt_cdp,
2187 Opt_cdpl2,
2188 Opt_mba_mbps,
2189 nr__rdt_params
2190};
2191
2192static const struct fs_parameter_spec rdt_fs_parameters[] = {
2193 fsparam_flag("cdp", Opt_cdp),
2194 fsparam_flag("cdpl2", Opt_cdpl2),
2195 fsparam_flag("mba_MBps", Opt_mba_mbps),
2196 {}
2197};
2198
2199static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param)
2200{
2201 struct rdt_fs_context *ctx = rdt_fc2context(fc);
2202 struct fs_parse_result result;
2203 int opt;
2204
2205 opt = fs_parse(fc, rdt_fs_parameters, param, &result);
2206 if (opt < 0)
2207 return opt;
2208
2209 switch (opt) {
2210 case Opt_cdp:
2211 ctx->enable_cdpl3 = true;
2212 return 0;
2213 case Opt_cdpl2:
2214 ctx->enable_cdpl2 = true;
2215 return 0;
2216 case Opt_mba_mbps:
2217 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2218 return -EINVAL;
2219 ctx->enable_mba_mbps = true;
2220 return 0;
2221 }
2222
2223 return -EINVAL;
2224}
2225
2226static void rdt_fs_context_free(struct fs_context *fc)
2227{
2228 struct rdt_fs_context *ctx = rdt_fc2context(fc);
2229
2230 kernfs_free_fs_context(fc);
2231 kfree(ctx);
2232}
2233
2234static const struct fs_context_operations rdt_fs_context_ops = {
2235 .free = rdt_fs_context_free,
2236 .parse_param = rdt_parse_param,
2237 .get_tree = rdt_get_tree,
2238};
2239
2240static int rdt_init_fs_context(struct fs_context *fc)
2241{
2242 struct rdt_fs_context *ctx;
2243
2244 ctx = kzalloc(sizeof(struct rdt_fs_context), GFP_KERNEL);
2245 if (!ctx)
2246 return -ENOMEM;
2247
2248 ctx->kfc.root = rdt_root;
2249 ctx->kfc.magic = RDTGROUP_SUPER_MAGIC;
2250 fc->fs_private = &ctx->kfc;
2251 fc->ops = &rdt_fs_context_ops;
2252 put_user_ns(fc->user_ns);
2253 fc->user_ns = get_user_ns(&init_user_ns);
2254 fc->global = true;
2255 return 0;
2256}
2257
2258static int reset_all_ctrls(struct rdt_resource *r)
2259{
2260 struct msr_param msr_param;
2261 cpumask_var_t cpu_mask;
2262 struct rdt_domain *d;
2263 int i, cpu;
2264
2265 if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
2266 return -ENOMEM;
2267
2268 msr_param.res = r;
2269 msr_param.low = 0;
2270 msr_param.high = r->num_closid;
2271
2272
2273
2274
2275
2276
2277 list_for_each_entry(d, &r->domains, list) {
2278 cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask);
2279
2280 for (i = 0; i < r->num_closid; i++)
2281 d->ctrl_val[i] = r->default_ctrl;
2282 }
2283 cpu = get_cpu();
2284
2285 if (cpumask_test_cpu(cpu, cpu_mask))
2286 rdt_ctrl_update(&msr_param);
2287
2288 smp_call_function_many(cpu_mask, rdt_ctrl_update, &msr_param, 1);
2289 put_cpu();
2290
2291 free_cpumask_var(cpu_mask);
2292
2293 return 0;
2294}
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304static void rdt_move_group_tasks(struct rdtgroup *from, struct rdtgroup *to,
2305 struct cpumask *mask)
2306{
2307 struct task_struct *p, *t;
2308
2309 read_lock(&tasklist_lock);
2310 for_each_process_thread(p, t) {
2311 if (!from || is_closid_match(t, from) ||
2312 is_rmid_match(t, from)) {
2313 WRITE_ONCE(t->closid, to->closid);
2314 WRITE_ONCE(t->rmid, to->mon.rmid);
2315
2316
2317
2318
2319
2320
2321
2322
2323 if (IS_ENABLED(CONFIG_SMP) && mask && task_curr(t))
2324 cpumask_set_cpu(task_cpu(t), mask);
2325 }
2326 }
2327 read_unlock(&tasklist_lock);
2328}
2329
2330static void free_all_child_rdtgrp(struct rdtgroup *rdtgrp)
2331{
2332 struct rdtgroup *sentry, *stmp;
2333 struct list_head *head;
2334
2335 head = &rdtgrp->mon.crdtgrp_list;
2336 list_for_each_entry_safe(sentry, stmp, head, mon.crdtgrp_list) {
2337 free_rmid(sentry->mon.rmid);
2338 list_del(&sentry->mon.crdtgrp_list);
2339
2340 if (atomic_read(&sentry->waitcount) != 0)
2341 sentry->flags = RDT_DELETED;
2342 else
2343 rdtgroup_remove(sentry);
2344 }
2345}
2346
2347
2348
2349
2350static void rmdir_all_sub(void)
2351{
2352 struct rdtgroup *rdtgrp, *tmp;
2353
2354
2355 rdt_move_group_tasks(NULL, &rdtgroup_default, NULL);
2356
2357 list_for_each_entry_safe(rdtgrp, tmp, &rdt_all_groups, rdtgroup_list) {
2358
2359 free_all_child_rdtgrp(rdtgrp);
2360
2361
2362 if (rdtgrp == &rdtgroup_default)
2363 continue;
2364
2365 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
2366 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)
2367 rdtgroup_pseudo_lock_remove(rdtgrp);
2368
2369
2370
2371
2372
2373
2374 cpumask_or(&rdtgroup_default.cpu_mask,
2375 &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask);
2376
2377 free_rmid(rdtgrp->mon.rmid);
2378
2379 kernfs_remove(rdtgrp->kn);
2380 list_del(&rdtgrp->rdtgroup_list);
2381
2382 if (atomic_read(&rdtgrp->waitcount) != 0)
2383 rdtgrp->flags = RDT_DELETED;
2384 else
2385 rdtgroup_remove(rdtgrp);
2386 }
2387
2388 update_closid_rmid(cpu_online_mask, &rdtgroup_default);
2389
2390 kernfs_remove(kn_info);
2391 kernfs_remove(kn_mongrp);
2392 kernfs_remove(kn_mondata);
2393}
2394
2395static void rdt_kill_sb(struct super_block *sb)
2396{
2397 struct rdt_resource *r;
2398
2399 cpus_read_lock();
2400 mutex_lock(&rdtgroup_mutex);
2401
2402 set_mba_sc(false);
2403
2404
2405 for_each_alloc_enabled_rdt_resource(r)
2406 reset_all_ctrls(r);
2407 cdp_disable_all();
2408 rmdir_all_sub();
2409 rdt_pseudo_lock_release();
2410 rdtgroup_default.mode = RDT_MODE_SHAREABLE;
2411 static_branch_disable_cpuslocked(&rdt_alloc_enable_key);
2412 static_branch_disable_cpuslocked(&rdt_mon_enable_key);
2413 static_branch_disable_cpuslocked(&rdt_enable_key);
2414 kernfs_kill_sb(sb);
2415 mutex_unlock(&rdtgroup_mutex);
2416 cpus_read_unlock();
2417}
2418
2419static struct file_system_type rdt_fs_type = {
2420 .name = "resctrl",
2421 .init_fs_context = rdt_init_fs_context,
2422 .parameters = rdt_fs_parameters,
2423 .kill_sb = rdt_kill_sb,
2424};
2425
2426static int mon_addfile(struct kernfs_node *parent_kn, const char *name,
2427 void *priv)
2428{
2429 struct kernfs_node *kn;
2430 int ret = 0;
2431
2432 kn = __kernfs_create_file(parent_kn, name, 0444,
2433 GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0,
2434 &kf_mondata_ops, priv, NULL, NULL);
2435 if (IS_ERR(kn))
2436 return PTR_ERR(kn);
2437
2438 ret = rdtgroup_kn_set_ugid(kn);
2439 if (ret) {
2440 kernfs_remove(kn);
2441 return ret;
2442 }
2443
2444 return ret;
2445}
2446
2447
2448
2449
2450
2451void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, unsigned int dom_id)
2452{
2453 struct rdtgroup *prgrp, *crgrp;
2454 char name[32];
2455
2456 if (!r->mon_enabled)
2457 return;
2458
2459 list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
2460 sprintf(name, "mon_%s_%02d", r->name, dom_id);
2461 kernfs_remove_by_name(prgrp->mon.mon_data_kn, name);
2462
2463 list_for_each_entry(crgrp, &prgrp->mon.crdtgrp_list, mon.crdtgrp_list)
2464 kernfs_remove_by_name(crgrp->mon.mon_data_kn, name);
2465 }
2466}
2467
2468static int mkdir_mondata_subdir(struct kernfs_node *parent_kn,
2469 struct rdt_domain *d,
2470 struct rdt_resource *r, struct rdtgroup *prgrp)
2471{
2472 union mon_data_bits priv;
2473 struct kernfs_node *kn;
2474 struct mon_evt *mevt;
2475 struct rmid_read rr;
2476 char name[32];
2477 int ret;
2478
2479 sprintf(name, "mon_%s_%02d", r->name, d->id);
2480
2481 kn = kernfs_create_dir(parent_kn, name, parent_kn->mode, prgrp);
2482 if (IS_ERR(kn))
2483 return PTR_ERR(kn);
2484
2485 ret = rdtgroup_kn_set_ugid(kn);
2486 if (ret)
2487 goto out_destroy;
2488
2489 if (WARN_ON(list_empty(&r->evt_list))) {
2490 ret = -EPERM;
2491 goto out_destroy;
2492 }
2493
2494 priv.u.rid = r->rid;
2495 priv.u.domid = d->id;
2496 list_for_each_entry(mevt, &r->evt_list, list) {
2497 priv.u.evtid = mevt->evtid;
2498 ret = mon_addfile(kn, mevt->name, priv.priv);
2499 if (ret)
2500 goto out_destroy;
2501
2502 if (is_mbm_event(mevt->evtid))
2503 mon_event_read(&rr, r, d, prgrp, mevt->evtid, true);
2504 }
2505 kernfs_activate(kn);
2506 return 0;
2507
2508out_destroy:
2509 kernfs_remove(kn);
2510 return ret;
2511}
2512
2513
2514
2515
2516
2517void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
2518 struct rdt_domain *d)
2519{
2520 struct kernfs_node *parent_kn;
2521 struct rdtgroup *prgrp, *crgrp;
2522 struct list_head *head;
2523
2524 if (!r->mon_enabled)
2525 return;
2526
2527 list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
2528 parent_kn = prgrp->mon.mon_data_kn;
2529 mkdir_mondata_subdir(parent_kn, d, r, prgrp);
2530
2531 head = &prgrp->mon.crdtgrp_list;
2532 list_for_each_entry(crgrp, head, mon.crdtgrp_list) {
2533 parent_kn = crgrp->mon.mon_data_kn;
2534 mkdir_mondata_subdir(parent_kn, d, r, crgrp);
2535 }
2536 }
2537}
2538
2539static int mkdir_mondata_subdir_alldom(struct kernfs_node *parent_kn,
2540 struct rdt_resource *r,
2541 struct rdtgroup *prgrp)
2542{
2543 struct rdt_domain *dom;
2544 int ret;
2545
2546 list_for_each_entry(dom, &r->domains, list) {
2547 ret = mkdir_mondata_subdir(parent_kn, dom, r, prgrp);
2548 if (ret)
2549 return ret;
2550 }
2551
2552 return 0;
2553}
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572static int mkdir_mondata_all(struct kernfs_node *parent_kn,
2573 struct rdtgroup *prgrp,
2574 struct kernfs_node **dest_kn)
2575{
2576 struct rdt_resource *r;
2577 struct kernfs_node *kn;
2578 int ret;
2579
2580
2581
2582
2583 ret = mongroup_create_dir(parent_kn, prgrp, "mon_data", &kn);
2584 if (ret)
2585 return ret;
2586
2587 if (dest_kn)
2588 *dest_kn = kn;
2589
2590
2591
2592
2593
2594 for_each_mon_enabled_rdt_resource(r) {
2595 ret = mkdir_mondata_subdir_alldom(kn, r, prgrp);
2596 if (ret)
2597 goto out_destroy;
2598 }
2599
2600 return 0;
2601
2602out_destroy:
2603 kernfs_remove(kn);
2604 return ret;
2605}
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622static u32 cbm_ensure_valid(u32 _val, struct rdt_resource *r)
2623{
2624 unsigned int cbm_len = r->cache.cbm_len;
2625 unsigned long first_bit, zero_bit;
2626 unsigned long val = _val;
2627
2628 if (!val)
2629 return 0;
2630
2631 first_bit = find_first_bit(&val, cbm_len);
2632 zero_bit = find_next_zero_bit(&val, cbm_len, first_bit);
2633
2634
2635 bitmap_clear(&val, zero_bit, cbm_len - zero_bit);
2636 return (u32)val;
2637}
2638
2639
2640
2641
2642
2643
2644
2645static int __init_one_rdt_domain(struct rdt_domain *d, struct rdt_resource *r,
2646 u32 closid)
2647{
2648 struct rdt_resource *r_cdp = NULL;
2649 struct rdt_domain *d_cdp = NULL;
2650 u32 used_b = 0, unused_b = 0;
2651 unsigned long tmp_cbm;
2652 enum rdtgrp_mode mode;
2653 u32 peer_ctl, *ctrl;
2654 int i;
2655
2656 rdt_cdp_peer_get(r, d, &r_cdp, &d_cdp);
2657 d->have_new_ctrl = false;
2658 d->new_ctrl = r->cache.shareable_bits;
2659 used_b = r->cache.shareable_bits;
2660 ctrl = d->ctrl_val;
2661 for (i = 0; i < closids_supported(); i++, ctrl++) {
2662 if (closid_allocated(i) && i != closid) {
2663 mode = rdtgroup_mode_by_closid(i);
2664 if (mode == RDT_MODE_PSEUDO_LOCKSETUP)
2665
2666
2667
2668
2669
2670 continue;
2671
2672
2673
2674
2675
2676 if (d_cdp)
2677 peer_ctl = d_cdp->ctrl_val[i];
2678 else
2679 peer_ctl = 0;
2680 used_b |= *ctrl | peer_ctl;
2681 if (mode == RDT_MODE_SHAREABLE)
2682 d->new_ctrl |= *ctrl | peer_ctl;
2683 }
2684 }
2685 if (d->plr && d->plr->cbm > 0)
2686 used_b |= d->plr->cbm;
2687 unused_b = used_b ^ (BIT_MASK(r->cache.cbm_len) - 1);
2688 unused_b &= BIT_MASK(r->cache.cbm_len) - 1;
2689 d->new_ctrl |= unused_b;
2690
2691
2692
2693
2694 d->new_ctrl = cbm_ensure_valid(d->new_ctrl, r);
2695
2696
2697
2698
2699 tmp_cbm = d->new_ctrl;
2700 if (bitmap_weight(&tmp_cbm, r->cache.cbm_len) < r->cache.min_cbm_bits) {
2701 rdt_last_cmd_printf("No space on %s:%d\n", r->name, d->id);
2702 return -ENOSPC;
2703 }
2704 d->have_new_ctrl = true;
2705
2706 return 0;
2707}
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719static int rdtgroup_init_cat(struct rdt_resource *r, u32 closid)
2720{
2721 struct rdt_domain *d;
2722 int ret;
2723
2724 list_for_each_entry(d, &r->domains, list) {
2725 ret = __init_one_rdt_domain(d, r, closid);
2726 if (ret < 0)
2727 return ret;
2728 }
2729
2730 return 0;
2731}
2732
2733
2734static void rdtgroup_init_mba(struct rdt_resource *r)
2735{
2736 struct rdt_domain *d;
2737
2738 list_for_each_entry(d, &r->domains, list) {
2739 d->new_ctrl = is_mba_sc(r) ? MBA_MAX_MBPS : r->default_ctrl;
2740 d->have_new_ctrl = true;
2741 }
2742}
2743
2744
2745static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp)
2746{
2747 struct rdt_resource *r;
2748 int ret;
2749
2750 for_each_alloc_enabled_rdt_resource(r) {
2751 if (r->rid == RDT_RESOURCE_MBA) {
2752 rdtgroup_init_mba(r);
2753 } else {
2754 ret = rdtgroup_init_cat(r, rdtgrp->closid);
2755 if (ret < 0)
2756 return ret;
2757 }
2758
2759 ret = update_domains(r, rdtgrp->closid);
2760 if (ret < 0) {
2761 rdt_last_cmd_puts("Failed to initialize allocations\n");
2762 return ret;
2763 }
2764
2765 }
2766
2767 rdtgrp->mode = RDT_MODE_SHAREABLE;
2768
2769 return 0;
2770}
2771
2772static int mkdir_rdt_prepare(struct kernfs_node *parent_kn,
2773 const char *name, umode_t mode,
2774 enum rdt_group_type rtype, struct rdtgroup **r)
2775{
2776 struct rdtgroup *prdtgrp, *rdtgrp;
2777 struct kernfs_node *kn;
2778 uint files = 0;
2779 int ret;
2780
2781 prdtgrp = rdtgroup_kn_lock_live(parent_kn);
2782 if (!prdtgrp) {
2783 ret = -ENODEV;
2784 goto out_unlock;
2785 }
2786
2787 if (rtype == RDTMON_GROUP &&
2788 (prdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
2789 prdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)) {
2790 ret = -EINVAL;
2791 rdt_last_cmd_puts("Pseudo-locking in progress\n");
2792 goto out_unlock;
2793 }
2794
2795
2796 rdtgrp = kzalloc(sizeof(*rdtgrp), GFP_KERNEL);
2797 if (!rdtgrp) {
2798 ret = -ENOSPC;
2799 rdt_last_cmd_puts("Kernel out of memory\n");
2800 goto out_unlock;
2801 }
2802 *r = rdtgrp;
2803 rdtgrp->mon.parent = prdtgrp;
2804 rdtgrp->type = rtype;
2805 INIT_LIST_HEAD(&rdtgrp->mon.crdtgrp_list);
2806
2807
2808 kn = kernfs_create_dir(parent_kn, name, mode, rdtgrp);
2809 if (IS_ERR(kn)) {
2810 ret = PTR_ERR(kn);
2811 rdt_last_cmd_puts("kernfs create error\n");
2812 goto out_free_rgrp;
2813 }
2814 rdtgrp->kn = kn;
2815
2816
2817
2818
2819
2820
2821
2822 kernfs_get(kn);
2823
2824 ret = rdtgroup_kn_set_ugid(kn);
2825 if (ret) {
2826 rdt_last_cmd_puts("kernfs perm error\n");
2827 goto out_destroy;
2828 }
2829
2830 files = RFTYPE_BASE | BIT(RF_CTRLSHIFT + rtype);
2831 ret = rdtgroup_add_files(kn, files);
2832 if (ret) {
2833 rdt_last_cmd_puts("kernfs fill error\n");
2834 goto out_destroy;
2835 }
2836
2837 if (rdt_mon_capable) {
2838 ret = alloc_rmid();
2839 if (ret < 0) {
2840 rdt_last_cmd_puts("Out of RMIDs\n");
2841 goto out_destroy;
2842 }
2843 rdtgrp->mon.rmid = ret;
2844
2845 ret = mkdir_mondata_all(kn, rdtgrp, &rdtgrp->mon.mon_data_kn);
2846 if (ret) {
2847 rdt_last_cmd_puts("kernfs subdir error\n");
2848 goto out_idfree;
2849 }
2850 }
2851 kernfs_activate(kn);
2852
2853
2854
2855
2856 return 0;
2857
2858out_idfree:
2859 free_rmid(rdtgrp->mon.rmid);
2860out_destroy:
2861 kernfs_put(rdtgrp->kn);
2862 kernfs_remove(rdtgrp->kn);
2863out_free_rgrp:
2864 kfree(rdtgrp);
2865out_unlock:
2866 rdtgroup_kn_unlock(parent_kn);
2867 return ret;
2868}
2869
2870static void mkdir_rdt_prepare_clean(struct rdtgroup *rgrp)
2871{
2872 kernfs_remove(rgrp->kn);
2873 free_rmid(rgrp->mon.rmid);
2874 rdtgroup_remove(rgrp);
2875}
2876
2877
2878
2879
2880
2881
2882static int rdtgroup_mkdir_mon(struct kernfs_node *parent_kn,
2883 const char *name, umode_t mode)
2884{
2885 struct rdtgroup *rdtgrp, *prgrp;
2886 int ret;
2887
2888 ret = mkdir_rdt_prepare(parent_kn, name, mode, RDTMON_GROUP, &rdtgrp);
2889 if (ret)
2890 return ret;
2891
2892 prgrp = rdtgrp->mon.parent;
2893 rdtgrp->closid = prgrp->closid;
2894
2895
2896
2897
2898
2899 list_add_tail(&rdtgrp->mon.crdtgrp_list, &prgrp->mon.crdtgrp_list);
2900
2901 rdtgroup_kn_unlock(parent_kn);
2902 return ret;
2903}
2904
2905
2906
2907
2908
2909static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn,
2910 const char *name, umode_t mode)
2911{
2912 struct rdtgroup *rdtgrp;
2913 struct kernfs_node *kn;
2914 u32 closid;
2915 int ret;
2916
2917 ret = mkdir_rdt_prepare(parent_kn, name, mode, RDTCTRL_GROUP, &rdtgrp);
2918 if (ret)
2919 return ret;
2920
2921 kn = rdtgrp->kn;
2922 ret = closid_alloc();
2923 if (ret < 0) {
2924 rdt_last_cmd_puts("Out of CLOSIDs\n");
2925 goto out_common_fail;
2926 }
2927 closid = ret;
2928 ret = 0;
2929
2930 rdtgrp->closid = closid;
2931 ret = rdtgroup_init_alloc(rdtgrp);
2932 if (ret < 0)
2933 goto out_id_free;
2934
2935 list_add(&rdtgrp->rdtgroup_list, &rdt_all_groups);
2936
2937 if (rdt_mon_capable) {
2938
2939
2940
2941
2942 ret = mongroup_create_dir(kn, rdtgrp, "mon_groups", NULL);
2943 if (ret) {
2944 rdt_last_cmd_puts("kernfs subdir error\n");
2945 goto out_del_list;
2946 }
2947 }
2948
2949 goto out_unlock;
2950
2951out_del_list:
2952 list_del(&rdtgrp->rdtgroup_list);
2953out_id_free:
2954 closid_free(closid);
2955out_common_fail:
2956 mkdir_rdt_prepare_clean(rdtgrp);
2957out_unlock:
2958 rdtgroup_kn_unlock(parent_kn);
2959 return ret;
2960}
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972static bool is_mon_groups(struct kernfs_node *kn, const char *name)
2973{
2974 return (!strcmp(kn->name, "mon_groups") &&
2975 strcmp(name, "mon_groups"));
2976}
2977
2978static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
2979 umode_t mode)
2980{
2981
2982 if (strchr(name, '\n'))
2983 return -EINVAL;
2984
2985
2986
2987
2988
2989
2990 if (rdt_alloc_capable && parent_kn == rdtgroup_default.kn)
2991 return rdtgroup_mkdir_ctrl_mon(parent_kn, name, mode);
2992
2993
2994
2995
2996
2997 if (rdt_mon_capable && is_mon_groups(parent_kn, name))
2998 return rdtgroup_mkdir_mon(parent_kn, name, mode);
2999
3000 return -EPERM;
3001}
3002
3003static int rdtgroup_rmdir_mon(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask)
3004{
3005 struct rdtgroup *prdtgrp = rdtgrp->mon.parent;
3006 int cpu;
3007
3008
3009 rdt_move_group_tasks(rdtgrp, prdtgrp, tmpmask);
3010
3011
3012 for_each_cpu(cpu, &rdtgrp->cpu_mask)
3013 per_cpu(pqr_state.default_rmid, cpu) = prdtgrp->mon.rmid;
3014
3015
3016
3017
3018 cpumask_or(tmpmask, tmpmask, &rdtgrp->cpu_mask);
3019 update_closid_rmid(tmpmask, NULL);
3020
3021 rdtgrp->flags = RDT_DELETED;
3022 free_rmid(rdtgrp->mon.rmid);
3023
3024
3025
3026
3027 WARN_ON(list_empty(&prdtgrp->mon.crdtgrp_list));
3028 list_del(&rdtgrp->mon.crdtgrp_list);
3029
3030 kernfs_remove(rdtgrp->kn);
3031
3032 return 0;
3033}
3034
3035static int rdtgroup_ctrl_remove(struct rdtgroup *rdtgrp)
3036{
3037 rdtgrp->flags = RDT_DELETED;
3038 list_del(&rdtgrp->rdtgroup_list);
3039
3040 kernfs_remove(rdtgrp->kn);
3041 return 0;
3042}
3043
3044static int rdtgroup_rmdir_ctrl(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask)
3045{
3046 int cpu;
3047
3048
3049 rdt_move_group_tasks(rdtgrp, &rdtgroup_default, tmpmask);
3050
3051
3052 cpumask_or(&rdtgroup_default.cpu_mask,
3053 &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask);
3054
3055
3056 for_each_cpu(cpu, &rdtgrp->cpu_mask) {
3057 per_cpu(pqr_state.default_closid, cpu) = rdtgroup_default.closid;
3058 per_cpu(pqr_state.default_rmid, cpu) = rdtgroup_default.mon.rmid;
3059 }
3060
3061
3062
3063
3064
3065 cpumask_or(tmpmask, tmpmask, &rdtgrp->cpu_mask);
3066 update_closid_rmid(tmpmask, NULL);
3067
3068 closid_free(rdtgrp->closid);
3069 free_rmid(rdtgrp->mon.rmid);
3070
3071 rdtgroup_ctrl_remove(rdtgrp);
3072
3073
3074
3075
3076 free_all_child_rdtgrp(rdtgrp);
3077
3078 return 0;
3079}
3080
3081static int rdtgroup_rmdir(struct kernfs_node *kn)
3082{
3083 struct kernfs_node *parent_kn = kn->parent;
3084 struct rdtgroup *rdtgrp;
3085 cpumask_var_t tmpmask;
3086 int ret = 0;
3087
3088 if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
3089 return -ENOMEM;
3090
3091 rdtgrp = rdtgroup_kn_lock_live(kn);
3092 if (!rdtgrp) {
3093 ret = -EPERM;
3094 goto out;
3095 }
3096
3097
3098
3099
3100
3101
3102
3103
3104 if (rdtgrp->type == RDTCTRL_GROUP && parent_kn == rdtgroup_default.kn &&
3105 rdtgrp != &rdtgroup_default) {
3106 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
3107 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
3108 ret = rdtgroup_ctrl_remove(rdtgrp);
3109 } else {
3110 ret = rdtgroup_rmdir_ctrl(rdtgrp, tmpmask);
3111 }
3112 } else if (rdtgrp->type == RDTMON_GROUP &&
3113 is_mon_groups(parent_kn, kn->name)) {
3114 ret = rdtgroup_rmdir_mon(rdtgrp, tmpmask);
3115 } else {
3116 ret = -EPERM;
3117 }
3118
3119out:
3120 rdtgroup_kn_unlock(kn);
3121 free_cpumask_var(tmpmask);
3122 return ret;
3123}
3124
3125static int rdtgroup_show_options(struct seq_file *seq, struct kernfs_root *kf)
3126{
3127 if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled)
3128 seq_puts(seq, ",cdp");
3129
3130 if (rdt_resources_all[RDT_RESOURCE_L2DATA].alloc_enabled)
3131 seq_puts(seq, ",cdpl2");
3132
3133 if (is_mba_sc(&rdt_resources_all[RDT_RESOURCE_MBA]))
3134 seq_puts(seq, ",mba_MBps");
3135
3136 return 0;
3137}
3138
3139static struct kernfs_syscall_ops rdtgroup_kf_syscall_ops = {
3140 .mkdir = rdtgroup_mkdir,
3141 .rmdir = rdtgroup_rmdir,
3142 .show_options = rdtgroup_show_options,
3143};
3144
3145static int __init rdtgroup_setup_root(void)
3146{
3147 int ret;
3148
3149 rdt_root = kernfs_create_root(&rdtgroup_kf_syscall_ops,
3150 KERNFS_ROOT_CREATE_DEACTIVATED |
3151 KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK,
3152 &rdtgroup_default);
3153 if (IS_ERR(rdt_root))
3154 return PTR_ERR(rdt_root);
3155
3156 mutex_lock(&rdtgroup_mutex);
3157
3158 rdtgroup_default.closid = 0;
3159 rdtgroup_default.mon.rmid = 0;
3160 rdtgroup_default.type = RDTCTRL_GROUP;
3161 INIT_LIST_HEAD(&rdtgroup_default.mon.crdtgrp_list);
3162
3163 list_add(&rdtgroup_default.rdtgroup_list, &rdt_all_groups);
3164
3165 ret = rdtgroup_add_files(rdt_root->kn, RF_CTRL_BASE);
3166 if (ret) {
3167 kernfs_destroy_root(rdt_root);
3168 goto out;
3169 }
3170
3171 rdtgroup_default.kn = rdt_root->kn;
3172 kernfs_activate(rdtgroup_default.kn);
3173
3174out:
3175 mutex_unlock(&rdtgroup_mutex);
3176
3177 return ret;
3178}
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188int __init rdtgroup_init(void)
3189{
3190 int ret = 0;
3191
3192 seq_buf_init(&last_cmd_status, last_cmd_status_buf,
3193 sizeof(last_cmd_status_buf));
3194
3195 ret = rdtgroup_setup_root();
3196 if (ret)
3197 return ret;
3198
3199 ret = sysfs_create_mount_point(fs_kobj, "resctrl");
3200 if (ret)
3201 goto cleanup_root;
3202
3203 ret = register_filesystem(&rdt_fs_type);
3204 if (ret)
3205 goto cleanup_mountpoint;
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228 debugfs_resctrl = debugfs_create_dir("resctrl", NULL);
3229
3230 return 0;
3231
3232cleanup_mountpoint:
3233 sysfs_remove_mount_point(fs_kobj, "resctrl");
3234cleanup_root:
3235 kernfs_destroy_root(rdt_root);
3236
3237 return ret;
3238}
3239
3240void __exit rdtgroup_exit(void)
3241{
3242 debugfs_remove_recursive(debugfs_resctrl);
3243 unregister_filesystem(&rdt_fs_type);
3244 sysfs_remove_mount_point(fs_kobj, "resctrl");
3245 kernfs_destroy_root(rdt_root);
3246}
3247