1
2
3
4
5
6
7
8
9
10
11
12
13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14
15#include <linux/cacheinfo.h>
16#include <linux/cpu.h>
17#include <linux/debugfs.h>
18#include <linux/fs.h>
19#include <linux/fs_parser.h>
20#include <linux/sysfs.h>
21#include <linux/kernfs.h>
22#include <linux/seq_buf.h>
23#include <linux/seq_file.h>
24#include <linux/sched/signal.h>
25#include <linux/sched/task.h>
26#include <linux/slab.h>
27#include <linux/task_work.h>
28#include <linux/user_namespace.h>
29
30#include <uapi/linux/magic.h>
31
32#include <asm/resctrl_sched.h>
33#include "internal.h"
34
35DEFINE_STATIC_KEY_FALSE(rdt_enable_key);
36DEFINE_STATIC_KEY_FALSE(rdt_mon_enable_key);
37DEFINE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
38static struct kernfs_root *rdt_root;
39struct rdtgroup rdtgroup_default;
40LIST_HEAD(rdt_all_groups);
41
42
43static struct kernfs_node *kn_info;
44
45
46static struct kernfs_node *kn_mongrp;
47
48
49static struct kernfs_node *kn_mondata;
50
51static struct seq_buf last_cmd_status;
52static char last_cmd_status_buf[512];
53
54struct dentry *debugfs_resctrl;
55
56void rdt_last_cmd_clear(void)
57{
58 lockdep_assert_held(&rdtgroup_mutex);
59 seq_buf_clear(&last_cmd_status);
60}
61
62void rdt_last_cmd_puts(const char *s)
63{
64 lockdep_assert_held(&rdtgroup_mutex);
65 seq_buf_puts(&last_cmd_status, s);
66}
67
68void rdt_last_cmd_printf(const char *fmt, ...)
69{
70 va_list ap;
71
72 va_start(ap, fmt);
73 lockdep_assert_held(&rdtgroup_mutex);
74 seq_buf_vprintf(&last_cmd_status, fmt, ap);
75 va_end(ap);
76}
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93static int closid_free_map;
94static int closid_free_map_len;
95
96int closids_supported(void)
97{
98 return closid_free_map_len;
99}
100
101static void closid_init(void)
102{
103 struct rdt_resource *r;
104 int rdt_min_closid = 32;
105
106
107 for_each_alloc_enabled_rdt_resource(r)
108 rdt_min_closid = min(rdt_min_closid, r->num_closid);
109
110 closid_free_map = BIT_MASK(rdt_min_closid) - 1;
111
112
113 closid_free_map &= ~1;
114 closid_free_map_len = rdt_min_closid;
115}
116
117static int closid_alloc(void)
118{
119 u32 closid = ffs(closid_free_map);
120
121 if (closid == 0)
122 return -ENOSPC;
123 closid--;
124 closid_free_map &= ~(1 << closid);
125
126 return closid;
127}
128
129void closid_free(int closid)
130{
131 closid_free_map |= 1 << closid;
132}
133
134
135
136
137
138
139
140
141static bool closid_allocated(unsigned int closid)
142{
143 return (closid_free_map & (1 << closid)) == 0;
144}
145
146
147
148
149
150
151
152
153
154
155enum rdtgrp_mode rdtgroup_mode_by_closid(int closid)
156{
157 struct rdtgroup *rdtgrp;
158
159 list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) {
160 if (rdtgrp->closid == closid)
161 return rdtgrp->mode;
162 }
163
164 return RDT_NUM_MODES;
165}
166
167static const char * const rdt_mode_str[] = {
168 [RDT_MODE_SHAREABLE] = "shareable",
169 [RDT_MODE_EXCLUSIVE] = "exclusive",
170 [RDT_MODE_PSEUDO_LOCKSETUP] = "pseudo-locksetup",
171 [RDT_MODE_PSEUDO_LOCKED] = "pseudo-locked",
172};
173
174
175
176
177
178
179
180static const char *rdtgroup_mode_str(enum rdtgrp_mode mode)
181{
182 if (mode < RDT_MODE_SHAREABLE || mode >= RDT_NUM_MODES)
183 return "unknown";
184
185 return rdt_mode_str[mode];
186}
187
188
189static int rdtgroup_kn_set_ugid(struct kernfs_node *kn)
190{
191 struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
192 .ia_uid = current_fsuid(),
193 .ia_gid = current_fsgid(), };
194
195 if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
196 gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
197 return 0;
198
199 return kernfs_setattr(kn, &iattr);
200}
201
202static int rdtgroup_add_file(struct kernfs_node *parent_kn, struct rftype *rft)
203{
204 struct kernfs_node *kn;
205 int ret;
206
207 kn = __kernfs_create_file(parent_kn, rft->name, rft->mode,
208 GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
209 0, rft->kf_ops, rft, NULL, NULL);
210 if (IS_ERR(kn))
211 return PTR_ERR(kn);
212
213 ret = rdtgroup_kn_set_ugid(kn);
214 if (ret) {
215 kernfs_remove(kn);
216 return ret;
217 }
218
219 return 0;
220}
221
222static int rdtgroup_seqfile_show(struct seq_file *m, void *arg)
223{
224 struct kernfs_open_file *of = m->private;
225 struct rftype *rft = of->kn->priv;
226
227 if (rft->seq_show)
228 return rft->seq_show(of, m, arg);
229 return 0;
230}
231
232static ssize_t rdtgroup_file_write(struct kernfs_open_file *of, char *buf,
233 size_t nbytes, loff_t off)
234{
235 struct rftype *rft = of->kn->priv;
236
237 if (rft->write)
238 return rft->write(of, buf, nbytes, off);
239
240 return -EINVAL;
241}
242
243static struct kernfs_ops rdtgroup_kf_single_ops = {
244 .atomic_write_len = PAGE_SIZE,
245 .write = rdtgroup_file_write,
246 .seq_show = rdtgroup_seqfile_show,
247};
248
249static struct kernfs_ops kf_mondata_ops = {
250 .atomic_write_len = PAGE_SIZE,
251 .seq_show = rdtgroup_mondata_show,
252};
253
254static bool is_cpu_list(struct kernfs_open_file *of)
255{
256 struct rftype *rft = of->kn->priv;
257
258 return rft->flags & RFTYPE_FLAGS_CPUS_LIST;
259}
260
261static int rdtgroup_cpus_show(struct kernfs_open_file *of,
262 struct seq_file *s, void *v)
263{
264 struct rdtgroup *rdtgrp;
265 struct cpumask *mask;
266 int ret = 0;
267
268 rdtgrp = rdtgroup_kn_lock_live(of->kn);
269
270 if (rdtgrp) {
271 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
272 if (!rdtgrp->plr->d) {
273 rdt_last_cmd_clear();
274 rdt_last_cmd_puts("Cache domain offline\n");
275 ret = -ENODEV;
276 } else {
277 mask = &rdtgrp->plr->d->cpu_mask;
278 seq_printf(s, is_cpu_list(of) ?
279 "%*pbl\n" : "%*pb\n",
280 cpumask_pr_args(mask));
281 }
282 } else {
283 seq_printf(s, is_cpu_list(of) ? "%*pbl\n" : "%*pb\n",
284 cpumask_pr_args(&rdtgrp->cpu_mask));
285 }
286 } else {
287 ret = -ENOENT;
288 }
289 rdtgroup_kn_unlock(of->kn);
290
291 return ret;
292}
293
294
295
296
297
298
299
300static void update_cpu_closid_rmid(void *info)
301{
302 struct rdtgroup *r = info;
303
304 if (r) {
305 this_cpu_write(pqr_state.default_closid, r->closid);
306 this_cpu_write(pqr_state.default_rmid, r->mon.rmid);
307 }
308
309
310
311
312
313
314 resctrl_sched_in();
315}
316
317
318
319
320
321
322static void
323update_closid_rmid(const struct cpumask *cpu_mask, struct rdtgroup *r)
324{
325 int cpu = get_cpu();
326
327 if (cpumask_test_cpu(cpu, cpu_mask))
328 update_cpu_closid_rmid(r);
329 smp_call_function_many(cpu_mask, update_cpu_closid_rmid, r, 1);
330 put_cpu();
331}
332
333static int cpus_mon_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask,
334 cpumask_var_t tmpmask)
335{
336 struct rdtgroup *prgrp = rdtgrp->mon.parent, *crgrp;
337 struct list_head *head;
338
339
340 cpumask_andnot(tmpmask, newmask, &prgrp->cpu_mask);
341 if (cpumask_weight(tmpmask)) {
342 rdt_last_cmd_puts("Can only add CPUs to mongroup that belong to parent\n");
343 return -EINVAL;
344 }
345
346
347 cpumask_andnot(tmpmask, &rdtgrp->cpu_mask, newmask);
348 if (cpumask_weight(tmpmask)) {
349
350 cpumask_or(&prgrp->cpu_mask, &prgrp->cpu_mask, tmpmask);
351 update_closid_rmid(tmpmask, prgrp);
352 }
353
354
355
356
357
358 cpumask_andnot(tmpmask, newmask, &rdtgrp->cpu_mask);
359 if (cpumask_weight(tmpmask)) {
360 head = &prgrp->mon.crdtgrp_list;
361 list_for_each_entry(crgrp, head, mon.crdtgrp_list) {
362 if (crgrp == rdtgrp)
363 continue;
364 cpumask_andnot(&crgrp->cpu_mask, &crgrp->cpu_mask,
365 tmpmask);
366 }
367 update_closid_rmid(tmpmask, rdtgrp);
368 }
369
370
371 cpumask_copy(&rdtgrp->cpu_mask, newmask);
372
373 return 0;
374}
375
376static void cpumask_rdtgrp_clear(struct rdtgroup *r, struct cpumask *m)
377{
378 struct rdtgroup *crgrp;
379
380 cpumask_andnot(&r->cpu_mask, &r->cpu_mask, m);
381
382 list_for_each_entry(crgrp, &r->mon.crdtgrp_list, mon.crdtgrp_list)
383 cpumask_and(&crgrp->cpu_mask, &r->cpu_mask, &crgrp->cpu_mask);
384}
385
386static int cpus_ctrl_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask,
387 cpumask_var_t tmpmask, cpumask_var_t tmpmask1)
388{
389 struct rdtgroup *r, *crgrp;
390 struct list_head *head;
391
392
393 cpumask_andnot(tmpmask, &rdtgrp->cpu_mask, newmask);
394 if (cpumask_weight(tmpmask)) {
395
396 if (rdtgrp == &rdtgroup_default) {
397 rdt_last_cmd_puts("Can't drop CPUs from default group\n");
398 return -EINVAL;
399 }
400
401
402 cpumask_or(&rdtgroup_default.cpu_mask,
403 &rdtgroup_default.cpu_mask, tmpmask);
404 update_closid_rmid(tmpmask, &rdtgroup_default);
405 }
406
407
408
409
410
411
412 cpumask_andnot(tmpmask, newmask, &rdtgrp->cpu_mask);
413 if (cpumask_weight(tmpmask)) {
414 list_for_each_entry(r, &rdt_all_groups, rdtgroup_list) {
415 if (r == rdtgrp)
416 continue;
417 cpumask_and(tmpmask1, &r->cpu_mask, tmpmask);
418 if (cpumask_weight(tmpmask1))
419 cpumask_rdtgrp_clear(r, tmpmask1);
420 }
421 update_closid_rmid(tmpmask, rdtgrp);
422 }
423
424
425 cpumask_copy(&rdtgrp->cpu_mask, newmask);
426
427
428
429
430
431 head = &rdtgrp->mon.crdtgrp_list;
432 list_for_each_entry(crgrp, head, mon.crdtgrp_list) {
433 cpumask_and(tmpmask, &rdtgrp->cpu_mask, &crgrp->cpu_mask);
434 update_closid_rmid(tmpmask, rdtgrp);
435 cpumask_clear(&crgrp->cpu_mask);
436 }
437
438 return 0;
439}
440
441static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of,
442 char *buf, size_t nbytes, loff_t off)
443{
444 cpumask_var_t tmpmask, newmask, tmpmask1;
445 struct rdtgroup *rdtgrp;
446 int ret;
447
448 if (!buf)
449 return -EINVAL;
450
451 if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
452 return -ENOMEM;
453 if (!zalloc_cpumask_var(&newmask, GFP_KERNEL)) {
454 free_cpumask_var(tmpmask);
455 return -ENOMEM;
456 }
457 if (!zalloc_cpumask_var(&tmpmask1, GFP_KERNEL)) {
458 free_cpumask_var(tmpmask);
459 free_cpumask_var(newmask);
460 return -ENOMEM;
461 }
462
463 rdtgrp = rdtgroup_kn_lock_live(of->kn);
464 rdt_last_cmd_clear();
465 if (!rdtgrp) {
466 ret = -ENOENT;
467 rdt_last_cmd_puts("Directory was removed\n");
468 goto unlock;
469 }
470
471 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED ||
472 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
473 ret = -EINVAL;
474 rdt_last_cmd_puts("Pseudo-locking in progress\n");
475 goto unlock;
476 }
477
478 if (is_cpu_list(of))
479 ret = cpulist_parse(buf, newmask);
480 else
481 ret = cpumask_parse(buf, newmask);
482
483 if (ret) {
484 rdt_last_cmd_puts("Bad CPU list/mask\n");
485 goto unlock;
486 }
487
488
489 cpumask_andnot(tmpmask, newmask, cpu_online_mask);
490 if (cpumask_weight(tmpmask)) {
491 ret = -EINVAL;
492 rdt_last_cmd_puts("Can only assign online CPUs\n");
493 goto unlock;
494 }
495
496 if (rdtgrp->type == RDTCTRL_GROUP)
497 ret = cpus_ctrl_write(rdtgrp, newmask, tmpmask, tmpmask1);
498 else if (rdtgrp->type == RDTMON_GROUP)
499 ret = cpus_mon_write(rdtgrp, newmask, tmpmask);
500 else
501 ret = -EINVAL;
502
503unlock:
504 rdtgroup_kn_unlock(of->kn);
505 free_cpumask_var(tmpmask);
506 free_cpumask_var(newmask);
507 free_cpumask_var(tmpmask1);
508
509 return ret ?: nbytes;
510}
511
512struct task_move_callback {
513 struct callback_head work;
514 struct rdtgroup *rdtgrp;
515};
516
517static void move_myself(struct callback_head *head)
518{
519 struct task_move_callback *callback;
520 struct rdtgroup *rdtgrp;
521
522 callback = container_of(head, struct task_move_callback, work);
523 rdtgrp = callback->rdtgrp;
524
525
526
527
528
529
530 if (atomic_dec_and_test(&rdtgrp->waitcount) &&
531 (rdtgrp->flags & RDT_DELETED)) {
532 current->closid = 0;
533 current->rmid = 0;
534 kfree(rdtgrp);
535 }
536
537 preempt_disable();
538
539 resctrl_sched_in();
540 preempt_enable();
541
542 kfree(callback);
543}
544
545static int __rdtgroup_move_task(struct task_struct *tsk,
546 struct rdtgroup *rdtgrp)
547{
548 struct task_move_callback *callback;
549 int ret;
550
551 callback = kzalloc(sizeof(*callback), GFP_KERNEL);
552 if (!callback)
553 return -ENOMEM;
554 callback->work.func = move_myself;
555 callback->rdtgrp = rdtgrp;
556
557
558
559
560
561 atomic_inc(&rdtgrp->waitcount);
562 ret = task_work_add(tsk, &callback->work, true);
563 if (ret) {
564
565
566
567
568
569 atomic_dec(&rdtgrp->waitcount);
570 kfree(callback);
571 rdt_last_cmd_puts("Task exited\n");
572 } else {
573
574
575
576
577
578 if (rdtgrp->type == RDTCTRL_GROUP) {
579 tsk->closid = rdtgrp->closid;
580 tsk->rmid = rdtgrp->mon.rmid;
581 } else if (rdtgrp->type == RDTMON_GROUP) {
582 if (rdtgrp->mon.parent->closid == tsk->closid) {
583 tsk->rmid = rdtgrp->mon.rmid;
584 } else {
585 rdt_last_cmd_puts("Can't move task to different control group\n");
586 ret = -EINVAL;
587 }
588 }
589 }
590 return ret;
591}
592
593
594
595
596
597
598
599int rdtgroup_tasks_assigned(struct rdtgroup *r)
600{
601 struct task_struct *p, *t;
602 int ret = 0;
603
604 lockdep_assert_held(&rdtgroup_mutex);
605
606 rcu_read_lock();
607 for_each_process_thread(p, t) {
608 if ((r->type == RDTCTRL_GROUP && t->closid == r->closid) ||
609 (r->type == RDTMON_GROUP && t->rmid == r->mon.rmid)) {
610 ret = 1;
611 break;
612 }
613 }
614 rcu_read_unlock();
615
616 return ret;
617}
618
619static int rdtgroup_task_write_permission(struct task_struct *task,
620 struct kernfs_open_file *of)
621{
622 const struct cred *tcred = get_task_cred(task);
623 const struct cred *cred = current_cred();
624 int ret = 0;
625
626
627
628
629
630 if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
631 !uid_eq(cred->euid, tcred->uid) &&
632 !uid_eq(cred->euid, tcred->suid)) {
633 rdt_last_cmd_printf("No permission to move task %d\n", task->pid);
634 ret = -EPERM;
635 }
636
637 put_cred(tcred);
638 return ret;
639}
640
641static int rdtgroup_move_task(pid_t pid, struct rdtgroup *rdtgrp,
642 struct kernfs_open_file *of)
643{
644 struct task_struct *tsk;
645 int ret;
646
647 rcu_read_lock();
648 if (pid) {
649 tsk = find_task_by_vpid(pid);
650 if (!tsk) {
651 rcu_read_unlock();
652 rdt_last_cmd_printf("No task %d\n", pid);
653 return -ESRCH;
654 }
655 } else {
656 tsk = current;
657 }
658
659 get_task_struct(tsk);
660 rcu_read_unlock();
661
662 ret = rdtgroup_task_write_permission(tsk, of);
663 if (!ret)
664 ret = __rdtgroup_move_task(tsk, rdtgrp);
665
666 put_task_struct(tsk);
667 return ret;
668}
669
670static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of,
671 char *buf, size_t nbytes, loff_t off)
672{
673 struct rdtgroup *rdtgrp;
674 int ret = 0;
675 pid_t pid;
676
677 if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
678 return -EINVAL;
679 rdtgrp = rdtgroup_kn_lock_live(of->kn);
680 if (!rdtgrp) {
681 rdtgroup_kn_unlock(of->kn);
682 return -ENOENT;
683 }
684 rdt_last_cmd_clear();
685
686 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED ||
687 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
688 ret = -EINVAL;
689 rdt_last_cmd_puts("Pseudo-locking in progress\n");
690 goto unlock;
691 }
692
693 ret = rdtgroup_move_task(pid, rdtgrp, of);
694
695unlock:
696 rdtgroup_kn_unlock(of->kn);
697
698 return ret ?: nbytes;
699}
700
701static void show_rdt_tasks(struct rdtgroup *r, struct seq_file *s)
702{
703 struct task_struct *p, *t;
704
705 rcu_read_lock();
706 for_each_process_thread(p, t) {
707 if ((r->type == RDTCTRL_GROUP && t->closid == r->closid) ||
708 (r->type == RDTMON_GROUP && t->rmid == r->mon.rmid))
709 seq_printf(s, "%d\n", t->pid);
710 }
711 rcu_read_unlock();
712}
713
714static int rdtgroup_tasks_show(struct kernfs_open_file *of,
715 struct seq_file *s, void *v)
716{
717 struct rdtgroup *rdtgrp;
718 int ret = 0;
719
720 rdtgrp = rdtgroup_kn_lock_live(of->kn);
721 if (rdtgrp)
722 show_rdt_tasks(rdtgrp, s);
723 else
724 ret = -ENOENT;
725 rdtgroup_kn_unlock(of->kn);
726
727 return ret;
728}
729
730static int rdt_last_cmd_status_show(struct kernfs_open_file *of,
731 struct seq_file *seq, void *v)
732{
733 int len;
734
735 mutex_lock(&rdtgroup_mutex);
736 len = seq_buf_used(&last_cmd_status);
737 if (len)
738 seq_printf(seq, "%.*s", len, last_cmd_status_buf);
739 else
740 seq_puts(seq, "ok\n");
741 mutex_unlock(&rdtgroup_mutex);
742 return 0;
743}
744
745static int rdt_num_closids_show(struct kernfs_open_file *of,
746 struct seq_file *seq, void *v)
747{
748 struct rdt_resource *r = of->kn->parent->priv;
749
750 seq_printf(seq, "%d\n", r->num_closid);
751 return 0;
752}
753
754static int rdt_default_ctrl_show(struct kernfs_open_file *of,
755 struct seq_file *seq, void *v)
756{
757 struct rdt_resource *r = of->kn->parent->priv;
758
759 seq_printf(seq, "%x\n", r->default_ctrl);
760 return 0;
761}
762
763static int rdt_min_cbm_bits_show(struct kernfs_open_file *of,
764 struct seq_file *seq, void *v)
765{
766 struct rdt_resource *r = of->kn->parent->priv;
767
768 seq_printf(seq, "%u\n", r->cache.min_cbm_bits);
769 return 0;
770}
771
772static int rdt_shareable_bits_show(struct kernfs_open_file *of,
773 struct seq_file *seq, void *v)
774{
775 struct rdt_resource *r = of->kn->parent->priv;
776
777 seq_printf(seq, "%x\n", r->cache.shareable_bits);
778 return 0;
779}
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795static int rdt_bit_usage_show(struct kernfs_open_file *of,
796 struct seq_file *seq, void *v)
797{
798 struct rdt_resource *r = of->kn->parent->priv;
799
800
801
802
803 unsigned long sw_shareable = 0, hw_shareable = 0;
804 unsigned long exclusive = 0, pseudo_locked = 0;
805 struct rdt_domain *dom;
806 int i, hwb, swb, excl, psl;
807 enum rdtgrp_mode mode;
808 bool sep = false;
809 u32 *ctrl;
810
811 mutex_lock(&rdtgroup_mutex);
812 hw_shareable = r->cache.shareable_bits;
813 list_for_each_entry(dom, &r->domains, list) {
814 if (sep)
815 seq_putc(seq, ';');
816 ctrl = dom->ctrl_val;
817 sw_shareable = 0;
818 exclusive = 0;
819 seq_printf(seq, "%d=", dom->id);
820 for (i = 0; i < closids_supported(); i++, ctrl++) {
821 if (!closid_allocated(i))
822 continue;
823 mode = rdtgroup_mode_by_closid(i);
824 switch (mode) {
825 case RDT_MODE_SHAREABLE:
826 sw_shareable |= *ctrl;
827 break;
828 case RDT_MODE_EXCLUSIVE:
829 exclusive |= *ctrl;
830 break;
831 case RDT_MODE_PSEUDO_LOCKSETUP:
832
833
834
835
836
837
838
839 break;
840 case RDT_MODE_PSEUDO_LOCKED:
841 case RDT_NUM_MODES:
842 WARN(1,
843 "invalid mode for closid %d\n", i);
844 break;
845 }
846 }
847 for (i = r->cache.cbm_len - 1; i >= 0; i--) {
848 pseudo_locked = dom->plr ? dom->plr->cbm : 0;
849 hwb = test_bit(i, &hw_shareable);
850 swb = test_bit(i, &sw_shareable);
851 excl = test_bit(i, &exclusive);
852 psl = test_bit(i, &pseudo_locked);
853 if (hwb && swb)
854 seq_putc(seq, 'X');
855 else if (hwb && !swb)
856 seq_putc(seq, 'H');
857 else if (!hwb && swb)
858 seq_putc(seq, 'S');
859 else if (excl)
860 seq_putc(seq, 'E');
861 else if (psl)
862 seq_putc(seq, 'P');
863 else
864 seq_putc(seq, '0');
865 }
866 sep = true;
867 }
868 seq_putc(seq, '\n');
869 mutex_unlock(&rdtgroup_mutex);
870 return 0;
871}
872
873static int rdt_min_bw_show(struct kernfs_open_file *of,
874 struct seq_file *seq, void *v)
875{
876 struct rdt_resource *r = of->kn->parent->priv;
877
878 seq_printf(seq, "%u\n", r->membw.min_bw);
879 return 0;
880}
881
882static int rdt_num_rmids_show(struct kernfs_open_file *of,
883 struct seq_file *seq, void *v)
884{
885 struct rdt_resource *r = of->kn->parent->priv;
886
887 seq_printf(seq, "%d\n", r->num_rmid);
888
889 return 0;
890}
891
892static int rdt_mon_features_show(struct kernfs_open_file *of,
893 struct seq_file *seq, void *v)
894{
895 struct rdt_resource *r = of->kn->parent->priv;
896 struct mon_evt *mevt;
897
898 list_for_each_entry(mevt, &r->evt_list, list)
899 seq_printf(seq, "%s\n", mevt->name);
900
901 return 0;
902}
903
904static int rdt_bw_gran_show(struct kernfs_open_file *of,
905 struct seq_file *seq, void *v)
906{
907 struct rdt_resource *r = of->kn->parent->priv;
908
909 seq_printf(seq, "%u\n", r->membw.bw_gran);
910 return 0;
911}
912
913static int rdt_delay_linear_show(struct kernfs_open_file *of,
914 struct seq_file *seq, void *v)
915{
916 struct rdt_resource *r = of->kn->parent->priv;
917
918 seq_printf(seq, "%u\n", r->membw.delay_linear);
919 return 0;
920}
921
922static int max_threshold_occ_show(struct kernfs_open_file *of,
923 struct seq_file *seq, void *v)
924{
925 struct rdt_resource *r = of->kn->parent->priv;
926
927 seq_printf(seq, "%u\n", resctrl_cqm_threshold * r->mon_scale);
928
929 return 0;
930}
931
932static ssize_t max_threshold_occ_write(struct kernfs_open_file *of,
933 char *buf, size_t nbytes, loff_t off)
934{
935 struct rdt_resource *r = of->kn->parent->priv;
936 unsigned int bytes;
937 int ret;
938
939 ret = kstrtouint(buf, 0, &bytes);
940 if (ret)
941 return ret;
942
943 if (bytes > (boot_cpu_data.x86_cache_size * 1024))
944 return -EINVAL;
945
946 resctrl_cqm_threshold = bytes / r->mon_scale;
947
948 return nbytes;
949}
950
951
952
953
954static int rdtgroup_mode_show(struct kernfs_open_file *of,
955 struct seq_file *s, void *v)
956{
957 struct rdtgroup *rdtgrp;
958
959 rdtgrp = rdtgroup_kn_lock_live(of->kn);
960 if (!rdtgrp) {
961 rdtgroup_kn_unlock(of->kn);
962 return -ENOENT;
963 }
964
965 seq_printf(s, "%s\n", rdtgroup_mode_str(rdtgrp->mode));
966
967 rdtgroup_kn_unlock(of->kn);
968 return 0;
969}
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994static int rdt_cdp_peer_get(struct rdt_resource *r, struct rdt_domain *d,
995 struct rdt_resource **r_cdp,
996 struct rdt_domain **d_cdp)
997{
998 struct rdt_resource *_r_cdp = NULL;
999 struct rdt_domain *_d_cdp = NULL;
1000 int ret = 0;
1001
1002 switch (r->rid) {
1003 case RDT_RESOURCE_L3DATA:
1004 _r_cdp = &rdt_resources_all[RDT_RESOURCE_L3CODE];
1005 break;
1006 case RDT_RESOURCE_L3CODE:
1007 _r_cdp = &rdt_resources_all[RDT_RESOURCE_L3DATA];
1008 break;
1009 case RDT_RESOURCE_L2DATA:
1010 _r_cdp = &rdt_resources_all[RDT_RESOURCE_L2CODE];
1011 break;
1012 case RDT_RESOURCE_L2CODE:
1013 _r_cdp = &rdt_resources_all[RDT_RESOURCE_L2DATA];
1014 break;
1015 default:
1016 ret = -ENOENT;
1017 goto out;
1018 }
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029 _d_cdp = rdt_find_domain(_r_cdp, d->id, NULL);
1030 if (WARN_ON(IS_ERR_OR_NULL(_d_cdp))) {
1031 _r_cdp = NULL;
1032 ret = -EINVAL;
1033 }
1034
1035out:
1036 *r_cdp = _r_cdp;
1037 *d_cdp = _d_cdp;
1038
1039 return ret;
1040}
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062static bool __rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d,
1063 unsigned long cbm, int closid, bool exclusive)
1064{
1065 enum rdtgrp_mode mode;
1066 unsigned long ctrl_b;
1067 u32 *ctrl;
1068 int i;
1069
1070
1071 if (!exclusive) {
1072 ctrl_b = r->cache.shareable_bits;
1073 if (bitmap_intersects(&cbm, &ctrl_b, r->cache.cbm_len))
1074 return true;
1075 }
1076
1077
1078 ctrl = d->ctrl_val;
1079 for (i = 0; i < closids_supported(); i++, ctrl++) {
1080 ctrl_b = *ctrl;
1081 mode = rdtgroup_mode_by_closid(i);
1082 if (closid_allocated(i) && i != closid &&
1083 mode != RDT_MODE_PSEUDO_LOCKSETUP) {
1084 if (bitmap_intersects(&cbm, &ctrl_b, r->cache.cbm_len)) {
1085 if (exclusive) {
1086 if (mode == RDT_MODE_EXCLUSIVE)
1087 return true;
1088 continue;
1089 }
1090 return true;
1091 }
1092 }
1093 }
1094
1095 return false;
1096}
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118bool rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d,
1119 unsigned long cbm, int closid, bool exclusive)
1120{
1121 struct rdt_resource *r_cdp;
1122 struct rdt_domain *d_cdp;
1123
1124 if (__rdtgroup_cbm_overlaps(r, d, cbm, closid, exclusive))
1125 return true;
1126
1127 if (rdt_cdp_peer_get(r, d, &r_cdp, &d_cdp) < 0)
1128 return false;
1129
1130 return __rdtgroup_cbm_overlaps(r_cdp, d_cdp, cbm, closid, exclusive);
1131}
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp)
1146{
1147 int closid = rdtgrp->closid;
1148 struct rdt_resource *r;
1149 bool has_cache = false;
1150 struct rdt_domain *d;
1151
1152 for_each_alloc_enabled_rdt_resource(r) {
1153 if (r->rid == RDT_RESOURCE_MBA)
1154 continue;
1155 has_cache = true;
1156 list_for_each_entry(d, &r->domains, list) {
1157 if (rdtgroup_cbm_overlaps(r, d, d->ctrl_val[closid],
1158 rdtgrp->closid, false)) {
1159 rdt_last_cmd_puts("Schemata overlaps\n");
1160 return false;
1161 }
1162 }
1163 }
1164
1165 if (!has_cache) {
1166 rdt_last_cmd_puts("Cannot be exclusive without CAT/CDP\n");
1167 return false;
1168 }
1169
1170 return true;
1171}
1172
1173
1174
1175
1176
1177static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of,
1178 char *buf, size_t nbytes, loff_t off)
1179{
1180 struct rdtgroup *rdtgrp;
1181 enum rdtgrp_mode mode;
1182 int ret = 0;
1183
1184
1185 if (nbytes == 0 || buf[nbytes - 1] != '\n')
1186 return -EINVAL;
1187 buf[nbytes - 1] = '\0';
1188
1189 rdtgrp = rdtgroup_kn_lock_live(of->kn);
1190 if (!rdtgrp) {
1191 rdtgroup_kn_unlock(of->kn);
1192 return -ENOENT;
1193 }
1194
1195 rdt_last_cmd_clear();
1196
1197 mode = rdtgrp->mode;
1198
1199 if ((!strcmp(buf, "shareable") && mode == RDT_MODE_SHAREABLE) ||
1200 (!strcmp(buf, "exclusive") && mode == RDT_MODE_EXCLUSIVE) ||
1201 (!strcmp(buf, "pseudo-locksetup") &&
1202 mode == RDT_MODE_PSEUDO_LOCKSETUP) ||
1203 (!strcmp(buf, "pseudo-locked") && mode == RDT_MODE_PSEUDO_LOCKED))
1204 goto out;
1205
1206 if (mode == RDT_MODE_PSEUDO_LOCKED) {
1207 rdt_last_cmd_puts("Cannot change pseudo-locked group\n");
1208 ret = -EINVAL;
1209 goto out;
1210 }
1211
1212 if (!strcmp(buf, "shareable")) {
1213 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
1214 ret = rdtgroup_locksetup_exit(rdtgrp);
1215 if (ret)
1216 goto out;
1217 }
1218 rdtgrp->mode = RDT_MODE_SHAREABLE;
1219 } else if (!strcmp(buf, "exclusive")) {
1220 if (!rdtgroup_mode_test_exclusive(rdtgrp)) {
1221 ret = -EINVAL;
1222 goto out;
1223 }
1224 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
1225 ret = rdtgroup_locksetup_exit(rdtgrp);
1226 if (ret)
1227 goto out;
1228 }
1229 rdtgrp->mode = RDT_MODE_EXCLUSIVE;
1230 } else if (!strcmp(buf, "pseudo-locksetup")) {
1231 ret = rdtgroup_locksetup_enter(rdtgrp);
1232 if (ret)
1233 goto out;
1234 rdtgrp->mode = RDT_MODE_PSEUDO_LOCKSETUP;
1235 } else {
1236 rdt_last_cmd_puts("Unknown or unsupported mode\n");
1237 ret = -EINVAL;
1238 }
1239
1240out:
1241 rdtgroup_kn_unlock(of->kn);
1242 return ret ?: nbytes;
1243}
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r,
1261 struct rdt_domain *d, unsigned long cbm)
1262{
1263 struct cpu_cacheinfo *ci;
1264 unsigned int size = 0;
1265 int num_b, i;
1266
1267 num_b = bitmap_weight(&cbm, r->cache.cbm_len);
1268 ci = get_cpu_cacheinfo(cpumask_any(&d->cpu_mask));
1269 for (i = 0; i < ci->num_leaves; i++) {
1270 if (ci->info_list[i].level == r->cache_level) {
1271 size = ci->info_list[i].size / r->cache.cbm_len * num_b;
1272 break;
1273 }
1274 }
1275
1276 return size;
1277}
1278
1279
1280
1281
1282
1283
1284
1285
1286static int rdtgroup_size_show(struct kernfs_open_file *of,
1287 struct seq_file *s, void *v)
1288{
1289 struct rdtgroup *rdtgrp;
1290 struct rdt_resource *r;
1291 struct rdt_domain *d;
1292 unsigned int size;
1293 int ret = 0;
1294 bool sep;
1295 u32 ctrl;
1296
1297 rdtgrp = rdtgroup_kn_lock_live(of->kn);
1298 if (!rdtgrp) {
1299 rdtgroup_kn_unlock(of->kn);
1300 return -ENOENT;
1301 }
1302
1303 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
1304 if (!rdtgrp->plr->d) {
1305 rdt_last_cmd_clear();
1306 rdt_last_cmd_puts("Cache domain offline\n");
1307 ret = -ENODEV;
1308 } else {
1309 seq_printf(s, "%*s:", max_name_width,
1310 rdtgrp->plr->r->name);
1311 size = rdtgroup_cbm_to_size(rdtgrp->plr->r,
1312 rdtgrp->plr->d,
1313 rdtgrp->plr->cbm);
1314 seq_printf(s, "%d=%u\n", rdtgrp->plr->d->id, size);
1315 }
1316 goto out;
1317 }
1318
1319 for_each_alloc_enabled_rdt_resource(r) {
1320 sep = false;
1321 seq_printf(s, "%*s:", max_name_width, r->name);
1322 list_for_each_entry(d, &r->domains, list) {
1323 if (sep)
1324 seq_putc(s, ';');
1325 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
1326 size = 0;
1327 } else {
1328 ctrl = (!is_mba_sc(r) ?
1329 d->ctrl_val[rdtgrp->closid] :
1330 d->mbps_val[rdtgrp->closid]);
1331 if (r->rid == RDT_RESOURCE_MBA)
1332 size = ctrl;
1333 else
1334 size = rdtgroup_cbm_to_size(r, d, ctrl);
1335 }
1336 seq_printf(s, "%d=%u", d->id, size);
1337 sep = true;
1338 }
1339 seq_putc(s, '\n');
1340 }
1341
1342out:
1343 rdtgroup_kn_unlock(of->kn);
1344
1345 return ret;
1346}
1347
1348
1349static struct rftype res_common_files[] = {
1350 {
1351 .name = "last_cmd_status",
1352 .mode = 0444,
1353 .kf_ops = &rdtgroup_kf_single_ops,
1354 .seq_show = rdt_last_cmd_status_show,
1355 .fflags = RF_TOP_INFO,
1356 },
1357 {
1358 .name = "num_closids",
1359 .mode = 0444,
1360 .kf_ops = &rdtgroup_kf_single_ops,
1361 .seq_show = rdt_num_closids_show,
1362 .fflags = RF_CTRL_INFO,
1363 },
1364 {
1365 .name = "mon_features",
1366 .mode = 0444,
1367 .kf_ops = &rdtgroup_kf_single_ops,
1368 .seq_show = rdt_mon_features_show,
1369 .fflags = RF_MON_INFO,
1370 },
1371 {
1372 .name = "num_rmids",
1373 .mode = 0444,
1374 .kf_ops = &rdtgroup_kf_single_ops,
1375 .seq_show = rdt_num_rmids_show,
1376 .fflags = RF_MON_INFO,
1377 },
1378 {
1379 .name = "cbm_mask",
1380 .mode = 0444,
1381 .kf_ops = &rdtgroup_kf_single_ops,
1382 .seq_show = rdt_default_ctrl_show,
1383 .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE,
1384 },
1385 {
1386 .name = "min_cbm_bits",
1387 .mode = 0444,
1388 .kf_ops = &rdtgroup_kf_single_ops,
1389 .seq_show = rdt_min_cbm_bits_show,
1390 .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE,
1391 },
1392 {
1393 .name = "shareable_bits",
1394 .mode = 0444,
1395 .kf_ops = &rdtgroup_kf_single_ops,
1396 .seq_show = rdt_shareable_bits_show,
1397 .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE,
1398 },
1399 {
1400 .name = "bit_usage",
1401 .mode = 0444,
1402 .kf_ops = &rdtgroup_kf_single_ops,
1403 .seq_show = rdt_bit_usage_show,
1404 .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE,
1405 },
1406 {
1407 .name = "min_bandwidth",
1408 .mode = 0444,
1409 .kf_ops = &rdtgroup_kf_single_ops,
1410 .seq_show = rdt_min_bw_show,
1411 .fflags = RF_CTRL_INFO | RFTYPE_RES_MB,
1412 },
1413 {
1414 .name = "bandwidth_gran",
1415 .mode = 0444,
1416 .kf_ops = &rdtgroup_kf_single_ops,
1417 .seq_show = rdt_bw_gran_show,
1418 .fflags = RF_CTRL_INFO | RFTYPE_RES_MB,
1419 },
1420 {
1421 .name = "delay_linear",
1422 .mode = 0444,
1423 .kf_ops = &rdtgroup_kf_single_ops,
1424 .seq_show = rdt_delay_linear_show,
1425 .fflags = RF_CTRL_INFO | RFTYPE_RES_MB,
1426 },
1427 {
1428 .name = "max_threshold_occupancy",
1429 .mode = 0644,
1430 .kf_ops = &rdtgroup_kf_single_ops,
1431 .write = max_threshold_occ_write,
1432 .seq_show = max_threshold_occ_show,
1433 .fflags = RF_MON_INFO | RFTYPE_RES_CACHE,
1434 },
1435 {
1436 .name = "cpus",
1437 .mode = 0644,
1438 .kf_ops = &rdtgroup_kf_single_ops,
1439 .write = rdtgroup_cpus_write,
1440 .seq_show = rdtgroup_cpus_show,
1441 .fflags = RFTYPE_BASE,
1442 },
1443 {
1444 .name = "cpus_list",
1445 .mode = 0644,
1446 .kf_ops = &rdtgroup_kf_single_ops,
1447 .write = rdtgroup_cpus_write,
1448 .seq_show = rdtgroup_cpus_show,
1449 .flags = RFTYPE_FLAGS_CPUS_LIST,
1450 .fflags = RFTYPE_BASE,
1451 },
1452 {
1453 .name = "tasks",
1454 .mode = 0644,
1455 .kf_ops = &rdtgroup_kf_single_ops,
1456 .write = rdtgroup_tasks_write,
1457 .seq_show = rdtgroup_tasks_show,
1458 .fflags = RFTYPE_BASE,
1459 },
1460 {
1461 .name = "schemata",
1462 .mode = 0644,
1463 .kf_ops = &rdtgroup_kf_single_ops,
1464 .write = rdtgroup_schemata_write,
1465 .seq_show = rdtgroup_schemata_show,
1466 .fflags = RF_CTRL_BASE,
1467 },
1468 {
1469 .name = "mode",
1470 .mode = 0644,
1471 .kf_ops = &rdtgroup_kf_single_ops,
1472 .write = rdtgroup_mode_write,
1473 .seq_show = rdtgroup_mode_show,
1474 .fflags = RF_CTRL_BASE,
1475 },
1476 {
1477 .name = "size",
1478 .mode = 0444,
1479 .kf_ops = &rdtgroup_kf_single_ops,
1480 .seq_show = rdtgroup_size_show,
1481 .fflags = RF_CTRL_BASE,
1482 },
1483
1484};
1485
1486static int rdtgroup_add_files(struct kernfs_node *kn, unsigned long fflags)
1487{
1488 struct rftype *rfts, *rft;
1489 int ret, len;
1490
1491 rfts = res_common_files;
1492 len = ARRAY_SIZE(res_common_files);
1493
1494 lockdep_assert_held(&rdtgroup_mutex);
1495
1496 for (rft = rfts; rft < rfts + len; rft++) {
1497 if ((fflags & rft->fflags) == rft->fflags) {
1498 ret = rdtgroup_add_file(kn, rft);
1499 if (ret)
1500 goto error;
1501 }
1502 }
1503
1504 return 0;
1505error:
1506 pr_warn("Failed to add %s, err=%d\n", rft->name, ret);
1507 while (--rft >= rfts) {
1508 if ((fflags & rft->fflags) == rft->fflags)
1509 kernfs_remove_by_name(kn, rft->name);
1510 }
1511 return ret;
1512}
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name)
1532{
1533 struct iattr iattr = {.ia_valid = ATTR_MODE,};
1534 struct kernfs_node *kn;
1535 int ret = 0;
1536
1537 kn = kernfs_find_and_get_ns(r->kn, name, NULL);
1538 if (!kn)
1539 return -ENOENT;
1540
1541 switch (kernfs_type(kn)) {
1542 case KERNFS_DIR:
1543 iattr.ia_mode = S_IFDIR;
1544 break;
1545 case KERNFS_FILE:
1546 iattr.ia_mode = S_IFREG;
1547 break;
1548 case KERNFS_LINK:
1549 iattr.ia_mode = S_IFLNK;
1550 break;
1551 }
1552
1553 ret = kernfs_setattr(kn, &iattr);
1554 kernfs_put(kn);
1555 return ret;
1556}
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name,
1570 umode_t mask)
1571{
1572 struct iattr iattr = {.ia_valid = ATTR_MODE,};
1573 struct kernfs_node *kn, *parent;
1574 struct rftype *rfts, *rft;
1575 int ret, len;
1576
1577 rfts = res_common_files;
1578 len = ARRAY_SIZE(res_common_files);
1579
1580 for (rft = rfts; rft < rfts + len; rft++) {
1581 if (!strcmp(rft->name, name))
1582 iattr.ia_mode = rft->mode & mask;
1583 }
1584
1585 kn = kernfs_find_and_get_ns(r->kn, name, NULL);
1586 if (!kn)
1587 return -ENOENT;
1588
1589 switch (kernfs_type(kn)) {
1590 case KERNFS_DIR:
1591 parent = kernfs_get_parent(kn);
1592 if (parent) {
1593 iattr.ia_mode |= parent->mode;
1594 kernfs_put(parent);
1595 }
1596 iattr.ia_mode |= S_IFDIR;
1597 break;
1598 case KERNFS_FILE:
1599 iattr.ia_mode |= S_IFREG;
1600 break;
1601 case KERNFS_LINK:
1602 iattr.ia_mode |= S_IFLNK;
1603 break;
1604 }
1605
1606 ret = kernfs_setattr(kn, &iattr);
1607 kernfs_put(kn);
1608 return ret;
1609}
1610
1611static int rdtgroup_mkdir_info_resdir(struct rdt_resource *r, char *name,
1612 unsigned long fflags)
1613{
1614 struct kernfs_node *kn_subdir;
1615 int ret;
1616
1617 kn_subdir = kernfs_create_dir(kn_info, name,
1618 kn_info->mode, r);
1619 if (IS_ERR(kn_subdir))
1620 return PTR_ERR(kn_subdir);
1621
1622 kernfs_get(kn_subdir);
1623 ret = rdtgroup_kn_set_ugid(kn_subdir);
1624 if (ret)
1625 return ret;
1626
1627 ret = rdtgroup_add_files(kn_subdir, fflags);
1628 if (!ret)
1629 kernfs_activate(kn_subdir);
1630
1631 return ret;
1632}
1633
1634static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn)
1635{
1636 struct rdt_resource *r;
1637 unsigned long fflags;
1638 char name[32];
1639 int ret;
1640
1641
1642 kn_info = kernfs_create_dir(parent_kn, "info", parent_kn->mode, NULL);
1643 if (IS_ERR(kn_info))
1644 return PTR_ERR(kn_info);
1645 kernfs_get(kn_info);
1646
1647 ret = rdtgroup_add_files(kn_info, RF_TOP_INFO);
1648 if (ret)
1649 goto out_destroy;
1650
1651 for_each_alloc_enabled_rdt_resource(r) {
1652 fflags = r->fflags | RF_CTRL_INFO;
1653 ret = rdtgroup_mkdir_info_resdir(r, r->name, fflags);
1654 if (ret)
1655 goto out_destroy;
1656 }
1657
1658 for_each_mon_enabled_rdt_resource(r) {
1659 fflags = r->fflags | RF_MON_INFO;
1660 sprintf(name, "%s_MON", r->name);
1661 ret = rdtgroup_mkdir_info_resdir(r, name, fflags);
1662 if (ret)
1663 goto out_destroy;
1664 }
1665
1666
1667
1668
1669
1670 kernfs_get(kn_info);
1671
1672 ret = rdtgroup_kn_set_ugid(kn_info);
1673 if (ret)
1674 goto out_destroy;
1675
1676 kernfs_activate(kn_info);
1677
1678 return 0;
1679
1680out_destroy:
1681 kernfs_remove(kn_info);
1682 return ret;
1683}
1684
1685static int
1686mongroup_create_dir(struct kernfs_node *parent_kn, struct rdtgroup *prgrp,
1687 char *name, struct kernfs_node **dest_kn)
1688{
1689 struct kernfs_node *kn;
1690 int ret;
1691
1692
1693 kn = kernfs_create_dir(parent_kn, name, parent_kn->mode, prgrp);
1694 if (IS_ERR(kn))
1695 return PTR_ERR(kn);
1696
1697 if (dest_kn)
1698 *dest_kn = kn;
1699
1700
1701
1702
1703
1704 kernfs_get(kn);
1705
1706 ret = rdtgroup_kn_set_ugid(kn);
1707 if (ret)
1708 goto out_destroy;
1709
1710 kernfs_activate(kn);
1711
1712 return 0;
1713
1714out_destroy:
1715 kernfs_remove(kn);
1716 return ret;
1717}
1718
1719static void l3_qos_cfg_update(void *arg)
1720{
1721 bool *enable = arg;
1722
1723 wrmsrl(MSR_IA32_L3_QOS_CFG, *enable ? L3_QOS_CDP_ENABLE : 0ULL);
1724}
1725
1726static void l2_qos_cfg_update(void *arg)
1727{
1728 bool *enable = arg;
1729
1730 wrmsrl(MSR_IA32_L2_QOS_CFG, *enable ? L2_QOS_CDP_ENABLE : 0ULL);
1731}
1732
1733static inline bool is_mba_linear(void)
1734{
1735 return rdt_resources_all[RDT_RESOURCE_MBA].membw.delay_linear;
1736}
1737
1738static int set_cache_qos_cfg(int level, bool enable)
1739{
1740 void (*update)(void *arg);
1741 struct rdt_resource *r_l;
1742 cpumask_var_t cpu_mask;
1743 struct rdt_domain *d;
1744 int cpu;
1745
1746 if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
1747 return -ENOMEM;
1748
1749 if (level == RDT_RESOURCE_L3)
1750 update = l3_qos_cfg_update;
1751 else if (level == RDT_RESOURCE_L2)
1752 update = l2_qos_cfg_update;
1753 else
1754 return -EINVAL;
1755
1756 r_l = &rdt_resources_all[level];
1757 list_for_each_entry(d, &r_l->domains, list) {
1758
1759 cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask);
1760 }
1761 cpu = get_cpu();
1762
1763 if (cpumask_test_cpu(cpu, cpu_mask))
1764 update(&enable);
1765
1766 smp_call_function_many(cpu_mask, update, &enable, 1);
1767 put_cpu();
1768
1769 free_cpumask_var(cpu_mask);
1770
1771 return 0;
1772}
1773
1774
1775
1776
1777
1778
1779
1780static int set_mba_sc(bool mba_sc)
1781{
1782 struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_MBA];
1783 struct rdt_domain *d;
1784
1785 if (!is_mbm_enabled() || !is_mba_linear() ||
1786 mba_sc == is_mba_sc(r))
1787 return -EINVAL;
1788
1789 r->membw.mba_sc = mba_sc;
1790 list_for_each_entry(d, &r->domains, list)
1791 setup_default_ctrlval(r, d->ctrl_val, d->mbps_val);
1792
1793 return 0;
1794}
1795
1796static int cdp_enable(int level, int data_type, int code_type)
1797{
1798 struct rdt_resource *r_ldata = &rdt_resources_all[data_type];
1799 struct rdt_resource *r_lcode = &rdt_resources_all[code_type];
1800 struct rdt_resource *r_l = &rdt_resources_all[level];
1801 int ret;
1802
1803 if (!r_l->alloc_capable || !r_ldata->alloc_capable ||
1804 !r_lcode->alloc_capable)
1805 return -EINVAL;
1806
1807 ret = set_cache_qos_cfg(level, true);
1808 if (!ret) {
1809 r_l->alloc_enabled = false;
1810 r_ldata->alloc_enabled = true;
1811 r_lcode->alloc_enabled = true;
1812 }
1813 return ret;
1814}
1815
1816static int cdpl3_enable(void)
1817{
1818 return cdp_enable(RDT_RESOURCE_L3, RDT_RESOURCE_L3DATA,
1819 RDT_RESOURCE_L3CODE);
1820}
1821
1822static int cdpl2_enable(void)
1823{
1824 return cdp_enable(RDT_RESOURCE_L2, RDT_RESOURCE_L2DATA,
1825 RDT_RESOURCE_L2CODE);
1826}
1827
1828static void cdp_disable(int level, int data_type, int code_type)
1829{
1830 struct rdt_resource *r = &rdt_resources_all[level];
1831
1832 r->alloc_enabled = r->alloc_capable;
1833
1834 if (rdt_resources_all[data_type].alloc_enabled) {
1835 rdt_resources_all[data_type].alloc_enabled = false;
1836 rdt_resources_all[code_type].alloc_enabled = false;
1837 set_cache_qos_cfg(level, false);
1838 }
1839}
1840
1841static void cdpl3_disable(void)
1842{
1843 cdp_disable(RDT_RESOURCE_L3, RDT_RESOURCE_L3DATA, RDT_RESOURCE_L3CODE);
1844}
1845
1846static void cdpl2_disable(void)
1847{
1848 cdp_disable(RDT_RESOURCE_L2, RDT_RESOURCE_L2DATA, RDT_RESOURCE_L2CODE);
1849}
1850
1851static void cdp_disable_all(void)
1852{
1853 if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled)
1854 cdpl3_disable();
1855 if (rdt_resources_all[RDT_RESOURCE_L2DATA].alloc_enabled)
1856 cdpl2_disable();
1857}
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867static struct rdtgroup *kernfs_to_rdtgroup(struct kernfs_node *kn)
1868{
1869 if (kernfs_type(kn) == KERNFS_DIR) {
1870
1871
1872
1873
1874
1875
1876 if (kn == kn_info || kn->parent == kn_info)
1877 return NULL;
1878 else
1879 return kn->priv;
1880 } else {
1881 return kn->parent->priv;
1882 }
1883}
1884
1885struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn)
1886{
1887 struct rdtgroup *rdtgrp = kernfs_to_rdtgroup(kn);
1888
1889 if (!rdtgrp)
1890 return NULL;
1891
1892 atomic_inc(&rdtgrp->waitcount);
1893 kernfs_break_active_protection(kn);
1894
1895 mutex_lock(&rdtgroup_mutex);
1896
1897
1898 if (rdtgrp->flags & RDT_DELETED)
1899 return NULL;
1900
1901 return rdtgrp;
1902}
1903
1904void rdtgroup_kn_unlock(struct kernfs_node *kn)
1905{
1906 struct rdtgroup *rdtgrp = kernfs_to_rdtgroup(kn);
1907
1908 if (!rdtgrp)
1909 return;
1910
1911 mutex_unlock(&rdtgroup_mutex);
1912
1913 if (atomic_dec_and_test(&rdtgrp->waitcount) &&
1914 (rdtgrp->flags & RDT_DELETED)) {
1915 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
1916 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)
1917 rdtgroup_pseudo_lock_remove(rdtgrp);
1918 kernfs_unbreak_active_protection(kn);
1919 kernfs_put(rdtgrp->kn);
1920 kfree(rdtgrp);
1921 } else {
1922 kernfs_unbreak_active_protection(kn);
1923 }
1924}
1925
1926static int mkdir_mondata_all(struct kernfs_node *parent_kn,
1927 struct rdtgroup *prgrp,
1928 struct kernfs_node **mon_data_kn);
1929
1930static int rdt_enable_ctx(struct rdt_fs_context *ctx)
1931{
1932 int ret = 0;
1933
1934 if (ctx->enable_cdpl2)
1935 ret = cdpl2_enable();
1936
1937 if (!ret && ctx->enable_cdpl3)
1938 ret = cdpl3_enable();
1939
1940 if (!ret && ctx->enable_mba_mbps)
1941 ret = set_mba_sc(true);
1942
1943 return ret;
1944}
1945
1946static int rdt_get_tree(struct fs_context *fc)
1947{
1948 struct rdt_fs_context *ctx = rdt_fc2context(fc);
1949 struct rdt_domain *dom;
1950 struct rdt_resource *r;
1951 int ret;
1952
1953 cpus_read_lock();
1954 mutex_lock(&rdtgroup_mutex);
1955
1956
1957
1958 if (static_branch_unlikely(&rdt_enable_key)) {
1959 ret = -EBUSY;
1960 goto out;
1961 }
1962
1963 ret = rdt_enable_ctx(ctx);
1964 if (ret < 0)
1965 goto out_cdp;
1966
1967 closid_init();
1968
1969 ret = rdtgroup_create_info_dir(rdtgroup_default.kn);
1970 if (ret < 0)
1971 goto out_mba;
1972
1973 if (rdt_mon_capable) {
1974 ret = mongroup_create_dir(rdtgroup_default.kn,
1975 NULL, "mon_groups",
1976 &kn_mongrp);
1977 if (ret < 0)
1978 goto out_info;
1979 kernfs_get(kn_mongrp);
1980
1981 ret = mkdir_mondata_all(rdtgroup_default.kn,
1982 &rdtgroup_default, &kn_mondata);
1983 if (ret < 0)
1984 goto out_mongrp;
1985 kernfs_get(kn_mondata);
1986 rdtgroup_default.mon.mon_data_kn = kn_mondata;
1987 }
1988
1989 ret = rdt_pseudo_lock_init();
1990 if (ret)
1991 goto out_mondata;
1992
1993 ret = kernfs_get_tree(fc);
1994 if (ret < 0)
1995 goto out_psl;
1996
1997 if (rdt_alloc_capable)
1998 static_branch_enable_cpuslocked(&rdt_alloc_enable_key);
1999 if (rdt_mon_capable)
2000 static_branch_enable_cpuslocked(&rdt_mon_enable_key);
2001
2002 if (rdt_alloc_capable || rdt_mon_capable)
2003 static_branch_enable_cpuslocked(&rdt_enable_key);
2004
2005 if (is_mbm_enabled()) {
2006 r = &rdt_resources_all[RDT_RESOURCE_L3];
2007 list_for_each_entry(dom, &r->domains, list)
2008 mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL);
2009 }
2010
2011 goto out;
2012
2013out_psl:
2014 rdt_pseudo_lock_release();
2015out_mondata:
2016 if (rdt_mon_capable)
2017 kernfs_remove(kn_mondata);
2018out_mongrp:
2019 if (rdt_mon_capable)
2020 kernfs_remove(kn_mongrp);
2021out_info:
2022 kernfs_remove(kn_info);
2023out_mba:
2024 if (ctx->enable_mba_mbps)
2025 set_mba_sc(false);
2026out_cdp:
2027 cdp_disable_all();
2028out:
2029 rdt_last_cmd_clear();
2030 mutex_unlock(&rdtgroup_mutex);
2031 cpus_read_unlock();
2032 return ret;
2033}
2034
2035enum rdt_param {
2036 Opt_cdp,
2037 Opt_cdpl2,
2038 Opt_mba_mbps,
2039 nr__rdt_params
2040};
2041
2042static const struct fs_parameter_spec rdt_param_specs[] = {
2043 fsparam_flag("cdp", Opt_cdp),
2044 fsparam_flag("cdpl2", Opt_cdpl2),
2045 fsparam_flag("mba_MBps", Opt_mba_mbps),
2046 {}
2047};
2048
2049static const struct fs_parameter_description rdt_fs_parameters = {
2050 .name = "rdt",
2051 .specs = rdt_param_specs,
2052};
2053
2054static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param)
2055{
2056 struct rdt_fs_context *ctx = rdt_fc2context(fc);
2057 struct fs_parse_result result;
2058 int opt;
2059
2060 opt = fs_parse(fc, &rdt_fs_parameters, param, &result);
2061 if (opt < 0)
2062 return opt;
2063
2064 switch (opt) {
2065 case Opt_cdp:
2066 ctx->enable_cdpl3 = true;
2067 return 0;
2068 case Opt_cdpl2:
2069 ctx->enable_cdpl2 = true;
2070 return 0;
2071 case Opt_mba_mbps:
2072 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2073 return -EINVAL;
2074 ctx->enable_mba_mbps = true;
2075 return 0;
2076 }
2077
2078 return -EINVAL;
2079}
2080
2081static void rdt_fs_context_free(struct fs_context *fc)
2082{
2083 struct rdt_fs_context *ctx = rdt_fc2context(fc);
2084
2085 kernfs_free_fs_context(fc);
2086 kfree(ctx);
2087}
2088
2089static const struct fs_context_operations rdt_fs_context_ops = {
2090 .free = rdt_fs_context_free,
2091 .parse_param = rdt_parse_param,
2092 .get_tree = rdt_get_tree,
2093};
2094
2095static int rdt_init_fs_context(struct fs_context *fc)
2096{
2097 struct rdt_fs_context *ctx;
2098
2099 ctx = kzalloc(sizeof(struct rdt_fs_context), GFP_KERNEL);
2100 if (!ctx)
2101 return -ENOMEM;
2102
2103 ctx->kfc.root = rdt_root;
2104 ctx->kfc.magic = RDTGROUP_SUPER_MAGIC;
2105 fc->fs_private = &ctx->kfc;
2106 fc->ops = &rdt_fs_context_ops;
2107 put_user_ns(fc->user_ns);
2108 fc->user_ns = get_user_ns(&init_user_ns);
2109 fc->global = true;
2110 return 0;
2111}
2112
2113static int reset_all_ctrls(struct rdt_resource *r)
2114{
2115 struct msr_param msr_param;
2116 cpumask_var_t cpu_mask;
2117 struct rdt_domain *d;
2118 int i, cpu;
2119
2120 if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
2121 return -ENOMEM;
2122
2123 msr_param.res = r;
2124 msr_param.low = 0;
2125 msr_param.high = r->num_closid;
2126
2127
2128
2129
2130
2131
2132 list_for_each_entry(d, &r->domains, list) {
2133 cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask);
2134
2135 for (i = 0; i < r->num_closid; i++)
2136 d->ctrl_val[i] = r->default_ctrl;
2137 }
2138 cpu = get_cpu();
2139
2140 if (cpumask_test_cpu(cpu, cpu_mask))
2141 rdt_ctrl_update(&msr_param);
2142
2143 smp_call_function_many(cpu_mask, rdt_ctrl_update, &msr_param, 1);
2144 put_cpu();
2145
2146 free_cpumask_var(cpu_mask);
2147
2148 return 0;
2149}
2150
2151static bool is_closid_match(struct task_struct *t, struct rdtgroup *r)
2152{
2153 return (rdt_alloc_capable &&
2154 (r->type == RDTCTRL_GROUP) && (t->closid == r->closid));
2155}
2156
2157static bool is_rmid_match(struct task_struct *t, struct rdtgroup *r)
2158{
2159 return (rdt_mon_capable &&
2160 (r->type == RDTMON_GROUP) && (t->rmid == r->mon.rmid));
2161}
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171static void rdt_move_group_tasks(struct rdtgroup *from, struct rdtgroup *to,
2172 struct cpumask *mask)
2173{
2174 struct task_struct *p, *t;
2175
2176 read_lock(&tasklist_lock);
2177 for_each_process_thread(p, t) {
2178 if (!from || is_closid_match(t, from) ||
2179 is_rmid_match(t, from)) {
2180 t->closid = to->closid;
2181 t->rmid = to->mon.rmid;
2182
2183#ifdef CONFIG_SMP
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193 if (mask && t->on_cpu)
2194 cpumask_set_cpu(task_cpu(t), mask);
2195#endif
2196 }
2197 }
2198 read_unlock(&tasklist_lock);
2199}
2200
2201static void free_all_child_rdtgrp(struct rdtgroup *rdtgrp)
2202{
2203 struct rdtgroup *sentry, *stmp;
2204 struct list_head *head;
2205
2206 head = &rdtgrp->mon.crdtgrp_list;
2207 list_for_each_entry_safe(sentry, stmp, head, mon.crdtgrp_list) {
2208 free_rmid(sentry->mon.rmid);
2209 list_del(&sentry->mon.crdtgrp_list);
2210 kfree(sentry);
2211 }
2212}
2213
2214
2215
2216
2217static void rmdir_all_sub(void)
2218{
2219 struct rdtgroup *rdtgrp, *tmp;
2220
2221
2222 rdt_move_group_tasks(NULL, &rdtgroup_default, NULL);
2223
2224 list_for_each_entry_safe(rdtgrp, tmp, &rdt_all_groups, rdtgroup_list) {
2225
2226 free_all_child_rdtgrp(rdtgrp);
2227
2228
2229 if (rdtgrp == &rdtgroup_default)
2230 continue;
2231
2232 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
2233 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)
2234 rdtgroup_pseudo_lock_remove(rdtgrp);
2235
2236
2237
2238
2239
2240
2241 cpumask_or(&rdtgroup_default.cpu_mask,
2242 &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask);
2243
2244 free_rmid(rdtgrp->mon.rmid);
2245
2246 kernfs_remove(rdtgrp->kn);
2247 list_del(&rdtgrp->rdtgroup_list);
2248 kfree(rdtgrp);
2249 }
2250
2251 update_closid_rmid(cpu_online_mask, &rdtgroup_default);
2252
2253 kernfs_remove(kn_info);
2254 kernfs_remove(kn_mongrp);
2255 kernfs_remove(kn_mondata);
2256}
2257
2258static void rdt_kill_sb(struct super_block *sb)
2259{
2260 struct rdt_resource *r;
2261
2262 cpus_read_lock();
2263 mutex_lock(&rdtgroup_mutex);
2264
2265 set_mba_sc(false);
2266
2267
2268 for_each_alloc_enabled_rdt_resource(r)
2269 reset_all_ctrls(r);
2270 cdp_disable_all();
2271 rmdir_all_sub();
2272 rdt_pseudo_lock_release();
2273 rdtgroup_default.mode = RDT_MODE_SHAREABLE;
2274 static_branch_disable_cpuslocked(&rdt_alloc_enable_key);
2275 static_branch_disable_cpuslocked(&rdt_mon_enable_key);
2276 static_branch_disable_cpuslocked(&rdt_enable_key);
2277 kernfs_kill_sb(sb);
2278 mutex_unlock(&rdtgroup_mutex);
2279 cpus_read_unlock();
2280}
2281
2282static struct file_system_type rdt_fs_type = {
2283 .name = "resctrl",
2284 .init_fs_context = rdt_init_fs_context,
2285 .parameters = &rdt_fs_parameters,
2286 .kill_sb = rdt_kill_sb,
2287};
2288
2289static int mon_addfile(struct kernfs_node *parent_kn, const char *name,
2290 void *priv)
2291{
2292 struct kernfs_node *kn;
2293 int ret = 0;
2294
2295 kn = __kernfs_create_file(parent_kn, name, 0444,
2296 GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0,
2297 &kf_mondata_ops, priv, NULL, NULL);
2298 if (IS_ERR(kn))
2299 return PTR_ERR(kn);
2300
2301 ret = rdtgroup_kn_set_ugid(kn);
2302 if (ret) {
2303 kernfs_remove(kn);
2304 return ret;
2305 }
2306
2307 return ret;
2308}
2309
2310
2311
2312
2313
2314void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, unsigned int dom_id)
2315{
2316 struct rdtgroup *prgrp, *crgrp;
2317 char name[32];
2318
2319 if (!r->mon_enabled)
2320 return;
2321
2322 list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
2323 sprintf(name, "mon_%s_%02d", r->name, dom_id);
2324 kernfs_remove_by_name(prgrp->mon.mon_data_kn, name);
2325
2326 list_for_each_entry(crgrp, &prgrp->mon.crdtgrp_list, mon.crdtgrp_list)
2327 kernfs_remove_by_name(crgrp->mon.mon_data_kn, name);
2328 }
2329}
2330
2331static int mkdir_mondata_subdir(struct kernfs_node *parent_kn,
2332 struct rdt_domain *d,
2333 struct rdt_resource *r, struct rdtgroup *prgrp)
2334{
2335 union mon_data_bits priv;
2336 struct kernfs_node *kn;
2337 struct mon_evt *mevt;
2338 struct rmid_read rr;
2339 char name[32];
2340 int ret;
2341
2342 sprintf(name, "mon_%s_%02d", r->name, d->id);
2343
2344 kn = kernfs_create_dir(parent_kn, name, parent_kn->mode, prgrp);
2345 if (IS_ERR(kn))
2346 return PTR_ERR(kn);
2347
2348
2349
2350
2351
2352 kernfs_get(kn);
2353 ret = rdtgroup_kn_set_ugid(kn);
2354 if (ret)
2355 goto out_destroy;
2356
2357 if (WARN_ON(list_empty(&r->evt_list))) {
2358 ret = -EPERM;
2359 goto out_destroy;
2360 }
2361
2362 priv.u.rid = r->rid;
2363 priv.u.domid = d->id;
2364 list_for_each_entry(mevt, &r->evt_list, list) {
2365 priv.u.evtid = mevt->evtid;
2366 ret = mon_addfile(kn, mevt->name, priv.priv);
2367 if (ret)
2368 goto out_destroy;
2369
2370 if (is_mbm_event(mevt->evtid))
2371 mon_event_read(&rr, d, prgrp, mevt->evtid, true);
2372 }
2373 kernfs_activate(kn);
2374 return 0;
2375
2376out_destroy:
2377 kernfs_remove(kn);
2378 return ret;
2379}
2380
2381
2382
2383
2384
2385void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
2386 struct rdt_domain *d)
2387{
2388 struct kernfs_node *parent_kn;
2389 struct rdtgroup *prgrp, *crgrp;
2390 struct list_head *head;
2391
2392 if (!r->mon_enabled)
2393 return;
2394
2395 list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
2396 parent_kn = prgrp->mon.mon_data_kn;
2397 mkdir_mondata_subdir(parent_kn, d, r, prgrp);
2398
2399 head = &prgrp->mon.crdtgrp_list;
2400 list_for_each_entry(crgrp, head, mon.crdtgrp_list) {
2401 parent_kn = crgrp->mon.mon_data_kn;
2402 mkdir_mondata_subdir(parent_kn, d, r, crgrp);
2403 }
2404 }
2405}
2406
2407static int mkdir_mondata_subdir_alldom(struct kernfs_node *parent_kn,
2408 struct rdt_resource *r,
2409 struct rdtgroup *prgrp)
2410{
2411 struct rdt_domain *dom;
2412 int ret;
2413
2414 list_for_each_entry(dom, &r->domains, list) {
2415 ret = mkdir_mondata_subdir(parent_kn, dom, r, prgrp);
2416 if (ret)
2417 return ret;
2418 }
2419
2420 return 0;
2421}
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440static int mkdir_mondata_all(struct kernfs_node *parent_kn,
2441 struct rdtgroup *prgrp,
2442 struct kernfs_node **dest_kn)
2443{
2444 struct rdt_resource *r;
2445 struct kernfs_node *kn;
2446 int ret;
2447
2448
2449
2450
2451 ret = mongroup_create_dir(parent_kn, NULL, "mon_data", &kn);
2452 if (ret)
2453 return ret;
2454
2455 if (dest_kn)
2456 *dest_kn = kn;
2457
2458
2459
2460
2461
2462 for_each_mon_enabled_rdt_resource(r) {
2463 ret = mkdir_mondata_subdir_alldom(kn, r, prgrp);
2464 if (ret)
2465 goto out_destroy;
2466 }
2467
2468 return 0;
2469
2470out_destroy:
2471 kernfs_remove(kn);
2472 return ret;
2473}
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490static u32 cbm_ensure_valid(u32 _val, struct rdt_resource *r)
2491{
2492 unsigned int cbm_len = r->cache.cbm_len;
2493 unsigned long first_bit, zero_bit;
2494 unsigned long val = _val;
2495
2496 if (!val)
2497 return 0;
2498
2499 first_bit = find_first_bit(&val, cbm_len);
2500 zero_bit = find_next_zero_bit(&val, cbm_len, first_bit);
2501
2502
2503 bitmap_clear(&val, zero_bit, cbm_len - zero_bit);
2504 return (u32)val;
2505}
2506
2507
2508
2509
2510
2511
2512
2513static int __init_one_rdt_domain(struct rdt_domain *d, struct rdt_resource *r,
2514 u32 closid)
2515{
2516 struct rdt_resource *r_cdp = NULL;
2517 struct rdt_domain *d_cdp = NULL;
2518 u32 used_b = 0, unused_b = 0;
2519 unsigned long tmp_cbm;
2520 enum rdtgrp_mode mode;
2521 u32 peer_ctl, *ctrl;
2522 int i;
2523
2524 rdt_cdp_peer_get(r, d, &r_cdp, &d_cdp);
2525 d->have_new_ctrl = false;
2526 d->new_ctrl = r->cache.shareable_bits;
2527 used_b = r->cache.shareable_bits;
2528 ctrl = d->ctrl_val;
2529 for (i = 0; i < closids_supported(); i++, ctrl++) {
2530 if (closid_allocated(i) && i != closid) {
2531 mode = rdtgroup_mode_by_closid(i);
2532 if (mode == RDT_MODE_PSEUDO_LOCKSETUP)
2533
2534
2535
2536
2537
2538 continue;
2539
2540
2541
2542
2543
2544 if (d_cdp)
2545 peer_ctl = d_cdp->ctrl_val[i];
2546 else
2547 peer_ctl = 0;
2548 used_b |= *ctrl | peer_ctl;
2549 if (mode == RDT_MODE_SHAREABLE)
2550 d->new_ctrl |= *ctrl | peer_ctl;
2551 }
2552 }
2553 if (d->plr && d->plr->cbm > 0)
2554 used_b |= d->plr->cbm;
2555 unused_b = used_b ^ (BIT_MASK(r->cache.cbm_len) - 1);
2556 unused_b &= BIT_MASK(r->cache.cbm_len) - 1;
2557 d->new_ctrl |= unused_b;
2558
2559
2560
2561
2562 d->new_ctrl = cbm_ensure_valid(d->new_ctrl, r);
2563
2564
2565
2566
2567 tmp_cbm = d->new_ctrl;
2568 if (bitmap_weight(&tmp_cbm, r->cache.cbm_len) < r->cache.min_cbm_bits) {
2569 rdt_last_cmd_printf("No space on %s:%d\n", r->name, d->id);
2570 return -ENOSPC;
2571 }
2572 d->have_new_ctrl = true;
2573
2574 return 0;
2575}
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587static int rdtgroup_init_cat(struct rdt_resource *r, u32 closid)
2588{
2589 struct rdt_domain *d;
2590 int ret;
2591
2592 list_for_each_entry(d, &r->domains, list) {
2593 ret = __init_one_rdt_domain(d, r, closid);
2594 if (ret < 0)
2595 return ret;
2596 }
2597
2598 return 0;
2599}
2600
2601
2602static void rdtgroup_init_mba(struct rdt_resource *r)
2603{
2604 struct rdt_domain *d;
2605
2606 list_for_each_entry(d, &r->domains, list) {
2607 d->new_ctrl = is_mba_sc(r) ? MBA_MAX_MBPS : r->default_ctrl;
2608 d->have_new_ctrl = true;
2609 }
2610}
2611
2612
2613static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp)
2614{
2615 struct rdt_resource *r;
2616 int ret;
2617
2618 for_each_alloc_enabled_rdt_resource(r) {
2619 if (r->rid == RDT_RESOURCE_MBA) {
2620 rdtgroup_init_mba(r);
2621 } else {
2622 ret = rdtgroup_init_cat(r, rdtgrp->closid);
2623 if (ret < 0)
2624 return ret;
2625 }
2626
2627 ret = update_domains(r, rdtgrp->closid);
2628 if (ret < 0) {
2629 rdt_last_cmd_puts("Failed to initialize allocations\n");
2630 return ret;
2631 }
2632
2633 }
2634
2635 rdtgrp->mode = RDT_MODE_SHAREABLE;
2636
2637 return 0;
2638}
2639
2640static int mkdir_rdt_prepare(struct kernfs_node *parent_kn,
2641 struct kernfs_node *prgrp_kn,
2642 const char *name, umode_t mode,
2643 enum rdt_group_type rtype, struct rdtgroup **r)
2644{
2645 struct rdtgroup *prdtgrp, *rdtgrp;
2646 struct kernfs_node *kn;
2647 uint files = 0;
2648 int ret;
2649
2650 prdtgrp = rdtgroup_kn_lock_live(prgrp_kn);
2651 rdt_last_cmd_clear();
2652 if (!prdtgrp) {
2653 ret = -ENODEV;
2654 rdt_last_cmd_puts("Directory was removed\n");
2655 goto out_unlock;
2656 }
2657
2658 if (rtype == RDTMON_GROUP &&
2659 (prdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
2660 prdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)) {
2661 ret = -EINVAL;
2662 rdt_last_cmd_puts("Pseudo-locking in progress\n");
2663 goto out_unlock;
2664 }
2665
2666
2667 rdtgrp = kzalloc(sizeof(*rdtgrp), GFP_KERNEL);
2668 if (!rdtgrp) {
2669 ret = -ENOSPC;
2670 rdt_last_cmd_puts("Kernel out of memory\n");
2671 goto out_unlock;
2672 }
2673 *r = rdtgrp;
2674 rdtgrp->mon.parent = prdtgrp;
2675 rdtgrp->type = rtype;
2676 INIT_LIST_HEAD(&rdtgrp->mon.crdtgrp_list);
2677
2678
2679 kn = kernfs_create_dir(parent_kn, name, mode, rdtgrp);
2680 if (IS_ERR(kn)) {
2681 ret = PTR_ERR(kn);
2682 rdt_last_cmd_puts("kernfs create error\n");
2683 goto out_free_rgrp;
2684 }
2685 rdtgrp->kn = kn;
2686
2687
2688
2689
2690
2691
2692
2693 kernfs_get(kn);
2694
2695 ret = rdtgroup_kn_set_ugid(kn);
2696 if (ret) {
2697 rdt_last_cmd_puts("kernfs perm error\n");
2698 goto out_destroy;
2699 }
2700
2701 files = RFTYPE_BASE | BIT(RF_CTRLSHIFT + rtype);
2702 ret = rdtgroup_add_files(kn, files);
2703 if (ret) {
2704 rdt_last_cmd_puts("kernfs fill error\n");
2705 goto out_destroy;
2706 }
2707
2708 if (rdt_mon_capable) {
2709 ret = alloc_rmid();
2710 if (ret < 0) {
2711 rdt_last_cmd_puts("Out of RMIDs\n");
2712 goto out_destroy;
2713 }
2714 rdtgrp->mon.rmid = ret;
2715
2716 ret = mkdir_mondata_all(kn, rdtgrp, &rdtgrp->mon.mon_data_kn);
2717 if (ret) {
2718 rdt_last_cmd_puts("kernfs subdir error\n");
2719 goto out_idfree;
2720 }
2721 }
2722 kernfs_activate(kn);
2723
2724
2725
2726
2727 return 0;
2728
2729out_idfree:
2730 free_rmid(rdtgrp->mon.rmid);
2731out_destroy:
2732 kernfs_remove(rdtgrp->kn);
2733out_free_rgrp:
2734 kfree(rdtgrp);
2735out_unlock:
2736 rdtgroup_kn_unlock(prgrp_kn);
2737 return ret;
2738}
2739
2740static void mkdir_rdt_prepare_clean(struct rdtgroup *rgrp)
2741{
2742 kernfs_remove(rgrp->kn);
2743 free_rmid(rgrp->mon.rmid);
2744 kfree(rgrp);
2745}
2746
2747
2748
2749
2750
2751
2752static int rdtgroup_mkdir_mon(struct kernfs_node *parent_kn,
2753 struct kernfs_node *prgrp_kn,
2754 const char *name,
2755 umode_t mode)
2756{
2757 struct rdtgroup *rdtgrp, *prgrp;
2758 int ret;
2759
2760 ret = mkdir_rdt_prepare(parent_kn, prgrp_kn, name, mode, RDTMON_GROUP,
2761 &rdtgrp);
2762 if (ret)
2763 return ret;
2764
2765 prgrp = rdtgrp->mon.parent;
2766 rdtgrp->closid = prgrp->closid;
2767
2768
2769
2770
2771
2772 list_add_tail(&rdtgrp->mon.crdtgrp_list, &prgrp->mon.crdtgrp_list);
2773
2774 rdtgroup_kn_unlock(prgrp_kn);
2775 return ret;
2776}
2777
2778
2779
2780
2781
2782static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn,
2783 struct kernfs_node *prgrp_kn,
2784 const char *name, umode_t mode)
2785{
2786 struct rdtgroup *rdtgrp;
2787 struct kernfs_node *kn;
2788 u32 closid;
2789 int ret;
2790
2791 ret = mkdir_rdt_prepare(parent_kn, prgrp_kn, name, mode, RDTCTRL_GROUP,
2792 &rdtgrp);
2793 if (ret)
2794 return ret;
2795
2796 kn = rdtgrp->kn;
2797 ret = closid_alloc();
2798 if (ret < 0) {
2799 rdt_last_cmd_puts("Out of CLOSIDs\n");
2800 goto out_common_fail;
2801 }
2802 closid = ret;
2803 ret = 0;
2804
2805 rdtgrp->closid = closid;
2806 ret = rdtgroup_init_alloc(rdtgrp);
2807 if (ret < 0)
2808 goto out_id_free;
2809
2810 list_add(&rdtgrp->rdtgroup_list, &rdt_all_groups);
2811
2812 if (rdt_mon_capable) {
2813
2814
2815
2816
2817 ret = mongroup_create_dir(kn, NULL, "mon_groups", NULL);
2818 if (ret) {
2819 rdt_last_cmd_puts("kernfs subdir error\n");
2820 goto out_del_list;
2821 }
2822 }
2823
2824 goto out_unlock;
2825
2826out_del_list:
2827 list_del(&rdtgrp->rdtgroup_list);
2828out_id_free:
2829 closid_free(closid);
2830out_common_fail:
2831 mkdir_rdt_prepare_clean(rdtgrp);
2832out_unlock:
2833 rdtgroup_kn_unlock(prgrp_kn);
2834 return ret;
2835}
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847static bool is_mon_groups(struct kernfs_node *kn, const char *name)
2848{
2849 return (!strcmp(kn->name, "mon_groups") &&
2850 strcmp(name, "mon_groups"));
2851}
2852
2853static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
2854 umode_t mode)
2855{
2856
2857 if (strchr(name, '\n'))
2858 return -EINVAL;
2859
2860
2861
2862
2863
2864
2865 if (rdt_alloc_capable && parent_kn == rdtgroup_default.kn)
2866 return rdtgroup_mkdir_ctrl_mon(parent_kn, parent_kn, name, mode);
2867
2868
2869
2870
2871
2872 if (rdt_mon_capable && is_mon_groups(parent_kn, name))
2873 return rdtgroup_mkdir_mon(parent_kn, parent_kn->parent, name, mode);
2874
2875 return -EPERM;
2876}
2877
2878static int rdtgroup_rmdir_mon(struct kernfs_node *kn, struct rdtgroup *rdtgrp,
2879 cpumask_var_t tmpmask)
2880{
2881 struct rdtgroup *prdtgrp = rdtgrp->mon.parent;
2882 int cpu;
2883
2884
2885 rdt_move_group_tasks(rdtgrp, prdtgrp, tmpmask);
2886
2887
2888 for_each_cpu(cpu, &rdtgrp->cpu_mask)
2889 per_cpu(pqr_state.default_rmid, cpu) = prdtgrp->mon.rmid;
2890
2891
2892
2893
2894 cpumask_or(tmpmask, tmpmask, &rdtgrp->cpu_mask);
2895 update_closid_rmid(tmpmask, NULL);
2896
2897 rdtgrp->flags = RDT_DELETED;
2898 free_rmid(rdtgrp->mon.rmid);
2899
2900
2901
2902
2903 WARN_ON(list_empty(&prdtgrp->mon.crdtgrp_list));
2904 list_del(&rdtgrp->mon.crdtgrp_list);
2905
2906
2907
2908
2909
2910 kernfs_get(kn);
2911 kernfs_remove(rdtgrp->kn);
2912
2913 return 0;
2914}
2915
2916static int rdtgroup_ctrl_remove(struct kernfs_node *kn,
2917 struct rdtgroup *rdtgrp)
2918{
2919 rdtgrp->flags = RDT_DELETED;
2920 list_del(&rdtgrp->rdtgroup_list);
2921
2922
2923
2924
2925
2926 kernfs_get(kn);
2927 kernfs_remove(rdtgrp->kn);
2928 return 0;
2929}
2930
2931static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp,
2932 cpumask_var_t tmpmask)
2933{
2934 int cpu;
2935
2936
2937 rdt_move_group_tasks(rdtgrp, &rdtgroup_default, tmpmask);
2938
2939
2940 cpumask_or(&rdtgroup_default.cpu_mask,
2941 &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask);
2942
2943
2944 for_each_cpu(cpu, &rdtgrp->cpu_mask) {
2945 per_cpu(pqr_state.default_closid, cpu) = rdtgroup_default.closid;
2946 per_cpu(pqr_state.default_rmid, cpu) = rdtgroup_default.mon.rmid;
2947 }
2948
2949
2950
2951
2952
2953 cpumask_or(tmpmask, tmpmask, &rdtgrp->cpu_mask);
2954 update_closid_rmid(tmpmask, NULL);
2955
2956 closid_free(rdtgrp->closid);
2957 free_rmid(rdtgrp->mon.rmid);
2958
2959
2960
2961
2962 free_all_child_rdtgrp(rdtgrp);
2963
2964 rdtgroup_ctrl_remove(kn, rdtgrp);
2965
2966 return 0;
2967}
2968
2969static int rdtgroup_rmdir(struct kernfs_node *kn)
2970{
2971 struct kernfs_node *parent_kn = kn->parent;
2972 struct rdtgroup *rdtgrp;
2973 cpumask_var_t tmpmask;
2974 int ret = 0;
2975
2976 if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
2977 return -ENOMEM;
2978
2979 rdtgrp = rdtgroup_kn_lock_live(kn);
2980 if (!rdtgrp) {
2981 ret = -EPERM;
2982 goto out;
2983 }
2984
2985
2986
2987
2988
2989
2990
2991
2992 if (rdtgrp->type == RDTCTRL_GROUP && parent_kn == rdtgroup_default.kn) {
2993 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
2994 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
2995 ret = rdtgroup_ctrl_remove(kn, rdtgrp);
2996 } else {
2997 ret = rdtgroup_rmdir_ctrl(kn, rdtgrp, tmpmask);
2998 }
2999 } else if (rdtgrp->type == RDTMON_GROUP &&
3000 is_mon_groups(parent_kn, kn->name)) {
3001 ret = rdtgroup_rmdir_mon(kn, rdtgrp, tmpmask);
3002 } else {
3003 ret = -EPERM;
3004 }
3005
3006out:
3007 rdtgroup_kn_unlock(kn);
3008 free_cpumask_var(tmpmask);
3009 return ret;
3010}
3011
3012static int rdtgroup_show_options(struct seq_file *seq, struct kernfs_root *kf)
3013{
3014 if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled)
3015 seq_puts(seq, ",cdp");
3016
3017 if (rdt_resources_all[RDT_RESOURCE_L2DATA].alloc_enabled)
3018 seq_puts(seq, ",cdpl2");
3019
3020 if (is_mba_sc(&rdt_resources_all[RDT_RESOURCE_MBA]))
3021 seq_puts(seq, ",mba_MBps");
3022
3023 return 0;
3024}
3025
3026static struct kernfs_syscall_ops rdtgroup_kf_syscall_ops = {
3027 .mkdir = rdtgroup_mkdir,
3028 .rmdir = rdtgroup_rmdir,
3029 .show_options = rdtgroup_show_options,
3030};
3031
3032static int __init rdtgroup_setup_root(void)
3033{
3034 int ret;
3035
3036 rdt_root = kernfs_create_root(&rdtgroup_kf_syscall_ops,
3037 KERNFS_ROOT_CREATE_DEACTIVATED |
3038 KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK,
3039 &rdtgroup_default);
3040 if (IS_ERR(rdt_root))
3041 return PTR_ERR(rdt_root);
3042
3043 mutex_lock(&rdtgroup_mutex);
3044
3045 rdtgroup_default.closid = 0;
3046 rdtgroup_default.mon.rmid = 0;
3047 rdtgroup_default.type = RDTCTRL_GROUP;
3048 INIT_LIST_HEAD(&rdtgroup_default.mon.crdtgrp_list);
3049
3050 list_add(&rdtgroup_default.rdtgroup_list, &rdt_all_groups);
3051
3052 ret = rdtgroup_add_files(rdt_root->kn, RF_CTRL_BASE);
3053 if (ret) {
3054 kernfs_destroy_root(rdt_root);
3055 goto out;
3056 }
3057
3058 rdtgroup_default.kn = rdt_root->kn;
3059 kernfs_activate(rdtgroup_default.kn);
3060
3061out:
3062 mutex_unlock(&rdtgroup_mutex);
3063
3064 return ret;
3065}
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075int __init rdtgroup_init(void)
3076{
3077 int ret = 0;
3078
3079 seq_buf_init(&last_cmd_status, last_cmd_status_buf,
3080 sizeof(last_cmd_status_buf));
3081
3082 ret = rdtgroup_setup_root();
3083 if (ret)
3084 return ret;
3085
3086 ret = sysfs_create_mount_point(fs_kobj, "resctrl");
3087 if (ret)
3088 goto cleanup_root;
3089
3090 ret = register_filesystem(&rdt_fs_type);
3091 if (ret)
3092 goto cleanup_mountpoint;
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115 debugfs_resctrl = debugfs_create_dir("resctrl", NULL);
3116
3117 return 0;
3118
3119cleanup_mountpoint:
3120 sysfs_remove_mount_point(fs_kobj, "resctrl");
3121cleanup_root:
3122 kernfs_destroy_root(rdt_root);
3123
3124 return ret;
3125}
3126
3127void __exit rdtgroup_exit(void)
3128{
3129 debugfs_remove_recursive(debugfs_resctrl);
3130 unregister_filesystem(&rdt_fs_type);
3131 sysfs_remove_mount_point(fs_kobj, "resctrl");
3132 kernfs_destroy_root(rdt_root);
3133}
3134