1
2
3
4
5
6
7
8
9
10#include <linux/init.h>
11#include <linux/slab.h>
12#include <linux/device.h>
13#include <linux/compiler.h>
14#include <linux/cpu.h>
15#include <linux/sched.h>
16#include <linux/pci.h>
17
18#include <asm/processor.h>
19#include <linux/smp.h>
20#include <asm/amd_nb.h>
21#include <asm/smp.h>
22
23#define LVL_1_INST 1
24#define LVL_1_DATA 2
25#define LVL_2 3
26#define LVL_3 4
27#define LVL_TRACE 5
28
29struct _cache_table {
30 unsigned char descriptor;
31 char cache_type;
32 short size;
33};
34
35#define MB(x) ((x) * 1024)
36
37
38
39
40static const struct _cache_table __cpuinitconst cache_table[] =
41{
42 { 0x06, LVL_1_INST, 8 },
43 { 0x08, LVL_1_INST, 16 },
44 { 0x09, LVL_1_INST, 32 },
45 { 0x0a, LVL_1_DATA, 8 },
46 { 0x0c, LVL_1_DATA, 16 },
47 { 0x0d, LVL_1_DATA, 16 },
48 { 0x0e, LVL_1_DATA, 24 },
49 { 0x21, LVL_2, 256 },
50 { 0x22, LVL_3, 512 },
51 { 0x23, LVL_3, MB(1) },
52 { 0x25, LVL_3, MB(2) },
53 { 0x29, LVL_3, MB(4) },
54 { 0x2c, LVL_1_DATA, 32 },
55 { 0x30, LVL_1_INST, 32 },
56 { 0x39, LVL_2, 128 },
57 { 0x3a, LVL_2, 192 },
58 { 0x3b, LVL_2, 128 },
59 { 0x3c, LVL_2, 256 },
60 { 0x3d, LVL_2, 384 },
61 { 0x3e, LVL_2, 512 },
62 { 0x3f, LVL_2, 256 },
63 { 0x41, LVL_2, 128 },
64 { 0x42, LVL_2, 256 },
65 { 0x43, LVL_2, 512 },
66 { 0x44, LVL_2, MB(1) },
67 { 0x45, LVL_2, MB(2) },
68 { 0x46, LVL_3, MB(4) },
69 { 0x47, LVL_3, MB(8) },
70 { 0x48, LVL_2, MB(3) },
71 { 0x49, LVL_3, MB(4) },
72 { 0x4a, LVL_3, MB(6) },
73 { 0x4b, LVL_3, MB(8) },
74 { 0x4c, LVL_3, MB(12) },
75 { 0x4d, LVL_3, MB(16) },
76 { 0x4e, LVL_2, MB(6) },
77 { 0x60, LVL_1_DATA, 16 },
78 { 0x66, LVL_1_DATA, 8 },
79 { 0x67, LVL_1_DATA, 16 },
80 { 0x68, LVL_1_DATA, 32 },
81 { 0x70, LVL_TRACE, 12 },
82 { 0x71, LVL_TRACE, 16 },
83 { 0x72, LVL_TRACE, 32 },
84 { 0x73, LVL_TRACE, 64 },
85 { 0x78, LVL_2, MB(1) },
86 { 0x79, LVL_2, 128 },
87 { 0x7a, LVL_2, 256 },
88 { 0x7b, LVL_2, 512 },
89 { 0x7c, LVL_2, MB(1) },
90 { 0x7d, LVL_2, MB(2) },
91 { 0x7f, LVL_2, 512 },
92 { 0x80, LVL_2, 512 },
93 { 0x82, LVL_2, 256 },
94 { 0x83, LVL_2, 512 },
95 { 0x84, LVL_2, MB(1) },
96 { 0x85, LVL_2, MB(2) },
97 { 0x86, LVL_2, 512 },
98 { 0x87, LVL_2, MB(1) },
99 { 0xd0, LVL_3, 512 },
100 { 0xd1, LVL_3, MB(1) },
101 { 0xd2, LVL_3, MB(2) },
102 { 0xd6, LVL_3, MB(1) },
103 { 0xd7, LVL_3, MB(2) },
104 { 0xd8, LVL_3, MB(4) },
105 { 0xdc, LVL_3, MB(2) },
106 { 0xdd, LVL_3, MB(4) },
107 { 0xde, LVL_3, MB(8) },
108 { 0xe2, LVL_3, MB(2) },
109 { 0xe3, LVL_3, MB(4) },
110 { 0xe4, LVL_3, MB(8) },
111 { 0xea, LVL_3, MB(12) },
112 { 0xeb, LVL_3, MB(18) },
113 { 0xec, LVL_3, MB(24) },
114 { 0x00, 0, 0}
115};
116
117
118enum _cache_type {
119 CACHE_TYPE_NULL = 0,
120 CACHE_TYPE_DATA = 1,
121 CACHE_TYPE_INST = 2,
122 CACHE_TYPE_UNIFIED = 3
123};
124
125union _cpuid4_leaf_eax {
126 struct {
127 enum _cache_type type:5;
128 unsigned int level:3;
129 unsigned int is_self_initializing:1;
130 unsigned int is_fully_associative:1;
131 unsigned int reserved:4;
132 unsigned int num_threads_sharing:12;
133 unsigned int num_cores_on_die:6;
134 } split;
135 u32 full;
136};
137
138union _cpuid4_leaf_ebx {
139 struct {
140 unsigned int coherency_line_size:12;
141 unsigned int physical_line_partition:10;
142 unsigned int ways_of_associativity:10;
143 } split;
144 u32 full;
145};
146
147union _cpuid4_leaf_ecx {
148 struct {
149 unsigned int number_of_sets:32;
150 } split;
151 u32 full;
152};
153
154struct _cpuid4_info_regs {
155 union _cpuid4_leaf_eax eax;
156 union _cpuid4_leaf_ebx ebx;
157 union _cpuid4_leaf_ecx ecx;
158 unsigned long size;
159 struct amd_northbridge *nb;
160};
161
162struct _cpuid4_info {
163 struct _cpuid4_info_regs base;
164 DECLARE_BITMAP(shared_cpu_map, NR_CPUS);
165};
166
167unsigned short num_cache_leaves;
168
169
170
171
172
173
174
175union l1_cache {
176 struct {
177 unsigned line_size:8;
178 unsigned lines_per_tag:8;
179 unsigned assoc:8;
180 unsigned size_in_kb:8;
181 };
182 unsigned val;
183};
184
185union l2_cache {
186 struct {
187 unsigned line_size:8;
188 unsigned lines_per_tag:4;
189 unsigned assoc:4;
190 unsigned size_in_kb:16;
191 };
192 unsigned val;
193};
194
195union l3_cache {
196 struct {
197 unsigned line_size:8;
198 unsigned lines_per_tag:4;
199 unsigned assoc:4;
200 unsigned res:2;
201 unsigned size_encoded:14;
202 };
203 unsigned val;
204};
205
206static const unsigned short __cpuinitconst assocs[] = {
207 [1] = 1,
208 [2] = 2,
209 [4] = 4,
210 [6] = 8,
211 [8] = 16,
212 [0xa] = 32,
213 [0xb] = 48,
214 [0xc] = 64,
215 [0xd] = 96,
216 [0xe] = 128,
217 [0xf] = 0xffff
218};
219
220static const unsigned char __cpuinitconst levels[] = { 1, 1, 2, 3 };
221static const unsigned char __cpuinitconst types[] = { 1, 2, 3, 3 };
222
223static void __cpuinit
224amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
225 union _cpuid4_leaf_ebx *ebx,
226 union _cpuid4_leaf_ecx *ecx)
227{
228 unsigned dummy;
229 unsigned line_size, lines_per_tag, assoc, size_in_kb;
230 union l1_cache l1i, l1d;
231 union l2_cache l2;
232 union l3_cache l3;
233 union l1_cache *l1 = &l1d;
234
235 eax->full = 0;
236 ebx->full = 0;
237 ecx->full = 0;
238
239 cpuid(0x80000005, &dummy, &dummy, &l1d.val, &l1i.val);
240 cpuid(0x80000006, &dummy, &dummy, &l2.val, &l3.val);
241
242 switch (leaf) {
243 case 1:
244 l1 = &l1i;
245 case 0:
246 if (!l1->val)
247 return;
248 assoc = assocs[l1->assoc];
249 line_size = l1->line_size;
250 lines_per_tag = l1->lines_per_tag;
251 size_in_kb = l1->size_in_kb;
252 break;
253 case 2:
254 if (!l2.val)
255 return;
256 assoc = assocs[l2.assoc];
257 line_size = l2.line_size;
258 lines_per_tag = l2.lines_per_tag;
259
260 size_in_kb = __this_cpu_read(cpu_info.x86_cache_size);
261 break;
262 case 3:
263 if (!l3.val)
264 return;
265 assoc = assocs[l3.assoc];
266 line_size = l3.line_size;
267 lines_per_tag = l3.lines_per_tag;
268 size_in_kb = l3.size_encoded * 512;
269 if (boot_cpu_has(X86_FEATURE_AMD_DCM)) {
270 size_in_kb = size_in_kb >> 1;
271 assoc = assoc >> 1;
272 }
273 break;
274 default:
275 return;
276 }
277
278 eax->split.is_self_initializing = 1;
279 eax->split.type = types[leaf];
280 eax->split.level = levels[leaf];
281 eax->split.num_threads_sharing = 0;
282 eax->split.num_cores_on_die = __this_cpu_read(cpu_info.x86_max_cores) - 1;
283
284
285 if (assoc == 0xffff)
286 eax->split.is_fully_associative = 1;
287 ebx->split.coherency_line_size = line_size - 1;
288 ebx->split.ways_of_associativity = assoc - 1;
289 ebx->split.physical_line_partition = lines_per_tag - 1;
290 ecx->split.number_of_sets = (size_in_kb * 1024) / line_size /
291 (ebx->split.ways_of_associativity + 1) - 1;
292}
293
294struct _cache_attr {
295 struct attribute attr;
296 ssize_t (*show)(struct _cpuid4_info *, char *, unsigned int);
297 ssize_t (*store)(struct _cpuid4_info *, const char *, size_t count,
298 unsigned int);
299};
300
301#if defined(CONFIG_AMD_NB) && defined(CONFIG_SYSFS)
302
303
304
305static void __cpuinit amd_calc_l3_indices(struct amd_northbridge *nb)
306{
307 struct amd_l3_cache *l3 = &nb->l3_cache;
308 unsigned int sc0, sc1, sc2, sc3;
309 u32 val = 0;
310
311 pci_read_config_dword(nb->misc, 0x1C4, &val);
312
313
314 l3->subcaches[0] = sc0 = !(val & BIT(0));
315 l3->subcaches[1] = sc1 = !(val & BIT(4));
316
317 if (boot_cpu_data.x86 == 0x15) {
318 l3->subcaches[0] = sc0 += !(val & BIT(1));
319 l3->subcaches[1] = sc1 += !(val & BIT(5));
320 }
321
322 l3->subcaches[2] = sc2 = !(val & BIT(8)) + !(val & BIT(9));
323 l3->subcaches[3] = sc3 = !(val & BIT(12)) + !(val & BIT(13));
324
325 l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1;
326}
327
328static void __cpuinit amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index)
329{
330 int node;
331
332
333 if (index < 3)
334 return;
335
336 node = amd_get_nb_id(smp_processor_id());
337 this_leaf->nb = node_to_amd_nb(node);
338 if (this_leaf->nb && !this_leaf->nb->l3_cache.indices)
339 amd_calc_l3_indices(this_leaf->nb);
340}
341
342
343
344
345
346
347
348
349int amd_get_l3_disable_slot(struct amd_northbridge *nb, unsigned slot)
350{
351 unsigned int reg = 0;
352
353 pci_read_config_dword(nb->misc, 0x1BC + slot * 4, ®);
354
355
356 if (reg & (3UL << 30))
357 return reg & 0xfff;
358
359 return -1;
360}
361
362static ssize_t show_cache_disable(struct _cpuid4_info *this_leaf, char *buf,
363 unsigned int slot)
364{
365 int index;
366
367 if (!this_leaf->base.nb || !amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
368 return -EINVAL;
369
370 index = amd_get_l3_disable_slot(this_leaf->base.nb, slot);
371 if (index >= 0)
372 return sprintf(buf, "%d\n", index);
373
374 return sprintf(buf, "FREE\n");
375}
376
377#define SHOW_CACHE_DISABLE(slot) \
378static ssize_t \
379show_cache_disable_##slot(struct _cpuid4_info *this_leaf, char *buf, \
380 unsigned int cpu) \
381{ \
382 return show_cache_disable(this_leaf, buf, slot); \
383}
384SHOW_CACHE_DISABLE(0)
385SHOW_CACHE_DISABLE(1)
386
387static void amd_l3_disable_index(struct amd_northbridge *nb, int cpu,
388 unsigned slot, unsigned long idx)
389{
390 int i;
391
392 idx |= BIT(30);
393
394
395
396
397 for (i = 0; i < 4; i++) {
398 u32 reg = idx | (i << 20);
399
400 if (!nb->l3_cache.subcaches[i])
401 continue;
402
403 pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg);
404
405
406
407
408
409
410 wbinvd_on_cpu(cpu);
411
412 reg |= BIT(31);
413 pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg);
414 }
415}
416
417
418
419
420
421
422
423
424
425
426
427int amd_set_l3_disable_slot(struct amd_northbridge *nb, int cpu, unsigned slot,
428 unsigned long index)
429{
430 int ret = 0;
431
432
433 ret = amd_get_l3_disable_slot(nb, slot);
434 if (ret >= 0)
435 return -EEXIST;
436
437 if (index > nb->l3_cache.indices)
438 return -EINVAL;
439
440
441 if (index == amd_get_l3_disable_slot(nb, !slot))
442 return -EEXIST;
443
444 amd_l3_disable_index(nb, cpu, slot, index);
445
446 return 0;
447}
448
449static ssize_t store_cache_disable(struct _cpuid4_info *this_leaf,
450 const char *buf, size_t count,
451 unsigned int slot)
452{
453 unsigned long val = 0;
454 int cpu, err = 0;
455
456 if (!capable(CAP_SYS_ADMIN))
457 return -EPERM;
458
459 if (!this_leaf->base.nb || !amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
460 return -EINVAL;
461
462 cpu = cpumask_first(to_cpumask(this_leaf->shared_cpu_map));
463
464 if (strict_strtoul(buf, 10, &val) < 0)
465 return -EINVAL;
466
467 err = amd_set_l3_disable_slot(this_leaf->base.nb, cpu, slot, val);
468 if (err) {
469 if (err == -EEXIST)
470 pr_warning("L3 slot %d in use/index already disabled!\n",
471 slot);
472 return err;
473 }
474 return count;
475}
476
477#define STORE_CACHE_DISABLE(slot) \
478static ssize_t \
479store_cache_disable_##slot(struct _cpuid4_info *this_leaf, \
480 const char *buf, size_t count, \
481 unsigned int cpu) \
482{ \
483 return store_cache_disable(this_leaf, buf, count, slot); \
484}
485STORE_CACHE_DISABLE(0)
486STORE_CACHE_DISABLE(1)
487
488static struct _cache_attr cache_disable_0 = __ATTR(cache_disable_0, 0644,
489 show_cache_disable_0, store_cache_disable_0);
490static struct _cache_attr cache_disable_1 = __ATTR(cache_disable_1, 0644,
491 show_cache_disable_1, store_cache_disable_1);
492
493static ssize_t
494show_subcaches(struct _cpuid4_info *this_leaf, char *buf, unsigned int cpu)
495{
496 if (!this_leaf->base.nb || !amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
497 return -EINVAL;
498
499 return sprintf(buf, "%x\n", amd_get_subcaches(cpu));
500}
501
502static ssize_t
503store_subcaches(struct _cpuid4_info *this_leaf, const char *buf, size_t count,
504 unsigned int cpu)
505{
506 unsigned long val;
507
508 if (!capable(CAP_SYS_ADMIN))
509 return -EPERM;
510
511 if (!this_leaf->base.nb || !amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
512 return -EINVAL;
513
514 if (strict_strtoul(buf, 16, &val) < 0)
515 return -EINVAL;
516
517 if (amd_set_subcaches(cpu, val))
518 return -EINVAL;
519
520 return count;
521}
522
523static struct _cache_attr subcaches =
524 __ATTR(subcaches, 0644, show_subcaches, store_subcaches);
525
526#else
527#define amd_init_l3_cache(x, y)
528#endif
529
530static int
531__cpuinit cpuid4_cache_lookup_regs(int index,
532 struct _cpuid4_info_regs *this_leaf)
533{
534 union _cpuid4_leaf_eax eax;
535 union _cpuid4_leaf_ebx ebx;
536 union _cpuid4_leaf_ecx ecx;
537 unsigned edx;
538
539 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
540 if (cpu_has_topoext)
541 cpuid_count(0x8000001d, index, &eax.full,
542 &ebx.full, &ecx.full, &edx);
543 else
544 amd_cpuid4(index, &eax, &ebx, &ecx);
545 amd_init_l3_cache(this_leaf, index);
546 } else {
547 cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx);
548 }
549
550 if (eax.split.type == CACHE_TYPE_NULL)
551 return -EIO;
552
553 this_leaf->eax = eax;
554 this_leaf->ebx = ebx;
555 this_leaf->ecx = ecx;
556 this_leaf->size = (ecx.split.number_of_sets + 1) *
557 (ebx.split.coherency_line_size + 1) *
558 (ebx.split.physical_line_partition + 1) *
559 (ebx.split.ways_of_associativity + 1);
560 return 0;
561}
562
563static int __cpuinit find_num_cache_leaves(struct cpuinfo_x86 *c)
564{
565 unsigned int eax, ebx, ecx, edx, op;
566 union _cpuid4_leaf_eax cache_eax;
567 int i = -1;
568
569 if (c->x86_vendor == X86_VENDOR_AMD)
570 op = 0x8000001d;
571 else
572 op = 4;
573
574 do {
575 ++i;
576
577 cpuid_count(op, i, &eax, &ebx, &ecx, &edx);
578 cache_eax.full = eax;
579 } while (cache_eax.split.type != CACHE_TYPE_NULL);
580 return i;
581}
582
583void __cpuinit init_amd_cacheinfo(struct cpuinfo_x86 *c)
584{
585
586 if (cpu_has_topoext) {
587 num_cache_leaves = find_num_cache_leaves(c);
588 } else if (c->extended_cpuid_level >= 0x80000006) {
589 if (cpuid_edx(0x80000006) & 0xf000)
590 num_cache_leaves = 4;
591 else
592 num_cache_leaves = 3;
593 }
594}
595
596unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c)
597{
598
599 unsigned int trace = 0, l1i = 0, l1d = 0, l2 = 0, l3 = 0;
600 unsigned int new_l1d = 0, new_l1i = 0;
601 unsigned int new_l2 = 0, new_l3 = 0, i;
602 unsigned int l2_id = 0, l3_id = 0, num_threads_sharing, index_msb;
603#ifdef CONFIG_X86_HT
604 unsigned int cpu = c->cpu_index;
605#endif
606
607 if (c->cpuid_level > 3) {
608 static int is_initialized;
609
610 if (is_initialized == 0) {
611
612 num_cache_leaves = find_num_cache_leaves(c);
613 is_initialized++;
614 }
615
616
617
618
619
620 for (i = 0; i < num_cache_leaves; i++) {
621 struct _cpuid4_info_regs this_leaf;
622 int retval;
623
624 retval = cpuid4_cache_lookup_regs(i, &this_leaf);
625 if (retval >= 0) {
626 switch (this_leaf.eax.split.level) {
627 case 1:
628 if (this_leaf.eax.split.type ==
629 CACHE_TYPE_DATA)
630 new_l1d = this_leaf.size/1024;
631 else if (this_leaf.eax.split.type ==
632 CACHE_TYPE_INST)
633 new_l1i = this_leaf.size/1024;
634 break;
635 case 2:
636 new_l2 = this_leaf.size/1024;
637 num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
638 index_msb = get_count_order(num_threads_sharing);
639 l2_id = c->apicid & ~((1 << index_msb) - 1);
640 break;
641 case 3:
642 new_l3 = this_leaf.size/1024;
643 num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
644 index_msb = get_count_order(
645 num_threads_sharing);
646 l3_id = c->apicid & ~((1 << index_msb) - 1);
647 break;
648 default:
649 break;
650 }
651 }
652 }
653 }
654
655
656
657
658 if ((num_cache_leaves == 0 || c->x86 == 15) && c->cpuid_level > 1) {
659
660 int j, n;
661 unsigned int regs[4];
662 unsigned char *dp = (unsigned char *)regs;
663 int only_trace = 0;
664
665 if (num_cache_leaves != 0 && c->x86 == 15)
666 only_trace = 1;
667
668
669 n = cpuid_eax(2) & 0xFF;
670
671 for (i = 0 ; i < n ; i++) {
672 cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]);
673
674
675 for (j = 0 ; j < 3 ; j++)
676 if (regs[j] & (1 << 31))
677 regs[j] = 0;
678
679
680 for (j = 1 ; j < 16 ; j++) {
681 unsigned char des = dp[j];
682 unsigned char k = 0;
683
684
685 while (cache_table[k].descriptor != 0) {
686 if (cache_table[k].descriptor == des) {
687 if (only_trace && cache_table[k].cache_type != LVL_TRACE)
688 break;
689 switch (cache_table[k].cache_type) {
690 case LVL_1_INST:
691 l1i += cache_table[k].size;
692 break;
693 case LVL_1_DATA:
694 l1d += cache_table[k].size;
695 break;
696 case LVL_2:
697 l2 += cache_table[k].size;
698 break;
699 case LVL_3:
700 l3 += cache_table[k].size;
701 break;
702 case LVL_TRACE:
703 trace += cache_table[k].size;
704 break;
705 }
706
707 break;
708 }
709
710 k++;
711 }
712 }
713 }
714 }
715
716 if (new_l1d)
717 l1d = new_l1d;
718
719 if (new_l1i)
720 l1i = new_l1i;
721
722 if (new_l2) {
723 l2 = new_l2;
724#ifdef CONFIG_X86_HT
725 per_cpu(cpu_llc_id, cpu) = l2_id;
726#endif
727 }
728
729 if (new_l3) {
730 l3 = new_l3;
731#ifdef CONFIG_X86_HT
732 per_cpu(cpu_llc_id, cpu) = l3_id;
733#endif
734 }
735
736 c->x86_cache_size = l3 ? l3 : (l2 ? l2 : (l1i+l1d));
737
738 return l2;
739}
740
741#ifdef CONFIG_SYSFS
742
743
744static DEFINE_PER_CPU(struct _cpuid4_info *, ici_cpuid4_info);
745#define CPUID4_INFO_IDX(x, y) (&((per_cpu(ici_cpuid4_info, x))[y]))
746
747#ifdef CONFIG_SMP
748
749static int __cpuinit cache_shared_amd_cpu_map_setup(unsigned int cpu, int index)
750{
751 struct _cpuid4_info *this_leaf;
752 int i, sibling;
753
754 if (cpu_has_topoext) {
755 unsigned int apicid, nshared, first, last;
756
757 if (!per_cpu(ici_cpuid4_info, cpu))
758 return 0;
759
760 this_leaf = CPUID4_INFO_IDX(cpu, index);
761 nshared = this_leaf->base.eax.split.num_threads_sharing + 1;
762 apicid = cpu_data(cpu).apicid;
763 first = apicid - (apicid % nshared);
764 last = first + nshared - 1;
765
766 for_each_online_cpu(i) {
767 apicid = cpu_data(i).apicid;
768 if ((apicid < first) || (apicid > last))
769 continue;
770 if (!per_cpu(ici_cpuid4_info, i))
771 continue;
772 this_leaf = CPUID4_INFO_IDX(i, index);
773
774 for_each_online_cpu(sibling) {
775 apicid = cpu_data(sibling).apicid;
776 if ((apicid < first) || (apicid > last))
777 continue;
778 set_bit(sibling, this_leaf->shared_cpu_map);
779 }
780 }
781 } else if (index == 3) {
782 for_each_cpu(i, cpu_llc_shared_mask(cpu)) {
783 if (!per_cpu(ici_cpuid4_info, i))
784 continue;
785 this_leaf = CPUID4_INFO_IDX(i, index);
786 for_each_cpu(sibling, cpu_llc_shared_mask(cpu)) {
787 if (!cpu_online(sibling))
788 continue;
789 set_bit(sibling, this_leaf->shared_cpu_map);
790 }
791 }
792 } else
793 return 0;
794
795 return 1;
796}
797
798static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
799{
800 struct _cpuid4_info *this_leaf, *sibling_leaf;
801 unsigned long num_threads_sharing;
802 int index_msb, i;
803 struct cpuinfo_x86 *c = &cpu_data(cpu);
804
805 if (c->x86_vendor == X86_VENDOR_AMD) {
806 if (cache_shared_amd_cpu_map_setup(cpu, index))
807 return;
808 }
809
810 this_leaf = CPUID4_INFO_IDX(cpu, index);
811 num_threads_sharing = 1 + this_leaf->base.eax.split.num_threads_sharing;
812
813 if (num_threads_sharing == 1)
814 cpumask_set_cpu(cpu, to_cpumask(this_leaf->shared_cpu_map));
815 else {
816 index_msb = get_count_order(num_threads_sharing);
817
818 for_each_online_cpu(i) {
819 if (cpu_data(i).apicid >> index_msb ==
820 c->apicid >> index_msb) {
821 cpumask_set_cpu(i,
822 to_cpumask(this_leaf->shared_cpu_map));
823 if (i != cpu && per_cpu(ici_cpuid4_info, i)) {
824 sibling_leaf =
825 CPUID4_INFO_IDX(i, index);
826 cpumask_set_cpu(cpu, to_cpumask(
827 sibling_leaf->shared_cpu_map));
828 }
829 }
830 }
831 }
832}
833static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index)
834{
835 struct _cpuid4_info *this_leaf, *sibling_leaf;
836 int sibling;
837
838 this_leaf = CPUID4_INFO_IDX(cpu, index);
839 for_each_cpu(sibling, to_cpumask(this_leaf->shared_cpu_map)) {
840 sibling_leaf = CPUID4_INFO_IDX(sibling, index);
841 cpumask_clear_cpu(cpu,
842 to_cpumask(sibling_leaf->shared_cpu_map));
843 }
844}
845#else
846static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
847{
848}
849
850static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index)
851{
852}
853#endif
854
855static void __cpuinit free_cache_attributes(unsigned int cpu)
856{
857 int i;
858
859 for (i = 0; i < num_cache_leaves; i++)
860 cache_remove_shared_cpu_map(cpu, i);
861
862 kfree(per_cpu(ici_cpuid4_info, cpu));
863 per_cpu(ici_cpuid4_info, cpu) = NULL;
864}
865
866static void __cpuinit get_cpu_leaves(void *_retval)
867{
868 int j, *retval = _retval, cpu = smp_processor_id();
869
870
871 for (j = 0; j < num_cache_leaves; j++) {
872 struct _cpuid4_info *this_leaf = CPUID4_INFO_IDX(cpu, j);
873
874 *retval = cpuid4_cache_lookup_regs(j, &this_leaf->base);
875 if (unlikely(*retval < 0)) {
876 int i;
877
878 for (i = 0; i < j; i++)
879 cache_remove_shared_cpu_map(cpu, i);
880 break;
881 }
882 cache_shared_cpu_map_setup(cpu, j);
883 }
884}
885
886static int __cpuinit detect_cache_attributes(unsigned int cpu)
887{
888 int retval;
889
890 if (num_cache_leaves == 0)
891 return -ENOENT;
892
893 per_cpu(ici_cpuid4_info, cpu) = kzalloc(
894 sizeof(struct _cpuid4_info) * num_cache_leaves, GFP_KERNEL);
895 if (per_cpu(ici_cpuid4_info, cpu) == NULL)
896 return -ENOMEM;
897
898 smp_call_function_single(cpu, get_cpu_leaves, &retval, true);
899 if (retval) {
900 kfree(per_cpu(ici_cpuid4_info, cpu));
901 per_cpu(ici_cpuid4_info, cpu) = NULL;
902 }
903
904 return retval;
905}
906
907#include <linux/kobject.h>
908#include <linux/sysfs.h>
909#include <linux/cpu.h>
910
911
912static DEFINE_PER_CPU(struct kobject *, ici_cache_kobject);
913
914struct _index_kobject {
915 struct kobject kobj;
916 unsigned int cpu;
917 unsigned short index;
918};
919
920
921static DEFINE_PER_CPU(struct _index_kobject *, ici_index_kobject);
922#define INDEX_KOBJECT_PTR(x, y) (&((per_cpu(ici_index_kobject, x))[y]))
923
924#define show_one_plus(file_name, object, val) \
925static ssize_t show_##file_name(struct _cpuid4_info *this_leaf, char *buf, \
926 unsigned int cpu) \
927{ \
928 return sprintf(buf, "%lu\n", (unsigned long)this_leaf->object + val); \
929}
930
931show_one_plus(level, base.eax.split.level, 0);
932show_one_plus(coherency_line_size, base.ebx.split.coherency_line_size, 1);
933show_one_plus(physical_line_partition, base.ebx.split.physical_line_partition, 1);
934show_one_plus(ways_of_associativity, base.ebx.split.ways_of_associativity, 1);
935show_one_plus(number_of_sets, base.ecx.split.number_of_sets, 1);
936
937static ssize_t show_size(struct _cpuid4_info *this_leaf, char *buf,
938 unsigned int cpu)
939{
940 return sprintf(buf, "%luK\n", this_leaf->base.size / 1024);
941}
942
943static ssize_t show_shared_cpu_map_func(struct _cpuid4_info *this_leaf,
944 int type, char *buf)
945{
946 ptrdiff_t len = PTR_ALIGN(buf + PAGE_SIZE - 1, PAGE_SIZE) - buf;
947 int n = 0;
948
949 if (len > 1) {
950 const struct cpumask *mask;
951
952 mask = to_cpumask(this_leaf->shared_cpu_map);
953 n = type ?
954 cpulist_scnprintf(buf, len-2, mask) :
955 cpumask_scnprintf(buf, len-2, mask);
956 buf[n++] = '\n';
957 buf[n] = '\0';
958 }
959 return n;
960}
961
962static inline ssize_t show_shared_cpu_map(struct _cpuid4_info *leaf, char *buf,
963 unsigned int cpu)
964{
965 return show_shared_cpu_map_func(leaf, 0, buf);
966}
967
968static inline ssize_t show_shared_cpu_list(struct _cpuid4_info *leaf, char *buf,
969 unsigned int cpu)
970{
971 return show_shared_cpu_map_func(leaf, 1, buf);
972}
973
974static ssize_t show_type(struct _cpuid4_info *this_leaf, char *buf,
975 unsigned int cpu)
976{
977 switch (this_leaf->base.eax.split.type) {
978 case CACHE_TYPE_DATA:
979 return sprintf(buf, "Data\n");
980 case CACHE_TYPE_INST:
981 return sprintf(buf, "Instruction\n");
982 case CACHE_TYPE_UNIFIED:
983 return sprintf(buf, "Unified\n");
984 default:
985 return sprintf(buf, "Unknown\n");
986 }
987}
988
989#define to_object(k) container_of(k, struct _index_kobject, kobj)
990#define to_attr(a) container_of(a, struct _cache_attr, attr)
991
992#define define_one_ro(_name) \
993static struct _cache_attr _name = \
994 __ATTR(_name, 0444, show_##_name, NULL)
995
996define_one_ro(level);
997define_one_ro(type);
998define_one_ro(coherency_line_size);
999define_one_ro(physical_line_partition);
1000define_one_ro(ways_of_associativity);
1001define_one_ro(number_of_sets);
1002define_one_ro(size);
1003define_one_ro(shared_cpu_map);
1004define_one_ro(shared_cpu_list);
1005
1006static struct attribute *default_attrs[] = {
1007 &type.attr,
1008 &level.attr,
1009 &coherency_line_size.attr,
1010 &physical_line_partition.attr,
1011 &ways_of_associativity.attr,
1012 &number_of_sets.attr,
1013 &size.attr,
1014 &shared_cpu_map.attr,
1015 &shared_cpu_list.attr,
1016 NULL
1017};
1018
1019#ifdef CONFIG_AMD_NB
1020static struct attribute ** __cpuinit amd_l3_attrs(void)
1021{
1022 static struct attribute **attrs;
1023 int n;
1024
1025 if (attrs)
1026 return attrs;
1027
1028 n = ARRAY_SIZE(default_attrs);
1029
1030 if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
1031 n += 2;
1032
1033 if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
1034 n += 1;
1035
1036 attrs = kzalloc(n * sizeof (struct attribute *), GFP_KERNEL);
1037 if (attrs == NULL)
1038 return attrs = default_attrs;
1039
1040 for (n = 0; default_attrs[n]; n++)
1041 attrs[n] = default_attrs[n];
1042
1043 if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) {
1044 attrs[n++] = &cache_disable_0.attr;
1045 attrs[n++] = &cache_disable_1.attr;
1046 }
1047
1048 if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
1049 attrs[n++] = &subcaches.attr;
1050
1051 return attrs;
1052}
1053#endif
1054
1055static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
1056{
1057 struct _cache_attr *fattr = to_attr(attr);
1058 struct _index_kobject *this_leaf = to_object(kobj);
1059 ssize_t ret;
1060
1061 ret = fattr->show ?
1062 fattr->show(CPUID4_INFO_IDX(this_leaf->cpu, this_leaf->index),
1063 buf, this_leaf->cpu) :
1064 0;
1065 return ret;
1066}
1067
1068static ssize_t store(struct kobject *kobj, struct attribute *attr,
1069 const char *buf, size_t count)
1070{
1071 struct _cache_attr *fattr = to_attr(attr);
1072 struct _index_kobject *this_leaf = to_object(kobj);
1073 ssize_t ret;
1074
1075 ret = fattr->store ?
1076 fattr->store(CPUID4_INFO_IDX(this_leaf->cpu, this_leaf->index),
1077 buf, count, this_leaf->cpu) :
1078 0;
1079 return ret;
1080}
1081
1082static const struct sysfs_ops sysfs_ops = {
1083 .show = show,
1084 .store = store,
1085};
1086
1087static struct kobj_type ktype_cache = {
1088 .sysfs_ops = &sysfs_ops,
1089 .default_attrs = default_attrs,
1090};
1091
1092static struct kobj_type ktype_percpu_entry = {
1093 .sysfs_ops = &sysfs_ops,
1094};
1095
1096static void __cpuinit cpuid4_cache_sysfs_exit(unsigned int cpu)
1097{
1098 kfree(per_cpu(ici_cache_kobject, cpu));
1099 kfree(per_cpu(ici_index_kobject, cpu));
1100 per_cpu(ici_cache_kobject, cpu) = NULL;
1101 per_cpu(ici_index_kobject, cpu) = NULL;
1102 free_cache_attributes(cpu);
1103}
1104
1105static int __cpuinit cpuid4_cache_sysfs_init(unsigned int cpu)
1106{
1107 int err;
1108
1109 if (num_cache_leaves == 0)
1110 return -ENOENT;
1111
1112 err = detect_cache_attributes(cpu);
1113 if (err)
1114 return err;
1115
1116
1117 per_cpu(ici_cache_kobject, cpu) =
1118 kzalloc(sizeof(struct kobject), GFP_KERNEL);
1119 if (unlikely(per_cpu(ici_cache_kobject, cpu) == NULL))
1120 goto err_out;
1121
1122 per_cpu(ici_index_kobject, cpu) = kzalloc(
1123 sizeof(struct _index_kobject) * num_cache_leaves, GFP_KERNEL);
1124 if (unlikely(per_cpu(ici_index_kobject, cpu) == NULL))
1125 goto err_out;
1126
1127 return 0;
1128
1129err_out:
1130 cpuid4_cache_sysfs_exit(cpu);
1131 return -ENOMEM;
1132}
1133
1134static DECLARE_BITMAP(cache_dev_map, NR_CPUS);
1135
1136
1137static int __cpuinit cache_add_dev(struct device *dev)
1138{
1139 unsigned int cpu = dev->id;
1140 unsigned long i, j;
1141 struct _index_kobject *this_object;
1142 struct _cpuid4_info *this_leaf;
1143 int retval;
1144
1145 retval = cpuid4_cache_sysfs_init(cpu);
1146 if (unlikely(retval < 0))
1147 return retval;
1148
1149 retval = kobject_init_and_add(per_cpu(ici_cache_kobject, cpu),
1150 &ktype_percpu_entry,
1151 &dev->kobj, "%s", "cache");
1152 if (retval < 0) {
1153 cpuid4_cache_sysfs_exit(cpu);
1154 return retval;
1155 }
1156
1157 for (i = 0; i < num_cache_leaves; i++) {
1158 this_object = INDEX_KOBJECT_PTR(cpu, i);
1159 this_object->cpu = cpu;
1160 this_object->index = i;
1161
1162 this_leaf = CPUID4_INFO_IDX(cpu, i);
1163
1164 ktype_cache.default_attrs = default_attrs;
1165#ifdef CONFIG_AMD_NB
1166 if (this_leaf->base.nb)
1167 ktype_cache.default_attrs = amd_l3_attrs();
1168#endif
1169 retval = kobject_init_and_add(&(this_object->kobj),
1170 &ktype_cache,
1171 per_cpu(ici_cache_kobject, cpu),
1172 "index%1lu", i);
1173 if (unlikely(retval)) {
1174 for (j = 0; j < i; j++)
1175 kobject_put(&(INDEX_KOBJECT_PTR(cpu, j)->kobj));
1176 kobject_put(per_cpu(ici_cache_kobject, cpu));
1177 cpuid4_cache_sysfs_exit(cpu);
1178 return retval;
1179 }
1180 kobject_uevent(&(this_object->kobj), KOBJ_ADD);
1181 }
1182 cpumask_set_cpu(cpu, to_cpumask(cache_dev_map));
1183
1184 kobject_uevent(per_cpu(ici_cache_kobject, cpu), KOBJ_ADD);
1185 return 0;
1186}
1187
1188static void __cpuinit cache_remove_dev(struct device *dev)
1189{
1190 unsigned int cpu = dev->id;
1191 unsigned long i;
1192
1193 if (per_cpu(ici_cpuid4_info, cpu) == NULL)
1194 return;
1195 if (!cpumask_test_cpu(cpu, to_cpumask(cache_dev_map)))
1196 return;
1197 cpumask_clear_cpu(cpu, to_cpumask(cache_dev_map));
1198
1199 for (i = 0; i < num_cache_leaves; i++)
1200 kobject_put(&(INDEX_KOBJECT_PTR(cpu, i)->kobj));
1201 kobject_put(per_cpu(ici_cache_kobject, cpu));
1202 cpuid4_cache_sysfs_exit(cpu);
1203}
1204
1205static int __cpuinit cacheinfo_cpu_callback(struct notifier_block *nfb,
1206 unsigned long action, void *hcpu)
1207{
1208 unsigned int cpu = (unsigned long)hcpu;
1209 struct device *dev;
1210
1211 dev = get_cpu_device(cpu);
1212 switch (action) {
1213 case CPU_ONLINE:
1214 case CPU_ONLINE_FROZEN:
1215 cache_add_dev(dev);
1216 break;
1217 case CPU_DEAD:
1218 case CPU_DEAD_FROZEN:
1219 cache_remove_dev(dev);
1220 break;
1221 }
1222 return NOTIFY_OK;
1223}
1224
1225static struct notifier_block __cpuinitdata cacheinfo_cpu_notifier = {
1226 .notifier_call = cacheinfo_cpu_callback,
1227};
1228
1229static int __cpuinit cache_sysfs_init(void)
1230{
1231 int i;
1232
1233 if (num_cache_leaves == 0)
1234 return 0;
1235
1236 for_each_online_cpu(i) {
1237 int err;
1238 struct device *dev = get_cpu_device(i);
1239
1240 err = cache_add_dev(dev);
1241 if (err)
1242 return err;
1243 }
1244 register_hotcpu_notifier(&cacheinfo_cpu_notifier);
1245 return 0;
1246}
1247
1248device_initcall(cache_sysfs_init);
1249
1250#endif
1251
