1
2
3
4
5
6
7
8
9
10
11#include <linux/acpi.h>
12#include <linux/bitops.h>
13#include <linux/device.h>
14#include <linux/init.h>
15#include <linux/list.h>
16#include <linux/list_sort.h>
17#include <linux/node.h>
18#include <linux/sysfs.h>
19
20static __initdata u8 hmat_revision;
21
22static __initdata LIST_HEAD(targets);
23static __initdata LIST_HEAD(initiators);
24static __initdata LIST_HEAD(localities);
25
26
27
28
29
30enum locality_types {
31 WRITE_LATENCY,
32 READ_LATENCY,
33 WRITE_BANDWIDTH,
34 READ_BANDWIDTH,
35};
36
37static struct memory_locality *localities_types[4];
38
39struct memory_target {
40 struct list_head node;
41 unsigned int memory_pxm;
42 unsigned int processor_pxm;
43 struct node_hmem_attrs hmem_attrs;
44};
45
46struct memory_initiator {
47 struct list_head node;
48 unsigned int processor_pxm;
49};
50
51struct memory_locality {
52 struct list_head node;
53 struct acpi_hmat_locality *hmat_loc;
54};
55
56static __init struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
57{
58 struct memory_initiator *initiator;
59
60 list_for_each_entry(initiator, &initiators, node)
61 if (initiator->processor_pxm == cpu_pxm)
62 return initiator;
63 return NULL;
64}
65
66static __init struct memory_target *find_mem_target(unsigned int mem_pxm)
67{
68 struct memory_target *target;
69
70 list_for_each_entry(target, &targets, node)
71 if (target->memory_pxm == mem_pxm)
72 return target;
73 return NULL;
74}
75
76static __init void alloc_memory_initiator(unsigned int cpu_pxm)
77{
78 struct memory_initiator *initiator;
79
80 if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
81 return;
82
83 initiator = find_mem_initiator(cpu_pxm);
84 if (initiator)
85 return;
86
87 initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
88 if (!initiator)
89 return;
90
91 initiator->processor_pxm = cpu_pxm;
92 list_add_tail(&initiator->node, &initiators);
93}
94
95static __init void alloc_memory_target(unsigned int mem_pxm)
96{
97 struct memory_target *target;
98
99 if (pxm_to_node(mem_pxm) == NUMA_NO_NODE)
100 return;
101
102 target = find_mem_target(mem_pxm);
103 if (target)
104 return;
105
106 target = kzalloc(sizeof(*target), GFP_KERNEL);
107 if (!target)
108 return;
109
110 target->memory_pxm = mem_pxm;
111 target->processor_pxm = PXM_INVAL;
112 list_add_tail(&target->node, &targets);
113}
114
115static __init const char *hmat_data_type(u8 type)
116{
117 switch (type) {
118 case ACPI_HMAT_ACCESS_LATENCY:
119 return "Access Latency";
120 case ACPI_HMAT_READ_LATENCY:
121 return "Read Latency";
122 case ACPI_HMAT_WRITE_LATENCY:
123 return "Write Latency";
124 case ACPI_HMAT_ACCESS_BANDWIDTH:
125 return "Access Bandwidth";
126 case ACPI_HMAT_READ_BANDWIDTH:
127 return "Read Bandwidth";
128 case ACPI_HMAT_WRITE_BANDWIDTH:
129 return "Write Bandwidth";
130 default:
131 return "Reserved";
132 }
133}
134
135static __init const char *hmat_data_type_suffix(u8 type)
136{
137 switch (type) {
138 case ACPI_HMAT_ACCESS_LATENCY:
139 case ACPI_HMAT_READ_LATENCY:
140 case ACPI_HMAT_WRITE_LATENCY:
141 return " nsec";
142 case ACPI_HMAT_ACCESS_BANDWIDTH:
143 case ACPI_HMAT_READ_BANDWIDTH:
144 case ACPI_HMAT_WRITE_BANDWIDTH:
145 return " MB/s";
146 default:
147 return "";
148 }
149}
150
151static __init u32 hmat_normalize(u16 entry, u64 base, u8 type)
152{
153 u32 value;
154
155
156
157
158 if (entry == 0xffff || !entry)
159 return 0;
160 else if (base > (UINT_MAX / (entry)))
161 return 0;
162
163
164
165
166
167 value = entry * base;
168 if (hmat_revision == 1) {
169 if (value < 10)
170 return 0;
171 value = DIV_ROUND_UP(value, 10);
172 } else if (hmat_revision == 2) {
173 switch (type) {
174 case ACPI_HMAT_ACCESS_LATENCY:
175 case ACPI_HMAT_READ_LATENCY:
176 case ACPI_HMAT_WRITE_LATENCY:
177 value = DIV_ROUND_UP(value, 1000);
178 break;
179 default:
180 break;
181 }
182 }
183 return value;
184}
185
186static __init void hmat_update_target_access(struct memory_target *target,
187 u8 type, u32 value)
188{
189 switch (type) {
190 case ACPI_HMAT_ACCESS_LATENCY:
191 target->hmem_attrs.read_latency = value;
192 target->hmem_attrs.write_latency = value;
193 break;
194 case ACPI_HMAT_READ_LATENCY:
195 target->hmem_attrs.read_latency = value;
196 break;
197 case ACPI_HMAT_WRITE_LATENCY:
198 target->hmem_attrs.write_latency = value;
199 break;
200 case ACPI_HMAT_ACCESS_BANDWIDTH:
201 target->hmem_attrs.read_bandwidth = value;
202 target->hmem_attrs.write_bandwidth = value;
203 break;
204 case ACPI_HMAT_READ_BANDWIDTH:
205 target->hmem_attrs.read_bandwidth = value;
206 break;
207 case ACPI_HMAT_WRITE_BANDWIDTH:
208 target->hmem_attrs.write_bandwidth = value;
209 break;
210 default:
211 break;
212 }
213}
214
215static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
216{
217 struct memory_locality *loc;
218
219 loc = kzalloc(sizeof(*loc), GFP_KERNEL);
220 if (!loc) {
221 pr_notice_once("Failed to allocate HMAT locality\n");
222 return;
223 }
224
225 loc->hmat_loc = hmat_loc;
226 list_add_tail(&loc->node, &localities);
227
228 switch (hmat_loc->data_type) {
229 case ACPI_HMAT_ACCESS_LATENCY:
230 localities_types[READ_LATENCY] = loc;
231 localities_types[WRITE_LATENCY] = loc;
232 break;
233 case ACPI_HMAT_READ_LATENCY:
234 localities_types[READ_LATENCY] = loc;
235 break;
236 case ACPI_HMAT_WRITE_LATENCY:
237 localities_types[WRITE_LATENCY] = loc;
238 break;
239 case ACPI_HMAT_ACCESS_BANDWIDTH:
240 localities_types[READ_BANDWIDTH] = loc;
241 localities_types[WRITE_BANDWIDTH] = loc;
242 break;
243 case ACPI_HMAT_READ_BANDWIDTH:
244 localities_types[READ_BANDWIDTH] = loc;
245 break;
246 case ACPI_HMAT_WRITE_BANDWIDTH:
247 localities_types[WRITE_BANDWIDTH] = loc;
248 break;
249 default:
250 break;
251 }
252}
253
254static __init int hmat_parse_locality(union acpi_subtable_headers *header,
255 const unsigned long end)
256{
257 struct acpi_hmat_locality *hmat_loc = (void *)header;
258 struct memory_target *target;
259 unsigned int init, targ, total_size, ipds, tpds;
260 u32 *inits, *targs, value;
261 u16 *entries;
262 u8 type, mem_hier;
263
264 if (hmat_loc->header.length < sizeof(*hmat_loc)) {
265 pr_notice("HMAT: Unexpected locality header length: %d\n",
266 hmat_loc->header.length);
267 return -EINVAL;
268 }
269
270 type = hmat_loc->data_type;
271 mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
272 ipds = hmat_loc->number_of_initiator_Pds;
273 tpds = hmat_loc->number_of_target_Pds;
274 total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
275 sizeof(*inits) * ipds + sizeof(*targs) * tpds;
276 if (hmat_loc->header.length < total_size) {
277 pr_notice("HMAT: Unexpected locality header length:%d, minimum required:%d\n",
278 hmat_loc->header.length, total_size);
279 return -EINVAL;
280 }
281
282 pr_info("HMAT: Locality: Flags:%02x Type:%s Initiator Domains:%d Target Domains:%d Base:%lld\n",
283 hmat_loc->flags, hmat_data_type(type), ipds, tpds,
284 hmat_loc->entry_base_unit);
285
286 inits = (u32 *)(hmat_loc + 1);
287 targs = inits + ipds;
288 entries = (u16 *)(targs + tpds);
289 for (init = 0; init < ipds; init++) {
290 alloc_memory_initiator(inits[init]);
291 for (targ = 0; targ < tpds; targ++) {
292 value = hmat_normalize(entries[init * tpds + targ],
293 hmat_loc->entry_base_unit,
294 type);
295 pr_info(" Initiator-Target[%d-%d]:%d%s\n",
296 inits[init], targs[targ], value,
297 hmat_data_type_suffix(type));
298
299 if (mem_hier == ACPI_HMAT_MEMORY) {
300 target = find_mem_target(targs[targ]);
301 if (target && target->processor_pxm == inits[init])
302 hmat_update_target_access(target, type, value);
303 }
304 }
305 }
306
307 if (mem_hier == ACPI_HMAT_MEMORY)
308 hmat_add_locality(hmat_loc);
309
310 return 0;
311}
312
313static __init int hmat_parse_cache(union acpi_subtable_headers *header,
314 const unsigned long end)
315{
316 struct acpi_hmat_cache *cache = (void *)header;
317 struct node_cache_attrs cache_attrs;
318 u32 attrs;
319
320 if (cache->header.length < sizeof(*cache)) {
321 pr_notice("HMAT: Unexpected cache header length: %d\n",
322 cache->header.length);
323 return -EINVAL;
324 }
325
326 attrs = cache->cache_attributes;
327 pr_info("HMAT: Cache: Domain:%d Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
328 cache->memory_PD, cache->cache_size, attrs,
329 cache->number_of_SMBIOShandles);
330
331 cache_attrs.size = cache->cache_size;
332 cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
333 cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
334
335 switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
336 case ACPI_HMAT_CA_DIRECT_MAPPED:
337 cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
338 break;
339 case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
340 cache_attrs.indexing = NODE_CACHE_INDEXED;
341 break;
342 case ACPI_HMAT_CA_NONE:
343 default:
344 cache_attrs.indexing = NODE_CACHE_OTHER;
345 break;
346 }
347
348 switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
349 case ACPI_HMAT_CP_WB:
350 cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
351 break;
352 case ACPI_HMAT_CP_WT:
353 cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
354 break;
355 case ACPI_HMAT_CP_NONE:
356 default:
357 cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
358 break;
359 }
360
361 node_add_cache(pxm_to_node(cache->memory_PD), &cache_attrs);
362 return 0;
363}
364
365static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
366 const unsigned long end)
367{
368 struct acpi_hmat_proximity_domain *p = (void *)header;
369 struct memory_target *target = NULL;
370
371 if (p->header.length != sizeof(*p)) {
372 pr_notice("HMAT: Unexpected address range header length: %d\n",
373 p->header.length);
374 return -EINVAL;
375 }
376
377 if (hmat_revision == 1)
378 pr_info("HMAT: Memory (%#llx length %#llx) Flags:%04x Processor Domain:%d Memory Domain:%d\n",
379 p->reserved3, p->reserved4, p->flags, p->processor_PD,
380 p->memory_PD);
381 else
382 pr_info("HMAT: Memory Flags:%04x Processor Domain:%d Memory Domain:%d\n",
383 p->flags, p->processor_PD, p->memory_PD);
384
385 if (p->flags & ACPI_HMAT_MEMORY_PD_VALID) {
386 target = find_mem_target(p->memory_PD);
387 if (!target) {
388 pr_debug("HMAT: Memory Domain missing from SRAT\n");
389 return -EINVAL;
390 }
391 }
392 if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
393 int p_node = pxm_to_node(p->processor_PD);
394
395 if (p_node == NUMA_NO_NODE) {
396 pr_debug("HMAT: Invalid Processor Domain\n");
397 return -EINVAL;
398 }
399 target->processor_pxm = p_node;
400 }
401
402 return 0;
403}
404
405static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
406 const unsigned long end)
407{
408 struct acpi_hmat_structure *hdr = (void *)header;
409
410 if (!hdr)
411 return -EINVAL;
412
413 switch (hdr->type) {
414 case ACPI_HMAT_TYPE_PROXIMITY:
415 return hmat_parse_proximity_domain(header, end);
416 case ACPI_HMAT_TYPE_LOCALITY:
417 return hmat_parse_locality(header, end);
418 case ACPI_HMAT_TYPE_CACHE:
419 return hmat_parse_cache(header, end);
420 default:
421 return -EINVAL;
422 }
423}
424
425static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
426 const unsigned long end)
427{
428 struct acpi_srat_mem_affinity *ma = (void *)header;
429
430 if (!ma)
431 return -EINVAL;
432 if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
433 return 0;
434 alloc_memory_target(ma->proximity_domain);
435 return 0;
436}
437
438static __init u32 hmat_initiator_perf(struct memory_target *target,
439 struct memory_initiator *initiator,
440 struct acpi_hmat_locality *hmat_loc)
441{
442 unsigned int ipds, tpds, i, idx = 0, tdx = 0;
443 u32 *inits, *targs;
444 u16 *entries;
445
446 ipds = hmat_loc->number_of_initiator_Pds;
447 tpds = hmat_loc->number_of_target_Pds;
448 inits = (u32 *)(hmat_loc + 1);
449 targs = inits + ipds;
450 entries = (u16 *)(targs + tpds);
451
452 for (i = 0; i < ipds; i++) {
453 if (inits[i] == initiator->processor_pxm) {
454 idx = i;
455 break;
456 }
457 }
458
459 if (i == ipds)
460 return 0;
461
462 for (i = 0; i < tpds; i++) {
463 if (targs[i] == target->memory_pxm) {
464 tdx = i;
465 break;
466 }
467 }
468 if (i == tpds)
469 return 0;
470
471 return hmat_normalize(entries[idx * tpds + tdx],
472 hmat_loc->entry_base_unit,
473 hmat_loc->data_type);
474}
475
476static __init bool hmat_update_best(u8 type, u32 value, u32 *best)
477{
478 bool updated = false;
479
480 if (!value)
481 return false;
482
483 switch (type) {
484 case ACPI_HMAT_ACCESS_LATENCY:
485 case ACPI_HMAT_READ_LATENCY:
486 case ACPI_HMAT_WRITE_LATENCY:
487 if (!*best || *best > value) {
488 *best = value;
489 updated = true;
490 }
491 break;
492 case ACPI_HMAT_ACCESS_BANDWIDTH:
493 case ACPI_HMAT_READ_BANDWIDTH:
494 case ACPI_HMAT_WRITE_BANDWIDTH:
495 if (!*best || *best < value) {
496 *best = value;
497 updated = true;
498 }
499 break;
500 }
501
502 return updated;
503}
504
505static int initiator_cmp(void *priv, struct list_head *a, struct list_head *b)
506{
507 struct memory_initiator *ia;
508 struct memory_initiator *ib;
509 unsigned long *p_nodes = priv;
510
511 ia = list_entry(a, struct memory_initiator, node);
512 ib = list_entry(b, struct memory_initiator, node);
513
514 set_bit(ia->processor_pxm, p_nodes);
515 set_bit(ib->processor_pxm, p_nodes);
516
517 return ia->processor_pxm - ib->processor_pxm;
518}
519
520static __init void hmat_register_target_initiators(struct memory_target *target)
521{
522 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
523 struct memory_initiator *initiator;
524 unsigned int mem_nid, cpu_nid;
525 struct memory_locality *loc = NULL;
526 u32 best = 0;
527 int i;
528
529 mem_nid = pxm_to_node(target->memory_pxm);
530
531
532
533
534
535 if (target->processor_pxm != PXM_INVAL) {
536 cpu_nid = pxm_to_node(target->processor_pxm);
537 register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
538 return;
539 }
540
541 if (list_empty(&localities))
542 return;
543
544
545
546
547
548
549
550 bitmap_zero(p_nodes, MAX_NUMNODES);
551 list_sort(p_nodes, &initiators, initiator_cmp);
552 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
553 loc = localities_types[i];
554 if (!loc)
555 continue;
556
557 best = 0;
558 list_for_each_entry(initiator, &initiators, node) {
559 u32 value;
560
561 if (!test_bit(initiator->processor_pxm, p_nodes))
562 continue;
563
564 value = hmat_initiator_perf(target, initiator, loc->hmat_loc);
565 if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
566 bitmap_clear(p_nodes, 0, initiator->processor_pxm);
567 if (value != best)
568 clear_bit(initiator->processor_pxm, p_nodes);
569 }
570 if (best)
571 hmat_update_target_access(target, loc->hmat_loc->data_type, best);
572 }
573
574 for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
575 cpu_nid = pxm_to_node(i);
576 register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
577 }
578}
579
580static __init void hmat_register_target_perf(struct memory_target *target)
581{
582 unsigned mem_nid = pxm_to_node(target->memory_pxm);
583 node_set_perf_attrs(mem_nid, &target->hmem_attrs, 0);
584}
585
586static __init void hmat_register_targets(void)
587{
588 struct memory_target *target;
589
590 list_for_each_entry(target, &targets, node) {
591 hmat_register_target_initiators(target);
592 hmat_register_target_perf(target);
593 }
594}
595
596static __init void hmat_free_structures(void)
597{
598 struct memory_target *target, *tnext;
599 struct memory_locality *loc, *lnext;
600 struct memory_initiator *initiator, *inext;
601
602 list_for_each_entry_safe(target, tnext, &targets, node) {
603 list_del(&target->node);
604 kfree(target);
605 }
606
607 list_for_each_entry_safe(initiator, inext, &initiators, node) {
608 list_del(&initiator->node);
609 kfree(initiator);
610 }
611
612 list_for_each_entry_safe(loc, lnext, &localities, node) {
613 list_del(&loc->node);
614 kfree(loc);
615 }
616}
617
618static __init int hmat_init(void)
619{
620 struct acpi_table_header *tbl;
621 enum acpi_hmat_type i;
622 acpi_status status;
623
624 if (srat_disabled())
625 return 0;
626
627 status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);
628 if (ACPI_FAILURE(status))
629 return 0;
630
631 if (acpi_table_parse_entries(ACPI_SIG_SRAT,
632 sizeof(struct acpi_table_srat),
633 ACPI_SRAT_TYPE_MEMORY_AFFINITY,
634 srat_parse_mem_affinity, 0) < 0)
635 goto out_put;
636 acpi_put_table(tbl);
637
638 status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);
639 if (ACPI_FAILURE(status))
640 goto out_put;
641
642 hmat_revision = tbl->revision;
643 switch (hmat_revision) {
644 case 1:
645 case 2:
646 break;
647 default:
648 pr_notice("Ignoring HMAT: Unknown revision:%d\n", hmat_revision);
649 goto out_put;
650 }
651
652 for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
653 if (acpi_table_parse_entries(ACPI_SIG_HMAT,
654 sizeof(struct acpi_table_hmat), i,
655 hmat_parse_subtable, 0) < 0) {
656 pr_notice("Ignoring HMAT: Invalid table");
657 goto out_put;
658 }
659 }
660 hmat_register_targets();
661out_put:
662 hmat_free_structures();
663 acpi_put_table(tbl);
664 return 0;
665}
666subsys_initcall(hmat_init);
667
