1
2
3
4
5
6
7
8
9
10
11#include <linux/types.h>
12#include <linux/device.h>
13#include <linux/irq.h>
14#include <linux/irqdomain.h>
15#include <linux/msi.h>
16#include <linux/slab.h>
17#include <linux/pci.h>
18
19#include "internals.h"
20
21
22
23
24
25
26
27
28
29
30
31
32struct msi_desc *alloc_msi_entry(struct device *dev, int nvec,
33 const struct irq_affinity_desc *affinity)
34{
35 struct msi_desc *desc;
36
37 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
38 if (!desc)
39 return NULL;
40
41 INIT_LIST_HEAD(&desc->list);
42 desc->dev = dev;
43 desc->nvec_used = nvec;
44 if (affinity) {
45 desc->affinity = kmemdup(affinity,
46 nvec * sizeof(*desc->affinity), GFP_KERNEL);
47 if (!desc->affinity) {
48 kfree(desc);
49 return NULL;
50 }
51 }
52
53 return desc;
54}
55
56void free_msi_entry(struct msi_desc *entry)
57{
58 kfree(entry->affinity);
59 kfree(entry);
60}
61
62void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
63{
64 *msg = entry->msg;
65}
66
67void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)
68{
69 struct msi_desc *entry = irq_get_msi_desc(irq);
70
71 __get_cached_msi_msg(entry, msg);
72}
73EXPORT_SYMBOL_GPL(get_cached_msi_msg);
74
75static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
76 char *buf)
77{
78 struct msi_desc *entry;
79 bool is_msix = false;
80 unsigned long irq;
81 int retval;
82
83 retval = kstrtoul(attr->attr.name, 10, &irq);
84 if (retval)
85 return retval;
86
87 entry = irq_get_msi_desc(irq);
88 if (!entry)
89 return -ENODEV;
90
91 if (dev_is_pci(dev))
92 is_msix = entry->msi_attrib.is_msix;
93
94 return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi");
95}
96
97
98
99
100
101
102
103
104
105const struct attribute_group **msi_populate_sysfs(struct device *dev)
106{
107 const struct attribute_group **msi_irq_groups;
108 struct attribute **msi_attrs, *msi_attr;
109 struct device_attribute *msi_dev_attr;
110 struct attribute_group *msi_irq_group;
111 struct msi_desc *entry;
112 int ret = -ENOMEM;
113 int num_msi = 0;
114 int count = 0;
115 int i;
116
117
118 for_each_msi_entry(entry, dev)
119 num_msi += entry->nvec_used;
120 if (!num_msi)
121 return NULL;
122
123
124 msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL);
125 if (!msi_attrs)
126 return ERR_PTR(-ENOMEM);
127
128 for_each_msi_entry(entry, dev) {
129 for (i = 0; i < entry->nvec_used; i++) {
130 msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
131 if (!msi_dev_attr)
132 goto error_attrs;
133 msi_attrs[count] = &msi_dev_attr->attr;
134
135 sysfs_attr_init(&msi_dev_attr->attr);
136 msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
137 entry->irq + i);
138 if (!msi_dev_attr->attr.name)
139 goto error_attrs;
140 msi_dev_attr->attr.mode = 0444;
141 msi_dev_attr->show = msi_mode_show;
142 ++count;
143 }
144 }
145
146 msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
147 if (!msi_irq_group)
148 goto error_attrs;
149 msi_irq_group->name = "msi_irqs";
150 msi_irq_group->attrs = msi_attrs;
151
152 msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL);
153 if (!msi_irq_groups)
154 goto error_irq_group;
155 msi_irq_groups[0] = msi_irq_group;
156
157 ret = sysfs_create_groups(&dev->kobj, msi_irq_groups);
158 if (ret)
159 goto error_irq_groups;
160
161 return msi_irq_groups;
162
163error_irq_groups:
164 kfree(msi_irq_groups);
165error_irq_group:
166 kfree(msi_irq_group);
167error_attrs:
168 count = 0;
169 msi_attr = msi_attrs[count];
170 while (msi_attr) {
171 msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
172 kfree(msi_attr->name);
173 kfree(msi_dev_attr);
174 ++count;
175 msi_attr = msi_attrs[count];
176 }
177 kfree(msi_attrs);
178 return ERR_PTR(ret);
179}
180
181
182
183
184
185
186void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups)
187{
188 struct device_attribute *dev_attr;
189 struct attribute **msi_attrs;
190 int count = 0;
191
192 if (msi_irq_groups) {
193 sysfs_remove_groups(&dev->kobj, msi_irq_groups);
194 msi_attrs = msi_irq_groups[0]->attrs;
195 while (msi_attrs[count]) {
196 dev_attr = container_of(msi_attrs[count],
197 struct device_attribute, attr);
198 kfree(dev_attr->attr.name);
199 kfree(dev_attr);
200 ++count;
201 }
202 kfree(msi_attrs);
203 kfree(msi_irq_groups[0]);
204 kfree(msi_irq_groups);
205 }
206}
207
208#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
209static inline void irq_chip_write_msi_msg(struct irq_data *data,
210 struct msi_msg *msg)
211{
212 data->chip->irq_write_msi_msg(data, msg);
213}
214
215static void msi_check_level(struct irq_domain *domain, struct msi_msg *msg)
216{
217 struct msi_domain_info *info = domain->host_data;
218
219
220
221
222
223 WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) &&
224 (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) &&
225 (msg[1].address_lo || msg[1].address_hi || msg[1].data));
226}
227
228
229
230
231
232
233
234
235
236
237
238
239int msi_domain_set_affinity(struct irq_data *irq_data,
240 const struct cpumask *mask, bool force)
241{
242 struct irq_data *parent = irq_data->parent_data;
243 struct msi_msg msg[2] = { [1] = { }, };
244 int ret;
245
246 ret = parent->chip->irq_set_affinity(parent, mask, force);
247 if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) {
248 BUG_ON(irq_chip_compose_msi_msg(irq_data, msg));
249 msi_check_level(irq_data->domain, msg);
250 irq_chip_write_msi_msg(irq_data, msg);
251 }
252
253 return ret;
254}
255
256static int msi_domain_activate(struct irq_domain *domain,
257 struct irq_data *irq_data, bool early)
258{
259 struct msi_msg msg[2] = { [1] = { }, };
260
261 BUG_ON(irq_chip_compose_msi_msg(irq_data, msg));
262 msi_check_level(irq_data->domain, msg);
263 irq_chip_write_msi_msg(irq_data, msg);
264 return 0;
265}
266
267static void msi_domain_deactivate(struct irq_domain *domain,
268 struct irq_data *irq_data)
269{
270 struct msi_msg msg[2];
271
272 memset(msg, 0, sizeof(msg));
273 irq_chip_write_msi_msg(irq_data, msg);
274}
275
276static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
277 unsigned int nr_irqs, void *arg)
278{
279 struct msi_domain_info *info = domain->host_data;
280 struct msi_domain_ops *ops = info->ops;
281 irq_hw_number_t hwirq = ops->get_hwirq(info, arg);
282 int i, ret;
283
284 if (irq_find_mapping(domain, hwirq) > 0)
285 return -EEXIST;
286
287 if (domain->parent) {
288 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
289 if (ret < 0)
290 return ret;
291 }
292
293 for (i = 0; i < nr_irqs; i++) {
294 ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg);
295 if (ret < 0) {
296 if (ops->msi_free) {
297 for (i--; i > 0; i--)
298 ops->msi_free(domain, info, virq + i);
299 }
300 irq_domain_free_irqs_top(domain, virq, nr_irqs);
301 return ret;
302 }
303 }
304
305 return 0;
306}
307
308static void msi_domain_free(struct irq_domain *domain, unsigned int virq,
309 unsigned int nr_irqs)
310{
311 struct msi_domain_info *info = domain->host_data;
312 int i;
313
314 if (info->ops->msi_free) {
315 for (i = 0; i < nr_irqs; i++)
316 info->ops->msi_free(domain, info, virq + i);
317 }
318 irq_domain_free_irqs_top(domain, virq, nr_irqs);
319}
320
321static const struct irq_domain_ops msi_domain_ops = {
322 .alloc = msi_domain_alloc,
323 .free = msi_domain_free,
324 .activate = msi_domain_activate,
325 .deactivate = msi_domain_deactivate,
326};
327
328static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info,
329 msi_alloc_info_t *arg)
330{
331 return arg->hwirq;
332}
333
334static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev,
335 int nvec, msi_alloc_info_t *arg)
336{
337 memset(arg, 0, sizeof(*arg));
338 return 0;
339}
340
341static void msi_domain_ops_set_desc(msi_alloc_info_t *arg,
342 struct msi_desc *desc)
343{
344 arg->desc = desc;
345}
346
347static int msi_domain_ops_init(struct irq_domain *domain,
348 struct msi_domain_info *info,
349 unsigned int virq, irq_hw_number_t hwirq,
350 msi_alloc_info_t *arg)
351{
352 irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip,
353 info->chip_data);
354 if (info->handler && info->handler_name) {
355 __irq_set_handler(virq, info->handler, 0, info->handler_name);
356 if (info->handler_data)
357 irq_set_handler_data(virq, info->handler_data);
358 }
359 return 0;
360}
361
362static int msi_domain_ops_check(struct irq_domain *domain,
363 struct msi_domain_info *info,
364 struct device *dev)
365{
366 return 0;
367}
368
369static struct msi_domain_ops msi_domain_ops_default = {
370 .get_hwirq = msi_domain_ops_get_hwirq,
371 .msi_init = msi_domain_ops_init,
372 .msi_check = msi_domain_ops_check,
373 .msi_prepare = msi_domain_ops_prepare,
374 .set_desc = msi_domain_ops_set_desc,
375 .domain_alloc_irqs = __msi_domain_alloc_irqs,
376 .domain_free_irqs = __msi_domain_free_irqs,
377};
378
379static void msi_domain_update_dom_ops(struct msi_domain_info *info)
380{
381 struct msi_domain_ops *ops = info->ops;
382
383 if (ops == NULL) {
384 info->ops = &msi_domain_ops_default;
385 return;
386 }
387
388 if (ops->domain_alloc_irqs == NULL)
389 ops->domain_alloc_irqs = msi_domain_ops_default.domain_alloc_irqs;
390 if (ops->domain_free_irqs == NULL)
391 ops->domain_free_irqs = msi_domain_ops_default.domain_free_irqs;
392
393 if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS))
394 return;
395
396 if (ops->get_hwirq == NULL)
397 ops->get_hwirq = msi_domain_ops_default.get_hwirq;
398 if (ops->msi_init == NULL)
399 ops->msi_init = msi_domain_ops_default.msi_init;
400 if (ops->msi_check == NULL)
401 ops->msi_check = msi_domain_ops_default.msi_check;
402 if (ops->msi_prepare == NULL)
403 ops->msi_prepare = msi_domain_ops_default.msi_prepare;
404 if (ops->set_desc == NULL)
405 ops->set_desc = msi_domain_ops_default.set_desc;
406}
407
408static void msi_domain_update_chip_ops(struct msi_domain_info *info)
409{
410 struct irq_chip *chip = info->chip;
411
412 BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask);
413 if (!chip->irq_set_affinity)
414 chip->irq_set_affinity = msi_domain_set_affinity;
415}
416
417
418
419
420
421
422
423
424
425struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode,
426 struct msi_domain_info *info,
427 struct irq_domain *parent)
428{
429 struct irq_domain *domain;
430
431 msi_domain_update_dom_ops(info);
432 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
433 msi_domain_update_chip_ops(info);
434
435 domain = irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0,
436 fwnode, &msi_domain_ops, info);
437
438 if (domain && !domain->name && info->chip)
439 domain->name = info->chip->name;
440
441 return domain;
442}
443
444int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,
445 int nvec, msi_alloc_info_t *arg)
446{
447 struct msi_domain_info *info = domain->host_data;
448 struct msi_domain_ops *ops = info->ops;
449 int ret;
450
451 ret = ops->msi_check(domain, info, dev);
452 if (ret == 0)
453 ret = ops->msi_prepare(domain, dev, nvec, arg);
454
455 return ret;
456}
457
458int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev,
459 int virq, int nvec, msi_alloc_info_t *arg)
460{
461 struct msi_domain_info *info = domain->host_data;
462 struct msi_domain_ops *ops = info->ops;
463 struct msi_desc *desc;
464 int ret = 0;
465
466 for_each_msi_entry(desc, dev) {
467
468 if (WARN_ON(!desc->irq || desc->nvec_used != 1)) {
469 ret = -EINVAL;
470 break;
471 }
472
473 if (!(desc->irq >= virq && desc->irq < (virq + nvec)))
474 continue;
475
476 ops->set_desc(arg, desc);
477
478 ret = irq_domain_alloc_irqs_hierarchy(domain, desc->irq, 1,
479 arg);
480 if (ret)
481 break;
482
483 irq_set_msi_desc_off(desc->irq, 0, desc);
484 }
485
486 if (ret) {
487
488 for_each_msi_entry(desc, dev) {
489 if (!(desc->irq >= virq && desc->irq < (virq + nvec)))
490 continue;
491
492 irq_domain_free_irqs_common(domain, desc->irq, 1);
493 }
494 }
495
496 return ret;
497}
498
499
500
501
502
503
504
505
506
507
508
509
510static bool msi_check_reservation_mode(struct irq_domain *domain,
511 struct msi_domain_info *info,
512 struct device *dev)
513{
514 struct msi_desc *desc;
515
516 switch(domain->bus_token) {
517 case DOMAIN_BUS_PCI_MSI:
518 case DOMAIN_BUS_VMD_MSI:
519 break;
520 default:
521 return false;
522 }
523
524 if (!(info->flags & MSI_FLAG_MUST_REACTIVATE))
525 return false;
526
527 if (IS_ENABLED(CONFIG_PCI_MSI) && pci_msi_ignore_mask)
528 return false;
529
530
531
532
533
534 desc = first_msi_entry(dev);
535 return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit;
536}
537
538int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
539 int nvec)
540{
541 struct msi_domain_info *info = domain->host_data;
542 struct msi_domain_ops *ops = info->ops;
543 struct irq_data *irq_data;
544 struct msi_desc *desc;
545 msi_alloc_info_t arg = { };
546 int i, ret, virq;
547 bool can_reserve;
548
549 ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg);
550 if (ret)
551 return ret;
552
553 for_each_msi_entry(desc, dev) {
554 ops->set_desc(&arg, desc);
555
556 virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used,
557 dev_to_node(dev), &arg, false,
558 desc->affinity);
559 if (virq < 0) {
560 ret = -ENOSPC;
561 if (ops->handle_error)
562 ret = ops->handle_error(domain, desc, ret);
563 if (ops->msi_finish)
564 ops->msi_finish(&arg, ret);
565 return ret;
566 }
567
568 for (i = 0; i < desc->nvec_used; i++) {
569 irq_set_msi_desc_off(virq, i, desc);
570 irq_debugfs_copy_devname(virq + i, dev);
571 }
572 }
573
574 if (ops->msi_finish)
575 ops->msi_finish(&arg, 0);
576
577 can_reserve = msi_check_reservation_mode(domain, info, dev);
578
579
580
581
582
583
584 if (!(info->flags & MSI_FLAG_ACTIVATE_EARLY))
585 goto skip_activate;
586
587 for_each_msi_vector(desc, i, dev) {
588 if (desc->irq == i) {
589 virq = desc->irq;
590 dev_dbg(dev, "irq [%d-%d] for MSI\n",
591 virq, virq + desc->nvec_used - 1);
592 }
593
594 irq_data = irq_domain_get_irq_data(domain, i);
595 if (!can_reserve) {
596 irqd_clr_can_reserve(irq_data);
597 if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK)
598 irqd_set_msi_nomask_quirk(irq_data);
599 }
600 ret = irq_domain_activate_irq(irq_data, can_reserve);
601 if (ret)
602 goto cleanup;
603 }
604
605skip_activate:
606
607
608
609
610 if (can_reserve) {
611 for_each_msi_vector(desc, i, dev) {
612 irq_data = irq_domain_get_irq_data(domain, i);
613 irqd_clr_activated(irq_data);
614 }
615 }
616 return 0;
617
618cleanup:
619 msi_domain_free_irqs(domain, dev);
620 return ret;
621}
622
623
624
625
626
627
628
629
630
631
632int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
633 int nvec)
634{
635 struct msi_domain_info *info = domain->host_data;
636 struct msi_domain_ops *ops = info->ops;
637
638 return ops->domain_alloc_irqs(domain, dev, nvec);
639}
640
641void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
642{
643 struct irq_data *irq_data;
644 struct msi_desc *desc;
645 int i;
646
647 for_each_msi_vector(desc, i, dev) {
648 irq_data = irq_domain_get_irq_data(domain, i);
649 if (irqd_is_activated(irq_data))
650 irq_domain_deactivate_irq(irq_data);
651 }
652
653 for_each_msi_entry(desc, dev) {
654
655
656
657
658
659 if (desc->irq) {
660 irq_domain_free_irqs(desc->irq, desc->nvec_used);
661 desc->irq = 0;
662 }
663 }
664}
665
666
667
668
669
670
671
672void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
673{
674 struct msi_domain_info *info = domain->host_data;
675 struct msi_domain_ops *ops = info->ops;
676
677 return ops->domain_free_irqs(domain, dev);
678}
679
680
681
682
683
684
685
686struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain)
687{
688 return (struct msi_domain_info *)domain->host_data;
689}
690
691#endif
692