1
2
3
4
5
6
7
8
9
10#define pr_fmt(fmt) "genirq/ipi: " fmt
11
12#include <linux/irqdomain.h>
13#include <linux/irq.h>
14
15
16
17
18
19
20
21
22
23
24int irq_reserve_ipi(struct irq_domain *domain,
25 const struct cpumask *dest)
26{
27 unsigned int nr_irqs, offset;
28 struct irq_data *data;
29 int virq, i;
30
31 if (!domain ||!irq_domain_is_ipi(domain)) {
32 pr_warn("Reservation on a non IPI domain\n");
33 return -EINVAL;
34 }
35
36 if (!cpumask_subset(dest, cpu_possible_mask)) {
37 pr_warn("Reservation is not in possible_cpu_mask\n");
38 return -EINVAL;
39 }
40
41 nr_irqs = cpumask_weight(dest);
42 if (!nr_irqs) {
43 pr_warn("Reservation for empty destination mask\n");
44 return -EINVAL;
45 }
46
47 if (irq_domain_is_ipi_single(domain)) {
48
49
50
51
52
53
54 nr_irqs = 1;
55 offset = 0;
56 } else {
57 unsigned int next;
58
59
60
61
62
63
64
65 offset = cpumask_first(dest);
66
67
68
69
70 next = cpumask_next_zero(offset, dest);
71 if (next < nr_cpu_ids)
72 next = cpumask_next(next, dest);
73 if (next < nr_cpu_ids) {
74 pr_warn("Destination mask has holes\n");
75 return -EINVAL;
76 }
77 }
78
79 virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
80 if (virq <= 0) {
81 pr_warn("Can't reserve IPI, failed to alloc descs\n");
82 return -ENOMEM;
83 }
84
85 virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
86 (void *) dest, true, NULL);
87
88 if (virq <= 0) {
89 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
90 goto free_descs;
91 }
92
93 for (i = 0; i < nr_irqs; i++) {
94 data = irq_get_irq_data(virq + i);
95 cpumask_copy(data->common->affinity, dest);
96 data->common->ipi_offset = offset;
97 irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
98 }
99 return virq;
100
101free_descs:
102 irq_free_descs(virq, nr_irqs);
103 return -EBUSY;
104}
105
106
107
108
109
110
111
112
113
114
115
116int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
117{
118 struct irq_data *data = irq_get_irq_data(irq);
119 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
120 struct irq_domain *domain;
121 unsigned int nr_irqs;
122
123 if (!irq || !data || !ipimask)
124 return -EINVAL;
125
126 domain = data->domain;
127 if (WARN_ON(domain == NULL))
128 return -EINVAL;
129
130 if (!irq_domain_is_ipi(domain)) {
131 pr_warn("Trying to destroy a non IPI domain!\n");
132 return -EINVAL;
133 }
134
135 if (WARN_ON(!cpumask_subset(dest, ipimask)))
136
137
138
139
140 return -EINVAL;
141
142 if (irq_domain_is_ipi_per_cpu(domain)) {
143 irq = irq + cpumask_first(dest) - data->common->ipi_offset;
144 nr_irqs = cpumask_weight(dest);
145 } else {
146 nr_irqs = 1;
147 }
148
149 irq_domain_free_irqs(irq, nr_irqs);
150 return 0;
151}
152
153
154
155
156
157
158
159
160
161
162
163irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
164{
165 struct irq_data *data = irq_get_irq_data(irq);
166 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
167
168 if (!data || !ipimask || cpu >= nr_cpu_ids)
169 return INVALID_HWIRQ;
170
171 if (!cpumask_test_cpu(cpu, ipimask))
172 return INVALID_HWIRQ;
173
174
175
176
177
178
179
180 if (irq_domain_is_ipi_per_cpu(data->domain))
181 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
182
183 return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
184}
185EXPORT_SYMBOL_GPL(ipi_get_hwirq);
186
187static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
188 const struct cpumask *dest, unsigned int cpu)
189{
190 struct cpumask *ipimask = irq_data_get_affinity_mask(data);
191
192 if (!chip || !ipimask)
193 return -EINVAL;
194
195 if (!chip->ipi_send_single && !chip->ipi_send_mask)
196 return -EINVAL;
197
198 if (cpu >= nr_cpu_ids)
199 return -EINVAL;
200
201 if (dest) {
202 if (!cpumask_subset(dest, ipimask))
203 return -EINVAL;
204 } else {
205 if (!cpumask_test_cpu(cpu, ipimask))
206 return -EINVAL;
207 }
208 return 0;
209}
210
211
212
213
214
215
216
217
218
219
220
221
222int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
223{
224 struct irq_data *data = irq_desc_get_irq_data(desc);
225 struct irq_chip *chip = irq_data_get_irq_chip(data);
226
227#ifdef DEBUG
228
229
230
231
232
233 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
234 return -EINVAL;
235#endif
236 if (!chip->ipi_send_single) {
237 chip->ipi_send_mask(data, cpumask_of(cpu));
238 return 0;
239 }
240
241
242 if (irq_domain_is_ipi_per_cpu(data->domain) &&
243 cpu != data->common->ipi_offset) {
244
245 unsigned irq = data->irq + cpu - data->common->ipi_offset;
246
247 data = irq_get_irq_data(irq);
248 }
249 chip->ipi_send_single(data, cpu);
250 return 0;
251}
252
253
254
255
256
257
258
259
260
261
262
263
264int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
265{
266 struct irq_data *data = irq_desc_get_irq_data(desc);
267 struct irq_chip *chip = irq_data_get_irq_chip(data);
268 unsigned int cpu;
269
270#ifdef DEBUG
271
272
273
274
275
276 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
277 return -EINVAL;
278#endif
279 if (chip->ipi_send_mask) {
280 chip->ipi_send_mask(data, dest);
281 return 0;
282 }
283
284 if (irq_domain_is_ipi_per_cpu(data->domain)) {
285 unsigned int base = data->irq;
286
287 for_each_cpu(cpu, dest) {
288 unsigned irq = base + cpu - data->common->ipi_offset;
289
290 data = irq_get_irq_data(irq);
291 chip->ipi_send_single(data, cpu);
292 }
293 } else {
294 for_each_cpu(cpu, dest)
295 chip->ipi_send_single(data, cpu);
296 }
297 return 0;
298}
299
300
301
302
303
304
305
306
307
308int ipi_send_single(unsigned int virq, unsigned int cpu)
309{
310 struct irq_desc *desc = irq_to_desc(virq);
311 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
312 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
313
314 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
315 return -EINVAL;
316
317 return __ipi_send_single(desc, cpu);
318}
319EXPORT_SYMBOL_GPL(ipi_send_single);
320
321
322
323
324
325
326
327
328
329int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
330{
331 struct irq_desc *desc = irq_to_desc(virq);
332 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
333 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
334
335 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
336 return -EINVAL;
337
338 return __ipi_send_mask(desc, dest);
339}
340EXPORT_SYMBOL_GPL(ipi_send_mask);
341