1
2
3
4
5
6
7#include <linux/interrupt.h>
8#include <linux/irq.h>
9#include <linux/irqdomain.h>
10#include <linux/msi.h>
11#include <linux/sched.h>
12
13#include <linux/irqchip/arm-gic-v4.h>
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86static struct irq_domain *gic_domain;
87static const struct irq_domain_ops *vpe_domain_ops;
88static const struct irq_domain_ops *sgi_domain_ops;
89
90#ifdef CONFIG_ARM64
91#include <asm/cpufeature.h>
92
93bool gic_cpuif_has_vsgi(void)
94{
95 unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
96
97 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_GIC_SHIFT);
98
99 return fld >= 0x3;
100}
101#else
102bool gic_cpuif_has_vsgi(void)
103{
104 return false;
105}
106#endif
107
108static bool has_v4_1(void)
109{
110 return !!sgi_domain_ops;
111}
112
113static bool has_v4_1_sgi(void)
114{
115 return has_v4_1() && gic_cpuif_has_vsgi();
116}
117
118static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
119{
120 char *name;
121 int sgi_base;
122
123 if (!has_v4_1_sgi())
124 return 0;
125
126 name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current));
127 if (!name)
128 goto err;
129
130 vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx);
131 if (!vpe->fwnode)
132 goto err;
133
134 kfree(name);
135 name = NULL;
136
137 vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16,
138 sgi_domain_ops, vpe);
139 if (!vpe->sgi_domain)
140 goto err;
141
142 sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16,
143 NUMA_NO_NODE, vpe,
144 false, NULL);
145 if (sgi_base <= 0)
146 goto err;
147
148 return 0;
149
150err:
151 if (vpe->sgi_domain)
152 irq_domain_remove(vpe->sgi_domain);
153 if (vpe->fwnode)
154 irq_domain_free_fwnode(vpe->fwnode);
155 kfree(name);
156 return -ENOMEM;
157}
158
159int its_alloc_vcpu_irqs(struct its_vm *vm)
160{
161 int vpe_base_irq, i;
162
163 vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
164 task_pid_nr(current));
165 if (!vm->fwnode)
166 goto err;
167
168 vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
169 vm->fwnode, vpe_domain_ops,
170 vm);
171 if (!vm->domain)
172 goto err;
173
174 for (i = 0; i < vm->nr_vpes; i++) {
175 vm->vpes[i]->its_vm = vm;
176 vm->vpes[i]->idai = true;
177 }
178
179 vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
180 NUMA_NO_NODE, vm,
181 false, NULL);
182 if (vpe_base_irq <= 0)
183 goto err;
184
185 for (i = 0; i < vm->nr_vpes; i++) {
186 int ret;
187 vm->vpes[i]->irq = vpe_base_irq + i;
188 ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
189 if (ret)
190 goto err;
191 }
192
193 return 0;
194
195err:
196 if (vm->domain)
197 irq_domain_remove(vm->domain);
198 if (vm->fwnode)
199 irq_domain_free_fwnode(vm->fwnode);
200
201 return -ENOMEM;
202}
203
204static void its_free_sgi_irqs(struct its_vm *vm)
205{
206 int i;
207
208 if (!has_v4_1_sgi())
209 return;
210
211 for (i = 0; i < vm->nr_vpes; i++) {
212 unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
213
214 if (WARN_ON(!irq))
215 continue;
216
217 irq_domain_free_irqs(irq, 16);
218 irq_domain_remove(vm->vpes[i]->sgi_domain);
219 irq_domain_free_fwnode(vm->vpes[i]->fwnode);
220 }
221}
222
223void its_free_vcpu_irqs(struct its_vm *vm)
224{
225 its_free_sgi_irqs(vm);
226 irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
227 irq_domain_remove(vm->domain);
228 irq_domain_free_fwnode(vm->fwnode);
229}
230
231static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
232{
233 return irq_set_vcpu_affinity(vpe->irq, info);
234}
235
236int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
237{
238 struct irq_desc *desc = irq_to_desc(vpe->irq);
239 struct its_cmd_info info = { };
240 int ret;
241
242 WARN_ON(preemptible());
243
244 info.cmd_type = DESCHEDULE_VPE;
245 if (has_v4_1()) {
246
247 info.req_db = db;
248 } else {
249
250 while (db && irqd_irq_disabled(&desc->irq_data))
251 enable_irq(vpe->irq);
252 }
253
254 ret = its_send_vpe_cmd(vpe, &info);
255 if (!ret)
256 vpe->resident = false;
257
258 vpe->ready = false;
259
260 return ret;
261}
262
263int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
264{
265 struct its_cmd_info info = { };
266 int ret;
267
268 WARN_ON(preemptible());
269
270 info.cmd_type = SCHEDULE_VPE;
271 if (has_v4_1()) {
272 info.g0en = g0en;
273 info.g1en = g1en;
274 } else {
275
276 disable_irq_nosync(vpe->irq);
277 }
278
279 ret = its_send_vpe_cmd(vpe, &info);
280 if (!ret)
281 vpe->resident = true;
282
283 return ret;
284}
285
286int its_commit_vpe(struct its_vpe *vpe)
287{
288 struct its_cmd_info info = {
289 .cmd_type = COMMIT_VPE,
290 };
291 int ret;
292
293 WARN_ON(preemptible());
294
295 ret = its_send_vpe_cmd(vpe, &info);
296 if (!ret)
297 vpe->ready = true;
298
299 return ret;
300}
301
302
303int its_invall_vpe(struct its_vpe *vpe)
304{
305 struct its_cmd_info info = {
306 .cmd_type = INVALL_VPE,
307 };
308
309 return its_send_vpe_cmd(vpe, &info);
310}
311
312int its_map_vlpi(int irq, struct its_vlpi_map *map)
313{
314 struct its_cmd_info info = {
315 .cmd_type = MAP_VLPI,
316 {
317 .map = map,
318 },
319 };
320 int ret;
321
322
323
324
325
326 irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
327
328 ret = irq_set_vcpu_affinity(irq, &info);
329 if (ret)
330 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
331
332 return ret;
333}
334
335int its_get_vlpi(int irq, struct its_vlpi_map *map)
336{
337 struct its_cmd_info info = {
338 .cmd_type = GET_VLPI,
339 {
340 .map = map,
341 },
342 };
343
344 return irq_set_vcpu_affinity(irq, &info);
345}
346
347int its_unmap_vlpi(int irq)
348{
349 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
350 return irq_set_vcpu_affinity(irq, NULL);
351}
352
353int its_prop_update_vlpi(int irq, u8 config, bool inv)
354{
355 struct its_cmd_info info = {
356 .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
357 {
358 .config = config,
359 },
360 };
361
362 return irq_set_vcpu_affinity(irq, &info);
363}
364
365int its_prop_update_vsgi(int irq, u8 priority, bool group)
366{
367 struct its_cmd_info info = {
368 .cmd_type = PROP_UPDATE_VSGI,
369 {
370 .priority = priority,
371 .group = group,
372 },
373 };
374
375 return irq_set_vcpu_affinity(irq, &info);
376}
377
378int its_init_v4(struct irq_domain *domain,
379 const struct irq_domain_ops *vpe_ops,
380 const struct irq_domain_ops *sgi_ops)
381{
382 if (domain) {
383 pr_info("ITS: Enabling GICv4 support\n");
384 gic_domain = domain;
385 vpe_domain_ops = vpe_ops;
386 sgi_domain_ops = sgi_ops;
387 return 0;
388 }
389
390 pr_err("ITS: No GICv4 VPE domain allocated\n");
391 return -ENODEV;
392}
393