1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47#include <linux/kernel.h>
48#include <linux/init.h>
49#include <linux/types.h>
50#include <linux/pci.h>
51#include <linux/slab.h>
52#include <linux/errno.h>
53#include <linux/interrupt.h>
54#include "adf_accel_devices.h"
55#include "adf_common_drv.h"
56#include "adf_cfg.h"
57#include "adf_cfg_strings.h"
58#include "adf_cfg_common.h"
59#include "adf_transport_access_macros.h"
60#include "adf_transport_internal.h"
61
62static int adf_enable_msix(struct adf_accel_dev *accel_dev)
63{
64 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
65 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
66 u32 msix_num_entries = 1;
67
68
69 if (!accel_dev->pf.vf_info) {
70 int i;
71
72 msix_num_entries += hw_data->num_banks;
73 for (i = 0; i < msix_num_entries; i++)
74 pci_dev_info->msix_entries.entries[i].entry = i;
75 } else {
76 pci_dev_info->msix_entries.entries[0].entry =
77 hw_data->num_banks;
78 }
79
80 if (pci_enable_msix_exact(pci_dev_info->pci_dev,
81 pci_dev_info->msix_entries.entries,
82 msix_num_entries)) {
83 dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n");
84 return -EFAULT;
85 }
86 return 0;
87}
88
89static void adf_disable_msix(struct adf_accel_pci *pci_dev_info)
90{
91 pci_disable_msix(pci_dev_info->pci_dev);
92}
93
94static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
95{
96 struct adf_etr_bank_data *bank = bank_ptr;
97
98 WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0);
99 tasklet_hi_schedule(&bank->resp_handler);
100 return IRQ_HANDLED;
101}
102
103static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
104{
105 struct adf_accel_dev *accel_dev = dev_ptr;
106
107#ifdef CONFIG_PCI_IOV
108
109 if (accel_dev->pf.vf_info) {
110 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
111 struct adf_bar *pmisc =
112 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
113 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
114 u32 vf_mask;
115
116
117 vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU5) &
118 0x0000FFFF) << 16) |
119 ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU3) &
120 0x01FFFE00) >> 9);
121
122 if (vf_mask) {
123 struct adf_accel_vf_info *vf_info;
124 bool irq_handled = false;
125 int i;
126
127
128 adf_disable_vf2pf_interrupts(accel_dev, vf_mask);
129
130
131
132
133
134
135 for_each_set_bit(i, (const unsigned long *)&vf_mask,
136 (sizeof(vf_mask) * BITS_PER_BYTE)) {
137 vf_info = accel_dev->pf.vf_info + i;
138
139 if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
140 dev_info(&GET_DEV(accel_dev),
141 "Too many ints from VF%d\n",
142 vf_info->vf_nr + 1);
143 continue;
144 }
145
146
147 tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet);
148 irq_handled = true;
149 }
150
151 if (irq_handled)
152 return IRQ_HANDLED;
153 }
154 }
155#endif
156
157 dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
158 accel_dev->accel_id);
159
160 return IRQ_NONE;
161}
162
163static int adf_request_irqs(struct adf_accel_dev *accel_dev)
164{
165 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
166 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
167 struct msix_entry *msixe = pci_dev_info->msix_entries.entries;
168 struct adf_etr_data *etr_data = accel_dev->transport;
169 int ret, i = 0;
170 char *name;
171
172
173 if (!accel_dev->pf.vf_info) {
174 for (i = 0; i < hw_data->num_banks; i++) {
175 struct adf_etr_bank_data *bank = &etr_data->banks[i];
176 unsigned int cpu, cpus = num_online_cpus();
177
178 name = *(pci_dev_info->msix_entries.names + i);
179 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
180 "qat%d-bundle%d", accel_dev->accel_id, i);
181 ret = request_irq(msixe[i].vector,
182 adf_msix_isr_bundle, 0, name, bank);
183 if (ret) {
184 dev_err(&GET_DEV(accel_dev),
185 "failed to enable irq %d for %s\n",
186 msixe[i].vector, name);
187 return ret;
188 }
189
190 cpu = ((accel_dev->accel_id * hw_data->num_banks) +
191 i) % cpus;
192 irq_set_affinity_hint(msixe[i].vector,
193 get_cpu_mask(cpu));
194 }
195 }
196
197
198 name = *(pci_dev_info->msix_entries.names + i);
199 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
200 "qat%d-ae-cluster", accel_dev->accel_id);
201 ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev);
202 if (ret) {
203 dev_err(&GET_DEV(accel_dev),
204 "failed to enable irq %d, for %s\n",
205 msixe[i].vector, name);
206 return ret;
207 }
208 return ret;
209}
210
211static void adf_free_irqs(struct adf_accel_dev *accel_dev)
212{
213 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
214 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
215 struct msix_entry *msixe = pci_dev_info->msix_entries.entries;
216 struct adf_etr_data *etr_data = accel_dev->transport;
217 int i = 0;
218
219 if (pci_dev_info->msix_entries.num_entries > 1) {
220 for (i = 0; i < hw_data->num_banks; i++) {
221 irq_set_affinity_hint(msixe[i].vector, NULL);
222 free_irq(msixe[i].vector, &etr_data->banks[i]);
223 }
224 }
225 irq_set_affinity_hint(msixe[i].vector, NULL);
226 free_irq(msixe[i].vector, accel_dev);
227}
228
229static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev)
230{
231 int i;
232 char **names;
233 struct msix_entry *entries;
234 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
235 u32 msix_num_entries = 1;
236
237
238 if (!accel_dev->pf.vf_info)
239 msix_num_entries += hw_data->num_banks;
240
241 entries = kzalloc_node(msix_num_entries * sizeof(*entries),
242 GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));
243 if (!entries)
244 return -ENOMEM;
245
246 names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL);
247 if (!names) {
248 kfree(entries);
249 return -ENOMEM;
250 }
251 for (i = 0; i < msix_num_entries; i++) {
252 *(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
253 if (!(*(names + i)))
254 goto err;
255 }
256 accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries;
257 accel_dev->accel_pci_dev.msix_entries.entries = entries;
258 accel_dev->accel_pci_dev.msix_entries.names = names;
259 return 0;
260err:
261 for (i = 0; i < msix_num_entries; i++)
262 kfree(*(names + i));
263 kfree(entries);
264 kfree(names);
265 return -ENOMEM;
266}
267
268static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev)
269{
270 char **names = accel_dev->accel_pci_dev.msix_entries.names;
271 int i;
272
273 kfree(accel_dev->accel_pci_dev.msix_entries.entries);
274 for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++)
275 kfree(*(names + i));
276 kfree(names);
277}
278
279static int adf_setup_bh(struct adf_accel_dev *accel_dev)
280{
281 struct adf_etr_data *priv_data = accel_dev->transport;
282 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
283 int i;
284
285 for (i = 0; i < hw_data->num_banks; i++)
286 tasklet_init(&priv_data->banks[i].resp_handler,
287 adf_response_handler,
288 (unsigned long)&priv_data->banks[i]);
289 return 0;
290}
291
292static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
293{
294 struct adf_etr_data *priv_data = accel_dev->transport;
295 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
296 int i;
297
298 for (i = 0; i < hw_data->num_banks; i++) {
299 tasklet_disable(&priv_data->banks[i].resp_handler);
300 tasklet_kill(&priv_data->banks[i].resp_handler);
301 }
302}
303
304
305
306
307
308
309
310void adf_isr_resource_free(struct adf_accel_dev *accel_dev)
311{
312 adf_free_irqs(accel_dev);
313 adf_cleanup_bh(accel_dev);
314 adf_disable_msix(&accel_dev->accel_pci_dev);
315 adf_isr_free_msix_entry_table(accel_dev);
316}
317EXPORT_SYMBOL_GPL(adf_isr_resource_free);
318
319
320
321
322
323
324
325
326
327int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
328{
329 int ret;
330
331 ret = adf_isr_alloc_msix_entry_table(accel_dev);
332 if (ret)
333 return ret;
334 if (adf_enable_msix(accel_dev))
335 goto err_out;
336
337 if (adf_setup_bh(accel_dev))
338 goto err_out;
339
340 if (adf_request_irqs(accel_dev))
341 goto err_out;
342
343 return 0;
344err_out:
345 adf_isr_resource_free(accel_dev);
346 return -EFAULT;
347}
348EXPORT_SYMBOL_GPL(adf_isr_resource_alloc);
349
