1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47#include <linux/module.h>
48#include <linux/slab.h>
49#include "adf_accel_devices.h"
50#include "adf_common_drv.h"
51#include "adf_transport.h"
52#include "adf_transport_access_macros.h"
53#include "adf_cfg.h"
54#include "adf_cfg_strings.h"
55#include "qat_crypto.h"
56#include "icp_qat_fw.h"
57
58#define SEC ADF_KERNEL_SEC
59
60static struct service_hndl qat_crypto;
61
62void qat_crypto_put_instance(struct qat_crypto_instance *inst)
63{
64 atomic_dec(&inst->refctr);
65 adf_dev_put(inst->accel_dev);
66}
67
68static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev)
69{
70 struct qat_crypto_instance *inst, *tmp;
71 int i;
72
73 list_for_each_entry_safe(inst, tmp, &accel_dev->crypto_list, list) {
74 for (i = 0; i < atomic_read(&inst->refctr); i++)
75 qat_crypto_put_instance(inst);
76
77 if (inst->sym_tx)
78 adf_remove_ring(inst->sym_tx);
79
80 if (inst->sym_rx)
81 adf_remove_ring(inst->sym_rx);
82
83 if (inst->pke_tx)
84 adf_remove_ring(inst->pke_tx);
85
86 if (inst->pke_rx)
87 adf_remove_ring(inst->pke_rx);
88
89 list_del(&inst->list);
90 kfree(inst);
91 }
92 return 0;
93}
94
95struct qat_crypto_instance *qat_crypto_get_instance_node(int node)
96{
97 struct adf_accel_dev *accel_dev = NULL, *tmp_dev;
98 struct qat_crypto_instance *inst = NULL, *tmp_inst;
99 unsigned long best = ~0;
100
101 list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) {
102 unsigned long ctr;
103
104 if ((node == dev_to_node(&GET_DEV(tmp_dev)) ||
105 dev_to_node(&GET_DEV(tmp_dev)) < 0) &&
106 adf_dev_started(tmp_dev) &&
107 !list_empty(&tmp_dev->crypto_list)) {
108 ctr = atomic_read(&tmp_dev->ref_count);
109 if (best > ctr) {
110 accel_dev = tmp_dev;
111 best = ctr;
112 }
113 }
114 }
115
116 if (!accel_dev) {
117 pr_info("QAT: Could not find a device on node %d\n", node);
118
119 list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) {
120 if (adf_dev_started(tmp_dev) &&
121 !list_empty(&tmp_dev->crypto_list)) {
122 accel_dev = tmp_dev;
123 break;
124 }
125 }
126 }
127
128 if (!accel_dev)
129 return NULL;
130
131 best = ~0;
132 list_for_each_entry(tmp_inst, &accel_dev->crypto_list, list) {
133 unsigned long ctr;
134
135 ctr = atomic_read(&tmp_inst->refctr);
136 if (best > ctr) {
137 inst = tmp_inst;
138 best = ctr;
139 }
140 }
141 if (inst) {
142 if (adf_dev_get(accel_dev)) {
143 dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n");
144 return NULL;
145 }
146 atomic_inc(&inst->refctr);
147 }
148 return inst;
149}
150
151
152
153
154
155
156
157
158
159
160int qat_crypto_dev_config(struct adf_accel_dev *accel_dev)
161{
162 int cpus = num_online_cpus();
163 int banks = GET_MAX_BANKS(accel_dev);
164 int instances = min(cpus, banks);
165 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
166 int i;
167 unsigned long val;
168
169 if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
170 goto err;
171 if (adf_cfg_section_add(accel_dev, "Accelerator0"))
172 goto err;
173 for (i = 0; i < instances; i++) {
174 val = i;
175 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
176 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
177 key, (void *)&val, ADF_DEC))
178 goto err;
179
180 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
181 i);
182 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
183 key, (void *)&val, ADF_DEC))
184 goto err;
185
186 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
187 val = 128;
188 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
189 key, (void *)&val, ADF_DEC))
190 goto err;
191
192 val = 512;
193 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
194 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
195 key, (void *)&val, ADF_DEC))
196 goto err;
197
198 val = 0;
199 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
200 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
201 key, (void *)&val, ADF_DEC))
202 goto err;
203
204 val = 2;
205 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
206 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
207 key, (void *)&val, ADF_DEC))
208 goto err;
209
210 val = 8;
211 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
212 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
213 key, (void *)&val, ADF_DEC))
214 goto err;
215
216 val = 10;
217 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
218 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
219 key, (void *)&val, ADF_DEC))
220 goto err;
221
222 val = ADF_COALESCING_DEF_TIME;
223 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
224 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
225 key, (void *)&val, ADF_DEC))
226 goto err;
227 }
228
229 val = i;
230 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
231 ADF_NUM_CY, (void *)&val, ADF_DEC))
232 goto err;
233
234 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
235 return 0;
236err:
237 dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
238 return -EINVAL;
239}
240EXPORT_SYMBOL_GPL(qat_crypto_dev_config);
241
242static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev)
243{
244 int i;
245 unsigned long bank;
246 unsigned long num_inst, num_msg_sym, num_msg_asym;
247 int msg_size;
248 struct qat_crypto_instance *inst;
249 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
250 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
251
252 INIT_LIST_HEAD(&accel_dev->crypto_list);
253 strlcpy(key, ADF_NUM_CY, sizeof(key));
254 if (adf_cfg_get_param_value(accel_dev, SEC, key, val))
255 return -EFAULT;
256
257 if (kstrtoul(val, 0, &num_inst))
258 return -EFAULT;
259
260 for (i = 0; i < num_inst; i++) {
261 inst = kzalloc_node(sizeof(*inst), GFP_KERNEL,
262 dev_to_node(&GET_DEV(accel_dev)));
263 if (!inst)
264 goto err;
265
266 list_add_tail(&inst->list, &accel_dev->crypto_list);
267 inst->id = i;
268 atomic_set(&inst->refctr, 0);
269 inst->accel_dev = accel_dev;
270 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
271 if (adf_cfg_get_param_value(accel_dev, SEC, key, val))
272 goto err;
273
274 if (kstrtoul(val, 10, &bank))
275 goto err;
276 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
277 if (adf_cfg_get_param_value(accel_dev, SEC, key, val))
278 goto err;
279
280 if (kstrtoul(val, 10, &num_msg_sym))
281 goto err;
282
283 num_msg_sym = num_msg_sym >> 1;
284
285 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
286 if (adf_cfg_get_param_value(accel_dev, SEC, key, val))
287 goto err;
288
289 if (kstrtoul(val, 10, &num_msg_asym))
290 goto err;
291 num_msg_asym = num_msg_asym >> 1;
292
293 msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ;
294 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
295 if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym,
296 msg_size, key, NULL, 0, &inst->sym_tx))
297 goto err;
298
299 msg_size = msg_size >> 1;
300 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
301 if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym,
302 msg_size, key, NULL, 0, &inst->pke_tx))
303 goto err;
304
305 msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ;
306 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
307 if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym,
308 msg_size, key, qat_alg_callback, 0,
309 &inst->sym_rx))
310 goto err;
311
312 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
313 if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym,
314 msg_size, key, qat_alg_asym_callback, 0,
315 &inst->pke_rx))
316 goto err;
317 }
318 return 0;
319err:
320 qat_crypto_free_instances(accel_dev);
321 return -ENOMEM;
322}
323
324static int qat_crypto_init(struct adf_accel_dev *accel_dev)
325{
326 if (qat_crypto_create_instances(accel_dev))
327 return -EFAULT;
328
329 return 0;
330}
331
332static int qat_crypto_shutdown(struct adf_accel_dev *accel_dev)
333{
334 return qat_crypto_free_instances(accel_dev);
335}
336
337static int qat_crypto_event_handler(struct adf_accel_dev *accel_dev,
338 enum adf_event event)
339{
340 int ret;
341
342 switch (event) {
343 case ADF_EVENT_INIT:
344 ret = qat_crypto_init(accel_dev);
345 break;
346 case ADF_EVENT_SHUTDOWN:
347 ret = qat_crypto_shutdown(accel_dev);
348 break;
349 case ADF_EVENT_RESTARTING:
350 case ADF_EVENT_RESTARTED:
351 case ADF_EVENT_START:
352 case ADF_EVENT_STOP:
353 default:
354 ret = 0;
355 }
356 return ret;
357}
358
359int qat_crypto_register(void)
360{
361 memset(&qat_crypto, 0, sizeof(qat_crypto));
362 qat_crypto.event_hld = qat_crypto_event_handler;
363 qat_crypto.name = "qat_crypto";
364 return adf_service_register(&qat_crypto);
365}
366
367int qat_crypto_unregister(void)
368{
369 return adf_service_unregister(&qat_crypto);
370}
371
