1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47#include <linux/module.h>
48#include <linux/slab.h>
49#include <linux/crypto.h>
50#include <crypto/internal/aead.h>
51#include <crypto/aes.h>
52#include <crypto/sha.h>
53#include <crypto/hash.h>
54#include <crypto/algapi.h>
55#include <crypto/authenc.h>
56#include <crypto/rng.h>
57#include <linux/dma-mapping.h>
58#include "adf_accel_devices.h"
59#include "adf_transport.h"
60#include "adf_common_drv.h"
61#include "qat_crypto.h"
62#include "icp_qat_hw.h"
63#include "icp_qat_fw.h"
64#include "icp_qat_fw_la.h"
65
66#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \
67 ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
68 ICP_QAT_HW_CIPHER_NO_CONVERT, \
69 ICP_QAT_HW_CIPHER_ENCRYPT)
70
71#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \
72 ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
73 ICP_QAT_HW_CIPHER_KEY_CONVERT, \
74 ICP_QAT_HW_CIPHER_DECRYPT)
75
76static DEFINE_MUTEX(algs_lock);
77static unsigned int active_devs;
78
79struct qat_alg_buf {
80 uint32_t len;
81 uint32_t resrvd;
82 uint64_t addr;
83} __packed;
84
85struct qat_alg_buf_list {
86 uint64_t resrvd;
87 uint32_t num_bufs;
88 uint32_t num_mapped_bufs;
89 struct qat_alg_buf bufers[];
90} __packed __aligned(64);
91
92
93struct qat_alg_cd {
94 union {
95 struct qat_enc {
96 struct icp_qat_hw_cipher_algo_blk cipher;
97 struct icp_qat_hw_auth_algo_blk hash;
98 } qat_enc_cd;
99 struct qat_dec {
100 struct icp_qat_hw_auth_algo_blk hash;
101 struct icp_qat_hw_cipher_algo_blk cipher;
102 } qat_dec_cd;
103 };
104} __aligned(64);
105
106struct qat_alg_aead_ctx {
107 struct qat_alg_cd *enc_cd;
108 struct qat_alg_cd *dec_cd;
109 dma_addr_t enc_cd_paddr;
110 dma_addr_t dec_cd_paddr;
111 struct icp_qat_fw_la_bulk_req enc_fw_req;
112 struct icp_qat_fw_la_bulk_req dec_fw_req;
113 struct crypto_shash *hash_tfm;
114 enum icp_qat_hw_auth_algo qat_hash_alg;
115 struct qat_crypto_instance *inst;
116 struct crypto_tfm *tfm;
117 uint8_t salt[AES_BLOCK_SIZE];
118 spinlock_t lock;
119};
120
121struct qat_alg_ablkcipher_ctx {
122 struct icp_qat_hw_cipher_algo_blk *enc_cd;
123 struct icp_qat_hw_cipher_algo_blk *dec_cd;
124 dma_addr_t enc_cd_paddr;
125 dma_addr_t dec_cd_paddr;
126 struct icp_qat_fw_la_bulk_req enc_fw_req;
127 struct icp_qat_fw_la_bulk_req dec_fw_req;
128 struct qat_crypto_instance *inst;
129 struct crypto_tfm *tfm;
130 spinlock_t lock;
131};
132
133static int get_current_node(void)
134{
135 return cpu_data(current_thread_info()->cpu).phys_proc_id;
136}
137
138static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
139{
140 switch (qat_hash_alg) {
141 case ICP_QAT_HW_AUTH_ALGO_SHA1:
142 return ICP_QAT_HW_SHA1_STATE1_SZ;
143 case ICP_QAT_HW_AUTH_ALGO_SHA256:
144 return ICP_QAT_HW_SHA256_STATE1_SZ;
145 case ICP_QAT_HW_AUTH_ALGO_SHA512:
146 return ICP_QAT_HW_SHA512_STATE1_SZ;
147 default:
148 return -EFAULT;
149 };
150 return -EFAULT;
151}
152
153static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
154 struct qat_alg_aead_ctx *ctx,
155 const uint8_t *auth_key,
156 unsigned int auth_keylen)
157{
158 SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
159 struct sha1_state sha1;
160 struct sha256_state sha256;
161 struct sha512_state sha512;
162 int block_size = crypto_shash_blocksize(ctx->hash_tfm);
163 int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
164 char ipad[block_size];
165 char opad[block_size];
166 __be32 *hash_state_out;
167 __be64 *hash512_state_out;
168 int i, offset;
169
170 memset(ipad, 0, block_size);
171 memset(opad, 0, block_size);
172 shash->tfm = ctx->hash_tfm;
173 shash->flags = 0x0;
174
175 if (auth_keylen > block_size) {
176 int ret = crypto_shash_digest(shash, auth_key,
177 auth_keylen, ipad);
178 if (ret)
179 return ret;
180
181 memcpy(opad, ipad, digest_size);
182 } else {
183 memcpy(ipad, auth_key, auth_keylen);
184 memcpy(opad, auth_key, auth_keylen);
185 }
186
187 for (i = 0; i < block_size; i++) {
188 char *ipad_ptr = ipad + i;
189 char *opad_ptr = opad + i;
190 *ipad_ptr ^= 0x36;
191 *opad_ptr ^= 0x5C;
192 }
193
194 if (crypto_shash_init(shash))
195 return -EFAULT;
196
197 if (crypto_shash_update(shash, ipad, block_size))
198 return -EFAULT;
199
200 hash_state_out = (__be32 *)hash->sha.state1;
201 hash512_state_out = (__be64 *)hash_state_out;
202
203 switch (ctx->qat_hash_alg) {
204 case ICP_QAT_HW_AUTH_ALGO_SHA1:
205 if (crypto_shash_export(shash, &sha1))
206 return -EFAULT;
207 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
208 *hash_state_out = cpu_to_be32(*(sha1.state + i));
209 break;
210 case ICP_QAT_HW_AUTH_ALGO_SHA256:
211 if (crypto_shash_export(shash, &sha256))
212 return -EFAULT;
213 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
214 *hash_state_out = cpu_to_be32(*(sha256.state + i));
215 break;
216 case ICP_QAT_HW_AUTH_ALGO_SHA512:
217 if (crypto_shash_export(shash, &sha512))
218 return -EFAULT;
219 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
220 *hash512_state_out = cpu_to_be64(*(sha512.state + i));
221 break;
222 default:
223 return -EFAULT;
224 }
225
226 if (crypto_shash_init(shash))
227 return -EFAULT;
228
229 if (crypto_shash_update(shash, opad, block_size))
230 return -EFAULT;
231
232 offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
233 hash_state_out = (__be32 *)(hash->sha.state1 + offset);
234 hash512_state_out = (__be64 *)hash_state_out;
235
236 switch (ctx->qat_hash_alg) {
237 case ICP_QAT_HW_AUTH_ALGO_SHA1:
238 if (crypto_shash_export(shash, &sha1))
239 return -EFAULT;
240 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
241 *hash_state_out = cpu_to_be32(*(sha1.state + i));
242 break;
243 case ICP_QAT_HW_AUTH_ALGO_SHA256:
244 if (crypto_shash_export(shash, &sha256))
245 return -EFAULT;
246 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
247 *hash_state_out = cpu_to_be32(*(sha256.state + i));
248 break;
249 case ICP_QAT_HW_AUTH_ALGO_SHA512:
250 if (crypto_shash_export(shash, &sha512))
251 return -EFAULT;
252 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
253 *hash512_state_out = cpu_to_be64(*(sha512.state + i));
254 break;
255 default:
256 return -EFAULT;
257 }
258 memzero_explicit(ipad, block_size);
259 memzero_explicit(opad, block_size);
260 return 0;
261}
262
263static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
264{
265 header->hdr_flags =
266 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
267 header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
268 header->comn_req_flags =
269 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
270 QAT_COMN_PTR_TYPE_SGL);
271 ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
272 ICP_QAT_FW_LA_PARTIAL_NONE);
273 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
274 ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
275 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
276 ICP_QAT_FW_LA_NO_PROTO);
277 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
278 ICP_QAT_FW_LA_NO_UPDATE_STATE);
279}
280
281static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx,
282 int alg,
283 struct crypto_authenc_keys *keys)
284{
285 struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
286 unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
287 struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
288 struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
289 struct icp_qat_hw_auth_algo_blk *hash =
290 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
291 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
292 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
293 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
294 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
295 void *ptr = &req_tmpl->cd_ctrl;
296 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
297 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
298
299
300 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
301 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
302 hash->sha.inner_setup.auth_config.config =
303 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
304 ctx->qat_hash_alg, digestsize);
305 hash->sha.inner_setup.auth_counter.counter =
306 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
307
308 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
309 return -EFAULT;
310
311
312 qat_alg_init_common_hdr(header);
313 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
314 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
315 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
316 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
317 ICP_QAT_FW_LA_RET_AUTH_RES);
318 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
319 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
320 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
321 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
322
323
324 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
325 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
326 cipher_cd_ctrl->cipher_cfg_offset = 0;
327 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
328 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
329
330 hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
331 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
332 hash_cd_ctrl->inner_res_sz = digestsize;
333 hash_cd_ctrl->final_sz = digestsize;
334
335 switch (ctx->qat_hash_alg) {
336 case ICP_QAT_HW_AUTH_ALGO_SHA1:
337 hash_cd_ctrl->inner_state1_sz =
338 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
339 hash_cd_ctrl->inner_state2_sz =
340 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
341 break;
342 case ICP_QAT_HW_AUTH_ALGO_SHA256:
343 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
344 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
345 break;
346 case ICP_QAT_HW_AUTH_ALGO_SHA512:
347 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
348 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
349 break;
350 default:
351 break;
352 }
353 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
354 ((sizeof(struct icp_qat_hw_auth_setup) +
355 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
356 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
357 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
358 return 0;
359}
360
361static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx,
362 int alg,
363 struct crypto_authenc_keys *keys)
364{
365 struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
366 unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
367 struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
368 struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
369 struct icp_qat_hw_cipher_algo_blk *cipher =
370 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
371 sizeof(struct icp_qat_hw_auth_setup) +
372 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
373 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
374 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
375 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
376 void *ptr = &req_tmpl->cd_ctrl;
377 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
378 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
379 struct icp_qat_fw_la_auth_req_params *auth_param =
380 (struct icp_qat_fw_la_auth_req_params *)
381 ((char *)&req_tmpl->serv_specif_rqpars +
382 sizeof(struct icp_qat_fw_la_cipher_req_params));
383
384
385 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
386 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
387 hash->sha.inner_setup.auth_config.config =
388 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
389 ctx->qat_hash_alg,
390 digestsize);
391 hash->sha.inner_setup.auth_counter.counter =
392 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
393
394 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
395 return -EFAULT;
396
397
398 qat_alg_init_common_hdr(header);
399 header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
400 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
401 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
402 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
403 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
404 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
405 ICP_QAT_FW_LA_CMP_AUTH_RES);
406 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
407 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
408
409
410 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
411 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
412 cipher_cd_ctrl->cipher_cfg_offset =
413 (sizeof(struct icp_qat_hw_auth_setup) +
414 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
415 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
416 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
417
418
419 hash_cd_ctrl->hash_cfg_offset = 0;
420 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
421 hash_cd_ctrl->inner_res_sz = digestsize;
422 hash_cd_ctrl->final_sz = digestsize;
423
424 switch (ctx->qat_hash_alg) {
425 case ICP_QAT_HW_AUTH_ALGO_SHA1:
426 hash_cd_ctrl->inner_state1_sz =
427 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
428 hash_cd_ctrl->inner_state2_sz =
429 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
430 break;
431 case ICP_QAT_HW_AUTH_ALGO_SHA256:
432 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
433 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
434 break;
435 case ICP_QAT_HW_AUTH_ALGO_SHA512:
436 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
437 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
438 break;
439 default:
440 break;
441 }
442
443 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
444 ((sizeof(struct icp_qat_hw_auth_setup) +
445 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
446 auth_param->auth_res_sz = digestsize;
447 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
448 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
449 return 0;
450}
451
452static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,
453 struct icp_qat_fw_la_bulk_req *req,
454 struct icp_qat_hw_cipher_algo_blk *cd,
455 const uint8_t *key, unsigned int keylen)
456{
457 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
458 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
459 struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
460
461 memcpy(cd->aes.key, key, keylen);
462 qat_alg_init_common_hdr(header);
463 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
464 cd_pars->u.s.content_desc_params_sz =
465 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
466
467 cd_ctrl->cipher_key_sz = keylen >> 3;
468 cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
469 cd_ctrl->cipher_cfg_offset = 0;
470 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
471 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
472}
473
474static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,
475 int alg, const uint8_t *key,
476 unsigned int keylen)
477{
478 struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
479 struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
480 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
481
482 qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);
483 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
484 enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
485}
486
487static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,
488 int alg, const uint8_t *key,
489 unsigned int keylen)
490{
491 struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
492 struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
493 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
494
495 qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);
496 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
497 dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
498}
499
500static int qat_alg_validate_key(int key_len, int *alg)
501{
502 switch (key_len) {
503 case AES_KEYSIZE_128:
504 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
505 break;
506 case AES_KEYSIZE_192:
507 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
508 break;
509 case AES_KEYSIZE_256:
510 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
511 break;
512 default:
513 return -EINVAL;
514 }
515 return 0;
516}
517
518static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx,
519 const uint8_t *key, unsigned int keylen)
520{
521 struct crypto_authenc_keys keys;
522 int alg;
523
524 if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))
525 return -EFAULT;
526
527 if (crypto_authenc_extractkeys(&keys, key, keylen))
528 goto bad_key;
529
530 if (qat_alg_validate_key(keys.enckeylen, &alg))
531 goto bad_key;
532
533 if (qat_alg_aead_init_enc_session(ctx, alg, &keys))
534 goto error;
535
536 if (qat_alg_aead_init_dec_session(ctx, alg, &keys))
537 goto error;
538
539 return 0;
540bad_key:
541 crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
542 return -EINVAL;
543error:
544 return -EFAULT;
545}
546
547static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,
548 const uint8_t *key,
549 unsigned int keylen)
550{
551 int alg;
552
553 if (qat_alg_validate_key(keylen, &alg))
554 goto bad_key;
555
556 qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen);
557 qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen);
558 return 0;
559bad_key:
560 crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
561 return -EINVAL;
562}
563
564static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
565 unsigned int keylen)
566{
567 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
568 struct device *dev;
569
570 spin_lock(&ctx->lock);
571 if (ctx->enc_cd) {
572
573 dev = &GET_DEV(ctx->inst->accel_dev);
574 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
575 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
576 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
577 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
578 } else {
579
580 int node = get_current_node();
581 struct qat_crypto_instance *inst =
582 qat_crypto_get_instance_node(node);
583 if (!inst) {
584 spin_unlock(&ctx->lock);
585 return -EINVAL;
586 }
587
588 dev = &GET_DEV(inst->accel_dev);
589 ctx->inst = inst;
590 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
591 &ctx->enc_cd_paddr,
592 GFP_ATOMIC);
593 if (!ctx->enc_cd) {
594 spin_unlock(&ctx->lock);
595 return -ENOMEM;
596 }
597 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
598 &ctx->dec_cd_paddr,
599 GFP_ATOMIC);
600 if (!ctx->dec_cd) {
601 spin_unlock(&ctx->lock);
602 goto out_free_enc;
603 }
604 }
605 spin_unlock(&ctx->lock);
606 if (qat_alg_aead_init_sessions(ctx, key, keylen))
607 goto out_free_all;
608
609 return 0;
610
611out_free_all:
612 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
613 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
614 ctx->dec_cd, ctx->dec_cd_paddr);
615 ctx->dec_cd = NULL;
616out_free_enc:
617 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
618 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
619 ctx->enc_cd, ctx->enc_cd_paddr);
620 ctx->enc_cd = NULL;
621 return -ENOMEM;
622}
623
624static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
625 struct qat_crypto_request *qat_req)
626{
627 struct device *dev = &GET_DEV(inst->accel_dev);
628 struct qat_alg_buf_list *bl = qat_req->buf.bl;
629 struct qat_alg_buf_list *blout = qat_req->buf.blout;
630 dma_addr_t blp = qat_req->buf.blp;
631 dma_addr_t blpout = qat_req->buf.bloutp;
632 size_t sz = qat_req->buf.sz;
633 size_t sz_out = qat_req->buf.sz_out;
634 int i;
635
636 for (i = 0; i < bl->num_bufs; i++)
637 dma_unmap_single(dev, bl->bufers[i].addr,
638 bl->bufers[i].len, DMA_BIDIRECTIONAL);
639
640 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
641 kfree(bl);
642 if (blp != blpout) {
643
644 int bufless = blout->num_bufs - blout->num_mapped_bufs;
645
646 for (i = bufless; i < blout->num_bufs; i++) {
647 dma_unmap_single(dev, blout->bufers[i].addr,
648 blout->bufers[i].len,
649 DMA_BIDIRECTIONAL);
650 }
651 dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
652 kfree(blout);
653 }
654}
655
656static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
657 struct scatterlist *assoc, int assoclen,
658 struct scatterlist *sgl,
659 struct scatterlist *sglout, uint8_t *iv,
660 uint8_t ivlen,
661 struct qat_crypto_request *qat_req)
662{
663 struct device *dev = &GET_DEV(inst->accel_dev);
664 int i, bufs = 0, sg_nctr = 0;
665 int n = sg_nents(sgl), assoc_n = sg_nents(assoc);
666 struct qat_alg_buf_list *bufl;
667 struct qat_alg_buf_list *buflout = NULL;
668 dma_addr_t blp;
669 dma_addr_t bloutp = 0;
670 struct scatterlist *sg;
671 size_t sz_out, sz = sizeof(struct qat_alg_buf_list) +
672 ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
673
674 if (unlikely(!n))
675 return -EINVAL;
676
677 bufl = kzalloc_node(sz, GFP_ATOMIC,
678 dev_to_node(&GET_DEV(inst->accel_dev)));
679 if (unlikely(!bufl))
680 return -ENOMEM;
681
682 blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE);
683 if (unlikely(dma_mapping_error(dev, blp)))
684 goto err;
685
686 for_each_sg(assoc, sg, assoc_n, i) {
687 if (!sg->length)
688 continue;
689
690 if (!(assoclen > 0))
691 break;
692
693 bufl->bufers[bufs].addr =
694 dma_map_single(dev, sg_virt(sg),
695 min_t(int, assoclen, sg->length),
696 DMA_BIDIRECTIONAL);
697 bufl->bufers[bufs].len = min_t(int, assoclen, sg->length);
698 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
699 goto err;
700 bufs++;
701 assoclen -= sg->length;
702 }
703
704 if (ivlen) {
705 bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen,
706 DMA_BIDIRECTIONAL);
707 bufl->bufers[bufs].len = ivlen;
708 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
709 goto err;
710 bufs++;
711 }
712
713 for_each_sg(sgl, sg, n, i) {
714 int y = sg_nctr + bufs;
715
716 if (!sg->length)
717 continue;
718
719 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
720 sg->length,
721 DMA_BIDIRECTIONAL);
722 bufl->bufers[y].len = sg->length;
723 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
724 goto err;
725 sg_nctr++;
726 }
727 bufl->num_bufs = sg_nctr + bufs;
728 qat_req->buf.bl = bufl;
729 qat_req->buf.blp = blp;
730 qat_req->buf.sz = sz;
731
732 if (sgl != sglout) {
733 struct qat_alg_buf *bufers;
734
735 n = sg_nents(sglout);
736 sz_out = sizeof(struct qat_alg_buf_list) +
737 ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
738 sg_nctr = 0;
739 buflout = kzalloc_node(sz_out, GFP_ATOMIC,
740 dev_to_node(&GET_DEV(inst->accel_dev)));
741 if (unlikely(!buflout))
742 goto err;
743 bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE);
744 if (unlikely(dma_mapping_error(dev, bloutp)))
745 goto err;
746 bufers = buflout->bufers;
747
748
749 for (i = 0; i < bufs; i++) {
750 bufers[i].len = bufl->bufers[i].len;
751 bufers[i].addr = bufl->bufers[i].addr;
752 }
753 for_each_sg(sglout, sg, n, i) {
754 int y = sg_nctr + bufs;
755
756 if (!sg->length)
757 continue;
758
759 bufers[y].addr = dma_map_single(dev, sg_virt(sg),
760 sg->length,
761 DMA_BIDIRECTIONAL);
762 if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
763 goto err;
764 bufers[y].len = sg->length;
765 sg_nctr++;
766 }
767 buflout->num_bufs = sg_nctr + bufs;
768 buflout->num_mapped_bufs = sg_nctr;
769 qat_req->buf.blout = buflout;
770 qat_req->buf.bloutp = bloutp;
771 qat_req->buf.sz_out = sz_out;
772 } else {
773
774 qat_req->buf.bloutp = qat_req->buf.blp;
775 qat_req->buf.sz_out = 0;
776 }
777 return 0;
778err:
779 dev_err(dev, "Failed to map buf for dma\n");
780 sg_nctr = 0;
781 for (i = 0; i < n + bufs; i++)
782 if (!dma_mapping_error(dev, bufl->bufers[i].addr))
783 dma_unmap_single(dev, bufl->bufers[i].addr,
784 bufl->bufers[i].len,
785 DMA_BIDIRECTIONAL);
786
787 if (!dma_mapping_error(dev, blp))
788 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
789 kfree(bufl);
790 if (sgl != sglout && buflout) {
791 n = sg_nents(sglout);
792 for (i = bufs; i < n + bufs; i++)
793 if (!dma_mapping_error(dev, buflout->bufers[i].addr))
794 dma_unmap_single(dev, buflout->bufers[i].addr,
795 buflout->bufers[i].len,
796 DMA_BIDIRECTIONAL);
797 if (!dma_mapping_error(dev, bloutp))
798 dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE);
799 kfree(buflout);
800 }
801 return -ENOMEM;
802}
803
804static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
805 struct qat_crypto_request *qat_req)
806{
807 struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
808 struct qat_crypto_instance *inst = ctx->inst;
809 struct aead_request *areq = qat_req->aead_req;
810 uint8_t stat_filed = qat_resp->comn_resp.comn_status;
811 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
812
813 qat_alg_free_bufl(inst, qat_req);
814 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
815 res = -EBADMSG;
816 areq->base.complete(&areq->base, res);
817}
818
819static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
820 struct qat_crypto_request *qat_req)
821{
822 struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx;
823 struct qat_crypto_instance *inst = ctx->inst;
824 struct ablkcipher_request *areq = qat_req->ablkcipher_req;
825 uint8_t stat_filed = qat_resp->comn_resp.comn_status;
826 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
827
828 qat_alg_free_bufl(inst, qat_req);
829 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
830 res = -EINVAL;
831 areq->base.complete(&areq->base, res);
832}
833
834void qat_alg_callback(void *resp)
835{
836 struct icp_qat_fw_la_resp *qat_resp = resp;
837 struct qat_crypto_request *qat_req =
838 (void *)(__force long)qat_resp->opaque_data;
839
840 qat_req->cb(qat_resp, qat_req);
841}
842
843static int qat_alg_aead_dec(struct aead_request *areq)
844{
845 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
846 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
847 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
848 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
849 struct icp_qat_fw_la_cipher_req_params *cipher_param;
850 struct icp_qat_fw_la_auth_req_params *auth_param;
851 struct icp_qat_fw_la_bulk_req *msg;
852 int digst_size = crypto_aead_crt(aead_tfm)->authsize;
853 int ret, ctr = 0;
854
855 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->assoclen,
856 areq->src, areq->dst, areq->iv,
857 AES_BLOCK_SIZE, qat_req);
858 if (unlikely(ret))
859 return ret;
860
861 msg = &qat_req->req;
862 *msg = ctx->dec_fw_req;
863 qat_req->aead_ctx = ctx;
864 qat_req->aead_req = areq;
865 qat_req->cb = qat_aead_alg_callback;
866 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
867 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
868 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
869 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
870 cipher_param->cipher_length = areq->cryptlen - digst_size;
871 cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE;
872 memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
873 auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
874 auth_param->auth_off = 0;
875 auth_param->auth_len = areq->assoclen +
876 cipher_param->cipher_length + AES_BLOCK_SIZE;
877 do {
878 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
879 } while (ret == -EAGAIN && ctr++ < 10);
880
881 if (ret == -EAGAIN) {
882 qat_alg_free_bufl(ctx->inst, qat_req);
883 return -EBUSY;
884 }
885 return -EINPROGRESS;
886}
887
888static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv,
889 int enc_iv)
890{
891 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
892 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
893 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
894 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
895 struct icp_qat_fw_la_cipher_req_params *cipher_param;
896 struct icp_qat_fw_la_auth_req_params *auth_param;
897 struct icp_qat_fw_la_bulk_req *msg;
898 int ret, ctr = 0;
899
900 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->assoclen,
901 areq->src, areq->dst, iv, AES_BLOCK_SIZE,
902 qat_req);
903 if (unlikely(ret))
904 return ret;
905
906 msg = &qat_req->req;
907 *msg = ctx->enc_fw_req;
908 qat_req->aead_ctx = ctx;
909 qat_req->aead_req = areq;
910 qat_req->cb = qat_aead_alg_callback;
911 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
912 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
913 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
914 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
915 auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
916
917 if (enc_iv) {
918 cipher_param->cipher_length = areq->cryptlen + AES_BLOCK_SIZE;
919 cipher_param->cipher_offset = areq->assoclen;
920 } else {
921 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
922 cipher_param->cipher_length = areq->cryptlen;
923 cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE;
924 }
925 auth_param->auth_off = 0;
926 auth_param->auth_len = areq->assoclen + areq->cryptlen + AES_BLOCK_SIZE;
927
928 do {
929 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
930 } while (ret == -EAGAIN && ctr++ < 10);
931
932 if (ret == -EAGAIN) {
933 qat_alg_free_bufl(ctx->inst, qat_req);
934 return -EBUSY;
935 }
936 return -EINPROGRESS;
937}
938
939static int qat_alg_aead_enc(struct aead_request *areq)
940{
941 return qat_alg_aead_enc_internal(areq, areq->iv, 0);
942}
943
944static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req)
945{
946 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq);
947 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
948 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
949 __be64 seq;
950
951 memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE);
952 seq = cpu_to_be64(req->seq);
953 memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t),
954 &seq, sizeof(uint64_t));
955 return qat_alg_aead_enc_internal(&req->areq, req->giv, 1);
956}
957
958static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
959 const uint8_t *key,
960 unsigned int keylen)
961{
962 struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
963 struct device *dev;
964
965 spin_lock(&ctx->lock);
966 if (ctx->enc_cd) {
967
968 dev = &GET_DEV(ctx->inst->accel_dev);
969 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
970 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
971 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
972 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
973 } else {
974
975 int node = get_current_node();
976 struct qat_crypto_instance *inst =
977 qat_crypto_get_instance_node(node);
978 if (!inst) {
979 spin_unlock(&ctx->lock);
980 return -EINVAL;
981 }
982
983 dev = &GET_DEV(inst->accel_dev);
984 ctx->inst = inst;
985 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
986 &ctx->enc_cd_paddr,
987 GFP_ATOMIC);
988 if (!ctx->enc_cd) {
989 spin_unlock(&ctx->lock);
990 return -ENOMEM;
991 }
992 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
993 &ctx->dec_cd_paddr,
994 GFP_ATOMIC);
995 if (!ctx->dec_cd) {
996 spin_unlock(&ctx->lock);
997 goto out_free_enc;
998 }
999 }
1000 spin_unlock(&ctx->lock);
1001 if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen))
1002 goto out_free_all;
1003
1004 return 0;
1005
1006out_free_all:
1007 memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd));
1008 dma_free_coherent(dev, sizeof(*ctx->enc_cd),
1009 ctx->dec_cd, ctx->dec_cd_paddr);
1010 ctx->dec_cd = NULL;
1011out_free_enc:
1012 memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd));
1013 dma_free_coherent(dev, sizeof(*ctx->dec_cd),
1014 ctx->enc_cd, ctx->enc_cd_paddr);
1015 ctx->enc_cd = NULL;
1016 return -ENOMEM;
1017}
1018
1019static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
1020{
1021 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
1022 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
1023 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1024 struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
1025 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1026 struct icp_qat_fw_la_bulk_req *msg;
1027 int ret, ctr = 0;
1028
1029 ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, 0, req->src, req->dst,
1030 NULL, 0, qat_req);
1031 if (unlikely(ret))
1032 return ret;
1033
1034 msg = &qat_req->req;
1035 *msg = ctx->enc_fw_req;
1036 qat_req->ablkcipher_ctx = ctx;
1037 qat_req->ablkcipher_req = req;
1038 qat_req->cb = qat_ablkcipher_alg_callback;
1039 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1040 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1041 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1042 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1043 cipher_param->cipher_length = req->nbytes;
1044 cipher_param->cipher_offset = 0;
1045 memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
1046 do {
1047 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1048 } while (ret == -EAGAIN && ctr++ < 10);
1049
1050 if (ret == -EAGAIN) {
1051 qat_alg_free_bufl(ctx->inst, qat_req);
1052 return -EBUSY;
1053 }
1054 return -EINPROGRESS;
1055}
1056
1057static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
1058{
1059 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
1060 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
1061 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1062 struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
1063 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1064 struct icp_qat_fw_la_bulk_req *msg;
1065 int ret, ctr = 0;
1066
1067 ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, 0, req->src, req->dst,
1068 NULL, 0, qat_req);
1069 if (unlikely(ret))
1070 return ret;
1071
1072 msg = &qat_req->req;
1073 *msg = ctx->dec_fw_req;
1074 qat_req->ablkcipher_ctx = ctx;
1075 qat_req->ablkcipher_req = req;
1076 qat_req->cb = qat_ablkcipher_alg_callback;
1077 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1078 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1079 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1080 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1081 cipher_param->cipher_length = req->nbytes;
1082 cipher_param->cipher_offset = 0;
1083 memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
1084 do {
1085 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1086 } while (ret == -EAGAIN && ctr++ < 10);
1087
1088 if (ret == -EAGAIN) {
1089 qat_alg_free_bufl(ctx->inst, qat_req);
1090 return -EBUSY;
1091 }
1092 return -EINPROGRESS;
1093}
1094
1095static int qat_alg_aead_init(struct crypto_tfm *tfm,
1096 enum icp_qat_hw_auth_algo hash,
1097 const char *hash_name)
1098{
1099 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
1100
1101 ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1102 if (IS_ERR(ctx->hash_tfm))
1103 return -EFAULT;
1104 spin_lock_init(&ctx->lock);
1105 ctx->qat_hash_alg = hash;
1106 crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
1107 sizeof(struct aead_request) +
1108 sizeof(struct qat_crypto_request));
1109 ctx->tfm = tfm;
1110 return 0;
1111}
1112
1113static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm)
1114{
1115 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1116}
1117
1118static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm)
1119{
1120 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1121}
1122
1123static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm)
1124{
1125 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1126}
1127
1128static void qat_alg_aead_exit(struct crypto_tfm *tfm)
1129{
1130 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
1131 struct qat_crypto_instance *inst = ctx->inst;
1132 struct device *dev;
1133
1134 if (!IS_ERR(ctx->hash_tfm))
1135 crypto_free_shash(ctx->hash_tfm);
1136
1137 if (!inst)
1138 return;
1139
1140 dev = &GET_DEV(inst->accel_dev);
1141 if (ctx->enc_cd) {
1142 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1143 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1144 ctx->enc_cd, ctx->enc_cd_paddr);
1145 }
1146 if (ctx->dec_cd) {
1147 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1148 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1149 ctx->dec_cd, ctx->dec_cd_paddr);
1150 }
1151 qat_crypto_put_instance(inst);
1152}
1153
1154static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm)
1155{
1156 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1157
1158 spin_lock_init(&ctx->lock);
1159 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
1160 sizeof(struct qat_crypto_request);
1161 ctx->tfm = tfm;
1162 return 0;
1163}
1164
1165static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm)
1166{
1167 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1168 struct qat_crypto_instance *inst = ctx->inst;
1169 struct device *dev;
1170
1171 if (!inst)
1172 return;
1173
1174 dev = &GET_DEV(inst->accel_dev);
1175 if (ctx->enc_cd) {
1176 memset(ctx->enc_cd, 0,
1177 sizeof(struct icp_qat_hw_cipher_algo_blk));
1178 dma_free_coherent(dev,
1179 sizeof(struct icp_qat_hw_cipher_algo_blk),
1180 ctx->enc_cd, ctx->enc_cd_paddr);
1181 }
1182 if (ctx->dec_cd) {
1183 memset(ctx->dec_cd, 0,
1184 sizeof(struct icp_qat_hw_cipher_algo_blk));
1185 dma_free_coherent(dev,
1186 sizeof(struct icp_qat_hw_cipher_algo_blk),
1187 ctx->dec_cd, ctx->dec_cd_paddr);
1188 }
1189 qat_crypto_put_instance(inst);
1190}
1191
1192static struct crypto_alg qat_algs[] = { {
1193 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1194 .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1195 .cra_priority = 4001,
1196 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1197 .cra_blocksize = AES_BLOCK_SIZE,
1198 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1199 .cra_alignmask = 0,
1200 .cra_type = &crypto_aead_type,
1201 .cra_module = THIS_MODULE,
1202 .cra_init = qat_alg_aead_sha1_init,
1203 .cra_exit = qat_alg_aead_exit,
1204 .cra_u = {
1205 .aead = {
1206 .setkey = qat_alg_aead_setkey,
1207 .decrypt = qat_alg_aead_dec,
1208 .encrypt = qat_alg_aead_enc,
1209 .givencrypt = qat_alg_aead_genivenc,
1210 .ivsize = AES_BLOCK_SIZE,
1211 .maxauthsize = SHA1_DIGEST_SIZE,
1212 },
1213 },
1214}, {
1215 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1216 .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1217 .cra_priority = 4001,
1218 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1219 .cra_blocksize = AES_BLOCK_SIZE,
1220 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1221 .cra_alignmask = 0,
1222 .cra_type = &crypto_aead_type,
1223 .cra_module = THIS_MODULE,
1224 .cra_init = qat_alg_aead_sha256_init,
1225 .cra_exit = qat_alg_aead_exit,
1226 .cra_u = {
1227 .aead = {
1228 .setkey = qat_alg_aead_setkey,
1229 .decrypt = qat_alg_aead_dec,
1230 .encrypt = qat_alg_aead_enc,
1231 .givencrypt = qat_alg_aead_genivenc,
1232 .ivsize = AES_BLOCK_SIZE,
1233 .maxauthsize = SHA256_DIGEST_SIZE,
1234 },
1235 },
1236}, {
1237 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1238 .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1239 .cra_priority = 4001,
1240 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1241 .cra_blocksize = AES_BLOCK_SIZE,
1242 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1243 .cra_alignmask = 0,
1244 .cra_type = &crypto_aead_type,
1245 .cra_module = THIS_MODULE,
1246 .cra_init = qat_alg_aead_sha512_init,
1247 .cra_exit = qat_alg_aead_exit,
1248 .cra_u = {
1249 .aead = {
1250 .setkey = qat_alg_aead_setkey,
1251 .decrypt = qat_alg_aead_dec,
1252 .encrypt = qat_alg_aead_enc,
1253 .givencrypt = qat_alg_aead_genivenc,
1254 .ivsize = AES_BLOCK_SIZE,
1255 .maxauthsize = SHA512_DIGEST_SIZE,
1256 },
1257 },
1258}, {
1259 .cra_name = "cbc(aes)",
1260 .cra_driver_name = "qat_aes_cbc",
1261 .cra_priority = 4001,
1262 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1263 .cra_blocksize = AES_BLOCK_SIZE,
1264 .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
1265 .cra_alignmask = 0,
1266 .cra_type = &crypto_ablkcipher_type,
1267 .cra_module = THIS_MODULE,
1268 .cra_init = qat_alg_ablkcipher_init,
1269 .cra_exit = qat_alg_ablkcipher_exit,
1270 .cra_u = {
1271 .ablkcipher = {
1272 .setkey = qat_alg_ablkcipher_setkey,
1273 .decrypt = qat_alg_ablkcipher_decrypt,
1274 .encrypt = qat_alg_ablkcipher_encrypt,
1275 .min_keysize = AES_MIN_KEY_SIZE,
1276 .max_keysize = AES_MAX_KEY_SIZE,
1277 .ivsize = AES_BLOCK_SIZE,
1278 },
1279 },
1280} };
1281
1282int qat_algs_register(void)
1283{
1284 int ret = 0;
1285
1286 mutex_lock(&algs_lock);
1287 if (++active_devs == 1) {
1288 int i;
1289
1290 for (i = 0; i < ARRAY_SIZE(qat_algs); i++)
1291 qat_algs[i].cra_flags =
1292 (qat_algs[i].cra_type == &crypto_aead_type) ?
1293 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC :
1294 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
1295
1296 ret = crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs));
1297 }
1298 mutex_unlock(&algs_lock);
1299 return ret;
1300}
1301
1302int qat_algs_unregister(void)
1303{
1304 int ret = 0;
1305
1306 mutex_lock(&algs_lock);
1307 if (--active_devs == 0)
1308 ret = crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs));
1309 mutex_unlock(&algs_lock);
1310 return ret;
1311}
1312
1313int qat_algs_init(void)
1314{
1315 crypto_get_default_rng();
1316 return 0;
1317}
1318
1319void qat_algs_exit(void)
1320{
1321 crypto_put_default_rng();
1322}
1323
