1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47#include <linux/module.h>
48#include <linux/slab.h>
49#include <linux/crypto.h>
50#include <crypto/internal/aead.h>
51#include <crypto/internal/skcipher.h>
52#include <crypto/aes.h>
53#include <crypto/sha.h>
54#include <crypto/hash.h>
55#include <crypto/hmac.h>
56#include <crypto/algapi.h>
57#include <crypto/authenc.h>
58#include <linux/dma-mapping.h>
59#include "adf_accel_devices.h"
60#include "adf_transport.h"
61#include "adf_common_drv.h"
62#include "qat_crypto.h"
63#include "icp_qat_hw.h"
64#include "icp_qat_fw.h"
65#include "icp_qat_fw_la.h"
66
67#define QAT_AES_HW_CONFIG_ENC(alg, mode) \
68 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
69 ICP_QAT_HW_CIPHER_NO_CONVERT, \
70 ICP_QAT_HW_CIPHER_ENCRYPT)
71
72#define QAT_AES_HW_CONFIG_DEC(alg, mode) \
73 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
74 ICP_QAT_HW_CIPHER_KEY_CONVERT, \
75 ICP_QAT_HW_CIPHER_DECRYPT)
76
77static DEFINE_MUTEX(algs_lock);
78static unsigned int active_devs;
79
80struct qat_alg_buf {
81 uint32_t len;
82 uint32_t resrvd;
83 uint64_t addr;
84} __packed;
85
86struct qat_alg_buf_list {
87 uint64_t resrvd;
88 uint32_t num_bufs;
89 uint32_t num_mapped_bufs;
90 struct qat_alg_buf bufers[];
91} __packed __aligned(64);
92
93
94struct qat_alg_cd {
95 union {
96 struct qat_enc {
97 struct icp_qat_hw_cipher_algo_blk cipher;
98 struct icp_qat_hw_auth_algo_blk hash;
99 } qat_enc_cd;
100 struct qat_dec {
101 struct icp_qat_hw_auth_algo_blk hash;
102 struct icp_qat_hw_cipher_algo_blk cipher;
103 } qat_dec_cd;
104 };
105} __aligned(64);
106
107struct qat_alg_aead_ctx {
108 struct qat_alg_cd *enc_cd;
109 struct qat_alg_cd *dec_cd;
110 dma_addr_t enc_cd_paddr;
111 dma_addr_t dec_cd_paddr;
112 struct icp_qat_fw_la_bulk_req enc_fw_req;
113 struct icp_qat_fw_la_bulk_req dec_fw_req;
114 struct crypto_shash *hash_tfm;
115 enum icp_qat_hw_auth_algo qat_hash_alg;
116 struct qat_crypto_instance *inst;
117 union {
118 struct sha1_state sha1;
119 struct sha256_state sha256;
120 struct sha512_state sha512;
121 };
122 char ipad[SHA512_BLOCK_SIZE];
123 char opad[SHA512_BLOCK_SIZE];
124};
125
126struct qat_alg_skcipher_ctx {
127 struct icp_qat_hw_cipher_algo_blk *enc_cd;
128 struct icp_qat_hw_cipher_algo_blk *dec_cd;
129 dma_addr_t enc_cd_paddr;
130 dma_addr_t dec_cd_paddr;
131 struct icp_qat_fw_la_bulk_req enc_fw_req;
132 struct icp_qat_fw_la_bulk_req dec_fw_req;
133 struct qat_crypto_instance *inst;
134 struct crypto_skcipher *tfm;
135};
136
137static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
138{
139 switch (qat_hash_alg) {
140 case ICP_QAT_HW_AUTH_ALGO_SHA1:
141 return ICP_QAT_HW_SHA1_STATE1_SZ;
142 case ICP_QAT_HW_AUTH_ALGO_SHA256:
143 return ICP_QAT_HW_SHA256_STATE1_SZ;
144 case ICP_QAT_HW_AUTH_ALGO_SHA512:
145 return ICP_QAT_HW_SHA512_STATE1_SZ;
146 default:
147 return -EFAULT;
148 };
149 return -EFAULT;
150}
151
152static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
153 struct qat_alg_aead_ctx *ctx,
154 const uint8_t *auth_key,
155 unsigned int auth_keylen)
156{
157 SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
158 int block_size = crypto_shash_blocksize(ctx->hash_tfm);
159 int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
160 __be32 *hash_state_out;
161 __be64 *hash512_state_out;
162 int i, offset;
163
164 memset(ctx->ipad, 0, block_size);
165 memset(ctx->opad, 0, block_size);
166 shash->tfm = ctx->hash_tfm;
167
168 if (auth_keylen > block_size) {
169 int ret = crypto_shash_digest(shash, auth_key,
170 auth_keylen, ctx->ipad);
171 if (ret)
172 return ret;
173
174 memcpy(ctx->opad, ctx->ipad, digest_size);
175 } else {
176 memcpy(ctx->ipad, auth_key, auth_keylen);
177 memcpy(ctx->opad, auth_key, auth_keylen);
178 }
179
180 for (i = 0; i < block_size; i++) {
181 char *ipad_ptr = ctx->ipad + i;
182 char *opad_ptr = ctx->opad + i;
183 *ipad_ptr ^= HMAC_IPAD_VALUE;
184 *opad_ptr ^= HMAC_OPAD_VALUE;
185 }
186
187 if (crypto_shash_init(shash))
188 return -EFAULT;
189
190 if (crypto_shash_update(shash, ctx->ipad, block_size))
191 return -EFAULT;
192
193 hash_state_out = (__be32 *)hash->sha.state1;
194 hash512_state_out = (__be64 *)hash_state_out;
195
196 switch (ctx->qat_hash_alg) {
197 case ICP_QAT_HW_AUTH_ALGO_SHA1:
198 if (crypto_shash_export(shash, &ctx->sha1))
199 return -EFAULT;
200 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
201 *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
202 break;
203 case ICP_QAT_HW_AUTH_ALGO_SHA256:
204 if (crypto_shash_export(shash, &ctx->sha256))
205 return -EFAULT;
206 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
207 *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
208 break;
209 case ICP_QAT_HW_AUTH_ALGO_SHA512:
210 if (crypto_shash_export(shash, &ctx->sha512))
211 return -EFAULT;
212 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
213 *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
214 break;
215 default:
216 return -EFAULT;
217 }
218
219 if (crypto_shash_init(shash))
220 return -EFAULT;
221
222 if (crypto_shash_update(shash, ctx->opad, block_size))
223 return -EFAULT;
224
225 offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
226 if (offset < 0)
227 return -EFAULT;
228
229 hash_state_out = (__be32 *)(hash->sha.state1 + offset);
230 hash512_state_out = (__be64 *)hash_state_out;
231
232 switch (ctx->qat_hash_alg) {
233 case ICP_QAT_HW_AUTH_ALGO_SHA1:
234 if (crypto_shash_export(shash, &ctx->sha1))
235 return -EFAULT;
236 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
237 *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
238 break;
239 case ICP_QAT_HW_AUTH_ALGO_SHA256:
240 if (crypto_shash_export(shash, &ctx->sha256))
241 return -EFAULT;
242 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
243 *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
244 break;
245 case ICP_QAT_HW_AUTH_ALGO_SHA512:
246 if (crypto_shash_export(shash, &ctx->sha512))
247 return -EFAULT;
248 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
249 *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
250 break;
251 default:
252 return -EFAULT;
253 }
254 memzero_explicit(ctx->ipad, block_size);
255 memzero_explicit(ctx->opad, block_size);
256 return 0;
257}
258
259static void qat_alg_init_hdr_iv_updt(struct icp_qat_fw_comn_req_hdr *header)
260{
261 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
262 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
263 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
264 ICP_QAT_FW_LA_UPDATE_STATE);
265}
266
267static void qat_alg_init_hdr_no_iv_updt(struct icp_qat_fw_comn_req_hdr *header)
268{
269 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
270 ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
271 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
272 ICP_QAT_FW_LA_NO_UPDATE_STATE);
273}
274
275static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
276 int aead)
277{
278 header->hdr_flags =
279 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
280 header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
281 header->comn_req_flags =
282 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
283 QAT_COMN_PTR_TYPE_SGL);
284 ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
285 ICP_QAT_FW_LA_PARTIAL_NONE);
286 if (aead)
287 qat_alg_init_hdr_no_iv_updt(header);
288 else
289 qat_alg_init_hdr_iv_updt(header);
290 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
291 ICP_QAT_FW_LA_NO_PROTO);
292}
293
294static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
295 int alg,
296 struct crypto_authenc_keys *keys,
297 int mode)
298{
299 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
300 unsigned int digestsize = crypto_aead_authsize(aead_tfm);
301 struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
302 struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
303 struct icp_qat_hw_auth_algo_blk *hash =
304 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
305 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
306 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
307 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
308 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
309 void *ptr = &req_tmpl->cd_ctrl;
310 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
311 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
312
313
314 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
315 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
316 hash->sha.inner_setup.auth_config.config =
317 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
318 ctx->qat_hash_alg, digestsize);
319 hash->sha.inner_setup.auth_counter.counter =
320 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
321
322 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
323 return -EFAULT;
324
325
326 qat_alg_init_common_hdr(header, 1);
327 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
328 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
329 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
330 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
331 ICP_QAT_FW_LA_RET_AUTH_RES);
332 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
333 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
334 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
335 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
336
337
338 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
339 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
340 cipher_cd_ctrl->cipher_cfg_offset = 0;
341 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
342 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
343
344 hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
345 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
346 hash_cd_ctrl->inner_res_sz = digestsize;
347 hash_cd_ctrl->final_sz = digestsize;
348
349 switch (ctx->qat_hash_alg) {
350 case ICP_QAT_HW_AUTH_ALGO_SHA1:
351 hash_cd_ctrl->inner_state1_sz =
352 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
353 hash_cd_ctrl->inner_state2_sz =
354 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
355 break;
356 case ICP_QAT_HW_AUTH_ALGO_SHA256:
357 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
358 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
359 break;
360 case ICP_QAT_HW_AUTH_ALGO_SHA512:
361 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
362 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
363 break;
364 default:
365 break;
366 }
367 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
368 ((sizeof(struct icp_qat_hw_auth_setup) +
369 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
370 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
371 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
372 return 0;
373}
374
375static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
376 int alg,
377 struct crypto_authenc_keys *keys,
378 int mode)
379{
380 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
381 unsigned int digestsize = crypto_aead_authsize(aead_tfm);
382 struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
383 struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
384 struct icp_qat_hw_cipher_algo_blk *cipher =
385 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
386 sizeof(struct icp_qat_hw_auth_setup) +
387 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
388 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
389 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
390 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
391 void *ptr = &req_tmpl->cd_ctrl;
392 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
393 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
394 struct icp_qat_fw_la_auth_req_params *auth_param =
395 (struct icp_qat_fw_la_auth_req_params *)
396 ((char *)&req_tmpl->serv_specif_rqpars +
397 sizeof(struct icp_qat_fw_la_cipher_req_params));
398
399
400 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
401 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
402 hash->sha.inner_setup.auth_config.config =
403 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
404 ctx->qat_hash_alg,
405 digestsize);
406 hash->sha.inner_setup.auth_counter.counter =
407 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
408
409 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
410 return -EFAULT;
411
412
413 qat_alg_init_common_hdr(header, 1);
414 header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
415 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
416 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
417 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
418 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
419 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
420 ICP_QAT_FW_LA_CMP_AUTH_RES);
421 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
422 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
423
424
425 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
426 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
427 cipher_cd_ctrl->cipher_cfg_offset =
428 (sizeof(struct icp_qat_hw_auth_setup) +
429 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
430 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
431 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
432
433
434 hash_cd_ctrl->hash_cfg_offset = 0;
435 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
436 hash_cd_ctrl->inner_res_sz = digestsize;
437 hash_cd_ctrl->final_sz = digestsize;
438
439 switch (ctx->qat_hash_alg) {
440 case ICP_QAT_HW_AUTH_ALGO_SHA1:
441 hash_cd_ctrl->inner_state1_sz =
442 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
443 hash_cd_ctrl->inner_state2_sz =
444 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
445 break;
446 case ICP_QAT_HW_AUTH_ALGO_SHA256:
447 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
448 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
449 break;
450 case ICP_QAT_HW_AUTH_ALGO_SHA512:
451 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
452 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
453 break;
454 default:
455 break;
456 }
457
458 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
459 ((sizeof(struct icp_qat_hw_auth_setup) +
460 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
461 auth_param->auth_res_sz = digestsize;
462 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
463 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
464 return 0;
465}
466
467static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx,
468 struct icp_qat_fw_la_bulk_req *req,
469 struct icp_qat_hw_cipher_algo_blk *cd,
470 const uint8_t *key, unsigned int keylen)
471{
472 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
473 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
474 struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
475
476 memcpy(cd->aes.key, key, keylen);
477 qat_alg_init_common_hdr(header, 0);
478 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
479 cd_pars->u.s.content_desc_params_sz =
480 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
481
482 cd_ctrl->cipher_key_sz = keylen >> 3;
483 cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
484 cd_ctrl->cipher_cfg_offset = 0;
485 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
486 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
487}
488
489static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx,
490 int alg, const uint8_t *key,
491 unsigned int keylen, int mode)
492{
493 struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
494 struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
495 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
496
497 qat_alg_skcipher_init_com(ctx, req, enc_cd, key, keylen);
498 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
499 enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
500}
501
502static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,
503 int alg, const uint8_t *key,
504 unsigned int keylen, int mode)
505{
506 struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
507 struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
508 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
509
510 qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen);
511 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
512
513 if (mode != ICP_QAT_HW_CIPHER_CTR_MODE)
514 dec_cd->aes.cipher_config.val =
515 QAT_AES_HW_CONFIG_DEC(alg, mode);
516 else
517 dec_cd->aes.cipher_config.val =
518 QAT_AES_HW_CONFIG_ENC(alg, mode);
519}
520
521static int qat_alg_validate_key(int key_len, int *alg, int mode)
522{
523 if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {
524 switch (key_len) {
525 case AES_KEYSIZE_128:
526 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
527 break;
528 case AES_KEYSIZE_192:
529 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
530 break;
531 case AES_KEYSIZE_256:
532 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
533 break;
534 default:
535 return -EINVAL;
536 }
537 } else {
538 switch (key_len) {
539 case AES_KEYSIZE_128 << 1:
540 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
541 break;
542 case AES_KEYSIZE_256 << 1:
543 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
544 break;
545 default:
546 return -EINVAL;
547 }
548 }
549 return 0;
550}
551
552static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key,
553 unsigned int keylen, int mode)
554{
555 struct crypto_authenc_keys keys;
556 int alg;
557
558 if (crypto_authenc_extractkeys(&keys, key, keylen))
559 goto bad_key;
560
561 if (qat_alg_validate_key(keys.enckeylen, &alg, mode))
562 goto bad_key;
563
564 if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode))
565 goto error;
566
567 if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode))
568 goto error;
569
570 memzero_explicit(&keys, sizeof(keys));
571 return 0;
572bad_key:
573 memzero_explicit(&keys, sizeof(keys));
574 return -EINVAL;
575error:
576 memzero_explicit(&keys, sizeof(keys));
577 return -EFAULT;
578}
579
580static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx,
581 const uint8_t *key,
582 unsigned int keylen,
583 int mode)
584{
585 int alg;
586
587 if (qat_alg_validate_key(keylen, &alg, mode))
588 return -EINVAL;
589
590 qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode);
591 qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode);
592 return 0;
593}
594
595static int qat_alg_aead_rekey(struct crypto_aead *tfm, const uint8_t *key,
596 unsigned int keylen)
597{
598 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
599
600 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
601 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
602 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
603 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
604
605 return qat_alg_aead_init_sessions(tfm, key, keylen,
606 ICP_QAT_HW_CIPHER_CBC_MODE);
607}
608
609static int qat_alg_aead_newkey(struct crypto_aead *tfm, const uint8_t *key,
610 unsigned int keylen)
611{
612 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
613 struct qat_crypto_instance *inst = NULL;
614 int node = get_current_node();
615 struct device *dev;
616 int ret;
617
618 inst = qat_crypto_get_instance_node(node);
619 if (!inst)
620 return -EINVAL;
621 dev = &GET_DEV(inst->accel_dev);
622 ctx->inst = inst;
623 ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
624 &ctx->enc_cd_paddr,
625 GFP_ATOMIC);
626 if (!ctx->enc_cd) {
627 ret = -ENOMEM;
628 goto out_free_inst;
629 }
630 ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
631 &ctx->dec_cd_paddr,
632 GFP_ATOMIC);
633 if (!ctx->dec_cd) {
634 ret = -ENOMEM;
635 goto out_free_enc;
636 }
637
638 ret = qat_alg_aead_init_sessions(tfm, key, keylen,
639 ICP_QAT_HW_CIPHER_CBC_MODE);
640 if (ret)
641 goto out_free_all;
642
643 return 0;
644
645out_free_all:
646 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
647 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
648 ctx->dec_cd, ctx->dec_cd_paddr);
649 ctx->dec_cd = NULL;
650out_free_enc:
651 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
652 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
653 ctx->enc_cd, ctx->enc_cd_paddr);
654 ctx->enc_cd = NULL;
655out_free_inst:
656 ctx->inst = NULL;
657 qat_crypto_put_instance(inst);
658 return ret;
659}
660
661static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
662 unsigned int keylen)
663{
664 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
665
666 if (ctx->enc_cd)
667 return qat_alg_aead_rekey(tfm, key, keylen);
668 else
669 return qat_alg_aead_newkey(tfm, key, keylen);
670}
671
672static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
673 struct qat_crypto_request *qat_req)
674{
675 struct device *dev = &GET_DEV(inst->accel_dev);
676 struct qat_alg_buf_list *bl = qat_req->buf.bl;
677 struct qat_alg_buf_list *blout = qat_req->buf.blout;
678 dma_addr_t blp = qat_req->buf.blp;
679 dma_addr_t blpout = qat_req->buf.bloutp;
680 size_t sz = qat_req->buf.sz;
681 size_t sz_out = qat_req->buf.sz_out;
682 int i;
683
684 for (i = 0; i < bl->num_bufs; i++)
685 dma_unmap_single(dev, bl->bufers[i].addr,
686 bl->bufers[i].len, DMA_BIDIRECTIONAL);
687
688 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
689 kfree(bl);
690 if (blp != blpout) {
691
692 int bufless = blout->num_bufs - blout->num_mapped_bufs;
693
694 for (i = bufless; i < blout->num_bufs; i++) {
695 dma_unmap_single(dev, blout->bufers[i].addr,
696 blout->bufers[i].len,
697 DMA_BIDIRECTIONAL);
698 }
699 dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
700 kfree(blout);
701 }
702}
703
704static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
705 struct scatterlist *sgl,
706 struct scatterlist *sglout,
707 struct qat_crypto_request *qat_req)
708{
709 struct device *dev = &GET_DEV(inst->accel_dev);
710 int i, sg_nctr = 0;
711 int n = sg_nents(sgl);
712 struct qat_alg_buf_list *bufl;
713 struct qat_alg_buf_list *buflout = NULL;
714 dma_addr_t blp;
715 dma_addr_t bloutp = 0;
716 struct scatterlist *sg;
717 size_t sz_out, sz = struct_size(bufl, bufers, n + 1);
718
719 if (unlikely(!n))
720 return -EINVAL;
721
722 bufl = kzalloc_node(sz, GFP_ATOMIC,
723 dev_to_node(&GET_DEV(inst->accel_dev)));
724 if (unlikely(!bufl))
725 return -ENOMEM;
726
727 blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE);
728 if (unlikely(dma_mapping_error(dev, blp)))
729 goto err_in;
730
731 for_each_sg(sgl, sg, n, i) {
732 int y = sg_nctr;
733
734 if (!sg->length)
735 continue;
736
737 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
738 sg->length,
739 DMA_BIDIRECTIONAL);
740 bufl->bufers[y].len = sg->length;
741 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
742 goto err_in;
743 sg_nctr++;
744 }
745 bufl->num_bufs = sg_nctr;
746 qat_req->buf.bl = bufl;
747 qat_req->buf.blp = blp;
748 qat_req->buf.sz = sz;
749
750 if (sgl != sglout) {
751 struct qat_alg_buf *bufers;
752
753 n = sg_nents(sglout);
754 sz_out = struct_size(buflout, bufers, n + 1);
755 sg_nctr = 0;
756 buflout = kzalloc_node(sz_out, GFP_ATOMIC,
757 dev_to_node(&GET_DEV(inst->accel_dev)));
758 if (unlikely(!buflout))
759 goto err_in;
760 bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE);
761 if (unlikely(dma_mapping_error(dev, bloutp)))
762 goto err_out;
763 bufers = buflout->bufers;
764 for_each_sg(sglout, sg, n, i) {
765 int y = sg_nctr;
766
767 if (!sg->length)
768 continue;
769
770 bufers[y].addr = dma_map_single(dev, sg_virt(sg),
771 sg->length,
772 DMA_BIDIRECTIONAL);
773 if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
774 goto err_out;
775 bufers[y].len = sg->length;
776 sg_nctr++;
777 }
778 buflout->num_bufs = sg_nctr;
779 buflout->num_mapped_bufs = sg_nctr;
780 qat_req->buf.blout = buflout;
781 qat_req->buf.bloutp = bloutp;
782 qat_req->buf.sz_out = sz_out;
783 } else {
784
785 qat_req->buf.bloutp = qat_req->buf.blp;
786 qat_req->buf.sz_out = 0;
787 }
788 return 0;
789
790err_out:
791 n = sg_nents(sglout);
792 for (i = 0; i < n; i++)
793 if (!dma_mapping_error(dev, buflout->bufers[i].addr))
794 dma_unmap_single(dev, buflout->bufers[i].addr,
795 buflout->bufers[i].len,
796 DMA_BIDIRECTIONAL);
797 if (!dma_mapping_error(dev, bloutp))
798 dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE);
799 kfree(buflout);
800
801err_in:
802 n = sg_nents(sgl);
803 for (i = 0; i < n; i++)
804 if (!dma_mapping_error(dev, bufl->bufers[i].addr))
805 dma_unmap_single(dev, bufl->bufers[i].addr,
806 bufl->bufers[i].len,
807 DMA_BIDIRECTIONAL);
808
809 if (!dma_mapping_error(dev, blp))
810 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
811 kfree(bufl);
812
813 dev_err(dev, "Failed to map buf for dma\n");
814 return -ENOMEM;
815}
816
817static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
818 struct qat_crypto_request *qat_req)
819{
820 struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
821 struct qat_crypto_instance *inst = ctx->inst;
822 struct aead_request *areq = qat_req->aead_req;
823 uint8_t stat_filed = qat_resp->comn_resp.comn_status;
824 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
825
826 qat_alg_free_bufl(inst, qat_req);
827 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
828 res = -EBADMSG;
829 areq->base.complete(&areq->base, res);
830}
831
832static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
833 struct qat_crypto_request *qat_req)
834{
835 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
836 struct qat_crypto_instance *inst = ctx->inst;
837 struct skcipher_request *sreq = qat_req->skcipher_req;
838 uint8_t stat_filed = qat_resp->comn_resp.comn_status;
839 struct device *dev = &GET_DEV(ctx->inst->accel_dev);
840 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
841
842 qat_alg_free_bufl(inst, qat_req);
843 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
844 res = -EINVAL;
845
846 memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE);
847 dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
848 qat_req->iv_paddr);
849
850 sreq->base.complete(&sreq->base, res);
851}
852
853void qat_alg_callback(void *resp)
854{
855 struct icp_qat_fw_la_resp *qat_resp = resp;
856 struct qat_crypto_request *qat_req =
857 (void *)(__force long)qat_resp->opaque_data;
858
859 qat_req->cb(qat_resp, qat_req);
860}
861
862static int qat_alg_aead_dec(struct aead_request *areq)
863{
864 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
865 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
866 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
867 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
868 struct icp_qat_fw_la_cipher_req_params *cipher_param;
869 struct icp_qat_fw_la_auth_req_params *auth_param;
870 struct icp_qat_fw_la_bulk_req *msg;
871 int digst_size = crypto_aead_authsize(aead_tfm);
872 int ret, ctr = 0;
873
874 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
875 if (unlikely(ret))
876 return ret;
877
878 msg = &qat_req->req;
879 *msg = ctx->dec_fw_req;
880 qat_req->aead_ctx = ctx;
881 qat_req->aead_req = areq;
882 qat_req->cb = qat_aead_alg_callback;
883 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
884 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
885 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
886 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
887 cipher_param->cipher_length = areq->cryptlen - digst_size;
888 cipher_param->cipher_offset = areq->assoclen;
889 memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
890 auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
891 auth_param->auth_off = 0;
892 auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
893 do {
894 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
895 } while (ret == -EAGAIN && ctr++ < 10);
896
897 if (ret == -EAGAIN) {
898 qat_alg_free_bufl(ctx->inst, qat_req);
899 return -EBUSY;
900 }
901 return -EINPROGRESS;
902}
903
904static int qat_alg_aead_enc(struct aead_request *areq)
905{
906 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
907 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
908 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
909 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
910 struct icp_qat_fw_la_cipher_req_params *cipher_param;
911 struct icp_qat_fw_la_auth_req_params *auth_param;
912 struct icp_qat_fw_la_bulk_req *msg;
913 uint8_t *iv = areq->iv;
914 int ret, ctr = 0;
915
916 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
917 if (unlikely(ret))
918 return ret;
919
920 msg = &qat_req->req;
921 *msg = ctx->enc_fw_req;
922 qat_req->aead_ctx = ctx;
923 qat_req->aead_req = areq;
924 qat_req->cb = qat_aead_alg_callback;
925 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
926 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
927 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
928 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
929 auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
930
931 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
932 cipher_param->cipher_length = areq->cryptlen;
933 cipher_param->cipher_offset = areq->assoclen;
934
935 auth_param->auth_off = 0;
936 auth_param->auth_len = areq->assoclen + areq->cryptlen;
937
938 do {
939 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
940 } while (ret == -EAGAIN && ctr++ < 10);
941
942 if (ret == -EAGAIN) {
943 qat_alg_free_bufl(ctx->inst, qat_req);
944 return -EBUSY;
945 }
946 return -EINPROGRESS;
947}
948
949static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx,
950 const u8 *key, unsigned int keylen,
951 int mode)
952{
953 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
954 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
955 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
956 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
957
958 return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
959}
960
961static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx,
962 const u8 *key, unsigned int keylen,
963 int mode)
964{
965 struct qat_crypto_instance *inst = NULL;
966 struct device *dev;
967 int node = get_current_node();
968 int ret;
969
970 inst = qat_crypto_get_instance_node(node);
971 if (!inst)
972 return -EINVAL;
973 dev = &GET_DEV(inst->accel_dev);
974 ctx->inst = inst;
975 ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
976 &ctx->enc_cd_paddr,
977 GFP_ATOMIC);
978 if (!ctx->enc_cd) {
979 ret = -ENOMEM;
980 goto out_free_instance;
981 }
982 ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
983 &ctx->dec_cd_paddr,
984 GFP_ATOMIC);
985 if (!ctx->dec_cd) {
986 ret = -ENOMEM;
987 goto out_free_enc;
988 }
989
990 ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
991 if (ret)
992 goto out_free_all;
993
994 return 0;
995
996out_free_all:
997 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
998 dma_free_coherent(dev, sizeof(*ctx->dec_cd),
999 ctx->dec_cd, ctx->dec_cd_paddr);
1000 ctx->dec_cd = NULL;
1001out_free_enc:
1002 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
1003 dma_free_coherent(dev, sizeof(*ctx->enc_cd),
1004 ctx->enc_cd, ctx->enc_cd_paddr);
1005 ctx->enc_cd = NULL;
1006out_free_instance:
1007 ctx->inst = NULL;
1008 qat_crypto_put_instance(inst);
1009 return ret;
1010}
1011
1012static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm,
1013 const u8 *key, unsigned int keylen,
1014 int mode)
1015{
1016 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1017
1018 if (ctx->enc_cd)
1019 return qat_alg_skcipher_rekey(ctx, key, keylen, mode);
1020 else
1021 return qat_alg_skcipher_newkey(ctx, key, keylen, mode);
1022}
1023
1024static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm,
1025 const u8 *key, unsigned int keylen)
1026{
1027 return qat_alg_skcipher_setkey(tfm, key, keylen,
1028 ICP_QAT_HW_CIPHER_CBC_MODE);
1029}
1030
1031static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm,
1032 const u8 *key, unsigned int keylen)
1033{
1034 return qat_alg_skcipher_setkey(tfm, key, keylen,
1035 ICP_QAT_HW_CIPHER_CTR_MODE);
1036}
1037
1038static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm,
1039 const u8 *key, unsigned int keylen)
1040{
1041 return qat_alg_skcipher_setkey(tfm, key, keylen,
1042 ICP_QAT_HW_CIPHER_XTS_MODE);
1043}
1044
1045static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
1046{
1047 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1048 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1049 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1050 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1051 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1052 struct icp_qat_fw_la_bulk_req *msg;
1053 struct device *dev = &GET_DEV(ctx->inst->accel_dev);
1054 int ret, ctr = 0;
1055
1056 if (req->cryptlen == 0)
1057 return 0;
1058
1059 qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE,
1060 &qat_req->iv_paddr, GFP_ATOMIC);
1061 if (!qat_req->iv)
1062 return -ENOMEM;
1063
1064 ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
1065 if (unlikely(ret)) {
1066 dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
1067 qat_req->iv_paddr);
1068 return ret;
1069 }
1070
1071 msg = &qat_req->req;
1072 *msg = ctx->enc_fw_req;
1073 qat_req->skcipher_ctx = ctx;
1074 qat_req->skcipher_req = req;
1075 qat_req->cb = qat_skcipher_alg_callback;
1076 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1077 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1078 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1079 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1080 cipher_param->cipher_length = req->cryptlen;
1081 cipher_param->cipher_offset = 0;
1082 cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr;
1083 memcpy(qat_req->iv, req->iv, AES_BLOCK_SIZE);
1084 do {
1085 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1086 } while (ret == -EAGAIN && ctr++ < 10);
1087
1088 if (ret == -EAGAIN) {
1089 qat_alg_free_bufl(ctx->inst, qat_req);
1090 dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
1091 qat_req->iv_paddr);
1092 return -EBUSY;
1093 }
1094 return -EINPROGRESS;
1095}
1096
1097static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req)
1098{
1099 if (req->cryptlen % AES_BLOCK_SIZE != 0)
1100 return -EINVAL;
1101
1102 return qat_alg_skcipher_encrypt(req);
1103}
1104
1105static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
1106{
1107 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1108 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1109 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1110 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1111 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1112 struct icp_qat_fw_la_bulk_req *msg;
1113 struct device *dev = &GET_DEV(ctx->inst->accel_dev);
1114 int ret, ctr = 0;
1115
1116 if (req->cryptlen == 0)
1117 return 0;
1118
1119 qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE,
1120 &qat_req->iv_paddr, GFP_ATOMIC);
1121 if (!qat_req->iv)
1122 return -ENOMEM;
1123
1124 ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
1125 if (unlikely(ret)) {
1126 dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
1127 qat_req->iv_paddr);
1128 return ret;
1129 }
1130
1131 msg = &qat_req->req;
1132 *msg = ctx->dec_fw_req;
1133 qat_req->skcipher_ctx = ctx;
1134 qat_req->skcipher_req = req;
1135 qat_req->cb = qat_skcipher_alg_callback;
1136 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1137 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1138 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1139 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1140 cipher_param->cipher_length = req->cryptlen;
1141 cipher_param->cipher_offset = 0;
1142 cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr;
1143 memcpy(qat_req->iv, req->iv, AES_BLOCK_SIZE);
1144 do {
1145 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1146 } while (ret == -EAGAIN && ctr++ < 10);
1147
1148 if (ret == -EAGAIN) {
1149 qat_alg_free_bufl(ctx->inst, qat_req);
1150 dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
1151 qat_req->iv_paddr);
1152 return -EBUSY;
1153 }
1154 return -EINPROGRESS;
1155}
1156
1157static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req)
1158{
1159 if (req->cryptlen % AES_BLOCK_SIZE != 0)
1160 return -EINVAL;
1161
1162 return qat_alg_skcipher_decrypt(req);
1163}
1164static int qat_alg_aead_init(struct crypto_aead *tfm,
1165 enum icp_qat_hw_auth_algo hash,
1166 const char *hash_name)
1167{
1168 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1169
1170 ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1171 if (IS_ERR(ctx->hash_tfm))
1172 return PTR_ERR(ctx->hash_tfm);
1173 ctx->qat_hash_alg = hash;
1174 crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1175 return 0;
1176}
1177
1178static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)
1179{
1180 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1181}
1182
1183static int qat_alg_aead_sha256_init(struct crypto_aead *tfm)
1184{
1185 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1186}
1187
1188static int qat_alg_aead_sha512_init(struct crypto_aead *tfm)
1189{
1190 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1191}
1192
1193static void qat_alg_aead_exit(struct crypto_aead *tfm)
1194{
1195 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1196 struct qat_crypto_instance *inst = ctx->inst;
1197 struct device *dev;
1198
1199 crypto_free_shash(ctx->hash_tfm);
1200
1201 if (!inst)
1202 return;
1203
1204 dev = &GET_DEV(inst->accel_dev);
1205 if (ctx->enc_cd) {
1206 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1207 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1208 ctx->enc_cd, ctx->enc_cd_paddr);
1209 }
1210 if (ctx->dec_cd) {
1211 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1212 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1213 ctx->dec_cd, ctx->dec_cd_paddr);
1214 }
1215 qat_crypto_put_instance(inst);
1216}
1217
1218static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm)
1219{
1220 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1221
1222 crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1223 ctx->tfm = tfm;
1224 return 0;
1225}
1226
1227static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)
1228{
1229 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1230 struct qat_crypto_instance *inst = ctx->inst;
1231 struct device *dev;
1232
1233 if (!inst)
1234 return;
1235
1236 dev = &GET_DEV(inst->accel_dev);
1237 if (ctx->enc_cd) {
1238 memset(ctx->enc_cd, 0,
1239 sizeof(struct icp_qat_hw_cipher_algo_blk));
1240 dma_free_coherent(dev,
1241 sizeof(struct icp_qat_hw_cipher_algo_blk),
1242 ctx->enc_cd, ctx->enc_cd_paddr);
1243 }
1244 if (ctx->dec_cd) {
1245 memset(ctx->dec_cd, 0,
1246 sizeof(struct icp_qat_hw_cipher_algo_blk));
1247 dma_free_coherent(dev,
1248 sizeof(struct icp_qat_hw_cipher_algo_blk),
1249 ctx->dec_cd, ctx->dec_cd_paddr);
1250 }
1251 qat_crypto_put_instance(inst);
1252}
1253
1254
1255static struct aead_alg qat_aeads[] = { {
1256 .base = {
1257 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1258 .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1259 .cra_priority = 4001,
1260 .cra_flags = CRYPTO_ALG_ASYNC,
1261 .cra_blocksize = AES_BLOCK_SIZE,
1262 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1263 .cra_module = THIS_MODULE,
1264 },
1265 .init = qat_alg_aead_sha1_init,
1266 .exit = qat_alg_aead_exit,
1267 .setkey = qat_alg_aead_setkey,
1268 .decrypt = qat_alg_aead_dec,
1269 .encrypt = qat_alg_aead_enc,
1270 .ivsize = AES_BLOCK_SIZE,
1271 .maxauthsize = SHA1_DIGEST_SIZE,
1272}, {
1273 .base = {
1274 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1275 .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1276 .cra_priority = 4001,
1277 .cra_flags = CRYPTO_ALG_ASYNC,
1278 .cra_blocksize = AES_BLOCK_SIZE,
1279 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1280 .cra_module = THIS_MODULE,
1281 },
1282 .init = qat_alg_aead_sha256_init,
1283 .exit = qat_alg_aead_exit,
1284 .setkey = qat_alg_aead_setkey,
1285 .decrypt = qat_alg_aead_dec,
1286 .encrypt = qat_alg_aead_enc,
1287 .ivsize = AES_BLOCK_SIZE,
1288 .maxauthsize = SHA256_DIGEST_SIZE,
1289}, {
1290 .base = {
1291 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1292 .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1293 .cra_priority = 4001,
1294 .cra_flags = CRYPTO_ALG_ASYNC,
1295 .cra_blocksize = AES_BLOCK_SIZE,
1296 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1297 .cra_module = THIS_MODULE,
1298 },
1299 .init = qat_alg_aead_sha512_init,
1300 .exit = qat_alg_aead_exit,
1301 .setkey = qat_alg_aead_setkey,
1302 .decrypt = qat_alg_aead_dec,
1303 .encrypt = qat_alg_aead_enc,
1304 .ivsize = AES_BLOCK_SIZE,
1305 .maxauthsize = SHA512_DIGEST_SIZE,
1306} };
1307
1308static struct skcipher_alg qat_skciphers[] = { {
1309 .base.cra_name = "cbc(aes)",
1310 .base.cra_driver_name = "qat_aes_cbc",
1311 .base.cra_priority = 4001,
1312 .base.cra_flags = CRYPTO_ALG_ASYNC,
1313 .base.cra_blocksize = AES_BLOCK_SIZE,
1314 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1315 .base.cra_alignmask = 0,
1316 .base.cra_module = THIS_MODULE,
1317
1318 .init = qat_alg_skcipher_init_tfm,
1319 .exit = qat_alg_skcipher_exit_tfm,
1320 .setkey = qat_alg_skcipher_cbc_setkey,
1321 .decrypt = qat_alg_skcipher_blk_decrypt,
1322 .encrypt = qat_alg_skcipher_blk_encrypt,
1323 .min_keysize = AES_MIN_KEY_SIZE,
1324 .max_keysize = AES_MAX_KEY_SIZE,
1325 .ivsize = AES_BLOCK_SIZE,
1326}, {
1327 .base.cra_name = "ctr(aes)",
1328 .base.cra_driver_name = "qat_aes_ctr",
1329 .base.cra_priority = 4001,
1330 .base.cra_flags = CRYPTO_ALG_ASYNC,
1331 .base.cra_blocksize = 1,
1332 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1333 .base.cra_alignmask = 0,
1334 .base.cra_module = THIS_MODULE,
1335
1336 .init = qat_alg_skcipher_init_tfm,
1337 .exit = qat_alg_skcipher_exit_tfm,
1338 .setkey = qat_alg_skcipher_ctr_setkey,
1339 .decrypt = qat_alg_skcipher_decrypt,
1340 .encrypt = qat_alg_skcipher_encrypt,
1341 .min_keysize = AES_MIN_KEY_SIZE,
1342 .max_keysize = AES_MAX_KEY_SIZE,
1343 .ivsize = AES_BLOCK_SIZE,
1344}, {
1345 .base.cra_name = "xts(aes)",
1346 .base.cra_driver_name = "qat_aes_xts",
1347 .base.cra_priority = 4001,
1348 .base.cra_flags = CRYPTO_ALG_ASYNC,
1349 .base.cra_blocksize = AES_BLOCK_SIZE,
1350 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1351 .base.cra_alignmask = 0,
1352 .base.cra_module = THIS_MODULE,
1353
1354 .init = qat_alg_skcipher_init_tfm,
1355 .exit = qat_alg_skcipher_exit_tfm,
1356 .setkey = qat_alg_skcipher_xts_setkey,
1357 .decrypt = qat_alg_skcipher_blk_decrypt,
1358 .encrypt = qat_alg_skcipher_blk_encrypt,
1359 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1360 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1361 .ivsize = AES_BLOCK_SIZE,
1362} };
1363
1364int qat_algs_register(void)
1365{
1366 int ret = 0;
1367
1368 mutex_lock(&algs_lock);
1369 if (++active_devs != 1)
1370 goto unlock;
1371
1372 ret = crypto_register_skciphers(qat_skciphers,
1373 ARRAY_SIZE(qat_skciphers));
1374 if (ret)
1375 goto unlock;
1376
1377 ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1378 if (ret)
1379 goto unreg_algs;
1380
1381unlock:
1382 mutex_unlock(&algs_lock);
1383 return ret;
1384
1385unreg_algs:
1386 crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1387 goto unlock;
1388}
1389
1390void qat_algs_unregister(void)
1391{
1392 mutex_lock(&algs_lock);
1393 if (--active_devs != 0)
1394 goto unlock;
1395
1396 crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1397 crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1398
1399unlock:
1400 mutex_unlock(&algs_lock);
1401}
1402
