1
2
3
4
5
6
7
8
9
10#include <crypto/aes.h>
11#include <crypto/algapi.h>
12#include <crypto/authenc.h>
13#include <crypto/cryptd.h>
14#include <crypto/crypto_wq.h>
15#include <crypto/des.h>
16#include <crypto/xts.h>
17#include <linux/crypto.h>
18#include <linux/err.h>
19#include <linux/list.h>
20#include <linux/scatterlist.h>
21
22#include "cptvf.h"
23#include "cptvf_algs.h"
24
25struct cpt_device_handle {
26 void *cdev[MAX_DEVICES];
27 u32 dev_count;
28};
29
30static struct cpt_device_handle dev_handle;
31
32static void cvm_callback(u32 status, void *arg)
33{
34 struct crypto_async_request *req = (struct crypto_async_request *)arg;
35
36 req->complete(req, !status);
37}
38
39static inline void update_input_iv(struct cpt_request_info *req_info,
40 u8 *iv, u32 enc_iv_len,
41 u32 *argcnt)
42{
43
44 req_info->in[*argcnt].vptr = (void *)iv;
45 req_info->in[*argcnt].size = enc_iv_len;
46 req_info->req.dlen += enc_iv_len;
47
48 ++(*argcnt);
49}
50
51static inline void update_output_iv(struct cpt_request_info *req_info,
52 u8 *iv, u32 enc_iv_len,
53 u32 *argcnt)
54{
55
56 req_info->out[*argcnt].vptr = (void *)iv;
57 req_info->out[*argcnt].size = enc_iv_len;
58 req_info->rlen += enc_iv_len;
59
60 ++(*argcnt);
61}
62
63static inline void update_input_data(struct cpt_request_info *req_info,
64 struct scatterlist *inp_sg,
65 u32 nbytes, u32 *argcnt)
66{
67 req_info->req.dlen += nbytes;
68
69 while (nbytes) {
70 u32 len = min(nbytes, inp_sg->length);
71 u8 *ptr = sg_virt(inp_sg);
72
73 req_info->in[*argcnt].vptr = (void *)ptr;
74 req_info->in[*argcnt].size = len;
75 nbytes -= len;
76
77 ++(*argcnt);
78 ++inp_sg;
79 }
80}
81
82static inline void update_output_data(struct cpt_request_info *req_info,
83 struct scatterlist *outp_sg,
84 u32 nbytes, u32 *argcnt)
85{
86 req_info->rlen += nbytes;
87
88 while (nbytes) {
89 u32 len = min(nbytes, outp_sg->length);
90 u8 *ptr = sg_virt(outp_sg);
91
92 req_info->out[*argcnt].vptr = (void *)ptr;
93 req_info->out[*argcnt].size = len;
94 nbytes -= len;
95 ++(*argcnt);
96 ++outp_sg;
97 }
98}
99
100static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
101 u32 *argcnt)
102{
103 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
104 struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
105 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
106 struct fc_context *fctx = &rctx->fctx;
107 u64 *offset_control = &rctx->control_word;
108 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
109 struct cpt_request_info *req_info = &rctx->cpt_req;
110 u64 *ctrl_flags = NULL;
111
112 req_info->ctrl.s.grp = 0;
113 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
114 req_info->ctrl.s.se_req = SE_CORE_REQ;
115
116 req_info->req.opcode.s.major = MAJOR_OP_FC |
117 DMA_MODE_FLAG(DMA_GATHER_SCATTER);
118 if (enc)
119 req_info->req.opcode.s.minor = 2;
120 else
121 req_info->req.opcode.s.minor = 3;
122
123 req_info->req.param1 = req->nbytes;
124 req_info->req.param2 = 0;
125
126 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
127 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
128 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
129
130 if (ctx->cipher_type == AES_XTS)
131 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
132 else
133 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
134 ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
135 *ctrl_flags = cpu_to_be64(*ctrl_flags);
136
137 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
138
139
140
141 req_info->in[*argcnt].vptr = (u8 *)offset_control;
142 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
143 req_info->req.dlen += CONTROL_WORD_LEN;
144 ++(*argcnt);
145
146 req_info->in[*argcnt].vptr = (u8 *)fctx;
147 req_info->in[*argcnt].size = sizeof(struct fc_context);
148 req_info->req.dlen += sizeof(struct fc_context);
149
150 ++(*argcnt);
151
152 return 0;
153}
154
155static inline u32 create_input_list(struct ablkcipher_request *req, u32 enc,
156 u32 enc_iv_len)
157{
158 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
159 struct cpt_request_info *req_info = &rctx->cpt_req;
160 u32 argcnt = 0;
161
162 create_ctx_hdr(req, enc, &argcnt);
163 update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
164 update_input_data(req_info, req->src, req->nbytes, &argcnt);
165 req_info->incnt = argcnt;
166
167 return 0;
168}
169
170static inline void store_cb_info(struct ablkcipher_request *req,
171 struct cpt_request_info *req_info)
172{
173 req_info->callback = (void *)cvm_callback;
174 req_info->callback_arg = (void *)&req->base;
175}
176
177static inline void create_output_list(struct ablkcipher_request *req,
178 u32 enc_iv_len)
179{
180 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
181 struct cpt_request_info *req_info = &rctx->cpt_req;
182 u32 argcnt = 0;
183
184
185
186
187
188
189
190
191
192 update_output_iv(req_info, req->info, enc_iv_len, &argcnt);
193 update_output_data(req_info, req->dst, req->nbytes, &argcnt);
194 req_info->outcnt = argcnt;
195}
196
197static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc)
198{
199 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
200 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
201 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
202 struct fc_context *fctx = &rctx->fctx;
203 struct cpt_request_info *req_info = &rctx->cpt_req;
204 void *cdev = NULL;
205 int status;
206
207 memset(req_info, 0, sizeof(struct cpt_request_info));
208 memset(fctx, 0, sizeof(struct fc_context));
209 create_input_list(req, enc, enc_iv_len);
210 create_output_list(req, enc_iv_len);
211 store_cb_info(req, req_info);
212 cdev = dev_handle.cdev[smp_processor_id()];
213 status = cptvf_do_request(cdev, req_info);
214
215
216
217
218
219 if (status)
220 return status;
221 else
222 return -EINPROGRESS;
223}
224
225static int cvm_encrypt(struct ablkcipher_request *req)
226{
227 return cvm_enc_dec(req, true);
228}
229
230static int cvm_decrypt(struct ablkcipher_request *req)
231{
232 return cvm_enc_dec(req, false);
233}
234
235static int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
236 u32 keylen)
237{
238 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
239 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
240 int err;
241 const u8 *key1 = key;
242 const u8 *key2 = key + (keylen / 2);
243
244 err = xts_check_key(tfm, key, keylen);
245 if (err)
246 return err;
247 ctx->key_len = keylen;
248 memcpy(ctx->enc_key, key1, keylen / 2);
249 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
250 ctx->cipher_type = AES_XTS;
251 switch (ctx->key_len) {
252 case 32:
253 ctx->key_type = AES_128_BIT;
254 break;
255 case 64:
256 ctx->key_type = AES_256_BIT;
257 break;
258 default:
259 return -EINVAL;
260 }
261
262 return 0;
263}
264
265static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
266{
267 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
268 ctx->key_len = keylen;
269 switch (ctx->key_len) {
270 case 16:
271 ctx->key_type = AES_128_BIT;
272 break;
273 case 24:
274 ctx->key_type = AES_192_BIT;
275 break;
276 case 32:
277 ctx->key_type = AES_256_BIT;
278 break;
279 default:
280 return -EINVAL;
281 }
282
283 if (ctx->cipher_type == DES3_CBC)
284 ctx->key_type = 0;
285
286 return 0;
287 }
288
289 return -EINVAL;
290}
291
292static int cvm_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
293 u32 keylen, u8 cipher_type)
294{
295 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
296 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
297
298 ctx->cipher_type = cipher_type;
299 if (!cvm_validate_keylen(ctx, keylen)) {
300 memcpy(ctx->enc_key, key, keylen);
301 return 0;
302 } else {
303 crypto_ablkcipher_set_flags(cipher,
304 CRYPTO_TFM_RES_BAD_KEY_LEN);
305 return -EINVAL;
306 }
307}
308
309static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
310 u32 keylen)
311{
312 return cvm_setkey(cipher, key, keylen, AES_CBC);
313}
314
315static int cvm_ecb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
316 u32 keylen)
317{
318 return cvm_setkey(cipher, key, keylen, AES_ECB);
319}
320
321static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
322 u32 keylen)
323{
324 return cvm_setkey(cipher, key, keylen, AES_CFB);
325}
326
327static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
328 u32 keylen)
329{
330 return cvm_setkey(cipher, key, keylen, DES3_CBC);
331}
332
333static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
334 u32 keylen)
335{
336 return cvm_setkey(cipher, key, keylen, DES3_ECB);
337}
338
339static int cvm_enc_dec_init(struct crypto_tfm *tfm)
340{
341 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
342
343 memset(ctx, 0, sizeof(*ctx));
344 tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx) +
345 sizeof(struct ablkcipher_request);
346
347
348
349
350
351 return 0;
352}
353
354struct crypto_alg algs[] = { {
355 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
356 .cra_blocksize = AES_BLOCK_SIZE,
357 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
358 .cra_alignmask = 7,
359 .cra_priority = 4001,
360 .cra_name = "xts(aes)",
361 .cra_driver_name = "cavium-xts-aes",
362 .cra_type = &crypto_ablkcipher_type,
363 .cra_u = {
364 .ablkcipher = {
365 .ivsize = AES_BLOCK_SIZE,
366 .min_keysize = 2 * AES_MIN_KEY_SIZE,
367 .max_keysize = 2 * AES_MAX_KEY_SIZE,
368 .setkey = cvm_xts_setkey,
369 .encrypt = cvm_encrypt,
370 .decrypt = cvm_decrypt,
371 },
372 },
373 .cra_init = cvm_enc_dec_init,
374 .cra_module = THIS_MODULE,
375}, {
376 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
377 .cra_blocksize = AES_BLOCK_SIZE,
378 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
379 .cra_alignmask = 7,
380 .cra_priority = 4001,
381 .cra_name = "cbc(aes)",
382 .cra_driver_name = "cavium-cbc-aes",
383 .cra_type = &crypto_ablkcipher_type,
384 .cra_u = {
385 .ablkcipher = {
386 .ivsize = AES_BLOCK_SIZE,
387 .min_keysize = AES_MIN_KEY_SIZE,
388 .max_keysize = AES_MAX_KEY_SIZE,
389 .setkey = cvm_cbc_aes_setkey,
390 .encrypt = cvm_encrypt,
391 .decrypt = cvm_decrypt,
392 },
393 },
394 .cra_init = cvm_enc_dec_init,
395 .cra_module = THIS_MODULE,
396}, {
397 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
398 .cra_blocksize = AES_BLOCK_SIZE,
399 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
400 .cra_alignmask = 7,
401 .cra_priority = 4001,
402 .cra_name = "ecb(aes)",
403 .cra_driver_name = "cavium-ecb-aes",
404 .cra_type = &crypto_ablkcipher_type,
405 .cra_u = {
406 .ablkcipher = {
407 .ivsize = AES_BLOCK_SIZE,
408 .min_keysize = AES_MIN_KEY_SIZE,
409 .max_keysize = AES_MAX_KEY_SIZE,
410 .setkey = cvm_ecb_aes_setkey,
411 .encrypt = cvm_encrypt,
412 .decrypt = cvm_decrypt,
413 },
414 },
415 .cra_init = cvm_enc_dec_init,
416 .cra_module = THIS_MODULE,
417}, {
418 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
419 .cra_blocksize = AES_BLOCK_SIZE,
420 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
421 .cra_alignmask = 7,
422 .cra_priority = 4001,
423 .cra_name = "cfb(aes)",
424 .cra_driver_name = "cavium-cfb-aes",
425 .cra_type = &crypto_ablkcipher_type,
426 .cra_u = {
427 .ablkcipher = {
428 .ivsize = AES_BLOCK_SIZE,
429 .min_keysize = AES_MIN_KEY_SIZE,
430 .max_keysize = AES_MAX_KEY_SIZE,
431 .setkey = cvm_cfb_aes_setkey,
432 .encrypt = cvm_encrypt,
433 .decrypt = cvm_decrypt,
434 },
435 },
436 .cra_init = cvm_enc_dec_init,
437 .cra_module = THIS_MODULE,
438}, {
439 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
440 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
441 .cra_ctxsize = sizeof(struct cvm_des3_ctx),
442 .cra_alignmask = 7,
443 .cra_priority = 4001,
444 .cra_name = "cbc(des3_ede)",
445 .cra_driver_name = "cavium-cbc-des3_ede",
446 .cra_type = &crypto_ablkcipher_type,
447 .cra_u = {
448 .ablkcipher = {
449 .min_keysize = DES3_EDE_KEY_SIZE,
450 .max_keysize = DES3_EDE_KEY_SIZE,
451 .ivsize = DES_BLOCK_SIZE,
452 .setkey = cvm_cbc_des3_setkey,
453 .encrypt = cvm_encrypt,
454 .decrypt = cvm_decrypt,
455 },
456 },
457 .cra_init = cvm_enc_dec_init,
458 .cra_module = THIS_MODULE,
459}, {
460 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
461 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
462 .cra_ctxsize = sizeof(struct cvm_des3_ctx),
463 .cra_alignmask = 7,
464 .cra_priority = 4001,
465 .cra_name = "ecb(des3_ede)",
466 .cra_driver_name = "cavium-ecb-des3_ede",
467 .cra_type = &crypto_ablkcipher_type,
468 .cra_u = {
469 .ablkcipher = {
470 .min_keysize = DES3_EDE_KEY_SIZE,
471 .max_keysize = DES3_EDE_KEY_SIZE,
472 .ivsize = DES_BLOCK_SIZE,
473 .setkey = cvm_ecb_des3_setkey,
474 .encrypt = cvm_encrypt,
475 .decrypt = cvm_decrypt,
476 },
477 },
478 .cra_init = cvm_enc_dec_init,
479 .cra_module = THIS_MODULE,
480} };
481
482static inline int cav_register_algs(void)
483{
484 int err = 0;
485
486 err = crypto_register_algs(algs, ARRAY_SIZE(algs));
487 if (err)
488 return err;
489
490 return 0;
491}
492
493static inline void cav_unregister_algs(void)
494{
495 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
496}
497
498int cvm_crypto_init(struct cpt_vf *cptvf)
499{
500 struct pci_dev *pdev = cptvf->pdev;
501 u32 dev_count;
502
503 dev_count = dev_handle.dev_count;
504 dev_handle.cdev[dev_count] = cptvf;
505 dev_handle.dev_count++;
506
507 if (dev_count == 3) {
508 if (cav_register_algs()) {
509 dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
510 return -EINVAL;
511 }
512 }
513
514 return 0;
515}
516
517void cvm_crypto_exit(void)
518{
519 u32 dev_count;
520
521 dev_count = --dev_handle.dev_count;
522 if (!dev_count)
523 cav_unregister_algs();
524}
525
