1
2
3
4
5
6
7
8
9
10#include <crypto/aes.h>
11#include <crypto/algapi.h>
12#include <crypto/authenc.h>
13#include <crypto/cryptd.h>
14#include <crypto/crypto_wq.h>
15#include <crypto/des.h>
16#include <crypto/xts.h>
17#include <linux/crypto.h>
18#include <linux/err.h>
19#include <linux/list.h>
20#include <linux/scatterlist.h>
21
22#include "cptvf.h"
23#include "cptvf_algs.h"
24
25struct cpt_device_handle {
26 void *cdev[MAX_DEVICES];
27 u32 dev_count;
28};
29
30static struct cpt_device_handle dev_handle;
31
32static void cvm_callback(u32 status, void *arg)
33{
34 struct crypto_async_request *req = (struct crypto_async_request *)arg;
35
36 req->complete(req, !status);
37}
38
39static inline void update_input_iv(struct cpt_request_info *req_info,
40 u8 *iv, u32 enc_iv_len,
41 u32 *argcnt)
42{
43
44 req_info->in[*argcnt].vptr = (void *)iv;
45 req_info->in[*argcnt].size = enc_iv_len;
46 req_info->req.dlen += enc_iv_len;
47
48 ++(*argcnt);
49}
50
51static inline void update_output_iv(struct cpt_request_info *req_info,
52 u8 *iv, u32 enc_iv_len,
53 u32 *argcnt)
54{
55
56 req_info->out[*argcnt].vptr = (void *)iv;
57 req_info->out[*argcnt].size = enc_iv_len;
58 req_info->rlen += enc_iv_len;
59
60 ++(*argcnt);
61}
62
63static inline void update_input_data(struct cpt_request_info *req_info,
64 struct scatterlist *inp_sg,
65 u32 nbytes, u32 *argcnt)
66{
67 req_info->req.dlen += nbytes;
68
69 while (nbytes) {
70 u32 len = min(nbytes, inp_sg->length);
71 u8 *ptr = sg_virt(inp_sg);
72
73 req_info->in[*argcnt].vptr = (void *)ptr;
74 req_info->in[*argcnt].size = len;
75 nbytes -= len;
76
77 ++(*argcnt);
78 ++inp_sg;
79 }
80}
81
82static inline void update_output_data(struct cpt_request_info *req_info,
83 struct scatterlist *outp_sg,
84 u32 nbytes, u32 *argcnt)
85{
86 req_info->rlen += nbytes;
87
88 while (nbytes) {
89 u32 len = min(nbytes, outp_sg->length);
90 u8 *ptr = sg_virt(outp_sg);
91
92 req_info->out[*argcnt].vptr = (void *)ptr;
93 req_info->out[*argcnt].size = len;
94 nbytes -= len;
95 ++(*argcnt);
96 ++outp_sg;
97 }
98}
99
100static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
101 u32 cipher_type, u32 aes_key_type,
102 u32 *argcnt)
103{
104 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
105 struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
106 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
107 struct fc_context *fctx = &rctx->fctx;
108 u64 *offset_control = &rctx->control_word;
109 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
110 struct cpt_request_info *req_info = &rctx->cpt_req;
111 u64 *ctrl_flags = NULL;
112
113 req_info->ctrl.s.grp = 0;
114 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
115 req_info->ctrl.s.se_req = SE_CORE_REQ;
116
117 req_info->req.opcode.s.major = MAJOR_OP_FC |
118 DMA_MODE_FLAG(DMA_GATHER_SCATTER);
119 if (enc)
120 req_info->req.opcode.s.minor = 2;
121 else
122 req_info->req.opcode.s.minor = 3;
123
124 req_info->req.param1 = req->nbytes;
125 req_info->req.param2 = 0;
126
127 fctx->enc.enc_ctrl.e.enc_cipher = cipher_type;
128 fctx->enc.enc_ctrl.e.aes_key = aes_key_type;
129 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
130
131 if (cipher_type == AES_XTS)
132 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
133 else
134 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
135 ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
136 *ctrl_flags = cpu_to_be64(*ctrl_flags);
137
138 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
139
140
141
142 req_info->in[*argcnt].vptr = (u8 *)offset_control;
143 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
144 req_info->req.dlen += CONTROL_WORD_LEN;
145 ++(*argcnt);
146
147 req_info->in[*argcnt].vptr = (u8 *)fctx;
148 req_info->in[*argcnt].size = sizeof(struct fc_context);
149 req_info->req.dlen += sizeof(struct fc_context);
150
151 ++(*argcnt);
152
153 return 0;
154}
155
156static inline u32 create_input_list(struct ablkcipher_request *req, u32 enc,
157 u32 cipher_type, u32 aes_key_type,
158 u32 enc_iv_len)
159{
160 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
161 struct cpt_request_info *req_info = &rctx->cpt_req;
162 u32 argcnt = 0;
163
164 create_ctx_hdr(req, enc, cipher_type, aes_key_type, &argcnt);
165 update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
166 update_input_data(req_info, req->src, req->nbytes, &argcnt);
167 req_info->incnt = argcnt;
168
169 return 0;
170}
171
172static inline void store_cb_info(struct ablkcipher_request *req,
173 struct cpt_request_info *req_info)
174{
175 req_info->callback = (void *)cvm_callback;
176 req_info->callback_arg = (void *)&req->base;
177}
178
179static inline void create_output_list(struct ablkcipher_request *req,
180 u32 cipher_type,
181 u32 enc_iv_len)
182{
183 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
184 struct cpt_request_info *req_info = &rctx->cpt_req;
185 u32 argcnt = 0;
186
187
188
189
190
191
192
193
194
195 update_output_iv(req_info, req->info, enc_iv_len, &argcnt);
196 update_output_data(req_info, req->dst, req->nbytes, &argcnt);
197 req_info->outcnt = argcnt;
198}
199
200static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc,
201 u32 cipher_type)
202{
203 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
204 struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
205 u32 key_type = AES_128_BIT;
206 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
207 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
208 struct fc_context *fctx = &rctx->fctx;
209 struct cpt_request_info *req_info = &rctx->cpt_req;
210 void *cdev = NULL;
211 int status;
212
213 switch (ctx->key_len) {
214 case 16:
215 key_type = AES_128_BIT;
216 break;
217 case 24:
218 key_type = AES_192_BIT;
219 break;
220 case 32:
221 if (cipher_type == AES_XTS)
222 key_type = AES_128_BIT;
223 else
224 key_type = AES_256_BIT;
225 break;
226 case 64:
227 if (cipher_type == AES_XTS)
228 key_type = AES_256_BIT;
229 else
230 return -EINVAL;
231 break;
232 default:
233 return -EINVAL;
234 }
235
236 if (cipher_type == DES3_CBC)
237 key_type = 0;
238
239 memset(req_info, 0, sizeof(struct cpt_request_info));
240 memset(fctx, 0, sizeof(struct fc_context));
241 create_input_list(req, enc, cipher_type, key_type, enc_iv_len);
242 create_output_list(req, cipher_type, enc_iv_len);
243 store_cb_info(req, req_info);
244 cdev = dev_handle.cdev[smp_processor_id()];
245 status = cptvf_do_request(cdev, req_info);
246
247
248
249
250
251 if (status)
252 return status;
253 else
254 return -EINPROGRESS;
255}
256
257int cvm_des3_encrypt_cbc(struct ablkcipher_request *req)
258{
259 return cvm_enc_dec(req, true, DES3_CBC);
260}
261
262int cvm_des3_decrypt_cbc(struct ablkcipher_request *req)
263{
264 return cvm_enc_dec(req, false, DES3_CBC);
265}
266
267int cvm_aes_encrypt_xts(struct ablkcipher_request *req)
268{
269 return cvm_enc_dec(req, true, AES_XTS);
270}
271
272int cvm_aes_decrypt_xts(struct ablkcipher_request *req)
273{
274 return cvm_enc_dec(req, false, AES_XTS);
275}
276
277int cvm_aes_encrypt_cbc(struct ablkcipher_request *req)
278{
279 return cvm_enc_dec(req, true, AES_CBC);
280}
281
282int cvm_aes_decrypt_cbc(struct ablkcipher_request *req)
283{
284 return cvm_enc_dec(req, false, AES_CBC);
285}
286
287int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
288 u32 keylen)
289{
290 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
291 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
292 int err;
293 const u8 *key1 = key;
294 const u8 *key2 = key + (keylen / 2);
295
296 err = xts_check_key(tfm, key, keylen);
297 if (err)
298 return err;
299 ctx->key_len = keylen;
300 memcpy(ctx->enc_key, key1, keylen / 2);
301 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
302
303 return 0;
304}
305
306int cvm_enc_dec_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
307 u32 keylen)
308{
309 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
310 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
311
312 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
313 ctx->key_len = keylen;
314 memcpy(ctx->enc_key, key, keylen);
315 return 0;
316 }
317 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
318
319 return -EINVAL;
320}
321
322int cvm_enc_dec_init(struct crypto_tfm *tfm)
323{
324 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
325
326 memset(ctx, 0, sizeof(*ctx));
327 tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx) +
328 sizeof(struct ablkcipher_request);
329
330
331
332
333
334 return 0;
335}
336
337struct crypto_alg algs[] = { {
338 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
339 .cra_blocksize = AES_BLOCK_SIZE,
340 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
341 .cra_alignmask = 7,
342 .cra_priority = 4001,
343 .cra_name = "xts(aes)",
344 .cra_driver_name = "cavium-xts-aes",
345 .cra_type = &crypto_ablkcipher_type,
346 .cra_u = {
347 .ablkcipher = {
348 .ivsize = AES_BLOCK_SIZE,
349 .min_keysize = 2 * AES_MIN_KEY_SIZE,
350 .max_keysize = 2 * AES_MAX_KEY_SIZE,
351 .setkey = cvm_xts_setkey,
352 .encrypt = cvm_aes_encrypt_xts,
353 .decrypt = cvm_aes_decrypt_xts,
354 },
355 },
356 .cra_init = cvm_enc_dec_init,
357 .cra_module = THIS_MODULE,
358}, {
359 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
360 .cra_blocksize = AES_BLOCK_SIZE,
361 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
362 .cra_alignmask = 7,
363 .cra_priority = 4001,
364 .cra_name = "cbc(aes)",
365 .cra_driver_name = "cavium-cbc-aes",
366 .cra_type = &crypto_ablkcipher_type,
367 .cra_u = {
368 .ablkcipher = {
369 .ivsize = AES_BLOCK_SIZE,
370 .min_keysize = AES_MIN_KEY_SIZE,
371 .max_keysize = AES_MAX_KEY_SIZE,
372 .setkey = cvm_enc_dec_setkey,
373 .encrypt = cvm_aes_encrypt_cbc,
374 .decrypt = cvm_aes_decrypt_cbc,
375 },
376 },
377 .cra_init = cvm_enc_dec_init,
378 .cra_module = THIS_MODULE,
379}, {
380 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
381 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
382 .cra_ctxsize = sizeof(struct cvm_des3_ctx),
383 .cra_alignmask = 7,
384 .cra_priority = 4001,
385 .cra_name = "cbc(des3_ede)",
386 .cra_driver_name = "cavium-cbc-des3_ede",
387 .cra_type = &crypto_ablkcipher_type,
388 .cra_u = {
389 .ablkcipher = {
390 .min_keysize = DES3_EDE_KEY_SIZE,
391 .max_keysize = DES3_EDE_KEY_SIZE,
392 .ivsize = DES_BLOCK_SIZE,
393 .setkey = cvm_enc_dec_setkey,
394 .encrypt = cvm_des3_encrypt_cbc,
395 .decrypt = cvm_des3_decrypt_cbc,
396 },
397 },
398 .cra_init = cvm_enc_dec_init,
399 .cra_module = THIS_MODULE,
400} };
401
402static inline int cav_register_algs(void)
403{
404 int err = 0;
405
406 err = crypto_register_algs(algs, ARRAY_SIZE(algs));
407 if (err)
408 return err;
409
410 return 0;
411}
412
413static inline void cav_unregister_algs(void)
414{
415 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
416}
417
418int cvm_crypto_init(struct cpt_vf *cptvf)
419{
420 struct pci_dev *pdev = cptvf->pdev;
421 u32 dev_count;
422
423 dev_count = dev_handle.dev_count;
424 dev_handle.cdev[dev_count] = cptvf;
425 dev_handle.dev_count++;
426
427 if (dev_count == 3) {
428 if (cav_register_algs()) {
429 dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
430 return -EINVAL;
431 }
432 }
433
434 return 0;
435}
436
437void cvm_crypto_exit(void)
438{
439 u32 dev_count;
440
441 dev_count = --dev_handle.dev_count;
442 if (!dev_count)
443 cav_unregister_algs();
444}
445
