1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46#include "common.h"
47#include "zip_crypto.h"
48
49#define DRV_NAME "ThunderX-ZIP"
50
51static struct zip_device *zip_dev[MAX_ZIP_DEVICES];
52
53static const struct pci_device_id zip_id_table[] = {
54 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDERX_ZIP) },
55 { 0, }
56};
57
58void zip_reg_write(u64 val, u64 __iomem *addr)
59{
60 writeq(val, addr);
61}
62
63u64 zip_reg_read(u64 __iomem *addr)
64{
65 return readq(addr);
66}
67
68
69
70
71
72static struct zip_device *zip_alloc_device(struct pci_dev *pdev)
73{
74 struct zip_device *zip = NULL;
75 int idx;
76
77 for (idx = 0; idx < MAX_ZIP_DEVICES; idx++) {
78 if (!zip_dev[idx])
79 break;
80 }
81
82
83 if (idx < MAX_ZIP_DEVICES)
84 zip = devm_kzalloc(&pdev->dev, sizeof(*zip), GFP_KERNEL);
85
86 if (!zip)
87 return NULL;
88
89 zip_dev[idx] = zip;
90 zip->index = idx;
91 return zip;
92}
93
94
95
96
97
98
99
100struct zip_device *zip_get_device(int node)
101{
102 if ((node < MAX_ZIP_DEVICES) && (node >= 0))
103 return zip_dev[node];
104
105 zip_err("ZIP device not found for node id %d\n", node);
106 return NULL;
107}
108
109
110
111
112
113
114int zip_get_node_id(void)
115{
116 return cpu_to_node(raw_smp_processor_id());
117}
118
119
120static int zip_init_hw(struct zip_device *zip)
121{
122 union zip_cmd_ctl cmd_ctl;
123 union zip_constants constants;
124 union zip_que_ena que_ena;
125 union zip_quex_map que_map;
126 union zip_que_pri que_pri;
127
128 union zip_quex_sbuf_addr que_sbuf_addr;
129 union zip_quex_sbuf_ctl que_sbuf_ctl;
130
131 int q = 0;
132
133
134 cmd_ctl.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CMD_CTL);
135 cmd_ctl.s.forceclk = 1;
136 zip_reg_write(cmd_ctl.u_reg64 & 0xFF, (zip->reg_base + ZIP_CMD_CTL));
137
138 zip_msg("ZIP_CMD_CTL : 0x%016llx",
139 zip_reg_read(zip->reg_base + ZIP_CMD_CTL));
140
141 constants.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CONSTANTS);
142 zip->depth = constants.s.depth;
143 zip->onfsize = constants.s.onfsize;
144 zip->ctxsize = constants.s.ctxsize;
145
146 zip_msg("depth: 0x%016llx , onfsize : 0x%016llx , ctxsize : 0x%016llx",
147 zip->depth, zip->onfsize, zip->ctxsize);
148
149
150
151
152
153
154 for (q = 0; q < ZIP_NUM_QUEUES; q++) {
155 que_sbuf_ctl.u_reg64 = 0ull;
156 que_sbuf_ctl.s.size = (ZIP_CMD_QBUF_SIZE / sizeof(u64));
157 que_sbuf_ctl.s.inst_be = 0;
158 que_sbuf_ctl.s.stream_id = 0;
159 zip_reg_write(que_sbuf_ctl.u_reg64,
160 (zip->reg_base + ZIP_QUEX_SBUF_CTL(q)));
161
162 zip_msg("QUEX_SBUF_CTL[%d]: 0x%016llx", q,
163 zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_CTL(q)));
164 }
165
166 for (q = 0; q < ZIP_NUM_QUEUES; q++) {
167 memset(&zip->iq[q], 0x0, sizeof(struct zip_iq));
168
169 spin_lock_init(&zip->iq[q].lock);
170
171 if (zip_cmd_qbuf_alloc(zip, q)) {
172 while (q != 0) {
173 q--;
174 zip_cmd_qbuf_free(zip, q);
175 }
176 return -ENOMEM;
177 }
178
179
180 zip->iq[q].sw_tail = zip->iq[q].sw_head;
181 zip->iq[q].hw_tail = zip->iq[q].sw_head;
182
183
184 que_sbuf_addr.u_reg64 = 0ull;
185 que_sbuf_addr.s.ptr = (__pa(zip->iq[q].sw_head) >>
186 ZIP_128B_ALIGN);
187
188 zip_msg("QUE[%d]_PTR(PHYS): 0x%016llx", q,
189 (u64)que_sbuf_addr.s.ptr);
190
191 zip_reg_write(que_sbuf_addr.u_reg64,
192 (zip->reg_base + ZIP_QUEX_SBUF_ADDR(q)));
193
194 zip_msg("QUEX_SBUF_ADDR[%d]: 0x%016llx", q,
195 zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_ADDR(q)));
196
197 zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx",
198 zip->iq[q].sw_head, zip->iq[q].sw_tail,
199 zip->iq[q].hw_tail);
200 zip_dbg("sw_head phy addr : 0x%lx", que_sbuf_addr.s.ptr);
201 }
202
203
204
205
206
207
208 que_ena.u_reg64 = 0x0ull;
209
210 for (q = 0; q < ZIP_NUM_QUEUES; q++)
211 que_ena.s.ena |= (0x1 << q);
212 zip_reg_write(que_ena.u_reg64, (zip->reg_base + ZIP_QUE_ENA));
213
214 zip_msg("QUE_ENA : 0x%016llx",
215 zip_reg_read(zip->reg_base + ZIP_QUE_ENA));
216
217 for (q = 0; q < ZIP_NUM_QUEUES; q++) {
218 que_map.u_reg64 = 0ull;
219
220 que_map.s.zce = 0x3;
221 zip_reg_write(que_map.u_reg64,
222 (zip->reg_base + ZIP_QUEX_MAP(q)));
223
224 zip_msg("QUE_MAP(%d) : 0x%016llx", q,
225 zip_reg_read(zip->reg_base + ZIP_QUEX_MAP(q)));
226 }
227
228 que_pri.u_reg64 = 0ull;
229 for (q = 0; q < ZIP_NUM_QUEUES; q++)
230 que_pri.s.pri |= (0x1 << q);
231 zip_reg_write(que_pri.u_reg64, (zip->reg_base + ZIP_QUE_PRI));
232
233 zip_msg("QUE_PRI %016llx", zip_reg_read(zip->reg_base + ZIP_QUE_PRI));
234
235 return 0;
236}
237
238static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
239{
240 struct device *dev = &pdev->dev;
241 struct zip_device *zip = NULL;
242 int err;
243
244 zip = zip_alloc_device(pdev);
245 if (!zip)
246 return -ENOMEM;
247
248 dev_info(dev, "Found ZIP device %d %x:%x on Node %d\n", zip->index,
249 pdev->vendor, pdev->device, dev_to_node(dev));
250
251 pci_set_drvdata(pdev, zip);
252 zip->pdev = pdev;
253
254 err = pci_enable_device(pdev);
255 if (err) {
256 dev_err(dev, "Failed to enable PCI device");
257 goto err_free_device;
258 }
259
260 err = pci_request_regions(pdev, DRV_NAME);
261 if (err) {
262 dev_err(dev, "PCI request regions failed 0x%x", err);
263 goto err_disable_device;
264 }
265
266 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
267 if (err) {
268 dev_err(dev, "Unable to get usable DMA configuration\n");
269 goto err_release_regions;
270 }
271
272 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
273 if (err) {
274 dev_err(dev, "Unable to get 48-bit DMA for allocations\n");
275 goto err_release_regions;
276 }
277
278
279 zip->reg_base = pci_ioremap_bar(pdev, PCI_CFG_ZIP_PF_BAR0);
280 if (!zip->reg_base) {
281 dev_err(dev, "ZIP: Cannot map BAR0 CSR memory space, aborting");
282 err = -ENOMEM;
283 goto err_release_regions;
284 }
285
286
287 err = zip_init_hw(zip);
288 if (err)
289 goto err_release_regions;
290
291 return 0;
292
293err_release_regions:
294 if (zip->reg_base)
295 iounmap(zip->reg_base);
296 pci_release_regions(pdev);
297
298err_disable_device:
299 pci_disable_device(pdev);
300
301err_free_device:
302 pci_set_drvdata(pdev, NULL);
303
304
305 zip_dev[zip->index] = NULL;
306 devm_kfree(dev, zip);
307
308 return err;
309}
310
311static void zip_remove(struct pci_dev *pdev)
312{
313 struct zip_device *zip = pci_get_drvdata(pdev);
314 union zip_cmd_ctl cmd_ctl;
315 int q = 0;
316
317 if (!zip)
318 return;
319
320 if (zip->reg_base) {
321 cmd_ctl.u_reg64 = 0x0ull;
322 cmd_ctl.s.reset = 1;
323 zip_reg_write(cmd_ctl.u_reg64, (zip->reg_base + ZIP_CMD_CTL));
324 iounmap(zip->reg_base);
325 }
326
327 pci_release_regions(pdev);
328 pci_disable_device(pdev);
329
330
331
332
333
334 for (q = 0; q < ZIP_NUM_QUEUES; q++)
335 zip_cmd_qbuf_free(zip, q);
336
337 pci_set_drvdata(pdev, NULL);
338
339 zip_dev[zip->index] = NULL;
340}
341
342
343static struct pci_driver zip_driver = {
344 .name = DRV_NAME,
345 .id_table = zip_id_table,
346 .probe = zip_probe,
347 .remove = zip_remove,
348};
349
350
351
352static struct crypto_alg zip_comp_deflate = {
353 .cra_name = "deflate",
354 .cra_driver_name = "deflate-cavium",
355 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
356 .cra_ctxsize = sizeof(struct zip_kernel_ctx),
357 .cra_priority = 300,
358 .cra_module = THIS_MODULE,
359 .cra_init = zip_alloc_comp_ctx_deflate,
360 .cra_exit = zip_free_comp_ctx,
361 .cra_u = { .compress = {
362 .coa_compress = zip_comp_compress,
363 .coa_decompress = zip_comp_decompress
364 } }
365};
366
367static struct crypto_alg zip_comp_lzs = {
368 .cra_name = "lzs",
369 .cra_driver_name = "lzs-cavium",
370 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
371 .cra_ctxsize = sizeof(struct zip_kernel_ctx),
372 .cra_priority = 300,
373 .cra_module = THIS_MODULE,
374 .cra_init = zip_alloc_comp_ctx_lzs,
375 .cra_exit = zip_free_comp_ctx,
376 .cra_u = { .compress = {
377 .coa_compress = zip_comp_compress,
378 .coa_decompress = zip_comp_decompress
379 } }
380};
381
382static struct scomp_alg zip_scomp_deflate = {
383 .alloc_ctx = zip_alloc_scomp_ctx_deflate,
384 .free_ctx = zip_free_scomp_ctx,
385 .compress = zip_scomp_compress,
386 .decompress = zip_scomp_decompress,
387 .base = {
388 .cra_name = "deflate",
389 .cra_driver_name = "deflate-scomp-cavium",
390 .cra_module = THIS_MODULE,
391 .cra_priority = 300,
392 }
393};
394
395static struct scomp_alg zip_scomp_lzs = {
396 .alloc_ctx = zip_alloc_scomp_ctx_lzs,
397 .free_ctx = zip_free_scomp_ctx,
398 .compress = zip_scomp_compress,
399 .decompress = zip_scomp_decompress,
400 .base = {
401 .cra_name = "lzs",
402 .cra_driver_name = "lzs-scomp-cavium",
403 .cra_module = THIS_MODULE,
404 .cra_priority = 300,
405 }
406};
407
408static int zip_register_compression_device(void)
409{
410 int ret;
411
412 ret = crypto_register_alg(&zip_comp_deflate);
413 if (ret < 0) {
414 zip_err("Deflate algorithm registration failed\n");
415 return ret;
416 }
417
418 ret = crypto_register_alg(&zip_comp_lzs);
419 if (ret < 0) {
420 zip_err("LZS algorithm registration failed\n");
421 goto err_unregister_alg_deflate;
422 }
423
424 ret = crypto_register_scomp(&zip_scomp_deflate);
425 if (ret < 0) {
426 zip_err("Deflate scomp algorithm registration failed\n");
427 goto err_unregister_alg_lzs;
428 }
429
430 ret = crypto_register_scomp(&zip_scomp_lzs);
431 if (ret < 0) {
432 zip_err("LZS scomp algorithm registration failed\n");
433 goto err_unregister_scomp_deflate;
434 }
435
436 return ret;
437
438err_unregister_scomp_deflate:
439 crypto_unregister_scomp(&zip_scomp_deflate);
440err_unregister_alg_lzs:
441 crypto_unregister_alg(&zip_comp_lzs);
442err_unregister_alg_deflate:
443 crypto_unregister_alg(&zip_comp_deflate);
444
445 return ret;
446}
447
448static void zip_unregister_compression_device(void)
449{
450 crypto_unregister_alg(&zip_comp_deflate);
451 crypto_unregister_alg(&zip_comp_lzs);
452 crypto_unregister_scomp(&zip_scomp_deflate);
453 crypto_unregister_scomp(&zip_scomp_lzs);
454}
455
456
457
458
459#ifdef CONFIG_DEBUG_FS
460#include <linux/debugfs.h>
461
462
463static int zip_show_stats(struct seq_file *s, void *unused)
464{
465 u64 val = 0ull;
466 u64 avg_chunk = 0ull, avg_cr = 0ull;
467 u32 q = 0;
468
469 int index = 0;
470 struct zip_device *zip;
471 struct zip_stats *st;
472
473 for (index = 0; index < MAX_ZIP_DEVICES; index++) {
474 u64 pending = 0;
475
476 if (zip_dev[index]) {
477 zip = zip_dev[index];
478 st = &zip->stats;
479
480
481 for (q = 0; q < ZIP_NUM_QUEUES; q++) {
482 val = zip_reg_read((zip->reg_base +
483 ZIP_DBG_QUEX_STA(q)));
484 pending += val >> 32 & 0xffffff;
485 }
486
487 val = atomic64_read(&st->comp_req_complete);
488 avg_chunk = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0;
489
490 val = atomic64_read(&st->comp_out_bytes);
491 avg_cr = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0;
492 seq_printf(s, " ZIP Device %d Stats\n"
493 "-----------------------------------\n"
494 "Comp Req Submitted : \t%lld\n"
495 "Comp Req Completed : \t%lld\n"
496 "Compress In Bytes : \t%lld\n"
497 "Compressed Out Bytes : \t%lld\n"
498 "Average Chunk size : \t%llu\n"
499 "Average Compression ratio : \t%llu\n"
500 "Decomp Req Submitted : \t%lld\n"
501 "Decomp Req Completed : \t%lld\n"
502 "Decompress In Bytes : \t%lld\n"
503 "Decompressed Out Bytes : \t%lld\n"
504 "Decompress Bad requests : \t%lld\n"
505 "Pending Req : \t%lld\n"
506 "---------------------------------\n",
507 index,
508 (u64)atomic64_read(&st->comp_req_submit),
509 (u64)atomic64_read(&st->comp_req_complete),
510 (u64)atomic64_read(&st->comp_in_bytes),
511 (u64)atomic64_read(&st->comp_out_bytes),
512 avg_chunk,
513 avg_cr,
514 (u64)atomic64_read(&st->decomp_req_submit),
515 (u64)atomic64_read(&st->decomp_req_complete),
516 (u64)atomic64_read(&st->decomp_in_bytes),
517 (u64)atomic64_read(&st->decomp_out_bytes),
518 (u64)atomic64_read(&st->decomp_bad_reqs),
519 pending);
520 }
521 }
522 return 0;
523}
524
525
526static int zip_clear_stats(struct seq_file *s, void *unused)
527{
528 int index = 0;
529
530 for (index = 0; index < MAX_ZIP_DEVICES; index++) {
531 if (zip_dev[index]) {
532 memset(&zip_dev[index]->stats, 0,
533 sizeof(struct zip_stats));
534 seq_printf(s, "Cleared stats for zip %d\n", index);
535 }
536 }
537
538 return 0;
539}
540
541static struct zip_registers zipregs[64] = {
542 {"ZIP_CMD_CTL ", 0x0000ull},
543 {"ZIP_THROTTLE ", 0x0010ull},
544 {"ZIP_CONSTANTS ", 0x00A0ull},
545 {"ZIP_QUE0_MAP ", 0x1400ull},
546 {"ZIP_QUE1_MAP ", 0x1408ull},
547 {"ZIP_QUE_ENA ", 0x0500ull},
548 {"ZIP_QUE_PRI ", 0x0508ull},
549 {"ZIP_QUE0_DONE ", 0x2000ull},
550 {"ZIP_QUE1_DONE ", 0x2008ull},
551 {"ZIP_QUE0_DOORBELL ", 0x4000ull},
552 {"ZIP_QUE1_DOORBELL ", 0x4008ull},
553 {"ZIP_QUE0_SBUF_ADDR ", 0x1000ull},
554 {"ZIP_QUE1_SBUF_ADDR ", 0x1008ull},
555 {"ZIP_QUE0_SBUF_CTL ", 0x1200ull},
556 {"ZIP_QUE1_SBUF_CTL ", 0x1208ull},
557 { NULL, 0}
558};
559
560
561static int zip_print_regs(struct seq_file *s, void *unused)
562{
563 u64 val = 0;
564 int i = 0, index = 0;
565
566 for (index = 0; index < MAX_ZIP_DEVICES; index++) {
567 if (zip_dev[index]) {
568 seq_printf(s, "--------------------------------\n"
569 " ZIP Device %d Registers\n"
570 "--------------------------------\n",
571 index);
572
573 i = 0;
574
575 while (zipregs[i].reg_name) {
576 val = zip_reg_read((zip_dev[index]->reg_base +
577 zipregs[i].reg_offset));
578 seq_printf(s, "%s: 0x%016llx\n",
579 zipregs[i].reg_name, val);
580 i++;
581 }
582 }
583 }
584 return 0;
585}
586
587static int zip_stats_open(struct inode *inode, struct file *file)
588{
589 return single_open(file, zip_show_stats, NULL);
590}
591
592static const struct file_operations zip_stats_fops = {
593 .owner = THIS_MODULE,
594 .open = zip_stats_open,
595 .read = seq_read,
596};
597
598static int zip_clear_open(struct inode *inode, struct file *file)
599{
600 return single_open(file, zip_clear_stats, NULL);
601}
602
603static const struct file_operations zip_clear_fops = {
604 .owner = THIS_MODULE,
605 .open = zip_clear_open,
606 .read = seq_read,
607};
608
609static int zip_regs_open(struct inode *inode, struct file *file)
610{
611 return single_open(file, zip_print_regs, NULL);
612}
613
614static const struct file_operations zip_regs_fops = {
615 .owner = THIS_MODULE,
616 .open = zip_regs_open,
617 .read = seq_read,
618};
619
620
621static struct dentry *zip_debugfs_root;
622
623static void __init zip_debugfs_init(void)
624{
625 if (!debugfs_initialized())
626 return;
627
628 zip_debugfs_root = debugfs_create_dir("thunderx_zip", NULL);
629
630
631 debugfs_create_file("zip_stats", 0444, zip_debugfs_root, NULL,
632 &zip_stats_fops);
633
634 debugfs_create_file("zip_clear", 0444, zip_debugfs_root, NULL,
635 &zip_clear_fops);
636
637 debugfs_create_file("zip_regs", 0444, zip_debugfs_root, NULL,
638 &zip_regs_fops);
639
640}
641
642static void __exit zip_debugfs_exit(void)
643{
644 debugfs_remove_recursive(zip_debugfs_root);
645}
646
647#else
648static void __init zip_debugfs_init(void) { }
649static void __exit zip_debugfs_exit(void) { }
650#endif
651
652
653static int __init zip_init_module(void)
654{
655 int ret;
656
657 zip_msg("%s\n", DRV_NAME);
658
659 ret = pci_register_driver(&zip_driver);
660 if (ret < 0) {
661 zip_err("ZIP: pci_register_driver() failed\n");
662 return ret;
663 }
664
665
666 ret = zip_register_compression_device();
667 if (ret < 0) {
668 zip_err("ZIP: Kernel Crypto Registration failed\n");
669 goto err_pci_unregister;
670 }
671
672
673 zip_debugfs_init();
674
675 return ret;
676
677err_pci_unregister:
678 pci_unregister_driver(&zip_driver);
679 return ret;
680}
681
682static void __exit zip_cleanup_module(void)
683{
684 zip_debugfs_exit();
685
686
687 zip_unregister_compression_device();
688
689
690 pci_unregister_driver(&zip_driver);
691}
692
693module_init(zip_init_module);
694module_exit(zip_cleanup_module);
695
696MODULE_AUTHOR("Cavium Inc");
697MODULE_DESCRIPTION("Cavium Inc ThunderX ZIP Driver");
698MODULE_LICENSE("GPL v2");
699MODULE_DEVICE_TABLE(pci, zip_id_table);
700
