1#include <linux/aer.h>
2#include <linux/delay.h>
3#include <linux/debugfs.h>
4#include <linux/firmware.h>
5#include <linux/list.h>
6#include <linux/module.h>
7#include <linux/mutex.h>
8#include <linux/pci.h>
9#include <linux/pci_ids.h>
10
11#include "nitrox_dev.h"
12#include "nitrox_common.h"
13#include "nitrox_csr.h"
14
15#define CNN55XX_DEV_ID 0x12
16#define MAX_PF_QUEUES 64
17#define UCODE_HLEN 48
18#define SE_GROUP 0
19
20#define DRIVER_VERSION "1.0"
21#define FW_DIR "cavium/"
22
23#define SE_FW FW_DIR "cnn55xx_se.fw"
24
25static const char nitrox_driver_name[] = "CNN55XX";
26
27static LIST_HEAD(ndevlist);
28static DEFINE_MUTEX(devlist_lock);
29static unsigned int num_devices;
30
31
32
33
34static const struct pci_device_id nitrox_pci_tbl[] = {
35 {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
36
37 {0, }
38};
39MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
40
41static unsigned int qlen = DEFAULT_CMD_QLEN;
42module_param(qlen, uint, 0644);
43MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
44
45
46
47
48
49
50
51
52
53struct ucode {
54 u8 id;
55 char version[VERSION_LEN - 1];
56 __be32 code_size;
57 u8 raz[12];
58 u64 code[0];
59};
60
61
62
63
64static void write_to_ucd_unit(struct nitrox_device *ndev,
65 struct ucode *ucode)
66{
67 u32 code_size = be32_to_cpu(ucode->code_size) * 2;
68 u64 offset, data;
69 int i = 0;
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87 offset = UCD_UCODE_LOAD_BLOCK_NUM;
88 nitrox_write_csr(ndev, offset, 0);
89
90 code_size = roundup(code_size, 8);
91 while (code_size) {
92 data = ucode->code[i];
93
94 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
95 nitrox_write_csr(ndev, offset, data);
96 code_size -= 8;
97 i++;
98 }
99
100
101 offset = POM_GRP_EXECMASKX(SE_GROUP);
102 nitrox_write_csr(ndev, offset, (~0ULL));
103
104 for (i = 0; i < ndev->hw.se_cores; i++) {
105
106
107
108
109
110
111 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
112 nitrox_write_csr(ndev, offset, 0x8);
113 }
114 usleep_range(300, 400);
115}
116
117static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
118{
119 const struct firmware *fw;
120 struct ucode *ucode;
121 int ret;
122
123 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
124
125 ret = request_firmware(&fw, fw_name, DEV(ndev));
126 if (ret < 0) {
127 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
128 return ret;
129 }
130
131 ucode = (struct ucode *)fw->data;
132
133 memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
134 ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
135
136 write_to_ucd_unit(ndev, ucode);
137 release_firmware(fw);
138
139 set_bit(NITROX_UCODE_LOADED, &ndev->status);
140
141 smp_mb__after_atomic();
142 return 0;
143}
144
145
146
147
148
149static int nitrox_add_to_devlist(struct nitrox_device *ndev)
150{
151 struct nitrox_device *dev;
152 int ret = 0;
153
154 INIT_LIST_HEAD(&ndev->list);
155 refcount_set(&ndev->refcnt, 1);
156
157 mutex_lock(&devlist_lock);
158 list_for_each_entry(dev, &ndevlist, list) {
159 if (dev == ndev) {
160 ret = -EEXIST;
161 goto unlock;
162 }
163 }
164 ndev->idx = num_devices++;
165 list_add_tail(&ndev->list, &ndevlist);
166unlock:
167 mutex_unlock(&devlist_lock);
168 return ret;
169}
170
171
172
173
174
175
176static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
177{
178 mutex_lock(&devlist_lock);
179 list_del(&ndev->list);
180 num_devices--;
181 mutex_unlock(&devlist_lock);
182}
183
184struct nitrox_device *nitrox_get_first_device(void)
185{
186 struct nitrox_device *ndev = NULL;
187
188 mutex_lock(&devlist_lock);
189 list_for_each_entry(ndev, &ndevlist, list) {
190 if (nitrox_ready(ndev))
191 break;
192 }
193 mutex_unlock(&devlist_lock);
194 if (!ndev)
195 return NULL;
196
197 refcount_inc(&ndev->refcnt);
198
199 smp_mb__after_atomic();
200 return ndev;
201}
202
203void nitrox_put_device(struct nitrox_device *ndev)
204{
205 if (!ndev)
206 return;
207
208 refcount_dec(&ndev->refcnt);
209
210 smp_mb__after_atomic();
211}
212
213static int nitrox_reset_device(struct pci_dev *pdev)
214{
215 int pos = 0;
216
217 pos = pci_save_state(pdev);
218 if (pos) {
219 dev_err(&pdev->dev, "Failed to save pci state\n");
220 return -ENOMEM;
221 }
222
223 pos = pci_pcie_cap(pdev);
224 if (!pos)
225 return -ENOTTY;
226
227 if (!pci_wait_for_pending_transaction(pdev))
228 dev_err(&pdev->dev, "waiting for pending transaction\n");
229
230 pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
231 msleep(100);
232 pci_restore_state(pdev);
233
234 return 0;
235}
236
237static int nitrox_pf_sw_init(struct nitrox_device *ndev)
238{
239 int err;
240
241 err = nitrox_common_sw_init(ndev);
242 if (err)
243 return err;
244
245 err = nitrox_pf_init_isr(ndev);
246 if (err)
247 nitrox_common_sw_cleanup(ndev);
248
249 return err;
250}
251
252static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
253{
254 nitrox_pf_cleanup_isr(ndev);
255 nitrox_common_sw_cleanup(ndev);
256}
257
258
259
260
261
262static int nitrox_bist_check(struct nitrox_device *ndev)
263{
264 u64 value = 0;
265 int i;
266
267 for (i = 0; i < NR_CLUSTERS; i++) {
268 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
269 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
270 }
271 value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
272 value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
273 value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
274 value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
275 value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
276 value += nitrox_read_csr(ndev, POM_BIST_REG);
277 value += nitrox_read_csr(ndev, BMI_BIST_REG);
278 value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
279 value += nitrox_read_csr(ndev, BMO_BIST_REG);
280 value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
281 value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
282 if (value)
283 return -EIO;
284 return 0;
285}
286
287static void nitrox_get_hwinfo(struct nitrox_device *ndev)
288{
289 union emu_fuse_map emu_fuse;
290 u64 offset;
291 int i;
292
293 for (i = 0; i < NR_CLUSTERS; i++) {
294 u8 dead_cores;
295
296 offset = EMU_FUSE_MAPX(i);
297 emu_fuse.value = nitrox_read_csr(ndev, offset);
298 if (emu_fuse.s.valid) {
299 dead_cores = hweight32(emu_fuse.s.ae_fuse);
300 ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
301 dead_cores = hweight16(emu_fuse.s.se_fuse);
302 ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
303 }
304 }
305}
306
307static int nitrox_pf_hw_init(struct nitrox_device *ndev)
308{
309 int err;
310
311 err = nitrox_bist_check(ndev);
312 if (err) {
313 dev_err(&ndev->pdev->dev, "BIST check failed\n");
314 return err;
315 }
316
317 nitrox_get_hwinfo(ndev);
318
319 nitrox_config_nps_unit(ndev);
320 nitrox_config_pom_unit(ndev);
321 nitrox_config_efl_unit(ndev);
322
323 nitrox_config_bmi_unit(ndev);
324 nitrox_config_bmo_unit(ndev);
325
326 nitrox_config_lbc_unit(ndev);
327 nitrox_config_rand_unit(ndev);
328
329
330 err = nitrox_load_fw(ndev, SE_FW);
331 if (err)
332 return err;
333
334 nitrox_config_emu_unit(ndev);
335
336 return 0;
337}
338
339#if IS_ENABLED(CONFIG_DEBUG_FS)
340static int registers_show(struct seq_file *s, void *v)
341{
342 struct nitrox_device *ndev = s->private;
343 u64 offset;
344
345
346 offset = NPS_STATS_PKT_DMA_RD_CNT;
347 seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT 0x%016llx\n",
348 nitrox_read_csr(ndev, offset));
349 offset = NPS_STATS_PKT_DMA_WR_CNT;
350 seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT 0x%016llx\n",
351 nitrox_read_csr(ndev, offset));
352
353
354 offset = BMI_NPS_PKT_CNT;
355 seq_printf(s, "BMI_NPS_PKT_CNT 0x%016llx\n",
356 nitrox_read_csr(ndev, offset));
357 offset = BMO_NPS_SLC_PKT_CNT;
358 seq_printf(s, "BMO_NPS_PKT_CNT 0x%016llx\n",
359 nitrox_read_csr(ndev, offset));
360
361 return 0;
362}
363
364static int registers_open(struct inode *inode, struct file *file)
365{
366 return single_open(file, registers_show, inode->i_private);
367}
368
369static const struct file_operations register_fops = {
370 .owner = THIS_MODULE,
371 .open = registers_open,
372 .read = seq_read,
373 .llseek = seq_lseek,
374 .release = single_release,
375};
376
377static int firmware_show(struct seq_file *s, void *v)
378{
379 struct nitrox_device *ndev = s->private;
380
381 seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
382 return 0;
383}
384
385static int firmware_open(struct inode *inode, struct file *file)
386{
387 return single_open(file, firmware_show, inode->i_private);
388}
389
390static const struct file_operations firmware_fops = {
391 .owner = THIS_MODULE,
392 .open = firmware_open,
393 .read = seq_read,
394 .llseek = seq_lseek,
395 .release = single_release,
396};
397
398static int nitrox_show(struct seq_file *s, void *v)
399{
400 struct nitrox_device *ndev = s->private;
401
402 seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
403 seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id);
404 seq_printf(s, " Cores [AE: %u SE: %u]\n",
405 ndev->hw.ae_cores, ndev->hw.se_cores);
406 seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues);
407 seq_printf(s, " Queue length: %u\n", ndev->qlen);
408 seq_printf(s, " Node: %u\n", ndev->node);
409
410 return 0;
411}
412
413static int nitrox_open(struct inode *inode, struct file *file)
414{
415 return single_open(file, nitrox_show, inode->i_private);
416}
417
418static const struct file_operations nitrox_fops = {
419 .owner = THIS_MODULE,
420 .open = nitrox_open,
421 .read = seq_read,
422 .llseek = seq_lseek,
423 .release = single_release,
424};
425
426static void nitrox_debugfs_exit(struct nitrox_device *ndev)
427{
428 debugfs_remove_recursive(ndev->debugfs_dir);
429 ndev->debugfs_dir = NULL;
430}
431
432static int nitrox_debugfs_init(struct nitrox_device *ndev)
433{
434 struct dentry *dir, *f;
435
436 dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
437 if (!dir)
438 return -ENOMEM;
439
440 ndev->debugfs_dir = dir;
441 f = debugfs_create_file("counters", 0400, dir, ndev, ®ister_fops);
442 if (!f)
443 goto err;
444 f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
445 if (!f)
446 goto err;
447 f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
448 if (!f)
449 goto err;
450
451 return 0;
452
453err:
454 nitrox_debugfs_exit(ndev);
455 return -ENODEV;
456}
457#else
458static int nitrox_debugfs_init(struct nitrox_device *ndev)
459{
460 return 0;
461}
462
463static void nitrox_debugfs_exit(struct nitrox_device *ndev)
464{
465}
466#endif
467
468
469
470
471
472
473
474
475
476static int nitrox_probe(struct pci_dev *pdev,
477 const struct pci_device_id *id)
478{
479 struct nitrox_device *ndev;
480 int err;
481
482 dev_info_once(&pdev->dev, "%s driver version %s\n",
483 nitrox_driver_name, DRIVER_VERSION);
484
485 err = pci_enable_device_mem(pdev);
486 if (err)
487 return err;
488
489
490 err = nitrox_reset_device(pdev);
491 if (err) {
492 dev_err(&pdev->dev, "FLR failed\n");
493 pci_disable_device(pdev);
494 return err;
495 }
496
497 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
498 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
499 } else {
500 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
501 if (err) {
502 dev_err(&pdev->dev, "DMA configuration failed\n");
503 pci_disable_device(pdev);
504 return err;
505 }
506 }
507
508 err = pci_request_mem_regions(pdev, nitrox_driver_name);
509 if (err) {
510 pci_disable_device(pdev);
511 return err;
512 }
513 pci_set_master(pdev);
514
515 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
516 if (!ndev) {
517 err = -ENOMEM;
518 goto ndev_fail;
519 }
520
521 pci_set_drvdata(pdev, ndev);
522 ndev->pdev = pdev;
523
524
525 nitrox_add_to_devlist(ndev);
526
527 ndev->hw.vendor_id = pdev->vendor;
528 ndev->hw.device_id = pdev->device;
529 ndev->hw.revision_id = pdev->revision;
530
531 ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
532 ndev->node = dev_to_node(&pdev->dev);
533 if (ndev->node == NUMA_NO_NODE)
534 ndev->node = 0;
535
536 ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
537 pci_resource_len(pdev, 0));
538 if (!ndev->bar_addr) {
539 err = -EIO;
540 goto ioremap_err;
541 }
542
543 ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
544 ndev->qlen = qlen;
545
546 err = nitrox_pf_sw_init(ndev);
547 if (err)
548 goto ioremap_err;
549
550 err = nitrox_pf_hw_init(ndev);
551 if (err)
552 goto pf_hw_fail;
553
554 err = nitrox_debugfs_init(ndev);
555 if (err)
556 goto pf_hw_fail;
557
558 set_bit(NITROX_READY, &ndev->status);
559
560 smp_mb__after_atomic();
561
562 err = nitrox_crypto_register();
563 if (err)
564 goto crypto_fail;
565
566 return 0;
567
568crypto_fail:
569 nitrox_debugfs_exit(ndev);
570 clear_bit(NITROX_READY, &ndev->status);
571
572 smp_mb__after_atomic();
573pf_hw_fail:
574 nitrox_pf_sw_cleanup(ndev);
575ioremap_err:
576 nitrox_remove_from_devlist(ndev);
577 kfree(ndev);
578 pci_set_drvdata(pdev, NULL);
579ndev_fail:
580 pci_release_mem_regions(pdev);
581 pci_disable_device(pdev);
582 return err;
583}
584
585
586
587
588
589static void nitrox_remove(struct pci_dev *pdev)
590{
591 struct nitrox_device *ndev = pci_get_drvdata(pdev);
592
593 if (!ndev)
594 return;
595
596 if (!refcount_dec_and_test(&ndev->refcnt)) {
597 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
598 refcount_read(&ndev->refcnt));
599 return;
600 }
601
602 dev_info(DEV(ndev), "Removing Device %x:%x\n",
603 ndev->hw.vendor_id, ndev->hw.device_id);
604
605 clear_bit(NITROX_READY, &ndev->status);
606
607 smp_mb__after_atomic();
608
609 nitrox_remove_from_devlist(ndev);
610 nitrox_crypto_unregister();
611 nitrox_debugfs_exit(ndev);
612 nitrox_pf_sw_cleanup(ndev);
613
614 iounmap(ndev->bar_addr);
615 kfree(ndev);
616
617 pci_set_drvdata(pdev, NULL);
618 pci_release_mem_regions(pdev);
619 pci_disable_device(pdev);
620}
621
622static void nitrox_shutdown(struct pci_dev *pdev)
623{
624 pci_set_drvdata(pdev, NULL);
625 pci_release_mem_regions(pdev);
626 pci_disable_device(pdev);
627}
628
629static struct pci_driver nitrox_driver = {
630 .name = nitrox_driver_name,
631 .id_table = nitrox_pci_tbl,
632 .probe = nitrox_probe,
633 .remove = nitrox_remove,
634 .shutdown = nitrox_shutdown,
635};
636
637module_pci_driver(nitrox_driver);
638
639MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
640MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
641MODULE_LICENSE("GPL");
642MODULE_VERSION(DRIVER_VERSION);
643MODULE_FIRMWARE(SE_FW);
644
