1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
44#include <linux/if.h>
45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
63#include <net/addrconf.h>
64#include <net/bonding.h>
65#include <net/addrconf.h>
66#include <asm/uaccess.h>
67#include <linux/crash_dump.h>
68
69#include "cxgb4.h"
70#include "cxgb4_filter.h"
71#include "t4_regs.h"
72#include "t4_values.h"
73#include "t4_msg.h"
74#include "t4fw_api.h"
75#include "t4fw_version.h"
76#include "cxgb4_dcb.h"
77#include "cxgb4_debugfs.h"
78#include "clip_tbl.h"
79#include "l2t.h"
80#include "sched.h"
81#include "cxgb4_tc_u32.h"
82
83char cxgb4_driver_name[] = KBUILD_MODNAME;
84
85#ifdef DRV_VERSION
86#undef DRV_VERSION
87#endif
88#define DRV_VERSION "2.0.0-ko"
89const char cxgb4_driver_version[] = DRV_VERSION;
90#define DRV_DESC "Chelsio T4/T5/T6 Network Driver"
91
92#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
93 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
94 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
95
96
97
98#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
99 static const struct pci_device_id cxgb4_pci_tbl[] = {
100#define CH_PCI_DEVICE_ID_FUNCTION 0x4
101
102
103
104
105#define CH_PCI_DEVICE_ID_FUNCTION2 0x0
106
107#define CH_PCI_ID_TABLE_ENTRY(devid) \
108 {PCI_VDEVICE(CHELSIO, (devid)), 4}
109
110#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
111 { 0, } \
112 }
113
114#include "t4_pci_id_tbl.h"
115
116#define FW4_FNAME "cxgb4/t4fw.bin"
117#define FW5_FNAME "cxgb4/t5fw.bin"
118#define FW6_FNAME "cxgb4/t6fw.bin"
119#define FW4_CFNAME "cxgb4/t4-config.txt"
120#define FW5_CFNAME "cxgb4/t5-config.txt"
121#define FW6_CFNAME "cxgb4/t6-config.txt"
122#define PHY_AQ1202_FIRMWARE "cxgb4/aq1202_fw.cld"
123#define PHY_BCM84834_FIRMWARE "cxgb4/bcm8483.bin"
124#define PHY_AQ1202_DEVICEID 0x4409
125#define PHY_BCM84834_DEVICEID 0x4486
126
127MODULE_DESCRIPTION(DRV_DESC);
128MODULE_AUTHOR("Chelsio Communications");
129MODULE_LICENSE("Dual BSD/GPL");
130MODULE_VERSION(DRV_VERSION);
131MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
132MODULE_FIRMWARE(FW4_FNAME);
133MODULE_FIRMWARE(FW5_FNAME);
134MODULE_FIRMWARE(FW6_FNAME);
135
136
137
138
139
140
141
142static uint force_init;
143
144module_param(force_init, uint, 0644);
145MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter,"
146 "deprecated parameter");
147
148static int dflt_msg_enable = DFLT_MSG_ENABLE;
149
150module_param(dflt_msg_enable, int, 0644);
151MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap, "
152 "deprecated parameter");
153
154
155
156
157
158
159
160
161
162
163static int msi = 2;
164
165module_param(msi, int, 0644);
166MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
167
168
169
170
171
172
173
174
175
176
177
178
179
180static int rx_dma_offset = 2;
181
182#ifdef CONFIG_PCI_IOV
183
184
185
186static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
187
188module_param_array(num_vf, uint, NULL, 0644);
189MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3, deprecated parameter - please use the pci sysfs interface.");
190#endif
191
192
193
194
195
196
197
198static int select_queue;
199module_param(select_queue, int, 0644);
200MODULE_PARM_DESC(select_queue,
201 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
202
203static struct dentry *cxgb4_debugfs_root;
204
205LIST_HEAD(adapter_list);
206DEFINE_MUTEX(uld_mutex);
207
208static void link_report(struct net_device *dev)
209{
210 if (!netif_carrier_ok(dev))
211 netdev_info(dev, "link down\n");
212 else {
213 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
214
215 const char *s;
216 const struct port_info *p = netdev_priv(dev);
217
218 switch (p->link_cfg.speed) {
219 case 10000:
220 s = "10Gbps";
221 break;
222 case 1000:
223 s = "1000Mbps";
224 break;
225 case 100:
226 s = "100Mbps";
227 break;
228 case 40000:
229 s = "40Gbps";
230 break;
231 default:
232 pr_info("%s: unsupported speed: %d\n",
233 dev->name, p->link_cfg.speed);
234 return;
235 }
236
237 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
238 fc[p->link_cfg.fc]);
239 }
240}
241
242#ifdef CONFIG_CHELSIO_T4_DCB
243
244static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
245{
246 struct port_info *pi = netdev_priv(dev);
247 struct adapter *adap = pi->adapter;
248 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
249 int i;
250
251
252
253
254 for (i = 0; i < pi->nqsets; i++, txq++) {
255 u32 name, value;
256 int err;
257
258 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
259 FW_PARAMS_PARAM_X_V(
260 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
261 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
262 value = enable ? i : 0xffffffff;
263
264
265
266
267
268 err = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1,
269 &name, &value,
270 -FW_CMD_MAX_TIMEOUT);
271
272 if (err)
273 dev_err(adap->pdev_dev,
274 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
275 enable ? "set" : "unset", pi->port_id, i, -err);
276 else
277 txq->dcb_prio = value;
278 }
279}
280
281static int cxgb4_dcb_enabled(const struct net_device *dev)
282{
283 struct port_info *pi = netdev_priv(dev);
284
285 if (!pi->dcb.enabled)
286 return 0;
287
288 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
289 (pi->dcb.state == CXGB4_DCB_STATE_HOST));
290}
291#endif
292
293void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
294{
295 struct net_device *dev = adapter->port[port_id];
296
297
298 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
299 if (link_stat)
300 netif_carrier_on(dev);
301 else {
302#ifdef CONFIG_CHELSIO_T4_DCB
303 if (cxgb4_dcb_enabled(dev)) {
304 cxgb4_dcb_state_init(dev);
305 dcb_tx_queue_prio_enable(dev, false);
306 }
307#endif
308 netif_carrier_off(dev);
309 }
310
311 link_report(dev);
312 }
313}
314
315void t4_os_portmod_changed(const struct adapter *adap, int port_id)
316{
317 static const char *mod_str[] = {
318 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
319 };
320
321 const struct net_device *dev = adap->port[port_id];
322 const struct port_info *pi = netdev_priv(dev);
323
324 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
325 netdev_info(dev, "port module unplugged\n");
326 else if (pi->mod_type < ARRAY_SIZE(mod_str))
327 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
328 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
329 netdev_info(dev, "%s: unsupported port module inserted\n",
330 dev->name);
331 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
332 netdev_info(dev, "%s: unknown port module inserted\n",
333 dev->name);
334 else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR)
335 netdev_info(dev, "%s: transceiver module error\n", dev->name);
336 else
337 netdev_info(dev, "%s: unknown module type %d inserted\n",
338 dev->name, pi->mod_type);
339}
340
341int dbfifo_int_thresh = 10;
342module_param(dbfifo_int_thresh, int, 0644);
343MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
344
345
346
347
348static int dbfifo_drain_delay = 1000;
349module_param(dbfifo_drain_delay, int, 0644);
350MODULE_PARM_DESC(dbfifo_drain_delay,
351 "usecs to sleep while draining the dbfifo");
352
353static inline int cxgb4_set_addr_hash(struct port_info *pi)
354{
355 struct adapter *adap = pi->adapter;
356 u64 vec = 0;
357 bool ucast = false;
358 struct hash_mac_addr *entry;
359
360
361 list_for_each_entry(entry, &adap->mac_hlist, list) {
362 ucast |= is_unicast_ether_addr(entry->addr);
363 vec |= (1ULL << hash_mac_addr(entry->addr));
364 }
365 return t4_set_addr_hash(adap, adap->mbox, pi->viid, ucast,
366 vec, false);
367}
368
369static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr)
370{
371 struct port_info *pi = netdev_priv(netdev);
372 struct adapter *adap = pi->adapter;
373 int ret;
374 u64 mhash = 0;
375 u64 uhash = 0;
376 bool free = false;
377 bool ucast = is_unicast_ether_addr(mac_addr);
378 const u8 *maclist[1] = {mac_addr};
379 struct hash_mac_addr *new_entry;
380
381 ret = t4_alloc_mac_filt(adap, adap->mbox, pi->viid, free, 1, maclist,
382 NULL, ucast ? &uhash : &mhash, false);
383 if (ret < 0)
384 goto out;
385
386
387
388
389 if (uhash || mhash) {
390 new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC);
391 if (!new_entry)
392 return -ENOMEM;
393 ether_addr_copy(new_entry->addr, mac_addr);
394 list_add_tail(&new_entry->list, &adap->mac_hlist);
395 ret = cxgb4_set_addr_hash(pi);
396 }
397out:
398 return ret < 0 ? ret : 0;
399}
400
401static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr)
402{
403 struct port_info *pi = netdev_priv(netdev);
404 struct adapter *adap = pi->adapter;
405 int ret;
406 const u8 *maclist[1] = {mac_addr};
407 struct hash_mac_addr *entry, *tmp;
408
409
410
411
412 list_for_each_entry_safe(entry, tmp, &adap->mac_hlist, list) {
413 if (ether_addr_equal(entry->addr, mac_addr)) {
414 list_del(&entry->list);
415 kfree(entry);
416 return cxgb4_set_addr_hash(pi);
417 }
418 }
419
420 ret = t4_free_mac_filt(adap, adap->mbox, pi->viid, 1, maclist, false);
421 return ret < 0 ? -EINVAL : 0;
422}
423
424
425
426
427
428static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
429{
430 struct port_info *pi = netdev_priv(dev);
431 struct adapter *adapter = pi->adapter;
432
433 __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
434 __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
435
436 return t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu,
437 (dev->flags & IFF_PROMISC) ? 1 : 0,
438 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
439 sleep_ok);
440}
441
442
443
444
445
446
447
448static int link_start(struct net_device *dev)
449{
450 int ret;
451 struct port_info *pi = netdev_priv(dev);
452 unsigned int mb = pi->adapter->pf;
453
454
455
456
457
458 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
459 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
460 if (ret == 0) {
461 ret = t4_change_mac(pi->adapter, mb, pi->viid,
462 pi->xact_addr_filt, dev->dev_addr, true,
463 true);
464 if (ret >= 0) {
465 pi->xact_addr_filt = ret;
466 ret = 0;
467 }
468 }
469 if (ret == 0)
470 ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan,
471 &pi->link_cfg);
472 if (ret == 0) {
473 local_bh_disable();
474 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
475 true, CXGB4_DCB_ENABLED);
476 local_bh_enable();
477 }
478
479 return ret;
480}
481
482#ifdef CONFIG_CHELSIO_T4_DCB
483
484static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
485{
486 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
487 struct net_device *dev = adap->port[adap->chan_map[port]];
488 int old_dcb_enabled = cxgb4_dcb_enabled(dev);
489 int new_dcb_enabled;
490
491 cxgb4_dcb_handle_fw_update(adap, pcmd);
492 new_dcb_enabled = cxgb4_dcb_enabled(dev);
493
494
495
496
497
498 if (new_dcb_enabled != old_dcb_enabled)
499 dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
500}
501#endif
502
503
504
505static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
506 const struct pkt_gl *gl)
507{
508 u8 opcode = ((const struct rss_header *)rsp)->opcode;
509
510 rsp++;
511
512
513
514 if (unlikely(opcode == CPL_FW4_MSG &&
515 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
516 rsp++;
517 opcode = ((const struct rss_header *)rsp)->opcode;
518 rsp++;
519 if (opcode != CPL_SGE_EGR_UPDATE) {
520 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
521 , opcode);
522 goto out;
523 }
524 }
525
526 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
527 const struct cpl_sge_egr_update *p = (void *)rsp;
528 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid));
529 struct sge_txq *txq;
530
531 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
532 txq->restarts++;
533 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
534 struct sge_eth_txq *eq;
535
536 eq = container_of(txq, struct sge_eth_txq, q);
537 netif_tx_wake_queue(eq->txq);
538 } else {
539 struct sge_ofld_txq *oq;
540
541 oq = container_of(txq, struct sge_ofld_txq, q);
542 tasklet_schedule(&oq->qresume_tsk);
543 }
544 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
545 const struct cpl_fw6_msg *p = (void *)rsp;
546
547#ifdef CONFIG_CHELSIO_T4_DCB
548 const struct fw_port_cmd *pcmd = (const void *)p->data;
549 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
550 unsigned int action =
551 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
552
553 if (cmd == FW_PORT_CMD &&
554 action == FW_PORT_ACTION_GET_PORT_INFO) {
555 int port = FW_PORT_CMD_PORTID_G(
556 be32_to_cpu(pcmd->op_to_portid));
557 struct net_device *dev =
558 q->adap->port[q->adap->chan_map[port]];
559 int state_input = ((pcmd->u.info.dcbxdis_pkd &
560 FW_PORT_CMD_DCBXDIS_F)
561 ? CXGB4_DCB_INPUT_FW_DISABLED
562 : CXGB4_DCB_INPUT_FW_ENABLED);
563
564 cxgb4_dcb_state_fsm(dev, state_input);
565 }
566
567 if (cmd == FW_PORT_CMD &&
568 action == FW_PORT_ACTION_L2_DCB_CFG)
569 dcb_rpl(q->adap, pcmd);
570 else
571#endif
572 if (p->type == 0)
573 t4_handle_fw_rpl(q->adap, p->data);
574 } else if (opcode == CPL_L2T_WRITE_RPL) {
575 const struct cpl_l2t_write_rpl *p = (void *)rsp;
576
577 do_l2t_write_rpl(q->adap, p);
578 } else if (opcode == CPL_SET_TCB_RPL) {
579 const struct cpl_set_tcb_rpl *p = (void *)rsp;
580
581 filter_rpl(q->adap, p);
582 } else
583 dev_err(q->adap->pdev_dev,
584 "unexpected CPL %#x on FW event queue\n", opcode);
585out:
586 return 0;
587}
588
589static void disable_msi(struct adapter *adapter)
590{
591 if (adapter->flags & USING_MSIX) {
592 pci_disable_msix(adapter->pdev);
593 adapter->flags &= ~USING_MSIX;
594 } else if (adapter->flags & USING_MSI) {
595 pci_disable_msi(adapter->pdev);
596 adapter->flags &= ~USING_MSI;
597 }
598}
599
600
601
602
603static irqreturn_t t4_nondata_intr(int irq, void *cookie)
604{
605 struct adapter *adap = cookie;
606 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
607
608 if (v & PFSW_F) {
609 adap->swintr = 1;
610 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
611 }
612 if (adap->flags & MASTER_PF)
613 t4_slow_intr_handler(adap);
614 return IRQ_HANDLED;
615}
616
617
618
619
620static void name_msix_vecs(struct adapter *adap)
621{
622 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
623
624
625 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
626
627
628 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
629 adap->port[0]->name);
630
631
632 for_each_port(adap, j) {
633 struct net_device *d = adap->port[j];
634 const struct port_info *pi = netdev_priv(d);
635
636 for (i = 0; i < pi->nqsets; i++, msi_idx++)
637 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
638 d->name, i);
639 }
640}
641
642static int request_msix_queue_irqs(struct adapter *adap)
643{
644 struct sge *s = &adap->sge;
645 int err, ethqidx;
646 int msi_index = 2;
647
648 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
649 adap->msix_info[1].desc, &s->fw_evtq);
650 if (err)
651 return err;
652
653 for_each_ethrxq(s, ethqidx) {
654 err = request_irq(adap->msix_info[msi_index].vec,
655 t4_sge_intr_msix, 0,
656 adap->msix_info[msi_index].desc,
657 &s->ethrxq[ethqidx].rspq);
658 if (err)
659 goto unwind;
660 msi_index++;
661 }
662 return 0;
663
664unwind:
665 while (--ethqidx >= 0)
666 free_irq(adap->msix_info[--msi_index].vec,
667 &s->ethrxq[ethqidx].rspq);
668 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
669 return err;
670}
671
672static void free_msix_queue_irqs(struct adapter *adap)
673{
674 int i, msi_index = 2;
675 struct sge *s = &adap->sge;
676
677 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
678 for_each_ethrxq(s, i)
679 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
680}
681
682
683
684
685
686
687
688
689
690
691int cxgb4_write_rss(const struct port_info *pi, const u16 *queues)
692{
693 u16 *rss;
694 int i, err;
695 struct adapter *adapter = pi->adapter;
696 const struct sge_eth_rxq *rxq;
697
698 rxq = &adapter->sge.ethrxq[pi->first_qset];
699 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
700 if (!rss)
701 return -ENOMEM;
702
703
704 for (i = 0; i < pi->rss_size; i++, queues++)
705 rss[i] = rxq[*queues].rspq.abs_id;
706
707 err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0,
708 pi->rss_size, rss, pi->rss_size);
709
710
711
712
713
714 if (!err)
715 err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid,
716 FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F |
717 FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F |
718 FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F |
719 FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F |
720 FW_RSS_VI_CONFIG_CMD_UDPEN_F,
721 rss[0]);
722 kfree(rss);
723 return err;
724}
725
726
727
728
729
730
731
732static int setup_rss(struct adapter *adap)
733{
734 int i, j, err;
735
736 for_each_port(adap, i) {
737 const struct port_info *pi = adap2pinfo(adap, i);
738
739
740 for (j = 0; j < pi->rss_size; j++)
741 pi->rss[j] = j % pi->nqsets;
742
743 err = cxgb4_write_rss(pi, pi->rss);
744 if (err)
745 return err;
746 }
747 return 0;
748}
749
750
751
752
753static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
754{
755 qid -= p->ingr_start;
756 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
757}
758
759
760
761
762static void quiesce_rx(struct adapter *adap)
763{
764 int i;
765
766 for (i = 0; i < adap->sge.ingr_sz; i++) {
767 struct sge_rspq *q = adap->sge.ingr_map[i];
768
769 if (q && q->handler) {
770 napi_disable(&q->napi);
771 local_bh_disable();
772 while (!cxgb_poll_lock_napi(q))
773 mdelay(1);
774 local_bh_enable();
775 }
776
777 }
778}
779
780
781static void disable_interrupts(struct adapter *adap)
782{
783 if (adap->flags & FULL_INIT_DONE) {
784 t4_intr_disable(adap);
785 if (adap->flags & USING_MSIX) {
786 free_msix_queue_irqs(adap);
787 free_irq(adap->msix_info[0].vec, adap);
788 } else {
789 free_irq(adap->pdev->irq, adap);
790 }
791 quiesce_rx(adap);
792 }
793}
794
795
796
797
798static void enable_rx(struct adapter *adap)
799{
800 int i;
801
802 for (i = 0; i < adap->sge.ingr_sz; i++) {
803 struct sge_rspq *q = adap->sge.ingr_map[i];
804
805 if (!q)
806 continue;
807 if (q->handler) {
808 cxgb_busy_poll_init_lock(q);
809 napi_enable(&q->napi);
810 }
811
812 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
813 SEINTARM_V(q->intr_params) |
814 INGRESSQID_V(q->cntxt_id));
815 }
816}
817
818
819static int setup_fw_sge_queues(struct adapter *adap)
820{
821 struct sge *s = &adap->sge;
822 int err = 0;
823
824 bitmap_zero(s->starving_fl, s->egr_sz);
825 bitmap_zero(s->txq_maperr, s->egr_sz);
826
827 if (adap->flags & USING_MSIX)
828 adap->msi_idx = 1;
829 else {
830 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
831 NULL, NULL, NULL, -1);
832 if (err)
833 return err;
834 adap->msi_idx = -((int)s->intrq.abs_id + 1);
835 }
836
837 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
838 adap->msi_idx, NULL, fwevtq_handler, NULL, -1);
839 if (err)
840 t4_free_sge_resources(adap);
841 return err;
842}
843
844
845
846
847
848
849
850
851
852static int setup_sge_queues(struct adapter *adap)
853{
854 int err, i, j;
855 struct sge *s = &adap->sge;
856 struct sge_uld_rxq_info *rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA];
857 unsigned int cmplqid = 0;
858
859 for_each_port(adap, i) {
860 struct net_device *dev = adap->port[i];
861 struct port_info *pi = netdev_priv(dev);
862 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
863 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
864
865 for (j = 0; j < pi->nqsets; j++, q++) {
866 if (adap->msi_idx > 0)
867 adap->msi_idx++;
868 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
869 adap->msi_idx, &q->fl,
870 t4_ethrx_handler,
871 NULL,
872 t4_get_mps_bg_map(adap,
873 pi->tx_chan));
874 if (err)
875 goto freeout;
876 q->rspq.idx = j;
877 memset(&q->stats, 0, sizeof(q->stats));
878 }
879 for (j = 0; j < pi->nqsets; j++, t++) {
880 err = t4_sge_alloc_eth_txq(adap, t, dev,
881 netdev_get_tx_queue(dev, j),
882 s->fw_evtq.cntxt_id);
883 if (err)
884 goto freeout;
885 }
886 }
887
888 j = s->ofldqsets / adap->params.nports;
889 for_each_ofldtxq(s, i) {
890 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i],
891 adap->port[i / j],
892 s->fw_evtq.cntxt_id);
893 if (err)
894 goto freeout;
895 }
896
897 for_each_port(adap, i) {
898
899
900
901 if (rxq_info)
902 cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id;
903
904 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
905 s->fw_evtq.cntxt_id, cmplqid);
906 if (err)
907 goto freeout;
908 }
909
910 t4_write_reg(adap, is_t4(adap->params.chip) ?
911 MPS_TRC_RSS_CONTROL_A :
912 MPS_T5_TRC_RSS_CONTROL_A,
913 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
914 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
915 return 0;
916freeout:
917 t4_free_sge_resources(adap);
918 return err;
919}
920
921
922
923
924
925void *t4_alloc_mem(size_t size)
926{
927 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
928
929 if (!p)
930 p = vzalloc(size);
931 return p;
932}
933
934
935
936
937void t4_free_mem(void *addr)
938{
939 kvfree(addr);
940}
941
942static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
943 void *accel_priv, select_queue_fallback_t fallback)
944{
945 int txq;
946
947#ifdef CONFIG_CHELSIO_T4_DCB
948
949
950
951
952
953 if (cxgb4_dcb_enabled(dev)) {
954 u16 vlan_tci;
955 int err;
956
957 err = vlan_get_tag(skb, &vlan_tci);
958 if (unlikely(err)) {
959 if (net_ratelimit())
960 netdev_warn(dev,
961 "TX Packet without VLAN Tag on DCB Link\n");
962 txq = 0;
963 } else {
964 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
965#ifdef CONFIG_CHELSIO_T4_FCOE
966 if (skb->protocol == htons(ETH_P_FCOE))
967 txq = skb->priority & 0x7;
968#endif
969 }
970 return txq;
971 }
972#endif
973
974 if (select_queue) {
975 txq = (skb_rx_queue_recorded(skb)
976 ? skb_get_rx_queue(skb)
977 : smp_processor_id());
978
979 while (unlikely(txq >= dev->real_num_tx_queues))
980 txq -= dev->real_num_tx_queues;
981
982 return txq;
983 }
984
985 return fallback(dev, skb) % dev->real_num_tx_queues;
986}
987
988static int closest_timer(const struct sge *s, int time)
989{
990 int i, delta, match = 0, min_delta = INT_MAX;
991
992 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
993 delta = time - s->timer_val[i];
994 if (delta < 0)
995 delta = -delta;
996 if (delta < min_delta) {
997 min_delta = delta;
998 match = i;
999 }
1000 }
1001 return match;
1002}
1003
1004static int closest_thres(const struct sge *s, int thres)
1005{
1006 int i, delta, match = 0, min_delta = INT_MAX;
1007
1008 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1009 delta = thres - s->counter_val[i];
1010 if (delta < 0)
1011 delta = -delta;
1012 if (delta < min_delta) {
1013 min_delta = delta;
1014 match = i;
1015 }
1016 }
1017 return match;
1018}
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029int cxgb4_set_rspq_intr_params(struct sge_rspq *q,
1030 unsigned int us, unsigned int cnt)
1031{
1032 struct adapter *adap = q->adap;
1033
1034 if ((us | cnt) == 0)
1035 cnt = 1;
1036
1037 if (cnt) {
1038 int err;
1039 u32 v, new_idx;
1040
1041 new_idx = closest_thres(&adap->sge, cnt);
1042 if (q->desc && q->pktcnt_idx != new_idx) {
1043
1044 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
1045 FW_PARAMS_PARAM_X_V(
1046 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1047 FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
1048 err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
1049 &v, &new_idx);
1050 if (err)
1051 return err;
1052 }
1053 q->pktcnt_idx = new_idx;
1054 }
1055
1056 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1057 q->intr_params = QINTR_TIMER_IDX_V(us) | QINTR_CNT_EN_V(cnt > 0);
1058 return 0;
1059}
1060
1061static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
1062{
1063 const struct port_info *pi = netdev_priv(dev);
1064 netdev_features_t changed = dev->features ^ features;
1065 int err;
1066
1067 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
1068 return 0;
1069
1070 err = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, -1,
1071 -1, -1, -1,
1072 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
1073 if (unlikely(err))
1074 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
1075 return err;
1076}
1077
1078static int setup_debugfs(struct adapter *adap)
1079{
1080 if (IS_ERR_OR_NULL(adap->debugfs_root))
1081 return -1;
1082
1083#ifdef CONFIG_DEBUG_FS
1084 t4_setup_debugfs(adap);
1085#endif
1086 return 0;
1087}
1088
1089
1090
1091
1092
1093
1094
1095
1096int cxgb4_alloc_atid(struct tid_info *t, void *data)
1097{
1098 int atid = -1;
1099
1100 spin_lock_bh(&t->atid_lock);
1101 if (t->afree) {
1102 union aopen_entry *p = t->afree;
1103
1104 atid = (p - t->atid_tab) + t->atid_base;
1105 t->afree = p->next;
1106 p->data = data;
1107 t->atids_in_use++;
1108 }
1109 spin_unlock_bh(&t->atid_lock);
1110 return atid;
1111}
1112EXPORT_SYMBOL(cxgb4_alloc_atid);
1113
1114
1115
1116
1117void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
1118{
1119 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
1120
1121 spin_lock_bh(&t->atid_lock);
1122 p->next = t->afree;
1123 t->afree = p;
1124 t->atids_in_use--;
1125 spin_unlock_bh(&t->atid_lock);
1126}
1127EXPORT_SYMBOL(cxgb4_free_atid);
1128
1129
1130
1131
1132int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
1133{
1134 int stid;
1135
1136 spin_lock_bh(&t->stid_lock);
1137 if (family == PF_INET) {
1138 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
1139 if (stid < t->nstids)
1140 __set_bit(stid, t->stid_bmap);
1141 else
1142 stid = -1;
1143 } else {
1144 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 1);
1145 if (stid < 0)
1146 stid = -1;
1147 }
1148 if (stid >= 0) {
1149 t->stid_tab[stid].data = data;
1150 stid += t->stid_base;
1151
1152
1153
1154
1155 if (family == PF_INET)
1156 t->stids_in_use++;
1157 else
1158 t->stids_in_use += 2;
1159 }
1160 spin_unlock_bh(&t->stid_lock);
1161 return stid;
1162}
1163EXPORT_SYMBOL(cxgb4_alloc_stid);
1164
1165
1166
1167int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
1168{
1169 int stid;
1170
1171 spin_lock_bh(&t->stid_lock);
1172 if (family == PF_INET) {
1173 stid = find_next_zero_bit(t->stid_bmap,
1174 t->nstids + t->nsftids, t->nstids);
1175 if (stid < (t->nstids + t->nsftids))
1176 __set_bit(stid, t->stid_bmap);
1177 else
1178 stid = -1;
1179 } else {
1180 stid = -1;
1181 }
1182 if (stid >= 0) {
1183 t->stid_tab[stid].data = data;
1184 stid -= t->nstids;
1185 stid += t->sftid_base;
1186 t->sftids_in_use++;
1187 }
1188 spin_unlock_bh(&t->stid_lock);
1189 return stid;
1190}
1191EXPORT_SYMBOL(cxgb4_alloc_sftid);
1192
1193
1194
1195void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
1196{
1197
1198 if (t->nsftids && (stid >= t->sftid_base)) {
1199 stid -= t->sftid_base;
1200 stid += t->nstids;
1201 } else {
1202 stid -= t->stid_base;
1203 }
1204
1205 spin_lock_bh(&t->stid_lock);
1206 if (family == PF_INET)
1207 __clear_bit(stid, t->stid_bmap);
1208 else
1209 bitmap_release_region(t->stid_bmap, stid, 1);
1210 t->stid_tab[stid].data = NULL;
1211 if (stid < t->nstids) {
1212 if (family == PF_INET)
1213 t->stids_in_use--;
1214 else
1215 t->stids_in_use -= 2;
1216 } else {
1217 t->sftids_in_use--;
1218 }
1219 spin_unlock_bh(&t->stid_lock);
1220}
1221EXPORT_SYMBOL(cxgb4_free_stid);
1222
1223
1224
1225
1226static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
1227 unsigned int tid)
1228{
1229 struct cpl_tid_release *req;
1230
1231 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1232 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
1233 INIT_TP_WR(req, tid);
1234 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
1235}
1236
1237
1238
1239
1240
1241static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
1242 unsigned int tid)
1243{
1244 void **p = &t->tid_tab[tid];
1245 struct adapter *adap = container_of(t, struct adapter, tids);
1246
1247 spin_lock_bh(&adap->tid_release_lock);
1248 *p = adap->tid_release_head;
1249
1250 adap->tid_release_head = (void **)((uintptr_t)p | chan);
1251 if (!adap->tid_release_task_busy) {
1252 adap->tid_release_task_busy = true;
1253 queue_work(adap->workq, &adap->tid_release_task);
1254 }
1255 spin_unlock_bh(&adap->tid_release_lock);
1256}
1257
1258
1259
1260
1261static void process_tid_release_list(struct work_struct *work)
1262{
1263 struct sk_buff *skb;
1264 struct adapter *adap;
1265
1266 adap = container_of(work, struct adapter, tid_release_task);
1267
1268 spin_lock_bh(&adap->tid_release_lock);
1269 while (adap->tid_release_head) {
1270 void **p = adap->tid_release_head;
1271 unsigned int chan = (uintptr_t)p & 3;
1272 p = (void *)p - chan;
1273
1274 adap->tid_release_head = *p;
1275 *p = NULL;
1276 spin_unlock_bh(&adap->tid_release_lock);
1277
1278 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
1279 GFP_KERNEL)))
1280 schedule_timeout_uninterruptible(1);
1281
1282 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
1283 t4_ofld_send(adap, skb);
1284 spin_lock_bh(&adap->tid_release_lock);
1285 }
1286 adap->tid_release_task_busy = false;
1287 spin_unlock_bh(&adap->tid_release_lock);
1288}
1289
1290
1291
1292
1293
1294void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
1295{
1296 struct sk_buff *skb;
1297 struct adapter *adap = container_of(t, struct adapter, tids);
1298
1299 WARN_ON(tid >= t->ntids);
1300
1301 if (t->tid_tab[tid]) {
1302 t->tid_tab[tid] = NULL;
1303 if (t->hash_base && (tid >= t->hash_base))
1304 atomic_dec(&t->hash_tids_in_use);
1305 else
1306 atomic_dec(&t->tids_in_use);
1307 }
1308
1309 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
1310 if (likely(skb)) {
1311 mk_tid_release(skb, chan, tid);
1312 t4_ofld_send(adap, skb);
1313 } else
1314 cxgb4_queue_tid_release(t, chan, tid);
1315}
1316EXPORT_SYMBOL(cxgb4_remove_tid);
1317
1318
1319
1320
1321static int tid_init(struct tid_info *t)
1322{
1323 struct adapter *adap = container_of(t, struct adapter, tids);
1324 unsigned int max_ftids = t->nftids + t->nsftids;
1325 unsigned int natids = t->natids;
1326 unsigned int stid_bmap_size;
1327 unsigned int ftid_bmap_size;
1328 size_t size;
1329
1330 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
1331 ftid_bmap_size = BITS_TO_LONGS(t->nftids);
1332 size = t->ntids * sizeof(*t->tid_tab) +
1333 natids * sizeof(*t->atid_tab) +
1334 t->nstids * sizeof(*t->stid_tab) +
1335 t->nsftids * sizeof(*t->stid_tab) +
1336 stid_bmap_size * sizeof(long) +
1337 max_ftids * sizeof(*t->ftid_tab) +
1338 ftid_bmap_size * sizeof(long);
1339
1340 t->tid_tab = t4_alloc_mem(size);
1341 if (!t->tid_tab)
1342 return -ENOMEM;
1343
1344 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
1345 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
1346 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
1347 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
1348 t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids];
1349 spin_lock_init(&t->stid_lock);
1350 spin_lock_init(&t->atid_lock);
1351 spin_lock_init(&t->ftid_lock);
1352
1353 t->stids_in_use = 0;
1354 t->sftids_in_use = 0;
1355 t->afree = NULL;
1356 t->atids_in_use = 0;
1357 atomic_set(&t->tids_in_use, 0);
1358 atomic_set(&t->hash_tids_in_use, 0);
1359
1360
1361 if (natids) {
1362 while (--natids)
1363 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
1364 t->afree = t->atid_tab;
1365 }
1366
1367 if (is_offload(adap)) {
1368 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
1369
1370 if (!t->stid_base &&
1371 CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
1372 __set_bit(0, t->stid_bmap);
1373 }
1374
1375 bitmap_zero(t->ftid_bmap, t->nftids);
1376 return 0;
1377}
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
1391 __be32 sip, __be16 sport, __be16 vlan,
1392 unsigned int queue)
1393{
1394 unsigned int chan;
1395 struct sk_buff *skb;
1396 struct adapter *adap;
1397 struct cpl_pass_open_req *req;
1398 int ret;
1399
1400 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
1401 if (!skb)
1402 return -ENOMEM;
1403
1404 adap = netdev2adap(dev);
1405 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
1406 INIT_TP_WR(req, 0);
1407 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
1408 req->local_port = sport;
1409 req->peer_port = htons(0);
1410 req->local_ip = sip;
1411 req->peer_ip = htonl(0);
1412 chan = rxq_to_chan(&adap->sge, queue);
1413 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
1414 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
1415 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
1416 ret = t4_mgmt_tx(adap, skb);
1417 return net_xmit_eval(ret);
1418}
1419EXPORT_SYMBOL(cxgb4_create_server);
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
1432 const struct in6_addr *sip, __be16 sport,
1433 unsigned int queue)
1434{
1435 unsigned int chan;
1436 struct sk_buff *skb;
1437 struct adapter *adap;
1438 struct cpl_pass_open_req6 *req;
1439 int ret;
1440
1441 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
1442 if (!skb)
1443 return -ENOMEM;
1444
1445 adap = netdev2adap(dev);
1446 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
1447 INIT_TP_WR(req, 0);
1448 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
1449 req->local_port = sport;
1450 req->peer_port = htons(0);
1451 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
1452 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
1453 req->peer_ip_hi = cpu_to_be64(0);
1454 req->peer_ip_lo = cpu_to_be64(0);
1455 chan = rxq_to_chan(&adap->sge, queue);
1456 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
1457 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
1458 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
1459 ret = t4_mgmt_tx(adap, skb);
1460 return net_xmit_eval(ret);
1461}
1462EXPORT_SYMBOL(cxgb4_create_server6);
1463
1464int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
1465 unsigned int queue, bool ipv6)
1466{
1467 struct sk_buff *skb;
1468 struct adapter *adap;
1469 struct cpl_close_listsvr_req *req;
1470 int ret;
1471
1472 adap = netdev2adap(dev);
1473
1474 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
1475 if (!skb)
1476 return -ENOMEM;
1477
1478 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
1479 INIT_TP_WR(req, 0);
1480 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
1481 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) :
1482 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue));
1483 ret = t4_mgmt_tx(adap, skb);
1484 return net_xmit_eval(ret);
1485}
1486EXPORT_SYMBOL(cxgb4_remove_server);
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
1499 unsigned int *idx)
1500{
1501 unsigned int i = 0;
1502
1503 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
1504 ++i;
1505 if (idx)
1506 *idx = i;
1507 return mtus[i];
1508}
1509EXPORT_SYMBOL(cxgb4_best_mtu);
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
1528 unsigned short header_size,
1529 unsigned short data_size_max,
1530 unsigned short data_size_align,
1531 unsigned int *mtu_idxp)
1532{
1533 unsigned short max_mtu = header_size + data_size_max;
1534 unsigned short data_size_align_mask = data_size_align - 1;
1535 int mtu_idx, aligned_mtu_idx;
1536
1537
1538
1539
1540
1541
1542 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
1543 unsigned short data_size = mtus[mtu_idx] - header_size;
1544
1545
1546
1547
1548 if ((data_size & data_size_align_mask) == 0)
1549 aligned_mtu_idx = mtu_idx;
1550
1551
1552
1553
1554
1555 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
1556 break;
1557 }
1558
1559
1560
1561
1562 if (mtu_idx == NMTUS)
1563 mtu_idx--;
1564
1565
1566
1567
1568
1569 if (aligned_mtu_idx >= 0 &&
1570 mtu_idx - aligned_mtu_idx <= 1)
1571 mtu_idx = aligned_mtu_idx;
1572
1573
1574
1575
1576 if (mtu_idxp)
1577 *mtu_idxp = mtu_idx;
1578 return mtus[mtu_idx];
1579}
1580EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
1581
1582
1583
1584
1585
1586
1587
1588
1589unsigned int cxgb4_tp_smt_idx(enum chip_type chip, unsigned int viid)
1590{
1591
1592
1593
1594
1595
1596
1597 if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5)
1598 return ((viid & 0x7f) << 1);
1599 else
1600 return (viid & 0x7f);
1601}
1602EXPORT_SYMBOL(cxgb4_tp_smt_idx);
1603
1604
1605
1606
1607
1608
1609
1610unsigned int cxgb4_port_chan(const struct net_device *dev)
1611{
1612 return netdev2pinfo(dev)->tx_chan;
1613}
1614EXPORT_SYMBOL(cxgb4_port_chan);
1615
1616unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
1617{
1618 struct adapter *adap = netdev2adap(dev);
1619 u32 v1, v2, lp_count, hp_count;
1620
1621 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
1622 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
1623 if (is_t4(adap->params.chip)) {
1624 lp_count = LP_COUNT_G(v1);
1625 hp_count = HP_COUNT_G(v1);
1626 } else {
1627 lp_count = LP_COUNT_T5_G(v1);
1628 hp_count = HP_COUNT_T5_G(v2);
1629 }
1630 return lpfifo ? lp_count : hp_count;
1631}
1632EXPORT_SYMBOL(cxgb4_dbfifo_count);
1633
1634
1635
1636
1637
1638
1639
1640unsigned int cxgb4_port_viid(const struct net_device *dev)
1641{
1642 return netdev2pinfo(dev)->viid;
1643}
1644EXPORT_SYMBOL(cxgb4_port_viid);
1645
1646
1647
1648
1649
1650
1651
1652unsigned int cxgb4_port_idx(const struct net_device *dev)
1653{
1654 return netdev2pinfo(dev)->port_id;
1655}
1656EXPORT_SYMBOL(cxgb4_port_idx);
1657
1658void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
1659 struct tp_tcp_stats *v6)
1660{
1661 struct adapter *adap = pci_get_drvdata(pdev);
1662
1663 spin_lock(&adap->stats_lock);
1664 t4_tp_get_tcp_stats(adap, v4, v6);
1665 spin_unlock(&adap->stats_lock);
1666}
1667EXPORT_SYMBOL(cxgb4_get_tcp_stats);
1668
1669void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
1670 const unsigned int *pgsz_order)
1671{
1672 struct adapter *adap = netdev2adap(dev);
1673
1674 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
1675 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
1676 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
1677 HPZ3_V(pgsz_order[3]));
1678}
1679EXPORT_SYMBOL(cxgb4_iscsi_init);
1680
1681int cxgb4_flush_eq_cache(struct net_device *dev)
1682{
1683 struct adapter *adap = netdev2adap(dev);
1684
1685 return t4_sge_ctxt_flush(adap, adap->mbox);
1686}
1687EXPORT_SYMBOL(cxgb4_flush_eq_cache);
1688
1689static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
1690{
1691 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
1692 __be64 indices;
1693 int ret;
1694
1695 spin_lock(&adap->win0_lock);
1696 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
1697 sizeof(indices), (__be32 *)&indices,
1698 T4_MEMORY_READ);
1699 spin_unlock(&adap->win0_lock);
1700 if (!ret) {
1701 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
1702 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
1703 }
1704 return ret;
1705}
1706
1707int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
1708 u16 size)
1709{
1710 struct adapter *adap = netdev2adap(dev);
1711 u16 hw_pidx, hw_cidx;
1712 int ret;
1713
1714 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
1715 if (ret)
1716 goto out;
1717
1718 if (pidx != hw_pidx) {
1719 u16 delta;
1720 u32 val;
1721
1722 if (pidx >= hw_pidx)
1723 delta = pidx - hw_pidx;
1724 else
1725 delta = size - hw_pidx + pidx;
1726
1727 if (is_t4(adap->params.chip))
1728 val = PIDX_V(delta);
1729 else
1730 val = PIDX_T5_V(delta);
1731 wmb();
1732 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
1733 QID_V(qid) | val);
1734 }
1735out:
1736 return ret;
1737}
1738EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
1739
1740int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
1741{
1742 struct adapter *adap;
1743 u32 offset, memtype, memaddr;
1744 u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
1745 u32 edc0_end, edc1_end, mc0_end, mc1_end;
1746 int ret;
1747
1748 adap = netdev2adap(dev);
1749
1750 offset = ((stag >> 8) * 32) + adap->vres.stag.start;
1751
1752
1753
1754
1755
1756
1757
1758 size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
1759 edc0_size = EDRAM0_SIZE_G(size) << 20;
1760 size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
1761 edc1_size = EDRAM1_SIZE_G(size) << 20;
1762 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
1763 mc0_size = EXT_MEM0_SIZE_G(size) << 20;
1764
1765 edc0_end = edc0_size;
1766 edc1_end = edc0_end + edc1_size;
1767 mc0_end = edc1_end + mc0_size;
1768
1769 if (offset < edc0_end) {
1770 memtype = MEM_EDC0;
1771 memaddr = offset;
1772 } else if (offset < edc1_end) {
1773 memtype = MEM_EDC1;
1774 memaddr = offset - edc0_end;
1775 } else {
1776 if (offset < mc0_end) {
1777 memtype = MEM_MC0;
1778 memaddr = offset - edc1_end;
1779 } else if (is_t5(adap->params.chip)) {
1780 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
1781 mc1_size = EXT_MEM1_SIZE_G(size) << 20;
1782 mc1_end = mc0_end + mc1_size;
1783 if (offset < mc1_end) {
1784 memtype = MEM_MC1;
1785 memaddr = offset - mc0_end;
1786 } else {
1787
1788 goto err;
1789 }
1790 } else {
1791
1792 goto err;
1793 }
1794 }
1795
1796 spin_lock(&adap->win0_lock);
1797 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
1798 spin_unlock(&adap->win0_lock);
1799 return ret;
1800
1801err:
1802 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
1803 stag, offset);
1804 return -EINVAL;
1805}
1806EXPORT_SYMBOL(cxgb4_read_tpte);
1807
1808u64 cxgb4_read_sge_timestamp(struct net_device *dev)
1809{
1810 u32 hi, lo;
1811 struct adapter *adap;
1812
1813 adap = netdev2adap(dev);
1814 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
1815 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
1816
1817 return ((u64)hi << 32) | (u64)lo;
1818}
1819EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
1820
1821int cxgb4_bar2_sge_qregs(struct net_device *dev,
1822 unsigned int qid,
1823 enum cxgb4_bar2_qtype qtype,
1824 int user,
1825 u64 *pbar2_qoffset,
1826 unsigned int *pbar2_qid)
1827{
1828 return t4_bar2_sge_qregs(netdev2adap(dev),
1829 qid,
1830 (qtype == CXGB4_BAR2_QTYPE_EGRESS
1831 ? T4_BAR2_QTYPE_EGRESS
1832 : T4_BAR2_QTYPE_INGRESS),
1833 user,
1834 pbar2_qoffset,
1835 pbar2_qid);
1836}
1837EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
1838
1839static struct pci_driver cxgb4_driver;
1840
1841static void check_neigh_update(struct neighbour *neigh)
1842{
1843 const struct device *parent;
1844 const struct net_device *netdev = neigh->dev;
1845
1846 if (netdev->priv_flags & IFF_802_1Q_VLAN)
1847 netdev = vlan_dev_real_dev(netdev);
1848 parent = netdev->dev.parent;
1849 if (parent && parent->driver == &cxgb4_driver.driver)
1850 t4_l2t_update(dev_get_drvdata(parent), neigh);
1851}
1852
1853static int netevent_cb(struct notifier_block *nb, unsigned long event,
1854 void *data)
1855{
1856 switch (event) {
1857 case NETEVENT_NEIGH_UPDATE:
1858 check_neigh_update(data);
1859 break;
1860 case NETEVENT_REDIRECT:
1861 default:
1862 break;
1863 }
1864 return 0;
1865}
1866
1867static bool netevent_registered;
1868static struct notifier_block cxgb4_netevent_nb = {
1869 .notifier_call = netevent_cb
1870};
1871
1872static void drain_db_fifo(struct adapter *adap, int usecs)
1873{
1874 u32 v1, v2, lp_count, hp_count;
1875
1876 do {
1877 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
1878 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
1879 if (is_t4(adap->params.chip)) {
1880 lp_count = LP_COUNT_G(v1);
1881 hp_count = HP_COUNT_G(v1);
1882 } else {
1883 lp_count = LP_COUNT_T5_G(v1);
1884 hp_count = HP_COUNT_T5_G(v2);
1885 }
1886
1887 if (lp_count == 0 && hp_count == 0)
1888 break;
1889 set_current_state(TASK_UNINTERRUPTIBLE);
1890 schedule_timeout(usecs_to_jiffies(usecs));
1891 } while (1);
1892}
1893
1894static void disable_txq_db(struct sge_txq *q)
1895{
1896 unsigned long flags;
1897
1898 spin_lock_irqsave(&q->db_lock, flags);
1899 q->db_disabled = 1;
1900 spin_unlock_irqrestore(&q->db_lock, flags);
1901}
1902
1903static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
1904{
1905 spin_lock_irq(&q->db_lock);
1906 if (q->db_pidx_inc) {
1907
1908
1909
1910 wmb();
1911 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
1912 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
1913 q->db_pidx_inc = 0;
1914 }
1915 q->db_disabled = 0;
1916 spin_unlock_irq(&q->db_lock);
1917}
1918
1919static void disable_dbs(struct adapter *adap)
1920{
1921 int i;
1922
1923 for_each_ethrxq(&adap->sge, i)
1924 disable_txq_db(&adap->sge.ethtxq[i].q);
1925 for_each_ofldtxq(&adap->sge, i)
1926 disable_txq_db(&adap->sge.ofldtxq[i].q);
1927 for_each_port(adap, i)
1928 disable_txq_db(&adap->sge.ctrlq[i].q);
1929}
1930
1931static void enable_dbs(struct adapter *adap)
1932{
1933 int i;
1934
1935 for_each_ethrxq(&adap->sge, i)
1936 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
1937 for_each_ofldtxq(&adap->sge, i)
1938 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
1939 for_each_port(adap, i)
1940 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
1941}
1942
1943static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
1944{
1945 enum cxgb4_uld type = CXGB4_ULD_RDMA;
1946
1947 if (adap->uld && adap->uld[type].handle)
1948 adap->uld[type].control(adap->uld[type].handle, cmd);
1949}
1950
1951static void process_db_full(struct work_struct *work)
1952{
1953 struct adapter *adap;
1954
1955 adap = container_of(work, struct adapter, db_full_task);
1956
1957 drain_db_fifo(adap, dbfifo_drain_delay);
1958 enable_dbs(adap);
1959 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
1960 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
1961 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
1962 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
1963 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
1964 else
1965 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
1966 DBFIFO_LP_INT_F, DBFIFO_LP_INT_F);
1967}
1968
1969static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
1970{
1971 u16 hw_pidx, hw_cidx;
1972 int ret;
1973
1974 spin_lock_irq(&q->db_lock);
1975 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
1976 if (ret)
1977 goto out;
1978 if (q->db_pidx != hw_pidx) {
1979 u16 delta;
1980 u32 val;
1981
1982 if (q->db_pidx >= hw_pidx)
1983 delta = q->db_pidx - hw_pidx;
1984 else
1985 delta = q->size - hw_pidx + q->db_pidx;
1986
1987 if (is_t4(adap->params.chip))
1988 val = PIDX_V(delta);
1989 else
1990 val = PIDX_T5_V(delta);
1991 wmb();
1992 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
1993 QID_V(q->cntxt_id) | val);
1994 }
1995out:
1996 q->db_disabled = 0;
1997 q->db_pidx_inc = 0;
1998 spin_unlock_irq(&q->db_lock);
1999 if (ret)
2000 CH_WARN(adap, "DB drop recovery failed.\n");
2001}
2002
2003static void recover_all_queues(struct adapter *adap)
2004{
2005 int i;
2006
2007 for_each_ethrxq(&adap->sge, i)
2008 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
2009 for_each_ofldtxq(&adap->sge, i)
2010 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
2011 for_each_port(adap, i)
2012 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
2013}
2014
2015static void process_db_drop(struct work_struct *work)
2016{
2017 struct adapter *adap;
2018
2019 adap = container_of(work, struct adapter, db_drop_task);
2020
2021 if (is_t4(adap->params.chip)) {
2022 drain_db_fifo(adap, dbfifo_drain_delay);
2023 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
2024 drain_db_fifo(adap, dbfifo_drain_delay);
2025 recover_all_queues(adap);
2026 drain_db_fifo(adap, dbfifo_drain_delay);
2027 enable_dbs(adap);
2028 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2029 } else if (is_t5(adap->params.chip)) {
2030 u32 dropped_db = t4_read_reg(adap, 0x010ac);
2031 u16 qid = (dropped_db >> 15) & 0x1ffff;
2032 u16 pidx_inc = dropped_db & 0x1fff;
2033 u64 bar2_qoffset;
2034 unsigned int bar2_qid;
2035 int ret;
2036
2037 ret = t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
2038 0, &bar2_qoffset, &bar2_qid);
2039 if (ret)
2040 dev_err(adap->pdev_dev, "doorbell drop recovery: "
2041 "qid=%d, pidx_inc=%d\n", qid, pidx_inc);
2042 else
2043 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
2044 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
2045
2046
2047 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
2048 }
2049
2050 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
2051 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
2052}
2053
2054void t4_db_full(struct adapter *adap)
2055{
2056 if (is_t4(adap->params.chip)) {
2057 disable_dbs(adap);
2058 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
2059 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
2060 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
2061 queue_work(adap->workq, &adap->db_full_task);
2062 }
2063}
2064
2065void t4_db_dropped(struct adapter *adap)
2066{
2067 if (is_t4(adap->params.chip)) {
2068 disable_dbs(adap);
2069 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
2070 }
2071 queue_work(adap->workq, &adap->db_drop_task);
2072}
2073
2074void t4_register_netevent_notifier(void)
2075{
2076 if (!netevent_registered) {
2077 register_netevent_notifier(&cxgb4_netevent_nb);
2078 netevent_registered = true;
2079 }
2080}
2081
2082static void detach_ulds(struct adapter *adap)
2083{
2084 unsigned int i;
2085
2086 mutex_lock(&uld_mutex);
2087 list_del(&adap->list_node);
2088 for (i = 0; i < CXGB4_ULD_MAX; i++)
2089 if (adap->uld && adap->uld[i].handle) {
2090 adap->uld[i].state_change(adap->uld[i].handle,
2091 CXGB4_STATE_DETACH);
2092 adap->uld[i].handle = NULL;
2093 }
2094 if (netevent_registered && list_empty(&adapter_list)) {
2095 unregister_netevent_notifier(&cxgb4_netevent_nb);
2096 netevent_registered = false;
2097 }
2098 mutex_unlock(&uld_mutex);
2099}
2100
2101static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2102{
2103 unsigned int i;
2104
2105 mutex_lock(&uld_mutex);
2106 for (i = 0; i < CXGB4_ULD_MAX; i++)
2107 if (adap->uld && adap->uld[i].handle)
2108 adap->uld[i].state_change(adap->uld[i].handle,
2109 new_state);
2110 mutex_unlock(&uld_mutex);
2111}
2112
2113#if IS_ENABLED(CONFIG_IPV6)
2114static int cxgb4_inet6addr_handler(struct notifier_block *this,
2115 unsigned long event, void *data)
2116{
2117 struct inet6_ifaddr *ifa = data;
2118 struct net_device *event_dev = ifa->idev->dev;
2119 const struct device *parent = NULL;
2120#if IS_ENABLED(CONFIG_BONDING)
2121 struct adapter *adap;
2122#endif
2123 if (event_dev->priv_flags & IFF_802_1Q_VLAN)
2124 event_dev = vlan_dev_real_dev(event_dev);
2125#if IS_ENABLED(CONFIG_BONDING)
2126 if (event_dev->flags & IFF_MASTER) {
2127 list_for_each_entry(adap, &adapter_list, list_node) {
2128 switch (event) {
2129 case NETDEV_UP:
2130 cxgb4_clip_get(adap->port[0],
2131 (const u32 *)ifa, 1);
2132 break;
2133 case NETDEV_DOWN:
2134 cxgb4_clip_release(adap->port[0],
2135 (const u32 *)ifa, 1);
2136 break;
2137 default:
2138 break;
2139 }
2140 }
2141 return NOTIFY_OK;
2142 }
2143#endif
2144
2145 if (event_dev)
2146 parent = event_dev->dev.parent;
2147
2148 if (parent && parent->driver == &cxgb4_driver.driver) {
2149 switch (event) {
2150 case NETDEV_UP:
2151 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1);
2152 break;
2153 case NETDEV_DOWN:
2154 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1);
2155 break;
2156 default:
2157 break;
2158 }
2159 }
2160 return NOTIFY_OK;
2161}
2162
2163static bool inet6addr_registered;
2164static struct notifier_block cxgb4_inet6addr_notifier = {
2165 .notifier_call = cxgb4_inet6addr_handler
2166};
2167
2168static void update_clip(const struct adapter *adap)
2169{
2170 int i;
2171 struct net_device *dev;
2172 int ret;
2173
2174 rcu_read_lock();
2175
2176 for (i = 0; i < MAX_NPORTS; i++) {
2177 dev = adap->port[i];
2178 ret = 0;
2179
2180 if (dev)
2181 ret = cxgb4_update_root_dev_clip(dev);
2182
2183 if (ret < 0)
2184 break;
2185 }
2186 rcu_read_unlock();
2187}
2188#endif
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200static int cxgb_up(struct adapter *adap)
2201{
2202 int err;
2203
2204 err = setup_sge_queues(adap);
2205 if (err)
2206 goto out;
2207 err = setup_rss(adap);
2208 if (err)
2209 goto freeq;
2210
2211 if (adap->flags & USING_MSIX) {
2212 name_msix_vecs(adap);
2213 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2214 adap->msix_info[0].desc, adap);
2215 if (err)
2216 goto irq_err;
2217 err = request_msix_queue_irqs(adap);
2218 if (err) {
2219 free_irq(adap->msix_info[0].vec, adap);
2220 goto irq_err;
2221 }
2222 } else {
2223 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2224 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2225 adap->port[0]->name, adap);
2226 if (err)
2227 goto irq_err;
2228 }
2229 enable_rx(adap);
2230 t4_sge_start(adap);
2231 t4_intr_enable(adap);
2232 adap->flags |= FULL_INIT_DONE;
2233 notify_ulds(adap, CXGB4_STATE_UP);
2234#if IS_ENABLED(CONFIG_IPV6)
2235 update_clip(adap);
2236#endif
2237
2238 INIT_LIST_HEAD(&adap->mac_hlist);
2239 out:
2240 return err;
2241 irq_err:
2242 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2243 freeq:
2244 t4_free_sge_resources(adap);
2245 goto out;
2246}
2247
2248static void cxgb_down(struct adapter *adapter)
2249{
2250 cancel_work_sync(&adapter->tid_release_task);
2251 cancel_work_sync(&adapter->db_full_task);
2252 cancel_work_sync(&adapter->db_drop_task);
2253 adapter->tid_release_task_busy = false;
2254 adapter->tid_release_head = NULL;
2255
2256 t4_sge_stop(adapter);
2257 t4_free_sge_resources(adapter);
2258 adapter->flags &= ~FULL_INIT_DONE;
2259}
2260
2261
2262
2263
2264static int cxgb_open(struct net_device *dev)
2265{
2266 int err;
2267 struct port_info *pi = netdev_priv(dev);
2268 struct adapter *adapter = pi->adapter;
2269
2270 netif_carrier_off(dev);
2271
2272 if (!(adapter->flags & FULL_INIT_DONE)) {
2273 err = cxgb_up(adapter);
2274 if (err < 0)
2275 return err;
2276 }
2277
2278 err = link_start(dev);
2279 if (!err)
2280 netif_tx_start_all_queues(dev);
2281 return err;
2282}
2283
2284static int cxgb_close(struct net_device *dev)
2285{
2286 struct port_info *pi = netdev_priv(dev);
2287 struct adapter *adapter = pi->adapter;
2288
2289 netif_tx_stop_all_queues(dev);
2290 netif_carrier_off(dev);
2291 return t4_enable_vi(adapter, adapter->pf, pi->viid, false, false);
2292}
2293
2294int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
2295 __be32 sip, __be16 sport, __be16 vlan,
2296 unsigned int queue, unsigned char port, unsigned char mask)
2297{
2298 int ret;
2299 struct filter_entry *f;
2300 struct adapter *adap;
2301 int i;
2302 u8 *val;
2303
2304 adap = netdev2adap(dev);
2305
2306
2307 stid -= adap->tids.sftid_base;
2308 stid += adap->tids.nftids;
2309
2310
2311
2312 f = &adap->tids.ftid_tab[stid];
2313 ret = writable_filter(f);
2314 if (ret)
2315 return ret;
2316
2317
2318
2319
2320 if (f->valid)
2321 clear_filter(adap, f);
2322
2323
2324 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
2325 f->fs.val.lport = cpu_to_be16(sport);
2326 f->fs.mask.lport = ~0;
2327 val = (u8 *)&sip;
2328 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
2329 for (i = 0; i < 4; i++) {
2330 f->fs.val.lip[i] = val[i];
2331 f->fs.mask.lip[i] = ~0;
2332 }
2333 if (adap->params.tp.vlan_pri_map & PORT_F) {
2334 f->fs.val.iport = port;
2335 f->fs.mask.iport = mask;
2336 }
2337 }
2338
2339 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
2340 f->fs.val.proto = IPPROTO_TCP;
2341 f->fs.mask.proto = ~0;
2342 }
2343
2344 f->fs.dirsteer = 1;
2345 f->fs.iq = queue;
2346
2347 f->locked = 1;
2348 f->fs.rpttid = 1;
2349
2350 ret = set_filter_wr(adap, stid);
2351 if (ret) {
2352 clear_filter(adap, f);
2353 return ret;
2354 }
2355
2356 return 0;
2357}
2358EXPORT_SYMBOL(cxgb4_create_server_filter);
2359
2360int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
2361 unsigned int queue, bool ipv6)
2362{
2363 struct filter_entry *f;
2364 struct adapter *adap;
2365
2366 adap = netdev2adap(dev);
2367
2368
2369 stid -= adap->tids.sftid_base;
2370 stid += adap->tids.nftids;
2371
2372 f = &adap->tids.ftid_tab[stid];
2373
2374 f->locked = 0;
2375
2376 return delete_filter(adap, stid);
2377}
2378EXPORT_SYMBOL(cxgb4_remove_server_filter);
2379
2380static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2381 struct rtnl_link_stats64 *ns)
2382{
2383 struct port_stats stats;
2384 struct port_info *p = netdev_priv(dev);
2385 struct adapter *adapter = p->adapter;
2386
2387
2388
2389
2390
2391 spin_lock(&adapter->stats_lock);
2392 if (!netif_device_present(dev)) {
2393 spin_unlock(&adapter->stats_lock);
2394 return ns;
2395 }
2396 t4_get_port_stats_offset(adapter, p->tx_chan, &stats,
2397 &p->stats_base);
2398 spin_unlock(&adapter->stats_lock);
2399
2400 ns->tx_bytes = stats.tx_octets;
2401 ns->tx_packets = stats.tx_frames;
2402 ns->rx_bytes = stats.rx_octets;
2403 ns->rx_packets = stats.rx_frames;
2404 ns->multicast = stats.rx_mcast_frames;
2405
2406
2407 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2408 stats.rx_runt;
2409 ns->rx_over_errors = 0;
2410 ns->rx_crc_errors = stats.rx_fcs_err;
2411 ns->rx_frame_errors = stats.rx_symbol_err;
2412 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2413 stats.rx_ovflow2 + stats.rx_ovflow3 +
2414 stats.rx_trunc0 + stats.rx_trunc1 +
2415 stats.rx_trunc2 + stats.rx_trunc3;
2416 ns->rx_missed_errors = 0;
2417
2418
2419 ns->tx_aborted_errors = 0;
2420 ns->tx_carrier_errors = 0;
2421 ns->tx_fifo_errors = 0;
2422 ns->tx_heartbeat_errors = 0;
2423 ns->tx_window_errors = 0;
2424
2425 ns->tx_errors = stats.tx_error_frames;
2426 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2427 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2428 return ns;
2429}
2430
2431static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2432{
2433 unsigned int mbox;
2434 int ret = 0, prtad, devad;
2435 struct port_info *pi = netdev_priv(dev);
2436 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2437
2438 switch (cmd) {
2439 case SIOCGMIIPHY:
2440 if (pi->mdio_addr < 0)
2441 return -EOPNOTSUPP;
2442 data->phy_id = pi->mdio_addr;
2443 break;
2444 case SIOCGMIIREG:
2445 case SIOCSMIIREG:
2446 if (mdio_phy_id_is_c45(data->phy_id)) {
2447 prtad = mdio_phy_id_prtad(data->phy_id);
2448 devad = mdio_phy_id_devad(data->phy_id);
2449 } else if (data->phy_id < 32) {
2450 prtad = data->phy_id;
2451 devad = 0;
2452 data->reg_num &= 0x1f;
2453 } else
2454 return -EINVAL;
2455
2456 mbox = pi->adapter->pf;
2457 if (cmd == SIOCGMIIREG)
2458 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
2459 data->reg_num, &data->val_out);
2460 else
2461 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
2462 data->reg_num, data->val_in);
2463 break;
2464 case SIOCGHWTSTAMP:
2465 return copy_to_user(req->ifr_data, &pi->tstamp_config,
2466 sizeof(pi->tstamp_config)) ?
2467 -EFAULT : 0;
2468 case SIOCSHWTSTAMP:
2469 if (copy_from_user(&pi->tstamp_config, req->ifr_data,
2470 sizeof(pi->tstamp_config)))
2471 return -EFAULT;
2472
2473 switch (pi->tstamp_config.rx_filter) {
2474 case HWTSTAMP_FILTER_NONE:
2475 pi->rxtstamp = false;
2476 break;
2477 case HWTSTAMP_FILTER_ALL:
2478 pi->rxtstamp = true;
2479 break;
2480 default:
2481 pi->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
2482 return -ERANGE;
2483 }
2484
2485 return copy_to_user(req->ifr_data, &pi->tstamp_config,
2486 sizeof(pi->tstamp_config)) ?
2487 -EFAULT : 0;
2488 default:
2489 return -EOPNOTSUPP;
2490 }
2491 return ret;
2492}
2493
2494static void cxgb_set_rxmode(struct net_device *dev)
2495{
2496
2497 set_rxmode(dev, -1, false);
2498}
2499
2500static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2501{
2502 int ret;
2503 struct port_info *pi = netdev_priv(dev);
2504
2505 if (new_mtu < 81 || new_mtu > MAX_MTU)
2506 return -EINVAL;
2507 ret = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, new_mtu, -1,
2508 -1, -1, -1, true);
2509 if (!ret)
2510 dev->mtu = new_mtu;
2511 return ret;
2512}
2513
2514#ifdef CONFIG_PCI_IOV
2515static int dummy_open(struct net_device *dev)
2516{
2517
2518
2519
2520 netif_carrier_off(dev);
2521 return 0;
2522}
2523
2524
2525static void fill_vf_station_mac_addr(struct adapter *adap)
2526{
2527 unsigned int i;
2528 u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN];
2529 int err;
2530 u8 *na;
2531 u16 a, b;
2532
2533 err = t4_get_raw_vpd_params(adap, &adap->params.vpd);
2534 if (!err) {
2535 na = adap->params.vpd.na;
2536 for (i = 0; i < ETH_ALEN; i++)
2537 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 +
2538 hex2val(na[2 * i + 1]));
2539 a = (hw_addr[0] << 8) | hw_addr[1];
2540 b = (hw_addr[1] << 8) | hw_addr[2];
2541 a ^= b;
2542 a |= 0x0200;
2543 a &= ~0x0100;
2544 macaddr[0] = a >> 8;
2545 macaddr[1] = a & 0xff;
2546
2547 for (i = 2; i < 5; i++)
2548 macaddr[i] = hw_addr[i + 1];
2549
2550 for (i = 0; i < adap->num_vfs; i++) {
2551 macaddr[5] = adap->pf * 16 + i;
2552 ether_addr_copy(adap->vfinfo[i].vf_mac_addr, macaddr);
2553 }
2554 }
2555}
2556
2557static int cxgb_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
2558{
2559 struct port_info *pi = netdev_priv(dev);
2560 struct adapter *adap = pi->adapter;
2561 int ret;
2562
2563
2564 if (!is_valid_ether_addr(mac)) {
2565 dev_err(pi->adapter->pdev_dev,
2566 "Invalid Ethernet address %pM for VF %d\n",
2567 mac, vf);
2568 return -EINVAL;
2569 }
2570
2571 dev_info(pi->adapter->pdev_dev,
2572 "Setting MAC %pM on VF %d\n", mac, vf);
2573 ret = t4_set_vf_mac_acl(adap, vf + 1, 1, mac);
2574 if (!ret)
2575 ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac);
2576 return ret;
2577}
2578
2579static int cxgb_get_vf_config(struct net_device *dev,
2580 int vf, struct ifla_vf_info *ivi)
2581{
2582 struct port_info *pi = netdev_priv(dev);
2583 struct adapter *adap = pi->adapter;
2584
2585 if (vf >= adap->num_vfs)
2586 return -EINVAL;
2587 ivi->vf = vf;
2588 ether_addr_copy(ivi->mac, adap->vfinfo[vf].vf_mac_addr);
2589 return 0;
2590}
2591#endif
2592
2593static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2594{
2595 int ret;
2596 struct sockaddr *addr = p;
2597 struct port_info *pi = netdev_priv(dev);
2598
2599 if (!is_valid_ether_addr(addr->sa_data))
2600 return -EADDRNOTAVAIL;
2601
2602 ret = t4_change_mac(pi->adapter, pi->adapter->pf, pi->viid,
2603 pi->xact_addr_filt, addr->sa_data, true, true);
2604 if (ret < 0)
2605 return ret;
2606
2607 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2608 pi->xact_addr_filt = ret;
2609 return 0;
2610}
2611
2612#ifdef CONFIG_NET_POLL_CONTROLLER
2613static void cxgb_netpoll(struct net_device *dev)
2614{
2615 struct port_info *pi = netdev_priv(dev);
2616 struct adapter *adap = pi->adapter;
2617
2618 if (adap->flags & USING_MSIX) {
2619 int i;
2620 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2621
2622 for (i = pi->nqsets; i; i--, rx++)
2623 t4_sge_intr_msix(0, &rx->rspq);
2624 } else
2625 t4_intr_handler(adap)(0, adap);
2626}
2627#endif
2628
2629static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
2630{
2631 struct port_info *pi = netdev_priv(dev);
2632 struct adapter *adap = pi->adapter;
2633 struct sched_class *e;
2634 struct ch_sched_params p;
2635 struct ch_sched_queue qe;
2636 u32 req_rate;
2637 int err = 0;
2638
2639 if (!can_sched(dev))
2640 return -ENOTSUPP;
2641
2642 if (index < 0 || index > pi->nqsets - 1)
2643 return -EINVAL;
2644
2645 if (!(adap->flags & FULL_INIT_DONE)) {
2646 dev_err(adap->pdev_dev,
2647 "Failed to rate limit on queue %d. Link Down?\n",
2648 index);
2649 return -EINVAL;
2650 }
2651
2652
2653 req_rate = rate << 10;
2654
2655
2656 if (req_rate >= SCHED_MAX_RATE_KBPS) {
2657 dev_err(adap->pdev_dev,
2658 "Invalid rate %u Mbps, Max rate is %u Gbps\n",
2659 rate, SCHED_MAX_RATE_KBPS);
2660 return -ERANGE;
2661 }
2662
2663
2664 memset(&qe, 0, sizeof(qe));
2665 qe.queue = index;
2666 qe.class = SCHED_CLS_NONE;
2667
2668 err = cxgb4_sched_class_unbind(dev, (void *)(&qe), SCHED_QUEUE);
2669 if (err) {
2670 dev_err(adap->pdev_dev,
2671 "Unbinding Queue %d on port %d fail. Err: %d\n",
2672 index, pi->port_id, err);
2673 return err;
2674 }
2675
2676
2677 if (!req_rate)
2678 return 0;
2679
2680
2681 memset(&p, 0, sizeof(p));
2682 p.type = SCHED_CLASS_TYPE_PACKET;
2683 p.u.params.level = SCHED_CLASS_LEVEL_CL_RL;
2684 p.u.params.mode = SCHED_CLASS_MODE_CLASS;
2685 p.u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS;
2686 p.u.params.ratemode = SCHED_CLASS_RATEMODE_ABS;
2687 p.u.params.channel = pi->tx_chan;
2688 p.u.params.class = SCHED_CLS_NONE;
2689 p.u.params.minrate = 0;
2690 p.u.params.maxrate = req_rate;
2691 p.u.params.weight = 0;
2692 p.u.params.pktsize = dev->mtu;
2693
2694 e = cxgb4_sched_class_alloc(dev, &p);
2695 if (!e)
2696 return -ENOMEM;
2697
2698
2699 memset(&qe, 0, sizeof(qe));
2700 qe.queue = index;
2701 qe.class = e->idx;
2702
2703 err = cxgb4_sched_class_bind(dev, (void *)(&qe), SCHED_QUEUE);
2704 if (err)
2705 dev_err(adap->pdev_dev,
2706 "Queue rate limiting failed. Err: %d\n", err);
2707 return err;
2708}
2709
2710static int cxgb_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
2711 struct tc_to_netdev *tc)
2712{
2713 struct port_info *pi = netdev2pinfo(dev);
2714 struct adapter *adap = netdev2adap(dev);
2715
2716 if (!(adap->flags & FULL_INIT_DONE)) {
2717 dev_err(adap->pdev_dev,
2718 "Failed to setup tc on port %d. Link Down?\n",
2719 pi->port_id);
2720 return -EINVAL;
2721 }
2722
2723 if (TC_H_MAJ(handle) == TC_H_MAJ(TC_H_INGRESS) &&
2724 tc->type == TC_SETUP_CLSU32) {
2725 switch (tc->cls_u32->command) {
2726 case TC_CLSU32_NEW_KNODE:
2727 case TC_CLSU32_REPLACE_KNODE:
2728 return cxgb4_config_knode(dev, proto, tc->cls_u32);
2729 case TC_CLSU32_DELETE_KNODE:
2730 return cxgb4_delete_knode(dev, proto, tc->cls_u32);
2731 default:
2732 return -EOPNOTSUPP;
2733 }
2734 }
2735
2736 return -EOPNOTSUPP;
2737}
2738
2739static const struct net_device_ops cxgb4_netdev_ops = {
2740 .ndo_open = cxgb_open,
2741 .ndo_stop = cxgb_close,
2742 .ndo_start_xmit = t4_eth_xmit,
2743 .ndo_select_queue = cxgb_select_queue,
2744 .ndo_get_stats64 = cxgb_get_stats,
2745 .ndo_set_rx_mode = cxgb_set_rxmode,
2746 .ndo_set_mac_address = cxgb_set_mac_addr,
2747 .ndo_set_features = cxgb_set_features,
2748 .ndo_validate_addr = eth_validate_addr,
2749 .ndo_do_ioctl = cxgb_ioctl,
2750 .ndo_change_mtu = cxgb_change_mtu,
2751#ifdef CONFIG_NET_POLL_CONTROLLER
2752 .ndo_poll_controller = cxgb_netpoll,
2753#endif
2754#ifdef CONFIG_CHELSIO_T4_FCOE
2755 .ndo_fcoe_enable = cxgb_fcoe_enable,
2756 .ndo_fcoe_disable = cxgb_fcoe_disable,
2757#endif
2758#ifdef CONFIG_NET_RX_BUSY_POLL
2759 .ndo_busy_poll = cxgb_busy_poll,
2760#endif
2761 .ndo_set_tx_maxrate = cxgb_set_tx_maxrate,
2762 .ndo_setup_tc = cxgb_setup_tc,
2763};
2764
2765#ifdef CONFIG_PCI_IOV
2766static const struct net_device_ops cxgb4_mgmt_netdev_ops = {
2767 .ndo_open = dummy_open,
2768 .ndo_set_vf_mac = cxgb_set_vf_mac,
2769 .ndo_get_vf_config = cxgb_get_vf_config,
2770};
2771#endif
2772
2773static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
2774{
2775 struct adapter *adapter = netdev2adap(dev);
2776
2777 strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
2778 strlcpy(info->version, cxgb4_driver_version,
2779 sizeof(info->version));
2780 strlcpy(info->bus_info, pci_name(adapter->pdev),
2781 sizeof(info->bus_info));
2782}
2783
2784static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = {
2785 .get_drvinfo = get_drvinfo,
2786};
2787
2788void t4_fatal_err(struct adapter *adap)
2789{
2790 t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0);
2791 t4_intr_disable(adap);
2792 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2793}
2794
2795static void setup_memwin(struct adapter *adap)
2796{
2797 u32 nic_win_base = t4_get_util_window(adap);
2798
2799 t4_setup_memwin(adap, nic_win_base, MEMWIN_NIC);
2800}
2801
2802static void setup_memwin_rdma(struct adapter *adap)
2803{
2804 if (adap->vres.ocq.size) {
2805 u32 start;
2806 unsigned int sz_kb;
2807
2808 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
2809 start &= PCI_BASE_ADDRESS_MEM_MASK;
2810 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2811 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2812 t4_write_reg(adap,
2813 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
2814 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
2815 t4_write_reg(adap,
2816 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
2817 adap->vres.ocq.start);
2818 t4_read_reg(adap,
2819 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
2820 }
2821}
2822
2823static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2824{
2825 u32 v;
2826 int ret;
2827
2828
2829 memset(c, 0, sizeof(*c));
2830 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
2831 FW_CMD_REQUEST_F | FW_CMD_READ_F);
2832 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
2833 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), c);
2834 if (ret < 0)
2835 return ret;
2836
2837 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
2838 FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
2839 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), NULL);
2840 if (ret < 0)
2841 return ret;
2842
2843 ret = t4_config_glbl_rss(adap, adap->pf,
2844 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2845 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
2846 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
2847 if (ret < 0)
2848 return ret;
2849
2850 ret = t4_cfg_pfvf(adap, adap->mbox, adap->pf, 0, adap->sge.egr_sz, 64,
2851 MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF,
2852 FW_CMD_CAP_PF);
2853 if (ret < 0)
2854 return ret;
2855
2856 t4_sge_init(adap);
2857
2858
2859 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
2860 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
2861 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
2862 v = t4_read_reg(adap, TP_PIO_DATA_A);
2863 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
2864
2865
2866 adap->params.tp.tx_modq_map = 0xE4;
2867 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
2868 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
2869
2870
2871 v = 0x84218421;
2872 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
2873 &v, 1, TP_TX_SCHED_HDR_A);
2874 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
2875 &v, 1, TP_TX_SCHED_FIFO_A);
2876 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
2877 &v, 1, TP_TX_SCHED_PCMD_A);
2878
2879#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16
2880 if (is_offload(adap)) {
2881 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
2882 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
2883 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
2884 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
2885 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
2886 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
2887 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
2888 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
2889 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
2890 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
2891 }
2892
2893
2894 return t4_early_init(adap, adap->pf);
2895}
2896
2897
2898
2899
2900#define MAX_ATIDS 8192U
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918static int adap_init0_tweaks(struct adapter *adapter)
2919{
2920
2921
2922
2923
2924
2925 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
2926
2927
2928
2929
2930 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
2931 dev_err(&adapter->pdev->dev,
2932 "Ignoring illegal rx_dma_offset=%d, using 2\n",
2933 rx_dma_offset);
2934 rx_dma_offset = 2;
2935 }
2936 t4_set_reg_field(adapter, SGE_CONTROL_A,
2937 PKTSHIFT_V(PKTSHIFT_M),
2938 PKTSHIFT_V(rx_dma_offset));
2939
2940
2941
2942
2943
2944 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
2945 CSUM_HAS_PSEUDO_HDR_F, 0);
2946
2947 return 0;
2948}
2949
2950
2951
2952
2953
2954static int phy_aq1202_version(const u8 *phy_fw_data,
2955 size_t phy_fw_size)
2956{
2957 int offset;
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968 #define be16(__p) (((__p)[0] << 8) | (__p)[1])
2969 #define le16(__p) ((__p)[0] | ((__p)[1] << 8))
2970 #define le24(__p) (le16(__p) | ((__p)[2] << 16))
2971
2972 offset = le24(phy_fw_data + 0x8) << 12;
2973 offset = le24(phy_fw_data + offset + 0xa);
2974 return be16(phy_fw_data + offset + 0x27e);
2975
2976 #undef be16
2977 #undef le16
2978 #undef le24
2979}
2980
2981static struct info_10gbt_phy_fw {
2982 unsigned int phy_fw_id;
2983 char *phy_fw_file;
2984 int (*phy_fw_version)(const u8 *phy_fw_data, size_t phy_fw_size);
2985 int phy_flash;
2986} phy_info_array[] = {
2987 {
2988 PHY_AQ1202_DEVICEID,
2989 PHY_AQ1202_FIRMWARE,
2990 phy_aq1202_version,
2991 1,
2992 },
2993 {
2994 PHY_BCM84834_DEVICEID,
2995 PHY_BCM84834_FIRMWARE,
2996 NULL,
2997 0,
2998 },
2999 { 0, NULL, NULL },
3000};
3001
3002static struct info_10gbt_phy_fw *find_phy_info(int devid)
3003{
3004 int i;
3005
3006 for (i = 0; i < ARRAY_SIZE(phy_info_array); i++) {
3007 if (phy_info_array[i].phy_fw_id == devid)
3008 return &phy_info_array[i];
3009 }
3010 return NULL;
3011}
3012
3013
3014
3015
3016
3017
3018static int adap_init0_phy(struct adapter *adap)
3019{
3020 const struct firmware *phyf;
3021 int ret;
3022 struct info_10gbt_phy_fw *phy_info;
3023
3024
3025
3026 phy_info = find_phy_info(adap->pdev->device);
3027 if (!phy_info) {
3028 dev_warn(adap->pdev_dev,
3029 "No PHY Firmware file found for this PHY\n");
3030 return -EOPNOTSUPP;
3031 }
3032
3033
3034
3035
3036
3037
3038 ret = request_firmware_direct(&phyf, phy_info->phy_fw_file,
3039 adap->pdev_dev);
3040 if (ret < 0) {
3041
3042
3043
3044
3045
3046
3047 dev_err(adap->pdev_dev, "unable to find PHY Firmware image "
3048 "/lib/firmware/%s, error %d\n",
3049 phy_info->phy_fw_file, -ret);
3050 if (phy_info->phy_flash) {
3051 int cur_phy_fw_ver = 0;
3052
3053 t4_phy_fw_ver(adap, &cur_phy_fw_ver);
3054 dev_warn(adap->pdev_dev, "continuing with, on-adapter "
3055 "FLASH copy, version %#x\n", cur_phy_fw_ver);
3056 ret = 0;
3057 }
3058
3059 return ret;
3060 }
3061
3062
3063
3064 ret = t4_load_phy_fw(adap, MEMWIN_NIC, &adap->win0_lock,
3065 phy_info->phy_fw_version,
3066 (u8 *)phyf->data, phyf->size);
3067 if (ret < 0)
3068 dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n",
3069 -ret);
3070 else if (ret > 0) {
3071 int new_phy_fw_ver = 0;
3072
3073 if (phy_info->phy_fw_version)
3074 new_phy_fw_ver = phy_info->phy_fw_version(phyf->data,
3075 phyf->size);
3076 dev_info(adap->pdev_dev, "Successfully transferred PHY "
3077 "Firmware /lib/firmware/%s, version %#x\n",
3078 phy_info->phy_fw_file, new_phy_fw_ver);
3079 }
3080
3081 release_firmware(phyf);
3082
3083 return ret;
3084}
3085
3086
3087
3088
3089static int adap_init0_config(struct adapter *adapter, int reset)
3090{
3091 struct fw_caps_config_cmd caps_cmd;
3092 const struct firmware *cf;
3093 unsigned long mtype = 0, maddr = 0;
3094 u32 finiver, finicsum, cfcsum;
3095 int ret;
3096 int config_issued = 0;
3097 char *fw_config_file, fw_config_file_path[256];
3098 char *config_name = NULL;
3099
3100
3101
3102
3103 if (reset) {
3104 ret = t4_fw_reset(adapter, adapter->mbox,
3105 PIORSTMODE_F | PIORST_F);
3106 if (ret < 0)
3107 goto bye;
3108 }
3109
3110
3111
3112
3113
3114
3115 if (is_10gbt_device(adapter->pdev->device)) {
3116 ret = adap_init0_phy(adapter);
3117 if (ret < 0)
3118 goto bye;
3119 }
3120
3121
3122
3123
3124
3125 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
3126 case CHELSIO_T4:
3127 fw_config_file = FW4_CFNAME;
3128 break;
3129 case CHELSIO_T5:
3130 fw_config_file = FW5_CFNAME;
3131 break;
3132 case CHELSIO_T6:
3133 fw_config_file = FW6_CFNAME;
3134 break;
3135 default:
3136 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
3137 adapter->pdev->device);
3138 ret = -EINVAL;
3139 goto bye;
3140 }
3141
3142 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
3143 if (ret < 0) {
3144 config_name = "On FLASH";
3145 mtype = FW_MEMTYPE_CF_FLASH;
3146 maddr = t4_flash_cfg_addr(adapter);
3147 } else {
3148 u32 params[7], val[7];
3149
3150 sprintf(fw_config_file_path,
3151 "/lib/firmware/%s", fw_config_file);
3152 config_name = fw_config_file_path;
3153
3154 if (cf->size >= FLASH_CFG_MAX_SIZE)
3155 ret = -ENOMEM;
3156 else {
3157 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3158 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
3159 ret = t4_query_params(adapter, adapter->mbox,
3160 adapter->pf, 0, 1, params, val);
3161 if (ret == 0) {
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172 size_t resid = cf->size & 0x3;
3173 size_t size = cf->size & ~0x3;
3174 __be32 *data = (__be32 *)cf->data;
3175
3176 mtype = FW_PARAMS_PARAM_Y_G(val[0]);
3177 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
3178
3179 spin_lock(&adapter->win0_lock);
3180 ret = t4_memory_rw(adapter, 0, mtype, maddr,
3181 size, data, T4_MEMORY_WRITE);
3182 if (ret == 0 && resid != 0) {
3183 union {
3184 __be32 word;
3185 char buf[4];
3186 } last;
3187 int i;
3188
3189 last.word = data[size >> 2];
3190 for (i = resid; i < 4; i++)
3191 last.buf[i] = 0;
3192 ret = t4_memory_rw(adapter, 0, mtype,
3193 maddr + size,
3194 4, &last.word,
3195 T4_MEMORY_WRITE);
3196 }
3197 spin_unlock(&adapter->win0_lock);
3198 }
3199 }
3200
3201 release_firmware(cf);
3202 if (ret)
3203 goto bye;
3204 }
3205
3206
3207
3208
3209
3210
3211
3212 memset(&caps_cmd, 0, sizeof(caps_cmd));
3213 caps_cmd.op_to_write =
3214 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
3215 FW_CMD_REQUEST_F |
3216 FW_CMD_READ_F);
3217 caps_cmd.cfvalid_to_len16 =
3218 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
3219 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
3220 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
3221 FW_LEN16(caps_cmd));
3222 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3223 &caps_cmd);
3224
3225
3226
3227
3228
3229
3230
3231 if (ret == -ENOENT) {
3232 memset(&caps_cmd, 0, sizeof(caps_cmd));
3233 caps_cmd.op_to_write =
3234 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
3235 FW_CMD_REQUEST_F |
3236 FW_CMD_READ_F);
3237 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
3238 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
3239 sizeof(caps_cmd), &caps_cmd);
3240 config_name = "Firmware Default";
3241 }
3242
3243 config_issued = 1;
3244 if (ret < 0)
3245 goto bye;
3246
3247 finiver = ntohl(caps_cmd.finiver);
3248 finicsum = ntohl(caps_cmd.finicsum);
3249 cfcsum = ntohl(caps_cmd.cfcsum);
3250 if (finicsum != cfcsum)
3251 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
3252 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
3253 finicsum, cfcsum);
3254
3255
3256
3257
3258 caps_cmd.op_to_write =
3259 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
3260 FW_CMD_REQUEST_F |
3261 FW_CMD_WRITE_F);
3262 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
3263 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3264 NULL);
3265 if (ret < 0)
3266 goto bye;
3267
3268
3269
3270
3271
3272 ret = adap_init0_tweaks(adapter);
3273 if (ret < 0)
3274 goto bye;
3275
3276
3277
3278
3279
3280 ret = t4_fw_initialize(adapter, adapter->mbox);
3281 if (ret < 0)
3282 goto bye;
3283
3284
3285
3286
3287 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
3288 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
3289 config_name, finiver, cfcsum);
3290 return 0;
3291
3292
3293
3294
3295
3296
3297bye:
3298 if (config_issued && ret != -ENOENT)
3299 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
3300 config_name, -ret);
3301 return ret;
3302}
3303
3304static struct fw_info fw_info_array[] = {
3305 {
3306 .chip = CHELSIO_T4,
3307 .fs_name = FW4_CFNAME,
3308 .fw_mod_name = FW4_FNAME,
3309 .fw_hdr = {
3310 .chip = FW_HDR_CHIP_T4,
3311 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
3312 .intfver_nic = FW_INTFVER(T4, NIC),
3313 .intfver_vnic = FW_INTFVER(T4, VNIC),
3314 .intfver_ri = FW_INTFVER(T4, RI),
3315 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
3316 .intfver_fcoe = FW_INTFVER(T4, FCOE),
3317 },
3318 }, {
3319 .chip = CHELSIO_T5,
3320 .fs_name = FW5_CFNAME,
3321 .fw_mod_name = FW5_FNAME,
3322 .fw_hdr = {
3323 .chip = FW_HDR_CHIP_T5,
3324 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
3325 .intfver_nic = FW_INTFVER(T5, NIC),
3326 .intfver_vnic = FW_INTFVER(T5, VNIC),
3327 .intfver_ri = FW_INTFVER(T5, RI),
3328 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
3329 .intfver_fcoe = FW_INTFVER(T5, FCOE),
3330 },
3331 }, {
3332 .chip = CHELSIO_T6,
3333 .fs_name = FW6_CFNAME,
3334 .fw_mod_name = FW6_FNAME,
3335 .fw_hdr = {
3336 .chip = FW_HDR_CHIP_T6,
3337 .fw_ver = __cpu_to_be32(FW_VERSION(T6)),
3338 .intfver_nic = FW_INTFVER(T6, NIC),
3339 .intfver_vnic = FW_INTFVER(T6, VNIC),
3340 .intfver_ofld = FW_INTFVER(T6, OFLD),
3341 .intfver_ri = FW_INTFVER(T6, RI),
3342 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU),
3343 .intfver_iscsi = FW_INTFVER(T6, ISCSI),
3344 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU),
3345 .intfver_fcoe = FW_INTFVER(T6, FCOE),
3346 },
3347 }
3348
3349};
3350
3351static struct fw_info *find_fw_info(int chip)
3352{
3353 int i;
3354
3355 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
3356 if (fw_info_array[i].chip == chip)
3357 return &fw_info_array[i];
3358 }
3359 return NULL;
3360}
3361
3362
3363
3364
3365static int adap_init0(struct adapter *adap)
3366{
3367 int ret;
3368 u32 v, port_vec;
3369 enum dev_state state;
3370 u32 params[7], val[7];
3371 struct fw_caps_config_cmd caps_cmd;
3372 int reset = 1;
3373
3374
3375
3376
3377 ret = t4_init_devlog_params(adap);
3378 if (ret < 0)
3379 return ret;
3380
3381
3382 ret = t4_fw_hello(adap, adap->mbox, adap->mbox,
3383 is_kdump_kernel() ? MASTER_MUST : MASTER_MAY, &state);
3384 if (ret < 0) {
3385 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
3386 ret);
3387 return ret;
3388 }
3389 if (ret == adap->mbox)
3390 adap->flags |= MASTER_PF;
3391
3392
3393
3394
3395
3396
3397
3398
3399 t4_get_fw_version(adap, &adap->params.fw_vers);
3400 t4_get_bs_version(adap, &adap->params.bs_vers);
3401 t4_get_tp_version(adap, &adap->params.tp_vers);
3402 t4_get_exprom_version(adap, &adap->params.er_vers);
3403
3404 ret = t4_check_fw_version(adap);
3405
3406 if (ret)
3407 state = DEV_STATE_UNINIT;
3408 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
3409 struct fw_info *fw_info;
3410 struct fw_hdr *card_fw;
3411 const struct firmware *fw;
3412 const u8 *fw_data = NULL;
3413 unsigned int fw_size = 0;
3414
3415
3416
3417
3418 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
3419 if (fw_info == NULL) {
3420 dev_err(adap->pdev_dev,
3421 "unable to get firmware info for chip %d.\n",
3422 CHELSIO_CHIP_VERSION(adap->params.chip));
3423 return -EINVAL;
3424 }
3425
3426
3427
3428
3429 card_fw = t4_alloc_mem(sizeof(*card_fw));
3430
3431
3432 ret = request_firmware(&fw, fw_info->fw_mod_name,
3433 adap->pdev_dev);
3434 if (ret < 0) {
3435 dev_err(adap->pdev_dev,
3436 "unable to load firmware image %s, error %d\n",
3437 fw_info->fw_mod_name, ret);
3438 } else {
3439 fw_data = fw->data;
3440 fw_size = fw->size;
3441 }
3442
3443
3444 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
3445 state, &reset);
3446
3447
3448 release_firmware(fw);
3449 t4_free_mem(card_fw);
3450
3451 if (ret < 0)
3452 goto bye;
3453 }
3454
3455
3456
3457
3458
3459
3460
3461
3462 ret = t4_get_vpd_params(adap, &adap->params.vpd);
3463 if (ret < 0)
3464 goto bye;
3465
3466
3467
3468
3469
3470
3471 v =
3472 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3473 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
3474 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec);
3475 if (ret < 0)
3476 goto bye;
3477
3478 adap->params.nports = hweight32(port_vec);
3479 adap->params.portvec = port_vec;
3480
3481
3482
3483
3484 if (state == DEV_STATE_INIT) {
3485 dev_info(adap->pdev_dev, "Coming up as %s: "\
3486 "Adapter already initialized\n",
3487 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
3488 } else {
3489 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
3490 "Initializing adapter\n");
3491
3492
3493
3494
3495 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3496 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
3497 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
3498 params, val);
3499
3500
3501
3502
3503 if (ret < 0) {
3504 dev_err(adap->pdev_dev, "firmware doesn't support "
3505 "Firmware Configuration Files\n");
3506 goto bye;
3507 }
3508
3509
3510
3511
3512
3513 ret = adap_init0_config(adap, reset);
3514 if (ret == -ENOENT) {
3515 dev_err(adap->pdev_dev, "no Configuration File "
3516 "present on adapter.\n");
3517 goto bye;
3518 }
3519 if (ret < 0) {
3520 dev_err(adap->pdev_dev, "could not initialize "
3521 "adapter, error %d\n", -ret);
3522 goto bye;
3523 }
3524 }
3525
3526
3527
3528
3529
3530 ret = t4_sge_init(adap);
3531 if (ret < 0)
3532 goto bye;
3533
3534 if (is_bypass_device(adap->pdev->device))
3535 adap->params.bypass = 1;
3536
3537
3538
3539
3540#define FW_PARAM_DEV(param) \
3541 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
3542 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
3543
3544#define FW_PARAM_PFVF(param) \
3545 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
3546 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \
3547 FW_PARAMS_PARAM_Y_V(0) | \
3548 FW_PARAMS_PARAM_Z_V(0)
3549
3550 params[0] = FW_PARAM_PFVF(EQ_START);
3551 params[1] = FW_PARAM_PFVF(L2T_START);
3552 params[2] = FW_PARAM_PFVF(L2T_END);
3553 params[3] = FW_PARAM_PFVF(FILTER_START);
3554 params[4] = FW_PARAM_PFVF(FILTER_END);
3555 params[5] = FW_PARAM_PFVF(IQFLINT_START);
3556 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, val);
3557 if (ret < 0)
3558 goto bye;
3559 adap->sge.egr_start = val[0];
3560 adap->l2t_start = val[1];
3561 adap->l2t_end = val[2];
3562 adap->tids.ftid_base = val[3];
3563 adap->tids.nftids = val[4] - val[3] + 1;
3564 adap->sge.ingr_start = val[5];
3565
3566
3567
3568
3569
3570
3571
3572 params[0] = FW_PARAM_PFVF(EQ_END);
3573 params[1] = FW_PARAM_PFVF(IQFLINT_END);
3574 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
3575 if (ret < 0)
3576 goto bye;
3577 adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1;
3578 adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1;
3579
3580 adap->sge.egr_map = kcalloc(adap->sge.egr_sz,
3581 sizeof(*adap->sge.egr_map), GFP_KERNEL);
3582 if (!adap->sge.egr_map) {
3583 ret = -ENOMEM;
3584 goto bye;
3585 }
3586
3587 adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz,
3588 sizeof(*adap->sge.ingr_map), GFP_KERNEL);
3589 if (!adap->sge.ingr_map) {
3590 ret = -ENOMEM;
3591 goto bye;
3592 }
3593
3594
3595
3596
3597 adap->sge.starving_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
3598 sizeof(long), GFP_KERNEL);
3599 if (!adap->sge.starving_fl) {
3600 ret = -ENOMEM;
3601 goto bye;
3602 }
3603
3604 adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
3605 sizeof(long), GFP_KERNEL);
3606 if (!adap->sge.txq_maperr) {
3607 ret = -ENOMEM;
3608 goto bye;
3609 }
3610
3611#ifdef CONFIG_DEBUG_FS
3612 adap->sge.blocked_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
3613 sizeof(long), GFP_KERNEL);
3614 if (!adap->sge.blocked_fl) {
3615 ret = -ENOMEM;
3616 goto bye;
3617 }
3618#endif
3619
3620 params[0] = FW_PARAM_PFVF(CLIP_START);
3621 params[1] = FW_PARAM_PFVF(CLIP_END);
3622 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
3623 if (ret < 0)
3624 goto bye;
3625 adap->clipt_start = val[0];
3626 adap->clipt_end = val[1];
3627
3628
3629
3630
3631
3632 adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16;
3633
3634
3635 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
3636 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
3637 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
3638
3639
3640
3641 if ((val[0] != val[1]) && (ret >= 0)) {
3642 adap->flags |= FW_OFLD_CONN;
3643 adap->tids.aftid_base = val[0];
3644 adap->tids.aftid_end = val[1];
3645 }
3646
3647
3648
3649
3650
3651
3652 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
3653 val[0] = 1;
3654 (void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val);
3655
3656
3657
3658
3659
3660
3661
3662 if (is_t4(adap->params.chip)) {
3663 adap->params.ulptx_memwrite_dsgl = false;
3664 } else {
3665 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
3666 ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
3667 1, params, val);
3668 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
3669 }
3670
3671
3672 params[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR);
3673 ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
3674 1, params, val);
3675 adap->params.fr_nsmr_tpte_wr_support = (ret == 0 && val[0] != 0);
3676
3677
3678
3679
3680
3681 memset(&caps_cmd, 0, sizeof(caps_cmd));
3682 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
3683 FW_CMD_REQUEST_F | FW_CMD_READ_F);
3684 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
3685 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
3686 &caps_cmd);
3687 if (ret < 0)
3688 goto bye;
3689
3690 if (caps_cmd.ofldcaps) {
3691
3692 params[0] = FW_PARAM_DEV(NTID);
3693 params[1] = FW_PARAM_PFVF(SERVER_START);
3694 params[2] = FW_PARAM_PFVF(SERVER_END);
3695 params[3] = FW_PARAM_PFVF(TDDP_START);
3696 params[4] = FW_PARAM_PFVF(TDDP_END);
3697 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3698 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6,
3699 params, val);
3700 if (ret < 0)
3701 goto bye;
3702 adap->tids.ntids = val[0];
3703 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3704 adap->tids.stid_base = val[1];
3705 adap->tids.nstids = val[2] - val[1] + 1;
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
3716 adap->tids.sftid_base = adap->tids.ftid_base +
3717 DIV_ROUND_UP(adap->tids.nftids, 3);
3718 adap->tids.nsftids = adap->tids.nftids -
3719 DIV_ROUND_UP(adap->tids.nftids, 3);
3720 adap->tids.nftids = adap->tids.sftid_base -
3721 adap->tids.ftid_base;
3722 }
3723 adap->vres.ddp.start = val[3];
3724 adap->vres.ddp.size = val[4] - val[3] + 1;
3725 adap->params.ofldq_wr_cred = val[5];
3726
3727 adap->params.offload = 1;
3728 adap->num_ofld_uld += 1;
3729 }
3730 if (caps_cmd.rdmacaps) {
3731 params[0] = FW_PARAM_PFVF(STAG_START);
3732 params[1] = FW_PARAM_PFVF(STAG_END);
3733 params[2] = FW_PARAM_PFVF(RQ_START);
3734 params[3] = FW_PARAM_PFVF(RQ_END);
3735 params[4] = FW_PARAM_PFVF(PBL_START);
3736 params[5] = FW_PARAM_PFVF(PBL_END);
3737 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6,
3738 params, val);
3739 if (ret < 0)
3740 goto bye;
3741 adap->vres.stag.start = val[0];
3742 adap->vres.stag.size = val[1] - val[0] + 1;
3743 adap->vres.rq.start = val[2];
3744 adap->vres.rq.size = val[3] - val[2] + 1;
3745 adap->vres.pbl.start = val[4];
3746 adap->vres.pbl.size = val[5] - val[4] + 1;
3747
3748 params[0] = FW_PARAM_PFVF(SQRQ_START);
3749 params[1] = FW_PARAM_PFVF(SQRQ_END);
3750 params[2] = FW_PARAM_PFVF(CQ_START);
3751 params[3] = FW_PARAM_PFVF(CQ_END);
3752 params[4] = FW_PARAM_PFVF(OCQ_START);
3753 params[5] = FW_PARAM_PFVF(OCQ_END);
3754 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params,
3755 val);
3756 if (ret < 0)
3757 goto bye;
3758 adap->vres.qp.start = val[0];
3759 adap->vres.qp.size = val[1] - val[0] + 1;
3760 adap->vres.cq.start = val[2];
3761 adap->vres.cq.size = val[3] - val[2] + 1;
3762 adap->vres.ocq.start = val[4];
3763 adap->vres.ocq.size = val[5] - val[4] + 1;
3764
3765 params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
3766 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
3767 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params,
3768 val);
3769 if (ret < 0) {
3770 adap->params.max_ordird_qp = 8;
3771 adap->params.max_ird_adapter = 32 * adap->tids.ntids;
3772 ret = 0;
3773 } else {
3774 adap->params.max_ordird_qp = val[0];
3775 adap->params.max_ird_adapter = val[1];
3776 }
3777 dev_info(adap->pdev_dev,
3778 "max_ordird_qp %d max_ird_adapter %d\n",
3779 adap->params.max_ordird_qp,
3780 adap->params.max_ird_adapter);
3781 adap->num_ofld_uld += 2;
3782 }
3783 if (caps_cmd.iscsicaps) {
3784 params[0] = FW_PARAM_PFVF(ISCSI_START);
3785 params[1] = FW_PARAM_PFVF(ISCSI_END);
3786 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
3787 params, val);
3788 if (ret < 0)
3789 goto bye;
3790 adap->vres.iscsi.start = val[0];
3791 adap->vres.iscsi.size = val[1] - val[0] + 1;
3792
3793 adap->num_ofld_uld += 2;
3794 }
3795 if (caps_cmd.cryptocaps) {
3796
3797 adap->params.crypto |= ULP_CRYPTO_LOOKASIDE;
3798 adap->num_uld += 1;
3799 }
3800#undef FW_PARAM_PFVF
3801#undef FW_PARAM_DEV
3802
3803
3804
3805
3806
3807
3808 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3809 if (state != DEV_STATE_INIT) {
3810 int i;
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829 for (i = 0; i < NMTUS; i++)
3830 if (adap->params.mtus[i] == 1492) {
3831 adap->params.mtus[i] = 1488;
3832 break;
3833 }
3834
3835 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3836 adap->params.b_wnd);
3837 }
3838 t4_init_sge_params(adap);
3839 adap->flags |= FW_OK;
3840 t4_init_tp_params(adap);
3841 return 0;
3842
3843
3844
3845
3846
3847
3848bye:
3849 kfree(adap->sge.egr_map);
3850 kfree(adap->sge.ingr_map);
3851 kfree(adap->sge.starving_fl);
3852 kfree(adap->sge.txq_maperr);
3853#ifdef CONFIG_DEBUG_FS
3854 kfree(adap->sge.blocked_fl);
3855#endif
3856 if (ret != -ETIMEDOUT && ret != -EIO)
3857 t4_fw_bye(adap, adap->mbox);
3858 return ret;
3859}
3860
3861
3862
3863static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3864 pci_channel_state_t state)
3865{
3866 int i;
3867 struct adapter *adap = pci_get_drvdata(pdev);
3868
3869 if (!adap)
3870 goto out;
3871
3872 rtnl_lock();
3873 adap->flags &= ~FW_OK;
3874 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3875 spin_lock(&adap->stats_lock);
3876 for_each_port(adap, i) {
3877 struct net_device *dev = adap->port[i];
3878
3879 netif_device_detach(dev);
3880 netif_carrier_off(dev);
3881 }
3882 spin_unlock(&adap->stats_lock);
3883 disable_interrupts(adap);
3884 if (adap->flags & FULL_INIT_DONE)
3885 cxgb_down(adap);
3886 rtnl_unlock();
3887 if ((adap->flags & DEV_ENABLED)) {
3888 pci_disable_device(pdev);
3889 adap->flags &= ~DEV_ENABLED;
3890 }
3891out: return state == pci_channel_io_perm_failure ?
3892 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3893}
3894
3895static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3896{
3897 int i, ret;
3898 struct fw_caps_config_cmd c;
3899 struct adapter *adap = pci_get_drvdata(pdev);
3900
3901 if (!adap) {
3902 pci_restore_state(pdev);
3903 pci_save_state(pdev);
3904 return PCI_ERS_RESULT_RECOVERED;
3905 }
3906
3907 if (!(adap->flags & DEV_ENABLED)) {
3908 if (pci_enable_device(pdev)) {
3909 dev_err(&pdev->dev, "Cannot reenable PCI "
3910 "device after reset\n");
3911 return PCI_ERS_RESULT_DISCONNECT;
3912 }
3913 adap->flags |= DEV_ENABLED;
3914 }
3915
3916 pci_set_master(pdev);
3917 pci_restore_state(pdev);
3918 pci_save_state(pdev);
3919 pci_cleanup_aer_uncorrect_error_status(pdev);
3920
3921 if (t4_wait_dev_ready(adap->regs) < 0)
3922 return PCI_ERS_RESULT_DISCONNECT;
3923 if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0)
3924 return PCI_ERS_RESULT_DISCONNECT;
3925 adap->flags |= FW_OK;
3926 if (adap_init1(adap, &c))
3927 return PCI_ERS_RESULT_DISCONNECT;
3928
3929 for_each_port(adap, i) {
3930 struct port_info *p = adap2pinfo(adap, i);
3931
3932 ret = t4_alloc_vi(adap, adap->mbox, p->tx_chan, adap->pf, 0, 1,
3933 NULL, NULL);
3934 if (ret < 0)
3935 return PCI_ERS_RESULT_DISCONNECT;
3936 p->viid = ret;
3937 p->xact_addr_filt = -1;
3938 }
3939
3940 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3941 adap->params.b_wnd);
3942 setup_memwin(adap);
3943 if (cxgb_up(adap))
3944 return PCI_ERS_RESULT_DISCONNECT;
3945 return PCI_ERS_RESULT_RECOVERED;
3946}
3947
3948static void eeh_resume(struct pci_dev *pdev)
3949{
3950 int i;
3951 struct adapter *adap = pci_get_drvdata(pdev);
3952
3953 if (!adap)
3954 return;
3955
3956 rtnl_lock();
3957 for_each_port(adap, i) {
3958 struct net_device *dev = adap->port[i];
3959
3960 if (netif_running(dev)) {
3961 link_start(dev);
3962 cxgb_set_rxmode(dev);
3963 }
3964 netif_device_attach(dev);
3965 }
3966 rtnl_unlock();
3967}
3968
3969static const struct pci_error_handlers cxgb4_eeh = {
3970 .error_detected = eeh_err_detected,
3971 .slot_reset = eeh_slot_reset,
3972 .resume = eeh_resume,
3973};
3974
3975
3976
3977
3978static inline bool is_x_10g_port(const struct link_config *lc)
3979{
3980 unsigned int speeds, high_speeds;
3981
3982 speeds = FW_PORT_CAP_SPEED_V(FW_PORT_CAP_SPEED_G(lc->supported));
3983 high_speeds = speeds & ~(FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G);
3984
3985 return high_speeds != 0;
3986}
3987
3988
3989
3990
3991
3992
3993static void cfg_queues(struct adapter *adap)
3994{
3995 struct sge *s = &adap->sge;
3996 int i, n10g = 0, qidx = 0;
3997#ifndef CONFIG_CHELSIO_T4_DCB
3998 int q10g = 0;
3999#endif
4000
4001
4002
4003 if (is_kdump_kernel()) {
4004 adap->params.offload = 0;
4005 adap->params.crypto = 0;
4006 } else if (is_uld(adap) && t4_uld_mem_alloc(adap)) {
4007 adap->params.offload = 0;
4008 adap->params.crypto = 0;
4009 }
4010
4011 for_each_port(adap, i)
4012 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
4013#ifdef CONFIG_CHELSIO_T4_DCB
4014
4015
4016
4017
4018 if (adap->params.nports * 8 > MAX_ETH_QSETS) {
4019 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
4020 MAX_ETH_QSETS, adap->params.nports * 8);
4021 BUG_ON(1);
4022 }
4023
4024 for_each_port(adap, i) {
4025 struct port_info *pi = adap2pinfo(adap, i);
4026
4027 pi->first_qset = qidx;
4028 pi->nqsets = 8;
4029 qidx += pi->nqsets;
4030 }
4031#else
4032
4033
4034
4035
4036 if (n10g)
4037 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
4038 if (q10g > netif_get_num_default_rss_queues())
4039 q10g = netif_get_num_default_rss_queues();
4040
4041 for_each_port(adap, i) {
4042 struct port_info *pi = adap2pinfo(adap, i);
4043
4044 pi->first_qset = qidx;
4045 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
4046 qidx += pi->nqsets;
4047 }
4048#endif
4049
4050 s->ethqsets = qidx;
4051 s->max_ethqsets = qidx;
4052
4053 if (is_uld(adap)) {
4054
4055
4056
4057
4058
4059 if (n10g) {
4060 i = min_t(int, MAX_OFLD_QSETS, num_online_cpus());
4061 s->ofldqsets = roundup(i, adap->params.nports);
4062 } else {
4063 s->ofldqsets = adap->params.nports;
4064 }
4065 }
4066
4067 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
4068 struct sge_eth_rxq *r = &s->ethrxq[i];
4069
4070 init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
4071 r->fl.size = 72;
4072 }
4073
4074 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
4075 s->ethtxq[i].q.size = 1024;
4076
4077 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
4078 s->ctrlq[i].q.size = 512;
4079
4080 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
4081 s->ofldtxq[i].q.size = 1024;
4082
4083 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
4084 init_rspq(adap, &s->intrq, 0, 1, 512, 64);
4085}
4086
4087
4088
4089
4090
4091static void reduce_ethqs(struct adapter *adap, int n)
4092{
4093 int i;
4094 struct port_info *pi;
4095
4096 while (n < adap->sge.ethqsets)
4097 for_each_port(adap, i) {
4098 pi = adap2pinfo(adap, i);
4099 if (pi->nqsets > 1) {
4100 pi->nqsets--;
4101 adap->sge.ethqsets--;
4102 if (adap->sge.ethqsets <= n)
4103 break;
4104 }
4105 }
4106
4107 n = 0;
4108 for_each_port(adap, i) {
4109 pi = adap2pinfo(adap, i);
4110 pi->first_qset = n;
4111 n += pi->nqsets;
4112 }
4113}
4114
4115static int get_msix_info(struct adapter *adap)
4116{
4117 struct uld_msix_info *msix_info;
4118 unsigned int max_ingq = 0;
4119
4120 if (is_offload(adap))
4121 max_ingq += MAX_OFLD_QSETS * adap->num_ofld_uld;
4122 if (is_pci_uld(adap))
4123 max_ingq += MAX_OFLD_QSETS * adap->num_uld;
4124
4125 if (!max_ingq)
4126 goto out;
4127
4128 msix_info = kcalloc(max_ingq, sizeof(*msix_info), GFP_KERNEL);
4129 if (!msix_info)
4130 return -ENOMEM;
4131
4132 adap->msix_bmap_ulds.msix_bmap = kcalloc(BITS_TO_LONGS(max_ingq),
4133 sizeof(long), GFP_KERNEL);
4134 if (!adap->msix_bmap_ulds.msix_bmap) {
4135 kfree(msix_info);
4136 return -ENOMEM;
4137 }
4138 spin_lock_init(&adap->msix_bmap_ulds.lock);
4139 adap->msix_info_ulds = msix_info;
4140out:
4141 return 0;
4142}
4143
4144static void free_msix_info(struct adapter *adap)
4145{
4146 if (!(adap->num_uld && adap->num_ofld_uld))
4147 return;
4148
4149 kfree(adap->msix_info_ulds);
4150 kfree(adap->msix_bmap_ulds.msix_bmap);
4151}
4152
4153
4154#define EXTRA_VECS 2
4155
4156static int enable_msix(struct adapter *adap)
4157{
4158 int ofld_need = 0, uld_need = 0;
4159 int i, j, want, need, allocated;
4160 struct sge *s = &adap->sge;
4161 unsigned int nchan = adap->params.nports;
4162 struct msix_entry *entries;
4163 int max_ingq = MAX_INGQ;
4164
4165 if (is_pci_uld(adap))
4166 max_ingq += (MAX_OFLD_QSETS * adap->num_uld);
4167 if (is_offload(adap))
4168 max_ingq += (MAX_OFLD_QSETS * adap->num_ofld_uld);
4169 entries = kmalloc(sizeof(*entries) * (max_ingq + 1),
4170 GFP_KERNEL);
4171 if (!entries)
4172 return -ENOMEM;
4173
4174
4175 if (get_msix_info(adap)) {
4176 adap->params.offload = 0;
4177 adap->params.crypto = 0;
4178 }
4179
4180 for (i = 0; i < max_ingq + 1; ++i)
4181 entries[i].entry = i;
4182
4183 want = s->max_ethqsets + EXTRA_VECS;
4184 if (is_offload(adap)) {
4185 want += adap->num_ofld_uld * s->ofldqsets;
4186 ofld_need = adap->num_ofld_uld * nchan;
4187 }
4188 if (is_pci_uld(adap)) {
4189 want += adap->num_uld * s->ofldqsets;
4190 uld_need = adap->num_uld * nchan;
4191 }
4192#ifdef CONFIG_CHELSIO_T4_DCB
4193
4194
4195
4196 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need + uld_need;
4197#else
4198 need = adap->params.nports + EXTRA_VECS + ofld_need + uld_need;
4199#endif
4200 allocated = pci_enable_msix_range(adap->pdev, entries, need, want);
4201 if (allocated < 0) {
4202 dev_info(adap->pdev_dev, "not enough MSI-X vectors left,"
4203 " not using MSI-X\n");
4204 kfree(entries);
4205 return allocated;
4206 }
4207
4208
4209
4210
4211
4212 i = allocated - EXTRA_VECS - ofld_need - uld_need;
4213 if (i < s->max_ethqsets) {
4214 s->max_ethqsets = i;
4215 if (i < s->ethqsets)
4216 reduce_ethqs(adap, i);
4217 }
4218 if (is_uld(adap)) {
4219 if (allocated < want)
4220 s->nqs_per_uld = nchan;
4221 else
4222 s->nqs_per_uld = s->ofldqsets;
4223 }
4224
4225 for (i = 0; i < (s->max_ethqsets + EXTRA_VECS); ++i)
4226 adap->msix_info[i].vec = entries[i].vector;
4227 if (is_uld(adap)) {
4228 for (j = 0 ; i < allocated; ++i, j++) {
4229 adap->msix_info_ulds[j].vec = entries[i].vector;
4230 adap->msix_info_ulds[j].idx = i;
4231 }
4232 adap->msix_bmap_ulds.mapsize = j;
4233 }
4234 dev_info(adap->pdev_dev, "%d MSI-X vectors allocated, "
4235 "nic %d per uld %d\n",
4236 allocated, s->max_ethqsets, s->nqs_per_uld);
4237
4238 kfree(entries);
4239 return 0;
4240}
4241
4242#undef EXTRA_VECS
4243
4244static int init_rss(struct adapter *adap)
4245{
4246 unsigned int i;
4247 int err;
4248
4249 err = t4_init_rss_mode(adap, adap->mbox);
4250 if (err)
4251 return err;
4252
4253 for_each_port(adap, i) {
4254 struct port_info *pi = adap2pinfo(adap, i);
4255
4256 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
4257 if (!pi->rss)
4258 return -ENOMEM;
4259 }
4260 return 0;
4261}
4262
4263static int cxgb4_get_pcie_dev_link_caps(struct adapter *adap,
4264 enum pci_bus_speed *speed,
4265 enum pcie_link_width *width)
4266{
4267 u32 lnkcap1, lnkcap2;
4268 int err1, err2;
4269
4270#define PCIE_MLW_CAP_SHIFT 4
4271
4272 *speed = PCI_SPEED_UNKNOWN;
4273 *width = PCIE_LNK_WIDTH_UNKNOWN;
4274
4275 err1 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP,
4276 &lnkcap1);
4277 err2 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP2,
4278 &lnkcap2);
4279 if (!err2 && lnkcap2) {
4280 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB)
4281 *speed = PCIE_SPEED_8_0GT;
4282 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB)
4283 *speed = PCIE_SPEED_5_0GT;
4284 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB)
4285 *speed = PCIE_SPEED_2_5GT;
4286 }
4287 if (!err1) {
4288 *width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT;
4289 if (!lnkcap2) {
4290 if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB)
4291 *speed = PCIE_SPEED_5_0GT;
4292 else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB)
4293 *speed = PCIE_SPEED_2_5GT;
4294 }
4295 }
4296
4297 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
4298 return err1 ? err1 : err2 ? err2 : -EINVAL;
4299 return 0;
4300}
4301
4302static void cxgb4_check_pcie_caps(struct adapter *adap)
4303{
4304 enum pcie_link_width width, width_cap;
4305 enum pci_bus_speed speed, speed_cap;
4306
4307#define PCIE_SPEED_STR(speed) \
4308 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \
4309 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \
4310 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \
4311 "Unknown")
4312
4313 if (cxgb4_get_pcie_dev_link_caps(adap, &speed_cap, &width_cap)) {
4314 dev_warn(adap->pdev_dev,
4315 "Unable to determine PCIe device BW capabilities\n");
4316 return;
4317 }
4318
4319 if (pcie_get_minimum_link(adap->pdev, &speed, &width) ||
4320 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) {
4321 dev_warn(adap->pdev_dev,
4322 "Unable to determine PCI Express bandwidth.\n");
4323 return;
4324 }
4325
4326 dev_info(adap->pdev_dev, "PCIe link speed is %s, device supports %s\n",
4327 PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap));
4328 dev_info(adap->pdev_dev, "PCIe link width is x%d, device supports x%d\n",
4329 width, width_cap);
4330 if (speed < speed_cap || width < width_cap)
4331 dev_info(adap->pdev_dev,
4332 "A slot with more lanes and/or higher speed is "
4333 "suggested for optimal performance.\n");
4334}
4335
4336
4337static void print_adapter_info(struct adapter *adapter)
4338{
4339
4340 dev_info(adapter->pdev_dev, "Chelsio %s rev %d\n",
4341 adapter->params.vpd.id,
4342 CHELSIO_CHIP_RELEASE(adapter->params.chip));
4343 dev_info(adapter->pdev_dev, "S/N: %s, P/N: %s\n",
4344 adapter->params.vpd.sn, adapter->params.vpd.pn);
4345
4346
4347 if (!adapter->params.fw_vers)
4348 dev_warn(adapter->pdev_dev, "No firmware loaded\n");
4349 else
4350 dev_info(adapter->pdev_dev, "Firmware version: %u.%u.%u.%u\n",
4351 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
4352 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
4353 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
4354 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers));
4355
4356
4357
4358
4359 if (!adapter->params.bs_vers)
4360 dev_info(adapter->pdev_dev, "No bootstrap loaded\n");
4361 else
4362 dev_info(adapter->pdev_dev, "Bootstrap version: %u.%u.%u.%u\n",
4363 FW_HDR_FW_VER_MAJOR_G(adapter->params.bs_vers),
4364 FW_HDR_FW_VER_MINOR_G(adapter->params.bs_vers),
4365 FW_HDR_FW_VER_MICRO_G(adapter->params.bs_vers),
4366 FW_HDR_FW_VER_BUILD_G(adapter->params.bs_vers));
4367
4368
4369 if (!adapter->params.tp_vers)
4370 dev_warn(adapter->pdev_dev, "No TP Microcode loaded\n");
4371 else
4372 dev_info(adapter->pdev_dev,
4373 "TP Microcode version: %u.%u.%u.%u\n",
4374 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
4375 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
4376 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
4377 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
4378
4379
4380 if (!adapter->params.er_vers)
4381 dev_info(adapter->pdev_dev, "No Expansion ROM loaded\n");
4382 else
4383 dev_info(adapter->pdev_dev,
4384 "Expansion ROM version: %u.%u.%u.%u\n",
4385 FW_HDR_FW_VER_MAJOR_G(adapter->params.er_vers),
4386 FW_HDR_FW_VER_MINOR_G(adapter->params.er_vers),
4387 FW_HDR_FW_VER_MICRO_G(adapter->params.er_vers),
4388 FW_HDR_FW_VER_BUILD_G(adapter->params.er_vers));
4389
4390
4391 dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n",
4392 is_offload(adapter) ? "R" : "",
4393 ((adapter->flags & USING_MSIX) ? "MSI-X" :
4394 (adapter->flags & USING_MSI) ? "MSI" : ""),
4395 is_offload(adapter) ? "Offload" : "non-Offload");
4396}
4397
4398static void print_port_info(const struct net_device *dev)
4399{
4400 char buf[80];
4401 char *bufp = buf;
4402 const char *spd = "";
4403 const struct port_info *pi = netdev_priv(dev);
4404 const struct adapter *adap = pi->adapter;
4405
4406 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
4407 spd = " 2.5 GT/s";
4408 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
4409 spd = " 5 GT/s";
4410 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
4411 spd = " 8 GT/s";
4412
4413 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
4414 bufp += sprintf(bufp, "100/");
4415 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
4416 bufp += sprintf(bufp, "1000/");
4417 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
4418 bufp += sprintf(bufp, "10G/");
4419 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_25G)
4420 bufp += sprintf(bufp, "25G/");
4421 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
4422 bufp += sprintf(bufp, "40G/");
4423 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100G)
4424 bufp += sprintf(bufp, "100G/");
4425 if (bufp != buf)
4426 --bufp;
4427 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
4428
4429 netdev_info(dev, "%s: Chelsio %s (%s) %s\n",
4430 dev->name, adap->params.vpd.id, adap->name, buf);
4431}
4432
4433static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
4434{
4435 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
4436}
4437
4438
4439
4440
4441
4442
4443
4444
4445static void free_some_resources(struct adapter *adapter)
4446{
4447 unsigned int i;
4448
4449 t4_free_mem(adapter->l2t);
4450 t4_cleanup_sched(adapter);
4451 t4_free_mem(adapter->tids.tid_tab);
4452 cxgb4_cleanup_tc_u32(adapter);
4453 kfree(adapter->sge.egr_map);
4454 kfree(adapter->sge.ingr_map);
4455 kfree(adapter->sge.starving_fl);
4456 kfree(adapter->sge.txq_maperr);
4457#ifdef CONFIG_DEBUG_FS
4458 kfree(adapter->sge.blocked_fl);
4459#endif
4460 disable_msi(adapter);
4461
4462 for_each_port(adapter, i)
4463 if (adapter->port[i]) {
4464 struct port_info *pi = adap2pinfo(adapter, i);
4465
4466 if (pi->viid != 0)
4467 t4_free_vi(adapter, adapter->mbox, adapter->pf,
4468 0, pi->viid);
4469 kfree(adap2pinfo(adapter, i)->rss);
4470 free_netdev(adapter->port[i]);
4471 }
4472 if (adapter->flags & FW_OK)
4473 t4_fw_bye(adapter, adapter->pf);
4474}
4475
4476#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
4477#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
4478 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
4479#define SEGMENT_SIZE 128
4480
4481static int get_chip_type(struct pci_dev *pdev, u32 pl_rev)
4482{
4483 u16 device_id;
4484
4485
4486 pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id);
4487
4488 switch (device_id >> 12) {
4489 case CHELSIO_T4:
4490 return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev);
4491 case CHELSIO_T5:
4492 return CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev);
4493 case CHELSIO_T6:
4494 return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev);
4495 default:
4496 dev_err(&pdev->dev, "Device %d is not supported\n",
4497 device_id);
4498 }
4499 return -EINVAL;
4500}
4501
4502#ifdef CONFIG_PCI_IOV
4503static void dummy_setup(struct net_device *dev)
4504{
4505 dev->type = ARPHRD_NONE;
4506 dev->mtu = 0;
4507 dev->hard_header_len = 0;
4508 dev->addr_len = 0;
4509 dev->tx_queue_len = 0;
4510 dev->flags |= IFF_NOARP;
4511 dev->priv_flags |= IFF_NO_QUEUE;
4512
4513
4514 dev->netdev_ops = &cxgb4_mgmt_netdev_ops;
4515 dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops;
4516 dev->destructor = free_netdev;
4517}
4518
4519static int config_mgmt_dev(struct pci_dev *pdev)
4520{
4521 struct adapter *adap = pci_get_drvdata(pdev);
4522 struct net_device *netdev;
4523 struct port_info *pi;
4524 char name[IFNAMSIZ];
4525 int err;
4526
4527 snprintf(name, IFNAMSIZ, "mgmtpf%d%d", adap->adap_idx, adap->pf);
4528 netdev = alloc_netdev(0, name, NET_NAME_UNKNOWN, dummy_setup);
4529 if (!netdev)
4530 return -ENOMEM;
4531
4532 pi = netdev_priv(netdev);
4533 pi->adapter = adap;
4534 SET_NETDEV_DEV(netdev, &pdev->dev);
4535
4536 adap->port[0] = netdev;
4537
4538 err = register_netdev(adap->port[0]);
4539 if (err) {
4540 pr_info("Unable to register VF mgmt netdev %s\n", name);
4541 free_netdev(adap->port[0]);
4542 adap->port[0] = NULL;
4543 return err;
4544 }
4545 return 0;
4546}
4547
4548static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
4549{
4550 struct adapter *adap = pci_get_drvdata(pdev);
4551 int err = 0;
4552 int current_vfs = pci_num_vf(pdev);
4553 u32 pcie_fw;
4554
4555 pcie_fw = readl(adap->regs + PCIE_FW_A);
4556
4557 if (!(pcie_fw & PCIE_FW_INIT_F) ||
4558 !(pcie_fw & PCIE_FW_MASTER_VLD_F) ||
4559 PCIE_FW_MASTER_G(pcie_fw) != 4) {
4560 dev_warn(&pdev->dev,
4561 "cxgb4 driver needs to be MASTER to support SRIOV\n");
4562 return -EOPNOTSUPP;
4563 }
4564
4565
4566
4567
4568 if (current_vfs && pci_vfs_assigned(pdev)) {
4569 dev_err(&pdev->dev,
4570 "Cannot modify SR-IOV while VFs are assigned\n");
4571 num_vfs = current_vfs;
4572 return num_vfs;
4573 }
4574
4575
4576
4577
4578
4579
4580 if (!num_vfs) {
4581 pci_disable_sriov(pdev);
4582 if (adap->port[0]) {
4583 unregister_netdev(adap->port[0]);
4584 adap->port[0] = NULL;
4585 }
4586
4587 kfree(adap->vfinfo);
4588 adap->vfinfo = NULL;
4589 adap->num_vfs = 0;
4590 return num_vfs;
4591 }
4592
4593 if (num_vfs != current_vfs) {
4594 err = pci_enable_sriov(pdev, num_vfs);
4595 if (err)
4596 return err;
4597
4598 adap->num_vfs = num_vfs;
4599 err = config_mgmt_dev(pdev);
4600 if (err)
4601 return err;
4602 }
4603
4604 adap->vfinfo = kcalloc(adap->num_vfs,
4605 sizeof(struct vf_info), GFP_KERNEL);
4606 if (adap->vfinfo)
4607 fill_vf_station_mac_addr(adap);
4608 return num_vfs;
4609}
4610#endif
4611
4612static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
4613{
4614 int func, i, err, s_qpp, qpp, num_seg;
4615 struct port_info *pi;
4616 bool highdma = false;
4617 struct adapter *adapter = NULL;
4618 struct net_device *netdev;
4619 void __iomem *regs;
4620 u32 whoami, pl_rev;
4621 enum chip_type chip;
4622 static int adap_idx = 1;
4623
4624 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
4625
4626 err = pci_request_regions(pdev, KBUILD_MODNAME);
4627 if (err) {
4628
4629 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
4630 return err;
4631 }
4632
4633 err = pci_enable_device(pdev);
4634 if (err) {
4635 dev_err(&pdev->dev, "cannot enable PCI device\n");
4636 goto out_release_regions;
4637 }
4638
4639 regs = pci_ioremap_bar(pdev, 0);
4640 if (!regs) {
4641 dev_err(&pdev->dev, "cannot map device registers\n");
4642 err = -ENOMEM;
4643 goto out_disable_device;
4644 }
4645
4646 err = t4_wait_dev_ready(regs);
4647 if (err < 0)
4648 goto out_unmap_bar0;
4649
4650
4651 whoami = readl(regs + PL_WHOAMI_A);
4652 pl_rev = REV_G(readl(regs + PL_REV_A));
4653 chip = get_chip_type(pdev, pl_rev);
4654 func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ?
4655 SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
4656 if (func != ent->driver_data) {
4657#ifndef CONFIG_PCI_IOV
4658 iounmap(regs);
4659#endif
4660 pci_disable_device(pdev);
4661 pci_save_state(pdev);
4662 goto sriov;
4663 }
4664
4665 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
4666 highdma = true;
4667 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
4668 if (err) {
4669 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
4670 "coherent allocations\n");
4671 goto out_unmap_bar0;
4672 }
4673 } else {
4674 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
4675 if (err) {
4676 dev_err(&pdev->dev, "no usable DMA configuration\n");
4677 goto out_unmap_bar0;
4678 }
4679 }
4680
4681 pci_enable_pcie_error_reporting(pdev);
4682 enable_pcie_relaxed_ordering(pdev);
4683 pci_set_master(pdev);
4684 pci_save_state(pdev);
4685
4686 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
4687 if (!adapter) {
4688 err = -ENOMEM;
4689 goto out_unmap_bar0;
4690 }
4691 adap_idx++;
4692
4693 adapter->workq = create_singlethread_workqueue("cxgb4");
4694 if (!adapter->workq) {
4695 err = -ENOMEM;
4696 goto out_free_adapter;
4697 }
4698
4699 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
4700 (sizeof(struct mbox_cmd) *
4701 T4_OS_LOG_MBOX_CMDS),
4702 GFP_KERNEL);
4703 if (!adapter->mbox_log) {
4704 err = -ENOMEM;
4705 goto out_free_adapter;
4706 }
4707 adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS;
4708
4709
4710 adapter->flags |= DEV_ENABLED;
4711
4712 adapter->regs = regs;
4713 adapter->pdev = pdev;
4714 adapter->pdev_dev = &pdev->dev;
4715 adapter->name = pci_name(pdev);
4716 adapter->mbox = func;
4717 adapter->pf = func;
4718 adapter->msg_enable = dflt_msg_enable;
4719 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
4720
4721 spin_lock_init(&adapter->stats_lock);
4722 spin_lock_init(&adapter->tid_release_lock);
4723 spin_lock_init(&adapter->win0_lock);
4724
4725 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
4726 INIT_WORK(&adapter->db_full_task, process_db_full);
4727 INIT_WORK(&adapter->db_drop_task, process_db_drop);
4728
4729 err = t4_prep_adapter(adapter);
4730 if (err)
4731 goto out_free_adapter;
4732
4733
4734 if (!is_t4(adapter->params.chip)) {
4735 s_qpp = (QUEUESPERPAGEPF0_S +
4736 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
4737 adapter->pf);
4738 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
4739 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
4740 num_seg = PAGE_SIZE / SEGMENT_SIZE;
4741
4742
4743
4744
4745
4746
4747 if (qpp > num_seg) {
4748 dev_err(&pdev->dev,
4749 "Incorrect number of egress queues per page\n");
4750 err = -EINVAL;
4751 goto out_free_adapter;
4752 }
4753 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
4754 pci_resource_len(pdev, 2));
4755 if (!adapter->bar2) {
4756 dev_err(&pdev->dev, "cannot map device bar2 region\n");
4757 err = -ENOMEM;
4758 goto out_free_adapter;
4759 }
4760 }
4761
4762 setup_memwin(adapter);
4763 err = adap_init0(adapter);
4764#ifdef CONFIG_DEBUG_FS
4765 bitmap_zero(adapter->sge.blocked_fl, adapter->sge.egr_sz);
4766#endif
4767 setup_memwin_rdma(adapter);
4768 if (err)
4769 goto out_unmap_bar;
4770
4771
4772 if (!is_t4(adapter->params.chip))
4773 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) |
4774 (is_t5(adapter->params.chip) ? STATMODE_V(0) :
4775 T6_STATMODE_V(0)));
4776
4777 for_each_port(adapter, i) {
4778 netdev = alloc_etherdev_mq(sizeof(struct port_info),
4779 MAX_ETH_QSETS);
4780 if (!netdev) {
4781 err = -ENOMEM;
4782 goto out_free_dev;
4783 }
4784
4785 SET_NETDEV_DEV(netdev, &pdev->dev);
4786
4787 adapter->port[i] = netdev;
4788 pi = netdev_priv(netdev);
4789 pi->adapter = adapter;
4790 pi->xact_addr_filt = -1;
4791 pi->port_id = i;
4792 netdev->irq = pdev->irq;
4793
4794 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
4795 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4796 NETIF_F_RXCSUM | NETIF_F_RXHASH |
4797 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
4798 NETIF_F_HW_TC;
4799 if (highdma)
4800 netdev->hw_features |= NETIF_F_HIGHDMA;
4801 netdev->features |= netdev->hw_features;
4802 netdev->vlan_features = netdev->features & VLAN_FEAT;
4803
4804 netdev->priv_flags |= IFF_UNICAST_FLT;
4805
4806 netdev->netdev_ops = &cxgb4_netdev_ops;
4807#ifdef CONFIG_CHELSIO_T4_DCB
4808 netdev->dcbnl_ops = &cxgb4_dcb_ops;
4809 cxgb4_dcb_state_init(netdev);
4810#endif
4811 cxgb4_set_ethtool_ops(netdev);
4812 }
4813
4814 pci_set_drvdata(pdev, adapter);
4815
4816 if (adapter->flags & FW_OK) {
4817 err = t4_port_init(adapter, func, func, 0);
4818 if (err)
4819 goto out_free_dev;
4820 } else if (adapter->params.nports == 1) {
4821
4822
4823
4824
4825
4826 u8 hw_addr[ETH_ALEN];
4827 u8 *na = adapter->params.vpd.na;
4828
4829 err = t4_get_raw_vpd_params(adapter, &adapter->params.vpd);
4830 if (!err) {
4831 for (i = 0; i < ETH_ALEN; i++)
4832 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 +
4833 hex2val(na[2 * i + 1]));
4834 t4_set_hw_addr(adapter, 0, hw_addr);
4835 }
4836 }
4837
4838
4839
4840
4841 cfg_queues(adapter);
4842
4843 adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end);
4844 if (!adapter->l2t) {
4845
4846 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
4847 adapter->params.offload = 0;
4848 }
4849
4850#if IS_ENABLED(CONFIG_IPV6)
4851 if ((CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) &&
4852 (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) {
4853
4854
4855
4856 dev_warn(&pdev->dev,
4857 "CLIP not enabled in hardware, continuing\n");
4858 adapter->params.offload = 0;
4859 } else {
4860 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
4861 adapter->clipt_end);
4862 if (!adapter->clipt) {
4863
4864
4865
4866 dev_warn(&pdev->dev,
4867 "could not allocate Clip table, continuing\n");
4868 adapter->params.offload = 0;
4869 }
4870 }
4871#endif
4872
4873 for_each_port(adapter, i) {
4874 pi = adap2pinfo(adapter, i);
4875 pi->sched_tbl = t4_init_sched(adapter->params.nsched_cls);
4876 if (!pi->sched_tbl)
4877 dev_warn(&pdev->dev,
4878 "could not activate scheduling on port %d\n",
4879 i);
4880 }
4881
4882 if (tid_init(&adapter->tids) < 0) {
4883 dev_warn(&pdev->dev, "could not allocate TID table, "
4884 "continuing\n");
4885 adapter->params.offload = 0;
4886 } else {
4887 adapter->tc_u32 = cxgb4_init_tc_u32(adapter,
4888 CXGB4_MAX_LINK_HANDLE);
4889 if (!adapter->tc_u32)
4890 dev_warn(&pdev->dev,
4891 "could not offload tc u32, continuing\n");
4892 }
4893
4894 if (is_offload(adapter)) {
4895 if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) {
4896 u32 hash_base, hash_reg;
4897
4898 if (chip <= CHELSIO_T5) {
4899 hash_reg = LE_DB_TID_HASHBASE_A;
4900 hash_base = t4_read_reg(adapter, hash_reg);
4901 adapter->tids.hash_base = hash_base / 4;
4902 } else {
4903 hash_reg = T6_LE_DB_HASH_TID_BASE_A;
4904 hash_base = t4_read_reg(adapter, hash_reg);
4905 adapter->tids.hash_base = hash_base;
4906 }
4907 }
4908 }
4909
4910
4911 if (msi > 1 && enable_msix(adapter) == 0)
4912 adapter->flags |= USING_MSIX;
4913 else if (msi > 0 && pci_enable_msi(pdev) == 0) {
4914 adapter->flags |= USING_MSI;
4915 if (msi > 1)
4916 free_msix_info(adapter);
4917 }
4918
4919
4920 cxgb4_check_pcie_caps(adapter);
4921
4922 err = init_rss(adapter);
4923 if (err)
4924 goto out_free_dev;
4925
4926
4927
4928
4929
4930
4931
4932 for_each_port(adapter, i) {
4933 pi = adap2pinfo(adapter, i);
4934 adapter->port[i]->dev_port = pi->lport;
4935 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
4936 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
4937
4938 err = register_netdev(adapter->port[i]);
4939 if (err)
4940 break;
4941 adapter->chan_map[pi->tx_chan] = i;
4942 print_port_info(adapter->port[i]);
4943 }
4944 if (i == 0) {
4945 dev_err(&pdev->dev, "could not register any net devices\n");
4946 goto out_free_dev;
4947 }
4948 if (err) {
4949 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
4950 err = 0;
4951 }
4952
4953 if (cxgb4_debugfs_root) {
4954 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
4955 cxgb4_debugfs_root);
4956 setup_debugfs(adapter);
4957 }
4958
4959
4960 pdev->needs_freset = 1;
4961
4962 if (is_uld(adapter)) {
4963 mutex_lock(&uld_mutex);
4964 list_add_tail(&adapter->list_node, &adapter_list);
4965 mutex_unlock(&uld_mutex);
4966 }
4967
4968 print_adapter_info(adapter);
4969 setup_fw_sge_queues(adapter);
4970 return 0;
4971
4972sriov:
4973#ifdef CONFIG_PCI_IOV
4974 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0) {
4975 dev_warn(&pdev->dev,
4976 "Enabling SR-IOV VFs using the num_vf module "
4977 "parameter is deprecated - please use the pci sysfs "
4978 "interface instead.\n");
4979 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
4980 dev_info(&pdev->dev,
4981 "instantiated %u virtual functions\n",
4982 num_vf[func]);
4983 }
4984
4985 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
4986 if (!adapter) {
4987 err = -ENOMEM;
4988 goto free_pci_region;
4989 }
4990
4991 adapter->pdev = pdev;
4992 adapter->pdev_dev = &pdev->dev;
4993 adapter->name = pci_name(pdev);
4994 adapter->mbox = func;
4995 adapter->pf = func;
4996 adapter->regs = regs;
4997 adapter->adap_idx = adap_idx;
4998 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
4999 (sizeof(struct mbox_cmd) *
5000 T4_OS_LOG_MBOX_CMDS),
5001 GFP_KERNEL);
5002 if (!adapter->mbox_log) {
5003 err = -ENOMEM;
5004 goto free_adapter;
5005 }
5006 pci_set_drvdata(pdev, adapter);
5007 return 0;
5008
5009 free_adapter:
5010 kfree(adapter);
5011 free_pci_region:
5012 iounmap(regs);
5013 pci_disable_sriov(pdev);
5014 pci_release_regions(pdev);
5015 return err;
5016#else
5017 return 0;
5018#endif
5019
5020 out_free_dev:
5021 free_some_resources(adapter);
5022 if (adapter->flags & USING_MSIX)
5023 free_msix_info(adapter);
5024 if (adapter->num_uld || adapter->num_ofld_uld)
5025 t4_uld_mem_free(adapter);
5026 out_unmap_bar:
5027 if (!is_t4(adapter->params.chip))
5028 iounmap(adapter->bar2);
5029 out_free_adapter:
5030 if (adapter->workq)
5031 destroy_workqueue(adapter->workq);
5032
5033 kfree(adapter->mbox_log);
5034 kfree(adapter);
5035 out_unmap_bar0:
5036 iounmap(regs);
5037 out_disable_device:
5038 pci_disable_pcie_error_reporting(pdev);
5039 pci_disable_device(pdev);
5040 out_release_regions:
5041 pci_release_regions(pdev);
5042 return err;
5043}
5044
5045static void remove_one(struct pci_dev *pdev)
5046{
5047 struct adapter *adapter = pci_get_drvdata(pdev);
5048
5049 if (!adapter) {
5050 pci_release_regions(pdev);
5051 return;
5052 }
5053
5054 if (adapter->pf == 4) {
5055 int i;
5056
5057
5058
5059
5060 destroy_workqueue(adapter->workq);
5061
5062 if (is_uld(adapter))
5063 detach_ulds(adapter);
5064
5065 disable_interrupts(adapter);
5066
5067 for_each_port(adapter, i)
5068 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
5069 unregister_netdev(adapter->port[i]);
5070
5071 debugfs_remove_recursive(adapter->debugfs_root);
5072
5073
5074
5075
5076 clear_all_filters(adapter);
5077
5078 if (adapter->flags & FULL_INIT_DONE)
5079 cxgb_down(adapter);
5080
5081 if (adapter->flags & USING_MSIX)
5082 free_msix_info(adapter);
5083 if (adapter->num_uld || adapter->num_ofld_uld)
5084 t4_uld_mem_free(adapter);
5085 free_some_resources(adapter);
5086#if IS_ENABLED(CONFIG_IPV6)
5087 t4_cleanup_clip_tbl(adapter);
5088#endif
5089 iounmap(adapter->regs);
5090 if (!is_t4(adapter->params.chip))
5091 iounmap(adapter->bar2);
5092 pci_disable_pcie_error_reporting(pdev);
5093 if ((adapter->flags & DEV_ENABLED)) {
5094 pci_disable_device(pdev);
5095 adapter->flags &= ~DEV_ENABLED;
5096 }
5097 pci_release_regions(pdev);
5098 kfree(adapter->mbox_log);
5099 synchronize_rcu();
5100 kfree(adapter);
5101 }
5102#ifdef CONFIG_PCI_IOV
5103 else {
5104 if (adapter->port[0])
5105 unregister_netdev(adapter->port[0]);
5106 iounmap(adapter->regs);
5107 kfree(adapter->vfinfo);
5108 kfree(adapter);
5109 pci_disable_sriov(pdev);
5110 pci_release_regions(pdev);
5111 }
5112#endif
5113}
5114
5115
5116
5117
5118
5119
5120static void shutdown_one(struct pci_dev *pdev)
5121{
5122 struct adapter *adapter = pci_get_drvdata(pdev);
5123
5124
5125
5126
5127
5128 if (!adapter) {
5129 pci_release_regions(pdev);
5130 return;
5131 }
5132
5133 if (adapter->pf == 4) {
5134 int i;
5135
5136 for_each_port(adapter, i)
5137 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
5138 cxgb_close(adapter->port[i]);
5139
5140 t4_uld_clean_up(adapter);
5141 disable_interrupts(adapter);
5142 disable_msi(adapter);
5143
5144 t4_sge_stop(adapter);
5145 if (adapter->flags & FW_OK)
5146 t4_fw_bye(adapter, adapter->mbox);
5147 }
5148#ifdef CONFIG_PCI_IOV
5149 else {
5150 if (adapter->port[0])
5151 unregister_netdev(adapter->port[0]);
5152 iounmap(adapter->regs);
5153 kfree(adapter->vfinfo);
5154 kfree(adapter);
5155 pci_disable_sriov(pdev);
5156 pci_release_regions(pdev);
5157 }
5158#endif
5159}
5160
5161static struct pci_driver cxgb4_driver = {
5162 .name = KBUILD_MODNAME,
5163 .id_table = cxgb4_pci_tbl,
5164 .probe = init_one,
5165 .remove = remove_one,
5166 .shutdown = shutdown_one,
5167#ifdef CONFIG_PCI_IOV
5168 .sriov_configure = cxgb4_iov_configure,
5169#endif
5170 .err_handler = &cxgb4_eeh,
5171};
5172
5173static int __init cxgb4_init_module(void)
5174{
5175 int ret;
5176
5177
5178 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
5179 if (!cxgb4_debugfs_root)
5180 pr_warn("could not create debugfs entry, continuing\n");
5181
5182 ret = pci_register_driver(&cxgb4_driver);
5183 if (ret < 0)
5184 debugfs_remove(cxgb4_debugfs_root);
5185
5186#if IS_ENABLED(CONFIG_IPV6)
5187 if (!inet6addr_registered) {
5188 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
5189 inet6addr_registered = true;
5190 }
5191#endif
5192
5193 return ret;
5194}
5195
5196static void __exit cxgb4_cleanup_module(void)
5197{
5198#if IS_ENABLED(CONFIG_IPV6)
5199 if (inet6addr_registered) {
5200 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
5201 inet6addr_registered = false;
5202 }
5203#endif
5204 pci_unregister_driver(&cxgb4_driver);
5205 debugfs_remove(cxgb4_debugfs_root);
5206}
5207
5208module_init(cxgb4_init_module);
5209module_exit(cxgb4_cleanup_module);
5210