1
2
3
4
5#include <sys/queue.h>
6#include <stdio.h>
7#include <errno.h>
8#include <stdint.h>
9#include <string.h>
10#include <unistd.h>
11#include <stdarg.h>
12#include <fcntl.h>
13#include <inttypes.h>
14#include <rte_byteorder.h>
15#include <rte_common.h>
16#include <rte_cycles.h>
17
18#include <rte_interrupts.h>
19#include <rte_log.h>
20#include <rte_debug.h>
21#include <rte_pci.h>
22#include <rte_bus_pci.h>
23#include <rte_branch_prediction.h>
24#include <rte_memory.h>
25#include <rte_memzone.h>
26#include <rte_eal.h>
27#include <rte_alarm.h>
28#include <rte_ether.h>
29#include <ethdev_driver.h>
30#include <ethdev_pci.h>
31#include <rte_string_fns.h>
32#include <rte_malloc.h>
33#include <rte_dev.h>
34
35#include "base/vmxnet3_defs.h"
36
37#include "vmxnet3_ring.h"
38#include "vmxnet3_logs.h"
39#include "vmxnet3_ethdev.h"
40
41#define VMXNET3_TX_MAX_SEG UINT8_MAX
42
43#define VMXNET3_TX_OFFLOAD_CAP \
44 (DEV_TX_OFFLOAD_VLAN_INSERT | \
45 DEV_TX_OFFLOAD_TCP_CKSUM | \
46 DEV_TX_OFFLOAD_UDP_CKSUM | \
47 DEV_TX_OFFLOAD_TCP_TSO | \
48 DEV_TX_OFFLOAD_MULTI_SEGS)
49
50#define VMXNET3_RX_OFFLOAD_CAP \
51 (DEV_RX_OFFLOAD_VLAN_STRIP | \
52 DEV_RX_OFFLOAD_VLAN_FILTER | \
53 DEV_RX_OFFLOAD_SCATTER | \
54 DEV_RX_OFFLOAD_UDP_CKSUM | \
55 DEV_RX_OFFLOAD_TCP_CKSUM | \
56 DEV_RX_OFFLOAD_TCP_LRO | \
57 DEV_RX_OFFLOAD_JUMBO_FRAME | \
58 DEV_RX_OFFLOAD_RSS_HASH)
59
60int vmxnet3_segs_dynfield_offset = -1;
61
62static int eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev);
63static int eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev);
64static int vmxnet3_dev_configure(struct rte_eth_dev *dev);
65static int vmxnet3_dev_start(struct rte_eth_dev *dev);
66static int vmxnet3_dev_stop(struct rte_eth_dev *dev);
67static int vmxnet3_dev_close(struct rte_eth_dev *dev);
68static int vmxnet3_dev_reset(struct rte_eth_dev *dev);
69static void vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set);
70static int vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev);
71static int vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev);
72static int vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev);
73static int vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev);
74static int __vmxnet3_dev_link_update(struct rte_eth_dev *dev,
75 int wait_to_complete);
76static int vmxnet3_dev_link_update(struct rte_eth_dev *dev,
77 int wait_to_complete);
78static void vmxnet3_hw_stats_save(struct vmxnet3_hw *hw);
79static int vmxnet3_dev_stats_get(struct rte_eth_dev *dev,
80 struct rte_eth_stats *stats);
81static int vmxnet3_dev_stats_reset(struct rte_eth_dev *dev);
82static int vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev,
83 struct rte_eth_xstat_name *xstats,
84 unsigned int n);
85static int vmxnet3_dev_xstats_get(struct rte_eth_dev *dev,
86 struct rte_eth_xstat *xstats, unsigned int n);
87static int vmxnet3_dev_info_get(struct rte_eth_dev *dev,
88 struct rte_eth_dev_info *dev_info);
89static const uint32_t *
90vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev);
91static int vmxnet3_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
92static int vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev,
93 uint16_t vid, int on);
94static int vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask);
95static int vmxnet3_mac_addr_set(struct rte_eth_dev *dev,
96 struct rte_ether_addr *mac_addr);
97static void vmxnet3_process_events(struct rte_eth_dev *dev);
98static void vmxnet3_interrupt_handler(void *param);
99static int vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
100 uint16_t queue_id);
101static int vmxnet3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
102 uint16_t queue_id);
103
104
105
106
107#define VMWARE_PCI_VENDOR_ID 0x15AD
108#define VMWARE_DEV_ID_VMXNET3 0x07B0
109static const struct rte_pci_id pci_id_vmxnet3_map[] = {
110 { RTE_PCI_DEVICE(VMWARE_PCI_VENDOR_ID, VMWARE_DEV_ID_VMXNET3) },
111 { .vendor_id = 0, },
112};
113
114static const struct eth_dev_ops vmxnet3_eth_dev_ops = {
115 .dev_configure = vmxnet3_dev_configure,
116 .dev_start = vmxnet3_dev_start,
117 .dev_stop = vmxnet3_dev_stop,
118 .dev_close = vmxnet3_dev_close,
119 .dev_reset = vmxnet3_dev_reset,
120 .promiscuous_enable = vmxnet3_dev_promiscuous_enable,
121 .promiscuous_disable = vmxnet3_dev_promiscuous_disable,
122 .allmulticast_enable = vmxnet3_dev_allmulticast_enable,
123 .allmulticast_disable = vmxnet3_dev_allmulticast_disable,
124 .link_update = vmxnet3_dev_link_update,
125 .stats_get = vmxnet3_dev_stats_get,
126 .xstats_get_names = vmxnet3_dev_xstats_get_names,
127 .xstats_get = vmxnet3_dev_xstats_get,
128 .stats_reset = vmxnet3_dev_stats_reset,
129 .mac_addr_set = vmxnet3_mac_addr_set,
130 .dev_infos_get = vmxnet3_dev_info_get,
131 .dev_supported_ptypes_get = vmxnet3_dev_supported_ptypes_get,
132 .mtu_set = vmxnet3_dev_mtu_set,
133 .vlan_filter_set = vmxnet3_dev_vlan_filter_set,
134 .vlan_offload_set = vmxnet3_dev_vlan_offload_set,
135 .rx_queue_setup = vmxnet3_dev_rx_queue_setup,
136 .rx_queue_release = vmxnet3_dev_rx_queue_release,
137 .tx_queue_setup = vmxnet3_dev_tx_queue_setup,
138 .tx_queue_release = vmxnet3_dev_tx_queue_release,
139 .rx_queue_intr_enable = vmxnet3_dev_rx_queue_intr_enable,
140 .rx_queue_intr_disable = vmxnet3_dev_rx_queue_intr_disable,
141};
142
143struct vmxnet3_xstats_name_off {
144 char name[RTE_ETH_XSTATS_NAME_SIZE];
145 unsigned int offset;
146};
147
148
149static const struct vmxnet3_xstats_name_off vmxnet3_txq_stat_strings[] = {
150 {"drop_total", offsetof(struct vmxnet3_txq_stats, drop_total)},
151 {"drop_too_many_segs", offsetof(struct vmxnet3_txq_stats, drop_too_many_segs)},
152 {"drop_tso", offsetof(struct vmxnet3_txq_stats, drop_tso)},
153 {"tx_ring_full", offsetof(struct vmxnet3_txq_stats, tx_ring_full)},
154};
155
156
157static const struct vmxnet3_xstats_name_off vmxnet3_rxq_stat_strings[] = {
158 {"drop_total", offsetof(struct vmxnet3_rxq_stats, drop_total)},
159 {"drop_err", offsetof(struct vmxnet3_rxq_stats, drop_err)},
160 {"drop_fcs", offsetof(struct vmxnet3_rxq_stats, drop_fcs)},
161 {"rx_buf_alloc_failure", offsetof(struct vmxnet3_rxq_stats, rx_buf_alloc_failure)},
162};
163
164static const struct rte_memzone *
165gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size,
166 const char *post_string, int socket_id,
167 uint16_t align, bool reuse)
168{
169 char z_name[RTE_MEMZONE_NAMESIZE];
170 const struct rte_memzone *mz;
171
172 snprintf(z_name, sizeof(z_name), "eth_p%d_%s",
173 dev->data->port_id, post_string);
174
175 mz = rte_memzone_lookup(z_name);
176 if (!reuse) {
177 if (mz)
178 rte_memzone_free(mz);
179 return rte_memzone_reserve_aligned(z_name, size, socket_id,
180 RTE_MEMZONE_IOVA_CONTIG, align);
181 }
182
183 if (mz)
184 return mz;
185
186 return rte_memzone_reserve_aligned(z_name, size, socket_id,
187 RTE_MEMZONE_IOVA_CONTIG, align);
188}
189
190
191
192
193static void
194vmxnet3_enable_intr(struct vmxnet3_hw *hw, unsigned int intr_idx)
195{
196 PMD_INIT_FUNC_TRACE();
197 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + intr_idx * 8, 0);
198}
199
200
201
202
203static void
204vmxnet3_disable_intr(struct vmxnet3_hw *hw, unsigned int intr_idx)
205{
206 PMD_INIT_FUNC_TRACE();
207 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + intr_idx * 8, 1);
208}
209
210
211
212
213static void
214vmxnet3_enable_all_intrs(struct vmxnet3_hw *hw)
215{
216 Vmxnet3_DSDevRead *devRead = &hw->shared->devRead;
217
218 PMD_INIT_FUNC_TRACE();
219
220 devRead->intrConf.intrCtrl &= rte_cpu_to_le_32(~VMXNET3_IC_DISABLE_ALL);
221
222 if (hw->intr.lsc_only) {
223 vmxnet3_enable_intr(hw, devRead->intrConf.eventIntrIdx);
224 } else {
225 int i;
226
227 for (i = 0; i < hw->intr.num_intrs; i++)
228 vmxnet3_enable_intr(hw, i);
229 }
230}
231
232
233
234
235static void
236vmxnet3_disable_all_intrs(struct vmxnet3_hw *hw)
237{
238 int i;
239
240 PMD_INIT_FUNC_TRACE();
241
242 hw->shared->devRead.intrConf.intrCtrl |=
243 rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL);
244 for (i = 0; i < hw->num_intrs; i++)
245 vmxnet3_disable_intr(hw, i);
246}
247
248
249
250
251static uint16_t
252eth_vmxnet3_txdata_get(struct vmxnet3_hw *hw)
253{
254 uint16 txdata_desc_size;
255
256 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
257 VMXNET3_CMD_GET_TXDATA_DESC_SIZE);
258 txdata_desc_size = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
259
260 return (txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE ||
261 txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE ||
262 txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK) ?
263 sizeof(struct Vmxnet3_TxDataDesc) : txdata_desc_size;
264}
265
266
267
268
269static int
270eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev)
271{
272 struct rte_pci_device *pci_dev;
273 struct vmxnet3_hw *hw = eth_dev->data->dev_private;
274 uint32_t mac_hi, mac_lo, ver;
275 struct rte_eth_link link;
276 static const struct rte_mbuf_dynfield vmxnet3_segs_dynfield_desc = {
277 .name = VMXNET3_SEGS_DYNFIELD_NAME,
278 .size = sizeof(vmxnet3_segs_dynfield_t),
279 .align = __alignof__(vmxnet3_segs_dynfield_t),
280 };
281
282 PMD_INIT_FUNC_TRACE();
283
284 eth_dev->dev_ops = &vmxnet3_eth_dev_ops;
285 eth_dev->rx_pkt_burst = &vmxnet3_recv_pkts;
286 eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts;
287 eth_dev->tx_pkt_prepare = vmxnet3_prep_pkts;
288 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
289
290
291 vmxnet3_segs_dynfield_offset =
292 rte_mbuf_dynfield_register(&vmxnet3_segs_dynfield_desc);
293 if (vmxnet3_segs_dynfield_offset < 0) {
294 PMD_INIT_LOG(ERR, "Cannot register mbuf field.");
295 return -rte_errno;
296 }
297
298
299
300
301
302 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
303 return 0;
304
305 rte_eth_copy_pci_info(eth_dev, pci_dev);
306 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
307
308
309 hw->device_id = pci_dev->id.device_id;
310 hw->vendor_id = pci_dev->id.vendor_id;
311 hw->hw_addr0 = (void *)pci_dev->mem_resource[0].addr;
312 hw->hw_addr1 = (void *)pci_dev->mem_resource[1].addr;
313
314 hw->num_rx_queues = 1;
315 hw->num_tx_queues = 1;
316 hw->bufs_per_pkt = 1;
317
318
319 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_VRRS);
320 PMD_INIT_LOG(DEBUG, "Hardware version : %d", ver);
321
322 if (ver & (1 << VMXNET3_REV_4)) {
323 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS,
324 1 << VMXNET3_REV_4);
325 hw->version = VMXNET3_REV_4 + 1;
326 } else if (ver & (1 << VMXNET3_REV_3)) {
327 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS,
328 1 << VMXNET3_REV_3);
329 hw->version = VMXNET3_REV_3 + 1;
330 } else if (ver & (1 << VMXNET3_REV_2)) {
331 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS,
332 1 << VMXNET3_REV_2);
333 hw->version = VMXNET3_REV_2 + 1;
334 } else if (ver & (1 << VMXNET3_REV_1)) {
335 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS,
336 1 << VMXNET3_REV_1);
337 hw->version = VMXNET3_REV_1 + 1;
338 } else {
339 PMD_INIT_LOG(ERR, "Incompatible hardware version: %d", ver);
340 return -EIO;
341 }
342
343 PMD_INIT_LOG(INFO, "Using device v%d", hw->version);
344
345
346 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_UVRS);
347 PMD_INIT_LOG(DEBUG, "UPT hardware version : %d", ver);
348 if (ver & 0x1)
349 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_UVRS, 1);
350 else {
351 PMD_INIT_LOG(ERR, "Incompatible UPT version.");
352 return -EIO;
353 }
354
355
356 mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL);
357 mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH);
358 memcpy(hw->perm_addr, &mac_lo, 4);
359 memcpy(hw->perm_addr + 4, &mac_hi, 2);
360
361
362 eth_dev->data->mac_addrs = rte_zmalloc("vmxnet3", RTE_ETHER_ADDR_LEN *
363 VMXNET3_MAX_MAC_ADDRS, 0);
364 if (eth_dev->data->mac_addrs == NULL) {
365 PMD_INIT_LOG(ERR,
366 "Failed to allocate %d bytes needed to store MAC addresses",
367 RTE_ETHER_ADDR_LEN * VMXNET3_MAX_MAC_ADDRS);
368 return -ENOMEM;
369 }
370
371 rte_ether_addr_copy((struct rte_ether_addr *)hw->perm_addr,
372 ð_dev->data->mac_addrs[0]);
373
374 PMD_INIT_LOG(DEBUG, "MAC Address : %02x:%02x:%02x:%02x:%02x:%02x",
375 hw->perm_addr[0], hw->perm_addr[1], hw->perm_addr[2],
376 hw->perm_addr[3], hw->perm_addr[4], hw->perm_addr[5]);
377
378
379 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
380
381
382 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, 0);
383
384 hw->txdata_desc_size = VMXNET3_VERSION_GE_3(hw) ?
385 eth_vmxnet3_txdata_get(hw) : sizeof(struct Vmxnet3_TxDataDesc);
386
387 hw->rxdata_desc_size = VMXNET3_VERSION_GE_3(hw) ?
388 VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
389 RTE_ASSERT((hw->rxdata_desc_size & ~VMXNET3_RXDATA_DESC_SIZE_MASK) ==
390 hw->rxdata_desc_size);
391
392
393 memset(hw->saved_tx_stats, 0, sizeof(hw->saved_tx_stats));
394 memset(hw->saved_rx_stats, 0, sizeof(hw->saved_rx_stats));
395
396
397 memset(hw->snapshot_tx_stats, 0, sizeof(hw->snapshot_tx_stats));
398 memset(hw->snapshot_rx_stats, 0, sizeof(hw->snapshot_rx_stats));
399
400
401 memset(&link, 0, sizeof(link));
402 link.link_duplex = ETH_LINK_FULL_DUPLEX;
403 link.link_speed = ETH_SPEED_NUM_10G;
404 link.link_autoneg = ETH_LINK_FIXED;
405 rte_eth_linkstatus_set(eth_dev, &link);
406
407 return 0;
408}
409
410static int
411eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev)
412{
413 struct vmxnet3_hw *hw = eth_dev->data->dev_private;
414
415 PMD_INIT_FUNC_TRACE();
416
417 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
418 return 0;
419
420 if (hw->adapter_stopped == 0) {
421 PMD_INIT_LOG(DEBUG, "Device has not been closed.");
422 return -EBUSY;
423 }
424
425 return 0;
426}
427
428static int eth_vmxnet3_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
429 struct rte_pci_device *pci_dev)
430{
431 return rte_eth_dev_pci_generic_probe(pci_dev,
432 sizeof(struct vmxnet3_hw), eth_vmxnet3_dev_init);
433}
434
435static int eth_vmxnet3_pci_remove(struct rte_pci_device *pci_dev)
436{
437 return rte_eth_dev_pci_generic_remove(pci_dev, eth_vmxnet3_dev_uninit);
438}
439
440static struct rte_pci_driver rte_vmxnet3_pmd = {
441 .id_table = pci_id_vmxnet3_map,
442 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
443 .probe = eth_vmxnet3_pci_probe,
444 .remove = eth_vmxnet3_pci_remove,
445};
446
447static void
448vmxnet3_alloc_intr_resources(struct rte_eth_dev *dev)
449{
450 struct vmxnet3_hw *hw = dev->data->dev_private;
451 uint32_t cfg;
452 int nvec = 1;
453
454
455 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
456 VMXNET3_CMD_GET_CONF_INTR);
457 cfg = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
458 hw->intr.type = cfg & 0x3;
459 hw->intr.mask_mode = (cfg >> 2) & 0x3;
460
461 if (hw->intr.type == VMXNET3_IT_AUTO)
462 hw->intr.type = VMXNET3_IT_MSIX;
463
464 if (hw->intr.type == VMXNET3_IT_MSIX) {
465
466 if (hw->num_tx_queues != hw->num_rx_queues)
467 goto msix_err;
468
469 nvec += hw->num_rx_queues;
470 hw->intr.num_intrs = nvec;
471 return;
472 }
473
474msix_err:
475
476 hw->intr.num_intrs = 2;
477 hw->intr.lsc_only = TRUE;
478 PMD_INIT_LOG(INFO, "Enabled MSI-X with %d vectors", hw->intr.num_intrs);
479}
480
481static int
482vmxnet3_dev_configure(struct rte_eth_dev *dev)
483{
484 const struct rte_memzone *mz;
485 struct vmxnet3_hw *hw = dev->data->dev_private;
486 size_t size;
487
488 PMD_INIT_FUNC_TRACE();
489
490 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
491 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
492
493 if (dev->data->nb_tx_queues > VMXNET3_MAX_TX_QUEUES ||
494 dev->data->nb_rx_queues > VMXNET3_MAX_RX_QUEUES) {
495 PMD_INIT_LOG(ERR, "ERROR: Number of queues not supported");
496 return -EINVAL;
497 }
498
499 if (!rte_is_power_of_2(dev->data->nb_rx_queues)) {
500 PMD_INIT_LOG(ERR, "ERROR: Number of rx queues not power of 2");
501 return -EINVAL;
502 }
503
504 size = dev->data->nb_rx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
505 dev->data->nb_tx_queues * sizeof(struct Vmxnet3_RxQueueDesc);
506
507 if (size > UINT16_MAX)
508 return -EINVAL;
509
510 hw->num_rx_queues = (uint8_t)dev->data->nb_rx_queues;
511 hw->num_tx_queues = (uint8_t)dev->data->nb_tx_queues;
512
513
514
515
516
517 mz = gpa_zone_reserve(dev, sizeof(struct Vmxnet3_DriverShared),
518 "shared", rte_socket_id(), 8, 1);
519
520 if (mz == NULL) {
521 PMD_INIT_LOG(ERR, "ERROR: Creating shared zone");
522 return -ENOMEM;
523 }
524 memset(mz->addr, 0, mz->len);
525
526 hw->shared = mz->addr;
527 hw->sharedPA = mz->iova;
528
529
530
531
532
533
534
535
536
537 mz = gpa_zone_reserve(dev, size, "queuedesc", rte_socket_id(),
538 VMXNET3_QUEUE_DESC_ALIGN, 0);
539 if (mz == NULL) {
540 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
541 return -ENOMEM;
542 }
543 memset(mz->addr, 0, mz->len);
544
545 hw->tqd_start = (Vmxnet3_TxQueueDesc *)mz->addr;
546 hw->rqd_start = (Vmxnet3_RxQueueDesc *)(hw->tqd_start + hw->num_tx_queues);
547
548 hw->queueDescPA = mz->iova;
549 hw->queue_desc_len = (uint16_t)size;
550
551 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
552
553 mz = gpa_zone_reserve(dev, sizeof(struct VMXNET3_RSSConf),
554 "rss_conf", rte_socket_id(),
555 RTE_CACHE_LINE_SIZE, 1);
556 if (mz == NULL) {
557 PMD_INIT_LOG(ERR,
558 "ERROR: Creating rss_conf structure zone");
559 return -ENOMEM;
560 }
561 memset(mz->addr, 0, mz->len);
562
563 hw->rss_conf = mz->addr;
564 hw->rss_confPA = mz->iova;
565 }
566
567 vmxnet3_alloc_intr_resources(dev);
568
569 return 0;
570}
571
572static void
573vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr)
574{
575 uint32_t val;
576
577 PMD_INIT_LOG(DEBUG,
578 "Writing MAC Address : %02x:%02x:%02x:%02x:%02x:%02x",
579 addr[0], addr[1], addr[2],
580 addr[3], addr[4], addr[5]);
581
582 memcpy(&val, addr, 4);
583 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACL, val);
584
585 memcpy(&val, addr + 4, 2);
586 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACH, val);
587}
588
589
590
591
592
593
594
595
596static int
597vmxnet3_configure_msix(struct rte_eth_dev *dev)
598{
599 struct vmxnet3_hw *hw = dev->data->dev_private;
600 struct rte_intr_handle *intr_handle = dev->intr_handle;
601 uint16_t intr_vector;
602 int i;
603
604 hw->intr.event_intr_idx = 0;
605
606
607 if (!rte_intr_cap_multiple(intr_handle) ||
608 dev->data->dev_conf.intr_conf.rxq == 0)
609 return -1;
610
611 intr_vector = dev->data->nb_rx_queues;
612 if (intr_vector > VMXNET3_MAX_RX_QUEUES) {
613 PMD_INIT_LOG(ERR, "At most %d intr queues supported",
614 VMXNET3_MAX_RX_QUEUES);
615 return -ENOTSUP;
616 }
617
618 if (rte_intr_efd_enable(intr_handle, intr_vector)) {
619 PMD_INIT_LOG(ERR, "Failed to enable fastpath event fd");
620 return -1;
621 }
622
623 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
624 intr_handle->intr_vec =
625 rte_zmalloc("intr_vec",
626 dev->data->nb_rx_queues * sizeof(int), 0);
627 if (intr_handle->intr_vec == NULL) {
628 PMD_INIT_LOG(ERR, "Failed to allocate %d Rx queues intr_vec",
629 dev->data->nb_rx_queues);
630 rte_intr_efd_disable(intr_handle);
631 return -ENOMEM;
632 }
633 }
634
635 if (!rte_intr_allow_others(intr_handle) &&
636 dev->data->dev_conf.intr_conf.lsc != 0) {
637 PMD_INIT_LOG(ERR, "not enough intr vector to support both Rx interrupt and LSC");
638 rte_free(intr_handle->intr_vec);
639 intr_handle->intr_vec = NULL;
640 rte_intr_efd_disable(intr_handle);
641 return -1;
642 }
643
644
645
646
647 if (hw->intr.num_intrs != (intr_handle->nb_efd + 1)) {
648 PMD_INIT_LOG(ERR, "Device configured with %d Rx intr vectors, expecting %d",
649 hw->intr.num_intrs, intr_handle->nb_efd + 1);
650 rte_free(intr_handle->intr_vec);
651 intr_handle->intr_vec = NULL;
652 rte_intr_efd_disable(intr_handle);
653 return -1;
654 }
655
656 for (i = 0; i < dev->data->nb_rx_queues; i++)
657 intr_handle->intr_vec[i] = i + 1;
658
659 for (i = 0; i < hw->intr.num_intrs; i++)
660 hw->intr.mod_levels[i] = UPT1_IML_ADAPTIVE;
661
662 PMD_INIT_LOG(INFO, "intr type %u, mode %u, %u vectors allocated",
663 hw->intr.type, hw->intr.mask_mode, hw->intr.num_intrs);
664
665 return 0;
666}
667
668static int
669vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev)
670{
671 struct vmxnet3_hw *hw = dev->data->dev_private;
672 Vmxnet3_DriverShared *shared = hw->shared;
673 Vmxnet3_CmdInfo *cmdInfo;
674 struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES];
675 uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES];
676 uint32_t num, i, j, size;
677
678 if (hw->memRegsPA == 0) {
679 const struct rte_memzone *mz;
680
681 size = sizeof(Vmxnet3_MemRegs) +
682 (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) *
683 sizeof(Vmxnet3_MemoryRegion);
684
685 mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8,
686 1);
687 if (mz == NULL) {
688 PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone");
689 return -ENOMEM;
690 }
691 memset(mz->addr, 0, mz->len);
692 hw->memRegs = mz->addr;
693 hw->memRegsPA = mz->iova;
694 }
695
696 num = hw->num_rx_queues;
697
698 for (i = 0; i < num; i++) {
699 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
700
701 mp[i] = rxq->mp;
702 index[i] = 1 << i;
703 }
704
705
706
707
708
709
710 for (i = 1; i < num; i++) {
711 for (j = 0; j < i; j++) {
712 if (mp[i] == mp[j]) {
713 mp[i] = NULL;
714 index[j] |= 1 << i;
715 break;
716 }
717 }
718 }
719
720 j = 0;
721 for (i = 0; i < num; i++) {
722 if (mp[i] == NULL)
723 continue;
724
725 Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j];
726
727 mr->startPA =
728 (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->iova;
729 mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ?
730 STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX;
731 mr->txQueueBits = index[i];
732 mr->rxQueueBits = index[i];
733
734 PMD_INIT_LOG(INFO,
735 "index: %u startPA: %" PRIu64 " length: %u, "
736 "rxBits: %x",
737 j, mr->startPA, mr->length, mr->rxQueueBits);
738 j++;
739 }
740 hw->memRegs->numRegs = j;
741 PMD_INIT_LOG(INFO, "numRegs: %u", j);
742
743 size = sizeof(Vmxnet3_MemRegs) +
744 (j - 1) * sizeof(Vmxnet3_MemoryRegion);
745
746 cmdInfo = &shared->cu.cmdInfo;
747 cmdInfo->varConf.confVer = 1;
748 cmdInfo->varConf.confLen = size;
749 cmdInfo->varConf.confPA = hw->memRegsPA;
750
751 return 0;
752}
753
754static int
755vmxnet3_setup_driver_shared(struct rte_eth_dev *dev)
756{
757 struct rte_eth_conf port_conf = dev->data->dev_conf;
758 struct vmxnet3_hw *hw = dev->data->dev_private;
759 struct rte_intr_handle *intr_handle = dev->intr_handle;
760 uint32_t mtu = dev->data->mtu;
761 Vmxnet3_DriverShared *shared = hw->shared;
762 Vmxnet3_DSDevRead *devRead = &shared->devRead;
763 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
764 uint32_t i;
765 int ret;
766
767 hw->mtu = mtu;
768
769 shared->magic = VMXNET3_REV1_MAGIC;
770 devRead->misc.driverInfo.version = VMXNET3_DRIVER_VERSION_NUM;
771
772
773 devRead->misc.driverInfo.gos.gosBits = sizeof(void *) == 4 ?
774 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64;
775 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
776 devRead->misc.driverInfo.vmxnet3RevSpt = 1;
777 devRead->misc.driverInfo.uptVerSpt = 1;
778
779 devRead->misc.mtu = rte_le_to_cpu_32(mtu);
780 devRead->misc.queueDescPA = hw->queueDescPA;
781 devRead->misc.queueDescLen = hw->queue_desc_len;
782 devRead->misc.numTxQueues = hw->num_tx_queues;
783 devRead->misc.numRxQueues = hw->num_rx_queues;
784
785 for (i = 0; i < hw->num_tx_queues; i++) {
786 Vmxnet3_TxQueueDesc *tqd = &hw->tqd_start[i];
787 vmxnet3_tx_queue_t *txq = dev->data->tx_queues[i];
788
789 txq->shared = &hw->tqd_start[i];
790
791 tqd->ctrl.txNumDeferred = 0;
792 tqd->ctrl.txThreshold = 1;
793 tqd->conf.txRingBasePA = txq->cmd_ring.basePA;
794 tqd->conf.compRingBasePA = txq->comp_ring.basePA;
795 tqd->conf.dataRingBasePA = txq->data_ring.basePA;
796
797 tqd->conf.txRingSize = txq->cmd_ring.size;
798 tqd->conf.compRingSize = txq->comp_ring.size;
799 tqd->conf.dataRingSize = txq->data_ring.size;
800 tqd->conf.txDataRingDescSize = txq->txdata_desc_size;
801
802 if (hw->intr.lsc_only)
803 tqd->conf.intrIdx = 1;
804 else
805 tqd->conf.intrIdx = intr_handle->intr_vec[i];
806 tqd->status.stopped = TRUE;
807 tqd->status.error = 0;
808 memset(&tqd->stats, 0, sizeof(tqd->stats));
809 }
810
811 for (i = 0; i < hw->num_rx_queues; i++) {
812 Vmxnet3_RxQueueDesc *rqd = &hw->rqd_start[i];
813 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
814
815 rxq->shared = &hw->rqd_start[i];
816
817 rqd->conf.rxRingBasePA[0] = rxq->cmd_ring[0].basePA;
818 rqd->conf.rxRingBasePA[1] = rxq->cmd_ring[1].basePA;
819 rqd->conf.compRingBasePA = rxq->comp_ring.basePA;
820
821 rqd->conf.rxRingSize[0] = rxq->cmd_ring[0].size;
822 rqd->conf.rxRingSize[1] = rxq->cmd_ring[1].size;
823 rqd->conf.compRingSize = rxq->comp_ring.size;
824
825 if (hw->intr.lsc_only)
826 rqd->conf.intrIdx = 1;
827 else
828 rqd->conf.intrIdx = intr_handle->intr_vec[i];
829 rqd->status.stopped = TRUE;
830 rqd->status.error = 0;
831 memset(&rqd->stats, 0, sizeof(rqd->stats));
832 }
833
834
835 devRead->intrConf.autoMask = hw->intr.mask_mode == VMXNET3_IMM_AUTO;
836 devRead->intrConf.numIntrs = hw->intr.num_intrs;
837 for (i = 0; i < hw->intr.num_intrs; i++)
838 devRead->intrConf.modLevels[i] = hw->intr.mod_levels[i];
839
840 devRead->intrConf.eventIntrIdx = hw->intr.event_intr_idx;
841 devRead->intrConf.intrCtrl |= rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL);
842
843
844 devRead->rxFilterConf.rxMode = 0;
845
846
847 if (rx_offloads & DEV_RX_OFFLOAD_CHECKSUM)
848 devRead->misc.uptFeatures |= VMXNET3_F_RXCSUM;
849
850 if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
851 devRead->misc.uptFeatures |= VMXNET3_F_LRO;
852 devRead->misc.maxNumRxSG = 0;
853 }
854
855 if (port_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
856 ret = vmxnet3_rss_configure(dev);
857 if (ret != VMXNET3_SUCCESS)
858 return ret;
859
860 devRead->misc.uptFeatures |= VMXNET3_F_RSS;
861 devRead->rssConfDesc.confVer = 1;
862 devRead->rssConfDesc.confLen = sizeof(struct VMXNET3_RSSConf);
863 devRead->rssConfDesc.confPA = hw->rss_confPA;
864 }
865
866 ret = vmxnet3_dev_vlan_offload_set(dev,
867 ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK);
868 if (ret)
869 return ret;
870
871 vmxnet3_write_mac(hw, dev->data->mac_addrs->addr_bytes);
872
873 return VMXNET3_SUCCESS;
874}
875
876
877
878
879
880
881static int
882vmxnet3_dev_start(struct rte_eth_dev *dev)
883{
884 int ret;
885 struct vmxnet3_hw *hw = dev->data->dev_private;
886
887 PMD_INIT_FUNC_TRACE();
888
889
890 vmxnet3_hw_stats_save(hw);
891
892
893 ret = vmxnet3_configure_msix(dev);
894 if (ret < 0) {
895
896 hw->intr.num_intrs = 2;
897 hw->intr.lsc_only = TRUE;
898 }
899
900 ret = vmxnet3_setup_driver_shared(dev);
901 if (ret != VMXNET3_SUCCESS)
902 return ret;
903
904
905 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL,
906 VMXNET3_GET_ADDR_LO(hw->sharedPA));
907 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH,
908 VMXNET3_GET_ADDR_HI(hw->sharedPA));
909
910
911 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
912 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
913
914 if (ret != 0) {
915 PMD_INIT_LOG(ERR, "Device activation: UNSUCCESSFUL");
916 return -EINVAL;
917 }
918
919
920 ret = vmxnet3_dev_setup_memreg(dev);
921 if (ret == 0) {
922 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
923 VMXNET3_CMD_REGISTER_MEMREGS);
924 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
925 if (ret != 0)
926 PMD_INIT_LOG(DEBUG,
927 "Failed in setup memory region cmd\n");
928 ret = 0;
929 } else {
930 PMD_INIT_LOG(DEBUG, "Failed to setup memory region\n");
931 }
932
933 if (VMXNET3_VERSION_GE_4(hw) &&
934 dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
935
936 ret = vmxnet3_v4_rss_configure(dev);
937 if (ret != VMXNET3_SUCCESS) {
938 PMD_INIT_LOG(ERR, "Failed to configure v4 RSS");
939 return ret;
940 }
941 }
942
943
944
945
946
947 ret = vmxnet3_dev_rxtx_init(dev);
948 if (ret != VMXNET3_SUCCESS) {
949 PMD_INIT_LOG(ERR, "Device queue init: UNSUCCESSFUL");
950 return ret;
951 }
952
953 hw->adapter_stopped = FALSE;
954
955
956 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1);
957
958
959 rte_intr_callback_register(dev->intr_handle,
960 vmxnet3_interrupt_handler, dev);
961
962 if (rte_intr_enable(dev->intr_handle) < 0) {
963 PMD_INIT_LOG(ERR, "interrupt enable failed");
964 return -EIO;
965 }
966
967
968 vmxnet3_enable_all_intrs(hw);
969
970 vmxnet3_process_events(dev);
971
972
973
974
975
976
977
978
979 __vmxnet3_dev_link_update(dev, 0);
980
981 return VMXNET3_SUCCESS;
982}
983
984
985
986
987static int
988vmxnet3_dev_stop(struct rte_eth_dev *dev)
989{
990 struct rte_eth_link link;
991 struct vmxnet3_hw *hw = dev->data->dev_private;
992 struct rte_intr_handle *intr_handle = dev->intr_handle;
993 int ret;
994
995 PMD_INIT_FUNC_TRACE();
996
997 if (hw->adapter_stopped == 1) {
998 PMD_INIT_LOG(DEBUG, "Device already stopped.");
999 return 0;
1000 }
1001
1002 do {
1003
1004
1005
1006
1007 ret = rte_intr_callback_unregister(intr_handle,
1008 vmxnet3_interrupt_handler,
1009 (void *)-1);
1010 } while (ret == -EAGAIN);
1011
1012 if (ret < 0)
1013 PMD_DRV_LOG(ERR, "Error attempting to unregister intr cb: %d",
1014 ret);
1015
1016 PMD_INIT_LOG(DEBUG, "Disabled %d intr callbacks", ret);
1017
1018
1019 vmxnet3_disable_all_intrs(hw);
1020
1021 rte_intr_disable(intr_handle);
1022
1023
1024 rte_intr_efd_disable(intr_handle);
1025 if (intr_handle->intr_vec != NULL) {
1026 rte_free(intr_handle->intr_vec);
1027 intr_handle->intr_vec = NULL;
1028 }
1029
1030
1031 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
1032 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 0);
1033 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 0);
1034
1035
1036 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
1037 PMD_INIT_LOG(DEBUG, "Device reset.");
1038
1039 vmxnet3_dev_clear_queues(dev);
1040
1041
1042 memset(&link, 0, sizeof(link));
1043 link.link_duplex = ETH_LINK_FULL_DUPLEX;
1044 link.link_speed = ETH_SPEED_NUM_10G;
1045 link.link_autoneg = ETH_LINK_FIXED;
1046 rte_eth_linkstatus_set(dev, &link);
1047
1048 hw->adapter_stopped = 1;
1049 dev->data->dev_started = 0;
1050
1051 return 0;
1052}
1053
1054static void
1055vmxnet3_free_queues(struct rte_eth_dev *dev)
1056{
1057 int i;
1058
1059 PMD_INIT_FUNC_TRACE();
1060
1061 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1062 void *rxq = dev->data->rx_queues[i];
1063
1064 vmxnet3_dev_rx_queue_release(rxq);
1065 }
1066 dev->data->nb_rx_queues = 0;
1067
1068 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1069 void *txq = dev->data->tx_queues[i];
1070
1071 vmxnet3_dev_tx_queue_release(txq);
1072 }
1073 dev->data->nb_tx_queues = 0;
1074}
1075
1076
1077
1078
1079static int
1080vmxnet3_dev_close(struct rte_eth_dev *dev)
1081{
1082 int ret;
1083 PMD_INIT_FUNC_TRACE();
1084 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1085 return 0;
1086
1087 ret = vmxnet3_dev_stop(dev);
1088 vmxnet3_free_queues(dev);
1089
1090 return ret;
1091}
1092
1093static int
1094vmxnet3_dev_reset(struct rte_eth_dev *dev)
1095{
1096 int ret;
1097
1098 ret = eth_vmxnet3_dev_uninit(dev);
1099 if (ret)
1100 return ret;
1101 ret = eth_vmxnet3_dev_init(dev);
1102 return ret;
1103}
1104
1105static void
1106vmxnet3_hw_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q,
1107 struct UPT1_TxStats *res)
1108{
1109#define VMXNET3_UPDATE_TX_STAT(h, i, f, r) \
1110 ((r)->f = (h)->tqd_start[(i)].stats.f + \
1111 (h)->saved_tx_stats[(i)].f)
1112
1113 VMXNET3_UPDATE_TX_STAT(hw, q, ucastPktsTxOK, res);
1114 VMXNET3_UPDATE_TX_STAT(hw, q, mcastPktsTxOK, res);
1115 VMXNET3_UPDATE_TX_STAT(hw, q, bcastPktsTxOK, res);
1116 VMXNET3_UPDATE_TX_STAT(hw, q, ucastBytesTxOK, res);
1117 VMXNET3_UPDATE_TX_STAT(hw, q, mcastBytesTxOK, res);
1118 VMXNET3_UPDATE_TX_STAT(hw, q, bcastBytesTxOK, res);
1119 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxError, res);
1120 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxDiscard, res);
1121
1122#undef VMXNET3_UPDATE_TX_STAT
1123}
1124
1125static void
1126vmxnet3_hw_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q,
1127 struct UPT1_RxStats *res)
1128{
1129#define VMXNET3_UPDATE_RX_STAT(h, i, f, r) \
1130 ((r)->f = (h)->rqd_start[(i)].stats.f + \
1131 (h)->saved_rx_stats[(i)].f)
1132
1133 VMXNET3_UPDATE_RX_STAT(hw, q, ucastPktsRxOK, res);
1134 VMXNET3_UPDATE_RX_STAT(hw, q, mcastPktsRxOK, res);
1135 VMXNET3_UPDATE_RX_STAT(hw, q, bcastPktsRxOK, res);
1136 VMXNET3_UPDATE_RX_STAT(hw, q, ucastBytesRxOK, res);
1137 VMXNET3_UPDATE_RX_STAT(hw, q, mcastBytesRxOK, res);
1138 VMXNET3_UPDATE_RX_STAT(hw, q, bcastBytesRxOK, res);
1139 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxError, res);
1140 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxOutOfBuf, res);
1141
1142#undef VMXNET3_UPDATE_RX_STAT
1143}
1144
1145static void
1146vmxnet3_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q,
1147 struct UPT1_TxStats *res)
1148{
1149 vmxnet3_hw_tx_stats_get(hw, q, res);
1150
1151#define VMXNET3_REDUCE_SNAPSHOT_TX_STAT(h, i, f, r) \
1152 ((r)->f -= (h)->snapshot_tx_stats[(i)].f)
1153
1154 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastPktsTxOK, res);
1155 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastPktsTxOK, res);
1156 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastPktsTxOK, res);
1157 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastBytesTxOK, res);
1158 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastBytesTxOK, res);
1159 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastBytesTxOK, res);
1160 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxError, res);
1161 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxDiscard, res);
1162
1163#undef VMXNET3_REDUCE_SNAPSHOT_TX_STAT
1164}
1165
1166static void
1167vmxnet3_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q,
1168 struct UPT1_RxStats *res)
1169{
1170 vmxnet3_hw_rx_stats_get(hw, q, res);
1171
1172#define VMXNET3_REDUCE_SNAPSHOT_RX_STAT(h, i, f, r) \
1173 ((r)->f -= (h)->snapshot_rx_stats[(i)].f)
1174
1175 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastPktsRxOK, res);
1176 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastPktsRxOK, res);
1177 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastPktsRxOK, res);
1178 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastBytesRxOK, res);
1179 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastBytesRxOK, res);
1180 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastBytesRxOK, res);
1181 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxError, res);
1182 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxOutOfBuf, res);
1183
1184#undef VMXNET3_REDUCE_SNAPSHOT_RX_STAT
1185}
1186
1187static void
1188vmxnet3_hw_stats_save(struct vmxnet3_hw *hw)
1189{
1190 unsigned int i;
1191
1192 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
1193
1194 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES);
1195
1196 for (i = 0; i < hw->num_tx_queues; i++)
1197 vmxnet3_hw_tx_stats_get(hw, i, &hw->saved_tx_stats[i]);
1198 for (i = 0; i < hw->num_rx_queues; i++)
1199 vmxnet3_hw_rx_stats_get(hw, i, &hw->saved_rx_stats[i]);
1200}
1201
1202static int
1203vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev,
1204 struct rte_eth_xstat_name *xstats_names,
1205 unsigned int n)
1206{
1207 unsigned int i, t, count = 0;
1208 unsigned int nstats =
1209 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) +
1210 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings);
1211
1212 if (!xstats_names || n < nstats)
1213 return nstats;
1214
1215 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1216 if (!dev->data->rx_queues[i])
1217 continue;
1218
1219 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) {
1220 snprintf(xstats_names[count].name,
1221 sizeof(xstats_names[count].name),
1222 "rx_q%u_%s", i,
1223 vmxnet3_rxq_stat_strings[t].name);
1224 count++;
1225 }
1226 }
1227
1228 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1229 if (!dev->data->tx_queues[i])
1230 continue;
1231
1232 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) {
1233 snprintf(xstats_names[count].name,
1234 sizeof(xstats_names[count].name),
1235 "tx_q%u_%s", i,
1236 vmxnet3_txq_stat_strings[t].name);
1237 count++;
1238 }
1239 }
1240
1241 return count;
1242}
1243
1244static int
1245vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1246 unsigned int n)
1247{
1248 unsigned int i, t, count = 0;
1249 unsigned int nstats =
1250 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) +
1251 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings);
1252
1253 if (n < nstats)
1254 return nstats;
1255
1256 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1257 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
1258
1259 if (rxq == NULL)
1260 continue;
1261
1262 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) {
1263 xstats[count].value = *(uint64_t *)(((char *)&rxq->stats) +
1264 vmxnet3_rxq_stat_strings[t].offset);
1265 xstats[count].id = count;
1266 count++;
1267 }
1268 }
1269
1270 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1271 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
1272
1273 if (txq == NULL)
1274 continue;
1275
1276 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) {
1277 xstats[count].value = *(uint64_t *)(((char *)&txq->stats) +
1278 vmxnet3_txq_stat_strings[t].offset);
1279 xstats[count].id = count;
1280 count++;
1281 }
1282 }
1283
1284 return count;
1285}
1286
1287static int
1288vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1289{
1290 unsigned int i;
1291 struct vmxnet3_hw *hw = dev->data->dev_private;
1292 struct UPT1_TxStats txStats;
1293 struct UPT1_RxStats rxStats;
1294
1295 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
1296
1297 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES);
1298 for (i = 0; i < hw->num_tx_queues; i++) {
1299 vmxnet3_tx_stats_get(hw, i, &txStats);
1300
1301 stats->q_opackets[i] = txStats.ucastPktsTxOK +
1302 txStats.mcastPktsTxOK +
1303 txStats.bcastPktsTxOK;
1304
1305 stats->q_obytes[i] = txStats.ucastBytesTxOK +
1306 txStats.mcastBytesTxOK +
1307 txStats.bcastBytesTxOK;
1308
1309 stats->opackets += stats->q_opackets[i];
1310 stats->obytes += stats->q_obytes[i];
1311 stats->oerrors += txStats.pktsTxError + txStats.pktsTxDiscard;
1312 }
1313
1314 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_RX_QUEUES);
1315 for (i = 0; i < hw->num_rx_queues; i++) {
1316 vmxnet3_rx_stats_get(hw, i, &rxStats);
1317
1318 stats->q_ipackets[i] = rxStats.ucastPktsRxOK +
1319 rxStats.mcastPktsRxOK +
1320 rxStats.bcastPktsRxOK;
1321
1322 stats->q_ibytes[i] = rxStats.ucastBytesRxOK +
1323 rxStats.mcastBytesRxOK +
1324 rxStats.bcastBytesRxOK;
1325
1326 stats->ipackets += stats->q_ipackets[i];
1327 stats->ibytes += stats->q_ibytes[i];
1328
1329 stats->q_errors[i] = rxStats.pktsRxError;
1330 stats->ierrors += rxStats.pktsRxError;
1331 stats->imissed += rxStats.pktsRxOutOfBuf;
1332 }
1333
1334 return 0;
1335}
1336
1337static int
1338vmxnet3_dev_stats_reset(struct rte_eth_dev *dev)
1339{
1340 unsigned int i;
1341 struct vmxnet3_hw *hw = dev->data->dev_private;
1342 struct UPT1_TxStats txStats = {0};
1343 struct UPT1_RxStats rxStats = {0};
1344
1345 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
1346
1347 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES);
1348
1349 for (i = 0; i < hw->num_tx_queues; i++) {
1350 vmxnet3_hw_tx_stats_get(hw, i, &txStats);
1351 memcpy(&hw->snapshot_tx_stats[i], &txStats,
1352 sizeof(hw->snapshot_tx_stats[0]));
1353 }
1354 for (i = 0; i < hw->num_rx_queues; i++) {
1355 vmxnet3_hw_rx_stats_get(hw, i, &rxStats);
1356 memcpy(&hw->snapshot_rx_stats[i], &rxStats,
1357 sizeof(hw->snapshot_rx_stats[0]));
1358 }
1359
1360 return 0;
1361}
1362
1363static int
1364vmxnet3_dev_info_get(struct rte_eth_dev *dev,
1365 struct rte_eth_dev_info *dev_info)
1366{
1367 struct vmxnet3_hw *hw = dev->data->dev_private;
1368
1369 dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES;
1370 dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES;
1371 dev_info->min_rx_bufsize = 1518 + RTE_PKTMBUF_HEADROOM;
1372 dev_info->max_rx_pktlen = 16384;
1373 dev_info->min_mtu = VMXNET3_MIN_MTU;
1374 dev_info->max_mtu = VMXNET3_MAX_MTU;
1375 dev_info->speed_capa = ETH_LINK_SPEED_10G;
1376 dev_info->max_mac_addrs = VMXNET3_MAX_MAC_ADDRS;
1377
1378 dev_info->flow_type_rss_offloads = VMXNET3_RSS_OFFLOAD_ALL;
1379
1380 if (VMXNET3_VERSION_GE_4(hw)) {
1381 dev_info->flow_type_rss_offloads |= VMXNET3_V4_RSS_MASK;
1382 }
1383
1384 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1385 .nb_max = VMXNET3_RX_RING_MAX_SIZE,
1386 .nb_min = VMXNET3_DEF_RX_RING_SIZE,
1387 .nb_align = 1,
1388 };
1389
1390 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1391 .nb_max = VMXNET3_TX_RING_MAX_SIZE,
1392 .nb_min = VMXNET3_DEF_TX_RING_SIZE,
1393 .nb_align = 1,
1394 .nb_seg_max = VMXNET3_TX_MAX_SEG,
1395 .nb_mtu_seg_max = VMXNET3_MAX_TXD_PER_PKT,
1396 };
1397
1398 dev_info->rx_offload_capa = VMXNET3_RX_OFFLOAD_CAP;
1399 dev_info->rx_queue_offload_capa = 0;
1400 dev_info->tx_offload_capa = VMXNET3_TX_OFFLOAD_CAP;
1401 dev_info->tx_queue_offload_capa = 0;
1402
1403 return 0;
1404}
1405
1406static const uint32_t *
1407vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1408{
1409 static const uint32_t ptypes[] = {
1410 RTE_PTYPE_L3_IPV4_EXT,
1411 RTE_PTYPE_L3_IPV4,
1412 RTE_PTYPE_UNKNOWN
1413 };
1414
1415 if (dev->rx_pkt_burst == vmxnet3_recv_pkts)
1416 return ptypes;
1417 return NULL;
1418}
1419
1420static int
1421vmxnet3_dev_mtu_set(struct rte_eth_dev *dev, __rte_unused uint16_t mtu)
1422{
1423 if (dev->data->dev_started) {
1424 PMD_DRV_LOG(ERR, "Port %d must be stopped to configure MTU",
1425 dev->data->port_id);
1426 return -EBUSY;
1427 }
1428
1429 return 0;
1430}
1431
1432static int
1433vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1434{
1435 struct vmxnet3_hw *hw = dev->data->dev_private;
1436
1437 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)(hw->perm_addr));
1438 vmxnet3_write_mac(hw, mac_addr->addr_bytes);
1439 return 0;
1440}
1441
1442
1443static int
1444__vmxnet3_dev_link_update(struct rte_eth_dev *dev,
1445 __rte_unused int wait_to_complete)
1446{
1447 struct vmxnet3_hw *hw = dev->data->dev_private;
1448 struct rte_eth_link link;
1449 uint32_t ret;
1450
1451 memset(&link, 0, sizeof(link));
1452
1453 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
1454 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
1455
1456 if (ret & 0x1)
1457 link.link_status = ETH_LINK_UP;
1458 link.link_duplex = ETH_LINK_FULL_DUPLEX;
1459 link.link_speed = ETH_SPEED_NUM_10G;
1460 link.link_autoneg = ETH_LINK_FIXED;
1461
1462 return rte_eth_linkstatus_set(dev, &link);
1463}
1464
1465static int
1466vmxnet3_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1467{
1468
1469 if (dev->data->dev_started == 0)
1470 return -1;
1471
1472 return __vmxnet3_dev_link_update(dev, wait_to_complete);
1473}
1474
1475
1476static void
1477vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set)
1478{
1479 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf;
1480
1481 if (set)
1482 rxConf->rxMode = rxConf->rxMode | feature;
1483 else
1484 rxConf->rxMode = rxConf->rxMode & (~feature);
1485
1486 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE);
1487}
1488
1489
1490static int
1491vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev)
1492{
1493 struct vmxnet3_hw *hw = dev->data->dev_private;
1494 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable;
1495
1496 memset(vf_table, 0, VMXNET3_VFT_TABLE_SIZE);
1497 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 1);
1498
1499 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1500 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1501
1502 return 0;
1503}
1504
1505
1506static int
1507vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev)
1508{
1509 struct vmxnet3_hw *hw = dev->data->dev_private;
1510 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable;
1511 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
1512
1513 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
1514 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE);
1515 else
1516 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE);
1517 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 0);
1518 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1519 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1520
1521 return 0;
1522}
1523
1524
1525static int
1526vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev)
1527{
1528 struct vmxnet3_hw *hw = dev->data->dev_private;
1529
1530 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 1);
1531
1532 return 0;
1533}
1534
1535
1536static int
1537vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev)
1538{
1539 struct vmxnet3_hw *hw = dev->data->dev_private;
1540
1541 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 0);
1542
1543 return 0;
1544}
1545
1546
1547static int
1548vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vid, int on)
1549{
1550 struct vmxnet3_hw *hw = dev->data->dev_private;
1551 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf;
1552 uint32_t *vf_table = rxConf->vfTable;
1553
1554
1555 if (on)
1556 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, vid);
1557 else
1558 VMXNET3_CLEAR_VFTABLE_ENTRY(hw->shadow_vfta, vid);
1559
1560
1561 if (rxConf->rxMode & VMXNET3_RXM_PROMISC)
1562 return 0;
1563
1564
1565 if (on)
1566 VMXNET3_SET_VFTABLE_ENTRY(vf_table, vid);
1567 else
1568 VMXNET3_CLEAR_VFTABLE_ENTRY(vf_table, vid);
1569
1570 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1571 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1572 return 0;
1573}
1574
1575static int
1576vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1577{
1578 struct vmxnet3_hw *hw = dev->data->dev_private;
1579 Vmxnet3_DSDevRead *devRead = &hw->shared->devRead;
1580 uint32_t *vf_table = devRead->rxFilterConf.vfTable;
1581 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
1582
1583 if (mask & ETH_VLAN_STRIP_MASK) {
1584 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
1585 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
1586 else
1587 devRead->misc.uptFeatures &= ~UPT1_F_RXVLAN;
1588
1589 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1590 VMXNET3_CMD_UPDATE_FEATURE);
1591 }
1592
1593 if (mask & ETH_VLAN_FILTER_MASK) {
1594 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
1595 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE);
1596 else
1597 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE);
1598
1599 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1600 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1601 }
1602
1603 return 0;
1604}
1605
1606static void
1607vmxnet3_process_events(struct rte_eth_dev *dev)
1608{
1609 struct vmxnet3_hw *hw = dev->data->dev_private;
1610 uint32_t events = hw->shared->ecr;
1611
1612 if (!events)
1613 return;
1614
1615
1616
1617
1618
1619 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_ECR, events);
1620
1621
1622 if (events & VMXNET3_ECR_LINK) {
1623 PMD_DRV_LOG(DEBUG, "Process events: VMXNET3_ECR_LINK event");
1624 if (vmxnet3_dev_link_update(dev, 0) == 0)
1625 rte_eth_dev_callback_process(dev,
1626 RTE_ETH_EVENT_INTR_LSC,
1627 NULL);
1628 }
1629
1630
1631 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
1632 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1633 VMXNET3_CMD_GET_QUEUE_STATUS);
1634
1635 if (hw->tqd_start->status.stopped)
1636 PMD_DRV_LOG(ERR, "tq error 0x%x",
1637 hw->tqd_start->status.error);
1638
1639 if (hw->rqd_start->status.stopped)
1640 PMD_DRV_LOG(ERR, "rq error 0x%x",
1641 hw->rqd_start->status.error);
1642
1643
1644
1645 }
1646
1647 if (events & VMXNET3_ECR_DIC)
1648 PMD_DRV_LOG(DEBUG, "Device implementation change event.");
1649
1650 if (events & VMXNET3_ECR_DEBUG)
1651 PMD_DRV_LOG(DEBUG, "Debug event generated by device.");
1652}
1653
1654static void
1655vmxnet3_interrupt_handler(void *param)
1656{
1657 struct rte_eth_dev *dev = param;
1658 struct vmxnet3_hw *hw = dev->data->dev_private;
1659 Vmxnet3_DSDevRead *devRead = &hw->shared->devRead;
1660 uint32_t events;
1661
1662 PMD_INIT_FUNC_TRACE();
1663 vmxnet3_disable_intr(hw, devRead->intrConf.eventIntrIdx);
1664
1665 events = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_ECR);
1666 if (events == 0)
1667 goto done;
1668
1669 RTE_LOG(DEBUG, PMD, "Reading events: 0x%X", events);
1670 vmxnet3_process_events(dev);
1671done:
1672 vmxnet3_enable_intr(hw, devRead->intrConf.eventIntrIdx);
1673}
1674
1675static int
1676vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1677{
1678 struct vmxnet3_hw *hw = dev->data->dev_private;
1679
1680 vmxnet3_enable_intr(hw, dev->intr_handle->intr_vec[queue_id]);
1681
1682 return 0;
1683}
1684
1685static int
1686vmxnet3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1687{
1688 struct vmxnet3_hw *hw = dev->data->dev_private;
1689
1690 vmxnet3_disable_intr(hw, dev->intr_handle->intr_vec[queue_id]);
1691
1692 return 0;
1693}
1694
1695RTE_PMD_REGISTER_PCI(net_vmxnet3, rte_vmxnet3_pmd);
1696RTE_PMD_REGISTER_PCI_TABLE(net_vmxnet3, pci_id_vmxnet3_map);
1697RTE_PMD_REGISTER_KMOD_DEP(net_vmxnet3, "* igb_uio | uio_pci_generic | vfio-pci");
1698RTE_LOG_REGISTER_SUFFIX(vmxnet3_logtype_init, init, NOTICE);
1699RTE_LOG_REGISTER_SUFFIX(vmxnet3_logtype_driver, driver, NOTICE);
1700
