linux/drivers/net/ethernet/broadcom/genet/bcmgenet.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Broadcom GENET (Gigabit Ethernet) controller driver
   4 *
   5 * Copyright (c) 2014-2017 Broadcom
   6 */
   7
   8#define pr_fmt(fmt)                             "bcmgenet: " fmt
   9
  10#include <linux/kernel.h>
  11#include <linux/module.h>
  12#include <linux/sched.h>
  13#include <linux/types.h>
  14#include <linux/fcntl.h>
  15#include <linux/interrupt.h>
  16#include <linux/string.h>
  17#include <linux/if_ether.h>
  18#include <linux/init.h>
  19#include <linux/errno.h>
  20#include <linux/delay.h>
  21#include <linux/platform_device.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/pm.h>
  24#include <linux/clk.h>
  25#include <linux/of.h>
  26#include <linux/of_address.h>
  27#include <linux/of_irq.h>
  28#include <linux/of_net.h>
  29#include <linux/of_platform.h>
  30#include <net/arp.h>
  31
  32#include <linux/mii.h>
  33#include <linux/ethtool.h>
  34#include <linux/netdevice.h>
  35#include <linux/inetdevice.h>
  36#include <linux/etherdevice.h>
  37#include <linux/skbuff.h>
  38#include <linux/in.h>
  39#include <linux/ip.h>
  40#include <linux/ipv6.h>
  41#include <linux/phy.h>
  42#include <linux/platform_data/bcmgenet.h>
  43
  44#include <asm/unaligned.h>
  45
  46#include "bcmgenet.h"
  47
  48/* Maximum number of hardware queues, downsized if needed */
  49#define GENET_MAX_MQ_CNT        4
  50
  51/* Default highest priority queue for multi queue support */
  52#define GENET_Q0_PRIORITY       0
  53
  54#define GENET_Q16_RX_BD_CNT     \
  55        (TOTAL_DESC - priv->hw_params->rx_queues * priv->hw_params->rx_bds_per_q)
  56#define GENET_Q16_TX_BD_CNT     \
  57        (TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->tx_bds_per_q)
  58
  59#define RX_BUF_LENGTH           2048
  60#define SKB_ALIGNMENT           32
  61
  62/* Tx/Rx DMA register offset, skip 256 descriptors */
  63#define WORDS_PER_BD(p)         (p->hw_params->words_per_bd)
  64#define DMA_DESC_SIZE           (WORDS_PER_BD(priv) * sizeof(u32))
  65
  66#define GENET_TDMA_REG_OFF      (priv->hw_params->tdma_offset + \
  67                                TOTAL_DESC * DMA_DESC_SIZE)
  68
  69#define GENET_RDMA_REG_OFF      (priv->hw_params->rdma_offset + \
  70                                TOTAL_DESC * DMA_DESC_SIZE)
  71
  72static inline void bcmgenet_writel(u32 value, void __iomem *offset)
  73{
  74        /* MIPS chips strapped for BE will automagically configure the
  75         * peripheral registers for CPU-native byte order.
  76         */
  77        if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
  78                __raw_writel(value, offset);
  79        else
  80                writel_relaxed(value, offset);
  81}
  82
  83static inline u32 bcmgenet_readl(void __iomem *offset)
  84{
  85        if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
  86                return __raw_readl(offset);
  87        else
  88                return readl_relaxed(offset);
  89}
  90
  91static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv,
  92                                             void __iomem *d, u32 value)
  93{
  94        bcmgenet_writel(value, d + DMA_DESC_LENGTH_STATUS);
  95}
  96
  97static inline u32 dmadesc_get_length_status(struct bcmgenet_priv *priv,
  98                                            void __iomem *d)
  99{
 100        return bcmgenet_readl(d + DMA_DESC_LENGTH_STATUS);
 101}
 102
 103static inline void dmadesc_set_addr(struct bcmgenet_priv *priv,
 104                                    void __iomem *d,
 105                                    dma_addr_t addr)
 106{
 107        bcmgenet_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO);
 108
 109        /* Register writes to GISB bus can take couple hundred nanoseconds
 110         * and are done for each packet, save these expensive writes unless
 111         * the platform is explicitly configured for 64-bits/LPAE.
 112         */
 113#ifdef CONFIG_PHYS_ADDR_T_64BIT
 114        if (priv->hw_params->flags & GENET_HAS_40BITS)
 115                bcmgenet_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI);
 116#endif
 117}
 118
 119/* Combined address + length/status setter */
 120static inline void dmadesc_set(struct bcmgenet_priv *priv,
 121                               void __iomem *d, dma_addr_t addr, u32 val)
 122{
 123        dmadesc_set_addr(priv, d, addr);
 124        dmadesc_set_length_status(priv, d, val);
 125}
 126
 127static inline dma_addr_t dmadesc_get_addr(struct bcmgenet_priv *priv,
 128                                          void __iomem *d)
 129{
 130        dma_addr_t addr;
 131
 132        addr = bcmgenet_readl(d + DMA_DESC_ADDRESS_LO);
 133
 134        /* Register writes to GISB bus can take couple hundred nanoseconds
 135         * and are done for each packet, save these expensive writes unless
 136         * the platform is explicitly configured for 64-bits/LPAE.
 137         */
 138#ifdef CONFIG_PHYS_ADDR_T_64BIT
 139        if (priv->hw_params->flags & GENET_HAS_40BITS)
 140                addr |= (u64)bcmgenet_readl(d + DMA_DESC_ADDRESS_HI) << 32;
 141#endif
 142        return addr;
 143}
 144
 145#define GENET_VER_FMT   "%1d.%1d EPHY: 0x%04x"
 146
 147#define GENET_MSG_DEFAULT       (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
 148                                NETIF_MSG_LINK)
 149
 150static inline u32 bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv *priv)
 151{
 152        if (GENET_IS_V1(priv))
 153                return bcmgenet_rbuf_readl(priv, RBUF_FLUSH_CTRL_V1);
 154        else
 155                return bcmgenet_sys_readl(priv, SYS_RBUF_FLUSH_CTRL);
 156}
 157
 158static inline void bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
 159{
 160        if (GENET_IS_V1(priv))
 161                bcmgenet_rbuf_writel(priv, val, RBUF_FLUSH_CTRL_V1);
 162        else
 163                bcmgenet_sys_writel(priv, val, SYS_RBUF_FLUSH_CTRL);
 164}
 165
 166/* These macros are defined to deal with register map change
 167 * between GENET1.1 and GENET2. Only those currently being used
 168 * by driver are defined.
 169 */
 170static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv)
 171{
 172        if (GENET_IS_V1(priv))
 173                return bcmgenet_rbuf_readl(priv, TBUF_CTRL_V1);
 174        else
 175                return bcmgenet_readl(priv->base +
 176                                      priv->hw_params->tbuf_offset + TBUF_CTRL);
 177}
 178
 179static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
 180{
 181        if (GENET_IS_V1(priv))
 182                bcmgenet_rbuf_writel(priv, val, TBUF_CTRL_V1);
 183        else
 184                bcmgenet_writel(val, priv->base +
 185                                priv->hw_params->tbuf_offset + TBUF_CTRL);
 186}
 187
 188static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv)
 189{
 190        if (GENET_IS_V1(priv))
 191                return bcmgenet_rbuf_readl(priv, TBUF_BP_MC_V1);
 192        else
 193                return bcmgenet_readl(priv->base +
 194                                      priv->hw_params->tbuf_offset + TBUF_BP_MC);
 195}
 196
 197static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val)
 198{
 199        if (GENET_IS_V1(priv))
 200                bcmgenet_rbuf_writel(priv, val, TBUF_BP_MC_V1);
 201        else
 202                bcmgenet_writel(val, priv->base +
 203                                priv->hw_params->tbuf_offset + TBUF_BP_MC);
 204}
 205
 206/* RX/TX DMA register accessors */
 207enum dma_reg {
 208        DMA_RING_CFG = 0,
 209        DMA_CTRL,
 210        DMA_STATUS,
 211        DMA_SCB_BURST_SIZE,
 212        DMA_ARB_CTRL,
 213        DMA_PRIORITY_0,
 214        DMA_PRIORITY_1,
 215        DMA_PRIORITY_2,
 216        DMA_INDEX2RING_0,
 217        DMA_INDEX2RING_1,
 218        DMA_INDEX2RING_2,
 219        DMA_INDEX2RING_3,
 220        DMA_INDEX2RING_4,
 221        DMA_INDEX2RING_5,
 222        DMA_INDEX2RING_6,
 223        DMA_INDEX2RING_7,
 224        DMA_RING0_TIMEOUT,
 225        DMA_RING1_TIMEOUT,
 226        DMA_RING2_TIMEOUT,
 227        DMA_RING3_TIMEOUT,
 228        DMA_RING4_TIMEOUT,
 229        DMA_RING5_TIMEOUT,
 230        DMA_RING6_TIMEOUT,
 231        DMA_RING7_TIMEOUT,
 232        DMA_RING8_TIMEOUT,
 233        DMA_RING9_TIMEOUT,
 234        DMA_RING10_TIMEOUT,
 235        DMA_RING11_TIMEOUT,
 236        DMA_RING12_TIMEOUT,
 237        DMA_RING13_TIMEOUT,
 238        DMA_RING14_TIMEOUT,
 239        DMA_RING15_TIMEOUT,
 240        DMA_RING16_TIMEOUT,
 241};
 242
 243static const u8 bcmgenet_dma_regs_v3plus[] = {
 244        [DMA_RING_CFG]          = 0x00,
 245        [DMA_CTRL]              = 0x04,
 246        [DMA_STATUS]            = 0x08,
 247        [DMA_SCB_BURST_SIZE]    = 0x0C,
 248        [DMA_ARB_CTRL]          = 0x2C,
 249        [DMA_PRIORITY_0]        = 0x30,
 250        [DMA_PRIORITY_1]        = 0x34,
 251        [DMA_PRIORITY_2]        = 0x38,
 252        [DMA_RING0_TIMEOUT]     = 0x2C,
 253        [DMA_RING1_TIMEOUT]     = 0x30,
 254        [DMA_RING2_TIMEOUT]     = 0x34,
 255        [DMA_RING3_TIMEOUT]     = 0x38,
 256        [DMA_RING4_TIMEOUT]     = 0x3c,
 257        [DMA_RING5_TIMEOUT]     = 0x40,
 258        [DMA_RING6_TIMEOUT]     = 0x44,
 259        [DMA_RING7_TIMEOUT]     = 0x48,
 260        [DMA_RING8_TIMEOUT]     = 0x4c,
 261        [DMA_RING9_TIMEOUT]     = 0x50,
 262        [DMA_RING10_TIMEOUT]    = 0x54,
 263        [DMA_RING11_TIMEOUT]    = 0x58,
 264        [DMA_RING12_TIMEOUT]    = 0x5c,
 265        [DMA_RING13_TIMEOUT]    = 0x60,
 266        [DMA_RING14_TIMEOUT]    = 0x64,
 267        [DMA_RING15_TIMEOUT]    = 0x68,
 268        [DMA_RING16_TIMEOUT]    = 0x6C,
 269        [DMA_INDEX2RING_0]      = 0x70,
 270        [DMA_INDEX2RING_1]      = 0x74,
 271        [DMA_INDEX2RING_2]      = 0x78,
 272        [DMA_INDEX2RING_3]      = 0x7C,
 273        [DMA_INDEX2RING_4]      = 0x80,
 274        [DMA_INDEX2RING_5]      = 0x84,
 275        [DMA_INDEX2RING_6]      = 0x88,
 276        [DMA_INDEX2RING_7]      = 0x8C,
 277};
 278
 279static const u8 bcmgenet_dma_regs_v2[] = {
 280        [DMA_RING_CFG]          = 0x00,
 281        [DMA_CTRL]              = 0x04,
 282        [DMA_STATUS]            = 0x08,
 283        [DMA_SCB_BURST_SIZE]    = 0x0C,
 284        [DMA_ARB_CTRL]          = 0x30,
 285        [DMA_PRIORITY_0]        = 0x34,
 286        [DMA_PRIORITY_1]        = 0x38,
 287        [DMA_PRIORITY_2]        = 0x3C,
 288        [DMA_RING0_TIMEOUT]     = 0x2C,
 289        [DMA_RING1_TIMEOUT]     = 0x30,
 290        [DMA_RING2_TIMEOUT]     = 0x34,
 291        [DMA_RING3_TIMEOUT]     = 0x38,
 292        [DMA_RING4_TIMEOUT]     = 0x3c,
 293        [DMA_RING5_TIMEOUT]     = 0x40,
 294        [DMA_RING6_TIMEOUT]     = 0x44,
 295        [DMA_RING7_TIMEOUT]     = 0x48,
 296        [DMA_RING8_TIMEOUT]     = 0x4c,
 297        [DMA_RING9_TIMEOUT]     = 0x50,
 298        [DMA_RING10_TIMEOUT]    = 0x54,
 299        [DMA_RING11_TIMEOUT]    = 0x58,
 300        [DMA_RING12_TIMEOUT]    = 0x5c,
 301        [DMA_RING13_TIMEOUT]    = 0x60,
 302        [DMA_RING14_TIMEOUT]    = 0x64,
 303        [DMA_RING15_TIMEOUT]    = 0x68,
 304        [DMA_RING16_TIMEOUT]    = 0x6C,
 305};
 306
 307static const u8 bcmgenet_dma_regs_v1[] = {
 308        [DMA_CTRL]              = 0x00,
 309        [DMA_STATUS]            = 0x04,
 310        [DMA_SCB_BURST_SIZE]    = 0x0C,
 311        [DMA_ARB_CTRL]          = 0x30,
 312        [DMA_PRIORITY_0]        = 0x34,
 313        [DMA_PRIORITY_1]        = 0x38,
 314        [DMA_PRIORITY_2]        = 0x3C,
 315        [DMA_RING0_TIMEOUT]     = 0x2C,
 316        [DMA_RING1_TIMEOUT]     = 0x30,
 317        [DMA_RING2_TIMEOUT]     = 0x34,
 318        [DMA_RING3_TIMEOUT]     = 0x38,
 319        [DMA_RING4_TIMEOUT]     = 0x3c,
 320        [DMA_RING5_TIMEOUT]     = 0x40,
 321        [DMA_RING6_TIMEOUT]     = 0x44,
 322        [DMA_RING7_TIMEOUT]     = 0x48,
 323        [DMA_RING8_TIMEOUT]     = 0x4c,
 324        [DMA_RING9_TIMEOUT]     = 0x50,
 325        [DMA_RING10_TIMEOUT]    = 0x54,
 326        [DMA_RING11_TIMEOUT]    = 0x58,
 327        [DMA_RING12_TIMEOUT]    = 0x5c,
 328        [DMA_RING13_TIMEOUT]    = 0x60,
 329        [DMA_RING14_TIMEOUT]    = 0x64,
 330        [DMA_RING15_TIMEOUT]    = 0x68,
 331        [DMA_RING16_TIMEOUT]    = 0x6C,
 332};
 333
 334/* Set at runtime once bcmgenet version is known */
 335static const u8 *bcmgenet_dma_regs;
 336
 337static inline struct bcmgenet_priv *dev_to_priv(struct device *dev)
 338{
 339        return netdev_priv(dev_get_drvdata(dev));
 340}
 341
 342static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv,
 343                                      enum dma_reg r)
 344{
 345        return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
 346                              DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 347}
 348
 349static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv,
 350                                        u32 val, enum dma_reg r)
 351{
 352        bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
 353                        DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 354}
 355
 356static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv,
 357                                      enum dma_reg r)
 358{
 359        return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
 360                              DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 361}
 362
 363static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv,
 364                                        u32 val, enum dma_reg r)
 365{
 366        bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
 367                        DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 368}
 369
 370/* RDMA/TDMA ring registers and accessors
 371 * we merge the common fields and just prefix with T/D the registers
 372 * having different meaning depending on the direction
 373 */
 374enum dma_ring_reg {
 375        TDMA_READ_PTR = 0,
 376        RDMA_WRITE_PTR = TDMA_READ_PTR,
 377        TDMA_READ_PTR_HI,
 378        RDMA_WRITE_PTR_HI = TDMA_READ_PTR_HI,
 379        TDMA_CONS_INDEX,
 380        RDMA_PROD_INDEX = TDMA_CONS_INDEX,
 381        TDMA_PROD_INDEX,
 382        RDMA_CONS_INDEX = TDMA_PROD_INDEX,
 383        DMA_RING_BUF_SIZE,
 384        DMA_START_ADDR,
 385        DMA_START_ADDR_HI,
 386        DMA_END_ADDR,
 387        DMA_END_ADDR_HI,
 388        DMA_MBUF_DONE_THRESH,
 389        TDMA_FLOW_PERIOD,
 390        RDMA_XON_XOFF_THRESH = TDMA_FLOW_PERIOD,
 391        TDMA_WRITE_PTR,
 392        RDMA_READ_PTR = TDMA_WRITE_PTR,
 393        TDMA_WRITE_PTR_HI,
 394        RDMA_READ_PTR_HI = TDMA_WRITE_PTR_HI
 395};
 396
 397/* GENET v4 supports 40-bits pointer addressing
 398 * for obvious reasons the LO and HI word parts
 399 * are contiguous, but this offsets the other
 400 * registers.
 401 */
 402static const u8 genet_dma_ring_regs_v4[] = {
 403        [TDMA_READ_PTR]                 = 0x00,
 404        [TDMA_READ_PTR_HI]              = 0x04,
 405        [TDMA_CONS_INDEX]               = 0x08,
 406        [TDMA_PROD_INDEX]               = 0x0C,
 407        [DMA_RING_BUF_SIZE]             = 0x10,
 408        [DMA_START_ADDR]                = 0x14,
 409        [DMA_START_ADDR_HI]             = 0x18,
 410        [DMA_END_ADDR]                  = 0x1C,
 411        [DMA_END_ADDR_HI]               = 0x20,
 412        [DMA_MBUF_DONE_THRESH]          = 0x24,
 413        [TDMA_FLOW_PERIOD]              = 0x28,
 414        [TDMA_WRITE_PTR]                = 0x2C,
 415        [TDMA_WRITE_PTR_HI]             = 0x30,
 416};
 417
 418static const u8 genet_dma_ring_regs_v123[] = {
 419        [TDMA_READ_PTR]                 = 0x00,
 420        [TDMA_CONS_INDEX]               = 0x04,
 421        [TDMA_PROD_INDEX]               = 0x08,
 422        [DMA_RING_BUF_SIZE]             = 0x0C,
 423        [DMA_START_ADDR]                = 0x10,
 424        [DMA_END_ADDR]                  = 0x14,
 425        [DMA_MBUF_DONE_THRESH]          = 0x18,
 426        [TDMA_FLOW_PERIOD]              = 0x1C,
 427        [TDMA_WRITE_PTR]                = 0x20,
 428};
 429
 430/* Set at runtime once GENET version is known */
 431static const u8 *genet_dma_ring_regs;
 432
 433static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv,
 434                                           unsigned int ring,
 435                                           enum dma_ring_reg r)
 436{
 437        return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
 438                              (DMA_RING_SIZE * ring) +
 439                              genet_dma_ring_regs[r]);
 440}
 441
 442static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv,
 443                                             unsigned int ring, u32 val,
 444                                             enum dma_ring_reg r)
 445{
 446        bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
 447                        (DMA_RING_SIZE * ring) +
 448                        genet_dma_ring_regs[r]);
 449}
 450
 451static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv,
 452                                           unsigned int ring,
 453                                           enum dma_ring_reg r)
 454{
 455        return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
 456                              (DMA_RING_SIZE * ring) +
 457                              genet_dma_ring_regs[r]);
 458}
 459
 460static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv,
 461                                             unsigned int ring, u32 val,
 462                                             enum dma_ring_reg r)
 463{
 464        bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
 465                        (DMA_RING_SIZE * ring) +
 466                        genet_dma_ring_regs[r]);
 467}
 468
 469static int bcmgenet_begin(struct net_device *dev)
 470{
 471        struct bcmgenet_priv *priv = netdev_priv(dev);
 472
 473        /* Turn on the clock */
 474        return clk_prepare_enable(priv->clk);
 475}
 476
 477static void bcmgenet_complete(struct net_device *dev)
 478{
 479        struct bcmgenet_priv *priv = netdev_priv(dev);
 480
 481        /* Turn off the clock */
 482        clk_disable_unprepare(priv->clk);
 483}
 484
 485static int bcmgenet_get_link_ksettings(struct net_device *dev,
 486                                       struct ethtool_link_ksettings *cmd)
 487{
 488        if (!netif_running(dev))
 489                return -EINVAL;
 490
 491        if (!dev->phydev)
 492                return -ENODEV;
 493
 494        phy_ethtool_ksettings_get(dev->phydev, cmd);
 495
 496        return 0;
 497}
 498
 499static int bcmgenet_set_link_ksettings(struct net_device *dev,
 500                                       const struct ethtool_link_ksettings *cmd)
 501{
 502        if (!netif_running(dev))
 503                return -EINVAL;
 504
 505        if (!dev->phydev)
 506                return -ENODEV;
 507
 508        return phy_ethtool_ksettings_set(dev->phydev, cmd);
 509}
 510
 511static int bcmgenet_set_rx_csum(struct net_device *dev,
 512                                netdev_features_t wanted)
 513{
 514        struct bcmgenet_priv *priv = netdev_priv(dev);
 515        u32 rbuf_chk_ctrl;
 516        bool rx_csum_en;
 517
 518        rx_csum_en = !!(wanted & NETIF_F_RXCSUM);
 519
 520        rbuf_chk_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CHK_CTRL);
 521
 522        /* enable rx checksumming */
 523        if (rx_csum_en)
 524                rbuf_chk_ctrl |= RBUF_RXCHK_EN;
 525        else
 526                rbuf_chk_ctrl &= ~RBUF_RXCHK_EN;
 527        priv->desc_rxchk_en = rx_csum_en;
 528
 529        /* If UniMAC forwards CRC, we need to skip over it to get
 530         * a valid CHK bit to be set in the per-packet status word
 531        */
 532        if (rx_csum_en && priv->crc_fwd_en)
 533                rbuf_chk_ctrl |= RBUF_SKIP_FCS;
 534        else
 535                rbuf_chk_ctrl &= ~RBUF_SKIP_FCS;
 536
 537        bcmgenet_rbuf_writel(priv, rbuf_chk_ctrl, RBUF_CHK_CTRL);
 538
 539        return 0;
 540}
 541
 542static int bcmgenet_set_tx_csum(struct net_device *dev,
 543                                netdev_features_t wanted)
 544{
 545        struct bcmgenet_priv *priv = netdev_priv(dev);
 546        bool desc_64b_en;
 547        u32 tbuf_ctrl, rbuf_ctrl;
 548
 549        tbuf_ctrl = bcmgenet_tbuf_ctrl_get(priv);
 550        rbuf_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
 551
 552        desc_64b_en = !!(wanted & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM));
 553
 554        /* enable 64 bytes descriptor in both directions (RBUF and TBUF) */
 555        if (desc_64b_en) {
 556                tbuf_ctrl |= RBUF_64B_EN;
 557                rbuf_ctrl |= RBUF_64B_EN;
 558        } else {
 559                tbuf_ctrl &= ~RBUF_64B_EN;
 560                rbuf_ctrl &= ~RBUF_64B_EN;
 561        }
 562        priv->desc_64b_en = desc_64b_en;
 563
 564        bcmgenet_tbuf_ctrl_set(priv, tbuf_ctrl);
 565        bcmgenet_rbuf_writel(priv, rbuf_ctrl, RBUF_CTRL);
 566
 567        return 0;
 568}
 569
 570static int bcmgenet_set_features(struct net_device *dev,
 571                                 netdev_features_t features)
 572{
 573        netdev_features_t changed = features ^ dev->features;
 574        netdev_features_t wanted = dev->wanted_features;
 575        int ret = 0;
 576
 577        if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))
 578                ret = bcmgenet_set_tx_csum(dev, wanted);
 579        if (changed & (NETIF_F_RXCSUM))
 580                ret = bcmgenet_set_rx_csum(dev, wanted);
 581
 582        return ret;
 583}
 584
 585static u32 bcmgenet_get_msglevel(struct net_device *dev)
 586{
 587        struct bcmgenet_priv *priv = netdev_priv(dev);
 588
 589        return priv->msg_enable;
 590}
 591
 592static void bcmgenet_set_msglevel(struct net_device *dev, u32 level)
 593{
 594        struct bcmgenet_priv *priv = netdev_priv(dev);
 595
 596        priv->msg_enable = level;
 597}
 598
 599static int bcmgenet_get_coalesce(struct net_device *dev,
 600                                 struct ethtool_coalesce *ec)
 601{
 602        struct bcmgenet_priv *priv = netdev_priv(dev);
 603        struct bcmgenet_rx_ring *ring;
 604        unsigned int i;
 605
 606        ec->tx_max_coalesced_frames =
 607                bcmgenet_tdma_ring_readl(priv, DESC_INDEX,
 608                                         DMA_MBUF_DONE_THRESH);
 609        ec->rx_max_coalesced_frames =
 610                bcmgenet_rdma_ring_readl(priv, DESC_INDEX,
 611                                         DMA_MBUF_DONE_THRESH);
 612        ec->rx_coalesce_usecs =
 613                bcmgenet_rdma_readl(priv, DMA_RING16_TIMEOUT) * 8192 / 1000;
 614
 615        for (i = 0; i < priv->hw_params->rx_queues; i++) {
 616                ring = &priv->rx_rings[i];
 617                ec->use_adaptive_rx_coalesce |= ring->dim.use_dim;
 618        }
 619        ring = &priv->rx_rings[DESC_INDEX];
 620        ec->use_adaptive_rx_coalesce |= ring->dim.use_dim;
 621
 622        return 0;
 623}
 624
 625static void bcmgenet_set_rx_coalesce(struct bcmgenet_rx_ring *ring,
 626                                     u32 usecs, u32 pkts)
 627{
 628        struct bcmgenet_priv *priv = ring->priv;
 629        unsigned int i = ring->index;
 630        u32 reg;
 631
 632        bcmgenet_rdma_ring_writel(priv, i, pkts, DMA_MBUF_DONE_THRESH);
 633
 634        reg = bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT + i);
 635        reg &= ~DMA_TIMEOUT_MASK;
 636        reg |= DIV_ROUND_UP(usecs * 1000, 8192);
 637        bcmgenet_rdma_writel(priv, reg, DMA_RING0_TIMEOUT + i);
 638}
 639
 640static void bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring *ring,
 641                                          struct ethtool_coalesce *ec)
 642{
 643        struct dim_cq_moder moder;
 644        u32 usecs, pkts;
 645
 646        ring->rx_coalesce_usecs = ec->rx_coalesce_usecs;
 647        ring->rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
 648        usecs = ring->rx_coalesce_usecs;
 649        pkts = ring->rx_max_coalesced_frames;
 650
 651        if (ec->use_adaptive_rx_coalesce && !ring->dim.use_dim) {
 652                moder = net_dim_get_def_rx_moderation(ring->dim.dim.mode);
 653                usecs = moder.usec;
 654                pkts = moder.pkts;
 655        }
 656
 657        ring->dim.use_dim = ec->use_adaptive_rx_coalesce;
 658        bcmgenet_set_rx_coalesce(ring, usecs, pkts);
 659}
 660
 661static int bcmgenet_set_coalesce(struct net_device *dev,
 662                                 struct ethtool_coalesce *ec)
 663{
 664        struct bcmgenet_priv *priv = netdev_priv(dev);
 665        unsigned int i;
 666
 667        /* Base system clock is 125Mhz, DMA timeout is this reference clock
 668         * divided by 1024, which yields roughly 8.192us, our maximum value
 669         * has to fit in the DMA_TIMEOUT_MASK (16 bits)
 670         */
 671        if (ec->tx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
 672            ec->tx_max_coalesced_frames == 0 ||
 673            ec->rx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
 674            ec->rx_coalesce_usecs > (DMA_TIMEOUT_MASK * 8) + 1)
 675                return -EINVAL;
 676
 677        if (ec->rx_coalesce_usecs == 0 && ec->rx_max_coalesced_frames == 0)
 678                return -EINVAL;
 679
 680        /* GENET TDMA hardware does not support a configurable timeout, but will
 681         * always generate an interrupt either after MBDONE packets have been
 682         * transmitted, or when the ring is empty.
 683         */
 684        if (ec->tx_coalesce_usecs || ec->tx_coalesce_usecs_high ||
 685            ec->tx_coalesce_usecs_irq || ec->tx_coalesce_usecs_low ||
 686            ec->use_adaptive_tx_coalesce)
 687                return -EOPNOTSUPP;
 688
 689        /* Program all TX queues with the same values, as there is no
 690         * ethtool knob to do coalescing on a per-queue basis
 691         */
 692        for (i = 0; i < priv->hw_params->tx_queues; i++)
 693                bcmgenet_tdma_ring_writel(priv, i,
 694                                          ec->tx_max_coalesced_frames,
 695                                          DMA_MBUF_DONE_THRESH);
 696        bcmgenet_tdma_ring_writel(priv, DESC_INDEX,
 697                                  ec->tx_max_coalesced_frames,
 698                                  DMA_MBUF_DONE_THRESH);
 699
 700        for (i = 0; i < priv->hw_params->rx_queues; i++)
 701                bcmgenet_set_ring_rx_coalesce(&priv->rx_rings[i], ec);
 702        bcmgenet_set_ring_rx_coalesce(&priv->rx_rings[DESC_INDEX], ec);
 703
 704        return 0;
 705}
 706
 707/* standard ethtool support functions. */
 708enum bcmgenet_stat_type {
 709        BCMGENET_STAT_NETDEV = -1,
 710        BCMGENET_STAT_MIB_RX,
 711        BCMGENET_STAT_MIB_TX,
 712        BCMGENET_STAT_RUNT,
 713        BCMGENET_STAT_MISC,
 714        BCMGENET_STAT_SOFT,
 715};
 716
 717struct bcmgenet_stats {
 718        char stat_string[ETH_GSTRING_LEN];
 719        int stat_sizeof;
 720        int stat_offset;
 721        enum bcmgenet_stat_type type;
 722        /* reg offset from UMAC base for misc counters */
 723        u16 reg_offset;
 724};
 725
 726#define STAT_NETDEV(m) { \
 727        .stat_string = __stringify(m), \
 728        .stat_sizeof = sizeof(((struct net_device_stats *)0)->m), \
 729        .stat_offset = offsetof(struct net_device_stats, m), \
 730        .type = BCMGENET_STAT_NETDEV, \
 731}
 732
 733#define STAT_GENET_MIB(str, m, _type) { \
 734        .stat_string = str, \
 735        .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
 736        .stat_offset = offsetof(struct bcmgenet_priv, m), \
 737        .type = _type, \
 738}
 739
 740#define STAT_GENET_MIB_RX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_RX)
 741#define STAT_GENET_MIB_TX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_TX)
 742#define STAT_GENET_RUNT(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_RUNT)
 743#define STAT_GENET_SOFT_MIB(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_SOFT)
 744
 745#define STAT_GENET_MISC(str, m, offset) { \
 746        .stat_string = str, \
 747        .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
 748        .stat_offset = offsetof(struct bcmgenet_priv, m), \
 749        .type = BCMGENET_STAT_MISC, \
 750        .reg_offset = offset, \
 751}
 752
 753#define STAT_GENET_Q(num) \
 754        STAT_GENET_SOFT_MIB("txq" __stringify(num) "_packets", \
 755                        tx_rings[num].packets), \
 756        STAT_GENET_SOFT_MIB("txq" __stringify(num) "_bytes", \
 757                        tx_rings[num].bytes), \
 758        STAT_GENET_SOFT_MIB("rxq" __stringify(num) "_bytes", \
 759                        rx_rings[num].bytes),    \
 760        STAT_GENET_SOFT_MIB("rxq" __stringify(num) "_packets", \
 761                        rx_rings[num].packets), \
 762        STAT_GENET_SOFT_MIB("rxq" __stringify(num) "_errors", \
 763                        rx_rings[num].errors), \
 764        STAT_GENET_SOFT_MIB("rxq" __stringify(num) "_dropped", \
 765                        rx_rings[num].dropped)
 766
 767/* There is a 0xC gap between the end of RX and beginning of TX stats and then
 768 * between the end of TX stats and the beginning of the RX RUNT
 769 */
 770#define BCMGENET_STAT_OFFSET    0xc
 771
 772/* Hardware counters must be kept in sync because the order/offset
 773 * is important here (order in structure declaration = order in hardware)
 774 */
 775static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = {
 776        /* general stats */
 777        STAT_NETDEV(rx_packets),
 778        STAT_NETDEV(tx_packets),
 779        STAT_NETDEV(rx_bytes),
 780        STAT_NETDEV(tx_bytes),
 781        STAT_NETDEV(rx_errors),
 782        STAT_NETDEV(tx_errors),
 783        STAT_NETDEV(rx_dropped),
 784        STAT_NETDEV(tx_dropped),
 785        STAT_NETDEV(multicast),
 786        /* UniMAC RSV counters */
 787        STAT_GENET_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64),
 788        STAT_GENET_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127),
 789        STAT_GENET_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255),
 790        STAT_GENET_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511),
 791        STAT_GENET_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023),
 792        STAT_GENET_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518),
 793        STAT_GENET_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv),
 794        STAT_GENET_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047),
 795        STAT_GENET_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095),
 796        STAT_GENET_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216),
 797        STAT_GENET_MIB_RX("rx_pkts", mib.rx.pkt),
 798        STAT_GENET_MIB_RX("rx_bytes", mib.rx.bytes),
 799        STAT_GENET_MIB_RX("rx_multicast", mib.rx.mca),
 800        STAT_GENET_MIB_RX("rx_broadcast", mib.rx.bca),
 801        STAT_GENET_MIB_RX("rx_fcs", mib.rx.fcs),
 802        STAT_GENET_MIB_RX("rx_control", mib.rx.cf),
 803        STAT_GENET_MIB_RX("rx_pause", mib.rx.pf),
 804        STAT_GENET_MIB_RX("rx_unknown", mib.rx.uo),
 805        STAT_GENET_MIB_RX("rx_align", mib.rx.aln),
 806        STAT_GENET_MIB_RX("rx_outrange", mib.rx.flr),
 807        STAT_GENET_MIB_RX("rx_code", mib.rx.cde),
 808        STAT_GENET_MIB_RX("rx_carrier", mib.rx.fcr),
 809        STAT_GENET_MIB_RX("rx_oversize", mib.rx.ovr),
 810        STAT_GENET_MIB_RX("rx_jabber", mib.rx.jbr),
 811        STAT_GENET_MIB_RX("rx_mtu_err", mib.rx.mtue),
 812        STAT_GENET_MIB_RX("rx_good_pkts", mib.rx.pok),
 813        STAT_GENET_MIB_RX("rx_unicast", mib.rx.uc),
 814        STAT_GENET_MIB_RX("rx_ppp", mib.rx.ppp),
 815        STAT_GENET_MIB_RX("rx_crc", mib.rx.rcrc),
 816        /* UniMAC TSV counters */
 817        STAT_GENET_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64),
 818        STAT_GENET_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127),
 819        STAT_GENET_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255),
 820        STAT_GENET_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511),
 821        STAT_GENET_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023),
 822        STAT_GENET_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518),
 823        STAT_GENET_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv),
 824        STAT_GENET_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047),
 825        STAT_GENET_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095),
 826        STAT_GENET_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216),
 827        STAT_GENET_MIB_TX("tx_pkts", mib.tx.pkts),
 828        STAT_GENET_MIB_TX("tx_multicast", mib.tx.mca),
 829        STAT_GENET_MIB_TX("tx_broadcast", mib.tx.bca),
 830        STAT_GENET_MIB_TX("tx_pause", mib.tx.pf),
 831        STAT_GENET_MIB_TX("tx_control", mib.tx.cf),
 832        STAT_GENET_MIB_TX("tx_fcs_err", mib.tx.fcs),
 833        STAT_GENET_MIB_TX("tx_oversize", mib.tx.ovr),
 834        STAT_GENET_MIB_TX("tx_defer", mib.tx.drf),
 835        STAT_GENET_MIB_TX("tx_excess_defer", mib.tx.edf),
 836        STAT_GENET_MIB_TX("tx_single_col", mib.tx.scl),
 837        STAT_GENET_MIB_TX("tx_multi_col", mib.tx.mcl),
 838        STAT_GENET_MIB_TX("tx_late_col", mib.tx.lcl),
 839        STAT_GENET_MIB_TX("tx_excess_col", mib.tx.ecl),
 840        STAT_GENET_MIB_TX("tx_frags", mib.tx.frg),
 841        STAT_GENET_MIB_TX("tx_total_col", mib.tx.ncl),
 842        STAT_GENET_MIB_TX("tx_jabber", mib.tx.jbr),
 843        STAT_GENET_MIB_TX("tx_bytes", mib.tx.bytes),
 844        STAT_GENET_MIB_TX("tx_good_pkts", mib.tx.pok),
 845        STAT_GENET_MIB_TX("tx_unicast", mib.tx.uc),
 846        /* UniMAC RUNT counters */
 847        STAT_GENET_RUNT("rx_runt_pkts", mib.rx_runt_cnt),
 848        STAT_GENET_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs),
 849        STAT_GENET_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align),
 850        STAT_GENET_RUNT("rx_runt_bytes", mib.rx_runt_bytes),
 851        /* Misc UniMAC counters */
 852        STAT_GENET_MISC("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt,
 853                        UMAC_RBUF_OVFL_CNT_V1),
 854        STAT_GENET_MISC("rbuf_err_cnt", mib.rbuf_err_cnt,
 855                        UMAC_RBUF_ERR_CNT_V1),
 856        STAT_GENET_MISC("mdf_err_cnt", mib.mdf_err_cnt, UMAC_MDF_ERR_CNT),
 857        STAT_GENET_SOFT_MIB("alloc_rx_buff_failed", mib.alloc_rx_buff_failed),
 858        STAT_GENET_SOFT_MIB("rx_dma_failed", mib.rx_dma_failed),
 859        STAT_GENET_SOFT_MIB("tx_dma_failed", mib.tx_dma_failed),
 860        /* Per TX queues */
 861        STAT_GENET_Q(0),
 862        STAT_GENET_Q(1),
 863        STAT_GENET_Q(2),
 864        STAT_GENET_Q(3),
 865        STAT_GENET_Q(16),
 866};
 867
 868#define BCMGENET_STATS_LEN      ARRAY_SIZE(bcmgenet_gstrings_stats)
 869
 870static void bcmgenet_get_drvinfo(struct net_device *dev,
 871                                 struct ethtool_drvinfo *info)
 872{
 873        strlcpy(info->driver, "bcmgenet", sizeof(info->driver));
 874        strlcpy(info->version, "v2.0", sizeof(info->version));
 875}
 876
 877static int bcmgenet_get_sset_count(struct net_device *dev, int string_set)
 878{
 879        switch (string_set) {
 880        case ETH_SS_STATS:
 881                return BCMGENET_STATS_LEN;
 882        default:
 883                return -EOPNOTSUPP;
 884        }
 885}
 886
 887static void bcmgenet_get_strings(struct net_device *dev, u32 stringset,
 888                                 u8 *data)
 889{
 890        int i;
 891
 892        switch (stringset) {
 893        case ETH_SS_STATS:
 894                for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 895                        memcpy(data + i * ETH_GSTRING_LEN,
 896                               bcmgenet_gstrings_stats[i].stat_string,
 897                               ETH_GSTRING_LEN);
 898                }
 899                break;
 900        }
 901}
 902
 903static u32 bcmgenet_update_stat_misc(struct bcmgenet_priv *priv, u16 offset)
 904{
 905        u16 new_offset;
 906        u32 val;
 907
 908        switch (offset) {
 909        case UMAC_RBUF_OVFL_CNT_V1:
 910                if (GENET_IS_V2(priv))
 911                        new_offset = RBUF_OVFL_CNT_V2;
 912                else
 913                        new_offset = RBUF_OVFL_CNT_V3PLUS;
 914
 915                val = bcmgenet_rbuf_readl(priv, new_offset);
 916                /* clear if overflowed */
 917                if (val == ~0)
 918                        bcmgenet_rbuf_writel(priv, 0, new_offset);
 919                break;
 920        case UMAC_RBUF_ERR_CNT_V1:
 921                if (GENET_IS_V2(priv))
 922                        new_offset = RBUF_ERR_CNT_V2;
 923                else
 924                        new_offset = RBUF_ERR_CNT_V3PLUS;
 925
 926                val = bcmgenet_rbuf_readl(priv, new_offset);
 927                /* clear if overflowed */
 928                if (val == ~0)
 929                        bcmgenet_rbuf_writel(priv, 0, new_offset);
 930                break;
 931        default:
 932                val = bcmgenet_umac_readl(priv, offset);
 933                /* clear if overflowed */
 934                if (val == ~0)
 935                        bcmgenet_umac_writel(priv, 0, offset);
 936                break;
 937        }
 938
 939        return val;
 940}
 941
 942static void bcmgenet_update_mib_counters(struct bcmgenet_priv *priv)
 943{
 944        int i, j = 0;
 945
 946        for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 947                const struct bcmgenet_stats *s;
 948                u8 offset = 0;
 949                u32 val = 0;
 950                char *p;
 951
 952                s = &bcmgenet_gstrings_stats[i];
 953                switch (s->type) {
 954                case BCMGENET_STAT_NETDEV:
 955                case BCMGENET_STAT_SOFT:
 956                        continue;
 957                case BCMGENET_STAT_RUNT:
 958                        offset += BCMGENET_STAT_OFFSET;
 959                        /* fall through */
 960                case BCMGENET_STAT_MIB_TX:
 961                        offset += BCMGENET_STAT_OFFSET;
 962                        /* fall through */
 963                case BCMGENET_STAT_MIB_RX:
 964                        val = bcmgenet_umac_readl(priv,
 965                                                  UMAC_MIB_START + j + offset);
 966                        offset = 0;     /* Reset Offset */
 967                        break;
 968                case BCMGENET_STAT_MISC:
 969                        if (GENET_IS_V1(priv)) {
 970                                val = bcmgenet_umac_readl(priv, s->reg_offset);
 971                                /* clear if overflowed */
 972                                if (val == ~0)
 973                                        bcmgenet_umac_writel(priv, 0,
 974                                                             s->reg_offset);
 975                        } else {
 976                                val = bcmgenet_update_stat_misc(priv,
 977                                                                s->reg_offset);
 978                        }
 979                        break;
 980                }
 981
 982                j += s->stat_sizeof;
 983                p = (char *)priv + s->stat_offset;
 984                *(u32 *)p = val;
 985        }
 986}
 987
 988static void bcmgenet_get_ethtool_stats(struct net_device *dev,
 989                                       struct ethtool_stats *stats,
 990                                       u64 *data)
 991{
 992        struct bcmgenet_priv *priv = netdev_priv(dev);
 993        int i;
 994
 995        if (netif_running(dev))
 996                bcmgenet_update_mib_counters(priv);
 997
 998        for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 999                const struct bcmgenet_stats *s;
1000                char *p;
1001
1002                s = &bcmgenet_gstrings_stats[i];
1003                if (s->type == BCMGENET_STAT_NETDEV)
1004                        p = (char *)&dev->stats;
1005                else
1006                        p = (char *)priv;
1007                p += s->stat_offset;
1008                if (sizeof(unsigned long) != sizeof(u32) &&
1009                    s->stat_sizeof == sizeof(unsigned long))
1010                        data[i] = *(unsigned long *)p;
1011                else
1012                        data[i] = *(u32 *)p;
1013        }
1014}
1015
1016static void bcmgenet_eee_enable_set(struct net_device *dev, bool enable)
1017{
1018        struct bcmgenet_priv *priv = netdev_priv(dev);
1019        u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL;
1020        u32 reg;
1021
1022        if (enable && !priv->clk_eee_enabled) {
1023                clk_prepare_enable(priv->clk_eee);
1024                priv->clk_eee_enabled = true;
1025        }
1026
1027        reg = bcmgenet_umac_readl(priv, UMAC_EEE_CTRL);
1028        if (enable)
1029                reg |= EEE_EN;
1030        else
1031                reg &= ~EEE_EN;
1032        bcmgenet_umac_writel(priv, reg, UMAC_EEE_CTRL);
1033
1034        /* Enable EEE and switch to a 27Mhz clock automatically */
1035        reg = bcmgenet_readl(priv->base + off);
1036        if (enable)
1037                reg |= TBUF_EEE_EN | TBUF_PM_EN;
1038        else
1039                reg &= ~(TBUF_EEE_EN | TBUF_PM_EN);
1040        bcmgenet_writel(reg, priv->base + off);
1041
1042        /* Do the same for thing for RBUF */
1043        reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL);
1044        if (enable)
1045                reg |= RBUF_EEE_EN | RBUF_PM_EN;
1046        else
1047                reg &= ~(RBUF_EEE_EN | RBUF_PM_EN);
1048        bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL);
1049
1050        if (!enable && priv->clk_eee_enabled) {
1051                clk_disable_unprepare(priv->clk_eee);
1052                priv->clk_eee_enabled = false;
1053        }
1054
1055        priv->eee.eee_enabled = enable;
1056        priv->eee.eee_active = enable;
1057}
1058
1059static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_eee *e)
1060{
1061        struct bcmgenet_priv *priv = netdev_priv(dev);
1062        struct ethtool_eee *p = &priv->eee;
1063
1064        if (GENET_IS_V1(priv))
1065                return -EOPNOTSUPP;
1066
1067        if (!dev->phydev)
1068                return -ENODEV;
1069
1070        e->eee_enabled = p->eee_enabled;
1071        e->eee_active = p->eee_active;
1072        e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER);
1073
1074        return phy_ethtool_get_eee(dev->phydev, e);
1075}
1076
1077static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e)
1078{
1079        struct bcmgenet_priv *priv = netdev_priv(dev);
1080        struct ethtool_eee *p = &priv->eee;
1081        int ret = 0;
1082
1083        if (GENET_IS_V1(priv))
1084                return -EOPNOTSUPP;
1085
1086        if (!dev->phydev)
1087                return -ENODEV;
1088
1089        p->eee_enabled = e->eee_enabled;
1090
1091        if (!p->eee_enabled) {
1092                bcmgenet_eee_enable_set(dev, false);
1093        } else {
1094                ret = phy_init_eee(dev->phydev, 0);
1095                if (ret) {
1096                        netif_err(priv, hw, dev, "EEE initialization failed\n");
1097                        return ret;
1098                }
1099
1100                bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER);
1101                bcmgenet_eee_enable_set(dev, true);
1102        }
1103
1104        return phy_ethtool_set_eee(dev->phydev, e);
1105}
1106
1107/* standard ethtool support functions. */
1108static const struct ethtool_ops bcmgenet_ethtool_ops = {
1109        .begin                  = bcmgenet_begin,
1110        .complete               = bcmgenet_complete,
1111        .get_strings            = bcmgenet_get_strings,
1112        .get_sset_count         = bcmgenet_get_sset_count,
1113        .get_ethtool_stats      = bcmgenet_get_ethtool_stats,
1114        .get_drvinfo            = bcmgenet_get_drvinfo,
1115        .get_link               = ethtool_op_get_link,
1116        .get_msglevel           = bcmgenet_get_msglevel,
1117        .set_msglevel           = bcmgenet_set_msglevel,
1118        .get_wol                = bcmgenet_get_wol,
1119        .set_wol                = bcmgenet_set_wol,
1120        .get_eee                = bcmgenet_get_eee,
1121        .set_eee                = bcmgenet_set_eee,
1122        .nway_reset             = phy_ethtool_nway_reset,
1123        .get_coalesce           = bcmgenet_get_coalesce,
1124        .set_coalesce           = bcmgenet_set_coalesce,
1125        .get_link_ksettings     = bcmgenet_get_link_ksettings,
1126        .set_link_ksettings     = bcmgenet_set_link_ksettings,
1127        .get_ts_info            = ethtool_op_get_ts_info,
1128};
1129
1130/* Power down the unimac, based on mode. */
1131static int bcmgenet_power_down(struct bcmgenet_priv *priv,
1132                                enum bcmgenet_power_mode mode)
1133{
1134        int ret = 0;
1135        u32 reg;
1136
1137        switch (mode) {
1138        case GENET_POWER_CABLE_SENSE:
1139                phy_detach(priv->dev->phydev);
1140                break;
1141
1142        case GENET_POWER_WOL_MAGIC:
1143                ret = bcmgenet_wol_power_down_cfg(priv, mode);
1144                break;
1145
1146        case GENET_POWER_PASSIVE:
1147                /* Power down LED */
1148                if (priv->hw_params->flags & GENET_HAS_EXT) {
1149                        reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
1150                        if (GENET_IS_V5(priv))
1151                                reg |= EXT_PWR_DOWN_PHY_EN |
1152                                       EXT_PWR_DOWN_PHY_RD |
1153                                       EXT_PWR_DOWN_PHY_SD |
1154                                       EXT_PWR_DOWN_PHY_RX |
1155                                       EXT_PWR_DOWN_PHY_TX |
1156                                       EXT_IDDQ_GLBL_PWR;
1157                        else
1158                                reg |= EXT_PWR_DOWN_PHY;
1159
1160                        reg |= (EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS);
1161                        bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1162
1163                        bcmgenet_phy_power_set(priv->dev, false);
1164                }
1165                break;
1166        default:
1167                break;
1168        }
1169
1170        return ret;
1171}
1172
1173static void bcmgenet_power_up(struct bcmgenet_priv *priv,
1174                              enum bcmgenet_power_mode mode)
1175{
1176        u32 reg;
1177
1178        if (!(priv->hw_params->flags & GENET_HAS_EXT))
1179                return;
1180
1181        reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
1182
1183        switch (mode) {
1184        case GENET_POWER_PASSIVE:
1185                reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS);
1186                if (GENET_IS_V5(priv)) {
1187                        reg &= ~(EXT_PWR_DOWN_PHY_EN |
1188                                 EXT_PWR_DOWN_PHY_RD |
1189                                 EXT_PWR_DOWN_PHY_SD |
1190                                 EXT_PWR_DOWN_PHY_RX |
1191                                 EXT_PWR_DOWN_PHY_TX |
1192                                 EXT_IDDQ_GLBL_PWR);
1193                        reg |=   EXT_PHY_RESET;
1194                        bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1195                        mdelay(1);
1196
1197                        reg &=  ~EXT_PHY_RESET;
1198                } else {
1199                        reg &= ~EXT_PWR_DOWN_PHY;
1200                        reg |= EXT_PWR_DN_EN_LD;
1201                }
1202                bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1203                bcmgenet_phy_power_set(priv->dev, true);
1204                break;
1205
1206        case GENET_POWER_CABLE_SENSE:
1207                /* enable APD */
1208                if (!GENET_IS_V5(priv)) {
1209                        reg |= EXT_PWR_DN_EN_LD;
1210                        bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1211                }
1212                break;
1213        case GENET_POWER_WOL_MAGIC:
1214                bcmgenet_wol_power_up_cfg(priv, mode);
1215                return;
1216        default:
1217                break;
1218        }
1219}
1220
1221/* ioctl handle special commands that are not present in ethtool. */
1222static int bcmgenet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1223{
1224        if (!netif_running(dev))
1225                return -EINVAL;
1226
1227        if (!dev->phydev)
1228                return -ENODEV;
1229
1230        return phy_mii_ioctl(dev->phydev, rq, cmd);
1231}
1232
1233static struct enet_cb *bcmgenet_get_txcb(struct bcmgenet_priv *priv,
1234                                         struct bcmgenet_tx_ring *ring)
1235{
1236        struct enet_cb *tx_cb_ptr;
1237
1238        tx_cb_ptr = ring->cbs;
1239        tx_cb_ptr += ring->write_ptr - ring->cb_ptr;
1240
1241        /* Advancing local write pointer */
1242        if (ring->write_ptr == ring->end_ptr)
1243                ring->write_ptr = ring->cb_ptr;
1244        else
1245                ring->write_ptr++;
1246
1247        return tx_cb_ptr;
1248}
1249
1250static struct enet_cb *bcmgenet_put_txcb(struct bcmgenet_priv *priv,
1251                                         struct bcmgenet_tx_ring *ring)
1252{
1253        struct enet_cb *tx_cb_ptr;
1254
1255        tx_cb_ptr = ring->cbs;
1256        tx_cb_ptr += ring->write_ptr - ring->cb_ptr;
1257
1258        /* Rewinding local write pointer */
1259        if (ring->write_ptr == ring->cb_ptr)
1260                ring->write_ptr = ring->end_ptr;
1261        else
1262                ring->write_ptr--;
1263
1264        return tx_cb_ptr;
1265}
1266
1267static inline void bcmgenet_rx_ring16_int_disable(struct bcmgenet_rx_ring *ring)
1268{
1269        bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,
1270                                 INTRL2_CPU_MASK_SET);
1271}
1272
1273static inline void bcmgenet_rx_ring16_int_enable(struct bcmgenet_rx_ring *ring)
1274{
1275        bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,
1276                                 INTRL2_CPU_MASK_CLEAR);
1277}
1278
1279static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring)
1280{
1281        bcmgenet_intrl2_1_writel(ring->priv,
1282                                 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
1283                                 INTRL2_CPU_MASK_SET);
1284}
1285
1286static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring)
1287{
1288        bcmgenet_intrl2_1_writel(ring->priv,
1289                                 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
1290                                 INTRL2_CPU_MASK_CLEAR);
1291}
1292
1293static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_tx_ring *ring)
1294{
1295        bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,
1296                                 INTRL2_CPU_MASK_SET);
1297}
1298
1299static inline void bcmgenet_tx_ring16_int_enable(struct bcmgenet_tx_ring *ring)
1300{
1301        bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,
1302                                 INTRL2_CPU_MASK_CLEAR);
1303}
1304
1305static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring *ring)
1306{
1307        bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
1308                                 INTRL2_CPU_MASK_CLEAR);
1309}
1310
1311static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring *ring)
1312{
1313        bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
1314                                 INTRL2_CPU_MASK_SET);
1315}
1316
1317/* Simple helper to free a transmit control block's resources
1318 * Returns an skb when the last transmit control block associated with the
1319 * skb is freed.  The skb should be freed by the caller if necessary.
1320 */
1321static struct sk_buff *bcmgenet_free_tx_cb(struct device *dev,
1322                                           struct enet_cb *cb)
1323{
1324        struct sk_buff *skb;
1325
1326        skb = cb->skb;
1327
1328        if (skb) {
1329                cb->skb = NULL;
1330                if (cb == GENET_CB(skb)->first_cb)
1331                        dma_unmap_single(dev, dma_unmap_addr(cb, dma_addr),
1332                                         dma_unmap_len(cb, dma_len),
1333                                         DMA_TO_DEVICE);
1334                else
1335                        dma_unmap_page(dev, dma_unmap_addr(cb, dma_addr),
1336                                       dma_unmap_len(cb, dma_len),
1337                                       DMA_TO_DEVICE);
1338                dma_unmap_addr_set(cb, dma_addr, 0);
1339
1340                if (cb == GENET_CB(skb)->last_cb)
1341                        return skb;
1342
1343        } else if (dma_unmap_addr(cb, dma_addr)) {
1344                dma_unmap_page(dev,
1345                               dma_unmap_addr(cb, dma_addr),
1346                               dma_unmap_len(cb, dma_len),
1347                               DMA_TO_DEVICE);
1348                dma_unmap_addr_set(cb, dma_addr, 0);
1349        }
1350
1351        return NULL;
1352}
1353
1354/* Simple helper to free a receive control block's resources */
1355static struct sk_buff *bcmgenet_free_rx_cb(struct device *dev,
1356                                           struct enet_cb *cb)
1357{
1358        struct sk_buff *skb;
1359
1360        skb = cb->skb;
1361        cb->skb = NULL;
1362
1363        if (dma_unmap_addr(cb, dma_addr)) {
1364                dma_unmap_single(dev, dma_unmap_addr(cb, dma_addr),
1365                                 dma_unmap_len(cb, dma_len), DMA_FROM_DEVICE);
1366                dma_unmap_addr_set(cb, dma_addr, 0);
1367        }
1368
1369        return skb;
1370}
1371
1372/* Unlocked version of the reclaim routine */
1373static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev,
1374                                          struct bcmgenet_tx_ring *ring)
1375{
1376        struct bcmgenet_priv *priv = netdev_priv(dev);
1377        unsigned int txbds_processed = 0;
1378        unsigned int bytes_compl = 0;
1379        unsigned int pkts_compl = 0;
1380        unsigned int txbds_ready;
1381        unsigned int c_index;
1382        struct sk_buff *skb;
1383
1384        /* Clear status before servicing to reduce spurious interrupts */
1385        if (ring->index == DESC_INDEX)
1386                bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_TXDMA_DONE,
1387                                         INTRL2_CPU_CLEAR);
1388        else
1389                bcmgenet_intrl2_1_writel(priv, (1 << ring->index),
1390                                         INTRL2_CPU_CLEAR);
1391
1392        /* Compute how many buffers are transmitted since last xmit call */
1393        c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX)
1394                & DMA_C_INDEX_MASK;
1395        txbds_ready = (c_index - ring->c_index) & DMA_C_INDEX_MASK;
1396
1397        netif_dbg(priv, tx_done, dev,
1398                  "%s ring=%d old_c_index=%u c_index=%u txbds_ready=%u\n",
1399                  __func__, ring->index, ring->c_index, c_index, txbds_ready);
1400
1401        /* Reclaim transmitted buffers */
1402        while (txbds_processed < txbds_ready) {
1403                skb = bcmgenet_free_tx_cb(&priv->pdev->dev,
1404                                          &priv->tx_cbs[ring->clean_ptr]);
1405                if (skb) {
1406                        pkts_compl++;
1407                        bytes_compl += GENET_CB(skb)->bytes_sent;
1408                        dev_consume_skb_any(skb);
1409                }
1410
1411                txbds_processed++;
1412                if (likely(ring->clean_ptr < ring->end_ptr))
1413                        ring->clean_ptr++;
1414                else
1415                        ring->clean_ptr = ring->cb_ptr;
1416        }
1417
1418        ring->free_bds += txbds_processed;
1419        ring->c_index = c_index;
1420
1421        ring->packets += pkts_compl;
1422        ring->bytes += bytes_compl;
1423
1424        netdev_tx_completed_queue(netdev_get_tx_queue(dev, ring->queue),
1425                                  pkts_compl, bytes_compl);
1426
1427        return txbds_processed;
1428}
1429
1430static unsigned int bcmgenet_tx_reclaim(struct net_device *dev,
1431                                struct bcmgenet_tx_ring *ring)
1432{
1433        unsigned int released;
1434
1435        spin_lock_bh(&ring->lock);
1436        released = __bcmgenet_tx_reclaim(dev, ring);
1437        spin_unlock_bh(&ring->lock);
1438
1439        return released;
1440}
1441
1442static int bcmgenet_tx_poll(struct napi_struct *napi, int budget)
1443{
1444        struct bcmgenet_tx_ring *ring =
1445                container_of(napi, struct bcmgenet_tx_ring, napi);
1446        unsigned int work_done = 0;
1447        struct netdev_queue *txq;
1448
1449        spin_lock(&ring->lock);
1450        work_done = __bcmgenet_tx_reclaim(ring->priv->dev, ring);
1451        if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {
1452                txq = netdev_get_tx_queue(ring->priv->dev, ring->queue);
1453                netif_tx_wake_queue(txq);
1454        }
1455        spin_unlock(&ring->lock);
1456
1457        if (work_done == 0) {
1458                napi_complete(napi);
1459                ring->int_enable(ring);
1460
1461                return 0;
1462        }
1463
1464        return budget;
1465}
1466
1467static void bcmgenet_tx_reclaim_all(struct net_device *dev)
1468{
1469        struct bcmgenet_priv *priv = netdev_priv(dev);
1470        int i;
1471
1472        if (netif_is_multiqueue(dev)) {
1473                for (i = 0; i < priv->hw_params->tx_queues; i++)
1474                        bcmgenet_tx_reclaim(dev, &priv->tx_rings[i]);
1475        }
1476
1477        bcmgenet_tx_reclaim(dev, &priv->tx_rings[DESC_INDEX]);
1478}
1479
1480/* Reallocate the SKB to put enough headroom in front of it and insert
1481 * the transmit checksum offsets in the descriptors
1482 */
1483static struct sk_buff *bcmgenet_put_tx_csum(struct net_device *dev,
1484                                            struct sk_buff *skb)
1485{
1486        struct status_64 *status = NULL;
1487        struct sk_buff *new_skb;
1488        u16 offset;
1489        u8 ip_proto;
1490        __be16 ip_ver;
1491        u32 tx_csum_info;
1492
1493        if (unlikely(skb_headroom(skb) < sizeof(*status))) {
1494                /* If 64 byte status block enabled, must make sure skb has
1495                 * enough headroom for us to insert 64B status block.
1496                 */
1497                new_skb = skb_realloc_headroom(skb, sizeof(*status));
1498                dev_kfree_skb(skb);
1499                if (!new_skb) {
1500                        dev->stats.tx_dropped++;
1501                        return NULL;
1502                }
1503                skb = new_skb;
1504        }
1505
1506        skb_push(skb, sizeof(*status));
1507        status = (struct status_64 *)skb->data;
1508
1509        if (skb->ip_summed  == CHECKSUM_PARTIAL) {
1510                ip_ver = skb->protocol;
1511                switch (ip_ver) {
1512                case htons(ETH_P_IP):
1513                        ip_proto = ip_hdr(skb)->protocol;
1514                        break;
1515                case htons(ETH_P_IPV6):
1516                        ip_proto = ipv6_hdr(skb)->nexthdr;
1517                        break;
1518                default:
1519                        return skb;
1520                }
1521
1522                offset = skb_checksum_start_offset(skb) - sizeof(*status);
1523                tx_csum_info = (offset << STATUS_TX_CSUM_START_SHIFT) |
1524                                (offset + skb->csum_offset);
1525
1526                /* Set the length valid bit for TCP and UDP and just set
1527                 * the special UDP flag for IPv4, else just set to 0.
1528                 */
1529                if (ip_proto == IPPROTO_TCP || ip_proto == IPPROTO_UDP) {
1530                        tx_csum_info |= STATUS_TX_CSUM_LV;
1531                        if (ip_proto == IPPROTO_UDP &&
1532                            ip_ver == htons(ETH_P_IP))
1533                                tx_csum_info |= STATUS_TX_CSUM_PROTO_UDP;
1534                } else {
1535                        tx_csum_info = 0;
1536                }
1537
1538                status->tx_csum_info = tx_csum_info;
1539        }
1540
1541        return skb;
1542}
1543
1544static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev)
1545{
1546        struct bcmgenet_priv *priv = netdev_priv(dev);
1547        struct device *kdev = &priv->pdev->dev;
1548        struct bcmgenet_tx_ring *ring = NULL;
1549        struct enet_cb *tx_cb_ptr;
1550        struct netdev_queue *txq;
1551        int nr_frags, index;
1552        dma_addr_t mapping;
1553        unsigned int size;
1554        skb_frag_t *frag;
1555        u32 len_stat;
1556        int ret;
1557        int i;
1558
1559        index = skb_get_queue_mapping(skb);
1560        /* Mapping strategy:
1561         * queue_mapping = 0, unclassified, packet xmited through ring16
1562         * queue_mapping = 1, goes to ring 0. (highest priority queue
1563         * queue_mapping = 2, goes to ring 1.
1564         * queue_mapping = 3, goes to ring 2.
1565         * queue_mapping = 4, goes to ring 3.
1566         */
1567        if (index == 0)
1568                index = DESC_INDEX;
1569        else
1570                index -= 1;
1571
1572        ring = &priv->tx_rings[index];
1573        txq = netdev_get_tx_queue(dev, ring->queue);
1574
1575        nr_frags = skb_shinfo(skb)->nr_frags;
1576
1577        spin_lock(&ring->lock);
1578        if (ring->free_bds <= (nr_frags + 1)) {
1579                if (!netif_tx_queue_stopped(txq)) {
1580                        netif_tx_stop_queue(txq);
1581                        netdev_err(dev,
1582                                   "%s: tx ring %d full when queue %d awake\n",
1583                                   __func__, index, ring->queue);
1584                }
1585                ret = NETDEV_TX_BUSY;
1586                goto out;
1587        }
1588
1589        if (skb_padto(skb, ETH_ZLEN)) {
1590                ret = NETDEV_TX_OK;
1591                goto out;
1592        }
1593
1594        /* Retain how many bytes will be sent on the wire, without TSB inserted
1595         * by transmit checksum offload
1596         */
1597        GENET_CB(skb)->bytes_sent = skb->len;
1598
1599        /* set the SKB transmit checksum */
1600        if (priv->desc_64b_en) {
1601                skb = bcmgenet_put_tx_csum(dev, skb);
1602                if (!skb) {
1603                        ret = NETDEV_TX_OK;
1604                        goto out;
1605                }
1606        }
1607
1608        for (i = 0; i <= nr_frags; i++) {
1609                tx_cb_ptr = bcmgenet_get_txcb(priv, ring);
1610
1611                BUG_ON(!tx_cb_ptr);
1612
1613                if (!i) {
1614                        /* Transmit single SKB or head of fragment list */
1615                        GENET_CB(skb)->first_cb = tx_cb_ptr;
1616                        size = skb_headlen(skb);
1617                        mapping = dma_map_single(kdev, skb->data, size,
1618                                                 DMA_TO_DEVICE);
1619                } else {
1620                        /* xmit fragment */
1621                        frag = &skb_shinfo(skb)->frags[i - 1];
1622                        size = skb_frag_size(frag);
1623                        mapping = skb_frag_dma_map(kdev, frag, 0, size,
1624                                                   DMA_TO_DEVICE);
1625                }
1626
1627                ret = dma_mapping_error(kdev, mapping);
1628                if (ret) {
1629                        priv->mib.tx_dma_failed++;
1630                        netif_err(priv, tx_err, dev, "Tx DMA map failed\n");
1631                        ret = NETDEV_TX_OK;
1632                        goto out_unmap_frags;
1633                }
1634                dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping);
1635                dma_unmap_len_set(tx_cb_ptr, dma_len, size);
1636
1637                tx_cb_ptr->skb = skb;
1638
1639                len_stat = (size << DMA_BUFLENGTH_SHIFT) |
1640                           (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT);
1641
1642                if (!i) {
1643                        len_stat |= DMA_TX_APPEND_CRC | DMA_SOP;
1644                        if (skb->ip_summed == CHECKSUM_PARTIAL)
1645                                len_stat |= DMA_TX_DO_CSUM;
1646                }
1647                if (i == nr_frags)
1648                        len_stat |= DMA_EOP;
1649
1650                dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping, len_stat);
1651        }
1652
1653        GENET_CB(skb)->last_cb = tx_cb_ptr;
1654        skb_tx_timestamp(skb);
1655
1656        /* Decrement total BD count and advance our write pointer */
1657        ring->free_bds -= nr_frags + 1;
1658        ring->prod_index += nr_frags + 1;
1659        ring->prod_index &= DMA_P_INDEX_MASK;
1660
1661        netdev_tx_sent_queue(txq, GENET_CB(skb)->bytes_sent);
1662
1663        if (ring->free_bds <= (MAX_SKB_FRAGS + 1))
1664                netif_tx_stop_queue(txq);
1665
1666        if (!netdev_xmit_more() || netif_xmit_stopped(txq))
1667                /* Packets are ready, update producer index */
1668                bcmgenet_tdma_ring_writel(priv, ring->index,
1669                                          ring->prod_index, TDMA_PROD_INDEX);
1670out:
1671        spin_unlock(&ring->lock);
1672
1673        return ret;
1674
1675out_unmap_frags:
1676        /* Back up for failed control block mapping */
1677        bcmgenet_put_txcb(priv, ring);
1678
1679        /* Unmap successfully mapped control blocks */
1680        while (i-- > 0) {
1681                tx_cb_ptr = bcmgenet_put_txcb(priv, ring);
1682                bcmgenet_free_tx_cb(kdev, tx_cb_ptr);
1683        }
1684
1685        dev_kfree_skb(skb);
1686        goto out;
1687}
1688
1689static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv,
1690                                          struct enet_cb *cb)
1691{
1692        struct device *kdev = &priv->pdev->dev;
1693        struct sk_buff *skb;
1694        struct sk_buff *rx_skb;
1695        dma_addr_t mapping;
1696
1697        /* Allocate a new Rx skb */
1698        skb = netdev_alloc_skb(priv->dev, priv->rx_buf_len + SKB_ALIGNMENT);
1699        if (!skb) {
1700                priv->mib.alloc_rx_buff_failed++;
1701                netif_err(priv, rx_err, priv->dev,
1702                          "%s: Rx skb allocation failed\n", __func__);
1703                return NULL;
1704        }
1705
1706        /* DMA-map the new Rx skb */
1707        mapping = dma_map_single(kdev, skb->data, priv->rx_buf_len,
1708                                 DMA_FROM_DEVICE);
1709        if (dma_mapping_error(kdev, mapping)) {
1710                priv->mib.rx_dma_failed++;
1711                dev_kfree_skb_any(skb);
1712                netif_err(priv, rx_err, priv->dev,
1713                          "%s: Rx skb DMA mapping failed\n", __func__);
1714                return NULL;
1715        }
1716
1717        /* Grab the current Rx skb from the ring and DMA-unmap it */
1718        rx_skb = bcmgenet_free_rx_cb(kdev, cb);
1719
1720        /* Put the new Rx skb on the ring */
1721        cb->skb = skb;
1722        dma_unmap_addr_set(cb, dma_addr, mapping);
1723        dma_unmap_len_set(cb, dma_len, priv->rx_buf_len);
1724        dmadesc_set_addr(priv, cb->bd_addr, mapping);
1725
1726        /* Return the current Rx skb to caller */
1727        return rx_skb;
1728}
1729
1730/* bcmgenet_desc_rx - descriptor based rx process.
1731 * this could be called from bottom half, or from NAPI polling method.
1732 */
1733static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
1734                                     unsigned int budget)
1735{
1736        struct bcmgenet_priv *priv = ring->priv;
1737        struct net_device *dev = priv->dev;
1738        struct enet_cb *cb;
1739        struct sk_buff *skb;
1740        u32 dma_length_status;
1741        unsigned long dma_flag;
1742        int len;
1743        unsigned int rxpktprocessed = 0, rxpkttoprocess;
1744        unsigned int bytes_processed = 0;
1745        unsigned int p_index, mask;
1746        unsigned int discards;
1747        unsigned int chksum_ok = 0;
1748
1749        /* Clear status before servicing to reduce spurious interrupts */
1750        if (ring->index == DESC_INDEX) {
1751                bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_RXDMA_DONE,
1752                                         INTRL2_CPU_CLEAR);
1753        } else {
1754                mask = 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index);
1755                bcmgenet_intrl2_1_writel(priv,
1756                                         mask,
1757                                         INTRL2_CPU_CLEAR);
1758        }
1759
1760        p_index = bcmgenet_rdma_ring_readl(priv, ring->index, RDMA_PROD_INDEX);
1761
1762        discards = (p_index >> DMA_P_INDEX_DISCARD_CNT_SHIFT) &
1763                   DMA_P_INDEX_DISCARD_CNT_MASK;
1764        if (discards > ring->old_discards) {
1765                discards = discards - ring->old_discards;
1766                ring->errors += discards;
1767                ring->old_discards += discards;
1768
1769                /* Clear HW register when we reach 75% of maximum 0xFFFF */
1770                if (ring->old_discards >= 0xC000) {
1771                        ring->old_discards = 0;
1772                        bcmgenet_rdma_ring_writel(priv, ring->index, 0,
1773                                                  RDMA_PROD_INDEX);
1774                }
1775        }
1776
1777        p_index &= DMA_P_INDEX_MASK;
1778        rxpkttoprocess = (p_index - ring->c_index) & DMA_C_INDEX_MASK;
1779
1780        netif_dbg(priv, rx_status, dev,
1781                  "RDMA: rxpkttoprocess=%d\n", rxpkttoprocess);
1782
1783        while ((rxpktprocessed < rxpkttoprocess) &&
1784               (rxpktprocessed < budget)) {
1785                cb = &priv->rx_cbs[ring->read_ptr];
1786                skb = bcmgenet_rx_refill(priv, cb);
1787
1788                if (unlikely(!skb)) {
1789                        ring->dropped++;
1790                        goto next;
1791                }
1792
1793                if (!priv->desc_64b_en) {
1794                        dma_length_status =
1795                                dmadesc_get_length_status(priv, cb->bd_addr);
1796                } else {
1797                        struct status_64 *status;
1798
1799                        status = (struct status_64 *)skb->data;
1800                        dma_length_status = status->length_status;
1801                }
1802
1803                /* DMA flags and length are still valid no matter how
1804                 * we got the Receive Status Vector (64B RSB or register)
1805                 */
1806                dma_flag = dma_length_status & 0xffff;
1807                len = dma_length_status >> DMA_BUFLENGTH_SHIFT;
1808
1809                netif_dbg(priv, rx_status, dev,
1810                          "%s:p_ind=%d c_ind=%d read_ptr=%d len_stat=0x%08x\n",
1811                          __func__, p_index, ring->c_index,
1812                          ring->read_ptr, dma_length_status);
1813
1814                if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) {
1815                        netif_err(priv, rx_status, dev,
1816                                  "dropping fragmented packet!\n");
1817                        ring->errors++;
1818                        dev_kfree_skb_any(skb);
1819                        goto next;
1820                }
1821
1822                /* report errors */
1823                if (unlikely(dma_flag & (DMA_RX_CRC_ERROR |
1824                                                DMA_RX_OV |
1825                                                DMA_RX_NO |
1826                                                DMA_RX_LG |
1827                                                DMA_RX_RXER))) {
1828                        netif_err(priv, rx_status, dev, "dma_flag=0x%x\n",
1829                                  (unsigned int)dma_flag);
1830                        if (dma_flag & DMA_RX_CRC_ERROR)
1831                                dev->stats.rx_crc_errors++;
1832                        if (dma_flag & DMA_RX_OV)
1833                                dev->stats.rx_over_errors++;
1834                        if (dma_flag & DMA_RX_NO)
1835                                dev->stats.rx_frame_errors++;
1836                        if (dma_flag & DMA_RX_LG)
1837                                dev->stats.rx_length_errors++;
1838                        dev->stats.rx_errors++;
1839                        dev_kfree_skb_any(skb);
1840                        goto next;
1841                } /* error packet */
1842
1843                chksum_ok = (dma_flag & priv->dma_rx_chk_bit) &&
1844                             priv->desc_rxchk_en;
1845
1846                skb_put(skb, len);
1847                if (priv->desc_64b_en) {
1848                        skb_pull(skb, 64);
1849                        len -= 64;
1850                }
1851
1852                if (likely(chksum_ok))
1853                        skb->ip_summed = CHECKSUM_UNNECESSARY;
1854
1855                /* remove hardware 2bytes added for IP alignment */
1856                skb_pull(skb, 2);
1857                len -= 2;
1858
1859                if (priv->crc_fwd_en) {
1860                        skb_trim(skb, len - ETH_FCS_LEN);
1861                        len -= ETH_FCS_LEN;
1862                }
1863
1864                bytes_processed += len;
1865
1866                /*Finish setting up the received SKB and send it to the kernel*/
1867                skb->protocol = eth_type_trans(skb, priv->dev);
1868                ring->packets++;
1869                ring->bytes += len;
1870                if (dma_flag & DMA_RX_MULT)
1871                        dev->stats.multicast++;
1872
1873                /* Notify kernel */
1874                napi_gro_receive(&ring->napi, skb);
1875                netif_dbg(priv, rx_status, dev, "pushed up to kernel\n");
1876
1877next:
1878                rxpktprocessed++;
1879                if (likely(ring->read_ptr < ring->end_ptr))
1880                        ring->read_ptr++;
1881                else
1882                        ring->read_ptr = ring->cb_ptr;
1883
1884                ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;
1885                bcmgenet_rdma_ring_writel(priv, ring->index, ring->c_index, RDMA_CONS_INDEX);
1886        }
1887
1888        ring->dim.bytes = bytes_processed;
1889        ring->dim.packets = rxpktprocessed;
1890
1891        return rxpktprocessed;
1892}
1893
1894/* Rx NAPI polling method */
1895static int bcmgenet_rx_poll(struct napi_struct *napi, int budget)
1896{
1897        struct bcmgenet_rx_ring *ring = container_of(napi,
1898                        struct bcmgenet_rx_ring, napi);
1899        struct dim_sample dim_sample = {};
1900        unsigned int work_done;
1901
1902        work_done = bcmgenet_desc_rx(ring, budget);
1903
1904        if (work_done < budget) {
1905                napi_complete_done(napi, work_done);
1906                ring->int_enable(ring);
1907        }
1908
1909        if (ring->dim.use_dim) {
1910                dim_update_sample(ring->dim.event_ctr, ring->dim.packets,
1911                                  ring->dim.bytes, &dim_sample);
1912                net_dim(&ring->dim.dim, dim_sample);
1913        }
1914
1915        return work_done;
1916}
1917
1918static void bcmgenet_dim_work(struct work_struct *work)
1919{
1920        struct dim *dim = container_of(work, struct dim, work);
1921        struct bcmgenet_net_dim *ndim =
1922                        container_of(dim, struct bcmgenet_net_dim, dim);
1923        struct bcmgenet_rx_ring *ring =
1924                        container_of(ndim, struct bcmgenet_rx_ring, dim);
1925        struct dim_cq_moder cur_profile =
1926                        net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1927
1928        bcmgenet_set_rx_coalesce(ring, cur_profile.usec, cur_profile.pkts);
1929        dim->state = DIM_START_MEASURE;
1930}
1931
1932/* Assign skb to RX DMA descriptor. */
1933static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv,
1934                                     struct bcmgenet_rx_ring *ring)
1935{
1936        struct enet_cb *cb;
1937        struct sk_buff *skb;
1938        int i;
1939
1940        netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
1941
1942        /* loop here for each buffer needing assign */
1943        for (i = 0; i < ring->size; i++) {
1944                cb = ring->cbs + i;
1945                skb = bcmgenet_rx_refill(priv, cb);
1946                if (skb)
1947                        dev_consume_skb_any(skb);
1948                if (!cb->skb)
1949                        return -ENOMEM;
1950        }
1951
1952        return 0;
1953}
1954
1955static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv)
1956{
1957        struct sk_buff *skb;
1958        struct enet_cb *cb;
1959        int i;
1960
1961        for (i = 0; i < priv->num_rx_bds; i++) {
1962                cb = &priv->rx_cbs[i];
1963
1964                skb = bcmgenet_free_rx_cb(&priv->pdev->dev, cb);
1965                if (skb)
1966                        dev_consume_skb_any(skb);
1967        }
1968}
1969
1970static void umac_enable_set(struct bcmgenet_priv *priv, u32 mask, bool enable)
1971{
1972        u32 reg;
1973
1974        reg = bcmgenet_umac_readl(priv, UMAC_CMD);
1975        if (enable)
1976                reg |= mask;
1977        else
1978                reg &= ~mask;
1979        bcmgenet_umac_writel(priv, reg, UMAC_CMD);
1980
1981        /* UniMAC stops on a packet boundary, wait for a full-size packet
1982         * to be processed
1983         */
1984        if (enable == 0)
1985                usleep_range(1000, 2000);
1986}
1987
1988static void reset_umac(struct bcmgenet_priv *priv)
1989{
1990        /* 7358a0/7552a0: bad default in RBUF_FLUSH_CTRL.umac_sw_rst */
1991        bcmgenet_rbuf_ctrl_set(priv, 0);
1992        udelay(10);
1993
1994        /* disable MAC while updating its registers */
1995        bcmgenet_umac_writel(priv, 0, UMAC_CMD);
1996
1997        /* issue soft reset with (rg)mii loopback to ensure a stable rxclk */
1998        bcmgenet_umac_writel(priv, CMD_SW_RESET | CMD_LCL_LOOP_EN, UMAC_CMD);
1999}
2000
2001static void bcmgenet_intr_disable(struct bcmgenet_priv *priv)
2002{
2003        /* Mask all interrupts.*/
2004        bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
2005        bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
2006        bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
2007        bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
2008}
2009
2010static void bcmgenet_link_intr_enable(struct bcmgenet_priv *priv)
2011{
2012        u32 int0_enable = 0;
2013
2014        /* Monitor cable plug/unplugged event for internal PHY, external PHY
2015         * and MoCA PHY
2016         */
2017        if (priv->internal_phy) {
2018                int0_enable |= UMAC_IRQ_LINK_EVENT;
2019                if (GENET_IS_V1(priv) || GENET_IS_V2(priv) || GENET_IS_V3(priv))
2020                        int0_enable |= UMAC_IRQ_PHY_DET_R;
2021        } else if (priv->ext_phy) {
2022                int0_enable |= UMAC_IRQ_LINK_EVENT;
2023        } else if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
2024                if (priv->hw_params->flags & GENET_HAS_MOCA_LINK_DET)
2025                        int0_enable |= UMAC_IRQ_LINK_EVENT;
2026        }
2027        bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
2028}
2029
2030static void init_umac(struct bcmgenet_priv *priv)
2031{
2032        struct device *kdev = &priv->pdev->dev;
2033        u32 reg;
2034        u32 int0_enable = 0;
2035
2036        dev_dbg(&priv->pdev->dev, "bcmgenet: init_umac\n");
2037
2038        reset_umac(priv);
2039
2040        /* clear tx/rx counter */
2041        bcmgenet_umac_writel(priv,
2042                             MIB_RESET_RX | MIB_RESET_TX | MIB_RESET_RUNT,
2043                             UMAC_MIB_CTRL);
2044        bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL);
2045
2046        bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
2047
2048        /* init rx registers, enable ip header optimization */
2049        reg = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
2050        reg |= RBUF_ALIGN_2B;
2051        bcmgenet_rbuf_writel(priv, reg, RBUF_CTRL);
2052
2053        if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv))
2054                bcmgenet_rbuf_writel(priv, 1, RBUF_TBUF_SIZE_CTRL);
2055
2056        bcmgenet_intr_disable(priv);
2057
2058        /* Configure backpressure vectors for MoCA */
2059        if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
2060                reg = bcmgenet_bp_mc_get(priv);
2061                reg |= BIT(priv->hw_params->bp_in_en_shift);
2062
2063                /* bp_mask: back pressure mask */
2064                if (netif_is_multiqueue(priv->dev))
2065                        reg |= priv->hw_params->bp_in_mask;
2066                else
2067                        reg &= ~priv->hw_params->bp_in_mask;
2068                bcmgenet_bp_mc_set(priv, reg);
2069        }
2070
2071        /* Enable MDIO interrupts on GENET v3+ */
2072        if (priv->hw_params->flags & GENET_HAS_MDIO_INTR)
2073                int0_enable |= (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR);
2074
2075        bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
2076
2077        dev_dbg(kdev, "done init umac\n");
2078}
2079
2080static void bcmgenet_init_dim(struct bcmgenet_rx_ring *ring,
2081                              void (*cb)(struct work_struct *work))
2082{
2083        struct bcmgenet_net_dim *dim = &ring->dim;
2084
2085        INIT_WORK(&dim->dim.work, cb);
2086        dim->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2087        dim->event_ctr = 0;
2088        dim->packets = 0;
2089        dim->bytes = 0;
2090}
2091
2092static void bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring *ring)
2093{
2094        struct bcmgenet_net_dim *dim = &ring->dim;
2095        struct dim_cq_moder moder;
2096        u32 usecs, pkts;
2097
2098        usecs = ring->rx_coalesce_usecs;
2099        pkts = ring->rx_max_coalesced_frames;
2100
2101        /* If DIM was enabled, re-apply default parameters */
2102        if (dim->use_dim) {
2103                moder = net_dim_get_def_rx_moderation(dim->dim.mode);
2104                usecs = moder.usec;
2105                pkts = moder.pkts;
2106        }
2107
2108        bcmgenet_set_rx_coalesce(ring, usecs, pkts);
2109}
2110
2111/* Initialize a Tx ring along with corresponding hardware registers */
2112static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv,
2113                                  unsigned int index, unsigned int size,
2114                                  unsigned int start_ptr, unsigned int end_ptr)
2115{
2116        struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];
2117        u32 words_per_bd = WORDS_PER_BD(priv);
2118        u32 flow_period_val = 0;
2119
2120        spin_lock_init(&ring->lock);
2121        ring->priv = priv;
2122        ring->index = index;
2123        if (index == DESC_INDEX) {
2124                ring->queue = 0;
2125                ring->int_enable = bcmgenet_tx_ring16_int_enable;
2126                ring->int_disable = bcmgenet_tx_ring16_int_disable;
2127        } else {
2128                ring->queue = index + 1;
2129                ring->int_enable = bcmgenet_tx_ring_int_enable;
2130                ring->int_disable = bcmgenet_tx_ring_int_disable;
2131        }
2132        ring->cbs = priv->tx_cbs + start_ptr;
2133        ring->size = size;
2134        ring->clean_ptr = start_ptr;
2135        ring->c_index = 0;
2136        ring->free_bds = size;
2137        ring->write_ptr = start_ptr;
2138        ring->cb_ptr = start_ptr;
2139        ring->end_ptr = end_ptr - 1;
2140        ring->prod_index = 0;
2141
2142        /* Set flow period for ring != 16 */
2143        if (index != DESC_INDEX)
2144                flow_period_val = ENET_MAX_MTU_SIZE << 16;
2145
2146        bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX);
2147        bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX);
2148        bcmgenet_tdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH);
2149        /* Disable rate control for now */
2150        bcmgenet_tdma_ring_writel(priv, index, flow_period_val,
2151                                  TDMA_FLOW_PERIOD);
2152        bcmgenet_tdma_ring_writel(priv, index,
2153                                  ((size << DMA_RING_SIZE_SHIFT) |
2154                                   RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
2155
2156        /* Set start and end address, read and write pointers */
2157        bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2158                                  DMA_START_ADDR);
2159        bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2160                                  TDMA_READ_PTR);
2161        bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2162                                  TDMA_WRITE_PTR);
2163        bcmgenet_tdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
2164                                  DMA_END_ADDR);
2165
2166        /* Initialize Tx NAPI */
2167        netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll,
2168                       NAPI_POLL_WEIGHT);
2169}
2170
2171/* Initialize a RDMA ring */
2172static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv,
2173                                 unsigned int index, unsigned int size,
2174                                 unsigned int start_ptr, unsigned int end_ptr)
2175{
2176        struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];
2177        u32 words_per_bd = WORDS_PER_BD(priv);
2178        int ret;
2179
2180        ring->priv = priv;
2181        ring->index = index;
2182        if (index == DESC_INDEX) {
2183                ring->int_enable = bcmgenet_rx_ring16_int_enable;
2184                ring->int_disable = bcmgenet_rx_ring16_int_disable;
2185        } else {
2186                ring->int_enable = bcmgenet_rx_ring_int_enable;
2187                ring->int_disable = bcmgenet_rx_ring_int_disable;
2188        }
2189        ring->cbs = priv->rx_cbs + start_ptr;
2190        ring->size = size;
2191        ring->c_index = 0;
2192        ring->read_ptr = start_ptr;
2193        ring->cb_ptr = start_ptr;
2194        ring->end_ptr = end_ptr - 1;
2195
2196        ret = bcmgenet_alloc_rx_buffers(priv, ring);
2197        if (ret)
2198                return ret;
2199
2200        bcmgenet_init_dim(ring, bcmgenet_dim_work);
2201        bcmgenet_init_rx_coalesce(ring);
2202
2203        /* Initialize Rx NAPI */
2204        netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll,
2205                       NAPI_POLL_WEIGHT);
2206
2207        bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX);
2208        bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX);
2209        bcmgenet_rdma_ring_writel(priv, index,
2210                                  ((size << DMA_RING_SIZE_SHIFT) |
2211                                   RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
2212        bcmgenet_rdma_ring_writel(priv, index,
2213                                  (DMA_FC_THRESH_LO <<
2214                                   DMA_XOFF_THRESHOLD_SHIFT) |
2215                                   DMA_FC_THRESH_HI, RDMA_XON_XOFF_THRESH);
2216
2217        /* Set start and end address, read and write pointers */
2218        bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2219                                  DMA_START_ADDR);
2220        bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2221                                  RDMA_READ_PTR);
2222        bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2223                                  RDMA_WRITE_PTR);
2224        bcmgenet_rdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
2225                                  DMA_END_ADDR);
2226
2227        return ret;
2228}
2229
2230static void bcmgenet_enable_tx_napi(struct bcmgenet_priv *priv)
2231{
2232        unsigned int i;
2233        struct bcmgenet_tx_ring *ring;
2234
2235        for (i = 0; i < priv->hw_params->tx_queues; ++i) {
2236                ring = &priv->tx_rings[i];
2237                napi_enable(&ring->napi);
2238                ring->int_enable(ring);
2239        }
2240
2241        ring = &priv->tx_rings[DESC_INDEX];
2242        napi_enable(&ring->napi);
2243        ring->int_enable(ring);
2244}
2245
2246static void bcmgenet_disable_tx_napi(struct bcmgenet_priv *priv)
2247{
2248        unsigned int i;
2249        struct bcmgenet_tx_ring *ring;
2250
2251        for (i = 0; i < priv->hw_params->tx_queues; ++i) {
2252                ring = &priv->tx_rings[i];
2253                napi_disable(&ring->napi);
2254        }
2255
2256        ring = &priv->tx_rings[DESC_INDEX];
2257        napi_disable(&ring->napi);
2258}
2259
2260static void bcmgenet_fini_tx_napi(struct bcmgenet_priv *priv)
2261{
2262        unsigned int i;
2263        struct bcmgenet_tx_ring *ring;
2264
2265        for (i = 0; i < priv->hw_params->tx_queues; ++i) {
2266                ring = &priv->tx_rings[i];
2267                netif_napi_del(&ring->napi);
2268        }
2269
2270        ring = &priv->tx_rings[DESC_INDEX];
2271        netif_napi_del(&ring->napi);
2272}
2273
2274/* Initialize Tx queues
2275 *
2276 * Queues 0-3 are priority-based, each one has 32 descriptors,
2277 * with queue 0 being the highest priority queue.
2278 *
2279 * Queue 16 is the default Tx queue with
2280 * GENET_Q16_TX_BD_CNT = 256 - 4 * 32 = 128 descriptors.
2281 *
2282 * The transmit control block pool is then partitioned as follows:
2283 * - Tx queue 0 uses tx_cbs[0..31]
2284 * - Tx queue 1 uses tx_cbs[32..63]
2285 * - Tx queue 2 uses tx_cbs[64..95]
2286 * - Tx queue 3 uses tx_cbs[96..127]
2287 * - Tx queue 16 uses tx_cbs[128..255]
2288 */
2289static void bcmgenet_init_tx_queues(struct net_device *dev)
2290{
2291        struct bcmgenet_priv *priv = netdev_priv(dev);
2292        u32 i, dma_enable;
2293        u32 dma_ctrl, ring_cfg;
2294        u32 dma_priority[3] = {0, 0, 0};
2295
2296        dma_ctrl = bcmgenet_tdma_readl(priv, DMA_CTRL);
2297        dma_enable = dma_ctrl & DMA_EN;
2298        dma_ctrl &= ~DMA_EN;
2299        bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL);
2300
2301        dma_ctrl = 0;
2302        ring_cfg = 0;
2303
2304        /* Enable strict priority arbiter mode */
2305        bcmgenet_tdma_writel(priv, DMA_ARBITER_SP, DMA_ARB_CTRL);
2306
2307        /* Initialize Tx priority queues */
2308        for (i = 0; i < priv->hw_params->tx_queues; i++) {
2309                bcmgenet_init_tx_ring(priv, i, priv->hw_params->tx_bds_per_q,
2310                                      i * priv->hw_params->tx_bds_per_q,
2311                                      (i + 1) * priv->hw_params->tx_bds_per_q);
2312                ring_cfg |= (1 << i);
2313                dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
2314                dma_priority[DMA_PRIO_REG_INDEX(i)] |=
2315                        ((GENET_Q0_PRIORITY + i) << DMA_PRIO_REG_SHIFT(i));
2316        }
2317
2318        /* Initialize Tx default queue 16 */
2319        bcmgenet_init_tx_ring(priv, DESC_INDEX, GENET_Q16_TX_BD_CNT,
2320                              priv->hw_params->tx_queues *
2321                              priv->hw_params->tx_bds_per_q,
2322                              TOTAL_DESC);
2323        ring_cfg |= (1 << DESC_INDEX);
2324        dma_ctrl |= (1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT));
2325        dma_priority[DMA_PRIO_REG_INDEX(DESC_INDEX)] |=
2326                ((GENET_Q0_PRIORITY + priv->hw_params->tx_queues) <<
2327                 DMA_PRIO_REG_SHIFT(DESC_INDEX));
2328
2329        /* Set Tx queue priorities */
2330        bcmgenet_tdma_writel(priv, dma_priority[0], DMA_PRIORITY_0);
2331        bcmgenet_tdma_writel(priv, dma_priority[1], DMA_PRIORITY_1);
2332        bcmgenet_tdma_writel(priv, dma_priority[2], DMA_PRIORITY_2);
2333
2334        /* Enable Tx queues */
2335        bcmgenet_tdma_writel(priv, ring_cfg, DMA_RING_CFG);
2336
2337        /* Enable Tx DMA */
2338        if (dma_enable)
2339                dma_ctrl |= DMA_EN;
2340        bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL);
2341}
2342
2343static void bcmgenet_enable_rx_napi(struct bcmgenet_priv *priv)
2344{
2345        unsigned int i;
2346        struct bcmgenet_rx_ring *ring;
2347
2348        for (i = 0; i < priv->hw_params->rx_queues; ++i) {
2349                ring = &priv->rx_rings[i];
2350                napi_enable(&ring->napi);
2351                ring->int_enable(ring);
2352        }
2353
2354        ring = &priv->rx_rings[DESC_INDEX];
2355        napi_enable(&ring->napi);
2356        ring->int_enable(ring);
2357}
2358
2359static void bcmgenet_disable_rx_napi(struct bcmgenet_priv *priv)
2360{
2361        unsigned int i;
2362        struct bcmgenet_rx_ring *ring;
2363
2364        for (i = 0; i < priv->hw_params->rx_queues; ++i) {
2365                ring = &priv->rx_rings[i];
2366                napi_disable(&ring->napi);
2367                cancel_work_sync(&ring->dim.dim.work);
2368        }
2369
2370        ring = &priv->rx_rings[DESC_INDEX];
2371        napi_disable(&ring->napi);
2372        cancel_work_sync(&ring->dim.dim.work);
2373}
2374
2375static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv)
2376{
2377        unsigned int i;
2378        struct bcmgenet_rx_ring *ring;
2379
2380        for (i = 0; i < priv->hw_params->rx_queues; ++i) {
2381                ring = &priv->rx_rings[i];
2382                netif_napi_del(&ring->napi);
2383        }
2384
2385        ring = &priv->rx_rings[DESC_INDEX];
2386        netif_napi_del(&ring->napi);
2387}
2388
2389/* Initialize Rx queues
2390 *
2391 * Queues 0-15 are priority queues. Hardware Filtering Block (HFB) can be
2392 * used to direct traffic to these queues.
2393 *
2394 * Queue 16 is the default Rx queue with GENET_Q16_RX_BD_CNT descriptors.
2395 */
2396static int bcmgenet_init_rx_queues(struct net_device *dev)
2397{
2398        struct bcmgenet_priv *priv = netdev_priv(dev);
2399        u32 i;
2400        u32 dma_enable;
2401        u32 dma_ctrl;
2402        u32 ring_cfg;
2403        int ret;
2404
2405        dma_ctrl = bcmgenet_rdma_readl(priv, DMA_CTRL);
2406        dma_enable = dma_ctrl & DMA_EN;
2407        dma_ctrl &= ~DMA_EN;
2408        bcmgenet_rdma_writel(priv, dma_ctrl, DMA_CTRL);
2409
2410        dma_ctrl = 0;
2411        ring_cfg = 0;
2412
2413        /* Initialize Rx priority queues */
2414        for (i = 0; i < priv->hw_params->rx_queues; i++) {
2415                ret = bcmgenet_init_rx_ring(priv, i,
2416                                            priv->hw_params->rx_bds_per_q,
2417                                            i * priv->hw_params->rx_bds_per_q,
2418                                            (i + 1) *
2419                                            priv->hw_params->rx_bds_per_q);
2420                if (ret)
2421                        return ret;
2422
2423                ring_cfg |= (1 << i);
2424                dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
2425        }
2426
2427        /* Initialize Rx default queue 16 */
2428        ret = bcmgenet_init_rx_ring(priv, DESC_INDEX, GENET_Q16_RX_BD_CNT,
2429                                    priv->hw_params->rx_queues *
2430                                    priv->hw_params->rx_bds_per_q,
2431                                    TOTAL_DESC);
2432        if (ret)
2433                return ret;
2434
2435        ring_cfg |= (1 << DESC_INDEX);
2436        dma_ctrl |= (1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT));
2437
2438        /* Enable rings */
2439        bcmgenet_rdma_writel(priv, ring_cfg, DMA_RING_CFG);
2440
2441        /* Configure ring as descriptor ring and re-enable DMA if enabled */
2442        if (dma_enable)
2443                dma_ctrl |= DMA_EN;
2444        bcmgenet_rdma_writel(priv, dma_ctrl, DMA_CTRL);
2445
2446        return 0;
2447}
2448
2449static int bcmgenet_dma_teardown(struct bcmgenet_priv *priv)
2450{
2451        int ret = 0;
2452        int timeout = 0;
2453        u32 reg;
2454        u32 dma_ctrl;
2455        int i;
2456
2457        /* Disable TDMA to stop add more frames in TX DMA */
2458        reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2459        reg &= ~DMA_EN;
2460        bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2461
2462        /* Check TDMA status register to confirm TDMA is disabled */
2463        while (timeout++ < DMA_TIMEOUT_VAL) {
2464                reg = bcmgenet_tdma_readl(priv, DMA_STATUS);
2465                if (reg & DMA_DISABLED)
2466                        break;
2467
2468                udelay(1);
2469        }
2470
2471        if (timeout == DMA_TIMEOUT_VAL) {
2472                netdev_warn(priv->dev, "Timed out while disabling TX DMA\n");
2473                ret = -ETIMEDOUT;
2474        }
2475
2476        /* Wait 10ms for packet drain in both tx and rx dma */
2477        usleep_range(10000, 20000);
2478
2479        /* Disable RDMA */
2480        reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2481        reg &= ~DMA_EN;
2482        bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2483
2484        timeout = 0;
2485        /* Check RDMA status register to confirm RDMA is disabled */
2486        while (timeout++ < DMA_TIMEOUT_VAL) {
2487                reg = bcmgenet_rdma_readl(priv, DMA_STATUS);
2488                if (reg & DMA_DISABLED)
2489                        break;
2490
2491                udelay(1);
2492        }
2493
2494        if (timeout == DMA_TIMEOUT_VAL) {
2495                netdev_warn(priv->dev, "Timed out while disabling RX DMA\n");
2496                ret = -ETIMEDOUT;
2497        }
2498
2499        dma_ctrl = 0;
2500        for (i = 0; i < priv->hw_params->rx_queues; i++)
2501                dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
2502        reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2503        reg &= ~dma_ctrl;
2504        bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2505
2506        dma_ctrl = 0;
2507        for (i = 0; i < priv->hw_params->tx_queues; i++)
2508                dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
2509        reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2510        reg &= ~dma_ctrl;
2511        bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2512
2513        return ret;
2514}
2515
2516static void bcmgenet_fini_dma(struct bcmgenet_priv *priv)
2517{
2518        struct netdev_queue *txq;
2519        int i;
2520
2521        bcmgenet_fini_rx_napi(priv);
2522        bcmgenet_fini_tx_napi(priv);
2523
2524        for (i = 0; i < priv->num_tx_bds; i++)
2525                dev_kfree_skb(bcmgenet_free_tx_cb(&priv->pdev->dev,
2526                                                  priv->tx_cbs + i));
2527
2528        for (i = 0; i < priv->hw_params->tx_queues; i++) {
2529                txq = netdev_get_tx_queue(priv->dev, priv->tx_rings[i].queue);
2530                netdev_tx_reset_queue(txq);
2531        }
2532
2533        txq = netdev_get_tx_queue(priv->dev, priv->tx_rings[DESC_INDEX].queue);
2534        netdev_tx_reset_queue(txq);
2535
2536        bcmgenet_free_rx_buffers(priv);
2537        kfree(priv->rx_cbs);
2538        kfree(priv->tx_cbs);
2539}
2540
2541/* init_edma: Initialize DMA control register */
2542static int bcmgenet_init_dma(struct bcmgenet_priv *priv)
2543{
2544        int ret;
2545        unsigned int i;
2546        struct enet_cb *cb;
2547
2548        netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
2549
2550        /* Initialize common Rx ring structures */
2551        priv->rx_bds = priv->base + priv->hw_params->rdma_offset;
2552        priv->num_rx_bds = TOTAL_DESC;
2553        priv->rx_cbs = kcalloc(priv->num_rx_bds, sizeof(struct enet_cb),
2554                               GFP_KERNEL);
2555        if (!priv->rx_cbs)
2556                return -ENOMEM;
2557
2558        for (i = 0; i < priv->num_rx_bds; i++) {
2559                cb = priv->rx_cbs + i;
2560                cb->bd_addr = priv->rx_bds + i * DMA_DESC_SIZE;
2561        }
2562
2563        /* Initialize common TX ring structures */
2564        priv->tx_bds = priv->base + priv->hw_params->tdma_offset;
2565        priv->num_tx_bds = TOTAL_DESC;
2566        priv->tx_cbs = kcalloc(priv->num_tx_bds, sizeof(struct enet_cb),
2567                               GFP_KERNEL);
2568        if (!priv->tx_cbs) {
2569                kfree(priv->rx_cbs);
2570                return -ENOMEM;
2571        }
2572
2573        for (i = 0; i < priv->num_tx_bds; i++) {
2574                cb = priv->tx_cbs + i;
2575                cb->bd_addr = priv->tx_bds + i * DMA_DESC_SIZE;
2576        }
2577
2578        /* Init rDma */
2579        bcmgenet_rdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE);
2580
2581        /* Initialize Rx queues */
2582        ret = bcmgenet_init_rx_queues(priv->dev);
2583        if (ret) {
2584                netdev_err(priv->dev, "failed to initialize Rx queues\n");
2585                bcmgenet_free_rx_buffers(priv);
2586                kfree(priv->rx_cbs);
2587                kfree(priv->tx_cbs);
2588                return ret;
2589        }
2590
2591        /* Init tDma */
2592        bcmgenet_tdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE);
2593
2594        /* Initialize Tx queues */
2595        bcmgenet_init_tx_queues(priv->dev);
2596
2597        return 0;
2598}
2599
2600/* Interrupt bottom half */
2601static void bcmgenet_irq_task(struct work_struct *work)
2602{
2603        unsigned int status;
2604        struct bcmgenet_priv *priv = container_of(
2605                        work, struct bcmgenet_priv, bcmgenet_irq_work);
2606
2607        netif_dbg(priv, intr, priv->dev, "%s\n", __func__);
2608
2609        spin_lock_irq(&priv->lock);
2610        status = priv->irq0_stat;
2611        priv->irq0_stat = 0;
2612        spin_unlock_irq(&priv->lock);
2613
2614        if (status & UMAC_IRQ_PHY_DET_R &&
2615            priv->dev->phydev->autoneg != AUTONEG_ENABLE) {
2616                phy_init_hw(priv->dev->phydev);
2617                genphy_config_aneg(priv->dev->phydev);
2618        }
2619
2620        /* Link UP/DOWN event */
2621        if (status & UMAC_IRQ_LINK_EVENT)
2622                phy_mac_interrupt(priv->dev->phydev);
2623
2624}
2625
2626/* bcmgenet_isr1: handle Rx and Tx priority queues */
2627static irqreturn_t bcmgenet_isr1(int irq, void *dev_id)
2628{
2629        struct bcmgenet_priv *priv = dev_id;
2630        struct bcmgenet_rx_ring *rx_ring;
2631        struct bcmgenet_tx_ring *tx_ring;
2632        unsigned int index, status;
2633
2634        /* Read irq status */
2635        status = bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) &
2636                ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
2637
2638        /* clear interrupts */
2639        bcmgenet_intrl2_1_writel(priv, status, INTRL2_CPU_CLEAR);
2640
2641        netif_dbg(priv, intr, priv->dev,
2642                  "%s: IRQ=0x%x\n", __func__, status);
2643
2644        /* Check Rx priority queue interrupts */
2645        for (index = 0; index < priv->hw_params->rx_queues; index++) {
2646                if (!(status & BIT(UMAC_IRQ1_RX_INTR_SHIFT + index)))
2647                        continue;
2648
2649                rx_ring = &priv->rx_rings[index];
2650                rx_ring->dim.event_ctr++;
2651
2652                if (likely(napi_schedule_prep(&rx_ring->napi))) {
2653                        rx_ring->int_disable(rx_ring);
2654                        __napi_schedule_irqoff(&rx_ring->napi);
2655                }
2656        }
2657
2658        /* Check Tx priority queue interrupts */
2659        for (index = 0; index < priv->hw_params->tx_queues; index++) {
2660                if (!(status & BIT(index)))
2661                        continue;
2662
2663                tx_ring = &priv->tx_rings[index];
2664
2665                if (likely(napi_schedule_prep(&tx_ring->napi))) {
2666                        tx_ring->int_disable(tx_ring);
2667                        __napi_schedule_irqoff(&tx_ring->napi);
2668                }
2669        }
2670
2671        return IRQ_HANDLED;
2672}
2673
2674/* bcmgenet_isr0: handle Rx and Tx default queues + other stuff */
2675static irqreturn_t bcmgenet_isr0(int irq, void *dev_id)
2676{
2677        struct bcmgenet_priv *priv = dev_id;
2678        struct bcmgenet_rx_ring *rx_ring;
2679        struct bcmgenet_tx_ring *tx_ring;
2680        unsigned int status;
2681        unsigned long flags;
2682
2683        /* Read irq status */
2684        status = bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) &
2685                ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
2686
2687        /* clear interrupts */
2688        bcmgenet_intrl2_0_writel(priv, status, INTRL2_CPU_CLEAR);
2689
2690        netif_dbg(priv, intr, priv->dev,
2691                  "IRQ=0x%x\n", status);
2692
2693        if (status & UMAC_IRQ_RXDMA_DONE) {
2694                rx_ring = &priv->rx_rings[DESC_INDEX];
2695                rx_ring->dim.event_ctr++;
2696
2697                if (likely(napi_schedule_prep(&rx_ring->napi))) {
2698                        rx_ring->int_disable(rx_ring);
2699                        __napi_schedule_irqoff(&rx_ring->napi);
2700                }
2701        }
2702
2703        if (status & UMAC_IRQ_TXDMA_DONE) {
2704                tx_ring = &priv->tx_rings[DESC_INDEX];
2705
2706                if (likely(napi_schedule_prep(&tx_ring->napi))) {
2707                        tx_ring->int_disable(tx_ring);
2708                        __napi_schedule_irqoff(&tx_ring->napi);
2709                }
2710        }
2711
2712        if ((priv->hw_params->flags & GENET_HAS_MDIO_INTR) &&
2713                status & (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR)) {
2714                wake_up(&priv->wq);
2715        }
2716
2717        /* all other interested interrupts handled in bottom half */
2718        status &= (UMAC_IRQ_LINK_EVENT | UMAC_IRQ_PHY_DET_R);
2719        if (status) {
2720                /* Save irq status for bottom-half processing. */
2721                spin_lock_irqsave(&priv->lock, flags);
2722                priv->irq0_stat |= status;
2723                spin_unlock_irqrestore(&priv->lock, flags);
2724
2725                schedule_work(&priv->bcmgenet_irq_work);
2726        }
2727
2728        return IRQ_HANDLED;
2729}
2730
2731static irqreturn_t bcmgenet_wol_isr(int irq, void *dev_id)
2732{
2733        struct bcmgenet_priv *priv = dev_id;
2734
2735        pm_wakeup_event(&priv->pdev->dev, 0);
2736
2737        return IRQ_HANDLED;
2738}
2739
2740#ifdef CONFIG_NET_POLL_CONTROLLER
2741static void bcmgenet_poll_controller(struct net_device *dev)
2742{
2743        struct bcmgenet_priv *priv = netdev_priv(dev);
2744
2745        /* Invoke the main RX/TX interrupt handler */
2746        disable_irq(priv->irq0);
2747        bcmgenet_isr0(priv->irq0, priv);
2748        enable_irq(priv->irq0);
2749
2750        /* And the interrupt handler for RX/TX priority queues */
2751        disable_irq(priv->irq1);
2752        bcmgenet_isr1(priv->irq1, priv);
2753        enable_irq(priv->irq1);
2754}
2755#endif
2756
2757static void bcmgenet_umac_reset(struct bcmgenet_priv *priv)
2758{
2759        u32 reg;
2760
2761        reg = bcmgenet_rbuf_ctrl_get(priv);
2762        reg |= BIT(1);
2763        bcmgenet_rbuf_ctrl_set(priv, reg);
2764        udelay(10);
2765
2766        reg &= ~BIT(1);
2767        bcmgenet_rbuf_ctrl_set(priv, reg);
2768        udelay(10);
2769}
2770
2771static void bcmgenet_set_hw_addr(struct bcmgenet_priv *priv,
2772                                 unsigned char *addr)
2773{
2774        bcmgenet_umac_writel(priv, (addr[0] << 24) | (addr[1] << 16) |
2775                        (addr[2] << 8) | addr[3], UMAC_MAC0);
2776        bcmgenet_umac_writel(priv, (addr[4] << 8) | addr[5], UMAC_MAC1);
2777}
2778
2779/* Returns a reusable dma control register value */
2780static u32 bcmgenet_dma_disable(struct bcmgenet_priv *priv)
2781{
2782        u32 reg;
2783        u32 dma_ctrl;
2784
2785        /* disable DMA */
2786        dma_ctrl = 1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT) | DMA_EN;
2787        reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2788        reg &= ~dma_ctrl;
2789        bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2790
2791        reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2792        reg &= ~dma_ctrl;
2793        bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2794
2795        bcmgenet_umac_writel(priv, 1, UMAC_TX_FLUSH);
2796        udelay(10);
2797        bcmgenet_umac_writel(priv, 0, UMAC_TX_FLUSH);
2798
2799        return dma_ctrl;
2800}
2801
2802static void bcmgenet_enable_dma(struct bcmgenet_priv *priv, u32 dma_ctrl)
2803{
2804        u32 reg;
2805
2806        reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2807        reg |= dma_ctrl;
2808        bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2809
2810        reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2811        reg |= dma_ctrl;
2812        bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2813}
2814
2815/* bcmgenet_hfb_clear
2816 *
2817 * Clear Hardware Filter Block and disable all filtering.
2818 */
2819static void bcmgenet_hfb_clear(struct bcmgenet_priv *priv)
2820{
2821        u32 i;
2822
2823        bcmgenet_hfb_reg_writel(priv, 0x0, HFB_CTRL);
2824        bcmgenet_hfb_reg_writel(priv, 0x0, HFB_FLT_ENABLE_V3PLUS);
2825        bcmgenet_hfb_reg_writel(priv, 0x0, HFB_FLT_ENABLE_V3PLUS + 4);
2826
2827        for (i = DMA_INDEX2RING_0; i <= DMA_INDEX2RING_7; i++)
2828                bcmgenet_rdma_writel(priv, 0x0, i);
2829
2830        for (i = 0; i < (priv->hw_params->hfb_filter_cnt / 4); i++)
2831                bcmgenet_hfb_reg_writel(priv, 0x0,
2832                                        HFB_FLT_LEN_V3PLUS + i * sizeof(u32));
2833
2834        for (i = 0; i < priv->hw_params->hfb_filter_cnt *
2835                        priv->hw_params->hfb_filter_size; i++)
2836                bcmgenet_hfb_writel(priv, 0x0, i * sizeof(u32));
2837}
2838
2839static void bcmgenet_hfb_init(struct bcmgenet_priv *priv)
2840{
2841        if (GENET_IS_V1(priv) || GENET_IS_V2(priv))
2842                return;
2843
2844        bcmgenet_hfb_clear(priv);
2845}
2846
2847static void bcmgenet_netif_start(struct net_device *dev)
2848{
2849        struct bcmgenet_priv *priv = netdev_priv(dev);
2850
2851        /* Start the network engine */
2852        bcmgenet_enable_rx_napi(priv);
2853
2854        umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, true);
2855
2856        bcmgenet_enable_tx_napi(priv);
2857
2858        /* Monitor link interrupts now */
2859        bcmgenet_link_intr_enable(priv);
2860
2861        phy_start(dev->phydev);
2862}
2863
2864static int bcmgenet_open(struct net_device *dev)
2865{
2866        struct bcmgenet_priv *priv = netdev_priv(dev);
2867        unsigned long dma_ctrl;
2868        u32 reg;
2869        int ret;
2870
2871        netif_dbg(priv, ifup, dev, "bcmgenet_open\n");
2872
2873        /* Turn on the clock */
2874        clk_prepare_enable(priv->clk);
2875
2876        /* If this is an internal GPHY, power it back on now, before UniMAC is
2877         * brought out of reset as absolutely no UniMAC activity is allowed
2878         */
2879        if (priv->internal_phy)
2880                bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
2881
2882        /* take MAC out of reset */
2883        bcmgenet_umac_reset(priv);
2884
2885        init_umac(priv);
2886
2887        /* Make sure we reflect the value of CRC_CMD_FWD */
2888        reg = bcmgenet_umac_readl(priv, UMAC_CMD);
2889        priv->crc_fwd_en = !!(reg & CMD_CRC_FWD);
2890
2891        bcmgenet_set_hw_addr(priv, dev->dev_addr);
2892
2893        if (priv->internal_phy) {
2894                reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
2895                reg |= EXT_ENERGY_DET_MASK;
2896                bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
2897        }
2898
2899        /* Disable RX/TX DMA and flush TX queues */
2900        dma_ctrl = bcmgenet_dma_disable(priv);
2901
2902        /* Reinitialize TDMA and RDMA and SW housekeeping */
2903        ret = bcmgenet_init_dma(priv);
2904        if (ret) {
2905                netdev_err(dev, "failed to initialize DMA\n");
2906                goto err_clk_disable;
2907        }
2908
2909        /* Always enable ring 16 - descriptor ring */
2910        bcmgenet_enable_dma(priv, dma_ctrl);
2911
2912        /* HFB init */
2913        bcmgenet_hfb_init(priv);
2914
2915        ret = request_irq(priv->irq0, bcmgenet_isr0, IRQF_SHARED,
2916                          dev->name, priv);
2917        if (ret < 0) {
2918                netdev_err(dev, "can't request IRQ %d\n", priv->irq0);
2919                goto err_fini_dma;
2920        }
2921
2922        ret = request_irq(priv->irq1, bcmgenet_isr1, IRQF_SHARED,
2923                          dev->name, priv);
2924        if (ret < 0) {
2925                netdev_err(dev, "can't request IRQ %d\n", priv->irq1);
2926                goto err_irq0;
2927        }
2928
2929        ret = bcmgenet_mii_probe(dev);
2930        if (ret) {
2931                netdev_err(dev, "failed to connect to PHY\n");
2932                goto err_irq1;
2933        }
2934
2935        bcmgenet_netif_start(dev);
2936
2937        netif_tx_start_all_queues(dev);
2938
2939        return 0;
2940
2941err_irq1:
2942        free_irq(priv->irq1, priv);
2943err_irq0:
2944        free_irq(priv->irq0, priv);
2945err_fini_dma:
2946        bcmgenet_dma_teardown(priv);
2947        bcmgenet_fini_dma(priv);
2948err_clk_disable:
2949        if (priv->internal_phy)
2950                bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
2951        clk_disable_unprepare(priv->clk);
2952        return ret;
2953}
2954
2955static void bcmgenet_netif_stop(struct net_device *dev)
2956{
2957        struct bcmgenet_priv *priv = netdev_priv(dev);
2958
2959        bcmgenet_disable_tx_napi(priv);
2960        netif_tx_disable(dev);
2961
2962        /* Disable MAC receive */
2963        umac_enable_set(priv, CMD_RX_EN, false);
2964
2965        bcmgenet_dma_teardown(priv);
2966
2967        /* Disable MAC transmit. TX DMA disabled must be done before this */
2968        umac_enable_set(priv, CMD_TX_EN, false);
2969
2970        phy_stop(dev->phydev);
2971        bcmgenet_disable_rx_napi(priv);
2972        bcmgenet_intr_disable(priv);
2973
2974        /* Wait for pending work items to complete. Since interrupts are
2975         * disabled no new work will be scheduled.
2976         */
2977        cancel_work_sync(&priv->bcmgenet_irq_work);
2978
2979        priv->old_link = -1;
2980        priv->old_speed = -1;
2981        priv->old_duplex = -1;
2982        priv->old_pause = -1;
2983
2984        /* tx reclaim */
2985        bcmgenet_tx_reclaim_all(dev);
2986        bcmgenet_fini_dma(priv);
2987}
2988
2989static int bcmgenet_close(struct net_device *dev)
2990{
2991        struct bcmgenet_priv *priv = netdev_priv(dev);
2992        int ret = 0;
2993
2994        netif_dbg(priv, ifdown, dev, "bcmgenet_close\n");
2995
2996        bcmgenet_netif_stop(dev);
2997
2998        /* Really kill the PHY state machine and disconnect from it */
2999        phy_disconnect(dev->phydev);
3000
3001        free_irq(priv->irq0, priv);
3002        free_irq(priv->irq1, priv);
3003
3004        if (priv->internal_phy)
3005                ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
3006
3007        clk_disable_unprepare(priv->clk);
3008
3009        return ret;
3010}
3011
3012static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring)
3013{
3014        struct bcmgenet_priv *priv = ring->priv;
3015        u32 p_index, c_index, intsts, intmsk;
3016        struct netdev_queue *txq;
3017        unsigned int free_bds;
3018        bool txq_stopped;
3019
3020        if (!netif_msg_tx_err(priv))
3021                return;
3022
3023        txq = netdev_get_tx_queue(priv->dev, ring->queue);
3024
3025        spin_lock(&ring->lock);
3026        if (ring->index == DESC_INDEX) {
3027                intsts = ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
3028                intmsk = UMAC_IRQ_TXDMA_DONE | UMAC_IRQ_TXDMA_MBDONE;
3029        } else {
3030                intsts = ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
3031                intmsk = 1 << ring->index;
3032        }
3033        c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);
3034        p_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_PROD_INDEX);
3035        txq_stopped = netif_tx_queue_stopped(txq);
3036        free_bds = ring->free_bds;
3037        spin_unlock(&ring->lock);
3038
3039        netif_err(priv, tx_err, priv->dev, "Ring %d queue %d status summary\n"
3040                  "TX queue status: %s, interrupts: %s\n"
3041                  "(sw)free_bds: %d (sw)size: %d\n"
3042                  "(sw)p_index: %d (hw)p_index: %d\n"
3043                  "(sw)c_index: %d (hw)c_index: %d\n"
3044                  "(sw)clean_p: %d (sw)write_p: %d\n"
3045                  "(sw)cb_ptr: %d (sw)end_ptr: %d\n",
3046                  ring->index, ring->queue,
3047                  txq_stopped ? "stopped" : "active",
3048                  intsts & intmsk ? "enabled" : "disabled",
3049                  free_bds, ring->size,
3050                  ring->prod_index, p_index & DMA_P_INDEX_MASK,
3051                  ring->c_index, c_index & DMA_C_INDEX_MASK,
3052                  ring->clean_ptr, ring->write_ptr,
3053                  ring->cb_ptr, ring->end_ptr);
3054}
3055
3056static void bcmgenet_timeout(struct net_device *dev)
3057{
3058        struct bcmgenet_priv *priv = netdev_priv(dev);
3059        u32 int0_enable = 0;
3060        u32 int1_enable = 0;
3061        unsigned int q;
3062
3063        netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n");
3064
3065        for (q = 0; q < priv->hw_params->tx_queues; q++)
3066                bcmgenet_dump_tx_queue(&priv->tx_rings[q]);
3067        bcmgenet_dump_tx_queue(&priv->tx_rings[DESC_INDEX]);
3068
3069        bcmgenet_tx_reclaim_all(dev);
3070
3071        for (q = 0; q < priv->hw_params->tx_queues; q++)
3072                int1_enable |= (1 << q);
3073
3074        int0_enable = UMAC_IRQ_TXDMA_DONE;
3075
3076        /* Re-enable TX interrupts if disabled */
3077        bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
3078        bcmgenet_intrl2_1_writel(priv, int1_enable, INTRL2_CPU_MASK_CLEAR);
3079
3080        netif_trans_update(dev);
3081
3082        dev->stats.tx_errors++;
3083
3084        netif_tx_wake_all_queues(dev);
3085}
3086
3087#define MAX_MDF_FILTER  17
3088
3089static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv,
3090                                         unsigned char *addr,
3091                                         int *i)
3092{
3093        bcmgenet_umac_writel(priv, addr[0] << 8 | addr[1],
3094                             UMAC_MDF_ADDR + (*i * 4));
3095        bcmgenet_umac_writel(priv, addr[2] << 24 | addr[3] << 16 |
3096                             addr[4] << 8 | addr[5],
3097                             UMAC_MDF_ADDR + ((*i + 1) * 4));
3098        *i += 2;
3099}
3100
3101static void bcmgenet_set_rx_mode(struct net_device *dev)
3102{
3103        struct bcmgenet_priv *priv = netdev_priv(dev);
3104        struct netdev_hw_addr *ha;
3105        int i, nfilter;
3106        u32 reg;
3107
3108        netif_dbg(priv, hw, dev, "%s: %08X\n", __func__, dev->flags);
3109
3110        /* Number of filters needed */
3111        nfilter = netdev_uc_count(dev) + netdev_mc_count(dev) + 2;
3112
3113        /*
3114         * Turn on promicuous mode for three scenarios
3115         * 1. IFF_PROMISC flag is set
3116         * 2. IFF_ALLMULTI flag is set
3117         * 3. The number of filters needed exceeds the number filters
3118         *    supported by the hardware.
3119        */
3120        reg = bcmgenet_umac_readl(priv, UMAC_CMD);
3121        if ((dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) ||
3122            (nfilter > MAX_MDF_FILTER)) {
3123                reg |= CMD_PROMISC;
3124                bcmgenet_umac_writel(priv, reg, UMAC_CMD);
3125                bcmgenet_umac_writel(priv, 0, UMAC_MDF_CTRL);
3126                return;
3127        } else {
3128                reg &= ~CMD_PROMISC;
3129                bcmgenet_umac_writel(priv, reg, UMAC_CMD);
3130        }
3131
3132        /* update MDF filter */
3133        i = 0;
3134        /* Broadcast */
3135        bcmgenet_set_mdf_addr(priv, dev->broadcast, &i);
3136        /* my own address.*/
3137        bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i);
3138
3139        /* Unicast */
3140        netdev_for_each_uc_addr(ha, dev)
3141                bcmgenet_set_mdf_addr(priv, ha->addr, &i);
3142
3143        /* Multicast */
3144        netdev_for_each_mc_addr(ha, dev)
3145                bcmgenet_set_mdf_addr(priv, ha->addr, &i);
3146
3147        /* Enable filters */
3148        reg = GENMASK(MAX_MDF_FILTER - 1, MAX_MDF_FILTER - nfilter);
3149        bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL);
3150}
3151
3152/* Set the hardware MAC address. */
3153static int bcmgenet_set_mac_addr(struct net_device *dev, void *p)
3154{
3155        struct sockaddr *addr = p;
3156
3157        /* Setting the MAC address at the hardware level is not possible
3158         * without disabling the UniMAC RX/TX enable bits.
3159         */
3160        if (netif_running(dev))
3161                return -EBUSY;
3162
3163        ether_addr_copy(dev->dev_addr, addr->sa_data);
3164
3165        return 0;
3166}
3167
3168static struct net_device_stats *bcmgenet_get_stats(struct net_device *dev)
3169{
3170        struct bcmgenet_priv *priv = netdev_priv(dev);
3171        unsigned long tx_bytes = 0, tx_packets = 0;
3172        unsigned long rx_bytes = 0, rx_packets = 0;
3173        unsigned long rx_errors = 0, rx_dropped = 0;
3174        struct bcmgenet_tx_ring *tx_ring;
3175        struct bcmgenet_rx_ring *rx_ring;
3176        unsigned int q;
3177
3178        for (q = 0; q < priv->hw_params->tx_queues; q++) {
3179                tx_ring = &priv->tx_rings[q];
3180                tx_bytes += tx_ring->bytes;
3181                tx_packets += tx_ring->packets;
3182        }
3183        tx_ring = &priv->tx_rings[DESC_INDEX];
3184        tx_bytes += tx_ring->bytes;
3185        tx_packets += tx_ring->packets;
3186
3187        for (q = 0; q < priv->hw_params->rx_queues; q++) {
3188                rx_ring = &priv->rx_rings[q];
3189
3190                rx_bytes += rx_ring->bytes;
3191                rx_packets += rx_ring->packets;
3192                rx_errors += rx_ring->errors;
3193                rx_dropped += rx_ring->dropped;
3194        }
3195        rx_ring = &priv->rx_rings[DESC_INDEX];
3196        rx_bytes += rx_ring->bytes;
3197        rx_packets += rx_ring->packets;
3198        rx_errors += rx_ring->errors;
3199        rx_dropped += rx_ring->dropped;
3200
3201        dev->stats.tx_bytes = tx_bytes;
3202        dev->stats.tx_packets = tx_packets;
3203        dev->stats.rx_bytes = rx_bytes;
3204        dev->stats.rx_packets = rx_packets;
3205        dev->stats.rx_errors = rx_errors;
3206        dev->stats.rx_missed_errors = rx_errors;
3207        return &dev->stats;
3208}
3209
3210static const struct net_device_ops bcmgenet_netdev_ops = {
3211        .ndo_open               = bcmgenet_open,
3212        .ndo_stop               = bcmgenet_close,
3213        .ndo_start_xmit         = bcmgenet_xmit,
3214        .ndo_tx_timeout         = bcmgenet_timeout,
3215        .ndo_set_rx_mode        = bcmgenet_set_rx_mode,
3216        .ndo_set_mac_address    = bcmgenet_set_mac_addr,
3217        .ndo_do_ioctl           = bcmgenet_ioctl,
3218        .ndo_set_features       = bcmgenet_set_features,
3219#ifdef CONFIG_NET_POLL_CONTROLLER
3220        .ndo_poll_controller    = bcmgenet_poll_controller,
3221#endif
3222        .ndo_get_stats          = bcmgenet_get_stats,
3223};
3224
3225/* Array of GENET hardware parameters/characteristics */
3226static struct bcmgenet_hw_params bcmgenet_hw_params[] = {
3227        [GENET_V1] = {
3228                .tx_queues = 0,
3229                .tx_bds_per_q = 0,
3230                .rx_queues = 0,
3231                .rx_bds_per_q = 0,
3232                .bp_in_en_shift = 16,
3233                .bp_in_mask = 0xffff,
3234                .hfb_filter_cnt = 16,
3235                .qtag_mask = 0x1F,
3236                .hfb_offset = 0x1000,
3237                .rdma_offset = 0x2000,
3238                .tdma_offset = 0x3000,
3239                .words_per_bd = 2,
3240        },
3241        [GENET_V2] = {
3242                .tx_queues = 4,
3243                .tx_bds_per_q = 32,
3244                .rx_queues = 0,
3245                .rx_bds_per_q = 0,
3246                .bp_in_en_shift = 16,
3247                .bp_in_mask = 0xffff,
3248                .hfb_filter_cnt = 16,
3249                .qtag_mask = 0x1F,
3250                .tbuf_offset = 0x0600,
3251                .hfb_offset = 0x1000,
3252                .hfb_reg_offset = 0x2000,
3253                .rdma_offset = 0x3000,
3254                .tdma_offset = 0x4000,
3255                .words_per_bd = 2,
3256                .flags = GENET_HAS_EXT,
3257        },
3258        [GENET_V3] = {
3259                .tx_queues = 4,
3260                .tx_bds_per_q = 32,
3261                .rx_queues = 0,
3262                .rx_bds_per_q = 0,
3263                .bp_in_en_shift = 17,
3264                .bp_in_mask = 0x1ffff,
3265                .hfb_filter_cnt = 48,
3266                .hfb_filter_size = 128,
3267                .qtag_mask = 0x3F,
3268                .tbuf_offset = 0x0600,
3269                .hfb_offset = 0x8000,
3270                .hfb_reg_offset = 0xfc00,
3271                .rdma_offset = 0x10000,
3272                .tdma_offset = 0x11000,
3273                .words_per_bd = 2,
3274                .flags = GENET_HAS_EXT | GENET_HAS_MDIO_INTR |
3275                         GENET_HAS_MOCA_LINK_DET,
3276        },
3277        [GENET_V4] = {
3278                .tx_queues = 4,
3279                .tx_bds_per_q = 32,
3280                .rx_queues = 0,
3281                .rx_bds_per_q = 0,
3282                .bp_in_en_shift = 17,
3283                .bp_in_mask = 0x1ffff,
3284                .hfb_filter_cnt = 48,
3285                .hfb_filter_size = 128,
3286                .qtag_mask = 0x3F,
3287                .tbuf_offset = 0x0600,
3288                .hfb_offset = 0x8000,
3289                .hfb_reg_offset = 0xfc00,
3290                .rdma_offset = 0x2000,
3291                .tdma_offset = 0x4000,
3292                .words_per_bd = 3,
3293                .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3294                         GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3295        },
3296        [GENET_V5] = {
3297                .tx_queues = 4,
3298                .tx_bds_per_q = 32,
3299                .rx_queues = 0,
3300                .rx_bds_per_q = 0,
3301                .bp_in_en_shift = 17,
3302                .bp_in_mask = 0x1ffff,
3303                .hfb_filter_cnt = 48,
3304                .hfb_filter_size = 128,
3305                .qtag_mask = 0x3F,
3306                .tbuf_offset = 0x0600,
3307                .hfb_offset = 0x8000,
3308                .hfb_reg_offset = 0xfc00,
3309                .rdma_offset = 0x2000,
3310                .tdma_offset = 0x4000,
3311                .words_per_bd = 3,
3312                .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3313                         GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3314        },
3315};
3316
3317/* Infer hardware parameters from the detected GENET version */
3318static void bcmgenet_set_hw_params(struct bcmgenet_priv *priv)
3319{
3320        struct bcmgenet_hw_params *params;
3321        u32 reg;
3322        u8 major;
3323        u16 gphy_rev;
3324
3325        if (GENET_IS_V5(priv) || GENET_IS_V4(priv)) {
3326                bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
3327                genet_dma_ring_regs = genet_dma_ring_regs_v4;
3328                priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS;
3329        } else if (GENET_IS_V3(priv)) {
3330                bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
3331                genet_dma_ring_regs = genet_dma_ring_regs_v123;
3332                priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS;
3333        } else if (GENET_IS_V2(priv)) {
3334                bcmgenet_dma_regs = bcmgenet_dma_regs_v2;
3335                genet_dma_ring_regs = genet_dma_ring_regs_v123;
3336                priv->dma_rx_chk_bit = DMA_RX_CHK_V12;
3337        } else if (GENET_IS_V1(priv)) {
3338                bcmgenet_dma_regs = bcmgenet_dma_regs_v1;
3339                genet_dma_ring_regs = genet_dma_ring_regs_v123;
3340                priv->dma_rx_chk_bit = DMA_RX_CHK_V12;
3341        }
3342
3343        /* enum genet_version starts at 1 */
3344        priv->hw_params = &bcmgenet_hw_params[priv->version];
3345        params = priv->hw_params;
3346
3347        /* Read GENET HW version */
3348        reg = bcmgenet_sys_readl(priv, SYS_REV_CTRL);
3349        major = (reg >> 24 & 0x0f);
3350        if (major == 6)
3351                major = 5;
3352        else if (major == 5)
3353                major = 4;
3354        else if (major == 0)
3355                major = 1;
3356        if (major != priv->version) {
3357                dev_err(&priv->pdev->dev,
3358                        "GENET version mismatch, got: %d, configured for: %d\n",
3359                        major, priv->version);
3360        }
3361
3362        /* Print the GENET core version */
3363        dev_info(&priv->pdev->dev, "GENET " GENET_VER_FMT,
3364                 major, (reg >> 16) & 0x0f, reg & 0xffff);
3365
3366        /* Store the integrated PHY revision for the MDIO probing function
3367         * to pass this information to the PHY driver. The PHY driver expects
3368         * to find the PHY major revision in bits 15:8 while the GENET register
3369         * stores that information in bits 7:0, account for that.
3370         *
3371         * On newer chips, starting with PHY revision G0, a new scheme is
3372         * deployed similar to the Starfighter 2 switch with GPHY major
3373         * revision in bits 15:8 and patch level in bits 7:0. Major revision 0
3374         * is reserved as well as special value 0x01ff, we have a small
3375         * heuristic to check for the new GPHY revision and re-arrange things
3376         * so the GPHY driver is happy.
3377         */
3378        gphy_rev = reg & 0xffff;
3379
3380        if (GENET_IS_V5(priv)) {
3381                /* The EPHY revision should come from the MDIO registers of
3382                 * the PHY not from GENET.
3383                 */
3384                if (gphy_rev != 0) {
3385                        pr_warn("GENET is reporting EPHY revision: 0x%04x\n",
3386                                gphy_rev);
3387                }
3388        /* This is reserved so should require special treatment */
3389        } else if (gphy_rev == 0 || gphy_rev == 0x01ff) {
3390                pr_warn("Invalid GPHY revision detected: 0x%04x\n", gphy_rev);
3391                return;
3392        /* This is the good old scheme, just GPHY major, no minor nor patch */
3393        } else if ((gphy_rev & 0xf0) != 0) {
3394                priv->gphy_rev = gphy_rev << 8;
3395        /* This is the new scheme, GPHY major rolls over with 0x10 = rev G0 */
3396        } else if ((gphy_rev & 0xff00) != 0) {
3397                priv->gphy_rev = gphy_rev;
3398        }
3399
3400#ifdef CONFIG_PHYS_ADDR_T_64BIT
3401        if (!(params->flags & GENET_HAS_40BITS))
3402                pr_warn("GENET does not support 40-bits PA\n");
3403#endif
3404
3405        pr_debug("Configuration for version: %d\n"
3406                "TXq: %1d, TXqBDs: %1d, RXq: %1d, RXqBDs: %1d\n"
3407                "BP << en: %2d, BP msk: 0x%05x\n"
3408                "HFB count: %2d, QTAQ msk: 0x%05x\n"
3409                "TBUF: 0x%04x, HFB: 0x%04x, HFBreg: 0x%04x\n"
3410                "RDMA: 0x%05x, TDMA: 0x%05x\n"
3411                "Words/BD: %d\n",
3412                priv->version,
3413                params->tx_queues, params->tx_bds_per_q,
3414                params->rx_queues, params->rx_bds_per_q,
3415                params->bp_in_en_shift, params->bp_in_mask,
3416                params->hfb_filter_cnt, params->qtag_mask,
3417                params->tbuf_offset, params->hfb_offset,
3418                params->hfb_reg_offset,
3419                params->rdma_offset, params->tdma_offset,
3420                params->words_per_bd);
3421}
3422
3423static const struct of_device_id bcmgenet_match[] = {
3424        { .compatible = "brcm,genet-v1", .data = (void *)GENET_V1 },
3425        { .compatible = "brcm,genet-v2", .data = (void *)GENET_V2 },
3426        { .compatible = "brcm,genet-v3", .data = (void *)GENET_V3 },
3427        { .compatible = "brcm,genet-v4", .data = (void *)GENET_V4 },
3428        { .compatible = "brcm,genet-v5", .data = (void *)GENET_V5 },
3429        { },
3430};
3431MODULE_DEVICE_TABLE(of, bcmgenet_match);
3432
3433static int bcmgenet_probe(struct platform_device *pdev)
3434{
3435        struct bcmgenet_platform_data *pd = pdev->dev.platform_data;
3436        struct device_node *dn = pdev->dev.of_node;
3437        const struct of_device_id *of_id = NULL;
3438        struct bcmgenet_priv *priv;
3439        struct net_device *dev;
3440        const void *macaddr;
3441        unsigned int i;
3442        int err = -EIO;
3443        const char *phy_mode_str;
3444
3445        /* Up to GENET_MAX_MQ_CNT + 1 TX queues and RX queues */
3446        dev = alloc_etherdev_mqs(sizeof(*priv), GENET_MAX_MQ_CNT + 1,
3447                                 GENET_MAX_MQ_CNT + 1);
3448        if (!dev) {
3449                dev_err(&pdev->dev, "can't allocate net device\n");
3450                return -ENOMEM;
3451        }
3452
3453        if (dn) {
3454                of_id = of_match_node(bcmgenet_match, dn);
3455                if (!of_id)
3456                        return -EINVAL;
3457        }
3458
3459        priv = netdev_priv(dev);
3460        priv->irq0 = platform_get_irq(pdev, 0);
3461        priv->irq1 = platform_get_irq(pdev, 1);
3462        priv->wol_irq = platform_get_irq(pdev, 2);
3463        if (!priv->irq0 || !priv->irq1) {
3464                dev_err(&pdev->dev, "can't find IRQs\n");
3465                err = -EINVAL;
3466                goto err;
3467        }
3468
3469        if (dn) {
3470                macaddr = of_get_mac_address(dn);
3471                if (IS_ERR(macaddr)) {
3472                        dev_err(&pdev->dev, "can't find MAC address\n");
3473                        err = -EINVAL;
3474                        goto err;
3475                }
3476        } else {
3477                macaddr = pd->mac_address;
3478        }
3479
3480        priv->base = devm_platform_ioremap_resource(pdev, 0);
3481        if (IS_ERR(priv->base)) {
3482                err = PTR_ERR(priv->base);
3483                goto err;
3484        }
3485
3486        spin_lock_init(&priv->lock);
3487
3488        SET_NETDEV_DEV(dev, &pdev->dev);
3489        dev_set_drvdata(&pdev->dev, dev);
3490        ether_addr_copy(dev->dev_addr, macaddr);
3491        dev->watchdog_timeo = 2 * HZ;
3492        dev->ethtool_ops = &bcmgenet_ethtool_ops;
3493        dev->netdev_ops = &bcmgenet_netdev_ops;
3494
3495        priv->msg_enable = netif_msg_init(-1, GENET_MSG_DEFAULT);
3496
3497        /* Set hardware features */
3498        dev->hw_features |= NETIF_F_SG | NETIF_F_IP_CSUM |
3499                NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM;
3500
3501        /* Request the WOL interrupt and advertise suspend if available */
3502        priv->wol_irq_disabled = true;
3503        err = devm_request_irq(&pdev->dev, priv->wol_irq, bcmgenet_wol_isr, 0,
3504                               dev->name, priv);
3505        if (!err)
3506                device_set_wakeup_capable(&pdev->dev, 1);
3507
3508        /* Set the needed headroom to account for any possible
3509         * features enabling/disabling at runtime
3510         */
3511        dev->needed_headroom += 64;
3512
3513        netdev_boot_setup_check(dev);
3514
3515        priv->dev = dev;
3516        priv->pdev = pdev;
3517        if (of_id)
3518                priv->version = (enum bcmgenet_version)of_id->data;
3519        else
3520                priv->version = pd->genet_version;
3521
3522        priv->clk = devm_clk_get(&priv->pdev->dev, "enet");
3523        if (IS_ERR(priv->clk)) {
3524                dev_warn(&priv->pdev->dev, "failed to get enet clock\n");
3525                priv->clk = NULL;
3526        }
3527
3528        clk_prepare_enable(priv->clk);
3529
3530        bcmgenet_set_hw_params(priv);
3531
3532        /* Mii wait queue */
3533        init_waitqueue_head(&priv->wq);
3534        /* Always use RX_BUF_LENGTH (2KB) buffer for all chips */
3535        priv->rx_buf_len = RX_BUF_LENGTH;
3536        INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task);
3537
3538        priv->clk_wol = devm_clk_get(&priv->pdev->dev, "enet-wol");
3539        if (IS_ERR(priv->clk_wol)) {
3540                dev_warn(&priv->pdev->dev, "failed to get enet-wol clock\n");
3541                priv->clk_wol = NULL;
3542        }
3543
3544        priv->clk_eee = devm_clk_get(&priv->pdev->dev, "enet-eee");
3545        if (IS_ERR(priv->clk_eee)) {
3546                dev_warn(&priv->pdev->dev, "failed to get enet-eee clock\n");
3547                priv->clk_eee = NULL;
3548        }
3549
3550        /* If this is an internal GPHY, power it on now, before UniMAC is
3551         * brought out of reset as absolutely no UniMAC activity is allowed
3552         */
3553        if (dn && !of_property_read_string(dn, "phy-mode", &phy_mode_str) &&
3554            !strcasecmp(phy_mode_str, "internal"))
3555                bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
3556
3557        reset_umac(priv);
3558
3559        err = bcmgenet_mii_init(dev);
3560        if (err)
3561                goto err_clk_disable;
3562
3563        /* setup number of real queues  + 1 (GENET_V1 has 0 hardware queues
3564         * just the ring 16 descriptor based TX
3565         */
3566        netif_set_real_num_tx_queues(priv->dev, priv->hw_params->tx_queues + 1);
3567        netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1);
3568
3569        /* Set default coalescing parameters */
3570        for (i = 0; i < priv->hw_params->rx_queues; i++)
3571                priv->rx_rings[i].rx_max_coalesced_frames = 1;
3572        priv->rx_rings[DESC_INDEX].rx_max_coalesced_frames = 1;
3573
3574        /* libphy will determine the link state */
3575        netif_carrier_off(dev);
3576
3577        /* Turn off the main clock, WOL clock is handled separately */
3578        clk_disable_unprepare(priv->clk);
3579
3580        err = register_netdev(dev);
3581        if (err)
3582                goto err;
3583
3584        return err;
3585
3586err_clk_disable:
3587        clk_disable_unprepare(priv->clk);
3588err:
3589        free_netdev(dev);
3590        return err;
3591}
3592
3593static int bcmgenet_remove(struct platform_device *pdev)
3594{
3595        struct bcmgenet_priv *priv = dev_to_priv(&pdev->dev);
3596
3597        dev_set_drvdata(&pdev->dev, NULL);
3598        unregister_netdev(priv->dev);
3599        bcmgenet_mii_exit(priv->dev);
3600        free_netdev(priv->dev);
3601
3602        return 0;
3603}
3604
3605#ifdef CONFIG_PM_SLEEP
3606static int bcmgenet_resume(struct device *d)
3607{
3608        struct net_device *dev = dev_get_drvdata(d);
3609        struct bcmgenet_priv *priv = netdev_priv(dev);
3610        unsigned long dma_ctrl;
3611        int ret;
3612        u32 reg;
3613
3614        if (!netif_running(dev))
3615                return 0;
3616
3617        /* Turn on the clock */
3618        ret = clk_prepare_enable(priv->clk);
3619        if (ret)
3620                return ret;
3621
3622        /* If this is an internal GPHY, power it back on now, before UniMAC is
3623         * brought out of reset as absolutely no UniMAC activity is allowed
3624         */
3625        if (priv->internal_phy)
3626                bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
3627
3628        bcmgenet_umac_reset(priv);
3629
3630        init_umac(priv);
3631
3632        /* From WOL-enabled suspend, switch to regular clock */
3633        if (priv->wolopts)
3634                clk_disable_unprepare(priv->clk_wol);
3635
3636        phy_init_hw(dev->phydev);
3637
3638        /* Speed settings must be restored */
3639        genphy_config_aneg(dev->phydev);
3640        bcmgenet_mii_config(priv->dev, false);
3641
3642        bcmgenet_set_hw_addr(priv, dev->dev_addr);
3643
3644        if (priv->internal_phy) {
3645                reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
3646                reg |= EXT_ENERGY_DET_MASK;
3647                bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
3648        }
3649
3650        if (priv->wolopts)
3651                bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC);
3652
3653        /* Disable RX/TX DMA and flush TX queues */
3654        dma_ctrl = bcmgenet_dma_disable(priv);
3655
3656        /* Reinitialize TDMA and RDMA and SW housekeeping */
3657        ret = bcmgenet_init_dma(priv);
3658        if (ret) {
3659                netdev_err(dev, "failed to initialize DMA\n");
3660                goto out_clk_disable;
3661        }
3662
3663        /* Always enable ring 16 - descriptor ring */
3664        bcmgenet_enable_dma(priv, dma_ctrl);
3665
3666        if (!device_may_wakeup(d))
3667                phy_resume(dev->phydev);
3668
3669        if (priv->eee.eee_enabled)
3670                bcmgenet_eee_enable_set(dev, true);
3671
3672        bcmgenet_netif_start(dev);
3673
3674        netif_device_attach(dev);
3675
3676        return 0;
3677
3678out_clk_disable:
3679        if (priv->internal_phy)
3680                bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
3681        clk_disable_unprepare(priv->clk);
3682        return ret;
3683}
3684
3685static int bcmgenet_suspend(struct device *d)
3686{
3687        struct net_device *dev = dev_get_drvdata(d);
3688        struct bcmgenet_priv *priv = netdev_priv(dev);
3689        int ret = 0;
3690
3691        if (!netif_running(dev))
3692                return 0;
3693
3694        netif_device_detach(dev);
3695
3696        bcmgenet_netif_stop(dev);
3697
3698        if (!device_may_wakeup(d))
3699                phy_suspend(dev->phydev);
3700
3701        /* Prepare the device for Wake-on-LAN and switch to the slow clock */
3702        if (device_may_wakeup(d) && priv->wolopts) {
3703                ret = bcmgenet_power_down(priv, GENET_POWER_WOL_MAGIC);
3704                clk_prepare_enable(priv->clk_wol);
3705        } else if (priv->internal_phy) {
3706                ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
3707        }
3708
3709        /* Turn off the clocks */
3710        clk_disable_unprepare(priv->clk);
3711
3712        if (ret)
3713                bcmgenet_resume(d);
3714
3715        return ret;
3716}
3717#endif /* CONFIG_PM_SLEEP */
3718
3719static SIMPLE_DEV_PM_OPS(bcmgenet_pm_ops, bcmgenet_suspend, bcmgenet_resume);
3720
3721static struct platform_driver bcmgenet_driver = {
3722        .probe  = bcmgenet_probe,
3723        .remove = bcmgenet_remove,
3724        .driver = {
3725                .name   = "bcmgenet",
3726                .of_match_table = bcmgenet_match,
3727                .pm     = &bcmgenet_pm_ops,
3728        },
3729};
3730module_platform_driver(bcmgenet_driver);
3731
3732MODULE_AUTHOR("Broadcom Corporation");
3733MODULE_DESCRIPTION("Broadcom GENET Ethernet controller driver");
3734MODULE_ALIAS("platform:bcmgenet");
3735MODULE_LICENSE("GPL");
3736