linux/drivers/net/ethernet/hisilicon/hns/hns_enet.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (c) 2014-2015 Hisilicon Limited.
   4 */
   5
   6#include <linux/clk.h>
   7#include <linux/cpumask.h>
   8#include <linux/etherdevice.h>
   9#include <linux/if_vlan.h>
  10#include <linux/interrupt.h>
  11#include <linux/io.h>
  12#include <linux/ip.h>
  13#include <linux/ipv6.h>
  14#include <linux/module.h>
  15#include <linux/phy.h>
  16#include <linux/platform_device.h>
  17#include <linux/skbuff.h>
  18
  19#include "hnae.h"
  20#include "hns_enet.h"
  21#include "hns_dsaf_mac.h"
  22
  23#define NIC_MAX_Q_PER_VF 16
  24#define HNS_NIC_TX_TIMEOUT (5 * HZ)
  25
  26#define SERVICE_TIMER_HZ (1 * HZ)
  27
  28#define RCB_IRQ_NOT_INITED 0
  29#define RCB_IRQ_INITED 1
  30#define HNS_BUFFER_SIZE_2048 2048
  31
  32#define BD_MAX_SEND_SIZE 8191
  33#define SKB_TMP_LEN(SKB) \
  34        (((SKB)->transport_header - (SKB)->mac_header) + tcp_hdrlen(SKB))
  35
  36static void fill_v2_desc_hw(struct hnae_ring *ring, void *priv, int size,
  37                            int send_sz, dma_addr_t dma, int frag_end,
  38                            int buf_num, enum hns_desc_type type, int mtu)
  39{
  40        struct hnae_desc *desc = &ring->desc[ring->next_to_use];
  41        struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
  42        struct iphdr *iphdr;
  43        struct ipv6hdr *ipv6hdr;
  44        struct sk_buff *skb;
  45        __be16 protocol;
  46        u8 bn_pid = 0;
  47        u8 rrcfv = 0;
  48        u8 ip_offset = 0;
  49        u8 tvsvsn = 0;
  50        u16 mss = 0;
  51        u8 l4_len = 0;
  52        u16 paylen = 0;
  53
  54        desc_cb->priv = priv;
  55        desc_cb->length = size;
  56        desc_cb->dma = dma;
  57        desc_cb->type = type;
  58
  59        desc->addr = cpu_to_le64(dma);
  60        desc->tx.send_size = cpu_to_le16((u16)send_sz);
  61
  62        /* config bd buffer end */
  63        hnae_set_bit(rrcfv, HNSV2_TXD_VLD_B, 1);
  64        hnae_set_field(bn_pid, HNSV2_TXD_BUFNUM_M, 0, buf_num - 1);
  65
  66        /* fill port_id in the tx bd for sending management pkts */
  67        hnae_set_field(bn_pid, HNSV2_TXD_PORTID_M,
  68                       HNSV2_TXD_PORTID_S, ring->q->handle->dport_id);
  69
  70        if (type == DESC_TYPE_SKB) {
  71                skb = (struct sk_buff *)priv;
  72
  73                if (skb->ip_summed == CHECKSUM_PARTIAL) {
  74                        skb_reset_mac_len(skb);
  75                        protocol = skb->protocol;
  76                        ip_offset = ETH_HLEN;
  77
  78                        if (protocol == htons(ETH_P_8021Q)) {
  79                                ip_offset += VLAN_HLEN;
  80                                protocol = vlan_get_protocol(skb);
  81                                skb->protocol = protocol;
  82                        }
  83
  84                        if (skb->protocol == htons(ETH_P_IP)) {
  85                                iphdr = ip_hdr(skb);
  86                                hnae_set_bit(rrcfv, HNSV2_TXD_L3CS_B, 1);
  87                                hnae_set_bit(rrcfv, HNSV2_TXD_L4CS_B, 1);
  88
  89                                /* check for tcp/udp header */
  90                                if (iphdr->protocol == IPPROTO_TCP &&
  91                                    skb_is_gso(skb)) {
  92                                        hnae_set_bit(tvsvsn,
  93                                                     HNSV2_TXD_TSE_B, 1);
  94                                        l4_len = tcp_hdrlen(skb);
  95                                        mss = skb_shinfo(skb)->gso_size;
  96                                        paylen = skb->len - SKB_TMP_LEN(skb);
  97                                }
  98                        } else if (skb->protocol == htons(ETH_P_IPV6)) {
  99                                hnae_set_bit(tvsvsn, HNSV2_TXD_IPV6_B, 1);
 100                                ipv6hdr = ipv6_hdr(skb);
 101                                hnae_set_bit(rrcfv, HNSV2_TXD_L4CS_B, 1);
 102
 103                                /* check for tcp/udp header */
 104                                if (ipv6hdr->nexthdr == IPPROTO_TCP &&
 105                                    skb_is_gso(skb) && skb_is_gso_v6(skb)) {
 106                                        hnae_set_bit(tvsvsn,
 107                                                     HNSV2_TXD_TSE_B, 1);
 108                                        l4_len = tcp_hdrlen(skb);
 109                                        mss = skb_shinfo(skb)->gso_size;
 110                                        paylen = skb->len - SKB_TMP_LEN(skb);
 111                                }
 112                        }
 113                        desc->tx.ip_offset = ip_offset;
 114                        desc->tx.tse_vlan_snap_v6_sctp_nth = tvsvsn;
 115                        desc->tx.mss = cpu_to_le16(mss);
 116                        desc->tx.l4_len = l4_len;
 117                        desc->tx.paylen = cpu_to_le16(paylen);
 118                }
 119        }
 120
 121        hnae_set_bit(rrcfv, HNSV2_TXD_FE_B, frag_end);
 122
 123        desc->tx.bn_pid = bn_pid;
 124        desc->tx.ra_ri_cs_fe_vld = rrcfv;
 125
 126        ring_ptr_move_fw(ring, next_to_use);
 127}
 128
 129static void fill_v2_desc(struct hnae_ring *ring, void *priv,
 130                         int size, dma_addr_t dma, int frag_end,
 131                         int buf_num, enum hns_desc_type type, int mtu)
 132{
 133        fill_v2_desc_hw(ring, priv, size, size, dma, frag_end,
 134                        buf_num, type, mtu);
 135}
 136
 137static const struct acpi_device_id hns_enet_acpi_match[] = {
 138        { "HISI00C1", 0 },
 139        { "HISI00C2", 0 },
 140        { },
 141};
 142MODULE_DEVICE_TABLE(acpi, hns_enet_acpi_match);
 143
 144static void fill_desc(struct hnae_ring *ring, void *priv,
 145                      int size, dma_addr_t dma, int frag_end,
 146                      int buf_num, enum hns_desc_type type, int mtu)
 147{
 148        struct hnae_desc *desc = &ring->desc[ring->next_to_use];
 149        struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
 150        struct sk_buff *skb;
 151        __be16 protocol;
 152        u32 ip_offset;
 153        u32 asid_bufnum_pid = 0;
 154        u32 flag_ipoffset = 0;
 155
 156        desc_cb->priv = priv;
 157        desc_cb->length = size;
 158        desc_cb->dma = dma;
 159        desc_cb->type = type;
 160
 161        desc->addr = cpu_to_le64(dma);
 162        desc->tx.send_size = cpu_to_le16((u16)size);
 163
 164        /*config bd buffer end */
 165        flag_ipoffset |= 1 << HNS_TXD_VLD_B;
 166
 167        asid_bufnum_pid |= buf_num << HNS_TXD_BUFNUM_S;
 168
 169        if (type == DESC_TYPE_SKB) {
 170                skb = (struct sk_buff *)priv;
 171
 172                if (skb->ip_summed == CHECKSUM_PARTIAL) {
 173                        protocol = skb->protocol;
 174                        ip_offset = ETH_HLEN;
 175
 176                        /*if it is a SW VLAN check the next protocol*/
 177                        if (protocol == htons(ETH_P_8021Q)) {
 178                                ip_offset += VLAN_HLEN;
 179                                protocol = vlan_get_protocol(skb);
 180                                skb->protocol = protocol;
 181                        }
 182
 183                        if (skb->protocol == htons(ETH_P_IP)) {
 184                                flag_ipoffset |= 1 << HNS_TXD_L3CS_B;
 185                                /* check for tcp/udp header */
 186                                flag_ipoffset |= 1 << HNS_TXD_L4CS_B;
 187
 188                        } else if (skb->protocol == htons(ETH_P_IPV6)) {
 189                                /* ipv6 has not l3 cs, check for L4 header */
 190                                flag_ipoffset |= 1 << HNS_TXD_L4CS_B;
 191                        }
 192
 193                        flag_ipoffset |= ip_offset << HNS_TXD_IPOFFSET_S;
 194                }
 195        }
 196
 197        flag_ipoffset |= frag_end << HNS_TXD_FE_B;
 198
 199        desc->tx.asid_bufnum_pid = cpu_to_le16(asid_bufnum_pid);
 200        desc->tx.flag_ipoffset = cpu_to_le32(flag_ipoffset);
 201
 202        ring_ptr_move_fw(ring, next_to_use);
 203}
 204
 205static void unfill_desc(struct hnae_ring *ring)
 206{
 207        ring_ptr_move_bw(ring, next_to_use);
 208}
 209
 210static int hns_nic_maybe_stop_tx(
 211        struct sk_buff **out_skb, int *bnum, struct hnae_ring *ring)
 212{
 213        struct sk_buff *skb = *out_skb;
 214        struct sk_buff *new_skb = NULL;
 215        int buf_num;
 216
 217        /* no. of segments (plus a header) */
 218        buf_num = skb_shinfo(skb)->nr_frags + 1;
 219
 220        if (unlikely(buf_num > ring->max_desc_num_per_pkt)) {
 221                if (ring_space(ring) < 1)
 222                        return -EBUSY;
 223
 224                new_skb = skb_copy(skb, GFP_ATOMIC);
 225                if (!new_skb)
 226                        return -ENOMEM;
 227
 228                dev_kfree_skb_any(skb);
 229                *out_skb = new_skb;
 230                buf_num = 1;
 231        } else if (buf_num > ring_space(ring)) {
 232                return -EBUSY;
 233        }
 234
 235        *bnum = buf_num;
 236        return 0;
 237}
 238
 239static int hns_nic_maybe_stop_tso(
 240        struct sk_buff **out_skb, int *bnum, struct hnae_ring *ring)
 241{
 242        int i;
 243        int size;
 244        int buf_num;
 245        int frag_num;
 246        struct sk_buff *skb = *out_skb;
 247        struct sk_buff *new_skb = NULL;
 248        skb_frag_t *frag;
 249
 250        size = skb_headlen(skb);
 251        buf_num = (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
 252
 253        frag_num = skb_shinfo(skb)->nr_frags;
 254        for (i = 0; i < frag_num; i++) {
 255                frag = &skb_shinfo(skb)->frags[i];
 256                size = skb_frag_size(frag);
 257                buf_num += (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
 258        }
 259
 260        if (unlikely(buf_num > ring->max_desc_num_per_pkt)) {
 261                buf_num = (skb->len + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
 262                if (ring_space(ring) < buf_num)
 263                        return -EBUSY;
 264                /* manual split the send packet */
 265                new_skb = skb_copy(skb, GFP_ATOMIC);
 266                if (!new_skb)
 267                        return -ENOMEM;
 268                dev_kfree_skb_any(skb);
 269                *out_skb = new_skb;
 270
 271        } else if (ring_space(ring) < buf_num) {
 272                return -EBUSY;
 273        }
 274
 275        *bnum = buf_num;
 276        return 0;
 277}
 278
 279static void fill_tso_desc(struct hnae_ring *ring, void *priv,
 280                          int size, dma_addr_t dma, int frag_end,
 281                          int buf_num, enum hns_desc_type type, int mtu)
 282{
 283        int frag_buf_num;
 284        int sizeoflast;
 285        int k;
 286
 287        frag_buf_num = (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
 288        sizeoflast = size % BD_MAX_SEND_SIZE;
 289        sizeoflast = sizeoflast ? sizeoflast : BD_MAX_SEND_SIZE;
 290
 291        /* when the frag size is bigger than hardware, split this frag */
 292        for (k = 0; k < frag_buf_num; k++)
 293                fill_v2_desc_hw(ring, priv, k == 0 ? size : 0,
 294                                (k == frag_buf_num - 1) ?
 295                                        sizeoflast : BD_MAX_SEND_SIZE,
 296                                dma + BD_MAX_SEND_SIZE * k,
 297                                frag_end && (k == frag_buf_num - 1) ? 1 : 0,
 298                                buf_num,
 299                                (type == DESC_TYPE_SKB && !k) ?
 300                                        DESC_TYPE_SKB : DESC_TYPE_PAGE,
 301                                mtu);
 302}
 303
 304netdev_tx_t hns_nic_net_xmit_hw(struct net_device *ndev,
 305                                struct sk_buff *skb,
 306                                struct hns_nic_ring_data *ring_data)
 307{
 308        struct hns_nic_priv *priv = netdev_priv(ndev);
 309        struct hnae_ring *ring = ring_data->ring;
 310        struct device *dev = ring_to_dev(ring);
 311        struct netdev_queue *dev_queue;
 312        skb_frag_t *frag;
 313        int buf_num;
 314        int seg_num;
 315        dma_addr_t dma;
 316        int size, next_to_use;
 317        int i;
 318
 319        switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
 320        case -EBUSY:
 321                ring->stats.tx_busy++;
 322                goto out_net_tx_busy;
 323        case -ENOMEM:
 324                ring->stats.sw_err_cnt++;
 325                netdev_err(ndev, "no memory to xmit!\n");
 326                goto out_err_tx_ok;
 327        default:
 328                break;
 329        }
 330
 331        /* no. of segments (plus a header) */
 332        seg_num = skb_shinfo(skb)->nr_frags + 1;
 333        next_to_use = ring->next_to_use;
 334
 335        /* fill the first part */
 336        size = skb_headlen(skb);
 337        dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
 338        if (dma_mapping_error(dev, dma)) {
 339                netdev_err(ndev, "TX head DMA map failed\n");
 340                ring->stats.sw_err_cnt++;
 341                goto out_err_tx_ok;
 342        }
 343        priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
 344                            buf_num, DESC_TYPE_SKB, ndev->mtu);
 345
 346        /* fill the fragments */
 347        for (i = 1; i < seg_num; i++) {
 348                frag = &skb_shinfo(skb)->frags[i - 1];
 349                size = skb_frag_size(frag);
 350                dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
 351                if (dma_mapping_error(dev, dma)) {
 352                        netdev_err(ndev, "TX frag(%d) DMA map failed\n", i);
 353                        ring->stats.sw_err_cnt++;
 354                        goto out_map_frag_fail;
 355                }
 356                priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
 357                                    seg_num - 1 == i ? 1 : 0, buf_num,
 358                                    DESC_TYPE_PAGE, ndev->mtu);
 359        }
 360
 361        /*complete translate all packets*/
 362        dev_queue = netdev_get_tx_queue(ndev, skb->queue_mapping);
 363        netdev_tx_sent_queue(dev_queue, skb->len);
 364
 365        netif_trans_update(ndev);
 366        ndev->stats.tx_bytes += skb->len;
 367        ndev->stats.tx_packets++;
 368
 369        wmb(); /* commit all data before submit */
 370        assert(skb->queue_mapping < priv->ae_handle->q_num);
 371        hnae_queue_xmit(priv->ae_handle->qs[skb->queue_mapping], buf_num);
 372
 373        return NETDEV_TX_OK;
 374
 375out_map_frag_fail:
 376
 377        while (ring->next_to_use != next_to_use) {
 378                unfill_desc(ring);
 379                if (ring->next_to_use != next_to_use)
 380                        dma_unmap_page(dev,
 381                                       ring->desc_cb[ring->next_to_use].dma,
 382                                       ring->desc_cb[ring->next_to_use].length,
 383                                       DMA_TO_DEVICE);
 384                else
 385                        dma_unmap_single(dev,
 386                                         ring->desc_cb[next_to_use].dma,
 387                                         ring->desc_cb[next_to_use].length,
 388                                         DMA_TO_DEVICE);
 389        }
 390
 391out_err_tx_ok:
 392
 393        dev_kfree_skb_any(skb);
 394        return NETDEV_TX_OK;
 395
 396out_net_tx_busy:
 397
 398        netif_stop_subqueue(ndev, skb->queue_mapping);
 399
 400        /* Herbert's original patch had:
 401         *  smp_mb__after_netif_stop_queue();
 402         * but since that doesn't exist yet, just open code it.
 403         */
 404        smp_mb();
 405        return NETDEV_TX_BUSY;
 406}
 407
 408static void hns_nic_reuse_page(struct sk_buff *skb, int i,
 409                               struct hnae_ring *ring, int pull_len,
 410                               struct hnae_desc_cb *desc_cb)
 411{
 412        struct hnae_desc *desc;
 413        u32 truesize;
 414        int size;
 415        int last_offset;
 416        bool twobufs;
 417
 418        twobufs = ((PAGE_SIZE < 8192) &&
 419                hnae_buf_size(ring) == HNS_BUFFER_SIZE_2048);
 420
 421        desc = &ring->desc[ring->next_to_clean];
 422        size = le16_to_cpu(desc->rx.size);
 423
 424        if (twobufs) {
 425                truesize = hnae_buf_size(ring);
 426        } else {
 427                truesize = ALIGN(size, L1_CACHE_BYTES);
 428                last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
 429        }
 430
 431        skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
 432                        size - pull_len, truesize);
 433
 434         /* avoid re-using remote pages,flag default unreuse */
 435        if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
 436                return;
 437
 438        if (twobufs) {
 439                /* if we are only owner of page we can reuse it */
 440                if (likely(page_count(desc_cb->priv) == 1)) {
 441                        /* flip page offset to other buffer */
 442                        desc_cb->page_offset ^= truesize;
 443
 444                        desc_cb->reuse_flag = 1;
 445                        /* bump ref count on page before it is given*/
 446                        get_page(desc_cb->priv);
 447                }
 448                return;
 449        }
 450
 451        /* move offset up to the next cache line */
 452        desc_cb->page_offset += truesize;
 453
 454        if (desc_cb->page_offset <= last_offset) {
 455                desc_cb->reuse_flag = 1;
 456                /* bump ref count on page before it is given*/
 457                get_page(desc_cb->priv);
 458        }
 459}
 460
 461static void get_v2rx_desc_bnum(u32 bnum_flag, int *out_bnum)
 462{
 463        *out_bnum = hnae_get_field(bnum_flag,
 464                                   HNS_RXD_BUFNUM_M, HNS_RXD_BUFNUM_S) + 1;
 465}
 466
 467static void get_rx_desc_bnum(u32 bnum_flag, int *out_bnum)
 468{
 469        *out_bnum = hnae_get_field(bnum_flag,
 470                                   HNS_RXD_BUFNUM_M, HNS_RXD_BUFNUM_S);
 471}
 472
 473static void hns_nic_rx_checksum(struct hns_nic_ring_data *ring_data,
 474                                struct sk_buff *skb, u32 flag)
 475{
 476        struct net_device *netdev = ring_data->napi.dev;
 477        u32 l3id;
 478        u32 l4id;
 479
 480        /* check if RX checksum offload is enabled */
 481        if (unlikely(!(netdev->features & NETIF_F_RXCSUM)))
 482                return;
 483
 484        /* In hardware, we only support checksum for the following protocols:
 485         * 1) IPv4,
 486         * 2) TCP(over IPv4 or IPv6),
 487         * 3) UDP(over IPv4 or IPv6),
 488         * 4) SCTP(over IPv4 or IPv6)
 489         * but we support many L3(IPv4, IPv6, MPLS, PPPoE etc) and L4(TCP,
 490         * UDP, GRE, SCTP, IGMP, ICMP etc.) protocols.
 491         *
 492         * Hardware limitation:
 493         * Our present hardware RX Descriptor lacks L3/L4 checksum "Status &
 494         * Error" bit (which usually can be used to indicate whether checksum
 495         * was calculated by the hardware and if there was any error encountered
 496         * during checksum calculation).
 497         *
 498         * Software workaround:
 499         * We do get info within the RX descriptor about the kind of L3/L4
 500         * protocol coming in the packet and the error status. These errors
 501         * might not just be checksum errors but could be related to version,
 502         * length of IPv4, UDP, TCP etc.
 503         * Because there is no-way of knowing if it is a L3/L4 error due to bad
 504         * checksum or any other L3/L4 error, we will not (cannot) convey
 505         * checksum status for such cases to upper stack and will not maintain
 506         * the RX L3/L4 checksum counters as well.
 507         */
 508
 509        l3id = hnae_get_field(flag, HNS_RXD_L3ID_M, HNS_RXD_L3ID_S);
 510        l4id = hnae_get_field(flag, HNS_RXD_L4ID_M, HNS_RXD_L4ID_S);
 511
 512        /*  check L3 protocol for which checksum is supported */
 513        if ((l3id != HNS_RX_FLAG_L3ID_IPV4) && (l3id != HNS_RX_FLAG_L3ID_IPV6))
 514                return;
 515
 516        /* check for any(not just checksum)flagged L3 protocol errors */
 517        if (unlikely(hnae_get_bit(flag, HNS_RXD_L3E_B)))
 518                return;
 519
 520        /* we do not support checksum of fragmented packets */
 521        if (unlikely(hnae_get_bit(flag, HNS_RXD_FRAG_B)))
 522                return;
 523
 524        /*  check L4 protocol for which checksum is supported */
 525        if ((l4id != HNS_RX_FLAG_L4ID_TCP) &&
 526            (l4id != HNS_RX_FLAG_L4ID_UDP) &&
 527            (l4id != HNS_RX_FLAG_L4ID_SCTP))
 528                return;
 529
 530        /* check for any(not just checksum)flagged L4 protocol errors */
 531        if (unlikely(hnae_get_bit(flag, HNS_RXD_L4E_B)))
 532                return;
 533
 534        /* now, this has to be a packet with valid RX checksum */
 535        skb->ip_summed = CHECKSUM_UNNECESSARY;
 536}
 537
 538static int hns_nic_poll_rx_skb(struct hns_nic_ring_data *ring_data,
 539                               struct sk_buff **out_skb, int *out_bnum)
 540{
 541        struct hnae_ring *ring = ring_data->ring;
 542        struct net_device *ndev = ring_data->napi.dev;
 543        struct hns_nic_priv *priv = netdev_priv(ndev);
 544        struct sk_buff *skb;
 545        struct hnae_desc *desc;
 546        struct hnae_desc_cb *desc_cb;
 547        unsigned char *va;
 548        int bnum, length, i;
 549        int pull_len;
 550        u32 bnum_flag;
 551
 552        desc = &ring->desc[ring->next_to_clean];
 553        desc_cb = &ring->desc_cb[ring->next_to_clean];
 554
 555        prefetch(desc);
 556
 557        va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
 558
 559        /* prefetch first cache line of first page */
 560        prefetch(va);
 561#if L1_CACHE_BYTES < 128
 562        prefetch(va + L1_CACHE_BYTES);
 563#endif
 564
 565        skb = *out_skb = napi_alloc_skb(&ring_data->napi,
 566                                        HNS_RX_HEAD_SIZE);
 567        if (unlikely(!skb)) {
 568                netdev_err(ndev, "alloc rx skb fail\n");
 569                ring->stats.sw_err_cnt++;
 570                return -ENOMEM;
 571        }
 572
 573        prefetchw(skb->data);
 574        length = le16_to_cpu(desc->rx.pkt_len);
 575        bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag);
 576        priv->ops.get_rxd_bnum(bnum_flag, &bnum);
 577        *out_bnum = bnum;
 578
 579        if (length <= HNS_RX_HEAD_SIZE) {
 580                memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
 581
 582                /* we can reuse buffer as-is, just make sure it is local */
 583                if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
 584                        desc_cb->reuse_flag = 1;
 585                else /* this page cannot be reused so discard it */
 586                        put_page(desc_cb->priv);
 587
 588                ring_ptr_move_fw(ring, next_to_clean);
 589
 590                if (unlikely(bnum != 1)) { /* check err*/
 591                        *out_bnum = 1;
 592                        goto out_bnum_err;
 593                }
 594        } else {
 595                ring->stats.seg_pkt_cnt++;
 596
 597                pull_len = eth_get_headlen(ndev, va, HNS_RX_HEAD_SIZE);
 598                memcpy(__skb_put(skb, pull_len), va,
 599                       ALIGN(pull_len, sizeof(long)));
 600
 601                hns_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
 602                ring_ptr_move_fw(ring, next_to_clean);
 603
 604                if (unlikely(bnum >= (int)MAX_SKB_FRAGS)) { /* check err*/
 605                        *out_bnum = 1;
 606                        goto out_bnum_err;
 607                }
 608                for (i = 1; i < bnum; i++) {
 609                        desc = &ring->desc[ring->next_to_clean];
 610                        desc_cb = &ring->desc_cb[ring->next_to_clean];
 611
 612                        hns_nic_reuse_page(skb, i, ring, 0, desc_cb);
 613                        ring_ptr_move_fw(ring, next_to_clean);
 614                }
 615        }
 616
 617        /* check except process, free skb and jump the desc */
 618        if (unlikely((!bnum) || (bnum > ring->max_desc_num_per_pkt))) {
 619out_bnum_err:
 620                *out_bnum = *out_bnum ? *out_bnum : 1; /* ntc moved,cannot 0*/
 621                netdev_err(ndev, "invalid bnum(%d,%d,%d,%d),%016llx,%016llx\n",
 622                           bnum, ring->max_desc_num_per_pkt,
 623                           length, (int)MAX_SKB_FRAGS,
 624                           ((u64 *)desc)[0], ((u64 *)desc)[1]);
 625                ring->stats.err_bd_num++;
 626                dev_kfree_skb_any(skb);
 627                return -EDOM;
 628        }
 629
 630        bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag);
 631
 632        if (unlikely(!hnae_get_bit(bnum_flag, HNS_RXD_VLD_B))) {
 633                netdev_err(ndev, "no valid bd,%016llx,%016llx\n",
 634                           ((u64 *)desc)[0], ((u64 *)desc)[1]);
 635                ring->stats.non_vld_descs++;
 636                dev_kfree_skb_any(skb);
 637                return -EINVAL;
 638        }
 639
 640        if (unlikely((!desc->rx.pkt_len) ||
 641                     hnae_get_bit(bnum_flag, HNS_RXD_DROP_B))) {
 642                ring->stats.err_pkt_len++;
 643                dev_kfree_skb_any(skb);
 644                return -EFAULT;
 645        }
 646
 647        if (unlikely(hnae_get_bit(bnum_flag, HNS_RXD_L2E_B))) {
 648                ring->stats.l2_err++;
 649                dev_kfree_skb_any(skb);
 650                return -EFAULT;
 651        }
 652
 653        ring->stats.rx_pkts++;
 654        ring->stats.rx_bytes += skb->len;
 655
 656        /* indicate to upper stack if our hardware has already calculated
 657         * the RX checksum
 658         */
 659        hns_nic_rx_checksum(ring_data, skb, bnum_flag);
 660
 661        return 0;
 662}
 663
 664static void
 665hns_nic_alloc_rx_buffers(struct hns_nic_ring_data *ring_data, int cleand_count)
 666{
 667        int i, ret;
 668        struct hnae_desc_cb res_cbs;
 669        struct hnae_desc_cb *desc_cb;
 670        struct hnae_ring *ring = ring_data->ring;
 671        struct net_device *ndev = ring_data->napi.dev;
 672
 673        for (i = 0; i < cleand_count; i++) {
 674                desc_cb = &ring->desc_cb[ring->next_to_use];
 675                if (desc_cb->reuse_flag) {
 676                        ring->stats.reuse_pg_cnt++;
 677                        hnae_reuse_buffer(ring, ring->next_to_use);
 678                } else {
 679                        ret = hnae_reserve_buffer_map(ring, &res_cbs);
 680                        if (ret) {
 681                                ring->stats.sw_err_cnt++;
 682                                netdev_err(ndev, "hnae reserve buffer map failed.\n");
 683                                break;
 684                        }
 685                        hnae_replace_buffer(ring, ring->next_to_use, &res_cbs);
 686                }
 687
 688                ring_ptr_move_fw(ring, next_to_use);
 689        }
 690
 691        wmb(); /* make all data has been write before submit */
 692        writel_relaxed(i, ring->io_base + RCB_REG_HEAD);
 693}
 694
 695/* return error number for error or number of desc left to take
 696 */
 697static void hns_nic_rx_up_pro(struct hns_nic_ring_data *ring_data,
 698                              struct sk_buff *skb)
 699{
 700        struct net_device *ndev = ring_data->napi.dev;
 701
 702        skb->protocol = eth_type_trans(skb, ndev);
 703        (void)napi_gro_receive(&ring_data->napi, skb);
 704}
 705
 706static int hns_desc_unused(struct hnae_ring *ring)
 707{
 708        int ntc = ring->next_to_clean;
 709        int ntu = ring->next_to_use;
 710
 711        return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
 712}
 713
 714#define HNS_LOWEST_LATENCY_RATE         27      /* 27 MB/s */
 715#define HNS_LOW_LATENCY_RATE                    80      /* 80 MB/s */
 716
 717#define HNS_COAL_BDNUM                  3
 718
 719static u32 hns_coal_rx_bdnum(struct hnae_ring *ring)
 720{
 721        bool coal_enable = ring->q->handle->coal_adapt_en;
 722
 723        if (coal_enable &&
 724            ring->coal_last_rx_bytes > HNS_LOWEST_LATENCY_RATE)
 725                return HNS_COAL_BDNUM;
 726        else
 727                return 0;
 728}
 729
 730static void hns_update_rx_rate(struct hnae_ring *ring)
 731{
 732        bool coal_enable = ring->q->handle->coal_adapt_en;
 733        u32 time_passed_ms;
 734        u64 total_bytes;
 735
 736        if (!coal_enable ||
 737            time_before(jiffies, ring->coal_last_jiffies + (HZ >> 4)))
 738                return;
 739
 740        /* ring->stats.rx_bytes overflowed */
 741        if (ring->coal_last_rx_bytes > ring->stats.rx_bytes) {
 742                ring->coal_last_rx_bytes = ring->stats.rx_bytes;
 743                ring->coal_last_jiffies = jiffies;
 744                return;
 745        }
 746
 747        total_bytes = ring->stats.rx_bytes - ring->coal_last_rx_bytes;
 748        time_passed_ms = jiffies_to_msecs(jiffies - ring->coal_last_jiffies);
 749        do_div(total_bytes, time_passed_ms);
 750        ring->coal_rx_rate = total_bytes >> 10;
 751
 752        ring->coal_last_rx_bytes = ring->stats.rx_bytes;
 753        ring->coal_last_jiffies = jiffies;
 754}
 755
 756/**
 757 * smooth_alg - smoothing algrithm for adjusting coalesce parameter
 758 **/
 759static u32 smooth_alg(u32 new_param, u32 old_param)
 760{
 761        u32 gap = (new_param > old_param) ? new_param - old_param
 762                                          : old_param - new_param;
 763
 764        if (gap > 8)
 765                gap >>= 3;
 766
 767        if (new_param > old_param)
 768                return old_param + gap;
 769        else
 770                return old_param - gap;
 771}
 772
 773/**
 774 * hns_nic_adp_coalesce - self adapte coalesce according to rx rate
 775 * @ring_data: pointer to hns_nic_ring_data
 776 **/
 777static void hns_nic_adpt_coalesce(struct hns_nic_ring_data *ring_data)
 778{
 779        struct hnae_ring *ring = ring_data->ring;
 780        struct hnae_handle *handle = ring->q->handle;
 781        u32 new_coal_param, old_coal_param = ring->coal_param;
 782
 783        if (ring->coal_rx_rate < HNS_LOWEST_LATENCY_RATE)
 784                new_coal_param = HNAE_LOWEST_LATENCY_COAL_PARAM;
 785        else if (ring->coal_rx_rate < HNS_LOW_LATENCY_RATE)
 786                new_coal_param = HNAE_LOW_LATENCY_COAL_PARAM;
 787        else
 788                new_coal_param = HNAE_BULK_LATENCY_COAL_PARAM;
 789
 790        if (new_coal_param == old_coal_param &&
 791            new_coal_param == handle->coal_param)
 792                return;
 793
 794        new_coal_param = smooth_alg(new_coal_param, old_coal_param);
 795        ring->coal_param = new_coal_param;
 796
 797        /**
 798         * Because all ring in one port has one coalesce param, when one ring
 799         * calculate its own coalesce param, it cannot write to hardware at
 800         * once. There are three conditions as follows:
 801         *       1. current ring's coalesce param is larger than the hardware.
 802         *       2. or ring which adapt last time can change again.
 803         *       3. timeout.
 804         */
 805        if (new_coal_param == handle->coal_param) {
 806                handle->coal_last_jiffies = jiffies;
 807                handle->coal_ring_idx = ring_data->queue_index;
 808        } else if (new_coal_param > handle->coal_param ||
 809                   handle->coal_ring_idx == ring_data->queue_index ||
 810                   time_after(jiffies, handle->coal_last_jiffies + (HZ >> 4))) {
 811                handle->dev->ops->set_coalesce_usecs(handle,
 812                                        new_coal_param);
 813                handle->dev->ops->set_coalesce_frames(handle,
 814                                        1, new_coal_param);
 815                handle->coal_param = new_coal_param;
 816                handle->coal_ring_idx = ring_data->queue_index;
 817                handle->coal_last_jiffies = jiffies;
 818        }
 819}
 820
 821static int hns_nic_rx_poll_one(struct hns_nic_ring_data *ring_data,
 822                               int budget, void *v)
 823{
 824        struct hnae_ring *ring = ring_data->ring;
 825        struct sk_buff *skb;
 826        int num, bnum;
 827#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
 828        int recv_pkts, recv_bds, clean_count, err;
 829        int unused_count = hns_desc_unused(ring);
 830
 831        num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 832        rmb(); /* make sure num taken effect before the other data is touched */
 833
 834        recv_pkts = 0, recv_bds = 0, clean_count = 0;
 835        num -= unused_count;
 836
 837        while (recv_pkts < budget && recv_bds < num) {
 838                /* reuse or realloc buffers */
 839                if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
 840                        hns_nic_alloc_rx_buffers(ring_data,
 841                                                 clean_count + unused_count);
 842                        clean_count = 0;
 843                        unused_count = hns_desc_unused(ring);
 844                }
 845
 846                /* poll one pkt */
 847                err = hns_nic_poll_rx_skb(ring_data, &skb, &bnum);
 848                if (unlikely(!skb)) /* this fault cannot be repaired */
 849                        goto out;
 850
 851                recv_bds += bnum;
 852                clean_count += bnum;
 853                if (unlikely(err)) {  /* do jump the err */
 854                        recv_pkts++;
 855                        continue;
 856                }
 857
 858                /* do update ip stack process*/
 859                ((void (*)(struct hns_nic_ring_data *, struct sk_buff *))v)(
 860                                                        ring_data, skb);
 861                recv_pkts++;
 862        }
 863
 864out:
 865        /* make all data has been write before submit */
 866        if (clean_count + unused_count > 0)
 867                hns_nic_alloc_rx_buffers(ring_data,
 868                                         clean_count + unused_count);
 869
 870        return recv_pkts;
 871}
 872
 873static bool hns_nic_rx_fini_pro(struct hns_nic_ring_data *ring_data)
 874{
 875        struct hnae_ring *ring = ring_data->ring;
 876        int num = 0;
 877        bool rx_stopped;
 878
 879        hns_update_rx_rate(ring);
 880
 881        /* for hardware bug fixed */
 882        ring_data->ring->q->handle->dev->ops->toggle_ring_irq(ring, 0);
 883        num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 884
 885        if (num <= hns_coal_rx_bdnum(ring)) {
 886                if (ring->q->handle->coal_adapt_en)
 887                        hns_nic_adpt_coalesce(ring_data);
 888
 889                rx_stopped = true;
 890        } else {
 891                ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 892                        ring_data->ring, 1);
 893
 894                rx_stopped = false;
 895        }
 896
 897        return rx_stopped;
 898}
 899
 900static bool hns_nic_rx_fini_pro_v2(struct hns_nic_ring_data *ring_data)
 901{
 902        struct hnae_ring *ring = ring_data->ring;
 903        int num;
 904
 905        hns_update_rx_rate(ring);
 906        num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 907
 908        if (num <= hns_coal_rx_bdnum(ring)) {
 909                if (ring->q->handle->coal_adapt_en)
 910                        hns_nic_adpt_coalesce(ring_data);
 911
 912                return true;
 913        }
 914
 915        return false;
 916}
 917
 918static inline void hns_nic_reclaim_one_desc(struct hnae_ring *ring,
 919                                            int *bytes, int *pkts)
 920{
 921        struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
 922
 923        (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
 924        (*bytes) += desc_cb->length;
 925        /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
 926        hnae_free_buffer_detach(ring, ring->next_to_clean);
 927
 928        ring_ptr_move_fw(ring, next_to_clean);
 929}
 930
 931static int is_valid_clean_head(struct hnae_ring *ring, int h)
 932{
 933        int u = ring->next_to_use;
 934        int c = ring->next_to_clean;
 935
 936        if (unlikely(h > ring->desc_num))
 937                return 0;
 938
 939        assert(u > 0 && u < ring->desc_num);
 940        assert(c > 0 && c < ring->desc_num);
 941        assert(u != c && h != c); /* must be checked before call this func */
 942
 943        return u > c ? (h > c && h <= u) : (h > c || h <= u);
 944}
 945
 946/* reclaim all desc in one budget
 947 * return error or number of desc left
 948 */
 949static int hns_nic_tx_poll_one(struct hns_nic_ring_data *ring_data,
 950                               int budget, void *v)
 951{
 952        struct hnae_ring *ring = ring_data->ring;
 953        struct net_device *ndev = ring_data->napi.dev;
 954        struct netdev_queue *dev_queue;
 955        struct hns_nic_priv *priv = netdev_priv(ndev);
 956        int head;
 957        int bytes, pkts;
 958
 959        head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
 960        rmb(); /* make sure head is ready before touch any data */
 961
 962        if (is_ring_empty(ring) || head == ring->next_to_clean)
 963                return 0; /* no data to poll */
 964
 965        if (!is_valid_clean_head(ring, head)) {
 966                netdev_err(ndev, "wrong head (%d, %d-%d)\n", head,
 967                           ring->next_to_use, ring->next_to_clean);
 968                ring->stats.io_err_cnt++;
 969                return -EIO;
 970        }
 971
 972        bytes = 0;
 973        pkts = 0;
 974        while (head != ring->next_to_clean) {
 975                hns_nic_reclaim_one_desc(ring, &bytes, &pkts);
 976                /* issue prefetch for next Tx descriptor */
 977                prefetch(&ring->desc_cb[ring->next_to_clean]);
 978        }
 979        /* update tx ring statistics. */
 980        ring->stats.tx_pkts += pkts;
 981        ring->stats.tx_bytes += bytes;
 982
 983        dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index);
 984        netdev_tx_completed_queue(dev_queue, pkts, bytes);
 985
 986        if (unlikely(priv->link && !netif_carrier_ok(ndev)))
 987                netif_carrier_on(ndev);
 988
 989        if (unlikely(pkts && netif_carrier_ok(ndev) &&
 990                     (ring_space(ring) >= ring->max_desc_num_per_pkt * 2))) {
 991                /* Make sure that anybody stopping the queue after this
 992                 * sees the new next_to_clean.
 993                 */
 994                smp_mb();
 995                if (netif_tx_queue_stopped(dev_queue) &&
 996                    !test_bit(NIC_STATE_DOWN, &priv->state)) {
 997                        netif_tx_wake_queue(dev_queue);
 998                        ring->stats.restart_queue++;
 999                }
1000        }
1001        return 0;
1002}
1003
1004static bool hns_nic_tx_fini_pro(struct hns_nic_ring_data *ring_data)
1005{
1006        struct hnae_ring *ring = ring_data->ring;
1007        int head;
1008
1009        ring_data->ring->q->handle->dev->ops->toggle_ring_irq(ring, 0);
1010
1011        head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
1012
1013        if (head != ring->next_to_clean) {
1014                ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
1015                        ring_data->ring, 1);
1016
1017                return false;
1018        } else {
1019                return true;
1020        }
1021}
1022
1023static bool hns_nic_tx_fini_pro_v2(struct hns_nic_ring_data *ring_data)
1024{
1025        struct hnae_ring *ring = ring_data->ring;
1026        int head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
1027
1028        if (head == ring->next_to_clean)
1029                return true;
1030        else
1031                return false;
1032}
1033
1034static void hns_nic_tx_clr_all_bufs(struct hns_nic_ring_data *ring_data)
1035{
1036        struct hnae_ring *ring = ring_data->ring;
1037        struct net_device *ndev = ring_data->napi.dev;
1038        struct netdev_queue *dev_queue;
1039        int head;
1040        int bytes, pkts;
1041
1042        head = ring->next_to_use; /* ntu :soft setted ring position*/
1043        bytes = 0;
1044        pkts = 0;
1045        while (head != ring->next_to_clean)
1046                hns_nic_reclaim_one_desc(ring, &bytes, &pkts);
1047
1048        dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index);
1049        netdev_tx_reset_queue(dev_queue);
1050}
1051
1052static int hns_nic_common_poll(struct napi_struct *napi, int budget)
1053{
1054        int clean_complete = 0;
1055        struct hns_nic_ring_data *ring_data =
1056                container_of(napi, struct hns_nic_ring_data, napi);
1057        struct hnae_ring *ring = ring_data->ring;
1058
1059try_again:
1060        clean_complete += ring_data->poll_one(
1061                                ring_data, budget - clean_complete,
1062                                ring_data->ex_process);
1063
1064        if (clean_complete < budget) {
1065                if (ring_data->fini_process(ring_data)) {
1066                        napi_complete(napi);
1067                        ring->q->handle->dev->ops->toggle_ring_irq(ring, 0);
1068                } else {
1069                        goto try_again;
1070                }
1071        }
1072
1073        return clean_complete;
1074}
1075
1076static irqreturn_t hns_irq_handle(int irq, void *dev)
1077{
1078        struct hns_nic_ring_data *ring_data = (struct hns_nic_ring_data *)dev;
1079
1080        ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
1081                ring_data->ring, 1);
1082        napi_schedule(&ring_data->napi);
1083
1084        return IRQ_HANDLED;
1085}
1086
1087/**
1088 *hns_nic_adjust_link - adjust net work mode by the phy stat or new param
1089 *@ndev: net device
1090 */
1091static void hns_nic_adjust_link(struct net_device *ndev)
1092{
1093        struct hns_nic_priv *priv = netdev_priv(ndev);
1094        struct hnae_handle *h = priv->ae_handle;
1095        int state = 1;
1096
1097        /* If there is no phy, do not need adjust link */
1098        if (ndev->phydev) {
1099                /* When phy link down, do nothing */
1100                if (ndev->phydev->link == 0)
1101                        return;
1102
1103                if (h->dev->ops->need_adjust_link(h, ndev->phydev->speed,
1104                                                  ndev->phydev->duplex)) {
1105                        /* because Hi161X chip don't support to change gmac
1106                         * speed and duplex with traffic. Delay 200ms to
1107                         * make sure there is no more data in chip FIFO.
1108                         */
1109                        netif_carrier_off(ndev);
1110                        msleep(200);
1111                        h->dev->ops->adjust_link(h, ndev->phydev->speed,
1112                                                 ndev->phydev->duplex);
1113                        netif_carrier_on(ndev);
1114                }
1115        }
1116
1117        state = state && h->dev->ops->get_status(h);
1118
1119        if (state != priv->link) {
1120                if (state) {
1121                        netif_carrier_on(ndev);
1122                        netif_tx_wake_all_queues(ndev);
1123                        netdev_info(ndev, "link up\n");
1124                } else {
1125                        netif_carrier_off(ndev);
1126                        netdev_info(ndev, "link down\n");
1127                }
1128                priv->link = state;
1129        }
1130}
1131
1132/**
1133 *hns_nic_init_phy - init phy
1134 *@ndev: net device
1135 *@h: ae handle
1136 * Return 0 on success, negative on failure
1137 */
1138int hns_nic_init_phy(struct net_device *ndev, struct hnae_handle *h)
1139{
1140        __ETHTOOL_DECLARE_LINK_MODE_MASK(supported) = { 0, };
1141        struct phy_device *phy_dev = h->phy_dev;
1142        int ret;
1143
1144        if (!h->phy_dev)
1145                return 0;
1146
1147        ethtool_convert_legacy_u32_to_link_mode(supported, h->if_support);
1148        linkmode_and(phy_dev->supported, phy_dev->supported, supported);
1149        linkmode_copy(phy_dev->advertising, phy_dev->supported);
1150
1151        if (h->phy_if == PHY_INTERFACE_MODE_XGMII)
1152                phy_dev->autoneg = false;
1153
1154        if (h->phy_if != PHY_INTERFACE_MODE_XGMII) {
1155                phy_dev->dev_flags = 0;
1156
1157                ret = phy_connect_direct(ndev, phy_dev, hns_nic_adjust_link,
1158                                         h->phy_if);
1159        } else {
1160                ret = phy_attach_direct(ndev, phy_dev, 0, h->phy_if);
1161        }
1162        if (unlikely(ret))
1163                return -ENODEV;
1164
1165        phy_attached_info(phy_dev);
1166
1167        return 0;
1168}
1169
1170static int hns_nic_ring_open(struct net_device *netdev, int idx)
1171{
1172        struct hns_nic_priv *priv = netdev_priv(netdev);
1173        struct hnae_handle *h = priv->ae_handle;
1174
1175        napi_enable(&priv->ring_data[idx].napi);
1176
1177        enable_irq(priv->ring_data[idx].ring->irq);
1178        h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 0);
1179
1180        return 0;
1181}
1182
1183static int hns_nic_net_set_mac_address(struct net_device *ndev, void *p)
1184{
1185        struct hns_nic_priv *priv = netdev_priv(ndev);
1186        struct hnae_handle *h = priv->ae_handle;
1187        struct sockaddr *mac_addr = p;
1188        int ret;
1189
1190        if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1191                return -EADDRNOTAVAIL;
1192
1193        ret = h->dev->ops->set_mac_addr(h, mac_addr->sa_data);
1194        if (ret) {
1195                netdev_err(ndev, "set_mac_address fail, ret=%d!\n", ret);
1196                return ret;
1197        }
1198
1199        memcpy(ndev->dev_addr, mac_addr->sa_data, ndev->addr_len);
1200
1201        return 0;
1202}
1203
1204static void hns_nic_update_stats(struct net_device *netdev)
1205{
1206        struct hns_nic_priv *priv = netdev_priv(netdev);
1207        struct hnae_handle *h = priv->ae_handle;
1208
1209        h->dev->ops->update_stats(h, &netdev->stats);
1210}
1211
1212/* set mac addr if it is configed. or leave it to the AE driver */
1213static void hns_init_mac_addr(struct net_device *ndev)
1214{
1215        struct hns_nic_priv *priv = netdev_priv(ndev);
1216
1217        if (!device_get_mac_address(priv->dev, ndev->dev_addr, ETH_ALEN)) {
1218                eth_hw_addr_random(ndev);
1219                dev_warn(priv->dev, "No valid mac, use random mac %pM",
1220                         ndev->dev_addr);
1221        }
1222}
1223
1224static void hns_nic_ring_close(struct net_device *netdev, int idx)
1225{
1226        struct hns_nic_priv *priv = netdev_priv(netdev);
1227        struct hnae_handle *h = priv->ae_handle;
1228
1229        h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 1);
1230        disable_irq(priv->ring_data[idx].ring->irq);
1231
1232        napi_disable(&priv->ring_data[idx].napi);
1233}
1234
1235static int hns_nic_init_affinity_mask(int q_num, int ring_idx,
1236                                      struct hnae_ring *ring, cpumask_t *mask)
1237{
1238        int cpu;
1239
1240        /* Diffrent irq banlance between 16core and 32core.
1241         * The cpu mask set by ring index according to the ring flag
1242         * which indicate the ring is tx or rx.
1243         */
1244        if (q_num == num_possible_cpus()) {
1245                if (is_tx_ring(ring))
1246                        cpu = ring_idx;
1247                else
1248                        cpu = ring_idx - q_num;
1249        } else {
1250                if (is_tx_ring(ring))
1251                        cpu = ring_idx * 2;
1252                else
1253                        cpu = (ring_idx - q_num) * 2 + 1;
1254        }
1255
1256        cpumask_clear(mask);
1257        cpumask_set_cpu(cpu, mask);
1258
1259        return cpu;
1260}
1261
1262static void hns_nic_free_irq(int q_num, struct hns_nic_priv *priv)
1263{
1264        int i;
1265
1266        for (i = 0; i < q_num * 2; i++) {
1267                if (priv->ring_data[i].ring->irq_init_flag == RCB_IRQ_INITED) {
1268                        irq_set_affinity_hint(priv->ring_data[i].ring->irq,
1269                                              NULL);
1270                        free_irq(priv->ring_data[i].ring->irq,
1271                                 &priv->ring_data[i]);
1272                        priv->ring_data[i].ring->irq_init_flag =
1273                                RCB_IRQ_NOT_INITED;
1274                }
1275        }
1276}
1277
1278static int hns_nic_init_irq(struct hns_nic_priv *priv)
1279{
1280        struct hnae_handle *h = priv->ae_handle;
1281        struct hns_nic_ring_data *rd;
1282        int i;
1283        int ret;
1284        int cpu;
1285
1286        for (i = 0; i < h->q_num * 2; i++) {
1287                rd = &priv->ring_data[i];
1288
1289                if (rd->ring->irq_init_flag == RCB_IRQ_INITED)
1290                        break;
1291
1292                snprintf(rd->ring->ring_name, RCB_RING_NAME_LEN,
1293                         "%s-%s%d", priv->netdev->name,
1294                         (is_tx_ring(rd->ring) ? "tx" : "rx"), rd->queue_index);
1295
1296                rd->ring->ring_name[RCB_RING_NAME_LEN - 1] = '\0';
1297
1298                ret = request_irq(rd->ring->irq,
1299                                  hns_irq_handle, 0, rd->ring->ring_name, rd);
1300                if (ret) {
1301                        netdev_err(priv->netdev, "request irq(%d) fail\n",
1302                                   rd->ring->irq);
1303                        goto out_free_irq;
1304                }
1305                disable_irq(rd->ring->irq);
1306
1307                cpu = hns_nic_init_affinity_mask(h->q_num, i,
1308                                                 rd->ring, &rd->mask);
1309
1310                if (cpu_online(cpu))
1311                        irq_set_affinity_hint(rd->ring->irq,
1312                                              &rd->mask);
1313
1314                rd->ring->irq_init_flag = RCB_IRQ_INITED;
1315        }
1316
1317        return 0;
1318
1319out_free_irq:
1320        hns_nic_free_irq(h->q_num, priv);
1321        return ret;
1322}
1323
1324static int hns_nic_net_up(struct net_device *ndev)
1325{
1326        struct hns_nic_priv *priv = netdev_priv(ndev);
1327        struct hnae_handle *h = priv->ae_handle;
1328        int i, j;
1329        int ret;
1330
1331        if (!test_bit(NIC_STATE_DOWN, &priv->state))
1332                return 0;
1333
1334        ret = hns_nic_init_irq(priv);
1335        if (ret != 0) {
1336                netdev_err(ndev, "hns init irq failed! ret=%d\n", ret);
1337                return ret;
1338        }
1339
1340        for (i = 0; i < h->q_num * 2; i++) {
1341                ret = hns_nic_ring_open(ndev, i);
1342                if (ret)
1343                        goto out_has_some_queues;
1344        }
1345
1346        ret = h->dev->ops->set_mac_addr(h, ndev->dev_addr);
1347        if (ret)
1348                goto out_set_mac_addr_err;
1349
1350        ret = h->dev->ops->start ? h->dev->ops->start(h) : 0;
1351        if (ret)
1352                goto out_start_err;
1353
1354        if (ndev->phydev)
1355                phy_start(ndev->phydev);
1356
1357        clear_bit(NIC_STATE_DOWN, &priv->state);
1358        (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
1359
1360        return 0;
1361
1362out_start_err:
1363        netif_stop_queue(ndev);
1364out_set_mac_addr_err:
1365out_has_some_queues:
1366        for (j = i - 1; j >= 0; j--)
1367                hns_nic_ring_close(ndev, j);
1368
1369        hns_nic_free_irq(h->q_num, priv);
1370        set_bit(NIC_STATE_DOWN, &priv->state);
1371
1372        return ret;
1373}
1374
1375static void hns_nic_net_down(struct net_device *ndev)
1376{
1377        int i;
1378        struct hnae_ae_ops *ops;
1379        struct hns_nic_priv *priv = netdev_priv(ndev);
1380
1381        if (test_and_set_bit(NIC_STATE_DOWN, &priv->state))
1382                return;
1383
1384        (void)del_timer_sync(&priv->service_timer);
1385        netif_tx_stop_all_queues(ndev);
1386        netif_carrier_off(ndev);
1387        netif_tx_disable(ndev);
1388        priv->link = 0;
1389
1390        if (ndev->phydev)
1391                phy_stop(ndev->phydev);
1392
1393        ops = priv->ae_handle->dev->ops;
1394
1395        if (ops->stop)
1396                ops->stop(priv->ae_handle);
1397
1398        netif_tx_stop_all_queues(ndev);
1399
1400        for (i = priv->ae_handle->q_num - 1; i >= 0; i--) {
1401                hns_nic_ring_close(ndev, i);
1402                hns_nic_ring_close(ndev, i + priv->ae_handle->q_num);
1403
1404                /* clean tx buffers*/
1405                hns_nic_tx_clr_all_bufs(priv->ring_data + i);
1406        }
1407}
1408
1409void hns_nic_net_reset(struct net_device *ndev)
1410{
1411        struct hns_nic_priv *priv = netdev_priv(ndev);
1412        struct hnae_handle *handle = priv->ae_handle;
1413
1414        while (test_and_set_bit(NIC_STATE_RESETTING, &priv->state))
1415                usleep_range(1000, 2000);
1416
1417        (void)hnae_reinit_handle(handle);
1418
1419        clear_bit(NIC_STATE_RESETTING, &priv->state);
1420}
1421
1422void hns_nic_net_reinit(struct net_device *netdev)
1423{
1424        struct hns_nic_priv *priv = netdev_priv(netdev);
1425        enum hnae_port_type type = priv->ae_handle->port_type;
1426
1427        netif_trans_update(priv->netdev);
1428        while (test_and_set_bit(NIC_STATE_REINITING, &priv->state))
1429                usleep_range(1000, 2000);
1430
1431        hns_nic_net_down(netdev);
1432
1433        /* Only do hns_nic_net_reset in debug mode
1434         * because of hardware limitation.
1435         */
1436        if (type == HNAE_PORT_DEBUG)
1437                hns_nic_net_reset(netdev);
1438
1439        (void)hns_nic_net_up(netdev);
1440        clear_bit(NIC_STATE_REINITING, &priv->state);
1441}
1442
1443static int hns_nic_net_open(struct net_device *ndev)
1444{
1445        struct hns_nic_priv *priv = netdev_priv(ndev);
1446        struct hnae_handle *h = priv->ae_handle;
1447        int ret;
1448
1449        if (test_bit(NIC_STATE_TESTING, &priv->state))
1450                return -EBUSY;
1451
1452        priv->link = 0;
1453        netif_carrier_off(ndev);
1454
1455        ret = netif_set_real_num_tx_queues(ndev, h->q_num);
1456        if (ret < 0) {
1457                netdev_err(ndev, "netif_set_real_num_tx_queues fail, ret=%d!\n",
1458                           ret);
1459                return ret;
1460        }
1461
1462        ret = netif_set_real_num_rx_queues(ndev, h->q_num);
1463        if (ret < 0) {
1464                netdev_err(ndev,
1465                           "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
1466                return ret;
1467        }
1468
1469        ret = hns_nic_net_up(ndev);
1470        if (ret) {
1471                netdev_err(ndev,
1472                           "hns net up fail, ret=%d!\n", ret);
1473                return ret;
1474        }
1475
1476        return 0;
1477}
1478
1479static int hns_nic_net_stop(struct net_device *ndev)
1480{
1481        hns_nic_net_down(ndev);
1482
1483        return 0;
1484}
1485
1486static void hns_tx_timeout_reset(struct hns_nic_priv *priv);
1487#define HNS_TX_TIMEO_LIMIT (40 * HZ)
1488static void hns_nic_net_timeout(struct net_device *ndev)
1489{
1490        struct hns_nic_priv *priv = netdev_priv(ndev);
1491
1492        if (ndev->watchdog_timeo < HNS_TX_TIMEO_LIMIT) {
1493                ndev->watchdog_timeo *= 2;
1494                netdev_info(ndev, "watchdog_timo changed to %d.\n",
1495                            ndev->watchdog_timeo);
1496        } else {
1497                ndev->watchdog_timeo = HNS_NIC_TX_TIMEOUT;
1498                hns_tx_timeout_reset(priv);
1499        }
1500}
1501
1502static int hns_nic_do_ioctl(struct net_device *netdev, struct ifreq *ifr,
1503                            int cmd)
1504{
1505        struct phy_device *phy_dev = netdev->phydev;
1506
1507        if (!netif_running(netdev))
1508                return -EINVAL;
1509
1510        if (!phy_dev)
1511                return -ENOTSUPP;
1512
1513        return phy_mii_ioctl(phy_dev, ifr, cmd);
1514}
1515
1516static netdev_tx_t hns_nic_net_xmit(struct sk_buff *skb,
1517                                    struct net_device *ndev)
1518{
1519        struct hns_nic_priv *priv = netdev_priv(ndev);
1520
1521        assert(skb->queue_mapping < ndev->ae_handle->q_num);
1522
1523        return hns_nic_net_xmit_hw(ndev, skb,
1524                                   &tx_ring_data(priv, skb->queue_mapping));
1525}
1526
1527static void hns_nic_drop_rx_fetch(struct hns_nic_ring_data *ring_data,
1528                                  struct sk_buff *skb)
1529{
1530        dev_kfree_skb_any(skb);
1531}
1532
1533#define HNS_LB_TX_RING  0
1534static struct sk_buff *hns_assemble_skb(struct net_device *ndev)
1535{
1536        struct sk_buff *skb;
1537        struct ethhdr *ethhdr;
1538        int frame_len;
1539
1540        /* allocate test skb */
1541        skb = alloc_skb(64, GFP_KERNEL);
1542        if (!skb)
1543                return NULL;
1544
1545        skb_put(skb, 64);
1546        skb->dev = ndev;
1547        memset(skb->data, 0xFF, skb->len);
1548
1549        /* must be tcp/ip package */
1550        ethhdr = (struct ethhdr *)skb->data;
1551        ethhdr->h_proto = htons(ETH_P_IP);
1552
1553        frame_len = skb->len & (~1ul);
1554        memset(&skb->data[frame_len / 2], 0xAA,
1555               frame_len / 2 - 1);
1556
1557        skb->queue_mapping = HNS_LB_TX_RING;
1558
1559        return skb;
1560}
1561
1562static int hns_enable_serdes_lb(struct net_device *ndev)
1563{
1564        struct hns_nic_priv *priv = netdev_priv(ndev);
1565        struct hnae_handle *h = priv->ae_handle;
1566        struct hnae_ae_ops *ops = h->dev->ops;
1567        int speed, duplex;
1568        int ret;
1569
1570        ret = ops->set_loopback(h, MAC_INTERNALLOOP_SERDES, 1);
1571        if (ret)
1572                return ret;
1573
1574        ret = ops->start ? ops->start(h) : 0;
1575        if (ret)
1576                return ret;
1577
1578        /* link adjust duplex*/
1579        if (h->phy_if != PHY_INTERFACE_MODE_XGMII)
1580                speed = 1000;
1581        else
1582                speed = 10000;
1583        duplex = 1;
1584
1585        ops->adjust_link(h, speed, duplex);
1586
1587        /* wait h/w ready */
1588        mdelay(300);
1589
1590        return 0;
1591}
1592
1593static void hns_disable_serdes_lb(struct net_device *ndev)
1594{
1595        struct hns_nic_priv *priv = netdev_priv(ndev);
1596        struct hnae_handle *h = priv->ae_handle;
1597        struct hnae_ae_ops *ops = h->dev->ops;
1598
1599        ops->stop(h);
1600        ops->set_loopback(h, MAC_INTERNALLOOP_SERDES, 0);
1601}
1602
1603/**
1604 *hns_nic_clear_all_rx_fetch - clear the chip fetched descriptions. The
1605 *function as follows:
1606 *    1. if one rx ring has found the page_offset is not equal 0 between head
1607 *       and tail, it means that the chip fetched the wrong descs for the ring
1608 *       which buffer size is 4096.
1609 *    2. we set the chip serdes loopback and set rss indirection to the ring.
1610 *    3. construct 64-bytes ip broadcast packages, wait the associated rx ring
1611 *       recieving all packages and it will fetch new descriptions.
1612 *    4. recover to the original state.
1613 *
1614 *@ndev: net device
1615 */
1616static int hns_nic_clear_all_rx_fetch(struct net_device *ndev)
1617{
1618        struct hns_nic_priv *priv = netdev_priv(ndev);
1619        struct hnae_handle *h = priv->ae_handle;
1620        struct hnae_ae_ops *ops = h->dev->ops;
1621        struct hns_nic_ring_data *rd;
1622        struct hnae_ring *ring;
1623        struct sk_buff *skb;
1624        u32 *org_indir;
1625        u32 *cur_indir;
1626        int indir_size;
1627        int head, tail;
1628        int fetch_num;
1629        int i, j;
1630        bool found;
1631        int retry_times;
1632        int ret = 0;
1633
1634        /* alloc indir memory */
1635        indir_size = ops->get_rss_indir_size(h) * sizeof(*org_indir);
1636        org_indir = kzalloc(indir_size, GFP_KERNEL);
1637        if (!org_indir)
1638                return -ENOMEM;
1639
1640        /* store the orginal indirection */
1641        ops->get_rss(h, org_indir, NULL, NULL);
1642
1643        cur_indir = kzalloc(indir_size, GFP_KERNEL);
1644        if (!cur_indir) {
1645                ret = -ENOMEM;
1646                goto cur_indir_alloc_err;
1647        }
1648
1649        /* set loopback */
1650        if (hns_enable_serdes_lb(ndev)) {
1651                ret = -EINVAL;
1652                goto enable_serdes_lb_err;
1653        }
1654
1655        /* foreach every rx ring to clear fetch desc */
1656        for (i = 0; i < h->q_num; i++) {
1657                ring = &h->qs[i]->rx_ring;
1658                head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
1659                tail = readl_relaxed(ring->io_base + RCB_REG_TAIL);
1660                found = false;
1661                fetch_num = ring_dist(ring, head, tail);
1662
1663                while (head != tail) {
1664                        if (ring->desc_cb[head].page_offset != 0) {
1665                                found = true;
1666                                break;
1667                        }
1668
1669                        head++;
1670                        if (head == ring->desc_num)
1671                                head = 0;
1672                }
1673
1674                if (found) {
1675                        for (j = 0; j < indir_size / sizeof(*org_indir); j++)
1676                                cur_indir[j] = i;
1677                        ops->set_rss(h, cur_indir, NULL, 0);
1678
1679                        for (j = 0; j < fetch_num; j++) {
1680                                /* alloc one skb and init */
1681                                skb = hns_assemble_skb(ndev);
1682                                if (!skb)
1683                                        goto out;
1684                                rd = &tx_ring_data(priv, skb->queue_mapping);
1685                                hns_nic_net_xmit_hw(ndev, skb, rd);
1686
1687                                retry_times = 0;
1688                                while (retry_times++ < 10) {
1689                                        mdelay(10);
1690                                        /* clean rx */
1691                                        rd = &rx_ring_data(priv, i);
1692                                        if (rd->poll_one(rd, fetch_num,
1693                                                         hns_nic_drop_rx_fetch))
1694                                                break;
1695                                }
1696
1697                                retry_times = 0;
1698                                while (retry_times++ < 10) {
1699                                        mdelay(10);
1700                                        /* clean tx ring 0 send package */
1701                                        rd = &tx_ring_data(priv,
1702                                                           HNS_LB_TX_RING);
1703                                        if (rd->poll_one(rd, fetch_num, NULL))
1704                                                break;
1705                                }
1706                        }
1707                }
1708        }
1709
1710out:
1711        /* restore everything */
1712        ops->set_rss(h, org_indir, NULL, 0);
1713        hns_disable_serdes_lb(ndev);
1714enable_serdes_lb_err:
1715        kfree(cur_indir);
1716cur_indir_alloc_err:
1717        kfree(org_indir);
1718
1719        return ret;
1720}
1721
1722static int hns_nic_change_mtu(struct net_device *ndev, int new_mtu)
1723{
1724        struct hns_nic_priv *priv = netdev_priv(ndev);
1725        struct hnae_handle *h = priv->ae_handle;
1726        bool if_running = netif_running(ndev);
1727        int ret;
1728
1729        /* MTU < 68 is an error and causes problems on some kernels */
1730        if (new_mtu < 68)
1731                return -EINVAL;
1732
1733        /* MTU no change */
1734        if (new_mtu == ndev->mtu)
1735                return 0;
1736
1737        if (!h->dev->ops->set_mtu)
1738                return -ENOTSUPP;
1739
1740        if (if_running) {
1741                (void)hns_nic_net_stop(ndev);
1742                msleep(100);
1743        }
1744
1745        if (priv->enet_ver != AE_VERSION_1 &&
1746            ndev->mtu <= BD_SIZE_2048_MAX_MTU &&
1747            new_mtu > BD_SIZE_2048_MAX_MTU) {
1748                /* update desc */
1749                hnae_reinit_all_ring_desc(h);
1750
1751                /* clear the package which the chip has fetched */
1752                ret = hns_nic_clear_all_rx_fetch(ndev);
1753
1754                /* the page offset must be consist with desc */
1755                hnae_reinit_all_ring_page_off(h);
1756
1757                if (ret) {
1758                        netdev_err(ndev, "clear the fetched desc fail\n");
1759                        goto out;
1760                }
1761        }
1762
1763        ret = h->dev->ops->set_mtu(h, new_mtu);
1764        if (ret) {
1765                netdev_err(ndev, "set mtu fail, return value %d\n",
1766                           ret);
1767                goto out;
1768        }
1769
1770        /* finally, set new mtu to netdevice */
1771        ndev->mtu = new_mtu;
1772
1773out:
1774        if (if_running) {
1775                if (hns_nic_net_open(ndev)) {
1776                        netdev_err(ndev, "hns net open fail\n");
1777                        ret = -EINVAL;
1778                }
1779        }
1780
1781        return ret;
1782}
1783
1784static int hns_nic_set_features(struct net_device *netdev,
1785                                netdev_features_t features)
1786{
1787        struct hns_nic_priv *priv = netdev_priv(netdev);
1788
1789        switch (priv->enet_ver) {
1790        case AE_VERSION_1:
1791                if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1792                        netdev_info(netdev, "enet v1 do not support tso!\n");
1793                break;
1794        default:
1795                if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1796                        priv->ops.fill_desc = fill_tso_desc;
1797                        priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tso;
1798                        /* The chip only support 7*4096 */
1799                        netif_set_gso_max_size(netdev, 7 * 4096);
1800                } else {
1801                        priv->ops.fill_desc = fill_v2_desc;
1802                        priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
1803                }
1804                break;
1805        }
1806        netdev->features = features;
1807        return 0;
1808}
1809
1810static netdev_features_t hns_nic_fix_features(
1811                struct net_device *netdev, netdev_features_t features)
1812{
1813        struct hns_nic_priv *priv = netdev_priv(netdev);
1814
1815        switch (priv->enet_ver) {
1816        case AE_VERSION_1:
1817                features &= ~(NETIF_F_TSO | NETIF_F_TSO6 |
1818                                NETIF_F_HW_VLAN_CTAG_FILTER);
1819                break;
1820        default:
1821                break;
1822        }
1823        return features;
1824}
1825
1826static int hns_nic_uc_sync(struct net_device *netdev, const unsigned char *addr)
1827{
1828        struct hns_nic_priv *priv = netdev_priv(netdev);
1829        struct hnae_handle *h = priv->ae_handle;
1830
1831        if (h->dev->ops->add_uc_addr)
1832                return h->dev->ops->add_uc_addr(h, addr);
1833
1834        return 0;
1835}
1836
1837static int hns_nic_uc_unsync(struct net_device *netdev,
1838                             const unsigned char *addr)
1839{
1840        struct hns_nic_priv *priv = netdev_priv(netdev);
1841        struct hnae_handle *h = priv->ae_handle;
1842
1843        if (h->dev->ops->rm_uc_addr)
1844                return h->dev->ops->rm_uc_addr(h, addr);
1845
1846        return 0;
1847}
1848
1849/**
1850 * nic_set_multicast_list - set mutl mac address
1851 * @netdev: net device
1852 * @p: mac address
1853 *
1854 * return void
1855 */
1856static void hns_set_multicast_list(struct net_device *ndev)
1857{
1858        struct hns_nic_priv *priv = netdev_priv(ndev);
1859        struct hnae_handle *h = priv->ae_handle;
1860        struct netdev_hw_addr *ha = NULL;
1861
1862        if (!h) {
1863                netdev_err(ndev, "hnae handle is null\n");
1864                return;
1865        }
1866
1867        if (h->dev->ops->clr_mc_addr)
1868                if (h->dev->ops->clr_mc_addr(h))
1869                        netdev_err(ndev, "clear multicast address fail\n");
1870
1871        if (h->dev->ops->set_mc_addr) {
1872                netdev_for_each_mc_addr(ha, ndev)
1873                        if (h->dev->ops->set_mc_addr(h, ha->addr))
1874                                netdev_err(ndev, "set multicast fail\n");
1875        }
1876}
1877
1878static void hns_nic_set_rx_mode(struct net_device *ndev)
1879{
1880        struct hns_nic_priv *priv = netdev_priv(ndev);
1881        struct hnae_handle *h = priv->ae_handle;
1882
1883        if (h->dev->ops->set_promisc_mode) {
1884                if (ndev->flags & IFF_PROMISC)
1885                        h->dev->ops->set_promisc_mode(h, 1);
1886                else
1887                        h->dev->ops->set_promisc_mode(h, 0);
1888        }
1889
1890        hns_set_multicast_list(ndev);
1891
1892        if (__dev_uc_sync(ndev, hns_nic_uc_sync, hns_nic_uc_unsync))
1893                netdev_err(ndev, "sync uc address fail\n");
1894}
1895
1896static void hns_nic_get_stats64(struct net_device *ndev,
1897                                struct rtnl_link_stats64 *stats)
1898{
1899        int idx = 0;
1900        u64 tx_bytes = 0;
1901        u64 rx_bytes = 0;
1902        u64 tx_pkts = 0;
1903        u64 rx_pkts = 0;
1904        struct hns_nic_priv *priv = netdev_priv(ndev);
1905        struct hnae_handle *h = priv->ae_handle;
1906
1907        for (idx = 0; idx < h->q_num; idx++) {
1908                tx_bytes += h->qs[idx]->tx_ring.stats.tx_bytes;
1909                tx_pkts += h->qs[idx]->tx_ring.stats.tx_pkts;
1910                rx_bytes += h->qs[idx]->rx_ring.stats.rx_bytes;
1911                rx_pkts += h->qs[idx]->rx_ring.stats.rx_pkts;
1912        }
1913
1914        stats->tx_bytes = tx_bytes;
1915        stats->tx_packets = tx_pkts;
1916        stats->rx_bytes = rx_bytes;
1917        stats->rx_packets = rx_pkts;
1918
1919        stats->rx_errors = ndev->stats.rx_errors;
1920        stats->multicast = ndev->stats.multicast;
1921        stats->rx_length_errors = ndev->stats.rx_length_errors;
1922        stats->rx_crc_errors = ndev->stats.rx_crc_errors;
1923        stats->rx_missed_errors = ndev->stats.rx_missed_errors;
1924
1925        stats->tx_errors = ndev->stats.tx_errors;
1926        stats->rx_dropped = ndev->stats.rx_dropped;
1927        stats->tx_dropped = ndev->stats.tx_dropped;
1928        stats->collisions = ndev->stats.collisions;
1929        stats->rx_over_errors = ndev->stats.rx_over_errors;
1930        stats->rx_frame_errors = ndev->stats.rx_frame_errors;
1931        stats->rx_fifo_errors = ndev->stats.rx_fifo_errors;
1932        stats->tx_aborted_errors = ndev->stats.tx_aborted_errors;
1933        stats->tx_carrier_errors = ndev->stats.tx_carrier_errors;
1934        stats->tx_fifo_errors = ndev->stats.tx_fifo_errors;
1935        stats->tx_heartbeat_errors = ndev->stats.tx_heartbeat_errors;
1936        stats->tx_window_errors = ndev->stats.tx_window_errors;
1937        stats->rx_compressed = ndev->stats.rx_compressed;
1938        stats->tx_compressed = ndev->stats.tx_compressed;
1939}
1940
1941static u16
1942hns_nic_select_queue(struct net_device *ndev, struct sk_buff *skb,
1943                     struct net_device *sb_dev)
1944{
1945        struct ethhdr *eth_hdr = (struct ethhdr *)skb->data;
1946        struct hns_nic_priv *priv = netdev_priv(ndev);
1947
1948        /* fix hardware broadcast/multicast packets queue loopback */
1949        if (!AE_IS_VER1(priv->enet_ver) &&
1950            is_multicast_ether_addr(eth_hdr->h_dest))
1951                return 0;
1952        else
1953                return netdev_pick_tx(ndev, skb, NULL);
1954}
1955
1956static const struct net_device_ops hns_nic_netdev_ops = {
1957        .ndo_open = hns_nic_net_open,
1958        .ndo_stop = hns_nic_net_stop,
1959        .ndo_start_xmit = hns_nic_net_xmit,
1960        .ndo_tx_timeout = hns_nic_net_timeout,
1961        .ndo_set_mac_address = hns_nic_net_set_mac_address,
1962        .ndo_change_mtu = hns_nic_change_mtu,
1963        .ndo_do_ioctl = hns_nic_do_ioctl,
1964        .ndo_set_features = hns_nic_set_features,
1965        .ndo_fix_features = hns_nic_fix_features,
1966        .ndo_get_stats64 = hns_nic_get_stats64,
1967        .ndo_set_rx_mode = hns_nic_set_rx_mode,
1968        .ndo_select_queue = hns_nic_select_queue,
1969};
1970
1971static void hns_nic_update_link_status(struct net_device *netdev)
1972{
1973        struct hns_nic_priv *priv = netdev_priv(netdev);
1974
1975        struct hnae_handle *h = priv->ae_handle;
1976
1977        if (h->phy_dev) {
1978                if (h->phy_if != PHY_INTERFACE_MODE_XGMII)
1979                        return;
1980
1981                (void)genphy_read_status(h->phy_dev);
1982        }
1983        hns_nic_adjust_link(netdev);
1984}
1985
1986/* for dumping key regs*/
1987static void hns_nic_dump(struct hns_nic_priv *priv)
1988{
1989        struct hnae_handle *h = priv->ae_handle;
1990        struct hnae_ae_ops *ops = h->dev->ops;
1991        u32 *data, reg_num, i;
1992
1993        if (ops->get_regs_len && ops->get_regs) {
1994                reg_num = ops->get_regs_len(priv->ae_handle);
1995                reg_num = (reg_num + 3ul) & ~3ul;
1996                data = kcalloc(reg_num, sizeof(u32), GFP_KERNEL);
1997                if (data) {
1998                        ops->get_regs(priv->ae_handle, data);
1999                        for (i = 0; i < reg_num; i += 4)
2000                                pr_info("0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
2001                                        i, data[i], data[i + 1],
2002                                        data[i + 2], data[i + 3]);
2003                        kfree(data);
2004                }
2005        }
2006
2007        for (i = 0; i < h->q_num; i++) {
2008                pr_info("tx_queue%d_next_to_clean:%d\n",
2009                        i, h->qs[i]->tx_ring.next_to_clean);
2010                pr_info("tx_queue%d_next_to_use:%d\n",
2011                        i, h->qs[i]->tx_ring.next_to_use);
2012                pr_info("rx_queue%d_next_to_clean:%d\n",
2013                        i, h->qs[i]->rx_ring.next_to_clean);
2014                pr_info("rx_queue%d_next_to_use:%d\n",
2015                        i, h->qs[i]->rx_ring.next_to_use);
2016        }
2017}
2018
2019/* for resetting subtask */
2020static void hns_nic_reset_subtask(struct hns_nic_priv *priv)
2021{
2022        enum hnae_port_type type = priv->ae_handle->port_type;
2023
2024        if (!test_bit(NIC_STATE2_RESET_REQUESTED, &priv->state))
2025                return;
2026        clear_bit(NIC_STATE2_RESET_REQUESTED, &priv->state);
2027
2028        /* If we're already down, removing or resetting, just bail */
2029        if (test_bit(NIC_STATE_DOWN, &priv->state) ||
2030            test_bit(NIC_STATE_REMOVING, &priv->state) ||
2031            test_bit(NIC_STATE_RESETTING, &priv->state))
2032                return;
2033
2034        hns_nic_dump(priv);
2035        netdev_info(priv->netdev, "try to reset %s port!\n",
2036                    (type == HNAE_PORT_DEBUG ? "debug" : "service"));
2037
2038        rtnl_lock();
2039        /* put off any impending NetWatchDogTimeout */
2040        netif_trans_update(priv->netdev);
2041        hns_nic_net_reinit(priv->netdev);
2042
2043        rtnl_unlock();
2044}
2045
2046/* for doing service complete*/
2047static void hns_nic_service_event_complete(struct hns_nic_priv *priv)
2048{
2049        WARN_ON(!test_bit(NIC_STATE_SERVICE_SCHED, &priv->state));
2050        /* make sure to commit the things */
2051        smp_mb__before_atomic();
2052        clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state);
2053}
2054
2055static void hns_nic_service_task(struct work_struct *work)
2056{
2057        struct hns_nic_priv *priv
2058                = container_of(work, struct hns_nic_priv, service_task);
2059        struct hnae_handle *h = priv->ae_handle;
2060
2061        hns_nic_reset_subtask(priv);
2062        hns_nic_update_link_status(priv->netdev);
2063        h->dev->ops->update_led_status(h);
2064        hns_nic_update_stats(priv->netdev);
2065
2066        hns_nic_service_event_complete(priv);
2067}
2068
2069static void hns_nic_task_schedule(struct hns_nic_priv *priv)
2070{
2071        if (!test_bit(NIC_STATE_DOWN, &priv->state) &&
2072            !test_bit(NIC_STATE_REMOVING, &priv->state) &&
2073            !test_and_set_bit(NIC_STATE_SERVICE_SCHED, &priv->state))
2074                (void)schedule_work(&priv->service_task);
2075}
2076
2077static void hns_nic_service_timer(struct timer_list *t)
2078{
2079        struct hns_nic_priv *priv = from_timer(priv, t, service_timer);
2080
2081        (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
2082
2083        hns_nic_task_schedule(priv);
2084}
2085
2086/**
2087 * hns_tx_timeout_reset - initiate reset due to Tx timeout
2088 * @priv: driver private struct
2089 **/
2090static void hns_tx_timeout_reset(struct hns_nic_priv *priv)
2091{
2092        /* Do the reset outside of interrupt context */
2093        if (!test_bit(NIC_STATE_DOWN, &priv->state)) {
2094                set_bit(NIC_STATE2_RESET_REQUESTED, &priv->state);
2095                netdev_warn(priv->netdev,
2096                            "initiating reset due to tx timeout(%llu,0x%lx)\n",
2097                            priv->tx_timeout_count, priv->state);
2098                priv->tx_timeout_count++;
2099                hns_nic_task_schedule(priv);
2100        }
2101}
2102
2103static int hns_nic_init_ring_data(struct hns_nic_priv *priv)
2104{
2105        struct hnae_handle *h = priv->ae_handle;
2106        struct hns_nic_ring_data *rd;
2107        bool is_ver1 = AE_IS_VER1(priv->enet_ver);
2108        int i;
2109
2110        if (h->q_num > NIC_MAX_Q_PER_VF) {
2111                netdev_err(priv->netdev, "too much queue (%d)\n", h->q_num);
2112                return -EINVAL;
2113        }
2114
2115        priv->ring_data = kzalloc(array3_size(h->q_num,
2116                                              sizeof(*priv->ring_data), 2),
2117                                  GFP_KERNEL);
2118        if (!priv->ring_data)
2119                return -ENOMEM;
2120
2121        for (i = 0; i < h->q_num; i++) {
2122                rd = &priv->ring_data[i];
2123                rd->queue_index = i;
2124                rd->ring = &h->qs[i]->tx_ring;
2125                rd->poll_one = hns_nic_tx_poll_one;
2126                rd->fini_process = is_ver1 ? hns_nic_tx_fini_pro :
2127                        hns_nic_tx_fini_pro_v2;
2128
2129                netif_napi_add(priv->netdev, &rd->napi,
2130                               hns_nic_common_poll, NAPI_POLL_WEIGHT);
2131                rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
2132        }
2133        for (i = h->q_num; i < h->q_num * 2; i++) {
2134                rd = &priv->ring_data[i];
2135                rd->queue_index = i - h->q_num;
2136                rd->ring = &h->qs[i - h->q_num]->rx_ring;
2137                rd->poll_one = hns_nic_rx_poll_one;
2138                rd->ex_process = hns_nic_rx_up_pro;
2139                rd->fini_process = is_ver1 ? hns_nic_rx_fini_pro :
2140                        hns_nic_rx_fini_pro_v2;
2141
2142                netif_napi_add(priv->netdev, &rd->napi,
2143                               hns_nic_common_poll, NAPI_POLL_WEIGHT);
2144                rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
2145        }
2146
2147        return 0;
2148}
2149
2150static void hns_nic_uninit_ring_data(struct hns_nic_priv *priv)
2151{
2152        struct hnae_handle *h = priv->ae_handle;
2153        int i;
2154
2155        for (i = 0; i < h->q_num * 2; i++) {
2156                netif_napi_del(&priv->ring_data[i].napi);
2157                if (priv->ring_data[i].ring->irq_init_flag == RCB_IRQ_INITED) {
2158                        (void)irq_set_affinity_hint(
2159                                priv->ring_data[i].ring->irq,
2160                                NULL);
2161                        free_irq(priv->ring_data[i].ring->irq,
2162                                 &priv->ring_data[i]);
2163                }
2164
2165                priv->ring_data[i].ring->irq_init_flag = RCB_IRQ_NOT_INITED;
2166        }
2167        kfree(priv->ring_data);
2168}
2169
2170static void hns_nic_set_priv_ops(struct net_device *netdev)
2171{
2172        struct hns_nic_priv *priv = netdev_priv(netdev);
2173        struct hnae_handle *h = priv->ae_handle;
2174
2175        if (AE_IS_VER1(priv->enet_ver)) {
2176                priv->ops.fill_desc = fill_desc;
2177                priv->ops.get_rxd_bnum = get_rx_desc_bnum;
2178                priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
2179        } else {
2180                priv->ops.get_rxd_bnum = get_v2rx_desc_bnum;
2181                if ((netdev->features & NETIF_F_TSO) ||
2182                    (netdev->features & NETIF_F_TSO6)) {
2183                        priv->ops.fill_desc = fill_tso_desc;
2184                        priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tso;
2185                        /* This chip only support 7*4096 */
2186                        netif_set_gso_max_size(netdev, 7 * 4096);
2187                } else {
2188                        priv->ops.fill_desc = fill_v2_desc;
2189                        priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
2190                }
2191                /* enable tso when init
2192                 * control tso on/off through TSE bit in bd
2193                 */
2194                h->dev->ops->set_tso_stats(h, 1);
2195        }
2196}
2197
2198static int hns_nic_try_get_ae(struct net_device *ndev)
2199{
2200        struct hns_nic_priv *priv = netdev_priv(ndev);
2201        struct hnae_handle *h;
2202        int ret;
2203
2204        h = hnae_get_handle(&priv->netdev->dev,
2205                            priv->fwnode, priv->port_id, NULL);
2206        if (IS_ERR_OR_NULL(h)) {
2207                ret = -ENODEV;
2208                dev_dbg(priv->dev, "has not handle, register notifier!\n");
2209                goto out;
2210        }
2211        priv->ae_handle = h;
2212
2213        ret = hns_nic_init_phy(ndev, h);
2214        if (ret) {
2215                dev_err(priv->dev, "probe phy device fail!\n");
2216                goto out_init_phy;
2217        }
2218
2219        ret = hns_nic_init_ring_data(priv);
2220        if (ret) {
2221                ret = -ENOMEM;
2222                goto out_init_ring_data;
2223        }
2224
2225        hns_nic_set_priv_ops(ndev);
2226
2227        ret = register_netdev(ndev);
2228        if (ret) {
2229                dev_err(priv->dev, "probe register netdev fail!\n");
2230                goto out_reg_ndev_fail;
2231        }
2232        return 0;
2233
2234out_reg_ndev_fail:
2235        hns_nic_uninit_ring_data(priv);
2236        priv->ring_data = NULL;
2237out_init_phy:
2238out_init_ring_data:
2239        hnae_put_handle(priv->ae_handle);
2240        priv->ae_handle = NULL;
2241out:
2242        return ret;
2243}
2244
2245static int hns_nic_notifier_action(struct notifier_block *nb,
2246                                   unsigned long action, void *data)
2247{
2248        struct hns_nic_priv *priv =
2249                container_of(nb, struct hns_nic_priv, notifier_block);
2250
2251        assert(action == HNAE_AE_REGISTER);
2252
2253        if (!hns_nic_try_get_ae(priv->netdev)) {
2254                hnae_unregister_notifier(&priv->notifier_block);
2255                priv->notifier_block.notifier_call = NULL;
2256        }
2257        return 0;
2258}
2259
2260static int hns_nic_dev_probe(struct platform_device *pdev)
2261{
2262        struct device *dev = &pdev->dev;
2263        struct net_device *ndev;
2264        struct hns_nic_priv *priv;
2265        u32 port_id;
2266        int ret;
2267
2268        ndev = alloc_etherdev_mq(sizeof(struct hns_nic_priv), NIC_MAX_Q_PER_VF);
2269        if (!ndev)
2270                return -ENOMEM;
2271
2272        platform_set_drvdata(pdev, ndev);
2273
2274        priv = netdev_priv(ndev);
2275        priv->dev = dev;
2276        priv->netdev = ndev;
2277
2278        if (dev_of_node(dev)) {
2279                struct device_node *ae_node;
2280
2281                if (of_device_is_compatible(dev->of_node,
2282                                            "hisilicon,hns-nic-v1"))
2283                        priv->enet_ver = AE_VERSION_1;
2284                else
2285                        priv->enet_ver = AE_VERSION_2;
2286
2287                ae_node = of_parse_phandle(dev->of_node, "ae-handle", 0);
2288                if (!ae_node) {
2289                        ret = -ENODEV;
2290                        dev_err(dev, "not find ae-handle\n");
2291                        goto out_read_prop_fail;
2292                }
2293                priv->fwnode = &ae_node->fwnode;
2294        } else if (is_acpi_node(dev->fwnode)) {
2295                struct fwnode_reference_args args;
2296
2297                if (acpi_dev_found(hns_enet_acpi_match[0].id))
2298                        priv->enet_ver = AE_VERSION_1;
2299                else if (acpi_dev_found(hns_enet_acpi_match[1].id))
2300                        priv->enet_ver = AE_VERSION_2;
2301                else
2302                        return -ENXIO;
2303
2304                /* try to find port-idx-in-ae first */
2305                ret = acpi_node_get_property_reference(dev->fwnode,
2306                                                       "ae-handle", 0, &args);
2307                if (ret) {
2308                        dev_err(dev, "not find ae-handle\n");
2309                        goto out_read_prop_fail;
2310                }
2311                if (!is_acpi_device_node(args.fwnode)) {
2312                        ret = -EINVAL;
2313                        goto out_read_prop_fail;
2314                }
2315                priv->fwnode = args.fwnode;
2316        } else {
2317                dev_err(dev, "cannot read cfg data from OF or acpi\n");
2318                return -ENXIO;
2319        }
2320
2321        ret = device_property_read_u32(dev, "port-idx-in-ae", &port_id);
2322        if (ret) {
2323                /* only for old code compatible */
2324                ret = device_property_read_u32(dev, "port-id", &port_id);
2325                if (ret)
2326                        goto out_read_prop_fail;
2327                /* for old dts, we need to caculate the port offset */
2328                port_id = port_id < HNS_SRV_OFFSET ? port_id + HNS_DEBUG_OFFSET
2329                        : port_id - HNS_SRV_OFFSET;
2330        }
2331        priv->port_id = port_id;
2332
2333        hns_init_mac_addr(ndev);
2334
2335        ndev->watchdog_timeo = HNS_NIC_TX_TIMEOUT;
2336        ndev->priv_flags |= IFF_UNICAST_FLT;
2337        ndev->netdev_ops = &hns_nic_netdev_ops;
2338        hns_ethtool_set_ops(ndev);
2339
2340        ndev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2341                NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2342                NETIF_F_GRO;
2343        ndev->vlan_features |=
2344                NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM;
2345        ndev->vlan_features |= NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;
2346
2347        /* MTU range: 68 - 9578 (v1) or 9706 (v2) */
2348        ndev->min_mtu = MAC_MIN_MTU;
2349        switch (priv->enet_ver) {
2350        case AE_VERSION_2:
2351                ndev->features |= NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_NTUPLE;
2352                ndev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2353                        NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2354                        NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6;
2355                ndev->vlan_features |= NETIF_F_TSO | NETIF_F_TSO6;
2356                ndev->max_mtu = MAC_MAX_MTU_V2 -
2357                                (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
2358                break;
2359        default:
2360                ndev->max_mtu = MAC_MAX_MTU -
2361                                (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
2362                break;
2363        }
2364
2365        SET_NETDEV_DEV(ndev, dev);
2366
2367        if (!dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
2368                dev_dbg(dev, "set mask to 64bit\n");
2369        else
2370                dev_err(dev, "set mask to 64bit fail!\n");
2371
2372        /* carrier off reporting is important to ethtool even BEFORE open */
2373        netif_carrier_off(ndev);
2374
2375        timer_setup(&priv->service_timer, hns_nic_service_timer, 0);
2376        INIT_WORK(&priv->service_task, hns_nic_service_task);
2377
2378        set_bit(NIC_STATE_SERVICE_INITED, &priv->state);
2379        clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state);
2380        set_bit(NIC_STATE_DOWN, &priv->state);
2381
2382        if (hns_nic_try_get_ae(priv->netdev)) {
2383                priv->notifier_block.notifier_call = hns_nic_notifier_action;
2384                ret = hnae_register_notifier(&priv->notifier_block);
2385                if (ret) {
2386                        dev_err(dev, "register notifier fail!\n");
2387                        goto out_notify_fail;
2388                }
2389                dev_dbg(dev, "has not handle, register notifier!\n");
2390        }
2391
2392        return 0;
2393
2394out_notify_fail:
2395        (void)cancel_work_sync(&priv->service_task);
2396out_read_prop_fail:
2397        /* safe for ACPI FW */
2398        of_node_put(to_of_node(priv->fwnode));
2399        free_netdev(ndev);
2400        return ret;
2401}
2402
2403static int hns_nic_dev_remove(struct platform_device *pdev)
2404{
2405        struct net_device *ndev = platform_get_drvdata(pdev);
2406        struct hns_nic_priv *priv = netdev_priv(ndev);
2407
2408        if (ndev->reg_state != NETREG_UNINITIALIZED)
2409                unregister_netdev(ndev);
2410
2411        if (priv->ring_data)
2412                hns_nic_uninit_ring_data(priv);
2413        priv->ring_data = NULL;
2414
2415        if (ndev->phydev)
2416                phy_disconnect(ndev->phydev);
2417
2418        if (!IS_ERR_OR_NULL(priv->ae_handle))
2419                hnae_put_handle(priv->ae_handle);
2420        priv->ae_handle = NULL;
2421        if (priv->notifier_block.notifier_call)
2422                hnae_unregister_notifier(&priv->notifier_block);
2423        priv->notifier_block.notifier_call = NULL;
2424
2425        set_bit(NIC_STATE_REMOVING, &priv->state);
2426        (void)cancel_work_sync(&priv->service_task);
2427
2428        /* safe for ACPI FW */
2429        of_node_put(to_of_node(priv->fwnode));
2430
2431        free_netdev(ndev);
2432        return 0;
2433}
2434
2435static const struct of_device_id hns_enet_of_match[] = {
2436        {.compatible = "hisilicon,hns-nic-v1",},
2437        {.compatible = "hisilicon,hns-nic-v2",},
2438        {},
2439};
2440
2441MODULE_DEVICE_TABLE(of, hns_enet_of_match);
2442
2443static struct platform_driver hns_nic_dev_driver = {
2444        .driver = {
2445                .name = "hns-nic",
2446                .of_match_table = hns_enet_of_match,
2447                .acpi_match_table = ACPI_PTR(hns_enet_acpi_match),
2448        },
2449        .probe = hns_nic_dev_probe,
2450        .remove = hns_nic_dev_remove,
2451};
2452
2453module_platform_driver(hns_nic_dev_driver);
2454
2455MODULE_DESCRIPTION("HISILICON HNS Ethernet driver");
2456MODULE_AUTHOR("Hisilicon, Inc.");
2457MODULE_LICENSE("GPL");
2458MODULE_ALIAS("platform:hns-nic");
2459