linux/drivers/net/wireless/brcm80211/brcmsmac/dma.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2010 Broadcom Corporation
   3 *
   4 * Permission to use, copy, modify, and/or distribute this software for any
   5 * purpose with or without fee is hereby granted, provided that the above
   6 * copyright notice and this permission notice appear in all copies.
   7 *
   8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  11 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  13 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  14 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15 */
  16
  17#include <linux/slab.h>
  18#include <linux/delay.h>
  19#include <linux/pci.h>
  20#include <net/cfg80211.h>
  21#include <net/mac80211.h>
  22
  23#include <brcmu_utils.h>
  24#include <aiutils.h>
  25#include "types.h"
  26#include "main.h"
  27#include "dma.h"
  28#include "soc.h"
  29#include "scb.h"
  30#include "ampdu.h"
  31#include "debug.h"
  32#include "brcms_trace_events.h"
  33
  34/*
  35 * dma register field offset calculation
  36 */
  37#define DMA64REGOFFS(field)             offsetof(struct dma64regs, field)
  38#define DMA64TXREGOFFS(di, field)       (di->d64txregbase + DMA64REGOFFS(field))
  39#define DMA64RXREGOFFS(di, field)       (di->d64rxregbase + DMA64REGOFFS(field))
  40
  41/*
  42 * DMA hardware requires each descriptor ring to be 8kB aligned, and fit within
  43 * a contiguous 8kB physical address.
  44 */
  45#define D64RINGALIGN_BITS       13
  46#define D64MAXRINGSZ            (1 << D64RINGALIGN_BITS)
  47#define D64RINGALIGN            (1 << D64RINGALIGN_BITS)
  48
  49#define D64MAXDD        (D64MAXRINGSZ / sizeof(struct dma64desc))
  50
  51/* transmit channel control */
  52#define D64_XC_XE               0x00000001      /* transmit enable */
  53#define D64_XC_SE               0x00000002      /* transmit suspend request */
  54#define D64_XC_LE               0x00000004      /* loopback enable */
  55#define D64_XC_FL               0x00000010      /* flush request */
  56#define D64_XC_PD               0x00000800      /* parity check disable */
  57#define D64_XC_AE               0x00030000      /* address extension bits */
  58#define D64_XC_AE_SHIFT         16
  59
  60/* transmit descriptor table pointer */
  61#define D64_XP_LD_MASK          0x00000fff      /* last valid descriptor */
  62
  63/* transmit channel status */
  64#define D64_XS0_CD_MASK         0x00001fff      /* current descriptor pointer */
  65#define D64_XS0_XS_MASK         0xf0000000      /* transmit state */
  66#define D64_XS0_XS_SHIFT                28
  67#define D64_XS0_XS_DISABLED     0x00000000      /* disabled */
  68#define D64_XS0_XS_ACTIVE       0x10000000      /* active */
  69#define D64_XS0_XS_IDLE         0x20000000      /* idle wait */
  70#define D64_XS0_XS_STOPPED      0x30000000      /* stopped */
  71#define D64_XS0_XS_SUSP         0x40000000      /* suspend pending */
  72
  73#define D64_XS1_AD_MASK         0x00001fff      /* active descriptor */
  74#define D64_XS1_XE_MASK         0xf0000000      /* transmit errors */
  75#define D64_XS1_XE_SHIFT                28
  76#define D64_XS1_XE_NOERR        0x00000000      /* no error */
  77#define D64_XS1_XE_DPE          0x10000000      /* descriptor protocol error */
  78#define D64_XS1_XE_DFU          0x20000000      /* data fifo underrun */
  79#define D64_XS1_XE_DTE          0x30000000      /* data transfer error */
  80#define D64_XS1_XE_DESRE        0x40000000      /* descriptor read error */
  81#define D64_XS1_XE_COREE        0x50000000      /* core error */
  82
  83/* receive channel control */
  84/* receive enable */
  85#define D64_RC_RE               0x00000001
  86/* receive frame offset */
  87#define D64_RC_RO_MASK          0x000000fe
  88#define D64_RC_RO_SHIFT         1
  89/* direct fifo receive (pio) mode */
  90#define D64_RC_FM               0x00000100
  91/* separate rx header descriptor enable */
  92#define D64_RC_SH               0x00000200
  93/* overflow continue */
  94#define D64_RC_OC               0x00000400
  95/* parity check disable */
  96#define D64_RC_PD               0x00000800
  97/* address extension bits */
  98#define D64_RC_AE               0x00030000
  99#define D64_RC_AE_SHIFT         16
 100
 101/* flags for dma controller */
 102/* partity enable */
 103#define DMA_CTRL_PEN            (1 << 0)
 104/* rx overflow continue */
 105#define DMA_CTRL_ROC            (1 << 1)
 106/* allow rx scatter to multiple descriptors */
 107#define DMA_CTRL_RXMULTI        (1 << 2)
 108/* Unframed Rx/Tx data */
 109#define DMA_CTRL_UNFRAMED       (1 << 3)
 110
 111/* receive descriptor table pointer */
 112#define D64_RP_LD_MASK          0x00000fff      /* last valid descriptor */
 113
 114/* receive channel status */
 115#define D64_RS0_CD_MASK         0x00001fff      /* current descriptor pointer */
 116#define D64_RS0_RS_MASK         0xf0000000      /* receive state */
 117#define D64_RS0_RS_SHIFT                28
 118#define D64_RS0_RS_DISABLED     0x00000000      /* disabled */
 119#define D64_RS0_RS_ACTIVE       0x10000000      /* active */
 120#define D64_RS0_RS_IDLE         0x20000000      /* idle wait */
 121#define D64_RS0_RS_STOPPED      0x30000000      /* stopped */
 122#define D64_RS0_RS_SUSP         0x40000000      /* suspend pending */
 123
 124#define D64_RS1_AD_MASK         0x0001ffff      /* active descriptor */
 125#define D64_RS1_RE_MASK         0xf0000000      /* receive errors */
 126#define D64_RS1_RE_SHIFT                28
 127#define D64_RS1_RE_NOERR        0x00000000      /* no error */
 128#define D64_RS1_RE_DPO          0x10000000      /* descriptor protocol error */
 129#define D64_RS1_RE_DFU          0x20000000      /* data fifo overflow */
 130#define D64_RS1_RE_DTE          0x30000000      /* data transfer error */
 131#define D64_RS1_RE_DESRE        0x40000000      /* descriptor read error */
 132#define D64_RS1_RE_COREE        0x50000000      /* core error */
 133
 134/* fifoaddr */
 135#define D64_FA_OFF_MASK         0xffff  /* offset */
 136#define D64_FA_SEL_MASK         0xf0000 /* select */
 137#define D64_FA_SEL_SHIFT        16
 138#define D64_FA_SEL_XDD          0x00000 /* transmit dma data */
 139#define D64_FA_SEL_XDP          0x10000 /* transmit dma pointers */
 140#define D64_FA_SEL_RDD          0x40000 /* receive dma data */
 141#define D64_FA_SEL_RDP          0x50000 /* receive dma pointers */
 142#define D64_FA_SEL_XFD          0x80000 /* transmit fifo data */
 143#define D64_FA_SEL_XFP          0x90000 /* transmit fifo pointers */
 144#define D64_FA_SEL_RFD          0xc0000 /* receive fifo data */
 145#define D64_FA_SEL_RFP          0xd0000 /* receive fifo pointers */
 146#define D64_FA_SEL_RSD          0xe0000 /* receive frame status data */
 147#define D64_FA_SEL_RSP          0xf0000 /* receive frame status pointers */
 148
 149/* descriptor control flags 1 */
 150#define D64_CTRL_COREFLAGS      0x0ff00000      /* core specific flags */
 151#define D64_CTRL1_EOT           ((u32)1 << 28)  /* end of descriptor table */
 152#define D64_CTRL1_IOC           ((u32)1 << 29)  /* interrupt on completion */
 153#define D64_CTRL1_EOF           ((u32)1 << 30)  /* end of frame */
 154#define D64_CTRL1_SOF           ((u32)1 << 31)  /* start of frame */
 155
 156/* descriptor control flags 2 */
 157/* buffer byte count. real data len must <= 16KB */
 158#define D64_CTRL2_BC_MASK       0x00007fff
 159/* address extension bits */
 160#define D64_CTRL2_AE            0x00030000
 161#define D64_CTRL2_AE_SHIFT      16
 162/* parity bit */
 163#define D64_CTRL2_PARITY        0x00040000
 164
 165/* control flags in the range [27:20] are core-specific and not defined here */
 166#define D64_CTRL_CORE_MASK      0x0ff00000
 167
 168#define D64_RX_FRM_STS_LEN      0x0000ffff      /* frame length mask */
 169#define D64_RX_FRM_STS_OVFL     0x00800000      /* RxOverFlow */
 170#define D64_RX_FRM_STS_DSCRCNT  0x0f000000  /* no. of descriptors used - 1 */
 171#define D64_RX_FRM_STS_DATATYPE 0xf0000000      /* core-dependent data type */
 172
 173/*
 174 * packet headroom necessary to accommodate the largest header
 175 * in the system, (i.e TXOFF). By doing, we avoid the need to
 176 * allocate an extra buffer for the header when bridging to WL.
 177 * There is a compile time check in wlc.c which ensure that this
 178 * value is at least as big as TXOFF. This value is used in
 179 * dma_rxfill().
 180 */
 181
 182#define BCMEXTRAHDROOM 172
 183
 184#define MAXNAMEL        8       /* 8 char names */
 185
 186/* macros to convert between byte offsets and indexes */
 187#define B2I(bytes, type)        ((bytes) / sizeof(type))
 188#define I2B(index, type)        ((index) * sizeof(type))
 189
 190#define PCI32ADDR_HIGH          0xc0000000      /* address[31:30] */
 191#define PCI32ADDR_HIGH_SHIFT    30      /* address[31:30] */
 192
 193#define PCI64ADDR_HIGH          0x80000000      /* address[63] */
 194#define PCI64ADDR_HIGH_SHIFT    31      /* address[63] */
 195
 196/*
 197 * DMA Descriptor
 198 * Descriptors are only read by the hardware, never written back.
 199 */
 200struct dma64desc {
 201        __le32 ctrl1;   /* misc control bits & bufcount */
 202        __le32 ctrl2;   /* buffer count and address extension */
 203        __le32 addrlow; /* memory address of the date buffer, bits 31:0 */
 204        __le32 addrhigh; /* memory address of the date buffer, bits 63:32 */
 205};
 206
 207/* dma engine software state */
 208struct dma_info {
 209        struct dma_pub dma; /* exported structure */
 210        char name[MAXNAMEL];    /* callers name for diag msgs */
 211
 212        struct bcma_device *core;
 213        struct device *dmadev;
 214
 215        /* session information for AMPDU */
 216        struct brcms_ampdu_session ampdu_session;
 217
 218        bool dma64;     /* this dma engine is operating in 64-bit mode */
 219        bool addrext;   /* this dma engine supports DmaExtendedAddrChanges */
 220
 221        /* 64-bit dma tx engine registers */
 222        uint d64txregbase;
 223        /* 64-bit dma rx engine registers */
 224        uint d64rxregbase;
 225        /* pointer to dma64 tx descriptor ring */
 226        struct dma64desc *txd64;
 227        /* pointer to dma64 rx descriptor ring */
 228        struct dma64desc *rxd64;
 229
 230        u16 dmadesc_align;      /* alignment requirement for dma descriptors */
 231
 232        u16 ntxd;               /* # tx descriptors tunable */
 233        u16 txin;               /* index of next descriptor to reclaim */
 234        u16 txout;              /* index of next descriptor to post */
 235        /* pointer to parallel array of pointers to packets */
 236        struct sk_buff **txp;
 237        /* Aligned physical address of descriptor ring */
 238        dma_addr_t txdpa;
 239        /* Original physical address of descriptor ring */
 240        dma_addr_t txdpaorig;
 241        u16 txdalign;   /* #bytes added to alloc'd mem to align txd */
 242        u32 txdalloc;   /* #bytes allocated for the ring */
 243        u32 xmtptrbase; /* When using unaligned descriptors, the ptr register
 244                         * is not just an index, it needs all 13 bits to be
 245                         * an offset from the addr register.
 246                         */
 247
 248        u16 nrxd;       /* # rx descriptors tunable */
 249        u16 rxin;       /* index of next descriptor to reclaim */
 250        u16 rxout;      /* index of next descriptor to post */
 251        /* pointer to parallel array of pointers to packets */
 252        struct sk_buff **rxp;
 253        /* Aligned physical address of descriptor ring */
 254        dma_addr_t rxdpa;
 255        /* Original physical address of descriptor ring */
 256        dma_addr_t rxdpaorig;
 257        u16 rxdalign;   /* #bytes added to alloc'd mem to align rxd */
 258        u32 rxdalloc;   /* #bytes allocated for the ring */
 259        u32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */
 260
 261        /* tunables */
 262        unsigned int rxbufsize; /* rx buffer size in bytes, not including
 263                                 * the extra headroom
 264                                 */
 265        uint rxextrahdrroom;    /* extra rx headroom, reverseved to assist upper
 266                                 * stack, e.g. some rx pkt buffers will be
 267                                 * bridged to tx side without byte copying.
 268                                 * The extra headroom needs to be large enough
 269                                 * to fit txheader needs. Some dongle driver may
 270                                 * not need it.
 271                                 */
 272        uint nrxpost;           /* # rx buffers to keep posted */
 273        unsigned int rxoffset;  /* rxcontrol offset */
 274        /* add to get dma address of descriptor ring, low 32 bits */
 275        uint ddoffsetlow;
 276        /*   high 32 bits */
 277        uint ddoffsethigh;
 278        /* add to get dma address of data buffer, low 32 bits */
 279        uint dataoffsetlow;
 280        /*   high 32 bits */
 281        uint dataoffsethigh;
 282        /* descriptor base need to be aligned or not */
 283        bool aligndesc_4k;
 284};
 285
 286/* Check for odd number of 1's */
 287static u32 parity32(__le32 data)
 288{
 289        /* no swap needed for counting 1's */
 290        u32 par_data = *(u32 *)&data;
 291
 292        par_data ^= par_data >> 16;
 293        par_data ^= par_data >> 8;
 294        par_data ^= par_data >> 4;
 295        par_data ^= par_data >> 2;
 296        par_data ^= par_data >> 1;
 297
 298        return par_data & 1;
 299}
 300
 301static bool dma64_dd_parity(struct dma64desc *dd)
 302{
 303        return parity32(dd->addrlow ^ dd->addrhigh ^ dd->ctrl1 ^ dd->ctrl2);
 304}
 305
 306/* descriptor bumping functions */
 307
 308static uint xxd(uint x, uint n)
 309{
 310        return x & (n - 1); /* faster than %, but n must be power of 2 */
 311}
 312
 313static uint txd(struct dma_info *di, uint x)
 314{
 315        return xxd(x, di->ntxd);
 316}
 317
 318static uint rxd(struct dma_info *di, uint x)
 319{
 320        return xxd(x, di->nrxd);
 321}
 322
 323static uint nexttxd(struct dma_info *di, uint i)
 324{
 325        return txd(di, i + 1);
 326}
 327
 328static uint prevtxd(struct dma_info *di, uint i)
 329{
 330        return txd(di, i - 1);
 331}
 332
 333static uint nextrxd(struct dma_info *di, uint i)
 334{
 335        return rxd(di, i + 1);
 336}
 337
 338static uint ntxdactive(struct dma_info *di, uint h, uint t)
 339{
 340        return txd(di, t-h);
 341}
 342
 343static uint nrxdactive(struct dma_info *di, uint h, uint t)
 344{
 345        return rxd(di, t-h);
 346}
 347
 348static uint _dma_ctrlflags(struct dma_info *di, uint mask, uint flags)
 349{
 350        uint dmactrlflags;
 351
 352        if (di == NULL)
 353                return 0;
 354
 355        dmactrlflags = di->dma.dmactrlflags;
 356        dmactrlflags &= ~mask;
 357        dmactrlflags |= flags;
 358
 359        /* If trying to enable parity, check if parity is actually supported */
 360        if (dmactrlflags & DMA_CTRL_PEN) {
 361                u32 control;
 362
 363                control = bcma_read32(di->core, DMA64TXREGOFFS(di, control));
 364                bcma_write32(di->core, DMA64TXREGOFFS(di, control),
 365                      control | D64_XC_PD);
 366                if (bcma_read32(di->core, DMA64TXREGOFFS(di, control)) &
 367                    D64_XC_PD)
 368                        /* We *can* disable it so it is supported,
 369                         * restore control register
 370                         */
 371                        bcma_write32(di->core, DMA64TXREGOFFS(di, control),
 372                                     control);
 373                else
 374                        /* Not supported, don't allow it to be enabled */
 375                        dmactrlflags &= ~DMA_CTRL_PEN;
 376        }
 377
 378        di->dma.dmactrlflags = dmactrlflags;
 379
 380        return dmactrlflags;
 381}
 382
 383static bool _dma64_addrext(struct dma_info *di, uint ctrl_offset)
 384{
 385        u32 w;
 386        bcma_set32(di->core, ctrl_offset, D64_XC_AE);
 387        w = bcma_read32(di->core, ctrl_offset);
 388        bcma_mask32(di->core, ctrl_offset, ~D64_XC_AE);
 389        return (w & D64_XC_AE) == D64_XC_AE;
 390}
 391
 392/*
 393 * return true if this dma engine supports DmaExtendedAddrChanges,
 394 * otherwise false
 395 */
 396static bool _dma_isaddrext(struct dma_info *di)
 397{
 398        /* DMA64 supports full 32- or 64-bit operation. AE is always valid */
 399
 400        /* not all tx or rx channel are available */
 401        if (di->d64txregbase != 0) {
 402                if (!_dma64_addrext(di, DMA64TXREGOFFS(di, control)))
 403                        brcms_dbg_dma(di->core,
 404                                      "%s: DMA64 tx doesn't have AE set\n",
 405                                      di->name);
 406                return true;
 407        } else if (di->d64rxregbase != 0) {
 408                if (!_dma64_addrext(di, DMA64RXREGOFFS(di, control)))
 409                        brcms_dbg_dma(di->core,
 410                                      "%s: DMA64 rx doesn't have AE set\n",
 411                                      di->name);
 412                return true;
 413        }
 414
 415        return false;
 416}
 417
 418static bool _dma_descriptor_align(struct dma_info *di)
 419{
 420        u32 addrl;
 421
 422        /* Check to see if the descriptors need to be aligned on 4K/8K or not */
 423        if (di->d64txregbase != 0) {
 424                bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 0xff0);
 425                addrl = bcma_read32(di->core, DMA64TXREGOFFS(di, addrlow));
 426                if (addrl != 0)
 427                        return false;
 428        } else if (di->d64rxregbase != 0) {
 429                bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 0xff0);
 430                addrl = bcma_read32(di->core, DMA64RXREGOFFS(di, addrlow));
 431                if (addrl != 0)
 432                        return false;
 433        }
 434        return true;
 435}
 436
 437/*
 438 * Descriptor table must start at the DMA hardware dictated alignment, so
 439 * allocated memory must be large enough to support this requirement.
 440 */
 441static void *dma_alloc_consistent(struct dma_info *di, uint size,
 442                                  u16 align_bits, uint *alloced,
 443                                  dma_addr_t *pap)
 444{
 445        if (align_bits) {
 446                u16 align = (1 << align_bits);
 447                if (!IS_ALIGNED(PAGE_SIZE, align))
 448                        size += align;
 449                *alloced = size;
 450        }
 451        return dma_alloc_coherent(di->dmadev, size, pap, GFP_ATOMIC);
 452}
 453
 454static
 455u8 dma_align_sizetobits(uint size)
 456{
 457        u8 bitpos = 0;
 458        while (size >>= 1)
 459                bitpos++;
 460        return bitpos;
 461}
 462
 463/* This function ensures that the DMA descriptor ring will not get allocated
 464 * across Page boundary. If the allocation is done across the page boundary
 465 * at the first time, then it is freed and the allocation is done at
 466 * descriptor ring size aligned location. This will ensure that the ring will
 467 * not cross page boundary
 468 */
 469static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size,
 470                           u16 *alignbits, uint *alloced,
 471                           dma_addr_t *descpa)
 472{
 473        void *va;
 474        u32 desc_strtaddr;
 475        u32 alignbytes = 1 << *alignbits;
 476
 477        va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa);
 478
 479        if (NULL == va)
 480                return NULL;
 481
 482        desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes);
 483        if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
 484                                                        & boundary)) {
 485                *alignbits = dma_align_sizetobits(size);
 486                dma_free_coherent(di->dmadev, size, va, *descpa);
 487                va = dma_alloc_consistent(di, size, *alignbits,
 488                        alloced, descpa);
 489        }
 490        return va;
 491}
 492
 493static bool dma64_alloc(struct dma_info *di, uint direction)
 494{
 495        u16 size;
 496        uint ddlen;
 497        void *va;
 498        uint alloced = 0;
 499        u16 align;
 500        u16 align_bits;
 501
 502        ddlen = sizeof(struct dma64desc);
 503
 504        size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
 505        align_bits = di->dmadesc_align;
 506        align = (1 << align_bits);
 507
 508        if (direction == DMA_TX) {
 509                va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
 510                        &alloced, &di->txdpaorig);
 511                if (va == NULL) {
 512                        brcms_dbg_dma(di->core,
 513                                      "%s: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
 514                                      di->name);
 515                        return false;
 516                }
 517                align = (1 << align_bits);
 518                di->txd64 = (struct dma64desc *)
 519                                        roundup((unsigned long)va, align);
 520                di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va);
 521                di->txdpa = di->txdpaorig + di->txdalign;
 522                di->txdalloc = alloced;
 523        } else {
 524                va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
 525                        &alloced, &di->rxdpaorig);
 526                if (va == NULL) {
 527                        brcms_dbg_dma(di->core,
 528                                      "%s: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
 529                                      di->name);
 530                        return false;
 531                }
 532                align = (1 << align_bits);
 533                di->rxd64 = (struct dma64desc *)
 534                                        roundup((unsigned long)va, align);
 535                di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va);
 536                di->rxdpa = di->rxdpaorig + di->rxdalign;
 537                di->rxdalloc = alloced;
 538        }
 539
 540        return true;
 541}
 542
 543static bool _dma_alloc(struct dma_info *di, uint direction)
 544{
 545        return dma64_alloc(di, direction);
 546}
 547
 548struct dma_pub *dma_attach(char *name, struct brcms_c_info *wlc,
 549                           uint txregbase, uint rxregbase, uint ntxd, uint nrxd,
 550                           uint rxbufsize, int rxextheadroom,
 551                           uint nrxpost, uint rxoffset)
 552{
 553        struct si_pub *sih = wlc->hw->sih;
 554        struct bcma_device *core = wlc->hw->d11core;
 555        struct dma_info *di;
 556        u8 rev = core->id.rev;
 557        uint size;
 558        struct si_info *sii = container_of(sih, struct si_info, pub);
 559
 560        /* allocate private info structure */
 561        di = kzalloc(sizeof(struct dma_info), GFP_ATOMIC);
 562        if (di == NULL)
 563                return NULL;
 564
 565        di->dma64 =
 566                ((bcma_aread32(core, BCMA_IOST) & SISF_DMA64) == SISF_DMA64);
 567
 568        /* init dma reg info */
 569        di->core = core;
 570        di->d64txregbase = txregbase;
 571        di->d64rxregbase = rxregbase;
 572
 573        /*
 574         * Default flags (which can be changed by the driver calling
 575         * dma_ctrlflags before enable): For backwards compatibility
 576         * both Rx Overflow Continue and Parity are DISABLED.
 577         */
 578        _dma_ctrlflags(di, DMA_CTRL_ROC | DMA_CTRL_PEN, 0);
 579
 580        brcms_dbg_dma(di->core, "%s: %s flags 0x%x ntxd %d nrxd %d "
 581                      "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d "
 582                      "txregbase %u rxregbase %u\n", name, "DMA64",
 583                      di->dma.dmactrlflags, ntxd, nrxd, rxbufsize,
 584                      rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase);
 585
 586        /* make a private copy of our callers name */
 587        strncpy(di->name, name, MAXNAMEL);
 588        di->name[MAXNAMEL - 1] = '\0';
 589
 590        di->dmadev = core->dma_dev;
 591
 592        /* save tunables */
 593        di->ntxd = (u16) ntxd;
 594        di->nrxd = (u16) nrxd;
 595
 596        /* the actual dma size doesn't include the extra headroom */
 597        di->rxextrahdrroom =
 598            (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
 599        if (rxbufsize > BCMEXTRAHDROOM)
 600                di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom);
 601        else
 602                di->rxbufsize = (u16) rxbufsize;
 603
 604        di->nrxpost = (u16) nrxpost;
 605        di->rxoffset = (u8) rxoffset;
 606
 607        /*
 608         * figure out the DMA physical address offset for dd and data
 609         *     PCI/PCIE: they map silicon backplace address to zero
 610         *     based memory, need offset
 611         *     Other bus: use zero SI_BUS BIGENDIAN kludge: use sdram
 612         *     swapped region for data buffer, not descriptor
 613         */
 614        di->ddoffsetlow = 0;
 615        di->dataoffsetlow = 0;
 616        /* for pci bus, add offset */
 617        if (sii->icbus->hosttype == BCMA_HOSTTYPE_PCI) {
 618                /* add offset for pcie with DMA64 bus */
 619                di->ddoffsetlow = 0;
 620                di->ddoffsethigh = SI_PCIE_DMA_H32;
 621        }
 622        di->dataoffsetlow = di->ddoffsetlow;
 623        di->dataoffsethigh = di->ddoffsethigh;
 624
 625        /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
 626        if ((core->id.id == BCMA_CORE_SDIO_DEV)
 627            && ((rev > 0) && (rev <= 2)))
 628                di->addrext = false;
 629        else if ((core->id.id == BCMA_CORE_I2S) &&
 630                 ((rev == 0) || (rev == 1)))
 631                di->addrext = false;
 632        else
 633                di->addrext = _dma_isaddrext(di);
 634
 635        /* does the descriptor need to be aligned and if yes, on 4K/8K or not */
 636        di->aligndesc_4k = _dma_descriptor_align(di);
 637        if (di->aligndesc_4k) {
 638                di->dmadesc_align = D64RINGALIGN_BITS;
 639                if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2))
 640                        /* for smaller dd table, HW relax alignment reqmnt */
 641                        di->dmadesc_align = D64RINGALIGN_BITS - 1;
 642        } else {
 643                di->dmadesc_align = 4;  /* 16 byte alignment */
 644        }
 645
 646        brcms_dbg_dma(di->core, "DMA descriptor align_needed %d, align %d\n",
 647                      di->aligndesc_4k, di->dmadesc_align);
 648
 649        /* allocate tx packet pointer vector */
 650        if (ntxd) {
 651                size = ntxd * sizeof(void *);
 652                di->txp = kzalloc(size, GFP_ATOMIC);
 653                if (di->txp == NULL)
 654                        goto fail;
 655        }
 656
 657        /* allocate rx packet pointer vector */
 658        if (nrxd) {
 659                size = nrxd * sizeof(void *);
 660                di->rxp = kzalloc(size, GFP_ATOMIC);
 661                if (di->rxp == NULL)
 662                        goto fail;
 663        }
 664
 665        /*
 666         * allocate transmit descriptor ring, only need ntxd descriptors
 667         * but it must be aligned
 668         */
 669        if (ntxd) {
 670                if (!_dma_alloc(di, DMA_TX))
 671                        goto fail;
 672        }
 673
 674        /*
 675         * allocate receive descriptor ring, only need nrxd descriptors
 676         * but it must be aligned
 677         */
 678        if (nrxd) {
 679                if (!_dma_alloc(di, DMA_RX))
 680                        goto fail;
 681        }
 682
 683        if ((di->ddoffsetlow != 0) && !di->addrext) {
 684                if (di->txdpa > SI_PCI_DMA_SZ) {
 685                        brcms_dbg_dma(di->core,
 686                                      "%s: txdpa 0x%x: addrext not supported\n",
 687                                      di->name, (u32)di->txdpa);
 688                        goto fail;
 689                }
 690                if (di->rxdpa > SI_PCI_DMA_SZ) {
 691                        brcms_dbg_dma(di->core,
 692                                      "%s: rxdpa 0x%x: addrext not supported\n",
 693                                      di->name, (u32)di->rxdpa);
 694                        goto fail;
 695                }
 696        }
 697
 698        /* Initialize AMPDU session */
 699        brcms_c_ampdu_reset_session(&di->ampdu_session, wlc);
 700
 701        brcms_dbg_dma(di->core,
 702                      "ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh 0x%x addrext %d\n",
 703                      di->ddoffsetlow, di->ddoffsethigh,
 704                      di->dataoffsetlow, di->dataoffsethigh,
 705                      di->addrext);
 706
 707        return (struct dma_pub *) di;
 708
 709 fail:
 710        dma_detach((struct dma_pub *)di);
 711        return NULL;
 712}
 713
 714static inline void
 715dma64_dd_upd(struct dma_info *di, struct dma64desc *ddring,
 716             dma_addr_t pa, uint outidx, u32 *flags, u32 bufcount)
 717{
 718        u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
 719
 720        /* PCI bus with big(>1G) physical address, use address extension */
 721        if ((di->dataoffsetlow == 0) || !(pa & PCI32ADDR_HIGH)) {
 722                ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
 723                ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
 724                ddring[outidx].ctrl1 = cpu_to_le32(*flags);
 725                ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
 726        } else {
 727                /* address extension for 32-bit PCI */
 728                u32 ae;
 729
 730                ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
 731                pa &= ~PCI32ADDR_HIGH;
 732
 733                ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
 734                ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
 735                ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
 736                ddring[outidx].ctrl1 = cpu_to_le32(*flags);
 737                ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
 738        }
 739        if (di->dma.dmactrlflags & DMA_CTRL_PEN) {
 740                if (dma64_dd_parity(&ddring[outidx]))
 741                        ddring[outidx].ctrl2 =
 742                             cpu_to_le32(ctrl2 | D64_CTRL2_PARITY);
 743        }
 744}
 745
 746/* !! may be called with core in reset */
 747void dma_detach(struct dma_pub *pub)
 748{
 749        struct dma_info *di = container_of(pub, struct dma_info, dma);
 750
 751        brcms_dbg_dma(di->core, "%s:\n", di->name);
 752
 753        /* free dma descriptor rings */
 754        if (di->txd64)
 755                dma_free_coherent(di->dmadev, di->txdalloc,
 756                                  ((s8 *)di->txd64 - di->txdalign),
 757                                  (di->txdpaorig));
 758        if (di->rxd64)
 759                dma_free_coherent(di->dmadev, di->rxdalloc,
 760                                  ((s8 *)di->rxd64 - di->rxdalign),
 761                                  (di->rxdpaorig));
 762
 763        /* free packet pointer vectors */
 764        kfree(di->txp);
 765        kfree(di->rxp);
 766
 767        /* free our private info structure */
 768        kfree(di);
 769
 770}
 771
 772/* initialize descriptor table base address */
 773static void
 774_dma_ddtable_init(struct dma_info *di, uint direction, dma_addr_t pa)
 775{
 776        if (!di->aligndesc_4k) {
 777                if (direction == DMA_TX)
 778                        di->xmtptrbase = pa;
 779                else
 780                        di->rcvptrbase = pa;
 781        }
 782
 783        if ((di->ddoffsetlow == 0)
 784            || !(pa & PCI32ADDR_HIGH)) {
 785                if (direction == DMA_TX) {
 786                        bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
 787                                     pa + di->ddoffsetlow);
 788                        bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
 789                                     di->ddoffsethigh);
 790                } else {
 791                        bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
 792                                     pa + di->ddoffsetlow);
 793                        bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
 794                                     di->ddoffsethigh);
 795                }
 796        } else {
 797                /* DMA64 32bits address extension */
 798                u32 ae;
 799
 800                /* shift the high bit(s) from pa to ae */
 801                ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
 802                pa &= ~PCI32ADDR_HIGH;
 803
 804                if (direction == DMA_TX) {
 805                        bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
 806                                     pa + di->ddoffsetlow);
 807                        bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
 808                                     di->ddoffsethigh);
 809                        bcma_maskset32(di->core, DMA64TXREGOFFS(di, control),
 810                                       D64_XC_AE, (ae << D64_XC_AE_SHIFT));
 811                } else {
 812                        bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
 813                                     pa + di->ddoffsetlow);
 814                        bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
 815                                     di->ddoffsethigh);
 816                        bcma_maskset32(di->core, DMA64RXREGOFFS(di, control),
 817                                       D64_RC_AE, (ae << D64_RC_AE_SHIFT));
 818                }
 819        }
 820}
 821
 822static void _dma_rxenable(struct dma_info *di)
 823{
 824        uint dmactrlflags = di->dma.dmactrlflags;
 825        u32 control;
 826
 827        brcms_dbg_dma(di->core, "%s:\n", di->name);
 828
 829        control = D64_RC_RE | (bcma_read32(di->core,
 830                                           DMA64RXREGOFFS(di, control)) &
 831                               D64_RC_AE);
 832
 833        if ((dmactrlflags & DMA_CTRL_PEN) == 0)
 834                control |= D64_RC_PD;
 835
 836        if (dmactrlflags & DMA_CTRL_ROC)
 837                control |= D64_RC_OC;
 838
 839        bcma_write32(di->core, DMA64RXREGOFFS(di, control),
 840                ((di->rxoffset << D64_RC_RO_SHIFT) | control));
 841}
 842
 843void dma_rxinit(struct dma_pub *pub)
 844{
 845        struct dma_info *di = container_of(pub, struct dma_info, dma);
 846
 847        brcms_dbg_dma(di->core, "%s:\n", di->name);
 848
 849        if (di->nrxd == 0)
 850                return;
 851
 852        di->rxin = di->rxout = 0;
 853
 854        /* clear rx descriptor ring */
 855        memset(di->rxd64, '\0', di->nrxd * sizeof(struct dma64desc));
 856
 857        /* DMA engine with out alignment requirement requires table to be inited
 858         * before enabling the engine
 859         */
 860        if (!di->aligndesc_4k)
 861                _dma_ddtable_init(di, DMA_RX, di->rxdpa);
 862
 863        _dma_rxenable(di);
 864
 865        if (di->aligndesc_4k)
 866                _dma_ddtable_init(di, DMA_RX, di->rxdpa);
 867}
 868
 869static struct sk_buff *dma64_getnextrxp(struct dma_info *di, bool forceall)
 870{
 871        uint i, curr;
 872        struct sk_buff *rxp;
 873        dma_addr_t pa;
 874
 875        i = di->rxin;
 876
 877        /* return if no packets posted */
 878        if (i == di->rxout)
 879                return NULL;
 880
 881        curr =
 882            B2I(((bcma_read32(di->core,
 883                              DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) -
 884                 di->rcvptrbase) & D64_RS0_CD_MASK, struct dma64desc);
 885
 886        /* ignore curr if forceall */
 887        if (!forceall && (i == curr))
 888                return NULL;
 889
 890        /* get the packet pointer that corresponds to the rx descriptor */
 891        rxp = di->rxp[i];
 892        di->rxp[i] = NULL;
 893
 894        pa = le32_to_cpu(di->rxd64[i].addrlow) - di->dataoffsetlow;
 895
 896        /* clear this packet from the descriptor ring */
 897        dma_unmap_single(di->dmadev, pa, di->rxbufsize, DMA_FROM_DEVICE);
 898
 899        di->rxd64[i].addrlow = cpu_to_le32(0xdeadbeef);
 900        di->rxd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
 901
 902        di->rxin = nextrxd(di, i);
 903
 904        return rxp;
 905}
 906
 907static struct sk_buff *_dma_getnextrxp(struct dma_info *di, bool forceall)
 908{
 909        if (di->nrxd == 0)
 910                return NULL;
 911
 912        return dma64_getnextrxp(di, forceall);
 913}
 914
 915/*
 916 * !! rx entry routine
 917 * returns the number packages in the next frame, or 0 if there are no more
 918 *   if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is
 919 *   supported with pkts chain
 920 *   otherwise, it's treated as giant pkt and will be tossed.
 921 *   The DMA scattering starts with normal DMA header, followed by first
 922 *   buffer data. After it reaches the max size of buffer, the data continues
 923 *   in next DMA descriptor buffer WITHOUT DMA header
 924 */
 925int dma_rx(struct dma_pub *pub, struct sk_buff_head *skb_list)
 926{
 927        struct dma_info *di = container_of(pub, struct dma_info, dma);
 928        struct sk_buff_head dma_frames;
 929        struct sk_buff *p, *next;
 930        uint len;
 931        uint pkt_len;
 932        int resid = 0;
 933        int pktcnt = 1;
 934
 935        skb_queue_head_init(&dma_frames);
 936 next_frame:
 937        p = _dma_getnextrxp(di, false);
 938        if (p == NULL)
 939                return 0;
 940
 941        len = le16_to_cpu(*(__le16 *) (p->data));
 942        brcms_dbg_dma(di->core, "%s: dma_rx len %d\n", di->name, len);
 943        dma_spin_for_len(len, p);
 944
 945        /* set actual length */
 946        pkt_len = min((di->rxoffset + len), di->rxbufsize);
 947        __skb_trim(p, pkt_len);
 948        skb_queue_tail(&dma_frames, p);
 949        resid = len - (di->rxbufsize - di->rxoffset);
 950
 951        /* check for single or multi-buffer rx */
 952        if (resid > 0) {
 953                while ((resid > 0) && (p = _dma_getnextrxp(di, false))) {
 954                        pkt_len = min_t(uint, resid, di->rxbufsize);
 955                        __skb_trim(p, pkt_len);
 956                        skb_queue_tail(&dma_frames, p);
 957                        resid -= di->rxbufsize;
 958                        pktcnt++;
 959                }
 960
 961#ifdef DEBUG
 962                if (resid > 0) {
 963                        uint cur;
 964                        cur =
 965                            B2I(((bcma_read32(di->core,
 966                                              DMA64RXREGOFFS(di, status0)) &
 967                                  D64_RS0_CD_MASK) - di->rcvptrbase) &
 968                                D64_RS0_CD_MASK, struct dma64desc);
 969                        brcms_dbg_dma(di->core,
 970                                      "rxin %d rxout %d, hw_curr %d\n",
 971                                      di->rxin, di->rxout, cur);
 972                }
 973#endif                          /* DEBUG */
 974
 975                if ((di->dma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
 976                        brcms_dbg_dma(di->core, "%s: bad frame length (%d)\n",
 977                                      di->name, len);
 978                        skb_queue_walk_safe(&dma_frames, p, next) {
 979                                skb_unlink(p, &dma_frames);
 980                                brcmu_pkt_buf_free_skb(p);
 981                        }
 982                        di->dma.rxgiants++;
 983                        pktcnt = 1;
 984                        goto next_frame;
 985                }
 986        }
 987
 988        skb_queue_splice_tail(&dma_frames, skb_list);
 989        return pktcnt;
 990}
 991
 992static bool dma64_rxidle(struct dma_info *di)
 993{
 994        brcms_dbg_dma(di->core, "%s:\n", di->name);
 995
 996        if (di->nrxd == 0)
 997                return true;
 998
 999        return ((bcma_read32(di->core,
1000                             DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) ==
1001                (bcma_read32(di->core, DMA64RXREGOFFS(di, ptr)) &
1002                 D64_RS0_CD_MASK));
1003}
1004
1005static bool dma64_txidle(struct dma_info *di)
1006{
1007        if (di->ntxd == 0)
1008                return true;
1009
1010        return ((bcma_read32(di->core,
1011                             DMA64TXREGOFFS(di, status0)) & D64_XS0_CD_MASK) ==
1012                (bcma_read32(di->core, DMA64TXREGOFFS(di, ptr)) &
1013                 D64_XS0_CD_MASK));
1014}
1015
1016/*
1017 * post receive buffers
1018 *  Return false if refill failed completely or dma mapping failed. The ring
1019 *  is empty, which will stall the rx dma and user might want to call rxfill
1020 *  again asap. This is unlikely to happen on a memory-rich NIC, but often on
1021 *  memory-constrained dongle.
1022 */
1023bool dma_rxfill(struct dma_pub *pub)
1024{
1025        struct dma_info *di = container_of(pub, struct dma_info, dma);
1026        struct sk_buff *p;
1027        u16 rxin, rxout;
1028        u32 flags = 0;
1029        uint n;
1030        uint i;
1031        dma_addr_t pa;
1032        uint extra_offset = 0;
1033        bool ring_empty;
1034
1035        ring_empty = false;
1036
1037        /*
1038         * Determine how many receive buffers we're lacking
1039         * from the full complement, allocate, initialize,
1040         * and post them, then update the chip rx lastdscr.
1041         */
1042
1043        rxin = di->rxin;
1044        rxout = di->rxout;
1045
1046        n = di->nrxpost - nrxdactive(di, rxin, rxout);
1047
1048        brcms_dbg_dma(di->core, "%s: post %d\n", di->name, n);
1049
1050        if (di->rxbufsize > BCMEXTRAHDROOM)
1051                extra_offset = di->rxextrahdrroom;
1052
1053        for (i = 0; i < n; i++) {
1054                /*
1055                 * the di->rxbufsize doesn't include the extra headroom,
1056                 * we need to add it to the size to be allocated
1057                 */
1058                p = brcmu_pkt_buf_get_skb(di->rxbufsize + extra_offset);
1059
1060                if (p == NULL) {
1061                        brcms_dbg_dma(di->core, "%s: out of rxbufs\n",
1062                                      di->name);
1063                        if (i == 0 && dma64_rxidle(di)) {
1064                                brcms_dbg_dma(di->core, "%s: ring is empty !\n",
1065                                              di->name);
1066                                ring_empty = true;
1067                        }
1068                        di->dma.rxnobuf++;
1069                        break;
1070                }
1071                /* reserve an extra headroom, if applicable */
1072                if (extra_offset)
1073                        skb_pull(p, extra_offset);
1074
1075                /* Do a cached write instead of uncached write since DMA_MAP
1076                 * will flush the cache.
1077                 */
1078                *(u32 *) (p->data) = 0;
1079
1080                pa = dma_map_single(di->dmadev, p->data, di->rxbufsize,
1081                                    DMA_FROM_DEVICE);
1082                if (dma_mapping_error(di->dmadev, pa))
1083                        return false;
1084
1085                /* save the free packet pointer */
1086                di->rxp[rxout] = p;
1087
1088                /* reset flags for each descriptor */
1089                flags = 0;
1090                if (rxout == (di->nrxd - 1))
1091                        flags = D64_CTRL1_EOT;
1092
1093                dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,
1094                             di->rxbufsize);
1095                rxout = nextrxd(di, rxout);
1096        }
1097
1098        di->rxout = rxout;
1099
1100        /* update the chip lastdscr pointer */
1101        bcma_write32(di->core, DMA64RXREGOFFS(di, ptr),
1102              di->rcvptrbase + I2B(rxout, struct dma64desc));
1103
1104        return ring_empty;
1105}
1106
1107void dma_rxreclaim(struct dma_pub *pub)
1108{
1109        struct dma_info *di = container_of(pub, struct dma_info, dma);
1110        struct sk_buff *p;
1111
1112        brcms_dbg_dma(di->core, "%s:\n", di->name);
1113
1114        while ((p = _dma_getnextrxp(di, true)))
1115                brcmu_pkt_buf_free_skb(p);
1116}
1117
1118void dma_counterreset(struct dma_pub *pub)
1119{
1120        /* reset all software counters */
1121        pub->rxgiants = 0;
1122        pub->rxnobuf = 0;
1123        pub->txnobuf = 0;
1124}
1125
1126/* get the address of the var in order to change later */
1127unsigned long dma_getvar(struct dma_pub *pub, const char *name)
1128{
1129        struct dma_info *di = container_of(pub, struct dma_info, dma);
1130
1131        if (!strcmp(name, "&txavail"))
1132                return (unsigned long)&(di->dma.txavail);
1133        return 0;
1134}
1135
1136/* 64-bit DMA functions */
1137
1138void dma_txinit(struct dma_pub *pub)
1139{
1140        struct dma_info *di = container_of(pub, struct dma_info, dma);
1141        u32 control = D64_XC_XE;
1142
1143        brcms_dbg_dma(di->core, "%s:\n", di->name);
1144
1145        if (di->ntxd == 0)
1146                return;
1147
1148        di->txin = di->txout = 0;
1149        di->dma.txavail = di->ntxd - 1;
1150
1151        /* clear tx descriptor ring */
1152        memset(di->txd64, '\0', (di->ntxd * sizeof(struct dma64desc)));
1153
1154        /* DMA engine with out alignment requirement requires table to be inited
1155         * before enabling the engine
1156         */
1157        if (!di->aligndesc_4k)
1158                _dma_ddtable_init(di, DMA_TX, di->txdpa);
1159
1160        if ((di->dma.dmactrlflags & DMA_CTRL_PEN) == 0)
1161                control |= D64_XC_PD;
1162        bcma_set32(di->core, DMA64TXREGOFFS(di, control), control);
1163
1164        /* DMA engine with alignment requirement requires table to be inited
1165         * before enabling the engine
1166         */
1167        if (di->aligndesc_4k)
1168                _dma_ddtable_init(di, DMA_TX, di->txdpa);
1169}
1170
1171void dma_txsuspend(struct dma_pub *pub)
1172{
1173        struct dma_info *di = container_of(pub, struct dma_info, dma);
1174
1175        brcms_dbg_dma(di->core, "%s:\n", di->name);
1176
1177        if (di->ntxd == 0)
1178                return;
1179
1180        bcma_set32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
1181}
1182
1183void dma_txresume(struct dma_pub *pub)
1184{
1185        struct dma_info *di = container_of(pub, struct dma_info, dma);
1186
1187        brcms_dbg_dma(di->core, "%s:\n", di->name);
1188
1189        if (di->ntxd == 0)
1190                return;
1191
1192        bcma_mask32(di->core, DMA64TXREGOFFS(di, control), ~D64_XC_SE);
1193}
1194
1195bool dma_txsuspended(struct dma_pub *pub)
1196{
1197        struct dma_info *di = container_of(pub, struct dma_info, dma);
1198
1199        return (di->ntxd == 0) ||
1200               ((bcma_read32(di->core,
1201                             DMA64TXREGOFFS(di, control)) & D64_XC_SE) ==
1202                D64_XC_SE);
1203}
1204
1205void dma_txreclaim(struct dma_pub *pub, enum txd_range range)
1206{
1207        struct dma_info *di = container_of(pub, struct dma_info, dma);
1208        struct sk_buff *p;
1209
1210        brcms_dbg_dma(di->core, "%s: %s\n",
1211                      di->name,
1212                      range == DMA_RANGE_ALL ? "all" :
1213                      range == DMA_RANGE_TRANSMITTED ? "transmitted" :
1214                      "transferred");
1215
1216        if (di->txin == di->txout)
1217                return;
1218
1219        while ((p = dma_getnexttxp(pub, range))) {
1220                /* For unframed data, we don't have any packets to free */
1221                if (!(di->dma.dmactrlflags & DMA_CTRL_UNFRAMED))
1222                        brcmu_pkt_buf_free_skb(p);
1223        }
1224}
1225
1226bool dma_txreset(struct dma_pub *pub)
1227{
1228        struct dma_info *di = container_of(pub, struct dma_info, dma);
1229        u32 status;
1230
1231        if (di->ntxd == 0)
1232                return true;
1233
1234        /* suspend tx DMA first */
1235        bcma_write32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
1236        SPINWAIT(((status =
1237                   (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
1238                    D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED) &&
1239                  (status != D64_XS0_XS_IDLE) && (status != D64_XS0_XS_STOPPED),
1240                 10000);
1241
1242        bcma_write32(di->core, DMA64TXREGOFFS(di, control), 0);
1243        SPINWAIT(((status =
1244                   (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
1245                    D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED), 10000);
1246
1247        /* wait for the last transaction to complete */
1248        udelay(300);
1249
1250        return status == D64_XS0_XS_DISABLED;
1251}
1252
1253bool dma_rxreset(struct dma_pub *pub)
1254{
1255        struct dma_info *di = container_of(pub, struct dma_info, dma);
1256        u32 status;
1257
1258        if (di->nrxd == 0)
1259                return true;
1260
1261        bcma_write32(di->core, DMA64RXREGOFFS(di, control), 0);
1262        SPINWAIT(((status =
1263                   (bcma_read32(di->core, DMA64RXREGOFFS(di, status0)) &
1264                    D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED), 10000);
1265
1266        return status == D64_RS0_RS_DISABLED;
1267}
1268
1269static void dma_txenq(struct dma_info *di, struct sk_buff *p)
1270{
1271        unsigned char *data;
1272        uint len;
1273        u16 txout;
1274        u32 flags = 0;
1275        dma_addr_t pa;
1276
1277        txout = di->txout;
1278
1279        if (WARN_ON(nexttxd(di, txout) == di->txin))
1280                return;
1281
1282        /*
1283         * obtain and initialize transmit descriptor entry.
1284         */
1285        data = p->data;
1286        len = p->len;
1287
1288        /* get physical address of buffer start */
1289        pa = dma_map_single(di->dmadev, data, len, DMA_TO_DEVICE);
1290        /* if mapping failed, free skb */
1291        if (dma_mapping_error(di->dmadev, pa)) {
1292                brcmu_pkt_buf_free_skb(p);
1293                return;
1294        }
1295        /* With a DMA segment list, Descriptor table is filled
1296         * using the segment list instead of looping over
1297         * buffers in multi-chain DMA. Therefore, EOF for SGLIST
1298         * is when end of segment list is reached.
1299         */
1300        flags = D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF;
1301        if (txout == (di->ntxd - 1))
1302                flags |= D64_CTRL1_EOT;
1303
1304        dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
1305
1306        txout = nexttxd(di, txout);
1307
1308        /* save the packet */
1309        di->txp[prevtxd(di, txout)] = p;
1310
1311        /* bump the tx descriptor index */
1312        di->txout = txout;
1313}
1314
1315static void ampdu_finalize(struct dma_info *di)
1316{
1317        struct brcms_ampdu_session *session = &di->ampdu_session;
1318        struct sk_buff *p;
1319
1320        trace_brcms_ampdu_session(&session->wlc->hw->d11core->dev,
1321                                  session->max_ampdu_len,
1322                                  session->max_ampdu_frames,
1323                                  session->ampdu_len,
1324                                  skb_queue_len(&session->skb_list),
1325                                  session->dma_len);
1326
1327        if (WARN_ON(skb_queue_empty(&session->skb_list)))
1328                return;
1329
1330        brcms_c_ampdu_finalize(session);
1331
1332        while (!skb_queue_empty(&session->skb_list)) {
1333                p = skb_dequeue(&session->skb_list);
1334                dma_txenq(di, p);
1335        }
1336
1337        bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
1338                     di->xmtptrbase + I2B(di->txout, struct dma64desc));
1339        brcms_c_ampdu_reset_session(session, session->wlc);
1340}
1341
1342static void prep_ampdu_frame(struct dma_info *di, struct sk_buff *p)
1343{
1344        struct brcms_ampdu_session *session = &di->ampdu_session;
1345        int ret;
1346
1347        ret = brcms_c_ampdu_add_frame(session, p);
1348        if (ret == -ENOSPC) {
1349                /*
1350                 * AMPDU cannot accomodate this frame. Close out the in-
1351                 * progress AMPDU session and start a new one.
1352                 */
1353                ampdu_finalize(di);
1354                ret = brcms_c_ampdu_add_frame(session, p);
1355        }
1356
1357        WARN_ON(ret);
1358}
1359
1360/* Update count of available tx descriptors based on current DMA state */
1361static void dma_update_txavail(struct dma_info *di)
1362{
1363        /*
1364         * Available space is number of descriptors less the number of
1365         * active descriptors and the number of queued AMPDU frames.
1366         */
1367        di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) -
1368                          skb_queue_len(&di->ampdu_session.skb_list) - 1;
1369}
1370
1371/*
1372 * !! tx entry routine
1373 * WARNING: call must check the return value for error.
1374 *   the error(toss frames) could be fatal and cause many subsequent hard
1375 *   to debug problems
1376 */
1377int dma_txfast(struct brcms_c_info *wlc, struct dma_pub *pub,
1378               struct sk_buff *p)
1379{
1380        struct dma_info *di = container_of(pub, struct dma_info, dma);
1381        struct brcms_ampdu_session *session = &di->ampdu_session;
1382        struct ieee80211_tx_info *tx_info;
1383        bool is_ampdu;
1384
1385        /* no use to transmit a zero length packet */
1386        if (p->len == 0)
1387                return 0;
1388
1389        /* return nonzero if out of tx descriptors */
1390        if (di->dma.txavail == 0 || nexttxd(di, di->txout) == di->txin)
1391                goto outoftxd;
1392
1393        tx_info = IEEE80211_SKB_CB(p);
1394        is_ampdu = tx_info->flags & IEEE80211_TX_CTL_AMPDU;
1395        if (is_ampdu)
1396                prep_ampdu_frame(di, p);
1397        else
1398                dma_txenq(di, p);
1399
1400        /* tx flow control */
1401        dma_update_txavail(di);
1402
1403        /* kick the chip */
1404        if (is_ampdu) {
1405                /*
1406                 * Start sending data if we've got a full AMPDU, there's
1407                 * no more space in the DMA ring, or the ring isn't
1408                 * currently transmitting.
1409                 */
1410                if (skb_queue_len(&session->skb_list) == session->max_ampdu_frames ||
1411                    di->dma.txavail == 0 || dma64_txidle(di))
1412                        ampdu_finalize(di);
1413        } else {
1414                bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
1415                             di->xmtptrbase + I2B(di->txout, struct dma64desc));
1416        }
1417
1418        return 0;
1419
1420 outoftxd:
1421        brcms_dbg_dma(di->core, "%s: out of txds !!!\n", di->name);
1422        brcmu_pkt_buf_free_skb(p);
1423        di->dma.txavail = 0;
1424        di->dma.txnobuf++;
1425        return -ENOSPC;
1426}
1427
1428void dma_txflush(struct dma_pub *pub)
1429{
1430        struct dma_info *di = container_of(pub, struct dma_info, dma);
1431        struct brcms_ampdu_session *session = &di->ampdu_session;
1432
1433        if (!skb_queue_empty(&session->skb_list))
1434                ampdu_finalize(di);
1435}
1436
1437int dma_txpending(struct dma_pub *pub)
1438{
1439        struct dma_info *di = container_of(pub, struct dma_info, dma);
1440        return ntxdactive(di, di->txin, di->txout);
1441}
1442
1443/*
1444 * If we have an active AMPDU session and are not transmitting,
1445 * this function will force tx to start.
1446 */
1447void dma_kick_tx(struct dma_pub *pub)
1448{
1449        struct dma_info *di = container_of(pub, struct dma_info, dma);
1450        struct brcms_ampdu_session *session = &di->ampdu_session;
1451
1452        if (!skb_queue_empty(&session->skb_list) && dma64_txidle(di))
1453                ampdu_finalize(di);
1454}
1455
1456/*
1457 * Reclaim next completed txd (txds if using chained buffers) in the range
1458 * specified and return associated packet.
1459 * If range is DMA_RANGE_TRANSMITTED, reclaim descriptors that have be
1460 * transmitted as noted by the hardware "CurrDescr" pointer.
1461 * If range is DMA_RANGE_TRANSFERED, reclaim descriptors that have be
1462 * transferred by the DMA as noted by the hardware "ActiveDescr" pointer.
1463 * If range is DMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
1464 * return associated packet regardless of the value of hardware pointers.
1465 */
1466struct sk_buff *dma_getnexttxp(struct dma_pub *pub, enum txd_range range)
1467{
1468        struct dma_info *di = container_of(pub, struct dma_info, dma);
1469        u16 start, end, i;
1470        u16 active_desc;
1471        struct sk_buff *txp;
1472
1473        brcms_dbg_dma(di->core, "%s: %s\n",
1474                      di->name,
1475                      range == DMA_RANGE_ALL ? "all" :
1476                      range == DMA_RANGE_TRANSMITTED ? "transmitted" :
1477                      "transferred");
1478
1479        if (di->ntxd == 0)
1480                return NULL;
1481
1482        txp = NULL;
1483
1484        start = di->txin;
1485        if (range == DMA_RANGE_ALL)
1486                end = di->txout;
1487        else {
1488                end = (u16) (B2I(((bcma_read32(di->core,
1489                                               DMA64TXREGOFFS(di, status0)) &
1490                                   D64_XS0_CD_MASK) - di->xmtptrbase) &
1491                                 D64_XS0_CD_MASK, struct dma64desc));
1492
1493                if (range == DMA_RANGE_TRANSFERED) {
1494                        active_desc =
1495                                (u16)(bcma_read32(di->core,
1496                                                  DMA64TXREGOFFS(di, status1)) &
1497                                      D64_XS1_AD_MASK);
1498                        active_desc =
1499                            (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
1500                        active_desc = B2I(active_desc, struct dma64desc);
1501                        if (end != active_desc)
1502                                end = prevtxd(di, active_desc);
1503                }
1504        }
1505
1506        if ((start == 0) && (end > di->txout))
1507                goto bogus;
1508
1509        for (i = start; i != end && !txp; i = nexttxd(di, i)) {
1510                dma_addr_t pa;
1511                uint size;
1512
1513                pa = le32_to_cpu(di->txd64[i].addrlow) - di->dataoffsetlow;
1514
1515                size =
1516                    (le32_to_cpu(di->txd64[i].ctrl2) &
1517                     D64_CTRL2_BC_MASK);
1518
1519                di->txd64[i].addrlow = cpu_to_le32(0xdeadbeef);
1520                di->txd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
1521
1522                txp = di->txp[i];
1523                di->txp[i] = NULL;
1524
1525                dma_unmap_single(di->dmadev, pa, size, DMA_TO_DEVICE);
1526        }
1527
1528        di->txin = i;
1529
1530        /* tx flow control */
1531        dma_update_txavail(di);
1532
1533        return txp;
1534
1535 bogus:
1536        brcms_dbg_dma(di->core, "bogus curr: start %d end %d txout %d\n",
1537                      start, end, di->txout);
1538        return NULL;
1539}
1540
1541/*
1542 * Mac80211 initiated actions sometimes require packets in the DMA queue to be
1543 * modified. The modified portion of the packet is not under control of the DMA
1544 * engine. This function calls a caller-supplied function for each packet in
1545 * the caller specified dma chain.
1546 */
1547void dma_walk_packets(struct dma_pub *dmah, void (*callback_fnc)
1548                      (void *pkt, void *arg_a), void *arg_a)
1549{
1550        struct dma_info *di = container_of(dmah, struct dma_info, dma);
1551        uint i =   di->txin;
1552        uint end = di->txout;
1553        struct sk_buff *skb;
1554        struct ieee80211_tx_info *tx_info;
1555
1556        while (i != end) {
1557                skb = di->txp[i];
1558                if (skb != NULL) {
1559                        tx_info = (struct ieee80211_tx_info *)skb->cb;
1560                        (callback_fnc)(tx_info, arg_a);
1561                }
1562                i = nexttxd(di, i);
1563        }
1564}
1565