linux/drivers/usb/gadget/udc/gr_udc.c
<<
>>
Prefs
   1/*
   2 * USB Peripheral Controller driver for Aeroflex Gaisler GRUSBDC.
   3 *
   4 * 2013 (c) Aeroflex Gaisler AB
   5 *
   6 * This driver supports GRUSBDC USB Device Controller cores available in the
   7 * GRLIB VHDL IP core library.
   8 *
   9 * Full documentation of the GRUSBDC core can be found here:
  10 * http://www.gaisler.com/products/grlib/grip.pdf
  11 *
  12 * This program is free software; you can redistribute it and/or modify it
  13 * under the terms of the GNU General Public License as published by the
  14 * Free Software Foundation; either version 2 of the License, or (at your
  15 * option) any later version.
  16 *
  17 * Contributors:
  18 * - Andreas Larsson <andreas@gaisler.com>
  19 * - Marko Isomaki
  20 */
  21
  22/*
  23 * A GRUSBDC core can have up to 16 IN endpoints and 16 OUT endpoints each
  24 * individually configurable to any of the four USB transfer types. This driver
  25 * only supports cores in DMA mode.
  26 */
  27
  28#include <linux/kernel.h>
  29#include <linux/module.h>
  30#include <linux/slab.h>
  31#include <linux/spinlock.h>
  32#include <linux/errno.h>
  33#include <linux/list.h>
  34#include <linux/interrupt.h>
  35#include <linux/device.h>
  36#include <linux/usb/ch9.h>
  37#include <linux/usb/gadget.h>
  38#include <linux/dma-mapping.h>
  39#include <linux/dmapool.h>
  40#include <linux/debugfs.h>
  41#include <linux/seq_file.h>
  42#include <linux/of_platform.h>
  43#include <linux/of_irq.h>
  44#include <linux/of_address.h>
  45
  46#include <asm/byteorder.h>
  47
  48#include "gr_udc.h"
  49
  50#define DRIVER_NAME     "gr_udc"
  51#define DRIVER_DESC     "Aeroflex Gaisler GRUSBDC USB Peripheral Controller"
  52
  53static const char driver_name[] = DRIVER_NAME;
  54static const char driver_desc[] = DRIVER_DESC;
  55
  56#define gr_read32(x) (ioread32be((x)))
  57#define gr_write32(x, v) (iowrite32be((v), (x)))
  58
  59/* USB speed and corresponding string calculated from status register value */
  60#define GR_SPEED(status) \
  61        ((status & GR_STATUS_SP) ? USB_SPEED_FULL : USB_SPEED_HIGH)
  62#define GR_SPEED_STR(status) usb_speed_string(GR_SPEED(status))
  63
  64/* Size of hardware buffer calculated from epctrl register value */
  65#define GR_BUFFER_SIZE(epctrl)                                        \
  66        ((((epctrl) & GR_EPCTRL_BUFSZ_MASK) >> GR_EPCTRL_BUFSZ_POS) * \
  67         GR_EPCTRL_BUFSZ_SCALER)
  68
  69/* ---------------------------------------------------------------------- */
  70/* Debug printout functionality */
  71
  72static const char * const gr_modestring[] = {"control", "iso", "bulk", "int"};
  73
  74static const char *gr_ep0state_string(enum gr_ep0state state)
  75{
  76        static const char *const names[] = {
  77                [GR_EP0_DISCONNECT] = "disconnect",
  78                [GR_EP0_SETUP] = "setup",
  79                [GR_EP0_IDATA] = "idata",
  80                [GR_EP0_ODATA] = "odata",
  81                [GR_EP0_ISTATUS] = "istatus",
  82                [GR_EP0_OSTATUS] = "ostatus",
  83                [GR_EP0_STALL] = "stall",
  84                [GR_EP0_SUSPEND] = "suspend",
  85        };
  86
  87        if (state < 0 || state >= ARRAY_SIZE(names))
  88                return "UNKNOWN";
  89
  90        return names[state];
  91}
  92
  93#ifdef VERBOSE_DEBUG
  94
  95static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
  96                                struct gr_request *req)
  97{
  98        int buflen = ep->is_in ? req->req.length : req->req.actual;
  99        int rowlen = 32;
 100        int plen = min(rowlen, buflen);
 101
 102        dev_dbg(ep->dev->dev, "%s: 0x%p, %d bytes data%s:\n", str, req, buflen,
 103                (buflen > plen ? " (truncated)" : ""));
 104        print_hex_dump_debug("   ", DUMP_PREFIX_NONE,
 105                             rowlen, 4, req->req.buf, plen, false);
 106}
 107
 108static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
 109                               u16 value, u16 index, u16 length)
 110{
 111        dev_vdbg(dev->dev, "REQ: %02x.%02x v%04x i%04x l%04x\n",
 112                 type, request, value, index, length);
 113}
 114#else /* !VERBOSE_DEBUG */
 115
 116static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
 117                                struct gr_request *req) {}
 118
 119static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
 120                               u16 value, u16 index, u16 length) {}
 121
 122#endif /* VERBOSE_DEBUG */
 123
 124/* ---------------------------------------------------------------------- */
 125/* Debugfs functionality */
 126
 127#ifdef CONFIG_USB_GADGET_DEBUG_FS
 128
 129static void gr_seq_ep_show(struct seq_file *seq, struct gr_ep *ep)
 130{
 131        u32 epctrl = gr_read32(&ep->regs->epctrl);
 132        u32 epstat = gr_read32(&ep->regs->epstat);
 133        int mode = (epctrl & GR_EPCTRL_TT_MASK) >> GR_EPCTRL_TT_POS;
 134        struct gr_request *req;
 135
 136        seq_printf(seq, "%s:\n", ep->ep.name);
 137        seq_printf(seq, "  mode = %s\n", gr_modestring[mode]);
 138        seq_printf(seq, "  halted: %d\n", !!(epctrl & GR_EPCTRL_EH));
 139        seq_printf(seq, "  disabled: %d\n", !!(epctrl & GR_EPCTRL_ED));
 140        seq_printf(seq, "  valid: %d\n", !!(epctrl & GR_EPCTRL_EV));
 141        seq_printf(seq, "  dma_start = %d\n", ep->dma_start);
 142        seq_printf(seq, "  stopped = %d\n", ep->stopped);
 143        seq_printf(seq, "  wedged = %d\n", ep->wedged);
 144        seq_printf(seq, "  callback = %d\n", ep->callback);
 145        seq_printf(seq, "  maxpacket = %d\n", ep->ep.maxpacket);
 146        seq_printf(seq, "  maxpacket_limit = %d\n", ep->ep.maxpacket_limit);
 147        seq_printf(seq, "  bytes_per_buffer = %d\n", ep->bytes_per_buffer);
 148        if (mode == 1 || mode == 3)
 149                seq_printf(seq, "  nt = %d\n",
 150                           (epctrl & GR_EPCTRL_NT_MASK) >> GR_EPCTRL_NT_POS);
 151
 152        seq_printf(seq, "  Buffer 0: %s %s%d\n",
 153                   epstat & GR_EPSTAT_B0 ? "valid" : "invalid",
 154                   epstat & GR_EPSTAT_BS ? " " : "selected ",
 155                   (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS);
 156        seq_printf(seq, "  Buffer 1: %s %s%d\n",
 157                   epstat & GR_EPSTAT_B1 ? "valid" : "invalid",
 158                   epstat & GR_EPSTAT_BS ? "selected " : " ",
 159                   (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS);
 160
 161        if (list_empty(&ep->queue)) {
 162                seq_puts(seq, "  Queue: empty\n\n");
 163                return;
 164        }
 165
 166        seq_puts(seq, "  Queue:\n");
 167        list_for_each_entry(req, &ep->queue, queue) {
 168                struct gr_dma_desc *desc;
 169                struct gr_dma_desc *next;
 170
 171                seq_printf(seq, "    0x%p: 0x%p %d %d\n", req,
 172                           &req->req.buf, req->req.actual, req->req.length);
 173
 174                next = req->first_desc;
 175                do {
 176                        desc = next;
 177                        next = desc->next_desc;
 178                        seq_printf(seq, "    %c 0x%p (0x%08x): 0x%05x 0x%08x\n",
 179                                   desc == req->curr_desc ? 'c' : ' ',
 180                                   desc, desc->paddr, desc->ctrl, desc->data);
 181                } while (desc != req->last_desc);
 182        }
 183        seq_puts(seq, "\n");
 184}
 185
 186
 187static int gr_seq_show(struct seq_file *seq, void *v)
 188{
 189        struct gr_udc *dev = seq->private;
 190        u32 control = gr_read32(&dev->regs->control);
 191        u32 status = gr_read32(&dev->regs->status);
 192        struct gr_ep *ep;
 193
 194        seq_printf(seq, "usb state = %s\n",
 195                   usb_state_string(dev->gadget.state));
 196        seq_printf(seq, "address = %d\n",
 197                   (control & GR_CONTROL_UA_MASK) >> GR_CONTROL_UA_POS);
 198        seq_printf(seq, "speed = %s\n", GR_SPEED_STR(status));
 199        seq_printf(seq, "ep0state = %s\n", gr_ep0state_string(dev->ep0state));
 200        seq_printf(seq, "irq_enabled = %d\n", dev->irq_enabled);
 201        seq_printf(seq, "remote_wakeup = %d\n", dev->remote_wakeup);
 202        seq_printf(seq, "test_mode = %d\n", dev->test_mode);
 203        seq_puts(seq, "\n");
 204
 205        list_for_each_entry(ep, &dev->ep_list, ep_list)
 206                gr_seq_ep_show(seq, ep);
 207
 208        return 0;
 209}
 210
 211static int gr_dfs_open(struct inode *inode, struct file *file)
 212{
 213        return single_open(file, gr_seq_show, inode->i_private);
 214}
 215
 216static const struct file_operations gr_dfs_fops = {
 217        .owner          = THIS_MODULE,
 218        .open           = gr_dfs_open,
 219        .read           = seq_read,
 220        .llseek         = seq_lseek,
 221        .release        = single_release,
 222};
 223
 224static void gr_dfs_create(struct gr_udc *dev)
 225{
 226        const char *name = "gr_udc_state";
 227
 228        dev->dfs_root = debugfs_create_dir(dev_name(dev->dev), NULL);
 229        dev->dfs_state = debugfs_create_file(name, 0444, dev->dfs_root, dev,
 230                                             &gr_dfs_fops);
 231}
 232
 233static void gr_dfs_delete(struct gr_udc *dev)
 234{
 235        /* Handles NULL and ERR pointers internally */
 236        debugfs_remove(dev->dfs_state);
 237        debugfs_remove(dev->dfs_root);
 238}
 239
 240#else /* !CONFIG_USB_GADGET_DEBUG_FS */
 241
 242static void gr_dfs_create(struct gr_udc *dev) {}
 243static void gr_dfs_delete(struct gr_udc *dev) {}
 244
 245#endif /* CONFIG_USB_GADGET_DEBUG_FS */
 246
 247/* ---------------------------------------------------------------------- */
 248/* DMA and request handling */
 249
 250/* Allocates a new struct gr_dma_desc, sets paddr and zeroes the rest */
 251static struct gr_dma_desc *gr_alloc_dma_desc(struct gr_ep *ep, gfp_t gfp_flags)
 252{
 253        dma_addr_t paddr;
 254        struct gr_dma_desc *dma_desc;
 255
 256        dma_desc = dma_pool_alloc(ep->dev->desc_pool, gfp_flags, &paddr);
 257        if (!dma_desc) {
 258                dev_err(ep->dev->dev, "Could not allocate from DMA pool\n");
 259                return NULL;
 260        }
 261
 262        memset(dma_desc, 0, sizeof(*dma_desc));
 263        dma_desc->paddr = paddr;
 264
 265        return dma_desc;
 266}
 267
 268static inline void gr_free_dma_desc(struct gr_udc *dev,
 269                                    struct gr_dma_desc *desc)
 270{
 271        dma_pool_free(dev->desc_pool, desc, (dma_addr_t)desc->paddr);
 272}
 273
 274/* Frees the chain of struct gr_dma_desc for the given request */
 275static void gr_free_dma_desc_chain(struct gr_udc *dev, struct gr_request *req)
 276{
 277        struct gr_dma_desc *desc;
 278        struct gr_dma_desc *next;
 279
 280        next = req->first_desc;
 281        if (!next)
 282                return;
 283
 284        do {
 285                desc = next;
 286                next = desc->next_desc;
 287                gr_free_dma_desc(dev, desc);
 288        } while (desc != req->last_desc);
 289
 290        req->first_desc = NULL;
 291        req->curr_desc = NULL;
 292        req->last_desc = NULL;
 293}
 294
 295static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req);
 296
 297/*
 298 * Frees allocated resources and calls the appropriate completion function/setup
 299 * package handler for a finished request.
 300 *
 301 * Must be called with dev->lock held and irqs disabled.
 302 */
 303static void gr_finish_request(struct gr_ep *ep, struct gr_request *req,
 304                              int status)
 305        __releases(&dev->lock)
 306        __acquires(&dev->lock)
 307{
 308        struct gr_udc *dev;
 309
 310        list_del_init(&req->queue);
 311
 312        if (likely(req->req.status == -EINPROGRESS))
 313                req->req.status = status;
 314        else
 315                status = req->req.status;
 316
 317        dev = ep->dev;
 318        usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);
 319        gr_free_dma_desc_chain(dev, req);
 320
 321        if (ep->is_in) { /* For OUT, req->req.actual gets updated bit by bit */
 322                req->req.actual = req->req.length;
 323        } else if (req->oddlen && req->req.actual > req->evenlen) {
 324                /*
 325                 * Copy to user buffer in this case where length was not evenly
 326                 * divisible by ep->ep.maxpacket and the last descriptor was
 327                 * actually used.
 328                 */
 329                char *buftail = ((char *)req->req.buf + req->evenlen);
 330
 331                memcpy(buftail, ep->tailbuf, req->oddlen);
 332
 333                if (req->req.actual > req->req.length) {
 334                        /* We got more data than was requested */
 335                        dev_dbg(ep->dev->dev, "Overflow for ep %s\n",
 336                                ep->ep.name);
 337                        gr_dbgprint_request("OVFL", ep, req);
 338                        req->req.status = -EOVERFLOW;
 339                }
 340        }
 341
 342        if (!status) {
 343                if (ep->is_in)
 344                        gr_dbgprint_request("SENT", ep, req);
 345                else
 346                        gr_dbgprint_request("RECV", ep, req);
 347        }
 348
 349        /* Prevent changes to ep->queue during callback */
 350        ep->callback = 1;
 351        if (req == dev->ep0reqo && !status) {
 352                if (req->setup)
 353                        gr_ep0_setup(dev, req);
 354                else
 355                        dev_err(dev->dev,
 356                                "Unexpected non setup packet on ep0in\n");
 357        } else if (req->req.complete) {
 358                spin_unlock(&dev->lock);
 359
 360                usb_gadget_giveback_request(&ep->ep, &req->req);
 361
 362                spin_lock(&dev->lock);
 363        }
 364        ep->callback = 0;
 365}
 366
 367static struct usb_request *gr_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
 368{
 369        struct gr_request *req;
 370
 371        req = kzalloc(sizeof(*req), gfp_flags);
 372        if (!req)
 373                return NULL;
 374
 375        INIT_LIST_HEAD(&req->queue);
 376
 377        return &req->req;
 378}
 379
 380/*
 381 * Starts DMA for endpoint ep if there are requests in the queue.
 382 *
 383 * Must be called with dev->lock held and with !ep->stopped.
 384 */
 385static void gr_start_dma(struct gr_ep *ep)
 386{
 387        struct gr_request *req;
 388        u32 dmactrl;
 389
 390        if (list_empty(&ep->queue)) {
 391                ep->dma_start = 0;
 392                return;
 393        }
 394
 395        req = list_first_entry(&ep->queue, struct gr_request, queue);
 396
 397        /* A descriptor should already have been allocated */
 398        BUG_ON(!req->curr_desc);
 399
 400        /*
 401         * The DMA controller can not handle smaller OUT buffers than
 402         * ep->ep.maxpacket. It could lead to buffer overruns if an unexpectedly
 403         * long packet are received. Therefore an internal bounce buffer gets
 404         * used when such a request gets enabled.
 405         */
 406        if (!ep->is_in && req->oddlen)
 407                req->last_desc->data = ep->tailbuf_paddr;
 408
 409        wmb(); /* Make sure all is settled before handing it over to DMA */
 410
 411        /* Set the descriptor pointer in the hardware */
 412        gr_write32(&ep->regs->dmaaddr, req->curr_desc->paddr);
 413
 414        /* Announce available descriptors */
 415        dmactrl = gr_read32(&ep->regs->dmactrl);
 416        gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_DA);
 417
 418        ep->dma_start = 1;
 419}
 420
 421/*
 422 * Finishes the first request in the ep's queue and, if available, starts the
 423 * next request in queue.
 424 *
 425 * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
 426 */
 427static void gr_dma_advance(struct gr_ep *ep, int status)
 428{
 429        struct gr_request *req;
 430
 431        req = list_first_entry(&ep->queue, struct gr_request, queue);
 432        gr_finish_request(ep, req, status);
 433        gr_start_dma(ep); /* Regardless of ep->dma_start */
 434}
 435
 436/*
 437 * Abort DMA for an endpoint. Sets the abort DMA bit which causes an ongoing DMA
 438 * transfer to be canceled and clears GR_DMACTRL_DA.
 439 *
 440 * Must be called with dev->lock held.
 441 */
 442static void gr_abort_dma(struct gr_ep *ep)
 443{
 444        u32 dmactrl;
 445
 446        dmactrl = gr_read32(&ep->regs->dmactrl);
 447        gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_AD);
 448}
 449
 450/*
 451 * Allocates and sets up a struct gr_dma_desc and putting it on the descriptor
 452 * chain.
 453 *
 454 * Size is not used for OUT endpoints. Hardware can not be instructed to handle
 455 * smaller buffer than MAXPL in the OUT direction.
 456 */
 457static int gr_add_dma_desc(struct gr_ep *ep, struct gr_request *req,
 458                           dma_addr_t data, unsigned size, gfp_t gfp_flags)
 459{
 460        struct gr_dma_desc *desc;
 461
 462        desc = gr_alloc_dma_desc(ep, gfp_flags);
 463        if (!desc)
 464                return -ENOMEM;
 465
 466        desc->data = data;
 467        if (ep->is_in)
 468                desc->ctrl =
 469                        (GR_DESC_IN_CTRL_LEN_MASK & size) | GR_DESC_IN_CTRL_EN;
 470        else
 471                desc->ctrl = GR_DESC_OUT_CTRL_IE;
 472
 473        if (!req->first_desc) {
 474                req->first_desc = desc;
 475                req->curr_desc = desc;
 476        } else {
 477                req->last_desc->next_desc = desc;
 478                req->last_desc->next = desc->paddr;
 479                req->last_desc->ctrl |= GR_DESC_OUT_CTRL_NX;
 480        }
 481        req->last_desc = desc;
 482
 483        return 0;
 484}
 485
 486/*
 487 * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
 488 * together covers req->req.length bytes of the buffer at DMA address
 489 * req->req.dma for the OUT direction.
 490 *
 491 * The first descriptor in the chain is enabled, the rest disabled. The
 492 * interrupt handler will later enable them one by one when needed so we can
 493 * find out when the transfer is finished. For OUT endpoints, all descriptors
 494 * therefore generate interrutps.
 495 */
 496static int gr_setup_out_desc_list(struct gr_ep *ep, struct gr_request *req,
 497                                  gfp_t gfp_flags)
 498{
 499        u16 bytes_left; /* Bytes left to provide descriptors for */
 500        u16 bytes_used; /* Bytes accommodated for */
 501        int ret = 0;
 502
 503        req->first_desc = NULL; /* Signals that no allocation is done yet */
 504        bytes_left = req->req.length;
 505        bytes_used = 0;
 506        while (bytes_left > 0) {
 507                dma_addr_t start = req->req.dma + bytes_used;
 508                u16 size = min(bytes_left, ep->bytes_per_buffer);
 509
 510                if (size < ep->bytes_per_buffer) {
 511                        /* Prepare using bounce buffer */
 512                        req->evenlen = req->req.length - bytes_left;
 513                        req->oddlen = size;
 514                }
 515
 516                ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
 517                if (ret)
 518                        goto alloc_err;
 519
 520                bytes_left -= size;
 521                bytes_used += size;
 522        }
 523
 524        req->first_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
 525
 526        return 0;
 527
 528alloc_err:
 529        gr_free_dma_desc_chain(ep->dev, req);
 530
 531        return ret;
 532}
 533
 534/*
 535 * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
 536 * together covers req->req.length bytes of the buffer at DMA address
 537 * req->req.dma for the IN direction.
 538 *
 539 * When more data is provided than the maximum payload size, the hardware splits
 540 * this up into several payloads automatically. Moreover, ep->bytes_per_buffer
 541 * is always set to a multiple of the maximum payload (restricted to the valid
 542 * number of maximum payloads during high bandwidth isochronous or interrupt
 543 * transfers)
 544 *
 545 * All descriptors are enabled from the beginning and we only generate an
 546 * interrupt for the last one indicating that the entire request has been pushed
 547 * to hardware.
 548 */
 549static int gr_setup_in_desc_list(struct gr_ep *ep, struct gr_request *req,
 550                                 gfp_t gfp_flags)
 551{
 552        u16 bytes_left; /* Bytes left in req to provide descriptors for */
 553        u16 bytes_used; /* Bytes in req accommodated for */
 554        int ret = 0;
 555
 556        req->first_desc = NULL; /* Signals that no allocation is done yet */
 557        bytes_left = req->req.length;
 558        bytes_used = 0;
 559        do { /* Allow for zero length packets */
 560                dma_addr_t start = req->req.dma + bytes_used;
 561                u16 size = min(bytes_left, ep->bytes_per_buffer);
 562
 563                ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
 564                if (ret)
 565                        goto alloc_err;
 566
 567                bytes_left -= size;
 568                bytes_used += size;
 569        } while (bytes_left > 0);
 570
 571        /*
 572         * Send an extra zero length packet to indicate that no more data is
 573         * available when req->req.zero is set and the data length is even
 574         * multiples of ep->ep.maxpacket.
 575         */
 576        if (req->req.zero && (req->req.length % ep->ep.maxpacket == 0)) {
 577                ret = gr_add_dma_desc(ep, req, 0, 0, gfp_flags);
 578                if (ret)
 579                        goto alloc_err;
 580        }
 581
 582        /*
 583         * For IN packets we only want to know when the last packet has been
 584         * transmitted (not just put into internal buffers).
 585         */
 586        req->last_desc->ctrl |= GR_DESC_IN_CTRL_PI;
 587
 588        return 0;
 589
 590alloc_err:
 591        gr_free_dma_desc_chain(ep->dev, req);
 592
 593        return ret;
 594}
 595
 596/* Must be called with dev->lock held */
 597static int gr_queue(struct gr_ep *ep, struct gr_request *req, gfp_t gfp_flags)
 598{
 599        struct gr_udc *dev = ep->dev;
 600        int ret;
 601
 602        if (unlikely(!ep->ep.desc && ep->num != 0)) {
 603                dev_err(dev->dev, "No ep descriptor for %s\n", ep->ep.name);
 604                return -EINVAL;
 605        }
 606
 607        if (unlikely(!req->req.buf || !list_empty(&req->queue))) {
 608                dev_err(dev->dev,
 609                        "Invalid request for %s: buf=%p list_empty=%d\n",
 610                        ep->ep.name, req->req.buf, list_empty(&req->queue));
 611                return -EINVAL;
 612        }
 613
 614        if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)) {
 615                dev_err(dev->dev, "-ESHUTDOWN");
 616                return -ESHUTDOWN;
 617        }
 618
 619        /* Can't touch registers when suspended */
 620        if (dev->ep0state == GR_EP0_SUSPEND) {
 621                dev_err(dev->dev, "-EBUSY");
 622                return -EBUSY;
 623        }
 624
 625        /* Set up DMA mapping in case the caller didn't */
 626        ret = usb_gadget_map_request(&dev->gadget, &req->req, ep->is_in);
 627        if (ret) {
 628                dev_err(dev->dev, "usb_gadget_map_request");
 629                return ret;
 630        }
 631
 632        if (ep->is_in)
 633                ret = gr_setup_in_desc_list(ep, req, gfp_flags);
 634        else
 635                ret = gr_setup_out_desc_list(ep, req, gfp_flags);
 636        if (ret)
 637                return ret;
 638
 639        req->req.status = -EINPROGRESS;
 640        req->req.actual = 0;
 641        list_add_tail(&req->queue, &ep->queue);
 642
 643        /* Start DMA if not started, otherwise interrupt handler handles it */
 644        if (!ep->dma_start && likely(!ep->stopped))
 645                gr_start_dma(ep);
 646
 647        return 0;
 648}
 649
 650/*
 651 * Queue a request from within the driver.
 652 *
 653 * Must be called with dev->lock held.
 654 */
 655static inline int gr_queue_int(struct gr_ep *ep, struct gr_request *req,
 656                               gfp_t gfp_flags)
 657{
 658        if (ep->is_in)
 659                gr_dbgprint_request("RESP", ep, req);
 660
 661        return gr_queue(ep, req, gfp_flags);
 662}
 663
 664/* ---------------------------------------------------------------------- */
 665/* General helper functions */
 666
 667/*
 668 * Dequeue ALL requests.
 669 *
 670 * Must be called with dev->lock held and irqs disabled.
 671 */
 672static void gr_ep_nuke(struct gr_ep *ep)
 673{
 674        struct gr_request *req;
 675
 676        ep->stopped = 1;
 677        ep->dma_start = 0;
 678        gr_abort_dma(ep);
 679
 680        while (!list_empty(&ep->queue)) {
 681                req = list_first_entry(&ep->queue, struct gr_request, queue);
 682                gr_finish_request(ep, req, -ESHUTDOWN);
 683        }
 684}
 685
 686/*
 687 * Reset the hardware state of this endpoint.
 688 *
 689 * Must be called with dev->lock held.
 690 */
 691static void gr_ep_reset(struct gr_ep *ep)
 692{
 693        gr_write32(&ep->regs->epctrl, 0);
 694        gr_write32(&ep->regs->dmactrl, 0);
 695
 696        ep->ep.maxpacket = MAX_CTRL_PL_SIZE;
 697        ep->ep.desc = NULL;
 698        ep->stopped = 1;
 699        ep->dma_start = 0;
 700}
 701
 702/*
 703 * Generate STALL on ep0in/out.
 704 *
 705 * Must be called with dev->lock held.
 706 */
 707static void gr_control_stall(struct gr_udc *dev)
 708{
 709        u32 epctrl;
 710
 711        epctrl = gr_read32(&dev->epo[0].regs->epctrl);
 712        gr_write32(&dev->epo[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
 713        epctrl = gr_read32(&dev->epi[0].regs->epctrl);
 714        gr_write32(&dev->epi[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
 715
 716        dev->ep0state = GR_EP0_STALL;
 717}
 718
 719/*
 720 * Halts, halts and wedges, or clears halt for an endpoint.
 721 *
 722 * Must be called with dev->lock held.
 723 */
 724static int gr_ep_halt_wedge(struct gr_ep *ep, int halt, int wedge, int fromhost)
 725{
 726        u32 epctrl;
 727        int retval = 0;
 728
 729        if (ep->num && !ep->ep.desc)
 730                return -EINVAL;
 731
 732        if (ep->num && ep->ep.desc->bmAttributes == USB_ENDPOINT_XFER_ISOC)
 733                return -EOPNOTSUPP;
 734
 735        /* Never actually halt ep0, and therefore never clear halt for ep0 */
 736        if (!ep->num) {
 737                if (halt && !fromhost) {
 738                        /* ep0 halt from gadget - generate protocol stall */
 739                        gr_control_stall(ep->dev);
 740                        dev_dbg(ep->dev->dev, "EP: stall ep0\n");
 741                        return 0;
 742                }
 743                return -EINVAL;
 744        }
 745
 746        dev_dbg(ep->dev->dev, "EP: %s halt %s\n",
 747                (halt ? (wedge ? "wedge" : "set") : "clear"), ep->ep.name);
 748
 749        epctrl = gr_read32(&ep->regs->epctrl);
 750        if (halt) {
 751                /* Set HALT */
 752                gr_write32(&ep->regs->epctrl, epctrl | GR_EPCTRL_EH);
 753                ep->stopped = 1;
 754                if (wedge)
 755                        ep->wedged = 1;
 756        } else {
 757                gr_write32(&ep->regs->epctrl, epctrl & ~GR_EPCTRL_EH);
 758                ep->stopped = 0;
 759                ep->wedged = 0;
 760
 761                /* Things might have been queued up in the meantime */
 762                if (!ep->dma_start)
 763                        gr_start_dma(ep);
 764        }
 765
 766        return retval;
 767}
 768
 769/* Must be called with dev->lock held */
 770static inline void gr_set_ep0state(struct gr_udc *dev, enum gr_ep0state value)
 771{
 772        if (dev->ep0state != value)
 773                dev_vdbg(dev->dev, "STATE:  ep0state=%s\n",
 774                         gr_ep0state_string(value));
 775        dev->ep0state = value;
 776}
 777
 778/*
 779 * Should only be called when endpoints can not generate interrupts.
 780 *
 781 * Must be called with dev->lock held.
 782 */
 783static void gr_disable_interrupts_and_pullup(struct gr_udc *dev)
 784{
 785        gr_write32(&dev->regs->control, 0);
 786        wmb(); /* Make sure that we do not deny one of our interrupts */
 787        dev->irq_enabled = 0;
 788}
 789
 790/*
 791 * Stop all device activity and disable data line pullup.
 792 *
 793 * Must be called with dev->lock held and irqs disabled.
 794 */
 795static void gr_stop_activity(struct gr_udc *dev)
 796{
 797        struct gr_ep *ep;
 798
 799        list_for_each_entry(ep, &dev->ep_list, ep_list)
 800                gr_ep_nuke(ep);
 801
 802        gr_disable_interrupts_and_pullup(dev);
 803
 804        gr_set_ep0state(dev, GR_EP0_DISCONNECT);
 805        usb_gadget_set_state(&dev->gadget, USB_STATE_NOTATTACHED);
 806}
 807
 808/* ---------------------------------------------------------------------- */
 809/* ep0 setup packet handling */
 810
 811static void gr_ep0_testmode_complete(struct usb_ep *_ep,
 812                                     struct usb_request *_req)
 813{
 814        struct gr_ep *ep;
 815        struct gr_udc *dev;
 816        u32 control;
 817
 818        ep = container_of(_ep, struct gr_ep, ep);
 819        dev = ep->dev;
 820
 821        spin_lock(&dev->lock);
 822
 823        control = gr_read32(&dev->regs->control);
 824        control |= GR_CONTROL_TM | (dev->test_mode << GR_CONTROL_TS_POS);
 825        gr_write32(&dev->regs->control, control);
 826
 827        spin_unlock(&dev->lock);
 828}
 829
 830static void gr_ep0_dummy_complete(struct usb_ep *_ep, struct usb_request *_req)
 831{
 832        /* Nothing needs to be done here */
 833}
 834
 835/*
 836 * Queue a response on ep0in.
 837 *
 838 * Must be called with dev->lock held.
 839 */
 840static int gr_ep0_respond(struct gr_udc *dev, u8 *buf, int length,
 841                          void (*complete)(struct usb_ep *ep,
 842                                           struct usb_request *req))
 843{
 844        u8 *reqbuf = dev->ep0reqi->req.buf;
 845        int status;
 846        int i;
 847
 848        for (i = 0; i < length; i++)
 849                reqbuf[i] = buf[i];
 850        dev->ep0reqi->req.length = length;
 851        dev->ep0reqi->req.complete = complete;
 852
 853        status = gr_queue_int(&dev->epi[0], dev->ep0reqi, GFP_ATOMIC);
 854        if (status < 0)
 855                dev_err(dev->dev,
 856                        "Could not queue ep0in setup response: %d\n", status);
 857
 858        return status;
 859}
 860
 861/*
 862 * Queue a 2 byte response on ep0in.
 863 *
 864 * Must be called with dev->lock held.
 865 */
 866static inline int gr_ep0_respond_u16(struct gr_udc *dev, u16 response)
 867{
 868        __le16 le_response = cpu_to_le16(response);
 869
 870        return gr_ep0_respond(dev, (u8 *)&le_response, 2,
 871                              gr_ep0_dummy_complete);
 872}
 873
 874/*
 875 * Queue a ZLP response on ep0in.
 876 *
 877 * Must be called with dev->lock held.
 878 */
 879static inline int gr_ep0_respond_empty(struct gr_udc *dev)
 880{
 881        return gr_ep0_respond(dev, NULL, 0, gr_ep0_dummy_complete);
 882}
 883
 884/*
 885 * This is run when a SET_ADDRESS request is received. First writes
 886 * the new address to the control register which is updated internally
 887 * when the next IN packet is ACKED.
 888 *
 889 * Must be called with dev->lock held.
 890 */
 891static void gr_set_address(struct gr_udc *dev, u8 address)
 892{
 893        u32 control;
 894
 895        control = gr_read32(&dev->regs->control) & ~GR_CONTROL_UA_MASK;
 896        control |= (address << GR_CONTROL_UA_POS) & GR_CONTROL_UA_MASK;
 897        control |= GR_CONTROL_SU;
 898        gr_write32(&dev->regs->control, control);
 899}
 900
 901/*
 902 * Returns negative for STALL, 0 for successful handling and positive for
 903 * delegation.
 904 *
 905 * Must be called with dev->lock held.
 906 */
 907static int gr_device_request(struct gr_udc *dev, u8 type, u8 request,
 908                             u16 value, u16 index)
 909{
 910        u16 response;
 911        u8 test;
 912
 913        switch (request) {
 914        case USB_REQ_SET_ADDRESS:
 915                dev_dbg(dev->dev, "STATUS: address %d\n", value & 0xff);
 916                gr_set_address(dev, value & 0xff);
 917                if (value)
 918                        usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
 919                else
 920                        usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
 921                return gr_ep0_respond_empty(dev);
 922
 923        case USB_REQ_GET_STATUS:
 924                /* Self powered | remote wakeup */
 925                response = 0x0001 | (dev->remote_wakeup ? 0x0002 : 0);
 926                return gr_ep0_respond_u16(dev, response);
 927
 928        case USB_REQ_SET_FEATURE:
 929                switch (value) {
 930                case USB_DEVICE_REMOTE_WAKEUP:
 931                        /* Allow remote wakeup */
 932                        dev->remote_wakeup = 1;
 933                        return gr_ep0_respond_empty(dev);
 934
 935                case USB_DEVICE_TEST_MODE:
 936                        /* The hardware does not support TEST_FORCE_EN */
 937                        test = index >> 8;
 938                        if (test >= TEST_J && test <= TEST_PACKET) {
 939                                dev->test_mode = test;
 940                                return gr_ep0_respond(dev, NULL, 0,
 941                                                      gr_ep0_testmode_complete);
 942                        }
 943                }
 944                break;
 945
 946        case USB_REQ_CLEAR_FEATURE:
 947                switch (value) {
 948                case USB_DEVICE_REMOTE_WAKEUP:
 949                        /* Disallow remote wakeup */
 950                        dev->remote_wakeup = 0;
 951                        return gr_ep0_respond_empty(dev);
 952                }
 953                break;
 954        }
 955
 956        return 1; /* Delegate the rest */
 957}
 958
 959/*
 960 * Returns negative for STALL, 0 for successful handling and positive for
 961 * delegation.
 962 *
 963 * Must be called with dev->lock held.
 964 */
 965static int gr_interface_request(struct gr_udc *dev, u8 type, u8 request,
 966                                u16 value, u16 index)
 967{
 968        if (dev->gadget.state != USB_STATE_CONFIGURED)
 969                return -1;
 970
 971        /*
 972         * Should return STALL for invalid interfaces, but udc driver does not
 973         * know anything about that. However, many gadget drivers do not handle
 974         * GET_STATUS so we need to take care of that.
 975         */
 976
 977        switch (request) {
 978        case USB_REQ_GET_STATUS:
 979                return gr_ep0_respond_u16(dev, 0x0000);
 980
 981        case USB_REQ_SET_FEATURE:
 982        case USB_REQ_CLEAR_FEATURE:
 983                /*
 984                 * No possible valid standard requests. Still let gadget drivers
 985                 * have a go at it.
 986                 */
 987                break;
 988        }
 989
 990        return 1; /* Delegate the rest */
 991}
 992
 993/*
 994 * Returns negative for STALL, 0 for successful handling and positive for
 995 * delegation.
 996 *
 997 * Must be called with dev->lock held.
 998 */
 999static int gr_endpoint_request(struct gr_udc *dev, u8 type, u8 request,
1000                               u16 value, u16 index)
1001{
1002        struct gr_ep *ep;
1003        int status;
1004        int halted;
1005        u8 epnum = index & USB_ENDPOINT_NUMBER_MASK;
1006        u8 is_in = index & USB_ENDPOINT_DIR_MASK;
1007
1008        if ((is_in && epnum >= dev->nepi) || (!is_in && epnum >= dev->nepo))
1009                return -1;
1010
1011        if (dev->gadget.state != USB_STATE_CONFIGURED && epnum != 0)
1012                return -1;
1013
1014        ep = (is_in ? &dev->epi[epnum] : &dev->epo[epnum]);
1015
1016        switch (request) {
1017        case USB_REQ_GET_STATUS:
1018                halted = gr_read32(&ep->regs->epctrl) & GR_EPCTRL_EH;
1019                return gr_ep0_respond_u16(dev, halted ? 0x0001 : 0);
1020
1021        case USB_REQ_SET_FEATURE:
1022                switch (value) {
1023                case USB_ENDPOINT_HALT:
1024                        status = gr_ep_halt_wedge(ep, 1, 0, 1);
1025                        if (status >= 0)
1026                                status = gr_ep0_respond_empty(dev);
1027                        return status;
1028                }
1029                break;
1030
1031        case USB_REQ_CLEAR_FEATURE:
1032                switch (value) {
1033                case USB_ENDPOINT_HALT:
1034                        if (ep->wedged)
1035                                return -1;
1036                        status = gr_ep_halt_wedge(ep, 0, 0, 1);
1037                        if (status >= 0)
1038                                status = gr_ep0_respond_empty(dev);
1039                        return status;
1040                }
1041                break;
1042        }
1043
1044        return 1; /* Delegate the rest */
1045}
1046
1047/* Must be called with dev->lock held */
1048static void gr_ep0out_requeue(struct gr_udc *dev)
1049{
1050        int ret = gr_queue_int(&dev->epo[0], dev->ep0reqo, GFP_ATOMIC);
1051
1052        if (ret)
1053                dev_err(dev->dev, "Could not queue ep0out setup request: %d\n",
1054                        ret);
1055}
1056
1057/*
1058 * The main function dealing with setup requests on ep0.
1059 *
1060 * Must be called with dev->lock held and irqs disabled
1061 */
1062static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req)
1063        __releases(&dev->lock)
1064        __acquires(&dev->lock)
1065{
1066        union {
1067                struct usb_ctrlrequest ctrl;
1068                u8 raw[8];
1069                u32 word[2];
1070        } u;
1071        u8 type;
1072        u8 request;
1073        u16 value;
1074        u16 index;
1075        u16 length;
1076        int i;
1077        int status;
1078
1079        /* Restore from ep0 halt */
1080        if (dev->ep0state == GR_EP0_STALL) {
1081                gr_set_ep0state(dev, GR_EP0_SETUP);
1082                if (!req->req.actual)
1083                        goto out;
1084        }
1085
1086        if (dev->ep0state == GR_EP0_ISTATUS) {
1087                gr_set_ep0state(dev, GR_EP0_SETUP);
1088                if (req->req.actual > 0)
1089                        dev_dbg(dev->dev,
1090                                "Unexpected setup packet at state %s\n",
1091                                gr_ep0state_string(GR_EP0_ISTATUS));
1092                else
1093                        goto out; /* Got expected ZLP */
1094        } else if (dev->ep0state != GR_EP0_SETUP) {
1095                dev_info(dev->dev,
1096                         "Unexpected ep0out request at state %s - stalling\n",
1097                         gr_ep0state_string(dev->ep0state));
1098                gr_control_stall(dev);
1099                gr_set_ep0state(dev, GR_EP0_SETUP);
1100                goto out;
1101        } else if (!req->req.actual) {
1102                dev_dbg(dev->dev, "Unexpected ZLP at state %s\n",
1103                        gr_ep0state_string(dev->ep0state));
1104                goto out;
1105        }
1106
1107        /* Handle SETUP packet */
1108        for (i = 0; i < req->req.actual; i++)
1109                u.raw[i] = ((u8 *)req->req.buf)[i];
1110
1111        type = u.ctrl.bRequestType;
1112        request = u.ctrl.bRequest;
1113        value = le16_to_cpu(u.ctrl.wValue);
1114        index = le16_to_cpu(u.ctrl.wIndex);
1115        length = le16_to_cpu(u.ctrl.wLength);
1116
1117        gr_dbgprint_devreq(dev, type, request, value, index, length);
1118
1119        /* Check for data stage */
1120        if (length) {
1121                if (type & USB_DIR_IN)
1122                        gr_set_ep0state(dev, GR_EP0_IDATA);
1123                else
1124                        gr_set_ep0state(dev, GR_EP0_ODATA);
1125        }
1126
1127        status = 1; /* Positive status flags delegation */
1128        if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1129                switch (type & USB_RECIP_MASK) {
1130                case USB_RECIP_DEVICE:
1131                        status = gr_device_request(dev, type, request,
1132                                                   value, index);
1133                        break;
1134                case USB_RECIP_ENDPOINT:
1135                        status =  gr_endpoint_request(dev, type, request,
1136                                                      value, index);
1137                        break;
1138                case USB_RECIP_INTERFACE:
1139                        status = gr_interface_request(dev, type, request,
1140                                                      value, index);
1141                        break;
1142                }
1143        }
1144
1145        if (status > 0) {
1146                spin_unlock(&dev->lock);
1147
1148                dev_vdbg(dev->dev, "DELEGATE\n");
1149                status = dev->driver->setup(&dev->gadget, &u.ctrl);
1150
1151                spin_lock(&dev->lock);
1152        }
1153
1154        /* Generate STALL on both ep0out and ep0in if requested */
1155        if (unlikely(status < 0)) {
1156                dev_vdbg(dev->dev, "STALL\n");
1157                gr_control_stall(dev);
1158        }
1159
1160        if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD &&
1161            request == USB_REQ_SET_CONFIGURATION) {
1162                if (!value) {
1163                        dev_dbg(dev->dev, "STATUS: deconfigured\n");
1164                        usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
1165                } else if (status >= 0) {
1166                        /* Not configured unless gadget OK:s it */
1167                        dev_dbg(dev->dev, "STATUS: configured: %d\n", value);
1168                        usb_gadget_set_state(&dev->gadget,
1169                                             USB_STATE_CONFIGURED);
1170                }
1171        }
1172
1173        /* Get ready for next stage */
1174        if (dev->ep0state == GR_EP0_ODATA)
1175                gr_set_ep0state(dev, GR_EP0_OSTATUS);
1176        else if (dev->ep0state == GR_EP0_IDATA)
1177                gr_set_ep0state(dev, GR_EP0_ISTATUS);
1178        else
1179                gr_set_ep0state(dev, GR_EP0_SETUP);
1180
1181out:
1182        gr_ep0out_requeue(dev);
1183}
1184
1185/* ---------------------------------------------------------------------- */
1186/* VBUS and USB reset handling */
1187
1188/* Must be called with dev->lock held and irqs disabled  */
1189static void gr_vbus_connected(struct gr_udc *dev, u32 status)
1190{
1191        u32 control;
1192
1193        dev->gadget.speed = GR_SPEED(status);
1194        usb_gadget_set_state(&dev->gadget, USB_STATE_POWERED);
1195
1196        /* Turn on full interrupts and pullup */
1197        control = (GR_CONTROL_SI | GR_CONTROL_UI | GR_CONTROL_VI |
1198                   GR_CONTROL_SP | GR_CONTROL_EP);
1199        gr_write32(&dev->regs->control, control);
1200}
1201
1202/* Must be called with dev->lock held */
1203static void gr_enable_vbus_detect(struct gr_udc *dev)
1204{
1205        u32 status;
1206
1207        dev->irq_enabled = 1;
1208        wmb(); /* Make sure we do not ignore an interrupt */
1209        gr_write32(&dev->regs->control, GR_CONTROL_VI);
1210
1211        /* Take care of the case we are already plugged in at this point */
1212        status = gr_read32(&dev->regs->status);
1213        if (status & GR_STATUS_VB)
1214                gr_vbus_connected(dev, status);
1215}
1216
1217/* Must be called with dev->lock held and irqs disabled */
1218static void gr_vbus_disconnected(struct gr_udc *dev)
1219{
1220        gr_stop_activity(dev);
1221
1222        /* Report disconnect */
1223        if (dev->driver && dev->driver->disconnect) {
1224                spin_unlock(&dev->lock);
1225
1226                dev->driver->disconnect(&dev->gadget);
1227
1228                spin_lock(&dev->lock);
1229        }
1230
1231        gr_enable_vbus_detect(dev);
1232}
1233
1234/* Must be called with dev->lock held and irqs disabled */
1235static void gr_udc_usbreset(struct gr_udc *dev, u32 status)
1236{
1237        gr_set_address(dev, 0);
1238        gr_set_ep0state(dev, GR_EP0_SETUP);
1239        usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
1240        dev->gadget.speed = GR_SPEED(status);
1241
1242        gr_ep_nuke(&dev->epo[0]);
1243        gr_ep_nuke(&dev->epi[0]);
1244        dev->epo[0].stopped = 0;
1245        dev->epi[0].stopped = 0;
1246        gr_ep0out_requeue(dev);
1247}
1248
1249/* ---------------------------------------------------------------------- */
1250/* Irq handling */
1251
1252/*
1253 * Handles interrupts from in endpoints. Returns whether something was handled.
1254 *
1255 * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1256 */
1257static int gr_handle_in_ep(struct gr_ep *ep)
1258{
1259        struct gr_request *req;
1260
1261        req = list_first_entry(&ep->queue, struct gr_request, queue);
1262        if (!req->last_desc)
1263                return 0;
1264
1265        if (ACCESS_ONCE(req->last_desc->ctrl) & GR_DESC_IN_CTRL_EN)
1266                return 0; /* Not put in hardware buffers yet */
1267
1268        if (gr_read32(&ep->regs->epstat) & (GR_EPSTAT_B1 | GR_EPSTAT_B0))
1269                return 0; /* Not transmitted yet, still in hardware buffers */
1270
1271        /* Write complete */
1272        gr_dma_advance(ep, 0);
1273
1274        return 1;
1275}
1276
1277/*
1278 * Handles interrupts from out endpoints. Returns whether something was handled.
1279 *
1280 * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1281 */
1282static int gr_handle_out_ep(struct gr_ep *ep)
1283{
1284        u32 ep_dmactrl;
1285        u32 ctrl;
1286        u16 len;
1287        struct gr_request *req;
1288        struct gr_udc *dev = ep->dev;
1289
1290        req = list_first_entry(&ep->queue, struct gr_request, queue);
1291        if (!req->curr_desc)
1292                return 0;
1293
1294        ctrl = ACCESS_ONCE(req->curr_desc->ctrl);
1295        if (ctrl & GR_DESC_OUT_CTRL_EN)
1296                return 0; /* Not received yet */
1297
1298        /* Read complete */
1299        len = ctrl & GR_DESC_OUT_CTRL_LEN_MASK;
1300        req->req.actual += len;
1301        if (ctrl & GR_DESC_OUT_CTRL_SE)
1302                req->setup = 1;
1303
1304        if (len < ep->ep.maxpacket || req->req.actual >= req->req.length) {
1305                /* Short packet or >= expected size - we are done */
1306
1307                if ((ep == &dev->epo[0]) && (dev->ep0state == GR_EP0_OSTATUS)) {
1308                        /*
1309                         * Send a status stage ZLP to ack the DATA stage in the
1310                         * OUT direction. This needs to be done before
1311                         * gr_dma_advance as that can lead to a call to
1312                         * ep0_setup that can change dev->ep0state.
1313                         */
1314                        gr_ep0_respond_empty(dev);
1315                        gr_set_ep0state(dev, GR_EP0_SETUP);
1316                }
1317
1318                gr_dma_advance(ep, 0);
1319        } else {
1320                /* Not done yet. Enable the next descriptor to receive more. */
1321                req->curr_desc = req->curr_desc->next_desc;
1322                req->curr_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
1323
1324                ep_dmactrl = gr_read32(&ep->regs->dmactrl);
1325                gr_write32(&ep->regs->dmactrl, ep_dmactrl | GR_DMACTRL_DA);
1326        }
1327
1328        return 1;
1329}
1330
1331/*
1332 * Handle state changes. Returns whether something was handled.
1333 *
1334 * Must be called with dev->lock held and irqs disabled.
1335 */
1336static int gr_handle_state_changes(struct gr_udc *dev)
1337{
1338        u32 status = gr_read32(&dev->regs->status);
1339        int handled = 0;
1340        int powstate = !(dev->gadget.state == USB_STATE_NOTATTACHED ||
1341                         dev->gadget.state == USB_STATE_ATTACHED);
1342
1343        /* VBUS valid detected */
1344        if (!powstate && (status & GR_STATUS_VB)) {
1345                dev_dbg(dev->dev, "STATUS: vbus valid detected\n");
1346                gr_vbus_connected(dev, status);
1347                handled = 1;
1348        }
1349
1350        /* Disconnect */
1351        if (powstate && !(status & GR_STATUS_VB)) {
1352                dev_dbg(dev->dev, "STATUS: vbus invalid detected\n");
1353                gr_vbus_disconnected(dev);
1354                handled = 1;
1355        }
1356
1357        /* USB reset detected */
1358        if (status & GR_STATUS_UR) {
1359                dev_dbg(dev->dev, "STATUS: USB reset - speed is %s\n",
1360                        GR_SPEED_STR(status));
1361                gr_write32(&dev->regs->status, GR_STATUS_UR);
1362                gr_udc_usbreset(dev, status);
1363                handled = 1;
1364        }
1365
1366        /* Speed change */
1367        if (dev->gadget.speed != GR_SPEED(status)) {
1368                dev_dbg(dev->dev, "STATUS: USB Speed change to %s\n",
1369                        GR_SPEED_STR(status));
1370                dev->gadget.speed = GR_SPEED(status);
1371                handled = 1;
1372        }
1373
1374        /* Going into suspend */
1375        if ((dev->ep0state != GR_EP0_SUSPEND) && !(status & GR_STATUS_SU)) {
1376                dev_dbg(dev->dev, "STATUS: USB suspend\n");
1377                gr_set_ep0state(dev, GR_EP0_SUSPEND);
1378                dev->suspended_from = dev->gadget.state;
1379                usb_gadget_set_state(&dev->gadget, USB_STATE_SUSPENDED);
1380
1381                if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1382                    dev->driver && dev->driver->suspend) {
1383                        spin_unlock(&dev->lock);
1384
1385                        dev->driver->suspend(&dev->gadget);
1386
1387                        spin_lock(&dev->lock);
1388                }
1389                handled = 1;
1390        }
1391
1392        /* Coming out of suspend */
1393        if ((dev->ep0state == GR_EP0_SUSPEND) && (status & GR_STATUS_SU)) {
1394                dev_dbg(dev->dev, "STATUS: USB resume\n");
1395                if (dev->suspended_from == USB_STATE_POWERED)
1396                        gr_set_ep0state(dev, GR_EP0_DISCONNECT);
1397                else
1398                        gr_set_ep0state(dev, GR_EP0_SETUP);
1399                usb_gadget_set_state(&dev->gadget, dev->suspended_from);
1400
1401                if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1402                    dev->driver && dev->driver->resume) {
1403                        spin_unlock(&dev->lock);
1404
1405                        dev->driver->resume(&dev->gadget);
1406
1407                        spin_lock(&dev->lock);
1408                }
1409                handled = 1;
1410        }
1411
1412        return handled;
1413}
1414
1415/* Non-interrupt context irq handler */
1416static irqreturn_t gr_irq_handler(int irq, void *_dev)
1417{
1418        struct gr_udc *dev = _dev;
1419        struct gr_ep *ep;
1420        int handled = 0;
1421        int i;
1422        unsigned long flags;
1423
1424        spin_lock_irqsave(&dev->lock, flags);
1425
1426        if (!dev->irq_enabled)
1427                goto out;
1428
1429        /*
1430         * Check IN ep interrupts. We check these before the OUT eps because
1431         * some gadgets reuse the request that might already be currently
1432         * outstanding and needs to be completed (mainly setup requests).
1433         */
1434        for (i = 0; i < dev->nepi; i++) {
1435                ep = &dev->epi[i];
1436                if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1437                        handled = gr_handle_in_ep(ep) || handled;
1438        }
1439
1440        /* Check OUT ep interrupts */
1441        for (i = 0; i < dev->nepo; i++) {
1442                ep = &dev->epo[i];
1443                if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1444                        handled = gr_handle_out_ep(ep) || handled;
1445        }
1446
1447        /* Check status interrupts */
1448        handled = gr_handle_state_changes(dev) || handled;
1449
1450        /*
1451         * Check AMBA DMA errors. Only check if we didn't find anything else to
1452         * handle because this shouldn't happen if we did everything right.
1453         */
1454        if (!handled) {
1455                list_for_each_entry(ep, &dev->ep_list, ep_list) {
1456                        if (gr_read32(&ep->regs->dmactrl) & GR_DMACTRL_AE) {
1457                                dev_err(dev->dev,
1458                                        "AMBA Error occurred for %s\n",
1459                                        ep->ep.name);
1460                                handled = 1;
1461                        }
1462                }
1463        }
1464
1465out:
1466        spin_unlock_irqrestore(&dev->lock, flags);
1467
1468        return handled ? IRQ_HANDLED : IRQ_NONE;
1469}
1470
1471/* Interrupt context irq handler */
1472static irqreturn_t gr_irq(int irq, void *_dev)
1473{
1474        struct gr_udc *dev = _dev;
1475
1476        if (!dev->irq_enabled)
1477                return IRQ_NONE;
1478
1479        return IRQ_WAKE_THREAD;
1480}
1481
1482/* ---------------------------------------------------------------------- */
1483/* USB ep ops */
1484
1485/* Enable endpoint. Not for ep0in and ep0out that are handled separately. */
1486static int gr_ep_enable(struct usb_ep *_ep,
1487                        const struct usb_endpoint_descriptor *desc)
1488{
1489        struct gr_udc *dev;
1490        struct gr_ep *ep;
1491        u8 mode;
1492        u8 nt;
1493        u16 max;
1494        u16 buffer_size = 0;
1495        u32 epctrl;
1496
1497        ep = container_of(_ep, struct gr_ep, ep);
1498        if (!_ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT)
1499                return -EINVAL;
1500
1501        dev = ep->dev;
1502
1503        /* 'ep0' IN and OUT are reserved */
1504        if (ep == &dev->epo[0] || ep == &dev->epi[0])
1505                return -EINVAL;
1506
1507        if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
1508                return -ESHUTDOWN;
1509
1510        /* Make sure we are clear for enabling */
1511        epctrl = gr_read32(&ep->regs->epctrl);
1512        if (epctrl & GR_EPCTRL_EV)
1513                return -EBUSY;
1514
1515        /* Check that directions match */
1516        if (!ep->is_in != !usb_endpoint_dir_in(desc))
1517                return -EINVAL;
1518
1519        /* Check ep num */
1520        if ((!ep->is_in && ep->num >= dev->nepo) ||
1521            (ep->is_in && ep->num >= dev->nepi))
1522                return -EINVAL;
1523
1524        if (usb_endpoint_xfer_control(desc)) {
1525                mode = 0;
1526        } else if (usb_endpoint_xfer_isoc(desc)) {
1527                mode = 1;
1528        } else if (usb_endpoint_xfer_bulk(desc)) {
1529                mode = 2;
1530        } else if (usb_endpoint_xfer_int(desc)) {
1531                mode = 3;
1532        } else {
1533                dev_err(dev->dev, "Unknown transfer type for %s\n",
1534                        ep->ep.name);
1535                return -EINVAL;
1536        }
1537
1538        /*
1539         * Bits 10-0 set the max payload. 12-11 set the number of
1540         * additional transactions.
1541         */
1542        max = 0x7ff & usb_endpoint_maxp(desc);
1543        nt = 0x3 & (usb_endpoint_maxp(desc) >> 11);
1544        buffer_size = GR_BUFFER_SIZE(epctrl);
1545        if (nt && (mode == 0 || mode == 2)) {
1546                dev_err(dev->dev,
1547                        "%s mode: multiple trans./microframe not valid\n",
1548                        (mode == 2 ? "Bulk" : "Control"));
1549                return -EINVAL;
1550        } else if (nt == 0x3) {
1551                dev_err(dev->dev,
1552                        "Invalid value 0x3 for additional trans./microframe\n");
1553                return -EINVAL;
1554        } else if ((nt + 1) * max > buffer_size) {
1555                dev_err(dev->dev, "Hw buffer size %d < max payload %d * %d\n",
1556                        buffer_size, (nt + 1), max);
1557                return -EINVAL;
1558        } else if (max == 0) {
1559                dev_err(dev->dev, "Max payload cannot be set to 0\n");
1560                return -EINVAL;
1561        } else if (max > ep->ep.maxpacket_limit) {
1562                dev_err(dev->dev, "Requested max payload %d > limit %d\n",
1563                        max, ep->ep.maxpacket_limit);
1564                return -EINVAL;
1565        }
1566
1567        spin_lock(&ep->dev->lock);
1568
1569        if (!ep->stopped) {
1570                spin_unlock(&ep->dev->lock);
1571                return -EBUSY;
1572        }
1573
1574        ep->stopped = 0;
1575        ep->wedged = 0;
1576        ep->ep.desc = desc;
1577        ep->ep.maxpacket = max;
1578        ep->dma_start = 0;
1579
1580
1581        if (nt) {
1582                /*
1583                 * Maximum possible size of all payloads in one microframe
1584                 * regardless of direction when using high-bandwidth mode.
1585                 */
1586                ep->bytes_per_buffer = (nt + 1) * max;
1587        } else if (ep->is_in) {
1588                /*
1589                 * The biggest multiple of maximum packet size that fits into
1590                 * the buffer. The hardware will split up into many packets in
1591                 * the IN direction.
1592                 */
1593                ep->bytes_per_buffer = (buffer_size / max) * max;
1594        } else {
1595                /*
1596                 * Only single packets will be placed the buffers in the OUT
1597                 * direction.
1598                 */
1599                ep->bytes_per_buffer = max;
1600        }
1601
1602        epctrl = (max << GR_EPCTRL_MAXPL_POS)
1603                | (nt << GR_EPCTRL_NT_POS)
1604                | (mode << GR_EPCTRL_TT_POS)
1605                | GR_EPCTRL_EV;
1606        if (ep->is_in)
1607                epctrl |= GR_EPCTRL_PI;
1608        gr_write32(&ep->regs->epctrl, epctrl);
1609
1610        gr_write32(&ep->regs->dmactrl, GR_DMACTRL_IE | GR_DMACTRL_AI);
1611
1612        spin_unlock(&ep->dev->lock);
1613
1614        dev_dbg(ep->dev->dev, "EP: %s enabled - %s with %d bytes/buffer\n",
1615                ep->ep.name, gr_modestring[mode], ep->bytes_per_buffer);
1616        return 0;
1617}
1618
1619/* Disable endpoint. Not for ep0in and ep0out that are handled separately. */
1620static int gr_ep_disable(struct usb_ep *_ep)
1621{
1622        struct gr_ep *ep;
1623        struct gr_udc *dev;
1624        unsigned long flags;
1625
1626        ep = container_of(_ep, struct gr_ep, ep);
1627        if (!_ep || !ep->ep.desc)
1628                return -ENODEV;
1629
1630        dev = ep->dev;
1631
1632        /* 'ep0' IN and OUT are reserved */
1633        if (ep == &dev->epo[0] || ep == &dev->epi[0])
1634                return -EINVAL;
1635
1636        if (dev->ep0state == GR_EP0_SUSPEND)
1637                return -EBUSY;
1638
1639        dev_dbg(ep->dev->dev, "EP: disable %s\n", ep->ep.name);
1640
1641        spin_lock_irqsave(&dev->lock, flags);
1642
1643        gr_ep_nuke(ep);
1644        gr_ep_reset(ep);
1645        ep->ep.desc = NULL;
1646
1647        spin_unlock_irqrestore(&dev->lock, flags);
1648
1649        return 0;
1650}
1651
1652/*
1653 * Frees a request, but not any DMA buffers associated with it
1654 * (gr_finish_request should already have taken care of that).
1655 */
1656static void gr_free_request(struct usb_ep *_ep, struct usb_request *_req)
1657{
1658        struct gr_request *req;
1659
1660        if (!_ep || !_req)
1661                return;
1662        req = container_of(_req, struct gr_request, req);
1663
1664        /* Leads to memory leak */
1665        WARN(!list_empty(&req->queue),
1666             "request not dequeued properly before freeing\n");
1667
1668        kfree(req);
1669}
1670
1671/* Queue a request from the gadget */
1672static int gr_queue_ext(struct usb_ep *_ep, struct usb_request *_req,
1673                        gfp_t gfp_flags)
1674{
1675        struct gr_ep *ep;
1676        struct gr_request *req;
1677        struct gr_udc *dev;
1678        int ret;
1679
1680        if (unlikely(!_ep || !_req))
1681                return -EINVAL;
1682
1683        ep = container_of(_ep, struct gr_ep, ep);
1684        req = container_of(_req, struct gr_request, req);
1685        dev = ep->dev;
1686
1687        spin_lock(&ep->dev->lock);
1688
1689        /*
1690         * The ep0 pointer in the gadget struct is used both for ep0in and
1691         * ep0out. In a data stage in the out direction ep0out needs to be used
1692         * instead of the default ep0in. Completion functions might use
1693         * driver_data, so that needs to be copied as well.
1694         */
1695        if ((ep == &dev->epi[0]) && (dev->ep0state == GR_EP0_ODATA)) {
1696                ep = &dev->epo[0];
1697                ep->ep.driver_data = dev->epi[0].ep.driver_data;
1698        }
1699
1700        if (ep->is_in)
1701                gr_dbgprint_request("EXTERN", ep, req);
1702
1703        ret = gr_queue(ep, req, GFP_ATOMIC);
1704
1705        spin_unlock(&ep->dev->lock);
1706
1707        return ret;
1708}
1709
1710/* Dequeue JUST ONE request */
1711static int gr_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1712{
1713        struct gr_request *req;
1714        struct gr_ep *ep;
1715        struct gr_udc *dev;
1716        int ret = 0;
1717        unsigned long flags;
1718
1719        ep = container_of(_ep, struct gr_ep, ep);
1720        if (!_ep || !_req || (!ep->ep.desc && ep->num != 0))
1721                return -EINVAL;
1722        dev = ep->dev;
1723        if (!dev->driver)
1724                return -ESHUTDOWN;
1725
1726        /* We can't touch (DMA) registers when suspended */
1727        if (dev->ep0state == GR_EP0_SUSPEND)
1728                return -EBUSY;
1729
1730        spin_lock_irqsave(&dev->lock, flags);
1731
1732        /* Make sure it's actually queued on this endpoint */
1733        list_for_each_entry(req, &ep->queue, queue) {
1734                if (&req->req == _req)
1735                        break;
1736        }
1737        if (&req->req != _req) {
1738                ret = -EINVAL;
1739                goto out;
1740        }
1741
1742        if (list_first_entry(&ep->queue, struct gr_request, queue) == req) {
1743                /* This request is currently being processed */
1744                gr_abort_dma(ep);
1745                if (ep->stopped)
1746                        gr_finish_request(ep, req, -ECONNRESET);
1747                else
1748                        gr_dma_advance(ep, -ECONNRESET);
1749        } else if (!list_empty(&req->queue)) {
1750                /* Not being processed - gr_finish_request dequeues it */
1751                gr_finish_request(ep, req, -ECONNRESET);
1752        } else {
1753                ret = -EOPNOTSUPP;
1754        }
1755
1756out:
1757        spin_unlock_irqrestore(&dev->lock, flags);
1758
1759        return ret;
1760}
1761
1762/* Helper for gr_set_halt and gr_set_wedge */
1763static int gr_set_halt_wedge(struct usb_ep *_ep, int halt, int wedge)
1764{
1765        int ret;
1766        struct gr_ep *ep;
1767
1768        if (!_ep)
1769                return -ENODEV;
1770        ep = container_of(_ep, struct gr_ep, ep);
1771
1772        spin_lock(&ep->dev->lock);
1773
1774        /* Halting an IN endpoint should fail if queue is not empty */
1775        if (halt && ep->is_in && !list_empty(&ep->queue)) {
1776                ret = -EAGAIN;
1777                goto out;
1778        }
1779
1780        ret = gr_ep_halt_wedge(ep, halt, wedge, 0);
1781
1782out:
1783        spin_unlock(&ep->dev->lock);
1784
1785        return ret;
1786}
1787
1788/* Halt endpoint */
1789static int gr_set_halt(struct usb_ep *_ep, int halt)
1790{
1791        return gr_set_halt_wedge(_ep, halt, 0);
1792}
1793
1794/* Halt and wedge endpoint */
1795static int gr_set_wedge(struct usb_ep *_ep)
1796{
1797        return gr_set_halt_wedge(_ep, 1, 1);
1798}
1799
1800/*
1801 * Return the total number of bytes currently stored in the internal buffers of
1802 * the endpoint.
1803 */
1804static int gr_fifo_status(struct usb_ep *_ep)
1805{
1806        struct gr_ep *ep;
1807        u32 epstat;
1808        u32 bytes = 0;
1809
1810        if (!_ep)
1811                return -ENODEV;
1812        ep = container_of(_ep, struct gr_ep, ep);
1813
1814        epstat = gr_read32(&ep->regs->epstat);
1815
1816        if (epstat & GR_EPSTAT_B0)
1817                bytes += (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS;
1818        if (epstat & GR_EPSTAT_B1)
1819                bytes += (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS;
1820
1821        return bytes;
1822}
1823
1824
1825/* Empty data from internal buffers of an endpoint. */
1826static void gr_fifo_flush(struct usb_ep *_ep)
1827{
1828        struct gr_ep *ep;
1829        u32 epctrl;
1830
1831        if (!_ep)
1832                return;
1833        ep = container_of(_ep, struct gr_ep, ep);
1834        dev_vdbg(ep->dev->dev, "EP: flush fifo %s\n", ep->ep.name);
1835
1836        spin_lock(&ep->dev->lock);
1837
1838        epctrl = gr_read32(&ep->regs->epctrl);
1839        epctrl |= GR_EPCTRL_CB;
1840        gr_write32(&ep->regs->epctrl, epctrl);
1841
1842        spin_unlock(&ep->dev->lock);
1843}
1844
1845static struct usb_ep_ops gr_ep_ops = {
1846        .enable         = gr_ep_enable,
1847        .disable        = gr_ep_disable,
1848
1849        .alloc_request  = gr_alloc_request,
1850        .free_request   = gr_free_request,
1851
1852        .queue          = gr_queue_ext,
1853        .dequeue        = gr_dequeue,
1854
1855        .set_halt       = gr_set_halt,
1856        .set_wedge      = gr_set_wedge,
1857        .fifo_status    = gr_fifo_status,
1858        .fifo_flush     = gr_fifo_flush,
1859};
1860
1861/* ---------------------------------------------------------------------- */
1862/* USB Gadget ops */
1863
1864static int gr_get_frame(struct usb_gadget *_gadget)
1865{
1866        struct gr_udc *dev;
1867
1868        if (!_gadget)
1869                return -ENODEV;
1870        dev = container_of(_gadget, struct gr_udc, gadget);
1871        return gr_read32(&dev->regs->status) & GR_STATUS_FN_MASK;
1872}
1873
1874static int gr_wakeup(struct usb_gadget *_gadget)
1875{
1876        struct gr_udc *dev;
1877
1878        if (!_gadget)
1879                return -ENODEV;
1880        dev = container_of(_gadget, struct gr_udc, gadget);
1881
1882        /* Remote wakeup feature not enabled by host*/
1883        if (!dev->remote_wakeup)
1884                return -EINVAL;
1885
1886        spin_lock(&dev->lock);
1887
1888        gr_write32(&dev->regs->control,
1889                   gr_read32(&dev->regs->control) | GR_CONTROL_RW);
1890
1891        spin_unlock(&dev->lock);
1892
1893        return 0;
1894}
1895
1896static int gr_pullup(struct usb_gadget *_gadget, int is_on)
1897{
1898        struct gr_udc *dev;
1899        u32 control;
1900
1901        if (!_gadget)
1902                return -ENODEV;
1903        dev = container_of(_gadget, struct gr_udc, gadget);
1904
1905        spin_lock(&dev->lock);
1906
1907        control = gr_read32(&dev->regs->control);
1908        if (is_on)
1909                control |= GR_CONTROL_EP;
1910        else
1911                control &= ~GR_CONTROL_EP;
1912        gr_write32(&dev->regs->control, control);
1913
1914        spin_unlock(&dev->lock);
1915
1916        return 0;
1917}
1918
1919static int gr_udc_start(struct usb_gadget *gadget,
1920                        struct usb_gadget_driver *driver)
1921{
1922        struct gr_udc *dev = to_gr_udc(gadget);
1923
1924        spin_lock(&dev->lock);
1925
1926        /* Hook up the driver */
1927        driver->driver.bus = NULL;
1928        dev->driver = driver;
1929
1930        /* Get ready for host detection */
1931        gr_enable_vbus_detect(dev);
1932
1933        spin_unlock(&dev->lock);
1934
1935        return 0;
1936}
1937
1938static int gr_udc_stop(struct usb_gadget *gadget)
1939{
1940        struct gr_udc *dev = to_gr_udc(gadget);
1941        unsigned long flags;
1942
1943        spin_lock_irqsave(&dev->lock, flags);
1944
1945        dev->driver = NULL;
1946        gr_stop_activity(dev);
1947
1948        spin_unlock_irqrestore(&dev->lock, flags);
1949
1950        return 0;
1951}
1952
1953static const struct usb_gadget_ops gr_ops = {
1954        .get_frame      = gr_get_frame,
1955        .wakeup         = gr_wakeup,
1956        .pullup         = gr_pullup,
1957        .udc_start      = gr_udc_start,
1958        .udc_stop       = gr_udc_stop,
1959        /* Other operations not supported */
1960};
1961
1962/* ---------------------------------------------------------------------- */
1963/* Module probe, removal and of-matching */
1964
1965static const char * const onames[] = {
1966        "ep0out", "ep1out", "ep2out", "ep3out", "ep4out", "ep5out",
1967        "ep6out", "ep7out", "ep8out", "ep9out", "ep10out", "ep11out",
1968        "ep12out", "ep13out", "ep14out", "ep15out"
1969};
1970
1971static const char * const inames[] = {
1972        "ep0in", "ep1in", "ep2in", "ep3in", "ep4in", "ep5in",
1973        "ep6in", "ep7in", "ep8in", "ep9in", "ep10in", "ep11in",
1974        "ep12in", "ep13in", "ep14in", "ep15in"
1975};
1976
1977/* Must be called with dev->lock held */
1978static int gr_ep_init(struct gr_udc *dev, int num, int is_in, u32 maxplimit)
1979{
1980        struct gr_ep *ep;
1981        struct gr_request *req;
1982        struct usb_request *_req;
1983        void *buf;
1984
1985        if (is_in) {
1986                ep = &dev->epi[num];
1987                ep->ep.name = inames[num];
1988                ep->regs = &dev->regs->epi[num];
1989        } else {
1990                ep = &dev->epo[num];
1991                ep->ep.name = onames[num];
1992                ep->regs = &dev->regs->epo[num];
1993        }
1994
1995        gr_ep_reset(ep);
1996        ep->num = num;
1997        ep->is_in = is_in;
1998        ep->dev = dev;
1999        ep->ep.ops = &gr_ep_ops;
2000        INIT_LIST_HEAD(&ep->queue);
2001
2002        if (num == 0) {
2003                _req = gr_alloc_request(&ep->ep, GFP_ATOMIC);
2004                buf = devm_kzalloc(dev->dev, PAGE_SIZE, GFP_DMA | GFP_ATOMIC);
2005                if (!_req || !buf) {
2006                        /* possible _req freed by gr_probe via gr_remove */
2007                        return -ENOMEM;
2008                }
2009
2010                req = container_of(_req, struct gr_request, req);
2011                req->req.buf = buf;
2012                req->req.length = MAX_CTRL_PL_SIZE;
2013
2014                if (is_in)
2015                        dev->ep0reqi = req; /* Complete gets set as used */
2016                else
2017                        dev->ep0reqo = req; /* Completion treated separately */
2018
2019                usb_ep_set_maxpacket_limit(&ep->ep, MAX_CTRL_PL_SIZE);
2020                ep->bytes_per_buffer = MAX_CTRL_PL_SIZE;
2021
2022                ep->ep.caps.type_control = true;
2023        } else {
2024                usb_ep_set_maxpacket_limit(&ep->ep, (u16)maxplimit);
2025                list_add_tail(&ep->ep.ep_list, &dev->gadget.ep_list);
2026
2027                ep->ep.caps.type_iso = true;
2028                ep->ep.caps.type_bulk = true;
2029                ep->ep.caps.type_int = true;
2030        }
2031        list_add_tail(&ep->ep_list, &dev->ep_list);
2032
2033        if (is_in)
2034                ep->ep.caps.dir_in = true;
2035        else
2036                ep->ep.caps.dir_out = true;
2037
2038        ep->tailbuf = dma_alloc_coherent(dev->dev, ep->ep.maxpacket_limit,
2039                                         &ep->tailbuf_paddr, GFP_ATOMIC);
2040        if (!ep->tailbuf)
2041                return -ENOMEM;
2042
2043        return 0;
2044}
2045
2046/* Must be called with dev->lock held */
2047static int gr_udc_init(struct gr_udc *dev)
2048{
2049        struct device_node *np = dev->dev->of_node;
2050        u32 epctrl_val;
2051        u32 dmactrl_val;
2052        int i;
2053        int ret = 0;
2054        u32 bufsize;
2055
2056        gr_set_address(dev, 0);
2057
2058        INIT_LIST_HEAD(&dev->gadget.ep_list);
2059        dev->gadget.speed = USB_SPEED_UNKNOWN;
2060        dev->gadget.ep0 = &dev->epi[0].ep;
2061
2062        INIT_LIST_HEAD(&dev->ep_list);
2063        gr_set_ep0state(dev, GR_EP0_DISCONNECT);
2064
2065        for (i = 0; i < dev->nepo; i++) {
2066                if (of_property_read_u32_index(np, "epobufsizes", i, &bufsize))
2067                        bufsize = 1024;
2068                ret = gr_ep_init(dev, i, 0, bufsize);
2069                if (ret)
2070                        return ret;
2071        }
2072
2073        for (i = 0; i < dev->nepi; i++) {
2074                if (of_property_read_u32_index(np, "epibufsizes", i, &bufsize))
2075                        bufsize = 1024;
2076                ret = gr_ep_init(dev, i, 1, bufsize);
2077                if (ret)
2078                        return ret;
2079        }
2080
2081        /* Must be disabled by default */
2082        dev->remote_wakeup = 0;
2083
2084        /* Enable ep0out and ep0in */
2085        epctrl_val = (MAX_CTRL_PL_SIZE << GR_EPCTRL_MAXPL_POS) | GR_EPCTRL_EV;
2086        dmactrl_val = GR_DMACTRL_IE | GR_DMACTRL_AI;
2087        gr_write32(&dev->epo[0].regs->epctrl, epctrl_val);
2088        gr_write32(&dev->epi[0].regs->epctrl, epctrl_val | GR_EPCTRL_PI);
2089        gr_write32(&dev->epo[0].regs->dmactrl, dmactrl_val);
2090        gr_write32(&dev->epi[0].regs->dmactrl, dmactrl_val);
2091
2092        return 0;
2093}
2094
2095static void gr_ep_remove(struct gr_udc *dev, int num, int is_in)
2096{
2097        struct gr_ep *ep;
2098
2099        if (is_in)
2100                ep = &dev->epi[num];
2101        else
2102                ep = &dev->epo[num];
2103
2104        if (ep->tailbuf)
2105                dma_free_coherent(dev->dev, ep->ep.maxpacket_limit,
2106                                  ep->tailbuf, ep->tailbuf_paddr);
2107}
2108
2109static int gr_remove(struct platform_device *pdev)
2110{
2111        struct gr_udc *dev = platform_get_drvdata(pdev);
2112        int i;
2113
2114        if (dev->added)
2115                usb_del_gadget_udc(&dev->gadget); /* Shuts everything down */
2116        if (dev->driver)
2117                return -EBUSY;
2118
2119        gr_dfs_delete(dev);
2120        dma_pool_destroy(dev->desc_pool);
2121        platform_set_drvdata(pdev, NULL);
2122
2123        gr_free_request(&dev->epi[0].ep, &dev->ep0reqi->req);
2124        gr_free_request(&dev->epo[0].ep, &dev->ep0reqo->req);
2125
2126        for (i = 0; i < dev->nepo; i++)
2127                gr_ep_remove(dev, i, 0);
2128        for (i = 0; i < dev->nepi; i++)
2129                gr_ep_remove(dev, i, 1);
2130
2131        return 0;
2132}
2133static int gr_request_irq(struct gr_udc *dev, int irq)
2134{
2135        return devm_request_threaded_irq(dev->dev, irq, gr_irq, gr_irq_handler,
2136                                         IRQF_SHARED, driver_name, dev);
2137}
2138
2139static int gr_probe(struct platform_device *pdev)
2140{
2141        struct gr_udc *dev;
2142        struct resource *res;
2143        struct gr_regs __iomem *regs;
2144        int retval;
2145        u32 status;
2146
2147        dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
2148        if (!dev)
2149                return -ENOMEM;
2150        dev->dev = &pdev->dev;
2151
2152        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2153        regs = devm_ioremap_resource(dev->dev, res);
2154        if (IS_ERR(regs))
2155                return PTR_ERR(regs);
2156
2157        dev->irq = platform_get_irq(pdev, 0);
2158        if (dev->irq <= 0) {
2159                dev_err(dev->dev, "No irq found\n");
2160                return -ENODEV;
2161        }
2162
2163        /* Some core configurations has separate irqs for IN and OUT events */
2164        dev->irqi = platform_get_irq(pdev, 1);
2165        if (dev->irqi > 0) {
2166                dev->irqo = platform_get_irq(pdev, 2);
2167                if (dev->irqo <= 0) {
2168                        dev_err(dev->dev, "Found irqi but not irqo\n");
2169                        return -ENODEV;
2170                }
2171        } else {
2172                dev->irqi = 0;
2173        }
2174
2175        dev->gadget.name = driver_name;
2176        dev->gadget.max_speed = USB_SPEED_HIGH;
2177        dev->gadget.ops = &gr_ops;
2178
2179        spin_lock_init(&dev->lock);
2180        dev->regs = regs;
2181
2182        platform_set_drvdata(pdev, dev);
2183
2184        /* Determine number of endpoints and data interface mode */
2185        status = gr_read32(&dev->regs->status);
2186        dev->nepi = ((status & GR_STATUS_NEPI_MASK) >> GR_STATUS_NEPI_POS) + 1;
2187        dev->nepo = ((status & GR_STATUS_NEPO_MASK) >> GR_STATUS_NEPO_POS) + 1;
2188
2189        if (!(status & GR_STATUS_DM)) {
2190                dev_err(dev->dev, "Slave mode cores are not supported\n");
2191                return -ENODEV;
2192        }
2193
2194        /* --- Effects of the following calls might need explicit cleanup --- */
2195
2196        /* Create DMA pool for descriptors */
2197        dev->desc_pool = dma_pool_create("desc_pool", dev->dev,
2198                                         sizeof(struct gr_dma_desc), 4, 0);
2199        if (!dev->desc_pool) {
2200                dev_err(dev->dev, "Could not allocate DMA pool");
2201                return -ENOMEM;
2202        }
2203
2204        spin_lock(&dev->lock);
2205
2206        /* Inside lock so that no gadget can use this udc until probe is done */
2207        retval = usb_add_gadget_udc(dev->dev, &dev->gadget);
2208        if (retval) {
2209                dev_err(dev->dev, "Could not add gadget udc");
2210                goto out;
2211        }
2212        dev->added = 1;
2213
2214        retval = gr_udc_init(dev);
2215        if (retval)
2216                goto out;
2217
2218        gr_dfs_create(dev);
2219
2220        /* Clear all interrupt enables that might be left on since last boot */
2221        gr_disable_interrupts_and_pullup(dev);
2222
2223        retval = gr_request_irq(dev, dev->irq);
2224        if (retval) {
2225                dev_err(dev->dev, "Failed to request irq %d\n", dev->irq);
2226                goto out;
2227        }
2228
2229        if (dev->irqi) {
2230                retval = gr_request_irq(dev, dev->irqi);
2231                if (retval) {
2232                        dev_err(dev->dev, "Failed to request irqi %d\n",
2233                                dev->irqi);
2234                        goto out;
2235                }
2236                retval = gr_request_irq(dev, dev->irqo);
2237                if (retval) {
2238                        dev_err(dev->dev, "Failed to request irqo %d\n",
2239                                dev->irqo);
2240                        goto out;
2241                }
2242        }
2243
2244        if (dev->irqi)
2245                dev_info(dev->dev, "regs: %p, irqs %d, %d, %d\n", dev->regs,
2246                         dev->irq, dev->irqi, dev->irqo);
2247        else
2248                dev_info(dev->dev, "regs: %p, irq %d\n", dev->regs, dev->irq);
2249
2250out:
2251        spin_unlock(&dev->lock);
2252
2253        if (retval)
2254                gr_remove(pdev);
2255
2256        return retval;
2257}
2258
2259static const struct of_device_id gr_match[] = {
2260        {.name = "GAISLER_USBDC"},
2261        {.name = "01_021"},
2262        {},
2263};
2264MODULE_DEVICE_TABLE(of, gr_match);
2265
2266static struct platform_driver gr_driver = {
2267        .driver = {
2268                .name = DRIVER_NAME,
2269                .of_match_table = gr_match,
2270        },
2271        .probe = gr_probe,
2272        .remove = gr_remove,
2273};
2274module_platform_driver(gr_driver);
2275
2276MODULE_AUTHOR("Aeroflex Gaisler AB.");
2277MODULE_DESCRIPTION(DRIVER_DESC);
2278MODULE_LICENSE("GPL");
2279