/*****************************************************************************
* Copyright 2004 - 2008 Broadcom Corporation.  All rights reserved.
*
* Unless you and Broadcom execute a separate written software license
* agreement governing use of this software, this software is licensed to you
* under the terms of the GNU General Public License version 2, available at
* http://www.broadcom.com/licenses/GPLv2.php (the "GPL").
*
* Notwithstanding the above, under no circumstances may you combine this
* software in any way with any other Broadcom software provided under a
* license other than the GPL, without Broadcom's express prior written
* consent.
*****************************************************************************/

/****************************************************************************/
/**
*   @file   dma.c
*
*   @brief  Implements the DMA interface.
*/
/****************************************************************************/

/* ---- Include Files ---------------------------------------------------- */

#include <linux/module.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/irqreturn.h>
#include <linux/proc_fs.h>

#include <mach/timer.h>

#include <linux/mm.h>
#include <linux/pfn.h>
#include <asm/atomic.h>
#include <mach/dma.h>

/* I don't quite understand why dc4 fails when this is set to 1 and DMA is enabled */
/* especially since dc4 doesn't use kmalloc'd memory. */

#define ALLOW_MAP_OF_KMALLOC_MEMORY 0

/* ---- Public Variables ------------------------------------------------- */

/* ---- Private Constants and Types -------------------------------------- */

#define MAKE_HANDLE(controllerIdx, channelIdx)    (((controllerIdx) << 4) | (channelIdx))

#define CONTROLLER_FROM_HANDLE(handle)    (((handle) >> 4) & 0x0f)
#define CHANNEL_FROM_HANDLE(handle)       ((handle) & 0x0f)

#define DMA_MAP_DEBUG   0

#if DMA_MAP_DEBUG
#   define  DMA_MAP_PRINT(fmt, args...)   printk("%s: " fmt, __func__,  ## args)
#else
#   define  DMA_MAP_PRINT(fmt, args...)
#endif

/* ---- Private Variables ------------------------------------------------ */

static DMA_Global_t gDMA;
static struct proc_dir_entry *gDmaDir;

static atomic_t gDmaStatMemTypeKmalloc = ATOMIC_INIT(0);
static atomic_t gDmaStatMemTypeVmalloc = ATOMIC_INIT(0);
static atomic_t gDmaStatMemTypeUser = ATOMIC_INIT(0);
static atomic_t gDmaStatMemTypeCoherent = ATOMIC_INIT(0);

#include "dma_device.c"

/* ---- Private Function Prototypes -------------------------------------- */

/* ---- Functions  ------------------------------------------------------- */

/****************************************************************************/
/**
*   Displays information for /proc/dma/mem-type
*/
/****************************************************************************/

static int dma_proc_read_mem_type(char *buf, char **start, off_t offset,
                                  int count, int *eof, void *data)
{
        int len = 0;

        len += sprintf(buf + len, "dma_map_mem statistics\n");
        len +=
            sprintf(buf + len, "coherent: %d\n",
                    atomic_read(&gDmaStatMemTypeCoherent));
        len +=
            sprintf(buf + len, "kmalloc:  %d\n",
                    atomic_read(&gDmaStatMemTypeKmalloc));
        len +=
            sprintf(buf + len, "vmalloc:  %d\n",
                    atomic_read(&gDmaStatMemTypeVmalloc));
        len +=
            sprintf(buf + len, "user:     %d\n",
                    atomic_read(&gDmaStatMemTypeUser));

        return len;
}

/****************************************************************************/
/**
*   Displays information for /proc/dma/channels
*/
/****************************************************************************/

static int dma_proc_read_channels(char *buf, char **start, off_t offset,
                                  int count, int *eof, void *data)
{
        int controllerIdx;
        int channelIdx;
        int limit = count - 200;
        int len = 0;
        DMA_Channel_t *channel;

        if (down_interruptible(&gDMA.lock) < 0) {
                return -ERESTARTSYS;
        }

        for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
             controllerIdx++) {
                for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
                     channelIdx++) {
                        if (len >= limit) {
                                break;
                        }

                        channel =
                            &gDMA.controller[controllerIdx].channel[channelIdx];

                        len +=
                            sprintf(buf + len, "%d:%d ", controllerIdx,
                                    channelIdx);

                        if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) !=
                            0) {
                                len +=
                                    sprintf(buf + len, "Dedicated for %s ",
                                            DMA_gDeviceAttribute[channel->
                                                                 devType].name);
                        } else {
                                len += sprintf(buf + len, "Shared ");
                        }

                        if ((channel->flags & DMA_CHANNEL_FLAG_NO_ISR) != 0) {
                                len += sprintf(buf + len, "No ISR ");
                        }

                        if ((channel->flags & DMA_CHANNEL_FLAG_LARGE_FIFO) != 0) {
                                len += sprintf(buf + len, "Fifo: 128 ");
                        } else {
                                len += sprintf(buf + len, "Fifo: 64  ");
                        }

                        if ((channel->flags & DMA_CHANNEL_FLAG_IN_USE) != 0) {
                                len +=
                                    sprintf(buf + len, "InUse by %s",
                                            DMA_gDeviceAttribute[channel->
                                                                 devType].name);
#if (DMA_DEBUG_TRACK_RESERVATION)
                                len +=
                                    sprintf(buf + len, " (%s:%d)",
                                            channel->fileName,
                                            channel->lineNum);
#endif
                        } else {
                                len += sprintf(buf + len, "Avail ");
                        }

                        if (channel->lastDevType != DMA_DEVICE_NONE) {
                                len +=
                                    sprintf(buf + len, "Last use: %s ",
                                            DMA_gDeviceAttribute[channel->
                                                                 lastDevType].
                                            name);
                        }

                        len += sprintf(buf + len, "\n");
                }
        }
        up(&gDMA.lock);
        *eof = 1;

        return len;
}

/****************************************************************************/
/**
*   Displays information for /proc/dma/devices
*/
/****************************************************************************/

static int dma_proc_read_devices(char *buf, char **start, off_t offset,
                                 int count, int *eof, void *data)
{
        int limit = count - 200;
        int len = 0;
        int devIdx;

        if (down_interruptible(&gDMA.lock) < 0) {
                return -ERESTARTSYS;
        }

        for (devIdx = 0; devIdx < DMA_NUM_DEVICE_ENTRIES; devIdx++) {
                DMA_DeviceAttribute_t *devAttr = &DMA_gDeviceAttribute[devIdx];

                if (devAttr->name == NULL) {
                        continue;
                }

                if (len >= limit) {
                        break;
                }

                len += sprintf(buf + len, "%-12s ", devAttr->name);

                if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
                        len +=
                            sprintf(buf + len, "Dedicated %d:%d ",
                                    devAttr->dedicatedController,
                                    devAttr->dedicatedChannel);
                } else {
                        len += sprintf(buf + len, "Shared DMA:");
                        if ((devAttr->flags & DMA_DEVICE_FLAG_ON_DMA0) != 0) {
                                len += sprintf(buf + len, "0");
                        }
                        if ((devAttr->flags & DMA_DEVICE_FLAG_ON_DMA1) != 0) {
                                len += sprintf(buf + len, "1");
                        }
                        len += sprintf(buf + len, " ");
                }
                if ((devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) != 0) {
                        len += sprintf(buf + len, "NoISR ");
                }
                if ((devAttr->flags & DMA_DEVICE_FLAG_ALLOW_LARGE_FIFO) != 0) {
                        len += sprintf(buf + len, "Allow-128 ");
                }

                len +=
                    sprintf(buf + len,
                            "Xfer #: %Lu Ticks: %Lu Bytes: %Lu DescLen: %u\n",
                            devAttr->numTransfers, devAttr->transferTicks,
                            devAttr->transferBytes,
                            devAttr->ring.bytesAllocated);

        }

        up(&gDMA.lock);
        *eof = 1;

        return len;
}

/****************************************************************************/
/**
*   Determines if a DMA_Device_t is "valid".
*
*   @return
*       TRUE        - dma device is valid
*       FALSE       - dma device isn't valid
*/
/****************************************************************************/

static inline int IsDeviceValid(DMA_Device_t device)
{
        return (device >= 0) && (device < DMA_NUM_DEVICE_ENTRIES);
}

/****************************************************************************/
/**
*   Translates a DMA handle into a pointer to a channel.
*
*   @return
*       non-NULL    - pointer to DMA_Channel_t
*       NULL        - DMA Handle was invalid
*/
/****************************************************************************/

static inline DMA_Channel_t *HandleToChannel(DMA_Handle_t handle)
{
        int controllerIdx;
        int channelIdx;

        controllerIdx = CONTROLLER_FROM_HANDLE(handle);
        channelIdx = CHANNEL_FROM_HANDLE(handle);

        if ((controllerIdx > DMA_NUM_CONTROLLERS)
            || (channelIdx > DMA_NUM_CHANNELS)) {
                return NULL;
        }
        return &gDMA.controller[controllerIdx].channel[channelIdx];
}

/****************************************************************************/
/**
*   Interrupt handler which is called to process DMA interrupts.
*/
/****************************************************************************/

static irqreturn_t dma_interrupt_handler(int irq, void *dev_id)
{
        DMA_Channel_t *channel;
        DMA_DeviceAttribute_t *devAttr;
        int irqStatus;

        channel = (DMA_Channel_t *) dev_id;

        /* Figure out why we were called, and knock down the interrupt */

        irqStatus = dmacHw_getInterruptStatus(channel->dmacHwHandle);
        dmacHw_clearInterrupt(channel->dmacHwHandle);

        if ((channel->devType < 0)
            || (channel->devType > DMA_NUM_DEVICE_ENTRIES)) {
                printk(KERN_ERR "dma_interrupt_handler: Invalid devType: %d\n",
                       channel->devType);
                return IRQ_NONE;
        }
        devAttr = &DMA_gDeviceAttribute[channel->devType];

        /* Update stats */

        if ((irqStatus & dmacHw_INTERRUPT_STATUS_TRANS) != 0) {
                devAttr->transferTicks +=
                    (timer_get_tick_count() - devAttr->transferStartTime);
        }

        if ((irqStatus & dmacHw_INTERRUPT_STATUS_ERROR) != 0) {
                printk(KERN_ERR
                       "dma_interrupt_handler: devType :%d DMA error (%s)\n",
                       channel->devType, devAttr->name);
        } else {
                devAttr->numTransfers++;
                devAttr->transferBytes += devAttr->numBytes;
        }

        /* Call any installed handler */

        if (devAttr->devHandler != NULL) {
                devAttr->devHandler(channel->devType, irqStatus,
                                    devAttr->userData);
        }

        return IRQ_HANDLED;
}

/****************************************************************************/
/**
*   Allocates memory to hold a descriptor ring. The descriptor ring then
*   needs to be populated by making one or more calls to
*   dna_add_descriptors.
*
*   The returned descriptor ring will be automatically initialized.
*
*   @return
*       0           Descriptor ring was allocated successfully
*       -EINVAL     Invalid parameters passed in
*       -ENOMEM     Unable to allocate memory for the desired number of descriptors.
*/
/****************************************************************************/

int dma_alloc_descriptor_ring(DMA_DescriptorRing_t *ring,       /* Descriptor ring to populate */
                              int numDescriptors        /* Number of descriptors that need to be allocated. */
    ) {
        size_t bytesToAlloc = dmacHw_descriptorLen(numDescriptors);

        if ((ring == NULL) || (numDescriptors <= 0)) {
                return -EINVAL;
        }

        ring->physAddr = 0;
        ring->descriptorsAllocated = 0;
        ring->bytesAllocated = 0;

        ring->virtAddr = dma_alloc_writecombine(NULL,
                                                     bytesToAlloc,
                                                     &ring->physAddr,
                                                     GFP_KERNEL);
        if (ring->virtAddr == NULL) {
                return -ENOMEM;
        }

        ring->bytesAllocated = bytesToAlloc;
        ring->descriptorsAllocated = numDescriptors;

        return dma_init_descriptor_ring(ring, numDescriptors);
}

EXPORT_SYMBOL(dma_alloc_descriptor_ring);

/****************************************************************************/
/**
*   Releases the memory which was previously allocated for a descriptor ring.
*/
/****************************************************************************/

void dma_free_descriptor_ring(DMA_DescriptorRing_t *ring        /* Descriptor to release */
    ) {
        if (ring->virtAddr != NULL) {
                dma_free_writecombine(NULL,
                                      ring->bytesAllocated,
                                      ring->virtAddr, ring->physAddr);
        }

        ring->bytesAllocated = 0;
        ring->descriptorsAllocated = 0;
        ring->virtAddr = NULL;
        ring->physAddr = 0;
}

EXPORT_SYMBOL(dma_free_descriptor_ring);

/****************************************************************************/
/**
*   Initializes a descriptor ring, so that descriptors can be added to it.
*   Once a descriptor ring has been allocated, it may be reinitialized for
*   use with additional/different regions of memory.
*
*   Note that if 7 descriptors are allocated, it's perfectly acceptable to
*   initialize the ring with a smaller number of descriptors. The amount
*   of memory allocated for the descriptor ring will not be reduced, and
*   the descriptor ring may be reinitialized later
*
*   @return
*       0           Descriptor ring was initialized successfully
*       -ENOMEM     The descriptor which was passed in has insufficient space
*                   to hold the desired number of descriptors.
*/
/****************************************************************************/

int dma_init_descriptor_ring(DMA_DescriptorRing_t *ring,        /* Descriptor ring to initialize */
                             int numDescriptors /* Number of descriptors to initialize. */
    ) {
        if (ring->virtAddr == NULL) {
                return -EINVAL;
        }
        if (dmacHw_initDescriptor(ring->virtAddr,
                                  ring->physAddr,
                                  ring->bytesAllocated, numDescriptors) < 0) {
                printk(KERN_ERR
                       "dma_init_descriptor_ring: dmacHw_initDescriptor failed\n");
                return -ENOMEM;
        }

        return 0;
}

EXPORT_SYMBOL(dma_init_descriptor_ring);

/****************************************************************************/
/**
*   Determines the number of descriptors which would be required for a
*   transfer of the indicated memory region.
*
*   This function also needs to know which DMA device this transfer will
*   be destined for, so that the appropriate DMA configuration can be retrieved.
*   DMA parameters such as transfer width, and whether this is a memory-to-memory
*   or memory-to-peripheral, etc can all affect the actual number of descriptors
*   required.
*
*   @return
*       > 0     Returns the number of descriptors required for the indicated transfer
*       -ENODEV - Device handed in is invalid.
*       -EINVAL Invalid parameters
*       -ENOMEM Memory exhausted
*/
/****************************************************************************/

int dma_calculate_descriptor_count(DMA_Device_t device, /* DMA Device that this will be associated with */
                                   dma_addr_t srcData,  /* Place to get data to write to device */
                                   dma_addr_t dstData,  /* Pointer to device data address */
                                   size_t numBytes      /* Number of bytes to transfer to the device */
    ) {
        int numDescriptors;
        DMA_DeviceAttribute_t *devAttr;

        if (!IsDeviceValid(device)) {
                return -ENODEV;
        }
        devAttr = &DMA_gDeviceAttribute[device];

        numDescriptors = dmacHw_calculateDescriptorCount(&devAttr->config,
                                                              (void *)srcData,
                                                              (void *)dstData,
                                                              numBytes);
        if (numDescriptors < 0) {
                printk(KERN_ERR
                       "dma_calculate_descriptor_count: dmacHw_calculateDescriptorCount failed\n");
                return -EINVAL;
        }

        return numDescriptors;
}

EXPORT_SYMBOL(dma_calculate_descriptor_count);

/****************************************************************************/
/**
*   Adds a region of memory to the descriptor ring. Note that it may take
*   multiple descriptors for each region of memory. It is the callers
*   responsibility to allocate a sufficiently large descriptor ring.
*
*   @return
*       0       Descriptors were added successfully
*       -ENODEV Device handed in is invalid.
*       -EINVAL Invalid parameters
*       -ENOMEM Memory exhausted
*/
/****************************************************************************/

int dma_add_descriptors(DMA_DescriptorRing_t *ring,     /* Descriptor ring to add descriptors to */
                        DMA_Device_t device,    /* DMA Device that descriptors are for */
                        dma_addr_t srcData,     /* Place to get data (memory or device) */
                        dma_addr_t dstData,     /* Place to put data (memory or device) */
                        size_t numBytes /* Number of bytes to transfer to the device */
    ) {
        int rc;
        DMA_DeviceAttribute_t *devAttr;

        if (!IsDeviceValid(device)) {
                return -ENODEV;
        }
        devAttr = &DMA_gDeviceAttribute[device];

        rc = dmacHw_setDataDescriptor(&devAttr->config,
                                      ring->virtAddr,
                                      (void *)srcData,
                                      (void *)dstData, numBytes);
        if (rc < 0) {
                printk(KERN_ERR
                       "dma_add_descriptors: dmacHw_setDataDescriptor failed with code: %d\n",
                       rc);
                return -ENOMEM;
        }

        return 0;
}

EXPORT_SYMBOL(dma_add_descriptors);

/****************************************************************************/
/**
*   Sets the descriptor ring associated with a device.
*
*   Once set, the descriptor ring will be associated with the device, even
*   across channel request/free calls. Passing in a NULL descriptor ring
*   will release any descriptor ring currently associated with the device.
*
*   Note: If you call dma_transfer, or one of the other dma_alloc_ functions
*         the descriptor ring may be released and reallocated.
*
*   Note: This function will release the descriptor memory for any current
*         descriptor ring associated with this device.
*
*   @return
*       0       Descriptors were added successfully
*       -ENODEV Device handed in is invalid.
*/
/****************************************************************************/

int dma_set_device_descriptor_ring(DMA_Device_t device, /* Device to update the descriptor ring for. */
                                   DMA_DescriptorRing_t *ring   /* Descriptor ring to add descriptors to */
    ) {
        DMA_DeviceAttribute_t *devAttr;

        if (!IsDeviceValid(device)) {
                return -ENODEV;
        }
        devAttr = &DMA_gDeviceAttribute[device];

        /* Free the previously allocated descriptor ring */

        dma_free_descriptor_ring(&devAttr->ring);

        if (ring != NULL) {
                /* Copy in the new one */

                devAttr->ring = *ring;
        }

        /* Set things up so that if dma_transfer is called then this descriptor */
        /* ring will get freed. */

        devAttr->prevSrcData = 0;
        devAttr->prevDstData = 0;
        devAttr->prevNumBytes = 0;

        return 0;
}

EXPORT_SYMBOL(dma_set_device_descriptor_ring);

/****************************************************************************/
/**
*   Retrieves the descriptor ring associated with a device.
*
*   @return
*       0       Descriptors were added successfully
*       -ENODEV Device handed in is invalid.
*/
/****************************************************************************/

int dma_get_device_descriptor_ring(DMA_Device_t device, /* Device to retrieve the descriptor ring for. */
                                   DMA_DescriptorRing_t *ring   /* Place to store retrieved ring */
    ) {
        DMA_DeviceAttribute_t *devAttr;

        memset(ring, 0, sizeof(*ring));

        if (!IsDeviceValid(device)) {
                return -ENODEV;
        }
        devAttr = &DMA_gDeviceAttribute[device];

        *ring = devAttr->ring;

        return 0;
}

EXPORT_SYMBOL(dma_get_device_descriptor_ring);

/****************************************************************************/
/**
*   Configures a DMA channel.
*
*   @return
*       >= 0    - Initialization was successfull.
*
*       -EBUSY  - Device is currently being used.
*       -ENODEV - Device handed in is invalid.
*/
/****************************************************************************/

static int ConfigChannel(DMA_Handle_t handle)
{
        DMA_Channel_t *channel;
        DMA_DeviceAttribute_t *devAttr;
        int controllerIdx;

        channel = HandleToChannel(handle);
        if (channel == NULL) {
                return -ENODEV;
        }
        devAttr = &DMA_gDeviceAttribute[channel->devType];
        controllerIdx = CONTROLLER_FROM_HANDLE(handle);

        if ((devAttr->flags & DMA_DEVICE_FLAG_PORT_PER_DMAC) != 0) {
                if (devAttr->config.transferType ==
                    dmacHw_TRANSFER_TYPE_MEM_TO_PERIPHERAL) {
                        devAttr->config.dstPeripheralPort =
                            devAttr->dmacPort[controllerIdx];
                } else if (devAttr->config.transferType ==
                           dmacHw_TRANSFER_TYPE_PERIPHERAL_TO_MEM) {
                        devAttr->config.srcPeripheralPort =
                            devAttr->dmacPort[controllerIdx];
                }
        }

        if (dmacHw_configChannel(channel->dmacHwHandle, &devAttr->config) != 0) {
                printk(KERN_ERR "ConfigChannel: dmacHw_configChannel failed\n");
                return -EIO;
        }

        return 0;
}

/****************************************************************************/
/**
*   Intializes all of the data structures associated with the DMA.
*   @return
*       >= 0    - Initialization was successfull.
*
*       -EBUSY  - Device is currently being used.
*       -ENODEV - Device handed in is invalid.
*/
/****************************************************************************/

int dma_init(void)
{
        int rc = 0;
        int controllerIdx;
        int channelIdx;
        DMA_Device_t devIdx;
        DMA_Channel_t *channel;
        DMA_Handle_t dedicatedHandle;

        memset(&gDMA, 0, sizeof(gDMA));

        init_MUTEX_LOCKED(&gDMA.lock);
        init_waitqueue_head(&gDMA.freeChannelQ);

        /* Initialize the Hardware */

        dmacHw_initDma();

        /* Start off by marking all of the DMA channels as shared. */

        for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
             controllerIdx++) {
                for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
                     channelIdx++) {
                        channel =
                            &gDMA.controller[controllerIdx].channel[channelIdx];

                        channel->flags = 0;
                        channel->devType = DMA_DEVICE_NONE;
                        channel->lastDevType = DMA_DEVICE_NONE;

#if (DMA_DEBUG_TRACK_RESERVATION)
                        channel->fileName = "";
                        channel->lineNum = 0;
#endif

                        channel->dmacHwHandle =
                            dmacHw_getChannelHandle(dmacHw_MAKE_CHANNEL_ID
                                                    (controllerIdx,
                                                     channelIdx));
                        dmacHw_initChannel(channel->dmacHwHandle);
                }
        }

        /* Record any special attributes that channels may have */

        gDMA.controller[0].channel[0].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
        gDMA.controller[0].channel[1].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
        gDMA.controller[1].channel[0].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;
        gDMA.controller[1].channel[1].flags |= DMA_CHANNEL_FLAG_LARGE_FIFO;

        /* Now walk through and record the dedicated channels. */

        for (devIdx = 0; devIdx < DMA_NUM_DEVICE_ENTRIES; devIdx++) {
                DMA_DeviceAttribute_t *devAttr = &DMA_gDeviceAttribute[devIdx];

                if (((devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) != 0)
                    && ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) == 0)) {
                        printk(KERN_ERR
                               "DMA Device: %s Can only request NO_ISR for dedicated devices\n",
                               devAttr->name);
                        rc = -EINVAL;
                        goto out;
                }

                if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
                        /* This is a dedicated device. Mark the channel as being reserved. */

                        if (devAttr->dedicatedController >= DMA_NUM_CONTROLLERS) {
                                printk(KERN_ERR
                                       "DMA Device: %s DMA Controller %d is out of range\n",
                                       devAttr->name,
                                       devAttr->dedicatedController);
                                rc = -EINVAL;
                                goto out;
                        }

                        if (devAttr->dedicatedChannel >= DMA_NUM_CHANNELS) {
                                printk(KERN_ERR
                                       "DMA Device: %s DMA Channel %d is out of range\n",
                                       devAttr->name,
                                       devAttr->dedicatedChannel);
                                rc = -EINVAL;
                                goto out;
                        }

                        dedicatedHandle =
                            MAKE_HANDLE(devAttr->dedicatedController,
                                        devAttr->dedicatedChannel);
                        channel = HandleToChannel(dedicatedHandle);

                        if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) !=
                            0) {
                                printk
                                    ("DMA Device: %s attempting to use same DMA Controller:Channel (%d:%d) as %s\n",
                                     devAttr->name,
                                     devAttr->dedicatedController,
                                     devAttr->dedicatedChannel,
                                     DMA_gDeviceAttribute[channel->devType].
                                     name);
                                rc = -EBUSY;
                                goto out;
                        }

                        channel->flags |= DMA_CHANNEL_FLAG_IS_DEDICATED;
                        channel->devType = devIdx;

                        if (devAttr->flags & DMA_DEVICE_FLAG_NO_ISR) {
                                channel->flags |= DMA_CHANNEL_FLAG_NO_ISR;
                        }

                        /* For dedicated channels, we can go ahead and configure the DMA channel now */
                        /* as well. */

                        ConfigChannel(dedicatedHandle);
                }
        }

        /* Go through and register the interrupt handlers */

        for (controllerIdx = 0; controllerIdx < DMA_NUM_CONTROLLERS;
             controllerIdx++) {
                for (channelIdx = 0; channelIdx < DMA_NUM_CHANNELS;
                     channelIdx++) {
                        channel =
                            &gDMA.controller[controllerIdx].channel[channelIdx];

                        if ((channel->flags & DMA_CHANNEL_FLAG_NO_ISR) == 0) {
                                snprintf(channel->name, sizeof(channel->name),
                                         "dma %d:%d %s", controllerIdx,
                                         channelIdx,
                                         channel->devType ==
                                         DMA_DEVICE_NONE ? "" :
                                         DMA_gDeviceAttribute[channel->devType].
                                         name);

                                rc =
                                     request_irq(IRQ_DMA0C0 +
                                                 (controllerIdx *
                                                  DMA_NUM_CHANNELS) +
                                                 channelIdx,
                                                 dma_interrupt_handler,
                                                 IRQF_DISABLED, channel->name,
                                                 channel);
                                if (rc != 0) {
                                        printk(KERN_ERR
                                               "request_irq for IRQ_DMA%dC%d failed\n",
                                               controllerIdx, channelIdx);
                                }
                        }
                }
        }

        /* Create /proc/dma/channels and /proc/dma/devices */

        gDmaDir = create_proc_entry("dma", S_IFDIR | S_IRUGO | S_IXUGO, NULL);

        if (gDmaDir == NULL) {
                printk(KERN_ERR "Unable to create /proc/dma\n");
        } else {
                create_proc_read_entry("channels", 0, gDmaDir,
                                       dma_proc_read_channels, NULL);
                create_proc_read_entry("devices", 0, gDmaDir,
                                       dma_proc_read_devices, NULL);
                create_proc_read_entry("mem-type", 0, gDmaDir,
                                       dma_proc_read_mem_type, NULL);
        }

out:

        up(&gDMA.lock);

        return rc;
}

/****************************************************************************/
/**
*   Reserves a channel for use with @a dev. If the device is setup to use
*   a shared channel, then this function will block until a free channel
*   becomes available.
*
*   @return
*       >= 0    - A valid DMA Handle.
*       -EBUSY  - Device is currently being used.
*       -ENODEV - Device handed in is invalid.
*/
/****************************************************************************/

#if (DMA_DEBUG_TRACK_RESERVATION)
DMA_Handle_t dma_request_channel_dbg
    (DMA_Device_t dev, const char *fileName, int lineNum)
#else
DMA_Handle_t dma_request_channel(DMA_Device_t dev)
#endif
{
        DMA_Handle_t handle;
        DMA_DeviceAttribute_t *devAttr;
        DMA_Channel_t *channel;
        int controllerIdx;
        int controllerIdx2;
        int channelIdx;

        if (down_interruptible(&gDMA.lock) < 0) {
                return -ERESTARTSYS;
        }

        if ((dev < 0) || (dev >= DMA_NUM_DEVICE_ENTRIES)) {
                handle = -ENODEV;
                goto out;
        }
        devAttr = &DMA_gDeviceAttribute[dev];

#if (DMA_DEBUG_TRACK_RESERVATION)
        {
                char *s;

                s = strrchr(fileName, '/');
                if (s != NULL) {
                        fileName = s + 1;
                }
        }
#endif
        if ((devAttr->flags & DMA_DEVICE_FLAG_IN_USE) != 0) {
                /* This device has already been requested and not been freed */

                printk(KERN_ERR "%s: device %s is already requested\n",
                       __func__, devAttr->name);
                handle = -EBUSY;
                goto out;
        }

        if ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) != 0) {
                /* This device has a dedicated channel. */

                channel =
                    &gDMA.controller[devAttr->dedicatedController].
                    channel[devAttr->dedicatedChannel];
                if ((channel->flags & DMA_CHANNEL_FLAG_IN_USE) != 0) {
                        handle = -EBUSY;
                        goto out;
                }

                channel->flags |= DMA_CHANNEL_FLAG_IN_USE;
                devAttr->flags |= DMA_DEVICE_FLAG_IN_USE;

#if (DMA_DEBUG_TRACK_RESERVATION)
                channel->fileName = fileName;
                channel->lineNum = lineNum;
#endif
                handle =
                    MAKE_HANDLE(devAttr->dedicatedController,
                                devAttr->dedicatedChannel);
                goto out;
        }

        /* This device needs to use one of the shared channels. */

        handle = DMA_INVALID_HANDLE;
        while (handle == DMA_INVALID_HANDLE) {
                /* Scan through the shared channels and see if one is available */

                for (controllerIdx2 = 0; controllerIdx2 < DMA_NUM_CONTROLLERS;
                     controllerIdx2++) {
                        /* Check to see if we should try on controller 1 first. */

                        controllerIdx = controllerIdx2;
                        if ((devAttr->
                             flags & DMA_DEVICE_FLAG_ALLOC_DMA1_FIRST) != 0) {
                                controllerIdx = 1 - controllerIdx;
                        }

                        /* See if the device is available on the controller being tested */

                        if ((devAttr->
                             flags & (DMA_DEVICE_FLAG_ON_DMA0 << controllerIdx))
                            != 0) {
                                for (channelIdx = 0;
                                     channelIdx < DMA_NUM_CHANNELS;
                                     channelIdx++) {
                                        channel =
                                            &gDMA.controller[controllerIdx].
                                            channel[channelIdx];

                                        if (((channel->
                                              flags &
                                              DMA_CHANNEL_FLAG_IS_DEDICATED) ==
                                             0)
                                            &&
                                            ((channel->
                                              flags & DMA_CHANNEL_FLAG_IN_USE)
                                             == 0)) {
                                                if (((channel->
                                                      flags &
                                                      DMA_CHANNEL_FLAG_LARGE_FIFO)
                                                     != 0)
                                                    &&
                                                    ((devAttr->
                                                      flags &
                                                      DMA_DEVICE_FLAG_ALLOW_LARGE_FIFO)
                                                     == 0)) {
                                                        /* This channel is a large fifo - don't tie it up */
                                                        /* with devices that we don't want using it. */

                                                        continue;
                                                }

                                                channel->flags |=
                                                    DMA_CHANNEL_FLAG_IN_USE;
                                                channel->devType = dev;
                                                devAttr->flags |=
                                                    DMA_DEVICE_FLAG_IN_USE;

#if (DMA_DEBUG_TRACK_RESERVATION)
                                                channel->fileName = fileName;
                                                channel->lineNum = lineNum;
#endif
                                                handle =
                                                    MAKE_HANDLE(controllerIdx,
                                                                channelIdx);

                                                /* Now that we've reserved the channel - we can go ahead and configure it */

                                                if (ConfigChannel(handle) != 0) {
                                                        handle = -EIO;
                                                        printk(KERN_ERR
                                                               "dma_request_channel: ConfigChannel failed\n");
                                                }
                                                goto out;
                                        }
                                }
                        }
                }

                /* No channels are currently available. Let's wait for one to free up. */

                {
                        DEFINE_WAIT(wait);

                        prepare_to_wait(&gDMA.freeChannelQ, &wait,
                                        TASK_INTERRUPTIBLE);
                        up(&gDMA.lock);
                        schedule();
                        finish_wait(&gDMA.freeChannelQ, &wait);

                        if (signal_pending(current)) {
                                /* We don't currently hold gDMA.lock, so we return directly */

                                return -ERESTARTSYS;
                        }
                }

                if (down_interruptible(&gDMA.lock)) {
                        return -ERESTARTSYS;
                }
        }

out:
        up(&gDMA.lock);

        return handle;
}

/* Create both _dbg and non _dbg functions for modules. */

#if (DMA_DEBUG_TRACK_RESERVATION)
#undef dma_request_channel
DMA_Handle_t dma_request_channel(DMA_Device_t dev)
{
        return dma_request_channel_dbg(dev, __FILE__, __LINE__);
}

EXPORT_SYMBOL(dma_request_channel_dbg);
#endif
EXPORT_SYMBOL(dma_request_channel);

/****************************************************************************/
/**
*   Frees a previously allocated DMA Handle.
*/
/****************************************************************************/

int dma_free_channel(DMA_Handle_t handle        /* DMA handle. */
    ) {
        int rc = 0;
        DMA_Channel_t *channel;
        DMA_DeviceAttribute_t *devAttr;

        if (down_interruptible(&gDMA.lock) < 0) {
                return -ERESTARTSYS;
        }

        channel = HandleToChannel(handle);
        if (channel == NULL) {
                rc = -EINVAL;
                goto out;
        }

        devAttr = &DMA_gDeviceAttribute[channel->devType];

        if ((channel->flags & DMA_CHANNEL_FLAG_IS_DEDICATED) == 0) {
                channel->lastDevType = channel->devType;
                channel->devType = DMA_DEVICE_NONE;
        }
        channel->flags &= ~DMA_CHANNEL_FLAG_IN_USE;
        devAttr->flags &= ~DMA_DEVICE_FLAG_IN_USE;

out:
        up(&gDMA.lock);

        wake_up_interruptible(&gDMA.freeChannelQ);

        return rc;
}

EXPORT_SYMBOL(dma_free_channel);

/****************************************************************************/
/**
*   Determines if a given device has been configured as using a shared
*   channel.
*
*   @return
*       0           Device uses a dedicated channel
*       > zero      Device uses a shared channel
*       < zero      Error code
*/
/****************************************************************************/

int dma_device_is_channel_shared(DMA_Device_t device    /* Device to check. */
    ) {
        DMA_DeviceAttribute_t *devAttr;

        if (!IsDeviceValid(device)) {
                return -ENODEV;
        }
        devAttr = &DMA_gDeviceAttribute[device];

        return ((devAttr->flags & DMA_DEVICE_FLAG_IS_DEDICATED) == 0);
}

EXPORT_SYMBOL(dma_device_is_channel_shared);

/****************************************************************************/
/**
*   Allocates buffers for the descriptors. This is normally done automatically
*   but needs to be done explicitly when initiating a dma from interrupt
*   context.
*
*   @return
*       0       Descriptors were allocated successfully
*       -EINVAL Invalid device type for this kind of transfer
*               (i.e. the device is _MEM_TO_DEV and not _DEV_TO_MEM)
*       -ENOMEM Memory exhausted
*/
/****************************************************************************/

int dma_alloc_descriptors(DMA_Handle_t handle,  /* DMA Handle */
                          dmacHw_TRANSFER_TYPE_e transferType,  /* Type of transfer being performed */
                          dma_addr_t srcData,   /* Place to get data to write to device */
                          dma_addr_t dstData,   /* Pointer to device data address */
                          size_t numBytes       /* Number of bytes to transfer to the device */
    ) {
        DMA_Channel_t *channel;
        DMA_DeviceAttribute_t *devAttr;
        int numDescriptors;
        size_t ringBytesRequired;
        int rc = 0;

        channel = HandleToChannel(handle);
        if (channel == NULL) {
                return -ENODEV;
        }

        devAttr = &DMA_gDeviceAttribute[channel->devType];

        if (devAttr->config.transferType != transferType) {
                return -EINVAL;
        }

        /* Figure out how many descriptors we need. */

        /* printk("srcData: 0x%08x dstData: 0x%08x, numBytes: %d\n", */
        /*        srcData, dstData, numBytes); */

        numDescriptors = dmacHw_calculateDescriptorCount(&devAttr->config,
                                                              (void *)srcData,
                                                              (void *)dstData,
                                                              numBytes);
        if (numDescriptors < 0) {
                printk(KERN_ERR "%s: dmacHw_calculateDescriptorCount failed\n",
                       __func__);
                return -EINVAL;
        }

        /* Check to see if we can reuse the existing descriptor ring, or if we need to allocate */
        /* a new one. */

        ringBytesRequired = dmacHw_descriptorLen(numDescriptors);

        /* printk("ringBytesRequired: %d\n", ringBytesRequired); */

        if (ringBytesRequired > devAttr->ring.bytesAllocated) {
                /* Make sure that this code path is never taken from interrupt context. */
                /* It's OK for an interrupt to initiate a DMA transfer, but the descriptor */
                /* allocation needs to have already been done. */

                might_sleep();

                /* Free the old descriptor ring and allocate a new one. */

                dma_free_descriptor_ring(&devAttr->ring);

                /* And allocate a new one. */

                rc =
                     dma_alloc_descriptor_ring(&devAttr->ring,
                                               numDescriptors);
                if (rc < 0) {
                        printk(KERN_ERR
                               "%s: dma_alloc_descriptor_ring(%d) failed\n",
                               __func__, numDescriptors);
                        return rc;
                }
                /* Setup the descriptor for this transfer */

                if (dmacHw_initDescriptor(devAttr->ring.virtAddr,
                                          devAttr->ring.physAddr,
                                          devAttr->ring.bytesAllocated,
                                          numDescriptors) < 0) {
                        printk(KERN_ERR "%s: dmacHw_initDescriptor failed\n",
                               __func__);
                        return -EINVAL;
                }
        } else {
                /* We've already got enough ring buffer allocated. All we need to do is reset */
                /* any control information, just in case the previous DMA was stopped. */

                dmacHw_resetDescriptorControl(devAttr->ring.virtAddr);
        }

        /* dma_alloc/free both set the prevSrc/DstData to 0. If they happen to be the same */
        /* as last time, then we don't need to call setDataDescriptor again. */

        if (dmacHw_setDataDescriptor(&devAttr->config,
                                     devAttr->ring.virtAddr,
                                     (void *)srcData,
                                     (void *)dstData, numBytes) < 0) {
                printk(KERN_ERR "%s: dmacHw_setDataDescriptor failed\n",
                       __func__);
                return -EINVAL;
        }

        /* Remember the critical information for this transfer so that we can eliminate */
        /* another call to dma_alloc_descriptors if the caller reuses the same buffers */

        devAttr->prevSrcData = srcData;
        devAttr->prevDstData = dstData;
        devAttr->prevNumBytes = numBytes;

        return 0;
}

EXPORT_SYMBOL(dma_alloc_descriptors);

/****************************************************************************/
/**
*   Allocates and sets up descriptors for a double buffered circular buffer.
*
*   This is primarily intended to be used for things like the ingress samples
*   from a microphone.
*
*   @return
*       > 0     Number of descriptors actually allocated.
*       -EINVAL Invalid device type for this kind of transfer
*               (i.e. the device is _MEM_TO_DEV and not _DEV_TO_MEM)
*       -ENOMEM Memory exhausted
*/
/****************************************************************************/

int dma_alloc_double_dst_descriptors(DMA_Handle_t handle,       /* DMA Handle */
                                     dma_addr_t srcData,        /* Physical address of source data */
                                     dma_addr_t dstData1,       /* Physical address of first destination buffer */
                                     dma_addr_t dstData2,       /* Physical address of second destination buffer */
                                     size_t numBytes    /* Number of bytes in each destination buffer */
    ) {
        DMA_Channel_t *channel;
        DMA_DeviceAttribute_t *devAttr;
        int numDst1Descriptors;
        int numDst2Descriptors;
        int numDescriptors;
        size_t ringBytesRequired;
        int rc = 0;

        channel = HandleToChannel(handle);
        if (channel == NULL) {
                return -ENODEV;
        }

        devAttr = &DMA_gDeviceAttribute[channel->devType];

        /* Figure out how many descriptors we need. */

        /* printk("srcData: 0x%08x dstData: 0x%08x, numBytes: %d\n", */
        /*        srcData, dstData, numBytes); */

        numDst1Descriptors =
             dmacHw_calculateDescriptorCount(&devAttr->config, (void *)srcData,
                                             (void *)dstData1, numBytes);
        if (numDst1Descriptors < 0) {
                return -EINVAL;
        }
        numDst2Descriptors =
             dmacHw_calculateDescriptorCount(&devAttr->config, (void *)srcData,
                                             (void *)dstData2, numBytes);
        if (numDst2Descriptors < 0) {
                return -EINVAL;
        }
        numDescriptors = numDst1Descriptors + numDst2Descriptors;
        /* printk("numDescriptors: %d\n", numDescriptors); */

        /* Check to see if we can reuse the existing descriptor ring, or if we need to allocate */
        /* a new one. */

        ringBytesRequired = dmacHw_descriptorLen(numDescriptors);

        /* printk("ringBytesRequired: %d\n", ringBytesRequired); */

        if (ringBytesRequired > devAttr->ring.bytesAllocated) {
                /* Make sure that this code path is never taken from interrupt context. */
                /* It's OK for an interrupt to initiate a DMA transfer, but the descriptor */
                /* allocation needs to have already been done. */

                might_sleep();

                /* Free the old descriptor ring and allocate a new one. */

                dma_free_descriptor_ring(&devAttr->ring);

                /* And allocate a new one. */

                rc =
                     dma_alloc_descriptor_ring(&devAttr->ring,
                                               numDescriptors);
                if (rc < 0) {
                        printk(KERN_ERR
                               "%s: dma_alloc_descriptor_ring(%d) failed\n",
                               __func__, ringBytesRequired);
                        return rc;
                }
        }

        /* Setup the descriptor for this transfer. Since this function is used with */
        /* CONTINUOUS DMA operations, we need to reinitialize every time, otherwise */
        /* setDataDescriptor will keep trying to append onto the end. */

        if (dmacHw_initDescriptor(devAttr->ring.virtAddr,
                                  devAttr->ring.physAddr,
                                  devAttr->ring.bytesAllocated,
                                  numDescriptors) < 0) {
                printk(KERN_ERR "%s: dmacHw_initDescriptor failed\n", __func__);
                return -EINVAL;
        }

        /* dma_alloc/free both set the prevSrc/DstData to 0. If they happen to be the same */
        /* as last time, then we don't need to call setDataDescriptor again. */

        if (dmacHw_setDataDescriptor(&devAttr->config,
                                     devAttr->ring.virtAddr,
                                     (void *)srcData,
                                     (void *)dstData1, numBytes) < 0) {
                printk(KERN_ERR "%s: dmacHw_setDataDescriptor 1 failed\n",
                       __func__);
                return -EINVAL;
        }
        if (dmacHw_setDataDescriptor(&devAttr->config,
                                     devAttr->ring.virtAddr,
                                     (void *)srcData,
                                     (void *)dstData2, numBytes) < 0) {
                printk(KERN_ERR "%s: dmacHw_setDataDescriptor 2 failed\n",
                       __func__);
                return -EINVAL;
        }

        /* You should use dma_start_transfer rather than dma_transfer_xxx so we don't */
        /* try to make the 'prev' variables right. */

        devAttr->prevSrcData = 0;
        devAttr->prevDstData = 0;
        devAttr->prevNumBytes = 0;

        return numDescriptors;
}

EXPORT_SYMBOL(dma_alloc_double_dst_descriptors);

/****************************************************************************/
/**
*   Initiates a transfer when the descriptors have already been setup.
*
*   This is a special case, and normally, the dma_transfer_xxx functions should
*   be used.
*
*   @return
*       0       Transfer was started successfully
*       -ENODEV Invalid handle
*/
/****************************************************************************/

int dma_start_transfer(DMA_Handle_t handle)
{
        DMA_Channel_t *channel;
        DMA_DeviceAttribute_t *devAttr;

        channel = HandleToChannel(handle);
        if (channel == NULL) {
                return -ENODEV;
        }
        devAttr = &DMA_gDeviceAttribute[channel->devType];

        dmacHw_initiateTransfer(channel->dmacHwHandle, &devAttr->config,
                                devAttr->ring.virtAddr);

        /* Since we got this far, everything went successfully */

        return 0;
}

EXPORT_SYMBOL(dma_start_transfer);

/****************************************************************************/
/**
*   Stops a previously started DMA transfer.
*
*   @return
*       0       Transfer was stopped successfully
*       -ENODEV Invalid handle
*/
/****************************************************************************/

int dma_stop_transfer(DMA_Handle_t handle)
{
        DMA_Channel_t *channel;

        channel = HandleToChannel(handle);
        if (channel == NULL) {
                return -ENODEV;
        }

        dmacHw_stopTransfer(channel->dmacHwHandle);

        return 0;
}

EXPORT_SYMBOL(dma_stop_transfer);

/****************************************************************************/
/**
*   Waits for a DMA to complete by polling. This function is only intended
*   to be used for testing. Interrupts should be used for most DMA operations.
*/
/****************************************************************************/

int dma_wait_transfer_done(DMA_Handle_t handle)
{
        DMA_Channel_t *channel;
        dmacHw_TRANSFER_STATUS_e status;

        channel = HandleToChannel(handle);
        if (channel == NULL) {
                return -ENODEV;
        }

        while ((status =
                dmacHw_transferCompleted(channel->dmacHwHandle)) ==
               dmacHw_TRANSFER_STATUS_BUSY) {
                ;
        }

        if (status == dmacHw_TRANSFER_STATUS_ERROR) {
                printk(KERN_ERR "%s: DMA transfer failed\n", __func__);
                return -EIO;
        }
        return 0;
}

EXPORT_SYMBOL(dma_wait_transfer_done);

/****************************************************************************/
/**
*   Initiates a DMA, allocating the descriptors as required.
*
*   @return
*       0       Transfer was started successfully
*       -EINVAL Invalid device type for this kind of transfer
*               (i.e. the device is _DEV_TO_MEM and not _MEM_TO_DEV)
*/
/****************************************************************************/

int dma_transfer(DMA_Handle_t handle,   /* DMA Handle */
                 dmacHw_TRANSFER_TYPE_e transferType,   /* Type of transfer being performed */
                 dma_addr_t srcData,    /* Place to get data to write to device */
                 dma_addr_t dstData,    /* Pointer to device data address */
                 size_t numBytes        /* Number of bytes to transfer to the device */
    ) {
        DMA_Channel_t *channel;
        DMA_DeviceAttribute_t *devAttr;
        int rc = 0;

        channel = HandleToChannel(handle);
        if (channel == NULL) {
                return -ENODEV;
        }

        devAttr = &DMA_gDeviceAttribute[channel->devType];

        if (devAttr->config.transferType != transferType) {
                return -EINVAL;
        }

        /* We keep track of the information about the previous request for this */
        /* device, and if the attributes match, then we can use the descriptors we setup */
        /* the last time, and not have to reinitialize everything. */

        {
                rc =
                     dma_alloc_descriptors(handle, transferType, srcData,
                                           dstData, numBytes);
                if (rc != 0) {
                        return rc;
                }
        }

        /* And kick off the transfer */

        devAttr->numBytes = numBytes;
        devAttr->transferStartTime = timer_get_tick_count();

        dmacHw_initiateTransfer(channel->dmacHwHandle, &devAttr->config,
                                devAttr->ring.virtAddr);

        /* Since we got this far, everything went successfully */

        return 0;
}

EXPORT_SYMBOL(dma_transfer);

/****************************************************************************/
/**
*   Set the callback function which will be called when a transfer completes.
*   If a NULL callback function is set, then no callback will occur.
*
*   @note   @a devHandler will be called from IRQ context.
*
*   @return
*       0       - Success
*       -ENODEV - Device handed in is invalid.
*/
/****************************************************************************/

int dma_set_device_handler(DMA_Device_t dev,    /* Device to set the callback for. */
                           DMA_DeviceHandler_t devHandler,      /* Function to call when the DMA completes */
                           void *userData       /* Pointer which will be passed to devHandler. */
    ) {
        DMA_DeviceAttribute_t *devAttr;
        unsigned long flags;

        if (!IsDeviceValid(dev)) {
                return -ENODEV;
        }
        devAttr = &DMA_gDeviceAttribute[dev];

        local_irq_save(flags);

        devAttr->userData = userData;
        devAttr->devHandler = devHandler;

        local_irq_restore(flags);

        return 0;
}

EXPORT_SYMBOL(dma_set_device_handler);

/****************************************************************************/
/**
*   Initializes a memory mapping structure
*/
/****************************************************************************/

int dma_init_mem_map(DMA_MemMap_t *memMap)
{
        memset(memMap, 0, sizeof(*memMap));

        init_MUTEX(&memMap->lock);

        return 0;
}

EXPORT_SYMBOL(dma_init_mem_map);

/****************************************************************************/
/**
*   Releases any memory currently being held by a memory mapping structure.
*/
/****************************************************************************/

int dma_term_mem_map(DMA_MemMap_t *memMap)
{
        down(&memMap->lock);    /* Just being paranoid */

        /* Free up any allocated memory */

        up(&memMap->lock);
        memset(memMap, 0, sizeof(*memMap));

        return 0;
}

EXPORT_SYMBOL(dma_term_mem_map);

/****************************************************************************/
/**
*   Looks at a memory address and categorizes it.
*
*   @return One of the values from the DMA_MemType_t enumeration.
*/
/****************************************************************************/

DMA_MemType_t dma_mem_type(void *addr)
{
        unsigned long addrVal = (unsigned long)addr;

        if (addrVal >= VMALLOC_END) {
                /* NOTE: DMA virtual memory space starts at 0xFFxxxxxx */

                /* dma_alloc_xxx pages are physically and virtually contiguous */

                return DMA_MEM_TYPE_DMA;
        }

        /* Technically, we could add one more classification. Addresses between VMALLOC_END */
        /* and the beginning of the DMA virtual address could be considered to be I/O space. */
        /* Right now, nobody cares about this particular classification, so we ignore it. */

        if (is_vmalloc_addr(addr)) {
                /* Address comes from the vmalloc'd region. Pages are virtually */
                /* contiguous but NOT physically contiguous */

                return DMA_MEM_TYPE_VMALLOC;
        }

        if (addrVal >= PAGE_OFFSET) {
                /* PAGE_OFFSET is typically 0xC0000000 */

                /* kmalloc'd pages are physically contiguous */

                return DMA_MEM_TYPE_KMALLOC;
        }

        return DMA_MEM_TYPE_USER;
}

EXPORT_SYMBOL(dma_mem_type);

/****************************************************************************/
/**
*   Looks at a memory address and determines if we support DMA'ing to/from
*   that type of memory.
*
*   @return boolean -
*               return value != 0 means dma supported
*               return value == 0 means dma not supported
*/
/****************************************************************************/

int dma_mem_supports_dma(void *addr)
{
        DMA_MemType_t memType = dma_mem_type(addr);

        return (memType == DMA_MEM_TYPE_DMA)
#if ALLOW_MAP_OF_KMALLOC_MEMORY
            || (memType == DMA_MEM_TYPE_KMALLOC)
#endif
            || (memType == DMA_MEM_TYPE_USER);
}

EXPORT_SYMBOL(dma_mem_supports_dma);

/****************************************************************************/
/**
*   Maps in a memory region such that it can be used for performing a DMA.
*
*   @return
*/
/****************************************************************************/

int dma_map_start(DMA_MemMap_t *memMap, /* Stores state information about the map */
                  enum dma_data_direction dir   /* Direction that the mapping will be going */
    ) {
        int rc;

        down(&memMap->lock);

        DMA_MAP_PRINT("memMap: %p\n", memMap);

        if (memMap->inUse) {
                printk(KERN_ERR "%s: memory map %p is already being used\n",
                       __func__, memMap);
                rc = -EBUSY;
                goto out;
        }

        memMap->inUse = 1;
        memMap->dir = dir;
        memMap->numRegionsUsed = 0;

        rc = 0;

out:

        DMA_MAP_PRINT("returning %d", rc);

        up(&memMap->lock);

        return rc;
}

EXPORT_SYMBOL(dma_map_start);

/****************************************************************************/
/**
*   Adds a segment of memory to a memory map. Each segment is both
*   physically and virtually contiguous.
*
*   @return     0 on success, error code otherwise.
*/
/****************************************************************************/

static int dma_map_add_segment(DMA_MemMap_t *memMap,    /* Stores state information about the map */
                               DMA_Region_t *region,    /* Region that the segment belongs to */
                               void *virtAddr,  /* Virtual address of the segment being added */
                               dma_addr_t physAddr,     /* Physical address of the segment being added */
                               size_t numBytes  /* Number of bytes of the segment being added */
    ) {
        DMA_Segment_t *segment;

        DMA_MAP_PRINT("memMap:%p va:%p pa:0x%x #:%d\n", memMap, virtAddr,
                      physAddr, numBytes);

        /* Sanity check */

        if (((unsigned long)virtAddr < (unsigned long)region->virtAddr)
            || (((unsigned long)virtAddr + numBytes)) >
            ((unsigned long)region->virtAddr + region->numBytes)) {
                printk(KERN_ERR
                       "%s: virtAddr %p is outside region @ %p len: %d\n",
                       __func__, virtAddr, region->virtAddr, region->numBytes);
                return -EINVAL;
        }

        if (region->numSegmentsUsed > 0) {
                /* Check to see if this segment is physically contiguous with the previous one */

                segment = &region->segment[region->numSegmentsUsed - 1];

                if ((segment->physAddr + segment->numBytes) == physAddr) {
                        /* It is - just add on to the end */

                        DMA_MAP_PRINT("appending %d bytes to last segment\n",
                                      numBytes);

                        segment->numBytes += numBytes;

                        return 0;
                }
        }

        /* Reallocate to hold more segments, if required. */

        if (region->numSegmentsUsed >= region->numSegmentsAllocated) {
                DMA_Segment_t *newSegment;
                size_t oldSize =
                    region->numSegmentsAllocated * sizeof(*newSegment);
                int newAlloc = region->numSegmentsAllocated + 4;
                size_t newSize = newAlloc * sizeof(*newSegment);

                newSegment = kmalloc(newSize, GFP_KERNEL);
                if (newSegment == NULL) {
                        return -ENOMEM;
                }
                memcpy(newSegment, region->segment, oldSize);
                memset(&((uint8_t *) newSegment)[oldSize], 0,
                       newSize - oldSize);
                kfree(region->segment);

                region->numSegmentsAllocated = newAlloc;
                region->segment = newSegment;
        }

        segment = &region->segment[region->numSegmentsUsed];
        region->numSegmentsUsed++;

        segment->virtAddr = virtAddr;
        segment->physAddr = physAddr;
        segment->numBytes = numBytes;

        DMA_MAP_PRINT("returning success\n");

        return 0;
}

/****************************************************************************/
/**
*   Adds a region of memory to a memory map. Each region is virtually
*   contiguous, but not necessarily physically contiguous.
*
*   @return     0 on success, error code otherwise.
*/
/****************************************************************************/

int dma_map_add_region(DMA_MemMap_t *memMap,    /* Stores state information about the map */
                       void *mem,       /* Virtual address that we want to get a map of */
                       size_t numBytes  /* Number of bytes being mapped */
    ) {
        unsigned long addr = (unsigned long)mem;
        unsigned int offset;
        int rc = 0;
        DMA_Region_t *region;
        dma_addr_t physAddr;

        down(&memMap->lock);

        DMA_MAP_PRINT("memMap:%p va:%p #:%d\n", memMap, mem, numBytes);

        if (!memMap->inUse) {
                printk(KERN_ERR "%s: Make sure you call dma_map_start first\n",
                       __func__);
                rc = -EINVAL;
                goto out;
        }

        /* Reallocate to hold more regions. */

        if (memMap->numRegionsUsed >= memMap->numRegionsAllocated) {
                DMA_Region_t *newRegion;
                size_t oldSize =
                    memMap->numRegionsAllocated * sizeof(*newRegion);
                int newAlloc = memMap->numRegionsAllocated + 4;
                size_t newSize = newAlloc * sizeof(*newRegion);

                newRegion = kmalloc(newSize, GFP_KERNEL);
                if (newRegion == NULL) {
                        rc = -ENOMEM;
                        goto out;
                }
                memcpy(newRegion, memMap->region, oldSize);
                memset(&((uint8_t *) newRegion)[oldSize], 0, newSize - oldSize);

                kfree(memMap->region);

                memMap->numRegionsAllocated = newAlloc;
                memMap->region = newRegion;
        }

        region = &memMap->region[memMap->numRegionsUsed];
        memMap->numRegionsUsed++;

        offset = addr & ~PAGE_MASK;

        region->memType = dma_mem_type(mem);
        region->virtAddr = mem;
        region->numBytes = numBytes;
        region->numSegmentsUsed = 0;
        region->numLockedPages = 0;
        region->lockedPages = NULL;

        switch (region->memType) {
        case DMA_MEM_TYPE_VMALLOC:
                {
                        atomic_inc(&gDmaStatMemTypeVmalloc);

                        /* printk(KERN_ERR "%s: vmalloc'd pages are not supported\n", __func__); */

                        /* vmalloc'd pages are not physically contiguous */

                        rc = -EINVAL;
                        break;
                }

        case DMA_MEM_TYPE_KMALLOC:
                {
                        atomic_inc(&gDmaStatMemTypeKmalloc);

                        /* kmalloc'd pages are physically contiguous, so they'll have exactly */
                        /* one segment */

#if ALLOW_MAP_OF_KMALLOC_MEMORY
                        physAddr =
                            dma_map_single(NULL, mem, numBytes, memMap->dir);
                        rc = dma_map_add_segment(memMap, region, mem, physAddr,
                                                 numBytes);
#else
                        rc = -EINVAL;
#endif
                        break;
                }

        case DMA_MEM_TYPE_DMA:
                {
                        /* dma_alloc_xxx pages are physically contiguous */

                        atomic_inc(&gDmaStatMemTypeCoherent);

                        physAddr = (vmalloc_to_pfn(mem) << PAGE_SHIFT) + offset;

                        dma_sync_single_for_cpu(NULL, physAddr, numBytes,
                                                memMap->dir);
                        rc = dma_map_add_segment(memMap, region, mem, physAddr,
                                                 numBytes);
                        break;
                }

        case DMA_MEM_TYPE_USER:
                {
                        size_t firstPageOffset;
                        size_t firstPageSize;
                        struct page **pages;
                        struct task_struct *userTask;

                        atomic_inc(&gDmaStatMemTypeUser);

#if 1
                        /* If the pages are user pages, then the dma_mem_map_set_user_task function */
                        /* must have been previously called. */

                        if (memMap->userTask == NULL) {
                                printk(KERN_ERR
                                       "%s: must call dma_mem_map_set_user_task when using user-mode memory\n",
                                       __func__);
                                return -EINVAL;
                        }

                        /* User pages need to be locked. */

                        firstPageOffset =
                            (unsigned long)region->virtAddr & (PAGE_SIZE - 1);
                        firstPageSize = PAGE_SIZE - firstPageOffset;

                        region->numLockedPages = (firstPageOffset
                                                  + region->numBytes +
                                                  PAGE_SIZE - 1) / PAGE_SIZE;
                        pages =
                            kmalloc(region->numLockedPages *
                                    sizeof(struct page *), GFP_KERNEL);

                        if (pages == NULL) {
                                region->numLockedPages = 0;
                                return -ENOMEM;
                        }

                        userTask = memMap->userTask;

                        down_read(&userTask->mm->mmap_sem);
                        rc = get_user_pages(userTask,   /* task */
                                            userTask->mm,       /* mm */
                                            (unsigned long)region->virtAddr,    /* start */
                                            region->numLockedPages,     /* len */
                                            memMap->dir == DMA_FROM_DEVICE,     /* write */
                                            0,  /* force */
                                            pages,      /* pages (array of pointers to page) */
                                            NULL);      /* vmas */
                        up_read(&userTask->mm->mmap_sem);

                        if (rc != region->numLockedPages) {
                                kfree(pages);
                                region->numLockedPages = 0;

                                if (rc >= 0) {
                                        rc = -EINVAL;
                                }
                        } else {
                                uint8_t *virtAddr = region->virtAddr;
                                size_t bytesRemaining;
                                int pageIdx;

                                rc = 0; /* Since get_user_pages returns +ve number */

                                region->lockedPages = pages;

                                /* We've locked the user pages. Now we need to walk them and figure */
                                /* out the physical addresses. */

                                /* The first page may be partial */

                                dma_map_add_segment(memMap,
                                                    region,
                                                    virtAddr,
                                                    PFN_PHYS(page_to_pfn
                                                             (pages[0])) +
                                                    firstPageOffset,
                                                    firstPageSize);

                                virtAddr += firstPageSize;
                                bytesRemaining =
                                    region->numBytes - firstPageSize;

                                for (pageIdx = 1;
                                     pageIdx < region->numLockedPages;
                                     pageIdx++) {
                                        size_t bytesThisPage =
                                            (bytesRemaining >
                                             PAGE_SIZE ? PAGE_SIZE :
                                             bytesRemaining);

                                        DMA_MAP_PRINT
                                            ("pageIdx:%d pages[pageIdx]=%p pfn=%u phys=%u\n",
                                             pageIdx, pages[pageIdx],
                                             page_to_pfn(pages[pageIdx]),

                                             PFN_PHYS(page_to_pfn
                                                      (pages[pageIdx])));

                                        dma_map_add_segment(memMap,
                                                            region,
                                                            virtAddr,
                                                            PFN_PHYS(page_to_pfn
                                                                     (pages
                                                                      [pageIdx])),
                                                            bytesThisPage);

                                        virtAddr += bytesThisPage;
                                        bytesRemaining -= bytesThisPage;
                                }
                        }
#else
                        printk(KERN_ERR
                               "%s: User mode pages are not yet supported\n",
                               __func__);

                        /* user pages are not physically contiguous */

                        rc = -EINVAL;
#endif
                        break;
                }

        default:
                {
                        printk(KERN_ERR "%s: Unsupported memory type: %d\n",
                               __func__, region->memType);

                        rc = -EINVAL;
                        break;
                }
        }

        if (rc != 0) {
                memMap->numRegionsUsed--;
        }

out:

        DMA_MAP_PRINT("returning %d\n", rc);

        up(&memMap->lock);

        return rc;
}

EXPORT_SYMBOL(dma_map_add_segment);

/****************************************************************************/
/**
*   Maps in a memory region such that it can be used for performing a DMA.
*
*   @return     0 on success, error code otherwise.
*/
/****************************************************************************/

int dma_map_mem(DMA_MemMap_t *memMap,   /* Stores state information about the map */
                void *mem,      /* Virtual address that we want to get a map of */
                size_t numBytes,        /* Number of bytes being mapped */
                enum dma_data_direction dir     /* Direction that the mapping will be going */
    ) {
        int rc;

        rc = dma_map_start(memMap, dir);
        if (rc == 0) {
                rc = dma_map_add_region(memMap, mem, numBytes);
                if (rc < 0) {
                        /* Since the add fails, this function will fail, and the caller won't */
                        /* call unmap, so we need to do it here. */

                        dma_unmap(memMap, 0);
                }
        }

        return rc;
}

EXPORT_SYMBOL(dma_map_mem);

/****************************************************************************/
/**
*   Setup a descriptor ring for a given memory map.
*
*   It is assumed that the descriptor ring has already been initialized, and
*   this routine will only reallocate a new descriptor ring if the existing
*   one is too small.
*
*   @return     0 on success, error code otherwise.
*/
/****************************************************************************/

int dma_map_create_descriptor_ring(DMA_Device_t dev,    /* DMA device (where the ring is stored) */
                                   DMA_MemMap_t *memMap,        /* Memory map that will be used */
                                   dma_addr_t devPhysAddr       /* Physical address of device */
    ) {
        int rc;
        int numDescriptors;
        DMA_DeviceAttribute_t *devAttr;
        DMA_Region_t *region;
        DMA_Segment_t *segment;
        dma_addr_t srcPhysAddr;
        dma_addr_t dstPhysAddr;
        int regionIdx;
        int segmentIdx;

        devAttr = &DMA_gDeviceAttribute[dev];

        down(&memMap->lock);

        /* Figure out how many descriptors we need */

        numDescriptors = 0;
        for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
                region = &memMap->region[regionIdx];

                for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
                     segmentIdx++) {
                        segment = &region->segment[segmentIdx];

                        if (memMap->dir == DMA_TO_DEVICE) {
                                srcPhysAddr = segment->physAddr;
                                dstPhysAddr = devPhysAddr;
                        } else {
                                srcPhysAddr = devPhysAddr;
                                dstPhysAddr = segment->physAddr;
                        }

                        rc =
                             dma_calculate_descriptor_count(dev, srcPhysAddr,
                                                            dstPhysAddr,
                                                            segment->
                                                            numBytes);
                        if (rc < 0) {
                                printk(KERN_ERR
                                       "%s: dma_calculate_descriptor_count failed: %d\n",
                                       __func__, rc);
                                goto out;
                        }
                        numDescriptors += rc;
                }
        }

        /* Adjust the size of the ring, if it isn't big enough */

        if (numDescriptors > devAttr->ring.descriptorsAllocated) {
                dma_free_descriptor_ring(&devAttr->ring);
                rc =
                     dma_alloc_descriptor_ring(&devAttr->ring,
                                               numDescriptors);
                if (rc < 0) {
                        printk(KERN_ERR
                               "%s: dma_alloc_descriptor_ring failed: %d\n",
                               __func__, rc);
                        goto out;
                }
        } else {
                rc =
                     dma_init_descriptor_ring(&devAttr->ring,
                                              numDescriptors);
                if (rc < 0) {
                        printk(KERN_ERR
                               "%s: dma_init_descriptor_ring failed: %d\n",
                               __func__, rc);
                        goto out;
                }
        }

        /* Populate the descriptors */

        for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
                region = &memMap->region[regionIdx];

                for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
                     segmentIdx++) {
                        segment = &region->segment[segmentIdx];

                        if (memMap->dir == DMA_TO_DEVICE) {
                                srcPhysAddr = segment->physAddr;
                                dstPhysAddr = devPhysAddr;
                        } else {
                                srcPhysAddr = devPhysAddr;
                                dstPhysAddr = segment->physAddr;
                        }

                        rc =
                             dma_add_descriptors(&devAttr->ring, dev,
                                                 srcPhysAddr, dstPhysAddr,
                                                 segment->numBytes);
                        if (rc < 0) {
                                printk(KERN_ERR
                                       "%s: dma_add_descriptors failed: %d\n",
                                       __func__, rc);
                                goto out;
                        }
                }
        }

        rc = 0;

out:

        up(&memMap->lock);
        return rc;
}

EXPORT_SYMBOL(dma_map_create_descriptor_ring);

/****************************************************************************/
/**
*   Maps in a memory region such that it can be used for performing a DMA.
*
*   @return
*/
/****************************************************************************/

int dma_unmap(DMA_MemMap_t *memMap,     /* Stores state information about the map */
              int dirtied       /* non-zero if any of the pages were modified */
    ) {
        int regionIdx;
        int segmentIdx;
        DMA_Region_t *region;
        DMA_Segment_t *segment;

        for (regionIdx = 0; regionIdx < memMap->numRegionsUsed; regionIdx++) {
                region = &memMap->region[regionIdx];

                for (segmentIdx = 0; segmentIdx < region->numSegmentsUsed;
                     segmentIdx++) {
                        segment = &region->segment[segmentIdx];

                        switch (region->memType) {
                        case DMA_MEM_TYPE_VMALLOC:
                                {
                                        printk(KERN_ERR
                                               "%s: vmalloc'd pages are not yet supported\n",
                                               __func__);
                                        return -EINVAL;
                                }

                        case DMA_MEM_TYPE_KMALLOC:
                                {
#if ALLOW_MAP_OF_KMALLOC_MEMORY
                                        dma_unmap_single(NULL,
                                                         segment->physAddr,
                                                         segment->numBytes,
                                                         memMap->dir);
#endif
                                        break;
                                }

                        case DMA_MEM_TYPE_DMA:
                                {
                                        dma_sync_single_for_cpu(NULL,
                                                                segment->
                                                                physAddr,
                                                                segment->
                                                                numBytes,
                                                                memMap->dir);
                                        break;
                                }

                        case DMA_MEM_TYPE_USER:
                                {
                                        /* Nothing to do here. */

                                        break;
                                }

                        default:
                                {
                                        printk(KERN_ERR
                                               "%s: Unsupported memory type: %d\n",
                                               __func__, region->memType);
                                        return -EINVAL;
                                }
                        }

                        segment->virtAddr = NULL;
                        segment->physAddr = 0;
                        segment->numBytes = 0;
                }

                if (region->numLockedPages > 0) {
                        int pageIdx;

                        /* Some user pages were locked. We need to go and unlock them now. */

                        for (pageIdx = 0; pageIdx < region->numLockedPages;
                             pageIdx++) {
                                struct page *page =
                                    region->lockedPages[pageIdx];

                                if (memMap->dir == DMA_FROM_DEVICE) {
                                        SetPageDirty(page);
                                }
                                page_cache_release(page);
                        }
                        kfree(region->lockedPages);
                        region->numLockedPages = 0;
                        region->lockedPages = NULL;
                }

                region->memType = DMA_MEM_TYPE_NONE;
                region->virtAddr = NULL;
                region->numBytes = 0;
                region->numSegmentsUsed = 0;
        }
        memMap->userTask = NULL;
        memMap->numRegionsUsed = 0;
        memMap->inUse = 0;

        up(&memMap->lock);

        return 0;
}

EXPORT_SYMBOL(dma_unmap);