1
2#include <linux/module.h>
3#include <linux/init.h>
4#include <linux/mm.h>
5#include <linux/kernel.h>
6#include <linux/slab.h>
7#include <linux/fs.h>
8#include <linux/errno.h>
9#include <linux/types.h>
10#include <linux/cdev.h>
11#include <linux/uaccess.h>
12#include <linux/highmem.h>
13#include <linux/pagemap.h>
14#include "kpc_dma_driver.h"
15#include "uapi.h"
16
17
18static inline
19unsigned int count_pages(unsigned long iov_base, size_t iov_len)
20{
21 unsigned long first = (iov_base & PAGE_MASK) >> PAGE_SHIFT;
22 unsigned long last = ((iov_base+iov_len-1) & PAGE_MASK) >> PAGE_SHIFT;
23
24 return last - first + 1;
25}
26
27static inline
28unsigned int count_parts_for_sge(struct scatterlist *sg)
29{
30 return DIV_ROUND_UP(sg_dma_len(sg), 0x80000);
31}
32
33
34static int kpc_dma_transfer(struct dev_private_data *priv,
35 unsigned long iov_base, size_t iov_len)
36{
37 unsigned int i = 0;
38 long rv = 0;
39 struct kpc_dma_device *ldev;
40 struct aio_cb_data *acd;
41 DECLARE_COMPLETION_ONSTACK(done);
42 u32 desc_needed = 0;
43 struct scatterlist *sg;
44 u32 num_descrs_avail;
45 struct kpc_dma_descriptor *desc;
46 unsigned int pcnt;
47 unsigned int p;
48 u64 card_addr;
49 u64 dma_addr;
50 u64 user_ctl;
51
52 BUG_ON(priv == NULL);
53 ldev = priv->ldev;
54 BUG_ON(ldev == NULL);
55
56 acd = kzalloc(sizeof(*acd), GFP_KERNEL);
57 if (!acd) {
58 dev_err(&priv->ldev->pldev->dev, "Couldn't kmalloc space for for the aio data\n");
59 return -ENOMEM;
60 }
61 memset(acd, 0x66, sizeof(struct aio_cb_data));
62
63 acd->priv = priv;
64 acd->ldev = priv->ldev;
65 acd->cpl = &done;
66 acd->flags = 0;
67 acd->len = iov_len;
68 acd->page_count = count_pages(iov_base, iov_len);
69
70
71 acd->user_pages = kzalloc(sizeof(struct page *) * acd->page_count, GFP_KERNEL);
72 if (!acd->user_pages) {
73 dev_err(&priv->ldev->pldev->dev, "Couldn't kmalloc space for for the page pointers\n");
74 rv = -ENOMEM;
75 goto err_alloc_userpages;
76 }
77
78
79 down_read(¤t->mm->mmap_sem);
80 rv = get_user_pages(iov_base, acd->page_count, FOLL_TOUCH | FOLL_WRITE | FOLL_GET, acd->user_pages, NULL);
81 up_read(¤t->mm->mmap_sem);
82 if (rv != acd->page_count) {
83 dev_err(&priv->ldev->pldev->dev, "Couldn't get_user_pages (%ld)\n", rv);
84 goto err_get_user_pages;
85 }
86
87
88 rv = sg_alloc_table_from_pages(&acd->sgt, acd->user_pages, acd->page_count, iov_base & (PAGE_SIZE-1), iov_len, GFP_KERNEL);
89 if (rv) {
90 dev_err(&priv->ldev->pldev->dev, "Couldn't alloc sg_table (%ld)\n", rv);
91 goto err_alloc_sg_table;
92 }
93
94
95 acd->mapped_entry_count = dma_map_sg(&ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, ldev->dir);
96 if (acd->mapped_entry_count <= 0) {
97 dev_err(&priv->ldev->pldev->dev, "Couldn't dma_map_sg (%d)\n", acd->mapped_entry_count);
98 goto err_dma_map_sg;
99 }
100
101
102 for_each_sg(acd->sgt.sgl, sg, acd->mapped_entry_count, i) {
103 desc_needed += count_parts_for_sge(sg);
104 }
105
106 lock_engine(ldev);
107
108
109 num_descrs_avail = count_descriptors_available(ldev);
110 dev_dbg(&priv->ldev->pldev->dev, " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
111 if (desc_needed >= ldev->desc_pool_cnt) {
112 dev_warn(&priv->ldev->pldev->dev, " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d TOO MANY to ever complete!\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
113 rv = -EAGAIN;
114 goto err_descr_too_many;
115 }
116 if (desc_needed > num_descrs_avail) {
117 dev_warn(&priv->ldev->pldev->dev, " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d Too many to complete right now.\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
118 rv = -EMSGSIZE;
119 goto err_descr_too_many;
120 }
121
122
123 desc = ldev->desc_next;
124 card_addr = acd->priv->card_addr;
125 for_each_sg(acd->sgt.sgl, sg, acd->mapped_entry_count, i) {
126 pcnt = count_parts_for_sge(sg);
127 for (p = 0 ; p < pcnt ; p++) {
128
129 BUG_ON(desc == NULL);
130 clear_desc(desc);
131 if (p != pcnt-1) {
132 desc->DescByteCount = 0x80000;
133 } else {
134 desc->DescByteCount = sg_dma_len(sg) - (p * 0x80000);
135 }
136 desc->DescBufferByteCount = desc->DescByteCount;
137
138 desc->DescControlFlags |= DMA_DESC_CTL_IRQONERR;
139 if (i == 0 && p == 0)
140 desc->DescControlFlags |= DMA_DESC_CTL_SOP;
141 if (i == acd->mapped_entry_count-1 && p == pcnt-1)
142 desc->DescControlFlags |= DMA_DESC_CTL_EOP | DMA_DESC_CTL_IRQONDONE;
143
144 desc->DescCardAddrLS = (card_addr & 0xFFFFFFFF);
145 desc->DescCardAddrMS = (card_addr >> 32) & 0xF;
146 card_addr += desc->DescByteCount;
147
148 dma_addr = sg_dma_address(sg) + (p * 0x80000);
149 desc->DescSystemAddrLS = (dma_addr & 0x00000000FFFFFFFF) >> 0;
150 desc->DescSystemAddrMS = (dma_addr & 0xFFFFFFFF00000000) >> 32;
151
152 user_ctl = acd->priv->user_ctl;
153 if (i == acd->mapped_entry_count-1 && p == pcnt-1) {
154 user_ctl = acd->priv->user_ctl_last;
155 }
156 desc->DescUserControlLS = (user_ctl & 0x00000000FFFFFFFF) >> 0;
157 desc->DescUserControlMS = (user_ctl & 0xFFFFFFFF00000000) >> 32;
158
159 if (i == acd->mapped_entry_count-1 && p == pcnt-1)
160 desc->acd = acd;
161
162 dev_dbg(&priv->ldev->pldev->dev, " Filled descriptor %p (acd = %p)\n", desc, desc->acd);
163
164 ldev->desc_next = desc->Next;
165 desc = desc->Next;
166 }
167 }
168
169
170 SetEngineSWPtr(ldev, ldev->desc_next);
171
172 unlock_engine(ldev);
173
174 rv = wait_for_completion_interruptible(&done);
175
176
177
178
179 if (rv == -ERESTARTSYS)
180 acd->cpl = NULL;
181 if (rv == 0) {
182 rv = acd->len;
183 kfree(acd);
184 }
185 return rv;
186
187 err_descr_too_many:
188 unlock_engine(ldev);
189 dma_unmap_sg(&ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, ldev->dir);
190 sg_free_table(&acd->sgt);
191 err_dma_map_sg:
192 err_alloc_sg_table:
193 for (i = 0 ; i < acd->page_count ; i++) {
194 put_page(acd->user_pages[i]);
195 }
196 err_get_user_pages:
197 kfree(acd->user_pages);
198 err_alloc_userpages:
199 kfree(acd);
200 dev_dbg(&priv->ldev->pldev->dev, "%s returning with error %ld\n", __func__, rv);
201 return rv;
202}
203
204void transfer_complete_cb(struct aio_cb_data *acd, size_t xfr_count, u32 flags)
205{
206 unsigned int i;
207
208 BUG_ON(acd == NULL);
209 BUG_ON(acd->user_pages == NULL);
210 BUG_ON(acd->sgt.sgl == NULL);
211 BUG_ON(acd->ldev == NULL);
212 BUG_ON(acd->ldev->pldev == NULL);
213
214 for (i = 0 ; i < acd->page_count ; i++) {
215 if (!PageReserved(acd->user_pages[i])) {
216 set_page_dirty(acd->user_pages[i]);
217 }
218 }
219
220 dma_unmap_sg(&acd->ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, acd->ldev->dir);
221
222 for (i = 0 ; i < acd->page_count ; i++) {
223 put_page(acd->user_pages[i]);
224 }
225
226 sg_free_table(&acd->sgt);
227
228 kfree(acd->user_pages);
229
230 acd->flags = flags;
231
232 if (acd->cpl) {
233 complete(acd->cpl);
234 } else {
235
236
237
238
239 kfree(acd);
240 }
241}
242
243
244static
245int kpc_dma_open(struct inode *inode, struct file *filp)
246{
247 struct dev_private_data *priv;
248 struct kpc_dma_device *ldev = kpc_dma_lookup_device(iminor(inode));
249
250 if (!ldev)
251 return -ENODEV;
252
253 if (!atomic_dec_and_test(&ldev->open_count)) {
254 atomic_inc(&ldev->open_count);
255 return -EBUSY;
256 }
257
258 priv = kzalloc(sizeof(struct dev_private_data), GFP_KERNEL);
259 if (!priv)
260 return -ENOMEM;
261
262 priv->ldev = ldev;
263 filp->private_data = priv;
264
265 return 0;
266}
267
268static
269int kpc_dma_close(struct inode *inode, struct file *filp)
270{
271 struct kpc_dma_descriptor *cur;
272 struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
273 struct kpc_dma_device *eng = priv->ldev;
274
275 lock_engine(eng);
276
277 stop_dma_engine(eng);
278
279 cur = eng->desc_completed->Next;
280 while (cur != eng->desc_next) {
281 dev_dbg(&eng->pldev->dev, "Aborting descriptor %p (acd = %p)\n", cur, cur->acd);
282 if (cur->DescControlFlags & DMA_DESC_CTL_EOP) {
283 if (cur->acd)
284 transfer_complete_cb(cur->acd, 0, ACD_FLAG_ABORT);
285 }
286
287 clear_desc(cur);
288 eng->desc_completed = cur;
289
290 cur = cur->Next;
291 }
292
293 start_dma_engine(eng);
294
295 unlock_engine(eng);
296
297 atomic_inc(&priv->ldev->open_count);
298 kfree(priv);
299 return 0;
300}
301
302static
303ssize_t kpc_dma_read(struct file *filp, char __user *user_buf, size_t count, loff_t *ppos)
304{
305 struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
306
307 if (priv->ldev->dir != DMA_FROM_DEVICE)
308 return -EMEDIUMTYPE;
309
310 return kpc_dma_transfer(priv, (unsigned long)user_buf, count);
311}
312
313static
314ssize_t kpc_dma_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos)
315{
316 struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
317
318 if (priv->ldev->dir != DMA_TO_DEVICE)
319 return -EMEDIUMTYPE;
320
321 return kpc_dma_transfer(priv, (unsigned long)user_buf, count);
322}
323
324static
325long kpc_dma_ioctl(struct file *filp, unsigned int ioctl_num, unsigned long ioctl_param)
326{
327 struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
328
329 switch (ioctl_num) {
330 case KND_IOCTL_SET_CARD_ADDR:
331 priv->card_addr = ioctl_param; return priv->card_addr;
332 case KND_IOCTL_SET_USER_CTL:
333 priv->user_ctl = ioctl_param; return priv->user_ctl;
334 case KND_IOCTL_SET_USER_CTL_LAST:
335 priv->user_ctl_last = ioctl_param; return priv->user_ctl_last;
336 case KND_IOCTL_GET_USER_STS:
337 return priv->user_sts;
338 }
339
340 return -ENOTTY;
341}
342
343const struct file_operations kpc_dma_fops = {
344 .owner = THIS_MODULE,
345 .open = kpc_dma_open,
346 .release = kpc_dma_close,
347 .read = kpc_dma_read,
348 .write = kpc_dma_write,
349 .unlocked_ioctl = kpc_dma_ioctl,
350};
351
352
