1
2
3
4
5
6
7
8
9#include <linux/errno.h>
10#include <linux/interrupt.h>
11#include <linux/kernel.h>
12#include <linux/module.h>
13#include <linux/param.h>
14#include <linux/proc_fs.h>
15#include <linux/sched.h>
16#include <linux/seq_file.h>
17#include <linux/spinlock.h>
18
19#include <asm/blackfin.h>
20#include <asm/cacheflush.h>
21#include <asm/dma.h>
22#include <asm/uaccess.h>
23#include <asm/early_printk.h>
24
25
26
27
28
29
30struct dma_channel dma_ch[MAX_DMA_CHANNELS];
31EXPORT_SYMBOL(dma_ch);
32
33static int __init blackfin_dma_init(void)
34{
35 int i;
36
37 printk(KERN_INFO "Blackfin DMA Controller\n");
38
39 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
40 dma_ch[i].chan_status = DMA_CHANNEL_FREE;
41 dma_ch[i].regs = dma_io_base_addr[i];
42 mutex_init(&(dma_ch[i].dmalock));
43 }
44
45 request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
46 request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
47
48#if defined(CONFIG_DEB_DMA_URGENT)
49 bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
50 | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
51#endif
52
53 return 0;
54}
55arch_initcall(blackfin_dma_init);
56
57#ifdef CONFIG_PROC_FS
58static int proc_dma_show(struct seq_file *m, void *v)
59{
60 int i;
61
62 for (i = 0; i < MAX_DMA_CHANNELS; ++i)
63 if (dma_ch[i].chan_status != DMA_CHANNEL_FREE)
64 seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
65
66 return 0;
67}
68
69static int proc_dma_open(struct inode *inode, struct file *file)
70{
71 return single_open(file, proc_dma_show, NULL);
72}
73
74static const struct file_operations proc_dma_operations = {
75 .open = proc_dma_open,
76 .read = seq_read,
77 .llseek = seq_lseek,
78 .release = single_release,
79};
80
81static int __init proc_dma_init(void)
82{
83 return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL;
84}
85late_initcall(proc_dma_init);
86#endif
87
88
89
90
91
92
93int request_dma(unsigned int channel, const char *device_id)
94{
95 pr_debug("request_dma() : BEGIN \n");
96
97 if (device_id == NULL)
98 printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
99
100#if defined(CONFIG_BF561) && ANOMALY_05000182
101 if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
102 if (get_cclk() > 500000000) {
103 printk(KERN_WARNING
104 "Request IMDMA failed due to ANOMALY 05000182\n");
105 return -EFAULT;
106 }
107 }
108#endif
109
110 mutex_lock(&(dma_ch[channel].dmalock));
111
112 if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
113 || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
114 mutex_unlock(&(dma_ch[channel].dmalock));
115 pr_debug("DMA CHANNEL IN USE \n");
116 return -EBUSY;
117 } else {
118 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
119 pr_debug("DMA CHANNEL IS ALLOCATED \n");
120 }
121
122 mutex_unlock(&(dma_ch[channel].dmalock));
123
124#ifdef CONFIG_BF54x
125 if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
126 unsigned int per_map;
127 per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
128 if (strncmp(device_id, "BFIN_UART", 9) == 0)
129 dma_ch[channel].regs->peripheral_map = per_map |
130 ((channel - CH_UART2_RX + 0xC)<<12);
131 else
132 dma_ch[channel].regs->peripheral_map = per_map |
133 ((channel - CH_UART2_RX + 0x6)<<12);
134 }
135#endif
136
137 dma_ch[channel].device_id = device_id;
138 dma_ch[channel].irq = 0;
139
140
141
142
143
144 pr_debug("request_dma() : END \n");
145 return 0;
146}
147EXPORT_SYMBOL(request_dma);
148
149int set_dma_callback(unsigned int channel, irq_handler_t callback, void *data)
150{
151 BUG_ON(channel >= MAX_DMA_CHANNELS ||
152 dma_ch[channel].chan_status == DMA_CHANNEL_FREE);
153
154 if (callback != NULL) {
155 int ret;
156 unsigned int irq = channel2irq(channel);
157
158 ret = request_irq(irq, callback, IRQF_DISABLED,
159 dma_ch[channel].device_id, data);
160 if (ret)
161 return ret;
162
163 dma_ch[channel].irq = irq;
164 dma_ch[channel].data = data;
165 }
166 return 0;
167}
168EXPORT_SYMBOL(set_dma_callback);
169
170
171
172
173
174
175
176static void clear_dma_buffer(unsigned int channel)
177{
178 dma_ch[channel].regs->cfg |= RESTART;
179 SSYNC();
180 dma_ch[channel].regs->cfg &= ~RESTART;
181}
182
183void free_dma(unsigned int channel)
184{
185 pr_debug("freedma() : BEGIN \n");
186 BUG_ON(channel >= MAX_DMA_CHANNELS ||
187 dma_ch[channel].chan_status == DMA_CHANNEL_FREE);
188
189
190 disable_dma(channel);
191 clear_dma_buffer(channel);
192
193 if (dma_ch[channel].irq)
194 free_irq(dma_ch[channel].irq, dma_ch[channel].data);
195
196
197 mutex_lock(&(dma_ch[channel].dmalock));
198 dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
199 mutex_unlock(&(dma_ch[channel].dmalock));
200
201 pr_debug("freedma() : END \n");
202}
203EXPORT_SYMBOL(free_dma);
204
205#ifdef CONFIG_PM
206# ifndef MAX_DMA_SUSPEND_CHANNELS
207# define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS
208# endif
209int blackfin_dma_suspend(void)
210{
211 int i;
212
213 for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i) {
214 if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
215 printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
216 return -EBUSY;
217 }
218
219 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
220 }
221
222 return 0;
223}
224
225void blackfin_dma_resume(void)
226{
227 int i;
228
229 for (i = 0; i < MAX_DMA_CHANNELS; ++i) {
230 dma_ch[i].regs->cfg = 0;
231
232 if (i < MAX_DMA_SUSPEND_CHANNELS)
233 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
234 }
235}
236#endif
237
238
239
240
241
242
243
244void __init blackfin_dma_early_init(void)
245{
246 early_shadow_stamp();
247 bfin_write_MDMA_S0_CONFIG(0);
248 bfin_write_MDMA_S1_CONFIG(0);
249}
250
251void __init early_dma_memcpy(void *pdst, const void *psrc, size_t size)
252{
253 unsigned long dst = (unsigned long)pdst;
254 unsigned long src = (unsigned long)psrc;
255 struct dma_register *dst_ch, *src_ch;
256
257 early_shadow_stamp();
258
259
260
261
262 BUG_ON(dst % 4);
263 BUG_ON(src % 4);
264 BUG_ON(size % 4);
265
266 src_ch = 0;
267
268 while (1) {
269 if (src_ch == (struct dma_register *)MDMA_S0_NEXT_DESC_PTR) {
270 dst_ch = (struct dma_register *)MDMA_D1_NEXT_DESC_PTR;
271 src_ch = (struct dma_register *)MDMA_S1_NEXT_DESC_PTR;
272 } else {
273 dst_ch = (struct dma_register *)MDMA_D0_NEXT_DESC_PTR;
274 src_ch = (struct dma_register *)MDMA_S0_NEXT_DESC_PTR;
275 }
276
277 if (!bfin_read16(&src_ch->cfg))
278 break;
279 else if (bfin_read16(&dst_ch->irq_status) & DMA_DONE) {
280 bfin_write16(&src_ch->cfg, 0);
281 break;
282 }
283 }
284
285
286
287
288
289 __builtin_bfin_ssync();
290
291
292 bfin_write32(&dst_ch->start_addr, dst);
293 bfin_write16(&dst_ch->x_count, size >> 2);
294 bfin_write16(&dst_ch->x_modify, 1 << 2);
295 bfin_write16(&dst_ch->irq_status, DMA_DONE | DMA_ERR);
296
297
298 bfin_write32(&src_ch->start_addr, src);
299 bfin_write16(&src_ch->x_count, size >> 2);
300 bfin_write16(&src_ch->x_modify, 1 << 2);
301 bfin_write16(&src_ch->irq_status, DMA_DONE | DMA_ERR);
302
303
304 bfin_write16(&src_ch->cfg, DMAEN | WDSIZE_32);
305 bfin_write16(&dst_ch->cfg, WNR | DI_EN | DMAEN | WDSIZE_32);
306
307
308 __builtin_bfin_ssync();
309}
310
311void __init early_dma_memcpy_done(void)
312{
313 early_shadow_stamp();
314
315 while ((bfin_read_MDMA_S0_CONFIG() && !(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE)) ||
316 (bfin_read_MDMA_S1_CONFIG() && !(bfin_read_MDMA_D1_IRQ_STATUS() & DMA_DONE)))
317 continue;
318
319 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
320 bfin_write_MDMA_D1_IRQ_STATUS(DMA_DONE | DMA_ERR);
321
322
323
324
325
326 bfin_write_MDMA_S0_CONFIG(0);
327 bfin_write_MDMA_S1_CONFIG(0);
328 bfin_write_MDMA_D0_CONFIG(0);
329 bfin_write_MDMA_D1_CONFIG(0);
330
331 __builtin_bfin_ssync();
332}
333
334
335
336
337
338
339
340
341
342static void __dma_memcpy(u32 daddr, s16 dmod, u32 saddr, s16 smod, size_t cnt, u32 conf)
343{
344 static DEFINE_SPINLOCK(mdma_lock);
345 unsigned long flags;
346
347 spin_lock_irqsave(&mdma_lock, flags);
348
349
350
351
352
353
354 __builtin_bfin_ssync();
355
356 if (bfin_read_MDMA_S0_CONFIG())
357 while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
358 continue;
359
360 if (conf & DMA2D) {
361
362
363
364
365
366
367
368
369 u32 shift = abs(dmod) >> 1;
370 size_t ycnt = cnt >> (16 - shift);
371 cnt = 1 << (16 - shift);
372 bfin_write_MDMA_D0_Y_COUNT(ycnt);
373 bfin_write_MDMA_S0_Y_COUNT(ycnt);
374 bfin_write_MDMA_D0_Y_MODIFY(dmod);
375 bfin_write_MDMA_S0_Y_MODIFY(smod);
376 }
377
378 bfin_write_MDMA_D0_START_ADDR(daddr);
379 bfin_write_MDMA_D0_X_COUNT(cnt);
380 bfin_write_MDMA_D0_X_MODIFY(dmod);
381 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
382
383 bfin_write_MDMA_S0_START_ADDR(saddr);
384 bfin_write_MDMA_S0_X_COUNT(cnt);
385 bfin_write_MDMA_S0_X_MODIFY(smod);
386 bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
387
388 bfin_write_MDMA_S0_CONFIG(DMAEN | conf);
389 bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | conf);
390
391 spin_unlock_irqrestore(&mdma_lock, flags);
392
393 SSYNC();
394
395 while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
396 if (bfin_read_MDMA_S0_CONFIG())
397 continue;
398 else
399 return;
400
401 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
402
403 bfin_write_MDMA_S0_CONFIG(0);
404 bfin_write_MDMA_D0_CONFIG(0);
405}
406
407
408
409
410
411
412
413static void *_dma_memcpy(void *pdst, const void *psrc, size_t size)
414{
415 u32 conf, shift;
416 s16 mod;
417 unsigned long dst = (unsigned long)pdst;
418 unsigned long src = (unsigned long)psrc;
419
420 if (size == 0)
421 return NULL;
422
423 if (dst % 4 == 0 && src % 4 == 0 && size % 4 == 0) {
424 conf = WDSIZE_32;
425 shift = 2;
426 } else if (dst % 2 == 0 && src % 2 == 0 && size % 2 == 0) {
427 conf = WDSIZE_16;
428 shift = 1;
429 } else {
430 conf = WDSIZE_8;
431 shift = 0;
432 }
433
434
435
436
437
438 mod = 1 << shift;
439 if (src < dst) {
440 mod *= -1;
441 dst += size + mod;
442 src += size + mod;
443 }
444 size >>= shift;
445
446 if (size > 0x10000)
447 conf |= DMA2D;
448
449 __dma_memcpy(dst, mod, src, mod, size, conf);
450
451 return pdst;
452}
453
454
455
456
457
458
459
460
461void *dma_memcpy(void *pdst, const void *psrc, size_t size)
462{
463 unsigned long dst = (unsigned long)pdst;
464 unsigned long src = (unsigned long)psrc;
465 size_t bulk, rest;
466
467 if (bfin_addr_dcacheable(src))
468 blackfin_dcache_flush_range(src, src + size);
469
470 if (bfin_addr_dcacheable(dst))
471 blackfin_dcache_invalidate_range(dst, dst + size);
472
473 bulk = size & ~0xffff;
474 rest = size - bulk;
475 if (bulk)
476 _dma_memcpy(pdst, psrc, bulk);
477 _dma_memcpy(pdst + bulk, psrc + bulk, rest);
478 return pdst;
479}
480EXPORT_SYMBOL(dma_memcpy);
481
482
483
484
485
486
487void *safe_dma_memcpy(void *dst, const void *src, size_t size)
488{
489 if (!access_ok(VERIFY_WRITE, dst, size))
490 return NULL;
491 if (!access_ok(VERIFY_READ, src, size))
492 return NULL;
493 return dma_memcpy(dst, src, size);
494}
495EXPORT_SYMBOL(safe_dma_memcpy);
496
497static void _dma_out(unsigned long addr, unsigned long buf, unsigned short len,
498 u16 size, u16 dma_size)
499{
500 blackfin_dcache_flush_range(buf, buf + len * size);
501 __dma_memcpy(addr, 0, buf, size, len, dma_size);
502}
503
504static void _dma_in(unsigned long addr, unsigned long buf, unsigned short len,
505 u16 size, u16 dma_size)
506{
507 blackfin_dcache_invalidate_range(buf, buf + len * size);
508 __dma_memcpy(buf, size, addr, 0, len, dma_size);
509}
510
511#define MAKE_DMA_IO(io, bwl, isize, dmasize, cnst) \
512void dma_##io##s##bwl(unsigned long addr, cnst void *buf, unsigned short len) \
513{ \
514 _dma_##io(addr, (unsigned long)buf, len, isize, WDSIZE_##dmasize); \
515} \
516EXPORT_SYMBOL(dma_##io##s##bwl)
517MAKE_DMA_IO(out, b, 1, 8, const);
518MAKE_DMA_IO(in, b, 1, 8, );
519MAKE_DMA_IO(out, w, 2, 16, const);
520MAKE_DMA_IO(in, w, 2, 16, );
521MAKE_DMA_IO(out, l, 4, 32, const);
522MAKE_DMA_IO(in, l, 4, 32, );
523