1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include "nouveau_drv.h"
28#include "nouveau_dma.h"
29#include "nouveau_vmm.h"
30
31void
32OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords)
33{
34 bool is_iomem;
35 u32 *mem = ttm_kmap_obj_virtual(&chan->push.buffer->kmap, &is_iomem);
36 mem = &mem[chan->dma.cur];
37 if (is_iomem)
38 memcpy_toio((void __force __iomem *)mem, data, nr_dwords * 4);
39 else
40 memcpy(mem, data, nr_dwords * 4);
41 chan->dma.cur += nr_dwords;
42}
43
44
45
46
47
48
49
50
51static inline int
52READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout)
53{
54 uint64_t val;
55
56 val = nvif_rd32(&chan->user, chan->user_get);
57 if (chan->user_get_hi)
58 val |= (uint64_t)nvif_rd32(&chan->user, chan->user_get_hi) << 32;
59
60
61
62
63
64 if (val != *prev_get) {
65 *prev_get = val;
66 *timeout = 0;
67 }
68
69 if ((++*timeout & 0xff) == 0) {
70 udelay(1);
71 if (*timeout > 100000)
72 return -EBUSY;
73 }
74
75 if (val < chan->push.addr ||
76 val > chan->push.addr + (chan->dma.max << 2))
77 return -EINVAL;
78
79 return (val - chan->push.addr) >> 2;
80}
81
82void
83nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
84 int delta, int length)
85{
86 struct nouveau_cli *cli = (void *)chan->user.client;
87 struct nouveau_bo *pb = chan->push.buffer;
88 struct nouveau_vma *vma;
89 int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base;
90 u64 offset;
91
92 vma = nouveau_vma_find(bo, &cli->vmm);
93 BUG_ON(!vma);
94 offset = vma->addr + delta;
95
96 BUG_ON(chan->dma.ib_free < 1);
97
98 nouveau_bo_wr32(pb, ip++, lower_32_bits(offset));
99 nouveau_bo_wr32(pb, ip++, upper_32_bits(offset) | length << 8);
100
101 chan->dma.ib_put = (chan->dma.ib_put + 1) & chan->dma.ib_max;
102
103 mb();
104
105 nouveau_bo_rd32(pb, 0);
106
107 nvif_wr32(&chan->user, 0x8c, chan->dma.ib_put);
108 chan->dma.ib_free--;
109}
110
111static int
112nv50_dma_push_wait(struct nouveau_channel *chan, int count)
113{
114 uint32_t cnt = 0, prev_get = 0;
115
116 while (chan->dma.ib_free < count) {
117 uint32_t get = nvif_rd32(&chan->user, 0x88);
118 if (get != prev_get) {
119 prev_get = get;
120 cnt = 0;
121 }
122
123 if ((++cnt & 0xff) == 0) {
124 DRM_UDELAY(1);
125 if (cnt > 100000)
126 return -EBUSY;
127 }
128
129 chan->dma.ib_free = get - chan->dma.ib_put;
130 if (chan->dma.ib_free <= 0)
131 chan->dma.ib_free += chan->dma.ib_max;
132 }
133
134 return 0;
135}
136
137static int
138nv50_dma_wait(struct nouveau_channel *chan, int slots, int count)
139{
140 uint64_t prev_get = 0;
141 int ret, cnt = 0;
142
143 ret = nv50_dma_push_wait(chan, slots + 1);
144 if (unlikely(ret))
145 return ret;
146
147 while (chan->dma.free < count) {
148 int get = READ_GET(chan, &prev_get, &cnt);
149 if (unlikely(get < 0)) {
150 if (get == -EINVAL)
151 continue;
152
153 return get;
154 }
155
156 if (get <= chan->dma.cur) {
157 chan->dma.free = chan->dma.max - chan->dma.cur;
158 if (chan->dma.free >= count)
159 break;
160
161 FIRE_RING(chan);
162 do {
163 get = READ_GET(chan, &prev_get, &cnt);
164 if (unlikely(get < 0)) {
165 if (get == -EINVAL)
166 continue;
167 return get;
168 }
169 } while (get == 0);
170 chan->dma.cur = 0;
171 chan->dma.put = 0;
172 }
173
174 chan->dma.free = get - chan->dma.cur - 1;
175 }
176
177 return 0;
178}
179
180int
181nouveau_dma_wait(struct nouveau_channel *chan, int slots, int size)
182{
183 uint64_t prev_get = 0;
184 int cnt = 0, get;
185
186 if (chan->dma.ib_max)
187 return nv50_dma_wait(chan, slots, size);
188
189 while (chan->dma.free < size) {
190 get = READ_GET(chan, &prev_get, &cnt);
191 if (unlikely(get == -EBUSY))
192 return -EBUSY;
193
194
195
196
197
198
199
200
201
202
203 if (unlikely(get == -EINVAL) || get < NOUVEAU_DMA_SKIPS)
204 continue;
205
206 if (get <= chan->dma.cur) {
207
208
209
210
211
212
213
214
215
216
217
218
219
220 chan->dma.free = chan->dma.max - chan->dma.cur;
221 if (chan->dma.free >= size)
222 break;
223
224
225
226
227
228 OUT_RING(chan, chan->push.addr | 0x20000000);
229
230
231
232
233
234
235 do {
236 get = READ_GET(chan, &prev_get, &cnt);
237 if (unlikely(get == -EBUSY))
238 return -EBUSY;
239 if (unlikely(get == -EINVAL))
240 continue;
241 } while (get <= NOUVEAU_DMA_SKIPS);
242 WRITE_PUT(NOUVEAU_DMA_SKIPS);
243
244
245
246
247 chan->dma.cur =
248 chan->dma.put = NOUVEAU_DMA_SKIPS;
249 }
250
251
252
253
254
255
256
257 chan->dma.free = get - chan->dma.cur - 1;
258 }
259
260 return 0;
261}
262
263