1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <drm/drmP.h>
29#include <drm/radeon_drm.h>
30#include "radeon.h"
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65int radeon_gart_table_ram_alloc(struct radeon_device *rdev)
66{
67 void *ptr;
68
69 ptr = pci_alloc_consistent(rdev->pdev, rdev->gart.table_size,
70 &rdev->gart.table_addr);
71 if (ptr == NULL) {
72 return -ENOMEM;
73 }
74#ifdef CONFIG_X86
75 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
76 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
77 set_memory_uc((unsigned long)ptr,
78 rdev->gart.table_size >> PAGE_SHIFT);
79 }
80#endif
81 rdev->gart.ptr = ptr;
82 memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size);
83 return 0;
84}
85
86
87
88
89
90
91
92
93
94
95void radeon_gart_table_ram_free(struct radeon_device *rdev)
96{
97 if (rdev->gart.ptr == NULL) {
98 return;
99 }
100#ifdef CONFIG_X86
101 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
102 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
103 set_memory_wb((unsigned long)rdev->gart.ptr,
104 rdev->gart.table_size >> PAGE_SHIFT);
105 }
106#endif
107 pci_free_consistent(rdev->pdev, rdev->gart.table_size,
108 (void *)rdev->gart.ptr,
109 rdev->gart.table_addr);
110 rdev->gart.ptr = NULL;
111 rdev->gart.table_addr = 0;
112}
113
114
115
116
117
118
119
120
121
122
123
124int radeon_gart_table_vram_alloc(struct radeon_device *rdev)
125{
126 int r;
127
128 if (rdev->gart.robj == NULL) {
129 r = radeon_bo_create(rdev, rdev->gart.table_size,
130 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
131 0, NULL, NULL, &rdev->gart.robj);
132 if (r) {
133 return r;
134 }
135 }
136 return 0;
137}
138
139
140
141
142
143
144
145
146
147
148
149int radeon_gart_table_vram_pin(struct radeon_device *rdev)
150{
151 uint64_t gpu_addr;
152 int r;
153
154 r = radeon_bo_reserve(rdev->gart.robj, false);
155 if (unlikely(r != 0))
156 return r;
157 r = radeon_bo_pin(rdev->gart.robj,
158 RADEON_GEM_DOMAIN_VRAM, &gpu_addr);
159 if (r) {
160 radeon_bo_unreserve(rdev->gart.robj);
161 return r;
162 }
163 r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr);
164 if (r)
165 radeon_bo_unpin(rdev->gart.robj);
166 radeon_bo_unreserve(rdev->gart.robj);
167 rdev->gart.table_addr = gpu_addr;
168
169 if (!r) {
170 int i;
171
172
173
174
175 for (i = 0; i < rdev->gart.num_gpu_pages; i++)
176 radeon_gart_set_page(rdev, i, rdev->gart.pages_entry[i]);
177 mb();
178 radeon_gart_tlb_flush(rdev);
179 }
180
181 return r;
182}
183
184
185
186
187
188
189
190
191
192void radeon_gart_table_vram_unpin(struct radeon_device *rdev)
193{
194 int r;
195
196 if (rdev->gart.robj == NULL) {
197 return;
198 }
199 r = radeon_bo_reserve(rdev->gart.robj, false);
200 if (likely(r == 0)) {
201 radeon_bo_kunmap(rdev->gart.robj);
202 radeon_bo_unpin(rdev->gart.robj);
203 radeon_bo_unreserve(rdev->gart.robj);
204 rdev->gart.ptr = NULL;
205 }
206}
207
208
209
210
211
212
213
214
215
216
217void radeon_gart_table_vram_free(struct radeon_device *rdev)
218{
219 if (rdev->gart.robj == NULL) {
220 return;
221 }
222 radeon_bo_unref(&rdev->gart.robj);
223}
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
239 int pages)
240{
241 unsigned t;
242 unsigned p;
243 int i, j;
244
245 if (!rdev->gart.ready) {
246 WARN(1, "trying to unbind memory from uninitialized GART !\n");
247 return;
248 }
249 t = offset / RADEON_GPU_PAGE_SIZE;
250 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
251 for (i = 0; i < pages; i++, p++) {
252 if (rdev->gart.pages[p]) {
253 rdev->gart.pages[p] = NULL;
254 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
255 rdev->gart.pages_entry[t] = rdev->dummy_page.entry;
256 if (rdev->gart.ptr) {
257 radeon_gart_set_page(rdev, t,
258 rdev->dummy_page.entry);
259 }
260 }
261 }
262 }
263 if (rdev->gart.ptr) {
264 mb();
265 radeon_gart_tlb_flush(rdev);
266 }
267}
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
284 int pages, struct page **pagelist, dma_addr_t *dma_addr,
285 uint32_t flags)
286{
287 unsigned t;
288 unsigned p;
289 uint64_t page_base, page_entry;
290 int i, j;
291
292 if (!rdev->gart.ready) {
293 WARN(1, "trying to bind memory to uninitialized GART !\n");
294 return -EINVAL;
295 }
296 t = offset / RADEON_GPU_PAGE_SIZE;
297 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
298
299 for (i = 0; i < pages; i++, p++) {
300 rdev->gart.pages[p] = pagelist[i];
301 page_base = dma_addr[i];
302 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
303 page_entry = radeon_gart_get_page_entry(page_base, flags);
304 rdev->gart.pages_entry[t] = page_entry;
305 if (rdev->gart.ptr) {
306 radeon_gart_set_page(rdev, t, page_entry);
307 }
308 page_base += RADEON_GPU_PAGE_SIZE;
309 }
310 }
311 if (rdev->gart.ptr) {
312 mb();
313 radeon_gart_tlb_flush(rdev);
314 }
315 return 0;
316}
317
318
319
320
321
322
323
324
325
326int radeon_gart_init(struct radeon_device *rdev)
327{
328 int r, i;
329
330 if (rdev->gart.pages) {
331 return 0;
332 }
333
334 if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) {
335 DRM_ERROR("Page size is smaller than GPU page size!\n");
336 return -EINVAL;
337 }
338 r = radeon_dummy_page_init(rdev);
339 if (r)
340 return r;
341
342 rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE;
343 rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE;
344 DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
345 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages);
346
347 rdev->gart.pages = vzalloc(sizeof(void *) * rdev->gart.num_cpu_pages);
348 if (rdev->gart.pages == NULL) {
349 radeon_gart_fini(rdev);
350 return -ENOMEM;
351 }
352 rdev->gart.pages_entry = vmalloc(sizeof(uint64_t) *
353 rdev->gart.num_gpu_pages);
354 if (rdev->gart.pages_entry == NULL) {
355 radeon_gart_fini(rdev);
356 return -ENOMEM;
357 }
358
359 for (i = 0; i < rdev->gart.num_gpu_pages; i++)
360 rdev->gart.pages_entry[i] = rdev->dummy_page.entry;
361 return 0;
362}
363
364
365
366
367
368
369
370
371void radeon_gart_fini(struct radeon_device *rdev)
372{
373 if (rdev->gart.ready) {
374
375 radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages);
376 }
377 rdev->gart.ready = false;
378 vfree(rdev->gart.pages);
379 vfree(rdev->gart.pages_entry);
380 rdev->gart.pages = NULL;
381 rdev->gart.pages_entry = NULL;
382
383 radeon_dummy_page_fini(rdev);
384}
385