1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#define nvkm_mem(p) container_of((p), struct nvkm_mem, memory)
23#include "mem.h"
24
25#include <core/memory.h>
26
27#include <nvif/if000a.h>
28#include <nvif/unpack.h>
29
30struct nvkm_mem {
31 struct nvkm_memory memory;
32 enum nvkm_memory_target target;
33 struct nvkm_mmu *mmu;
34 u64 pages;
35 struct page **mem;
36 union {
37 struct scatterlist *sgl;
38 dma_addr_t *dma;
39 };
40};
41
42static enum nvkm_memory_target
43nvkm_mem_target(struct nvkm_memory *memory)
44{
45 return nvkm_mem(memory)->target;
46}
47
48static u8
49nvkm_mem_page(struct nvkm_memory *memory)
50{
51 return PAGE_SHIFT;
52}
53
54static u64
55nvkm_mem_addr(struct nvkm_memory *memory)
56{
57 struct nvkm_mem *mem = nvkm_mem(memory);
58 if (mem->pages == 1 && mem->mem)
59 return mem->dma[0];
60 return ~0ULL;
61}
62
63static u64
64nvkm_mem_size(struct nvkm_memory *memory)
65{
66 return nvkm_mem(memory)->pages << PAGE_SHIFT;
67}
68
69static int
70nvkm_mem_map_dma(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
71 struct nvkm_vma *vma, void *argv, u32 argc)
72{
73 struct nvkm_mem *mem = nvkm_mem(memory);
74 struct nvkm_vmm_map map = {
75 .memory = &mem->memory,
76 .offset = offset,
77 .dma = mem->dma,
78 };
79 return nvkm_vmm_map(vmm, vma, argv, argc, &map);
80}
81
82static void *
83nvkm_mem_dtor(struct nvkm_memory *memory)
84{
85 struct nvkm_mem *mem = nvkm_mem(memory);
86 if (mem->mem) {
87 while (mem->pages--) {
88 dma_unmap_page(mem->mmu->subdev.device->dev,
89 mem->dma[mem->pages], PAGE_SIZE,
90 DMA_BIDIRECTIONAL);
91 __free_page(mem->mem[mem->pages]);
92 }
93 kvfree(mem->dma);
94 kvfree(mem->mem);
95 }
96 return mem;
97}
98
99static const struct nvkm_memory_func
100nvkm_mem_dma = {
101 .dtor = nvkm_mem_dtor,
102 .target = nvkm_mem_target,
103 .page = nvkm_mem_page,
104 .addr = nvkm_mem_addr,
105 .size = nvkm_mem_size,
106 .map = nvkm_mem_map_dma,
107};
108
109static int
110nvkm_mem_map_sgl(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
111 struct nvkm_vma *vma, void *argv, u32 argc)
112{
113 struct nvkm_mem *mem = nvkm_mem(memory);
114 struct nvkm_vmm_map map = {
115 .memory = &mem->memory,
116 .offset = offset,
117 .sgl = mem->sgl,
118 };
119 return nvkm_vmm_map(vmm, vma, argv, argc, &map);
120}
121
122static const struct nvkm_memory_func
123nvkm_mem_sgl = {
124 .dtor = nvkm_mem_dtor,
125 .target = nvkm_mem_target,
126 .page = nvkm_mem_page,
127 .addr = nvkm_mem_addr,
128 .size = nvkm_mem_size,
129 .map = nvkm_mem_map_sgl,
130};
131
132int
133nvkm_mem_map_host(struct nvkm_memory *memory, void **pmap)
134{
135 struct nvkm_mem *mem = nvkm_mem(memory);
136 if (mem->mem) {
137 *pmap = vmap(mem->mem, mem->pages, VM_MAP, PAGE_KERNEL);
138 return *pmap ? 0 : -EFAULT;
139 }
140 return -EINVAL;
141}
142
143static int
144nvkm_mem_new_host(struct nvkm_mmu *mmu, int type, u8 page, u64 size,
145 void *argv, u32 argc, struct nvkm_memory **pmemory)
146{
147 struct device *dev = mmu->subdev.device->dev;
148 union {
149 struct nvif_mem_ram_vn vn;
150 struct nvif_mem_ram_v0 v0;
151 } *args = argv;
152 int ret = -ENOSYS;
153 enum nvkm_memory_target target;
154 struct nvkm_mem *mem;
155 gfp_t gfp = GFP_USER | __GFP_ZERO;
156
157 if ( (mmu->type[type].type & NVKM_MEM_COHERENT) &&
158 !(mmu->type[type].type & NVKM_MEM_UNCACHED))
159 target = NVKM_MEM_TARGET_HOST;
160 else
161 target = NVKM_MEM_TARGET_NCOH;
162
163 if (page != PAGE_SHIFT)
164 return -EINVAL;
165
166 if (!(mem = kzalloc(sizeof(*mem), GFP_KERNEL)))
167 return -ENOMEM;
168 mem->target = target;
169 mem->mmu = mmu;
170 *pmemory = &mem->memory;
171
172 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
173 if (args->v0.dma) {
174 nvkm_memory_ctor(&nvkm_mem_dma, &mem->memory);
175 mem->dma = args->v0.dma;
176 } else {
177 nvkm_memory_ctor(&nvkm_mem_sgl, &mem->memory);
178 mem->sgl = args->v0.sgl;
179 }
180
181 if (!IS_ALIGNED(size, PAGE_SIZE))
182 return -EINVAL;
183 mem->pages = size >> PAGE_SHIFT;
184 return 0;
185 } else
186 if ( (ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
187 kfree(mem);
188 return ret;
189 }
190
191 nvkm_memory_ctor(&nvkm_mem_dma, &mem->memory);
192 size = ALIGN(size, PAGE_SIZE) >> PAGE_SHIFT;
193
194 if (!(mem->mem = kvmalloc_array(size, sizeof(*mem->mem), GFP_KERNEL)))
195 return -ENOMEM;
196 if (!(mem->dma = kvmalloc_array(size, sizeof(*mem->dma), GFP_KERNEL)))
197 return -ENOMEM;
198
199 if (mmu->dma_bits > 32)
200 gfp |= GFP_HIGHUSER;
201 else
202 gfp |= GFP_DMA32;
203
204 for (mem->pages = 0; size; size--, mem->pages++) {
205 struct page *p = alloc_page(gfp);
206 if (!p)
207 return -ENOMEM;
208
209 mem->dma[mem->pages] = dma_map_page(mmu->subdev.device->dev,
210 p, 0, PAGE_SIZE,
211 DMA_BIDIRECTIONAL);
212 if (dma_mapping_error(dev, mem->dma[mem->pages])) {
213 __free_page(p);
214 return -ENOMEM;
215 }
216
217 mem->mem[mem->pages] = p;
218 }
219
220 return 0;
221}
222
223int
224nvkm_mem_new_type(struct nvkm_mmu *mmu, int type, u8 page, u64 size,
225 void *argv, u32 argc, struct nvkm_memory **pmemory)
226{
227 struct nvkm_memory *memory = NULL;
228 int ret;
229
230 if (mmu->type[type].type & NVKM_MEM_VRAM) {
231 ret = mmu->func->mem.vram(mmu, type, page, size,
232 argv, argc, &memory);
233 } else {
234 ret = nvkm_mem_new_host(mmu, type, page, size,
235 argv, argc, &memory);
236 }
237
238 if (ret)
239 nvkm_memory_unref(&memory);
240 *pmemory = memory;
241 return ret;
242}
243