1#ifndef __NVKM_VMM_H__
2#define __NVKM_VMM_H__
3#include "priv.h"
4#include <core/memory.h>
5enum nvkm_memory_target;
6
7struct nvkm_vmm_pt {
8
9
10
11
12
13
14 struct nvkm_mmu_pt *pt[2];
15 u32 refs[2];
16
17
18
19
20
21
22 u8 page;
23
24
25
26
27
28 bool sparse:1;
29
30
31
32
33
34
35#define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde)
36#define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde)
37#define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY)
38 struct nvkm_vmm_pt **pde;
39
40
41
42
43
44
45
46
47#define NVKM_VMM_PTE_SPARSE 0x80
48#define NVKM_VMM_PTE_VALID 0x40
49#define NVKM_VMM_PTE_SPTES 0x3f
50 u8 pte[];
51};
52
53typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *,
54 struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
55typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *,
56 struct nvkm_vmm_pt *, u32 pdei);
57typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *,
58 u32 ptei, u32 ptes, struct nvkm_vmm_map *);
59
60struct nvkm_vmm_desc_func {
61 nvkm_vmm_pxe_func invalid;
62 nvkm_vmm_pxe_func unmap;
63 nvkm_vmm_pxe_func sparse;
64
65 nvkm_vmm_pde_func pde;
66
67 nvkm_vmm_pte_func mem;
68 nvkm_vmm_pte_func dma;
69 nvkm_vmm_pte_func sgl;
70};
71
72extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;
73void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32);
74extern const struct nvkm_vmm_desc_func gf100_vmm_pgt;
75void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
76void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
77 struct nvkm_vmm_map *);
78void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
79 struct nvkm_vmm_map *);
80void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
81 struct nvkm_vmm_map *);
82
83void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
84
85struct nvkm_vmm_desc {
86 enum {
87 PGD,
88 PGT,
89 SPT,
90 LPT,
91 } type;
92 u8 bits;
93 u8 size;
94 u32 align;
95 const struct nvkm_vmm_desc_func *func;
96};
97
98extern const struct nvkm_vmm_desc nv50_vmm_desc_12[];
99extern const struct nvkm_vmm_desc nv50_vmm_desc_16[];
100
101extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[];
102extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[];
103extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[];
104extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[];
105
106extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[];
107extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[];
108extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[];
109extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[];
110
111extern const struct nvkm_vmm_desc gp100_vmm_desc_12[];
112extern const struct nvkm_vmm_desc gp100_vmm_desc_16[];
113
114struct nvkm_vmm_page {
115 u8 shift;
116 const struct nvkm_vmm_desc *desc;
117#define NVKM_VMM_PAGE_SPARSE 0x01
118#define NVKM_VMM_PAGE_VRAM 0x02
119#define NVKM_VMM_PAGE_HOST 0x04
120#define NVKM_VMM_PAGE_COMP 0x08
121#define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE)
122#define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM)
123#define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM)
124#define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST)
125#define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST)
126#define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST)
127#define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST)
128#define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP)
129#define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP)
130#define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP)
131#define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP)
132 u8 type;
133};
134
135struct nvkm_vmm_func {
136 int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst);
137 void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst);
138
139 int (*aper)(enum nvkm_memory_target);
140 int (*valid)(struct nvkm_vmm *, void *argv, u32 argc,
141 struct nvkm_vmm_map *);
142 void (*flush)(struct nvkm_vmm *, int depth);
143
144 u64 page_block;
145 const struct nvkm_vmm_page page[];
146};
147
148struct nvkm_vmm_join {
149 struct nvkm_memory *inst;
150 struct list_head head;
151};
152
153int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *,
154 u32 pd_header, u64 addr, u64 size, struct lock_class_key *,
155 const char *name, struct nvkm_vmm **);
156int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *,
157 u32 pd_header, u64 addr, u64 size, struct lock_class_key *,
158 const char *name, struct nvkm_vmm *);
159struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr);
160int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
161 bool sparse, u8 page, u8 align, u64 size,
162 struct nvkm_vma **pvma);
163void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
164void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *);
165void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma);
166
167struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
168void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *);
169
170int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32,
171 u64, u64, void *, u32, struct lock_class_key *,
172 const char *, struct nvkm_vmm **);
173int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
174
175int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
176void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
177int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
178void nv50_vmm_flush(struct nvkm_vmm *, int);
179
180int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
181 struct nvkm_mmu *, u64, u64, void *, u32,
182 struct lock_class_key *, const char *, struct nvkm_vmm **);
183int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
184int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
185void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
186int gf100_vmm_aper(enum nvkm_memory_target);
187int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
188void gf100_vmm_flush_(struct nvkm_vmm *, int);
189void gf100_vmm_flush(struct nvkm_vmm *, int);
190
191int gk20a_vmm_aper(enum nvkm_memory_target);
192
193int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
194 struct nvkm_mmu *, u64, u64, void *, u32,
195 struct lock_class_key *, const char *, struct nvkm_vmm **);
196int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
197int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
198
199int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
200int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
201void gp100_vmm_flush(struct nvkm_vmm *, int);
202
203int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
204 struct lock_class_key *, const char *, struct nvkm_vmm **);
205int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
206 struct lock_class_key *, const char *, struct nvkm_vmm **);
207int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
208 struct lock_class_key *, const char *, struct nvkm_vmm **);
209int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
210 struct lock_class_key *, const char *, struct nvkm_vmm **);
211int mcp77_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
212 struct lock_class_key *, const char *, struct nvkm_vmm **);
213int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
214 struct lock_class_key *, const char *, struct nvkm_vmm **);
215int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
216 struct lock_class_key *, const char *, struct nvkm_vmm **);
217int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
218 struct lock_class_key *, const char *, struct nvkm_vmm **);
219int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
220 struct lock_class_key *, const char *, struct nvkm_vmm **);
221int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32,
222 struct lock_class_key *, const char *,
223 struct nvkm_vmm **);
224int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
225 struct lock_class_key *, const char *,
226 struct nvkm_vmm **);
227int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32,
228 struct lock_class_key *, const char *,
229 struct nvkm_vmm **);
230int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
231 struct lock_class_key *, const char *,
232 struct nvkm_vmm **);
233int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
234 struct lock_class_key *, const char *,
235 struct nvkm_vmm **);
236int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
237 struct lock_class_key *, const char *,
238 struct nvkm_vmm **);
239
240#define VMM_PRINT(l,v,p,f,a...) do { \
241 struct nvkm_vmm *_vmm = (v); \
242 if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \
243 nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \
244 _vmm->name, ##a); \
245 } \
246} while(0)
247#define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a)
248#define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a)
249#define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a)
250
251#define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \
252 nvkm_kmap((PT)->memory); \
253 while (PTEN) { \
254 u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \
255 u64 _addr = ((BASE) + MAP->off); \
256 \
257 if (_ptes > PTEN) { \
258 MAP->off += PTEN << MAP->page->shift; \
259 _ptes = PTEN; \
260 } else { \
261 MAP->off = 0; \
262 NEXT; \
263 } \
264 \
265 VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \
266 \
267 FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \
268 PTEI += _ptes; \
269 PTEN -= _ptes; \
270 }; \
271 nvkm_done((PT)->memory); \
272} while(0)
273
274#define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \
275 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
276 ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \
277 ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \
278 (MAP->mem = MAP->mem->next))
279#define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \
280 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
281 *MAP->dma, PAGE_SIZE, MAP->dma++)
282#define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \
283 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
284 sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \
285 (MAP->sgl = sg_next(MAP->sgl)))
286
287#define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c))
288#define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d))
289#define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \
290 const u32 _pteo = (o); u##b _data = (d); \
291 VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \
292 VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \
293} while(0)
294
295#define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x")
296#define VMM_FO032(m,v,o,d,c) \
297 VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c))
298
299#define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx")
300#define VMM_FO064(m,v,o,d,c) \
301 VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c))
302
303#define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \
304 u32 _pteo = (o), _ptes = (c); \
305 const u64 _addr = (m)->addr + _pteo; \
306 VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \
307 while (_ptes--) { \
308 nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \
309 nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \
310 _pteo += 0x10; \
311 } \
312} while(0)
313
314#define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "")
315#define VMM_FO128(m,v,o,lo,hi,c) do { \
316 nvkm_kmap((m)->memory); \
317 VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \
318 nvkm_done((m)->memory); \
319} while(0)
320#endif
321