1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/printk.h>
25#include <linux/slab.h>
26#include "kfd_priv.h"
27#include "kfd_mqd_manager.h"
28#include "cik_regs.h"
29#include "cik_structs.h"
30
31static inline struct cik_mqd *get_mqd(void *mqd)
32{
33 return (struct cik_mqd *)mqd;
34}
35
36static int init_mqd(struct mqd_manager *mm, void **mqd,
37 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
38 struct queue_properties *q)
39{
40 uint64_t addr;
41 struct cik_mqd *m;
42 int retval;
43
44 BUG_ON(!mm || !q || !mqd);
45
46 pr_debug("kfd: In func %s\n", __func__);
47
48 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
49 mqd_mem_obj);
50
51 if (retval != 0)
52 return -ENOMEM;
53
54 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
55 addr = (*mqd_mem_obj)->gpu_addr;
56
57 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
58
59 m->header = 0xC0310800;
60 m->compute_pipelinestat_enable = 1;
61 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
62 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
63 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
64 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
65
66
67
68
69
70
71 m->cp_hqd_persistent_state =
72 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ;
73
74 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
75 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
76 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
77
78 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN;
79
80 m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE;
81
82 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
83 QUANTUM_DURATION(10);
84
85
86
87
88
89
90
91
92
93
94 m->cp_hqd_pipe_priority = 1;
95 m->cp_hqd_queue_priority = 15;
96
97 if (q->format == KFD_QUEUE_FORMAT_AQL)
98 m->cp_hqd_iq_rptr = AQL_ENABLE;
99
100 *mqd = m;
101 if (gart_addr != NULL)
102 *gart_addr = addr;
103 retval = mm->update_mqd(mm, m, q);
104
105 return retval;
106}
107
108static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
109 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
110 struct queue_properties *q)
111{
112 int retval;
113 struct cik_sdma_rlc_registers *m;
114
115 BUG_ON(!mm || !mqd || !mqd_mem_obj);
116
117 retval = kfd_gtt_sa_allocate(mm->dev,
118 sizeof(struct cik_sdma_rlc_registers),
119 mqd_mem_obj);
120
121 if (retval != 0)
122 return -ENOMEM;
123
124 m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr;
125
126 memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
127
128 *mqd = m;
129 if (gart_addr != NULL)
130 *gart_addr = (*mqd_mem_obj)->gpu_addr;
131
132 retval = mm->update_mqd(mm, m, q);
133
134 return retval;
135}
136
137static void uninit_mqd(struct mqd_manager *mm, void *mqd,
138 struct kfd_mem_obj *mqd_mem_obj)
139{
140 BUG_ON(!mm || !mqd);
141 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
142}
143
144static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
145 struct kfd_mem_obj *mqd_mem_obj)
146{
147 BUG_ON(!mm || !mqd);
148 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
149}
150
151static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
152 uint32_t queue_id, uint32_t __user *wptr)
153{
154 return mm->dev->kfd2kgd->hqd_load
155 (mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
156}
157
158static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
159 uint32_t pipe_id, uint32_t queue_id,
160 uint32_t __user *wptr)
161{
162 return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd);
163}
164
165static int update_mqd(struct mqd_manager *mm, void *mqd,
166 struct queue_properties *q)
167{
168 struct cik_mqd *m;
169
170 BUG_ON(!mm || !q || !mqd);
171
172 pr_debug("kfd: In func %s\n", __func__);
173
174 m = get_mqd(mqd);
175 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
176 DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
177
178
179
180
181
182 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
183 - 1 - 1;
184 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
185 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
186 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
187 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
188 m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
189 DOORBELL_OFFSET(q->doorbell_off);
190
191 m->cp_hqd_vmid = q->vmid;
192
193 if (q->format == KFD_QUEUE_FORMAT_AQL) {
194 m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
195 }
196
197 m->cp_hqd_active = 0;
198 q->is_active = false;
199 if (q->queue_size > 0 &&
200 q->queue_address != 0 &&
201 q->queue_percent > 0) {
202 m->cp_hqd_active = 1;
203 q->is_active = true;
204 }
205
206 return 0;
207}
208
209static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
210 struct queue_properties *q)
211{
212 struct cik_sdma_rlc_registers *m;
213
214 BUG_ON(!mm || !mqd || !q);
215
216 m = get_sdma_mqd(mqd);
217 m->sdma_rlc_rb_cntl =
218 SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) |
219 SDMA_RB_VMID(q->vmid) |
220 SDMA_RPTR_WRITEBACK_ENABLE |
221 SDMA_RPTR_WRITEBACK_TIMER(6);
222
223 m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8);
224 m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
225 m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
226 m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
227 m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE;
228 m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
229
230 m->sdma_engine_id = q->sdma_engine_id;
231 m->sdma_queue_id = q->sdma_queue_id;
232
233 q->is_active = false;
234 if (q->queue_size > 0 &&
235 q->queue_address != 0 &&
236 q->queue_percent > 0) {
237 m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE;
238 q->is_active = true;
239 }
240
241 return 0;
242}
243
244static int destroy_mqd(struct mqd_manager *mm, void *mqd,
245 enum kfd_preempt_type type,
246 unsigned int timeout, uint32_t pipe_id,
247 uint32_t queue_id)
248{
249 return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout,
250 pipe_id, queue_id);
251}
252
253
254
255
256
257static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
258 enum kfd_preempt_type type,
259 unsigned int timeout, uint32_t pipe_id,
260 uint32_t queue_id)
261{
262 return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
263}
264
265static bool is_occupied(struct mqd_manager *mm, void *mqd,
266 uint64_t queue_address, uint32_t pipe_id,
267 uint32_t queue_id)
268{
269
270 return mm->dev->kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address,
271 pipe_id, queue_id);
272
273}
274
275static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
276 uint64_t queue_address, uint32_t pipe_id,
277 uint32_t queue_id)
278{
279 return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
280}
281
282
283
284
285
286
287
288static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
289 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
290 struct queue_properties *q)
291{
292 uint64_t addr;
293 struct cik_mqd *m;
294 int retval;
295
296 BUG_ON(!mm || !q || !mqd || !mqd_mem_obj);
297
298 pr_debug("kfd: In func %s\n", __func__);
299
300 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
301 mqd_mem_obj);
302
303 if (retval != 0)
304 return -ENOMEM;
305
306 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
307 addr = (*mqd_mem_obj)->gpu_addr;
308
309 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
310
311 m->header = 0xC0310800;
312 m->compute_pipelinestat_enable = 1;
313 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
314 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
315 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
316 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
317
318 m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE |
319 PRELOAD_REQ;
320 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
321 QUANTUM_DURATION(10);
322
323 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
324 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
325 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
326
327 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
328
329
330
331
332
333
334
335
336
337
338 m->cp_hqd_pipe_priority = 1;
339 m->cp_hqd_queue_priority = 15;
340
341 *mqd = m;
342 if (gart_addr)
343 *gart_addr = addr;
344 retval = mm->update_mqd(mm, m, q);
345
346 return retval;
347}
348
349static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
350 struct queue_properties *q)
351{
352 struct cik_mqd *m;
353
354 BUG_ON(!mm || !q || !mqd);
355
356 pr_debug("kfd: In func %s\n", __func__);
357
358 m = get_mqd(mqd);
359 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
360 DEFAULT_MIN_AVAIL_SIZE |
361 PRIV_STATE |
362 KMD_QUEUE;
363
364
365
366
367
368 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
369 - 1 - 1;
370 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
371 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
372 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
373 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
374 m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
375 DOORBELL_OFFSET(q->doorbell_off);
376
377 m->cp_hqd_vmid = q->vmid;
378
379 m->cp_hqd_active = 0;
380 q->is_active = false;
381 if (q->queue_size > 0 &&
382 q->queue_address != 0 &&
383 q->queue_percent > 0) {
384 m->cp_hqd_active = 1;
385 q->is_active = true;
386 }
387
388 return 0;
389}
390
391struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
392{
393 struct cik_sdma_rlc_registers *m;
394
395 BUG_ON(!mqd);
396
397 m = (struct cik_sdma_rlc_registers *)mqd;
398
399 return m;
400}
401
402struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
403 struct kfd_dev *dev)
404{
405 struct mqd_manager *mqd;
406
407 BUG_ON(!dev);
408 BUG_ON(type >= KFD_MQD_TYPE_MAX);
409
410 pr_debug("kfd: In func %s\n", __func__);
411
412 mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
413 if (!mqd)
414 return NULL;
415
416 mqd->dev = dev;
417
418 switch (type) {
419 case KFD_MQD_TYPE_CP:
420 case KFD_MQD_TYPE_COMPUTE:
421 mqd->init_mqd = init_mqd;
422 mqd->uninit_mqd = uninit_mqd;
423 mqd->load_mqd = load_mqd;
424 mqd->update_mqd = update_mqd;
425 mqd->destroy_mqd = destroy_mqd;
426 mqd->is_occupied = is_occupied;
427 break;
428 case KFD_MQD_TYPE_HIQ:
429 mqd->init_mqd = init_mqd_hiq;
430 mqd->uninit_mqd = uninit_mqd;
431 mqd->load_mqd = load_mqd;
432 mqd->update_mqd = update_mqd_hiq;
433 mqd->destroy_mqd = destroy_mqd;
434 mqd->is_occupied = is_occupied;
435 break;
436 case KFD_MQD_TYPE_SDMA:
437 mqd->init_mqd = init_mqd_sdma;
438 mqd->uninit_mqd = uninit_mqd_sdma;
439 mqd->load_mqd = load_mqd_sdma;
440 mqd->update_mqd = update_mqd_sdma;
441 mqd->destroy_mqd = destroy_mqd_sdma;
442 mqd->is_occupied = is_occupied_sdma;
443 break;
444 default:
445 kfree(mqd);
446 return NULL;
447 }
448
449 return mqd;
450}
451
452