1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/printk.h>
25#include <linux/slab.h>
26#include <linux/mm_types.h>
27
28#include "kfd_priv.h"
29#include "kfd_mqd_manager.h"
30#include "vi_structs.h"
31#include "gca/gfx_8_0_sh_mask.h"
32#include "gca/gfx_8_0_enum.h"
33#include "oss/oss_3_0_sh_mask.h"
34#define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
35
36static inline struct vi_mqd *get_mqd(void *mqd)
37{
38 return (struct vi_mqd *)mqd;
39}
40
41static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd)
42{
43 return (struct vi_sdma_mqd *)mqd;
44}
45
46static int init_mqd(struct mqd_manager *mm, void **mqd,
47 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
48 struct queue_properties *q)
49{
50 int retval;
51 uint64_t addr;
52 struct vi_mqd *m;
53
54 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct vi_mqd),
55 mqd_mem_obj);
56 if (retval != 0)
57 return -ENOMEM;
58
59 m = (struct vi_mqd *) (*mqd_mem_obj)->cpu_ptr;
60 addr = (*mqd_mem_obj)->gpu_addr;
61
62 memset(m, 0, sizeof(struct vi_mqd));
63
64 m->header = 0xC0310800;
65 m->compute_pipelinestat_enable = 1;
66 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
67 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
68 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
69 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
70
71 m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
72 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
73
74 m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT |
75 MTYPE_UC << CP_MQD_CONTROL__MTYPE__SHIFT;
76
77 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
78 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
79
80 m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
81 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
82 10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
83
84 m->cp_hqd_pipe_priority = 1;
85 m->cp_hqd_queue_priority = 15;
86
87 m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT;
88
89 if (q->format == KFD_QUEUE_FORMAT_AQL)
90 m->cp_hqd_iq_rptr = 1;
91
92 if (q->tba_addr) {
93 m->compute_tba_lo = lower_32_bits(q->tba_addr >> 8);
94 m->compute_tba_hi = upper_32_bits(q->tba_addr >> 8);
95 m->compute_tma_lo = lower_32_bits(q->tma_addr >> 8);
96 m->compute_tma_hi = upper_32_bits(q->tma_addr >> 8);
97 m->compute_pgm_rsrc2 |=
98 (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
99 }
100
101 if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address) {
102 m->cp_hqd_persistent_state |=
103 (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
104 m->cp_hqd_ctx_save_base_addr_lo =
105 lower_32_bits(q->ctx_save_restore_area_address);
106 m->cp_hqd_ctx_save_base_addr_hi =
107 upper_32_bits(q->ctx_save_restore_area_address);
108 m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
109 m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
110 m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
111 m->cp_hqd_wg_state_offset = q->ctl_stack_size;
112 }
113
114 *mqd = m;
115 if (gart_addr)
116 *gart_addr = addr;
117 retval = mm->update_mqd(mm, m, q);
118
119 return retval;
120}
121
122static int load_mqd(struct mqd_manager *mm, void *mqd,
123 uint32_t pipe_id, uint32_t queue_id,
124 struct queue_properties *p, struct mm_struct *mms)
125{
126
127 uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
128 uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1);
129
130 return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
131 (uint32_t __user *)p->write_ptr,
132 wptr_shift, wptr_mask, mms);
133}
134
135static int __update_mqd(struct mqd_manager *mm, void *mqd,
136 struct queue_properties *q, unsigned int mtype,
137 unsigned int atc_bit)
138{
139 struct vi_mqd *m;
140
141 m = get_mqd(mqd);
142
143 m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT |
144 atc_bit << CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT |
145 mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT;
146 m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
147 pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
148
149 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
150 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
151
152 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
153 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
154 m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
155 m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
156
157 m->cp_hqd_pq_doorbell_control =
158 q->doorbell_off <<
159 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
160 pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
161 m->cp_hqd_pq_doorbell_control);
162
163 m->cp_hqd_eop_control = atc_bit << CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT |
164 mtype << CP_HQD_EOP_CONTROL__MTYPE__SHIFT;
165
166 m->cp_hqd_ib_control = atc_bit << CP_HQD_IB_CONTROL__IB_ATC__SHIFT |
167 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT |
168 mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT;
169
170
171
172
173
174
175
176
177 m->cp_hqd_eop_control |= min(0xA,
178 order_base_2(q->eop_ring_buffer_size / 4) - 1);
179 m->cp_hqd_eop_base_addr_lo =
180 lower_32_bits(q->eop_ring_buffer_address >> 8);
181 m->cp_hqd_eop_base_addr_hi =
182 upper_32_bits(q->eop_ring_buffer_address >> 8);
183
184 m->cp_hqd_iq_timer = atc_bit << CP_HQD_IQ_TIMER__IQ_ATC__SHIFT |
185 mtype << CP_HQD_IQ_TIMER__MTYPE__SHIFT;
186
187 m->cp_hqd_vmid = q->vmid;
188
189 if (q->format == KFD_QUEUE_FORMAT_AQL) {
190 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
191 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
192 }
193
194 if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)
195 m->cp_hqd_ctx_save_control =
196 atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
197 mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
198
199 q->is_active = (q->queue_size > 0 &&
200 q->queue_address != 0 &&
201 q->queue_percent > 0 &&
202 !q->is_evicted);
203
204 return 0;
205}
206
207
208static int update_mqd(struct mqd_manager *mm, void *mqd,
209 struct queue_properties *q)
210{
211 return __update_mqd(mm, mqd, q, MTYPE_CC, 1);
212}
213
214static int update_mqd_tonga(struct mqd_manager *mm, void *mqd,
215 struct queue_properties *q)
216{
217 return __update_mqd(mm, mqd, q, MTYPE_UC, 0);
218}
219
220static int destroy_mqd(struct mqd_manager *mm, void *mqd,
221 enum kfd_preempt_type type,
222 unsigned int timeout, uint32_t pipe_id,
223 uint32_t queue_id)
224{
225 return mm->dev->kfd2kgd->hqd_destroy
226 (mm->dev->kgd, mqd, type, timeout,
227 pipe_id, queue_id);
228}
229
230static void uninit_mqd(struct mqd_manager *mm, void *mqd,
231 struct kfd_mem_obj *mqd_mem_obj)
232{
233 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
234}
235
236static bool is_occupied(struct mqd_manager *mm, void *mqd,
237 uint64_t queue_address, uint32_t pipe_id,
238 uint32_t queue_id)
239{
240 return mm->dev->kfd2kgd->hqd_is_occupied(
241 mm->dev->kgd, queue_address,
242 pipe_id, queue_id);
243}
244
245static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
246 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
247 struct queue_properties *q)
248{
249 struct vi_mqd *m;
250 int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
251
252 if (retval != 0)
253 return retval;
254
255 m = get_mqd(*mqd);
256
257 m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
258 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
259
260 return retval;
261}
262
263static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
264 struct queue_properties *q)
265{
266 struct vi_mqd *m;
267 int retval = __update_mqd(mm, mqd, q, MTYPE_UC, 0);
268
269 if (retval != 0)
270 return retval;
271
272 m = get_mqd(mqd);
273 m->cp_hqd_vmid = q->vmid;
274 return retval;
275}
276
277static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
278 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
279 struct queue_properties *q)
280{
281 int retval;
282 struct vi_sdma_mqd *m;
283
284
285 retval = kfd_gtt_sa_allocate(mm->dev,
286 sizeof(struct vi_sdma_mqd),
287 mqd_mem_obj);
288
289 if (retval != 0)
290 return -ENOMEM;
291
292 m = (struct vi_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
293
294 memset(m, 0, sizeof(struct vi_sdma_mqd));
295
296 *mqd = m;
297 if (gart_addr != NULL)
298 *gart_addr = (*mqd_mem_obj)->gpu_addr;
299
300 retval = mm->update_mqd(mm, m, q);
301
302 return retval;
303}
304
305static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
306 struct kfd_mem_obj *mqd_mem_obj)
307{
308 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
309}
310
311static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
312 uint32_t pipe_id, uint32_t queue_id,
313 struct queue_properties *p, struct mm_struct *mms)
314{
315 return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
316 (uint32_t __user *)p->write_ptr,
317 mms);
318}
319
320static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
321 struct queue_properties *q)
322{
323 struct vi_sdma_mqd *m;
324
325 m = get_sdma_mqd(mqd);
326 m->sdmax_rlcx_rb_cntl = order_base_2(q->queue_size / 4)
327 << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
328 q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
329 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
330 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
331
332 m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
333 m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
334 m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
335 m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
336 m->sdmax_rlcx_doorbell =
337 q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
338
339 m->sdmax_rlcx_virtual_addr = q->sdma_vm_addr;
340
341 m->sdma_engine_id = q->sdma_engine_id;
342 m->sdma_queue_id = q->sdma_queue_id;
343
344 q->is_active = (q->queue_size > 0 &&
345 q->queue_address != 0 &&
346 q->queue_percent > 0 &&
347 !q->is_evicted);
348
349 return 0;
350}
351
352
353
354
355
356static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
357 enum kfd_preempt_type type,
358 unsigned int timeout, uint32_t pipe_id,
359 uint32_t queue_id)
360{
361 return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
362}
363
364static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
365 uint64_t queue_address, uint32_t pipe_id,
366 uint32_t queue_id)
367{
368 return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
369}
370
371#if defined(CONFIG_DEBUG_FS)
372
373static int debugfs_show_mqd(struct seq_file *m, void *data)
374{
375 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
376 data, sizeof(struct vi_mqd), false);
377 return 0;
378}
379
380static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
381{
382 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
383 data, sizeof(struct vi_sdma_mqd), false);
384 return 0;
385}
386
387#endif
388
389struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
390 struct kfd_dev *dev)
391{
392 struct mqd_manager *mqd;
393
394 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
395 return NULL;
396
397 mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
398 if (!mqd)
399 return NULL;
400
401 mqd->dev = dev;
402
403 switch (type) {
404 case KFD_MQD_TYPE_CP:
405 case KFD_MQD_TYPE_COMPUTE:
406 mqd->init_mqd = init_mqd;
407 mqd->uninit_mqd = uninit_mqd;
408 mqd->load_mqd = load_mqd;
409 mqd->update_mqd = update_mqd;
410 mqd->destroy_mqd = destroy_mqd;
411 mqd->is_occupied = is_occupied;
412#if defined(CONFIG_DEBUG_FS)
413 mqd->debugfs_show_mqd = debugfs_show_mqd;
414#endif
415 break;
416 case KFD_MQD_TYPE_HIQ:
417 mqd->init_mqd = init_mqd_hiq;
418 mqd->uninit_mqd = uninit_mqd;
419 mqd->load_mqd = load_mqd;
420 mqd->update_mqd = update_mqd_hiq;
421 mqd->destroy_mqd = destroy_mqd;
422 mqd->is_occupied = is_occupied;
423#if defined(CONFIG_DEBUG_FS)
424 mqd->debugfs_show_mqd = debugfs_show_mqd;
425#endif
426 break;
427 case KFD_MQD_TYPE_SDMA:
428 mqd->init_mqd = init_mqd_sdma;
429 mqd->uninit_mqd = uninit_mqd_sdma;
430 mqd->load_mqd = load_mqd_sdma;
431 mqd->update_mqd = update_mqd_sdma;
432 mqd->destroy_mqd = destroy_mqd_sdma;
433 mqd->is_occupied = is_occupied_sdma;
434#if defined(CONFIG_DEBUG_FS)
435 mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
436#endif
437 break;
438 default:
439 kfree(mqd);
440 return NULL;
441 }
442
443 return mqd;
444}
445
446struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
447 struct kfd_dev *dev)
448{
449 struct mqd_manager *mqd;
450
451 mqd = mqd_manager_init_vi(type, dev);
452 if (!mqd)
453 return NULL;
454 if ((type == KFD_MQD_TYPE_CP) || (type == KFD_MQD_TYPE_COMPUTE))
455 mqd->update_mqd = update_mqd_tonga;
456 return mqd;
457}
458
