1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/slab.h>
25#include <linux/list.h>
26#include "kfd_device_queue_manager.h"
27#include "kfd_priv.h"
28#include "kfd_kernel_queue.h"
29
30static inline struct process_queue_node *get_queue_by_qid(
31 struct process_queue_manager *pqm, unsigned int qid)
32{
33 struct process_queue_node *pqn;
34
35 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
36 if ((pqn->q && pqn->q->properties.queue_id == qid) ||
37 (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
38 return pqn;
39 }
40
41 return NULL;
42}
43
44static int find_available_queue_slot(struct process_queue_manager *pqm,
45 unsigned int *qid)
46{
47 unsigned long found;
48
49 found = find_first_zero_bit(pqm->queue_slot_bitmap,
50 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
51
52 pr_debug("The new slot id %lu\n", found);
53
54 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
55 pr_info("Cannot open more queues for process with pasid %d\n",
56 pqm->process->pasid);
57 return -ENOMEM;
58 }
59
60 set_bit(found, pqm->queue_slot_bitmap);
61 *qid = found;
62
63 return 0;
64}
65
66void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
67{
68 struct kfd_dev *dev = pdd->dev;
69
70 if (pdd->already_dequeued)
71 return;
72
73 dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
74 pdd->already_dequeued = true;
75}
76
77void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
78{
79 struct kfd_process_device *pdd;
80
81 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
82 kfd_process_dequeue_from_device(pdd);
83}
84
85int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
86{
87 INIT_LIST_HEAD(&pqm->queues);
88 pqm->queue_slot_bitmap =
89 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
90 BITS_PER_BYTE), GFP_KERNEL);
91 if (!pqm->queue_slot_bitmap)
92 return -ENOMEM;
93 pqm->process = p;
94
95 return 0;
96}
97
98void pqm_uninit(struct process_queue_manager *pqm)
99{
100 struct process_queue_node *pqn, *next;
101
102 list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
103 uninit_queue(pqn->q);
104 list_del(&pqn->process_queue_list);
105 kfree(pqn);
106 }
107
108 kfree(pqm->queue_slot_bitmap);
109 pqm->queue_slot_bitmap = NULL;
110}
111
112static int create_cp_queue(struct process_queue_manager *pqm,
113 struct kfd_dev *dev, struct queue **q,
114 struct queue_properties *q_properties,
115 struct file *f, unsigned int qid)
116{
117 int retval;
118
119
120 q_properties->doorbell_ptr = NULL;
121
122
123 q_properties->vmid = 0;
124 q_properties->queue_id = qid;
125
126 retval = init_queue(q, q_properties);
127 if (retval != 0)
128 return retval;
129
130 (*q)->device = dev;
131 (*q)->process = pqm->process;
132
133 pr_debug("PQM After init queue");
134
135 return retval;
136}
137
138int pqm_create_queue(struct process_queue_manager *pqm,
139 struct kfd_dev *dev,
140 struct file *f,
141 struct queue_properties *properties,
142 unsigned int *qid)
143{
144 int retval;
145 struct kfd_process_device *pdd;
146 struct queue *q;
147 struct process_queue_node *pqn;
148 struct kernel_queue *kq;
149 enum kfd_queue_type type = properties->type;
150 unsigned int max_queues = 127;
151
152 q = NULL;
153 kq = NULL;
154
155 pdd = kfd_get_process_device_data(dev, pqm->process);
156 if (!pdd) {
157 pr_err("Process device data doesn't exist\n");
158 return -1;
159 }
160
161
162
163
164
165
166
167 if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
168 max_queues = dev->device_info->max_no_of_hqd/2;
169
170 if (pdd->qpd.queue_count >= max_queues)
171 return -ENOSPC;
172
173 retval = find_available_queue_slot(pqm, qid);
174 if (retval != 0)
175 return retval;
176
177 if (list_empty(&pdd->qpd.queues_list) &&
178 list_empty(&pdd->qpd.priv_queue_list))
179 dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
180
181 pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
182 if (!pqn) {
183 retval = -ENOMEM;
184 goto err_allocate_pqn;
185 }
186
187 switch (type) {
188 case KFD_QUEUE_TYPE_SDMA:
189 if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) {
190 pr_err("Over-subscription is not allowed for SDMA.\n");
191 retval = -EPERM;
192 goto err_create_queue;
193 }
194
195 retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
196 if (retval != 0)
197 goto err_create_queue;
198 pqn->q = q;
199 pqn->kq = NULL;
200 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
201 pr_debug("DQM returned %d for create_queue\n", retval);
202 print_queue(q);
203 break;
204
205 case KFD_QUEUE_TYPE_COMPUTE:
206
207 if ((dev->dqm->sched_policy ==
208 KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
209 ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
210 (dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
211 pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
212 retval = -EPERM;
213 goto err_create_queue;
214 }
215
216 retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
217 if (retval != 0)
218 goto err_create_queue;
219 pqn->q = q;
220 pqn->kq = NULL;
221 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
222 pr_debug("DQM returned %d for create_queue\n", retval);
223 print_queue(q);
224 break;
225 case KFD_QUEUE_TYPE_DIQ:
226 kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
227 if (!kq) {
228 retval = -ENOMEM;
229 goto err_create_queue;
230 }
231 kq->queue->properties.queue_id = *qid;
232 pqn->kq = kq;
233 pqn->q = NULL;
234 retval = dev->dqm->ops.create_kernel_queue(dev->dqm,
235 kq, &pdd->qpd);
236 break;
237 default:
238 WARN(1, "Invalid queue type %d", type);
239 retval = -EINVAL;
240 }
241
242 if (retval != 0) {
243 pr_err("Pasid %d DQM create queue %d failed. ret %d\n",
244 pqm->process->pasid, type, retval);
245 goto err_create_queue;
246 }
247
248 if (q)
249
250
251
252
253 properties->doorbell_off =
254 (q->properties.doorbell_off * sizeof(uint32_t)) &
255 (kfd_doorbell_process_slice(dev) - 1);
256
257 pr_debug("PQM After DQM create queue\n");
258
259 list_add(&pqn->process_queue_list, &pqm->queues);
260
261 if (q) {
262 pr_debug("PQM done creating queue\n");
263 print_queue_properties(&q->properties);
264 }
265
266 return retval;
267
268err_create_queue:
269 kfree(pqn);
270err_allocate_pqn:
271
272 clear_bit(*qid, pqm->queue_slot_bitmap);
273 if (list_empty(&pdd->qpd.queues_list) &&
274 list_empty(&pdd->qpd.priv_queue_list))
275 dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
276 return retval;
277}
278
279int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
280{
281 struct process_queue_node *pqn;
282 struct kfd_process_device *pdd;
283 struct device_queue_manager *dqm;
284 struct kfd_dev *dev;
285 int retval;
286
287 dqm = NULL;
288
289 retval = 0;
290
291 pqn = get_queue_by_qid(pqm, qid);
292 if (!pqn) {
293 pr_err("Queue id does not match any known queue\n");
294 return -EINVAL;
295 }
296
297 dev = NULL;
298 if (pqn->kq)
299 dev = pqn->kq->dev;
300 if (pqn->q)
301 dev = pqn->q->device;
302 if (WARN_ON(!dev))
303 return -ENODEV;
304
305 pdd = kfd_get_process_device_data(dev, pqm->process);
306 if (!pdd) {
307 pr_err("Process device data doesn't exist\n");
308 return -1;
309 }
310
311 if (pqn->kq) {
312
313 dqm = pqn->kq->dev->dqm;
314 dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
315 kernel_queue_uninit(pqn->kq);
316 }
317
318 if (pqn->q) {
319 dqm = pqn->q->device->dqm;
320 retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
321 if (retval) {
322 pr_err("Pasid %d destroy queue %d failed, ret %d\n",
323 pqm->process->pasid,
324 pqn->q->properties.queue_id, retval);
325 if (retval != -ETIME)
326 goto err_destroy_queue;
327 }
328 kfree(pqn->q->properties.cu_mask);
329 pqn->q->properties.cu_mask = NULL;
330 uninit_queue(pqn->q);
331 }
332
333 list_del(&pqn->process_queue_list);
334 kfree(pqn);
335 clear_bit(qid, pqm->queue_slot_bitmap);
336
337 if (list_empty(&pdd->qpd.queues_list) &&
338 list_empty(&pdd->qpd.priv_queue_list))
339 dqm->ops.unregister_process(dqm, &pdd->qpd);
340
341err_destroy_queue:
342 return retval;
343}
344
345int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
346 struct queue_properties *p)
347{
348 int retval;
349 struct process_queue_node *pqn;
350
351 pqn = get_queue_by_qid(pqm, qid);
352 if (!pqn) {
353 pr_debug("No queue %d exists for update operation\n", qid);
354 return -EFAULT;
355 }
356
357 pqn->q->properties.queue_address = p->queue_address;
358 pqn->q->properties.queue_size = p->queue_size;
359 pqn->q->properties.queue_percent = p->queue_percent;
360 pqn->q->properties.priority = p->priority;
361
362 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
363 pqn->q);
364 if (retval != 0)
365 return retval;
366
367 return 0;
368}
369
370int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
371 struct queue_properties *p)
372{
373 int retval;
374 struct process_queue_node *pqn;
375
376 pqn = get_queue_by_qid(pqm, qid);
377 if (!pqn) {
378 pr_debug("No queue %d exists for update operation\n", qid);
379 return -EFAULT;
380 }
381
382
383
384
385 kfree(pqn->q->properties.cu_mask);
386
387 pqn->q->properties.cu_mask_count = p->cu_mask_count;
388 pqn->q->properties.cu_mask = p->cu_mask;
389
390 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
391 pqn->q);
392 if (retval != 0)
393 return retval;
394
395 return 0;
396}
397
398struct kernel_queue *pqm_get_kernel_queue(
399 struct process_queue_manager *pqm,
400 unsigned int qid)
401{
402 struct process_queue_node *pqn;
403
404 pqn = get_queue_by_qid(pqm, qid);
405 if (pqn && pqn->kq)
406 return pqn->kq;
407
408 return NULL;
409}
410
411int pqm_get_wave_state(struct process_queue_manager *pqm,
412 unsigned int qid,
413 void __user *ctl_stack,
414 u32 *ctl_stack_used_size,
415 u32 *save_area_used_size)
416{
417 struct process_queue_node *pqn;
418
419 pqn = get_queue_by_qid(pqm, qid);
420 if (!pqn) {
421 pr_debug("amdkfd: No queue %d exists for operation\n",
422 qid);
423 return -EFAULT;
424 }
425
426 return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,
427 pqn->q,
428 ctl_stack,
429 ctl_stack_used_size,
430 save_area_used_size);
431}
432
433#if defined(CONFIG_DEBUG_FS)
434
435int pqm_debugfs_mqds(struct seq_file *m, void *data)
436{
437 struct process_queue_manager *pqm = data;
438 struct process_queue_node *pqn;
439 struct queue *q;
440 enum KFD_MQD_TYPE mqd_type;
441 struct mqd_manager *mqd_mgr;
442 int r = 0;
443
444 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
445 if (pqn->q) {
446 q = pqn->q;
447 switch (q->properties.type) {
448 case KFD_QUEUE_TYPE_SDMA:
449 seq_printf(m, " SDMA queue on device %x\n",
450 q->device->id);
451 mqd_type = KFD_MQD_TYPE_SDMA;
452 break;
453 case KFD_QUEUE_TYPE_COMPUTE:
454 seq_printf(m, " Compute queue on device %x\n",
455 q->device->id);
456 mqd_type = KFD_MQD_TYPE_CP;
457 break;
458 default:
459 seq_printf(m,
460 " Bad user queue type %d on device %x\n",
461 q->properties.type, q->device->id);
462 continue;
463 }
464 mqd_mgr = q->device->dqm->ops.get_mqd_manager(
465 q->device->dqm, mqd_type);
466 } else if (pqn->kq) {
467 q = pqn->kq->queue;
468 mqd_mgr = pqn->kq->mqd_mgr;
469 switch (q->properties.type) {
470 case KFD_QUEUE_TYPE_DIQ:
471 seq_printf(m, " DIQ on device %x\n",
472 pqn->kq->dev->id);
473 mqd_type = KFD_MQD_TYPE_HIQ;
474 break;
475 default:
476 seq_printf(m,
477 " Bad kernel queue type %d on device %x\n",
478 q->properties.type,
479 pqn->kq->dev->id);
480 continue;
481 }
482 } else {
483 seq_printf(m,
484 " Weird: Queue node with neither kernel nor user queue\n");
485 continue;
486 }
487
488 r = mqd_mgr->debugfs_show_mqd(m, q->mqd);
489 if (r != 0)
490 break;
491 }
492
493 return r;
494}
495
496#endif
497
