LXR linux/drivers/gpu/drm/amd/amdkfd/kfd_device_queue

   1/*
   2 * Copyright 2014 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#ifndef KFD_DEVICE_QUEUE_MANAGER_H_
  25#define KFD_DEVICE_QUEUE_MANAGER_H_
  26
  27#include <linux/rwsem.h>
  28#include <linux/list.h>
  29#include <linux/mutex.h>
  30#include <linux/sched/mm.h>
  31#include "kfd_priv.h"
  32#include "kfd_mqd_manager.h"
  33
  34#define KFD_UNMAP_LATENCY_MS                    (4000)
  35#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
  36
  37struct device_process_node {
  38        struct qcm_process_device *qpd;
  39        struct list_head list;
  40};
  41
  42/**
  43 * struct device_queue_manager_ops
  44 *
  45 * @create_queue: Queue creation routine.
  46 *
  47 * @destroy_queue: Queue destruction routine.
  48 *
  49 * @update_queue: Queue update routine.
  50 *
  51 * @get_mqd_manager: Returns the mqd manager according to the mqd type.
  52 *
  53 * @exeute_queues: Dispatches the queues list to the H/W.
  54 *
  55 * @register_process: This routine associates a specific process with device.
  56 *
  57 * @unregister_process: destroys the associations between process to device.
  58 *
  59 * @initialize: Initializes the pipelines and memory module for that device.
  60 *
  61 * @start: Initializes the resources/modules the the device needs for queues
  62 * execution. This function is called on device initialization and after the
  63 * system woke up after suspension.
  64 *
  65 * @stop: This routine stops execution of all the active queue running on the
  66 * H/W and basically this function called on system suspend.
  67 *
  68 * @uninitialize: Destroys all the device queue manager resources allocated in
  69 * initialize routine.
  70 *
  71 * @create_kernel_queue: Creates kernel queue. Used for debug queue.
  72 *
  73 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
  74 *
  75 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
  76 * memory apertures.
  77 *
  78 * @process_termination: Clears all process queues belongs to that device.
  79 *
  80 * @evict_process_queues: Evict all active queues of a process
  81 *
  82 * @restore_process_queues: Restore all evicted queues queues of a process
  83 *
  84 * @get_wave_state: Retrieves context save state and optionally copies the
  85 * control stack, if kept in the MQD, to the given userspace address.
  86 */
  87
  88struct device_queue_manager_ops {
  89        int     (*create_queue)(struct device_queue_manager *dqm,
  90                                struct queue *q,
  91                                struct qcm_process_device *qpd);
  92
  93        int     (*destroy_queue)(struct device_queue_manager *dqm,
  94                                struct qcm_process_device *qpd,
  95                                struct queue *q);
  96
  97        int     (*update_queue)(struct device_queue_manager *dqm,
  98                                struct queue *q);
  99
 100        struct mqd_manager * (*get_mqd_manager)
 101                                        (struct device_queue_manager *dqm,
 102                                        enum KFD_MQD_TYPE type);
 103
 104        int     (*register_process)(struct device_queue_manager *dqm,
 105                                        struct qcm_process_device *qpd);
 106
 107        int     (*unregister_process)(struct device_queue_manager *dqm,
 108                                        struct qcm_process_device *qpd);
 109
 110        int     (*initialize)(struct device_queue_manager *dqm);
 111        int     (*start)(struct device_queue_manager *dqm);
 112        int     (*stop)(struct device_queue_manager *dqm);
 113        void    (*uninitialize)(struct device_queue_manager *dqm);
 114        int     (*create_kernel_queue)(struct device_queue_manager *dqm,
 115                                        struct kernel_queue *kq,
 116                                        struct qcm_process_device *qpd);
 117
 118        void    (*destroy_kernel_queue)(struct device_queue_manager *dqm,
 119                                        struct kernel_queue *kq,
 120                                        struct qcm_process_device *qpd);
 121
 122        bool    (*set_cache_memory_policy)(struct device_queue_manager *dqm,
 123                                           struct qcm_process_device *qpd,
 124                                           enum cache_policy default_policy,
 125                                           enum cache_policy alternate_policy,
 126                                           void __user *alternate_aperture_base,
 127                                           uint64_t alternate_aperture_size);
 128
 129        int     (*set_trap_handler)(struct device_queue_manager *dqm,
 130                                    struct qcm_process_device *qpd,
 131                                    uint64_t tba_addr,
 132                                    uint64_t tma_addr);
 133
 134        int (*process_termination)(struct device_queue_manager *dqm,
 135                        struct qcm_process_device *qpd);
 136
 137        int (*evict_process_queues)(struct device_queue_manager *dqm,
 138                                    struct qcm_process_device *qpd);
 139        int (*restore_process_queues)(struct device_queue_manager *dqm,
 140                                      struct qcm_process_device *qpd);
 141
 142        int     (*get_wave_state)(struct device_queue_manager *dqm,
 143                                  struct queue *q,
 144                                  void __user *ctl_stack,
 145                                  u32 *ctl_stack_used_size,
 146                                  u32 *save_area_used_size);
 147};
 148
 149struct device_queue_manager_asic_ops {
 150        int     (*update_qpd)(struct device_queue_manager *dqm,
 151                                        struct qcm_process_device *qpd);
 152        bool    (*set_cache_memory_policy)(struct device_queue_manager *dqm,
 153                                           struct qcm_process_device *qpd,
 154                                           enum cache_policy default_policy,
 155                                           enum cache_policy alternate_policy,
 156                                           void __user *alternate_aperture_base,
 157                                           uint64_t alternate_aperture_size);
 158        void    (*init_sdma_vm)(struct device_queue_manager *dqm,
 159                                struct queue *q,
 160                                struct qcm_process_device *qpd);
 161};
 162
 163/**
 164 * struct device_queue_manager
 165 *
 166 * This struct is a base class for the kfd queues scheduler in the
 167 * device level. The device base class should expose the basic operations
 168 * for queue creation and queue destruction. This base class hides the
 169 * scheduling mode of the driver and the specific implementation of the
 170 * concrete device. This class is the only class in the queues scheduler
 171 * that configures the H/W.
 172 *
 173 */
 174
 175struct device_queue_manager {
 176        struct device_queue_manager_ops ops;
 177        struct device_queue_manager_asic_ops asic_ops;
 178
 179        struct mqd_manager      *mqd_mgrs[KFD_MQD_TYPE_MAX];
 180        struct packet_manager   packets;
 181        struct kfd_dev          *dev;
 182        struct mutex            lock_hidden; /* use dqm_lock/unlock(dqm) */
 183        struct list_head        queues;
 184        unsigned int            saved_flags;
 185        unsigned int            processes_count;
 186        unsigned int            queue_count;
 187        unsigned int            sdma_queue_count;
 188        unsigned int            total_queue_count;
 189        unsigned int            next_pipe_to_allocate;
 190        unsigned int            *allocated_queues;
 191        unsigned int            sdma_bitmap;
 192        unsigned int            vmid_bitmap;
 193        uint64_t                pipelines_addr;
 194        struct kfd_mem_obj      *pipeline_mem;
 195        uint64_t                fence_gpu_addr;
 196        unsigned int            *fence_addr;
 197        struct kfd_mem_obj      *fence_mem;
 198        bool                    active_runlist;
 199        int                     sched_policy;
 200
 201        /* hw exception  */
 202        bool                    is_hws_hang;
 203        struct work_struct      hw_exception_work;
 204};
 205
 206void device_queue_manager_init_cik(
 207                struct device_queue_manager_asic_ops *asic_ops);
 208void device_queue_manager_init_cik_hawaii(
 209                struct device_queue_manager_asic_ops *asic_ops);
 210void device_queue_manager_init_vi(
 211                struct device_queue_manager_asic_ops *asic_ops);
 212void device_queue_manager_init_vi_tonga(
 213                struct device_queue_manager_asic_ops *asic_ops);
 214void device_queue_manager_init_v9(
 215                struct device_queue_manager_asic_ops *asic_ops);
 216void program_sh_mem_settings(struct device_queue_manager *dqm,
 217                                        struct qcm_process_device *qpd);
 218unsigned int get_queues_num(struct device_queue_manager *dqm);
 219unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
 220unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
 221unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
 222
 223static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
 224{
 225        return (pdd->lds_base >> 16) & 0xFF;
 226}
 227
 228static inline unsigned int
 229get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
 230{
 231        return (pdd->lds_base >> 60) & 0x0E;
 232}
 233
 234/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
 235 * happens while holding this lock anywhere to prevent deadlocks when
 236 * an MMU notifier runs in reclaim-FS context.
 237 */
 238static inline void dqm_lock(struct device_queue_manager *dqm)
 239{
 240        mutex_lock(&dqm->lock_hidden);
 241        dqm->saved_flags = memalloc_nofs_save();
 242}
 243static inline void dqm_unlock(struct device_queue_manager *dqm)
 244{
 245        memalloc_nofs_restore(dqm->saved_flags);
 246        mutex_unlock(&dqm->lock_hidden);
 247}
 248
 249#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
 250