1
2
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/device.h>
8#include <linux/sched/task.h>
9#include <linux/intel-svm.h>
10#include <linux/io-64-nonatomic-lo-hi.h>
11#include <linux/cdev.h>
12#include <linux/fs.h>
13#include <linux/poll.h>
14#include <linux/iommu.h>
15#include <uapi/linux/idxd.h>
16#include "registers.h"
17#include "idxd.h"
18
19struct idxd_cdev_context {
20 const char *name;
21 dev_t devt;
22 struct ida minor_ida;
23};
24
25
26
27
28
29static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
30 { .name = "dsa" },
31 { .name = "iax" }
32};
33
34struct idxd_user_context {
35 struct idxd_wq *wq;
36 struct task_struct *task;
37 unsigned int pasid;
38 unsigned int flags;
39 struct iommu_sva *sva;
40};
41
42static void idxd_cdev_dev_release(struct device *dev)
43{
44 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
45 struct idxd_cdev_context *cdev_ctx;
46 struct idxd_wq *wq = idxd_cdev->wq;
47
48 cdev_ctx = &ictx[wq->idxd->data->type];
49 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
50 kfree(idxd_cdev);
51}
52
53static struct device_type idxd_cdev_device_type = {
54 .name = "idxd_cdev",
55 .release = idxd_cdev_dev_release,
56};
57
58static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
59{
60 struct cdev *cdev = inode->i_cdev;
61
62 return container_of(cdev, struct idxd_cdev, cdev);
63}
64
65static inline struct idxd_wq *inode_wq(struct inode *inode)
66{
67 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
68
69 return idxd_cdev->wq;
70}
71
72static int idxd_cdev_open(struct inode *inode, struct file *filp)
73{
74 struct idxd_user_context *ctx;
75 struct idxd_device *idxd;
76 struct idxd_wq *wq;
77 struct device *dev;
78 int rc = 0;
79 struct iommu_sva *sva;
80 unsigned int pasid;
81
82 wq = inode_wq(inode);
83 idxd = wq->idxd;
84 dev = &idxd->pdev->dev;
85
86 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
87
88 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
89 if (!ctx)
90 return -ENOMEM;
91
92 mutex_lock(&wq->wq_lock);
93
94 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
95 rc = -EBUSY;
96 goto failed;
97 }
98
99 ctx->wq = wq;
100 filp->private_data = ctx;
101
102 if (device_pasid_enabled(idxd)) {
103 sva = iommu_sva_bind_device(dev, current->mm, NULL);
104 if (IS_ERR(sva)) {
105 rc = PTR_ERR(sva);
106 dev_err(dev, "pasid allocation failed: %d\n", rc);
107 goto failed;
108 }
109
110 pasid = iommu_sva_get_pasid(sva);
111 if (pasid == IOMMU_PASID_INVALID) {
112 iommu_sva_unbind_device(sva);
113 rc = -EINVAL;
114 goto failed;
115 }
116
117 ctx->sva = sva;
118 ctx->pasid = pasid;
119
120 if (wq_dedicated(wq)) {
121 rc = idxd_wq_set_pasid(wq, pasid);
122 if (rc < 0) {
123 iommu_sva_unbind_device(sva);
124 dev_err(dev, "wq set pasid failed: %d\n", rc);
125 goto failed;
126 }
127 }
128 }
129
130 idxd_wq_get(wq);
131 mutex_unlock(&wq->wq_lock);
132 return 0;
133
134 failed:
135 mutex_unlock(&wq->wq_lock);
136 kfree(ctx);
137 return rc;
138}
139
140static int idxd_cdev_release(struct inode *node, struct file *filep)
141{
142 struct idxd_user_context *ctx = filep->private_data;
143 struct idxd_wq *wq = ctx->wq;
144 struct idxd_device *idxd = wq->idxd;
145 struct device *dev = &idxd->pdev->dev;
146 int rc;
147
148 dev_dbg(dev, "%s called\n", __func__);
149 filep->private_data = NULL;
150
151
152 if (wq_shared(wq)) {
153 idxd_device_drain_pasid(idxd, ctx->pasid);
154 } else {
155 if (device_pasid_enabled(idxd)) {
156
157 rc = idxd_wq_disable_pasid(wq);
158 if (rc < 0)
159 dev_err(dev, "wq disable pasid failed.\n");
160 } else {
161 idxd_wq_drain(wq);
162 }
163 }
164
165 if (ctx->sva)
166 iommu_sva_unbind_device(ctx->sva);
167 kfree(ctx);
168 mutex_lock(&wq->wq_lock);
169 idxd_wq_put(wq);
170 mutex_unlock(&wq->wq_lock);
171 return 0;
172}
173
174static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
175 const char *func)
176{
177 struct device *dev = &wq->idxd->pdev->dev;
178
179 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
180 dev_info_ratelimited(dev,
181 "%s: %s: mapping too large: %lu\n",
182 current->comm, func,
183 vma->vm_end - vma->vm_start);
184 return -EINVAL;
185 }
186
187 return 0;
188}
189
190static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
191{
192 struct idxd_user_context *ctx = filp->private_data;
193 struct idxd_wq *wq = ctx->wq;
194 struct idxd_device *idxd = wq->idxd;
195 struct pci_dev *pdev = idxd->pdev;
196 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
197 unsigned long pfn;
198 int rc;
199
200 dev_dbg(&pdev->dev, "%s called\n", __func__);
201 rc = check_vma(wq, vma, __func__);
202 if (rc < 0)
203 return rc;
204
205 vma->vm_flags |= VM_DONTCOPY;
206 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
207 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
208 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
209 vma->vm_private_data = ctx;
210
211 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
212 vma->vm_page_prot);
213}
214
215static __poll_t idxd_cdev_poll(struct file *filp,
216 struct poll_table_struct *wait)
217{
218 struct idxd_user_context *ctx = filp->private_data;
219 struct idxd_wq *wq = ctx->wq;
220 struct idxd_device *idxd = wq->idxd;
221 __poll_t out = 0;
222
223 poll_wait(filp, &wq->err_queue, wait);
224 spin_lock(&idxd->dev_lock);
225 if (idxd->sw_err.valid)
226 out = EPOLLIN | EPOLLRDNORM;
227 spin_unlock(&idxd->dev_lock);
228
229 return out;
230}
231
232static const struct file_operations idxd_cdev_fops = {
233 .owner = THIS_MODULE,
234 .open = idxd_cdev_open,
235 .release = idxd_cdev_release,
236 .mmap = idxd_cdev_mmap,
237 .poll = idxd_cdev_poll,
238};
239
240int idxd_cdev_get_major(struct idxd_device *idxd)
241{
242 return MAJOR(ictx[idxd->data->type].devt);
243}
244
245int idxd_wq_add_cdev(struct idxd_wq *wq)
246{
247 struct idxd_device *idxd = wq->idxd;
248 struct idxd_cdev *idxd_cdev;
249 struct cdev *cdev;
250 struct device *dev;
251 struct idxd_cdev_context *cdev_ctx;
252 int rc, minor;
253
254 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
255 if (!idxd_cdev)
256 return -ENOMEM;
257
258 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
259 idxd_cdev->wq = wq;
260 cdev = &idxd_cdev->cdev;
261 dev = cdev_dev(idxd_cdev);
262 cdev_ctx = &ictx[wq->idxd->data->type];
263 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
264 if (minor < 0) {
265 kfree(idxd_cdev);
266 return minor;
267 }
268 idxd_cdev->minor = minor;
269
270 device_initialize(dev);
271 dev->parent = wq_confdev(wq);
272 dev->bus = &dsa_bus_type;
273 dev->type = &idxd_cdev_device_type;
274 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
275
276 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
277 if (rc < 0)
278 goto err;
279
280 wq->idxd_cdev = idxd_cdev;
281 cdev_init(cdev, &idxd_cdev_fops);
282 rc = cdev_device_add(cdev, dev);
283 if (rc) {
284 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
285 goto err;
286 }
287
288 return 0;
289
290 err:
291 put_device(dev);
292 wq->idxd_cdev = NULL;
293 return rc;
294}
295
296void idxd_wq_del_cdev(struct idxd_wq *wq)
297{
298 struct idxd_cdev *idxd_cdev;
299
300 idxd_cdev = wq->idxd_cdev;
301 wq->idxd_cdev = NULL;
302 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
303 put_device(cdev_dev(idxd_cdev));
304}
305
306static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
307{
308 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
309 struct idxd_device *idxd = wq->idxd;
310 int rc;
311
312 if (idxd->state != IDXD_DEV_ENABLED)
313 return -ENXIO;
314
315 mutex_lock(&wq->wq_lock);
316 wq->type = IDXD_WQT_USER;
317 rc = __drv_enable_wq(wq);
318 if (rc < 0)
319 goto err;
320
321 rc = idxd_wq_add_cdev(wq);
322 if (rc < 0) {
323 idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
324 goto err_cdev;
325 }
326
327 idxd->cmd_status = 0;
328 mutex_unlock(&wq->wq_lock);
329 return 0;
330
331err_cdev:
332 __drv_disable_wq(wq);
333err:
334 wq->type = IDXD_WQT_NONE;
335 mutex_unlock(&wq->wq_lock);
336 return rc;
337}
338
339static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
340{
341 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
342
343 mutex_lock(&wq->wq_lock);
344 idxd_wq_del_cdev(wq);
345 __drv_disable_wq(wq);
346 wq->type = IDXD_WQT_NONE;
347 mutex_unlock(&wq->wq_lock);
348}
349
350static enum idxd_dev_type dev_types[] = {
351 IDXD_DEV_WQ,
352 IDXD_DEV_NONE,
353};
354
355struct idxd_device_driver idxd_user_drv = {
356 .probe = idxd_user_drv_probe,
357 .remove = idxd_user_drv_remove,
358 .name = "user",
359 .type = dev_types,
360};
361EXPORT_SYMBOL_GPL(idxd_user_drv);
362
363int idxd_cdev_register(void)
364{
365 int rc, i;
366
367 for (i = 0; i < IDXD_TYPE_MAX; i++) {
368 ida_init(&ictx[i].minor_ida);
369 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
370 ictx[i].name);
371 if (rc)
372 return rc;
373 }
374
375 return 0;
376}
377
378void idxd_cdev_remove(void)
379{
380 int i;
381
382 for (i = 0; i < IDXD_TYPE_MAX; i++) {
383 unregister_chrdev_region(ictx[i].devt, MINORMASK);
384 ida_destroy(&ictx[i].minor_ida);
385 }
386}
387