1
2
3
4
5
6
7
8
9
10#include <linux/pci.h>
11#include <linux/slab.h>
12#include <linux/file.h>
13#include <misc/cxl.h>
14#include <linux/module.h>
15#include <linux/mount.h>
16#include <linux/sched/mm.h>
17#include <linux/mmu_context.h>
18
19#include "cxl.h"
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#define CXL_PSEUDO_FS_MAGIC 0x1697697f
36
37static int cxl_fs_cnt;
38static struct vfsmount *cxl_vfs_mount;
39
40static const struct dentry_operations cxl_fs_dops = {
41 .d_dname = simple_dname,
42};
43
44static struct dentry *cxl_fs_mount(struct file_system_type *fs_type, int flags,
45 const char *dev_name, void *data)
46{
47 return mount_pseudo(fs_type, "cxl:", NULL, &cxl_fs_dops,
48 CXL_PSEUDO_FS_MAGIC);
49}
50
51static struct file_system_type cxl_fs_type = {
52 .name = "cxl",
53 .owner = THIS_MODULE,
54 .mount = cxl_fs_mount,
55 .kill_sb = kill_anon_super,
56};
57
58
59void cxl_release_mapping(struct cxl_context *ctx)
60{
61 if (ctx->kernelapi && ctx->mapping)
62 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
63}
64
65static struct file *cxl_getfile(const char *name,
66 const struct file_operations *fops,
67 void *priv, int flags)
68{
69 struct file *file;
70 struct inode *inode;
71 int rc;
72
73
74
75 if (fops->owner && !try_module_get(fops->owner))
76 return ERR_PTR(-ENOENT);
77
78 rc = simple_pin_fs(&cxl_fs_type, &cxl_vfs_mount, &cxl_fs_cnt);
79 if (rc < 0) {
80 pr_err("Cannot mount cxl pseudo filesystem: %d\n", rc);
81 file = ERR_PTR(rc);
82 goto err_module;
83 }
84
85 inode = alloc_anon_inode(cxl_vfs_mount->mnt_sb);
86 if (IS_ERR(inode)) {
87 file = ERR_CAST(inode);
88 goto err_fs;
89 }
90
91 file = alloc_file_pseudo(inode, cxl_vfs_mount, name,
92 flags & (O_ACCMODE | O_NONBLOCK), fops);
93 if (IS_ERR(file))
94 goto err_inode;
95
96 file->private_data = priv;
97
98 return file;
99
100err_inode:
101 iput(inode);
102err_fs:
103 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
104err_module:
105 module_put(fops->owner);
106 return file;
107}
108
109struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)
110{
111 struct cxl_afu *afu;
112 struct cxl_context *ctx;
113 int rc;
114
115 afu = cxl_pci_to_afu(dev);
116 if (IS_ERR(afu))
117 return ERR_CAST(afu);
118
119 ctx = cxl_context_alloc();
120 if (!ctx)
121 return ERR_PTR(-ENOMEM);
122
123 ctx->kernelapi = true;
124
125
126 rc = cxl_context_init(ctx, afu, false);
127 if (rc)
128 goto err_ctx;
129
130 return ctx;
131
132err_ctx:
133 kfree(ctx);
134 return ERR_PTR(rc);
135}
136EXPORT_SYMBOL_GPL(cxl_dev_context_init);
137
138struct cxl_context *cxl_get_context(struct pci_dev *dev)
139{
140 return dev->dev.archdata.cxl_ctx;
141}
142EXPORT_SYMBOL_GPL(cxl_get_context);
143
144int cxl_release_context(struct cxl_context *ctx)
145{
146 if (ctx->status >= STARTED)
147 return -EBUSY;
148
149 cxl_context_free(ctx);
150
151 return 0;
152}
153EXPORT_SYMBOL_GPL(cxl_release_context);
154
155static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)
156{
157 __u16 range;
158 int r;
159
160 for (r = 0; r < CXL_IRQ_RANGES; r++) {
161 range = ctx->irqs.range[r];
162 if (num < range) {
163 return ctx->irqs.offset[r] + num;
164 }
165 num -= range;
166 }
167 return 0;
168}
169
170
171int cxl_set_priv(struct cxl_context *ctx, void *priv)
172{
173 if (!ctx)
174 return -EINVAL;
175
176 ctx->priv = priv;
177
178 return 0;
179}
180EXPORT_SYMBOL_GPL(cxl_set_priv);
181
182void *cxl_get_priv(struct cxl_context *ctx)
183{
184 if (!ctx)
185 return ERR_PTR(-EINVAL);
186
187 return ctx->priv;
188}
189EXPORT_SYMBOL_GPL(cxl_get_priv);
190
191int cxl_allocate_afu_irqs(struct cxl_context *ctx, int num)
192{
193 int res;
194 irq_hw_number_t hwirq;
195
196 if (num == 0)
197 num = ctx->afu->pp_irqs;
198 res = afu_allocate_irqs(ctx, num);
199 if (res)
200 return res;
201
202 if (!cpu_has_feature(CPU_FTR_HVMODE)) {
203
204
205
206 hwirq = cxl_find_afu_irq(ctx, 0);
207 if (hwirq)
208 cxl_map_irq(ctx->afu->adapter, hwirq, cxl_ops->psl_interrupt, ctx, "psl");
209 }
210
211 if (ctx->status == STARTED) {
212 if (cxl_ops->update_ivtes)
213 cxl_ops->update_ivtes(ctx);
214 else WARN(1, "BUG: cxl_allocate_afu_irqs must be called prior to starting the context on this platform\n");
215 }
216
217 return res;
218}
219EXPORT_SYMBOL_GPL(cxl_allocate_afu_irqs);
220
221void cxl_free_afu_irqs(struct cxl_context *ctx)
222{
223 irq_hw_number_t hwirq;
224 unsigned int virq;
225
226 if (!cpu_has_feature(CPU_FTR_HVMODE)) {
227 hwirq = cxl_find_afu_irq(ctx, 0);
228 if (hwirq) {
229 virq = irq_find_mapping(NULL, hwirq);
230 if (virq)
231 cxl_unmap_irq(virq, ctx);
232 }
233 }
234 afu_irq_name_free(ctx);
235 cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
236}
237EXPORT_SYMBOL_GPL(cxl_free_afu_irqs);
238
239int cxl_map_afu_irq(struct cxl_context *ctx, int num,
240 irq_handler_t handler, void *cookie, char *name)
241{
242 irq_hw_number_t hwirq;
243
244
245
246
247 hwirq = cxl_find_afu_irq(ctx, num);
248 if (!hwirq)
249 return -ENOENT;
250
251 return cxl_map_irq(ctx->afu->adapter, hwirq, handler, cookie, name);
252}
253EXPORT_SYMBOL_GPL(cxl_map_afu_irq);
254
255void cxl_unmap_afu_irq(struct cxl_context *ctx, int num, void *cookie)
256{
257 irq_hw_number_t hwirq;
258 unsigned int virq;
259
260 hwirq = cxl_find_afu_irq(ctx, num);
261 if (!hwirq)
262 return;
263
264 virq = irq_find_mapping(NULL, hwirq);
265 if (virq)
266 cxl_unmap_irq(virq, cookie);
267}
268EXPORT_SYMBOL_GPL(cxl_unmap_afu_irq);
269
270
271
272
273
274int cxl_start_context(struct cxl_context *ctx, u64 wed,
275 struct task_struct *task)
276{
277 int rc = 0;
278 bool kernel = true;
279
280 pr_devel("%s: pe: %i\n", __func__, ctx->pe);
281
282 mutex_lock(&ctx->status_mutex);
283 if (ctx->status == STARTED)
284 goto out;
285
286
287
288
289
290 rc = cxl_adapter_context_get(ctx->afu->adapter);
291 if (rc)
292 goto out;
293
294 if (task) {
295 ctx->pid = get_task_pid(task, PIDTYPE_PID);
296 kernel = false;
297
298
299 ctx->mm = get_task_mm(current);
300
301
302 cxl_context_mm_count_get(ctx);
303
304 if (ctx->mm) {
305
306 mmput(ctx->mm);
307
308 mm_context_add_copro(ctx->mm);
309 }
310 }
311
312
313
314
315
316 cxl_ctx_get();
317
318
319 smp_mb();
320
321 if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) {
322 put_pid(ctx->pid);
323 ctx->pid = NULL;
324 cxl_adapter_context_put(ctx->afu->adapter);
325 cxl_ctx_put();
326 if (task) {
327 cxl_context_mm_count_put(ctx);
328 if (ctx->mm)
329 mm_context_remove_copro(ctx->mm);
330 }
331 goto out;
332 }
333
334 ctx->status = STARTED;
335out:
336 mutex_unlock(&ctx->status_mutex);
337 return rc;
338}
339EXPORT_SYMBOL_GPL(cxl_start_context);
340
341int cxl_process_element(struct cxl_context *ctx)
342{
343 return ctx->external_pe;
344}
345EXPORT_SYMBOL_GPL(cxl_process_element);
346
347
348int cxl_stop_context(struct cxl_context *ctx)
349{
350 return __detach_context(ctx);
351}
352EXPORT_SYMBOL_GPL(cxl_stop_context);
353
354void cxl_set_master(struct cxl_context *ctx)
355{
356 ctx->master = true;
357}
358EXPORT_SYMBOL_GPL(cxl_set_master);
359
360
361int cxl_fd_open(struct inode *inode, struct file *file)
362{
363 return afu_open(inode, file);
364}
365EXPORT_SYMBOL_GPL(cxl_fd_open);
366int cxl_fd_release(struct inode *inode, struct file *file)
367{
368 return afu_release(inode, file);
369}
370EXPORT_SYMBOL_GPL(cxl_fd_release);
371long cxl_fd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
372{
373 return afu_ioctl(file, cmd, arg);
374}
375EXPORT_SYMBOL_GPL(cxl_fd_ioctl);
376int cxl_fd_mmap(struct file *file, struct vm_area_struct *vm)
377{
378 return afu_mmap(file, vm);
379}
380EXPORT_SYMBOL_GPL(cxl_fd_mmap);
381__poll_t cxl_fd_poll(struct file *file, struct poll_table_struct *poll)
382{
383 return afu_poll(file, poll);
384}
385EXPORT_SYMBOL_GPL(cxl_fd_poll);
386ssize_t cxl_fd_read(struct file *file, char __user *buf, size_t count,
387 loff_t *off)
388{
389 return afu_read(file, buf, count, off);
390}
391EXPORT_SYMBOL_GPL(cxl_fd_read);
392
393#define PATCH_FOPS(NAME) if (!fops->NAME) fops->NAME = afu_fops.NAME
394
395
396struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops,
397 int *fd)
398{
399 struct file *file;
400 int rc, flags, fdtmp;
401 char *name = NULL;
402
403
404 if (ctx->mapping)
405 return ERR_PTR(-EEXIST);
406
407 flags = O_RDWR | O_CLOEXEC;
408
409
410 rc = get_unused_fd_flags(flags);
411 if (rc < 0)
412 return ERR_PTR(rc);
413 fdtmp = rc;
414
415
416
417
418 if (fops) {
419 PATCH_FOPS(open);
420 PATCH_FOPS(poll);
421 PATCH_FOPS(read);
422 PATCH_FOPS(release);
423 PATCH_FOPS(unlocked_ioctl);
424 PATCH_FOPS(compat_ioctl);
425 PATCH_FOPS(mmap);
426 } else
427 fops = (struct file_operations *)&afu_fops;
428
429 name = kasprintf(GFP_KERNEL, "cxl:%d", ctx->pe);
430 file = cxl_getfile(name, fops, ctx, flags);
431 kfree(name);
432 if (IS_ERR(file))
433 goto err_fd;
434
435 cxl_context_set_mapping(ctx, file->f_mapping);
436 *fd = fdtmp;
437 return file;
438
439err_fd:
440 put_unused_fd(fdtmp);
441 return NULL;
442}
443EXPORT_SYMBOL_GPL(cxl_get_fd);
444
445struct cxl_context *cxl_fops_get_context(struct file *file)
446{
447 return file->private_data;
448}
449EXPORT_SYMBOL_GPL(cxl_fops_get_context);
450
451void cxl_set_driver_ops(struct cxl_context *ctx,
452 struct cxl_afu_driver_ops *ops)
453{
454 WARN_ON(!ops->fetch_event || !ops->event_delivered);
455 atomic_set(&ctx->afu_driver_events, 0);
456 ctx->afu_driver_ops = ops;
457}
458EXPORT_SYMBOL_GPL(cxl_set_driver_ops);
459
460void cxl_context_events_pending(struct cxl_context *ctx,
461 unsigned int new_events)
462{
463 atomic_add(new_events, &ctx->afu_driver_events);
464 wake_up_all(&ctx->wq);
465}
466EXPORT_SYMBOL_GPL(cxl_context_events_pending);
467
468int cxl_start_work(struct cxl_context *ctx,
469 struct cxl_ioctl_start_work *work)
470{
471 int rc;
472
473
474 if (!(work->flags & CXL_START_WORK_NUM_IRQS))
475 work->num_interrupts = ctx->afu->pp_irqs;
476 else if ((work->num_interrupts < ctx->afu->pp_irqs) ||
477 (work->num_interrupts > ctx->afu->irqs_max)) {
478 return -EINVAL;
479 }
480
481 rc = afu_register_irqs(ctx, work->num_interrupts);
482 if (rc)
483 return rc;
484
485 rc = cxl_start_context(ctx, work->work_element_descriptor, current);
486 if (rc < 0) {
487 afu_release_irqs(ctx, ctx);
488 return rc;
489 }
490
491 return 0;
492}
493EXPORT_SYMBOL_GPL(cxl_start_work);
494
495void __iomem *cxl_psa_map(struct cxl_context *ctx)
496{
497 if (ctx->status != STARTED)
498 return NULL;
499
500 pr_devel("%s: psn_phys%llx size:%llx\n",
501 __func__, ctx->psn_phys, ctx->psn_size);
502 return ioremap(ctx->psn_phys, ctx->psn_size);
503}
504EXPORT_SYMBOL_GPL(cxl_psa_map);
505
506void cxl_psa_unmap(void __iomem *addr)
507{
508 iounmap(addr);
509}
510EXPORT_SYMBOL_GPL(cxl_psa_unmap);
511
512int cxl_afu_reset(struct cxl_context *ctx)
513{
514 struct cxl_afu *afu = ctx->afu;
515 int rc;
516
517 rc = cxl_ops->afu_reset(afu);
518 if (rc)
519 return rc;
520
521 return cxl_ops->afu_check_and_enable(afu);
522}
523EXPORT_SYMBOL_GPL(cxl_afu_reset);
524
525void cxl_perst_reloads_same_image(struct cxl_afu *afu,
526 bool perst_reloads_same_image)
527{
528 afu->adapter->perst_same_image = perst_reloads_same_image;
529}
530EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image);
531
532ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)
533{
534 struct cxl_afu *afu = cxl_pci_to_afu(dev);
535 if (IS_ERR(afu))
536 return -ENODEV;
537
538 return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);
539}
540EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);
541