1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27#include <linux/kernel.h>
28#include <linux/errno.h>
29#include <linux/slab.h>
30#include <linux/mm.h>
31#include <linux/io.h>
32#include <linux/spinlock.h>
33#include <linux/device.h>
34#include <linux/miscdevice.h>
35#include <linux/interrupt.h>
36#include <linux/proc_fs.h>
37#include <linux/uaccess.h>
38#ifdef CONFIG_X86_64
39#include <asm/uv/uv_irq.h>
40#endif
41#include <asm/uv/uv.h>
42#include "gru.h"
43#include "grulib.h"
44#include "grutables.h"
45
46#include <asm/uv/uv_hub.h>
47#include <asm/uv/uv_mmrs.h>
48
49struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly;
50unsigned long gru_start_paddr __read_mostly;
51void *gru_start_vaddr __read_mostly;
52unsigned long gru_end_paddr __read_mostly;
53unsigned int gru_max_gids __read_mostly;
54struct gru_stats_s gru_stats;
55
56
57static int max_user_cbrs, max_user_dsr_bytes;
58
59static struct miscdevice gru_miscdev;
60
61static int gru_supported(void)
62{
63 return is_uv_system() &&
64 (uv_hub_info->hub_revision < UV3_HUB_REVISION_BASE);
65}
66
67
68
69
70
71
72
73static void gru_vma_close(struct vm_area_struct *vma)
74{
75 struct gru_vma_data *vdata;
76 struct gru_thread_state *gts;
77 struct list_head *entry, *next;
78
79 if (!vma->vm_private_data)
80 return;
81
82 vdata = vma->vm_private_data;
83 vma->vm_private_data = NULL;
84 gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file,
85 vdata);
86 list_for_each_safe(entry, next, &vdata->vd_head) {
87 gts =
88 list_entry(entry, struct gru_thread_state, ts_next);
89 list_del(>s->ts_next);
90 mutex_lock(>s->ts_ctxlock);
91 if (gts->ts_gru)
92 gru_unload_context(gts, 0);
93 mutex_unlock(>s->ts_ctxlock);
94 gts_drop(gts);
95 }
96 kfree(vdata);
97 STAT(vdata_free);
98}
99
100
101
102
103
104
105
106
107static int gru_file_mmap(struct file *file, struct vm_area_struct *vma)
108{
109 if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) != (VM_SHARED | VM_WRITE))
110 return -EPERM;
111
112 if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) ||
113 vma->vm_end & (GRU_GSEG_PAGESIZE - 1))
114 return -EINVAL;
115
116 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_LOCKED |
117 VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
118 vma->vm_page_prot = PAGE_SHARED;
119 vma->vm_ops = &gru_vm_ops;
120
121 vma->vm_private_data = gru_alloc_vma_data(vma, 0);
122 if (!vma->vm_private_data)
123 return -ENOMEM;
124
125 gru_dbg(grudev, "file %p, vaddr 0x%lx, vma %p, vdata %p\n",
126 file, vma->vm_start, vma, vma->vm_private_data);
127 return 0;
128}
129
130
131
132
133static int gru_create_new_context(unsigned long arg)
134{
135 struct gru_create_context_req req;
136 struct vm_area_struct *vma;
137 struct gru_vma_data *vdata;
138 int ret = -EINVAL;
139
140 if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
141 return -EFAULT;
142
143 if (req.data_segment_bytes > max_user_dsr_bytes)
144 return -EINVAL;
145 if (req.control_blocks > max_user_cbrs || !req.maximum_thread_count)
146 return -EINVAL;
147
148 if (!(req.options & GRU_OPT_MISS_MASK))
149 req.options |= GRU_OPT_MISS_FMM_INTR;
150
151 down_write(¤t->mm->mmap_sem);
152 vma = gru_find_vma(req.gseg);
153 if (vma) {
154 vdata = vma->vm_private_data;
155 vdata->vd_user_options = req.options;
156 vdata->vd_dsr_au_count =
157 GRU_DS_BYTES_TO_AU(req.data_segment_bytes);
158 vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks);
159 vdata->vd_tlb_preload_count = req.tlb_preload_count;
160 ret = 0;
161 }
162 up_write(¤t->mm->mmap_sem);
163
164 return ret;
165}
166
167
168
169
170static long gru_get_config_info(unsigned long arg)
171{
172 struct gru_config_info info;
173 int nodesperblade;
174
175 if (num_online_nodes() > 1 &&
176 (uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
177 nodesperblade = 2;
178 else
179 nodesperblade = 1;
180 memset(&info, 0, sizeof(info));
181 info.cpus = num_online_cpus();
182 info.nodes = num_online_nodes();
183 info.blades = info.nodes / nodesperblade;
184 info.chiplets = GRU_CHIPLETS_PER_BLADE * info.blades;
185
186 if (copy_to_user((void __user *)arg, &info, sizeof(info)))
187 return -EFAULT;
188 return 0;
189}
190
191
192
193
194
195
196static long gru_file_unlocked_ioctl(struct file *file, unsigned int req,
197 unsigned long arg)
198{
199 int err = -EBADRQC;
200
201 gru_dbg(grudev, "file %p, req 0x%x, 0x%lx\n", file, req, arg);
202
203 switch (req) {
204 case GRU_CREATE_CONTEXT:
205 err = gru_create_new_context(arg);
206 break;
207 case GRU_SET_CONTEXT_OPTION:
208 err = gru_set_context_option(arg);
209 break;
210 case GRU_USER_GET_EXCEPTION_DETAIL:
211 err = gru_get_exception_detail(arg);
212 break;
213 case GRU_USER_UNLOAD_CONTEXT:
214 err = gru_user_unload_context(arg);
215 break;
216 case GRU_USER_FLUSH_TLB:
217 err = gru_user_flush_tlb(arg);
218 break;
219 case GRU_USER_CALL_OS:
220 err = gru_handle_user_call_os(arg);
221 break;
222 case GRU_GET_GSEG_STATISTICS:
223 err = gru_get_gseg_statistics(arg);
224 break;
225 case GRU_KTEST:
226 err = gru_ktest(arg);
227 break;
228 case GRU_GET_CONFIG_INFO:
229 err = gru_get_config_info(arg);
230 break;
231 case GRU_DUMP_CHIPLET_STATE:
232 err = gru_dump_chiplet_request(arg);
233 break;
234 }
235 return err;
236}
237
238
239
240
241
242static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr,
243 void *vaddr, int blade_id, int chiplet_id)
244{
245 spin_lock_init(&gru->gs_lock);
246 spin_lock_init(&gru->gs_asid_lock);
247 gru->gs_gru_base_paddr = paddr;
248 gru->gs_gru_base_vaddr = vaddr;
249 gru->gs_gid = blade_id * GRU_CHIPLETS_PER_BLADE + chiplet_id;
250 gru->gs_blade = gru_base[blade_id];
251 gru->gs_blade_id = blade_id;
252 gru->gs_chiplet_id = chiplet_id;
253 gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1;
254 gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1;
255 gru->gs_asid_limit = MAX_ASID;
256 gru_tgh_flush_init(gru);
257 if (gru->gs_gid >= gru_max_gids)
258 gru_max_gids = gru->gs_gid + 1;
259 gru_dbg(grudev, "bid %d, gid %d, vaddr %p (0x%lx)\n",
260 blade_id, gru->gs_gid, gru->gs_gru_base_vaddr,
261 gru->gs_gru_base_paddr);
262}
263
264static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr)
265{
266 int pnode, nid, bid, chip;
267 int cbrs, dsrbytes, n;
268 int order = get_order(sizeof(struct gru_blade_state));
269 struct page *page;
270 struct gru_state *gru;
271 unsigned long paddr;
272 void *vaddr;
273
274 max_user_cbrs = GRU_NUM_CB;
275 max_user_dsr_bytes = GRU_NUM_DSR_BYTES;
276 for_each_possible_blade(bid) {
277 pnode = uv_blade_to_pnode(bid);
278 nid = uv_blade_to_memory_nid(bid);
279 page = alloc_pages_node(nid, GFP_KERNEL, order);
280 if (!page)
281 goto fail;
282 gru_base[bid] = page_address(page);
283 memset(gru_base[bid], 0, sizeof(struct gru_blade_state));
284 gru_base[bid]->bs_lru_gru = &gru_base[bid]->bs_grus[0];
285 spin_lock_init(&gru_base[bid]->bs_lock);
286 init_rwsem(&gru_base[bid]->bs_kgts_sema);
287
288 dsrbytes = 0;
289 cbrs = 0;
290 for (gru = gru_base[bid]->bs_grus, chip = 0;
291 chip < GRU_CHIPLETS_PER_BLADE;
292 chip++, gru++) {
293 paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip);
294 vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip);
295 gru_init_chiplet(gru, paddr, vaddr, bid, chip);
296 n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE;
297 cbrs = max(cbrs, n);
298 n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES;
299 dsrbytes = max(dsrbytes, n);
300 }
301 max_user_cbrs = min(max_user_cbrs, cbrs);
302 max_user_dsr_bytes = min(max_user_dsr_bytes, dsrbytes);
303 }
304
305 return 0;
306
307fail:
308 for (bid--; bid >= 0; bid--)
309 free_pages((unsigned long)gru_base[bid], order);
310 return -ENOMEM;
311}
312
313static void gru_free_tables(void)
314{
315 int bid;
316 int order = get_order(sizeof(struct gru_state) *
317 GRU_CHIPLETS_PER_BLADE);
318
319 for (bid = 0; bid < GRU_MAX_BLADES; bid++)
320 free_pages((unsigned long)gru_base[bid], order);
321}
322
323static unsigned long gru_chiplet_cpu_to_mmr(int chiplet, int cpu, int *corep)
324{
325 unsigned long mmr = 0;
326 int core;
327
328
329
330
331
332
333
334 core = uv_cpu_core_number(cpu) + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu);
335 if (core >= GRU_NUM_TFM || uv_cpu_ht_number(cpu))
336 return 0;
337
338 if (chiplet == 0) {
339 mmr = UVH_GR0_TLB_INT0_CONFIG +
340 core * (UVH_GR0_TLB_INT1_CONFIG - UVH_GR0_TLB_INT0_CONFIG);
341 } else if (chiplet == 1) {
342 mmr = UVH_GR1_TLB_INT0_CONFIG +
343 core * (UVH_GR1_TLB_INT1_CONFIG - UVH_GR1_TLB_INT0_CONFIG);
344 } else {
345 BUG();
346 }
347
348 *corep = core;
349 return mmr;
350}
351
352#ifdef CONFIG_IA64
353
354static int gru_irq_count[GRU_CHIPLETS_PER_BLADE];
355
356static void gru_noop(struct irq_data *d)
357{
358}
359
360static struct irq_chip gru_chip[GRU_CHIPLETS_PER_BLADE] = {
361 [0 ... GRU_CHIPLETS_PER_BLADE - 1] {
362 .irq_mask = gru_noop,
363 .irq_unmask = gru_noop,
364 .irq_ack = gru_noop
365 }
366};
367
368static int gru_chiplet_setup_tlb_irq(int chiplet, char *irq_name,
369 irq_handler_t irq_handler, int cpu, int blade)
370{
371 unsigned long mmr;
372 int irq = IRQ_GRU + chiplet;
373 int ret, core;
374
375 mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
376 if (mmr == 0)
377 return 0;
378
379 if (gru_irq_count[chiplet] == 0) {
380 gru_chip[chiplet].name = irq_name;
381 ret = irq_set_chip(irq, &gru_chip[chiplet]);
382 if (ret) {
383 printk(KERN_ERR "%s: set_irq_chip failed, errno=%d\n",
384 GRU_DRIVER_ID_STR, -ret);
385 return ret;
386 }
387
388 ret = request_irq(irq, irq_handler, 0, irq_name, NULL);
389 if (ret) {
390 printk(KERN_ERR "%s: request_irq failed, errno=%d\n",
391 GRU_DRIVER_ID_STR, -ret);
392 return ret;
393 }
394 }
395 gru_irq_count[chiplet]++;
396
397 return 0;
398}
399
400static void gru_chiplet_teardown_tlb_irq(int chiplet, int cpu, int blade)
401{
402 unsigned long mmr;
403 int core, irq = IRQ_GRU + chiplet;
404
405 if (gru_irq_count[chiplet] == 0)
406 return;
407
408 mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
409 if (mmr == 0)
410 return;
411
412 if (--gru_irq_count[chiplet] == 0)
413 free_irq(irq, NULL);
414}
415
416#elif defined CONFIG_X86_64
417
418static int gru_chiplet_setup_tlb_irq(int chiplet, char *irq_name,
419 irq_handler_t irq_handler, int cpu, int blade)
420{
421 unsigned long mmr;
422 int irq, core;
423 int ret;
424
425 mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
426 if (mmr == 0)
427 return 0;
428
429 irq = uv_setup_irq(irq_name, cpu, blade, mmr, UV_AFFINITY_CPU);
430 if (irq < 0) {
431 printk(KERN_ERR "%s: uv_setup_irq failed, errno=%d\n",
432 GRU_DRIVER_ID_STR, -irq);
433 return irq;
434 }
435
436 ret = request_irq(irq, irq_handler, 0, irq_name, NULL);
437 if (ret) {
438 uv_teardown_irq(irq);
439 printk(KERN_ERR "%s: request_irq failed, errno=%d\n",
440 GRU_DRIVER_ID_STR, -ret);
441 return ret;
442 }
443 gru_base[blade]->bs_grus[chiplet].gs_irq[core] = irq;
444 return 0;
445}
446
447static void gru_chiplet_teardown_tlb_irq(int chiplet, int cpu, int blade)
448{
449 int irq, core;
450 unsigned long mmr;
451
452 mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
453 if (mmr) {
454 irq = gru_base[blade]->bs_grus[chiplet].gs_irq[core];
455 if (irq) {
456 free_irq(irq, NULL);
457 uv_teardown_irq(irq);
458 }
459 }
460}
461
462#endif
463
464static void gru_teardown_tlb_irqs(void)
465{
466 int blade;
467 int cpu;
468
469 for_each_online_cpu(cpu) {
470 blade = uv_cpu_to_blade_id(cpu);
471 gru_chiplet_teardown_tlb_irq(0, cpu, blade);
472 gru_chiplet_teardown_tlb_irq(1, cpu, blade);
473 }
474 for_each_possible_blade(blade) {
475 if (uv_blade_nr_possible_cpus(blade))
476 continue;
477 gru_chiplet_teardown_tlb_irq(0, 0, blade);
478 gru_chiplet_teardown_tlb_irq(1, 0, blade);
479 }
480}
481
482static int gru_setup_tlb_irqs(void)
483{
484 int blade;
485 int cpu;
486 int ret;
487
488 for_each_online_cpu(cpu) {
489 blade = uv_cpu_to_blade_id(cpu);
490 ret = gru_chiplet_setup_tlb_irq(0, "GRU0_TLB", gru0_intr, cpu, blade);
491 if (ret != 0)
492 goto exit1;
493
494 ret = gru_chiplet_setup_tlb_irq(1, "GRU1_TLB", gru1_intr, cpu, blade);
495 if (ret != 0)
496 goto exit1;
497 }
498 for_each_possible_blade(blade) {
499 if (uv_blade_nr_possible_cpus(blade))
500 continue;
501 ret = gru_chiplet_setup_tlb_irq(0, "GRU0_TLB", gru_intr_mblade, 0, blade);
502 if (ret != 0)
503 goto exit1;
504
505 ret = gru_chiplet_setup_tlb_irq(1, "GRU1_TLB", gru_intr_mblade, 0, blade);
506 if (ret != 0)
507 goto exit1;
508 }
509
510 return 0;
511
512exit1:
513 gru_teardown_tlb_irqs();
514 return ret;
515}
516
517
518
519
520
521
522static int __init gru_init(void)
523{
524 int ret;
525
526 if (!gru_supported())
527 return 0;
528
529#if defined CONFIG_IA64
530 gru_start_paddr = 0xd000000000UL;
531#else
532 gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR) &
533 0x7fffffffffffUL;
534#endif
535 gru_start_vaddr = __va(gru_start_paddr);
536 gru_end_paddr = gru_start_paddr + GRU_MAX_BLADES * GRU_SIZE;
537 printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n",
538 gru_start_paddr, gru_end_paddr);
539 ret = misc_register(&gru_miscdev);
540 if (ret) {
541 printk(KERN_ERR "%s: misc_register failed\n",
542 GRU_DRIVER_ID_STR);
543 goto exit0;
544 }
545
546 ret = gru_proc_init();
547 if (ret) {
548 printk(KERN_ERR "%s: proc init failed\n", GRU_DRIVER_ID_STR);
549 goto exit1;
550 }
551
552 ret = gru_init_tables(gru_start_paddr, gru_start_vaddr);
553 if (ret) {
554 printk(KERN_ERR "%s: init tables failed\n", GRU_DRIVER_ID_STR);
555 goto exit2;
556 }
557
558 ret = gru_setup_tlb_irqs();
559 if (ret != 0)
560 goto exit3;
561
562 gru_kservices_init();
563
564 printk(KERN_INFO "%s: v%s\n", GRU_DRIVER_ID_STR,
565 GRU_DRIVER_VERSION_STR);
566 return 0;
567
568exit3:
569 gru_free_tables();
570exit2:
571 gru_proc_exit();
572exit1:
573 misc_deregister(&gru_miscdev);
574exit0:
575 return ret;
576
577}
578
579static void __exit gru_exit(void)
580{
581 if (!gru_supported())
582 return;
583
584 gru_teardown_tlb_irqs();
585 gru_kservices_exit();
586 gru_free_tables();
587 misc_deregister(&gru_miscdev);
588 gru_proc_exit();
589}
590
591static const struct file_operations gru_fops = {
592 .owner = THIS_MODULE,
593 .unlocked_ioctl = gru_file_unlocked_ioctl,
594 .mmap = gru_file_mmap,
595 .llseek = noop_llseek,
596};
597
598static struct miscdevice gru_miscdev = {
599 .minor = MISC_DYNAMIC_MINOR,
600 .name = "gru",
601 .fops = &gru_fops,
602};
603
604const struct vm_operations_struct gru_vm_ops = {
605 .close = gru_vma_close,
606 .fault = gru_fault,
607};
608
609#ifndef MODULE
610fs_initcall(gru_init);
611#else
612module_init(gru_init);
613#endif
614module_exit(gru_exit);
615
616module_param(gru_options, ulong, 0644);
617MODULE_PARM_DESC(gru_options, "Various debug options");
618
619MODULE_AUTHOR("Silicon Graphics, Inc.");
620MODULE_LICENSE("GPL");
621MODULE_DESCRIPTION(GRU_DRIVER_ID_STR GRU_DRIVER_VERSION_STR);
622MODULE_VERSION(GRU_DRIVER_VERSION_STR);
623
624
