1
2
3
4
5
6
7
8#include <linux/compiler.h>
9#include <linux/delay.h>
10#include <linux/device.h>
11#include <linux/fs.h>
12#include <linux/init.h>
13#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/moduleparam.h>
16#include <linux/pci.h>
17#include <linux/printk.h>
18#include <linux/sched.h>
19#include <linux/uaccess.h>
20
21#include "apex.h"
22
23#include "gasket_core.h"
24#include "gasket_interrupt.h"
25#include "gasket_page_table.h"
26#include "gasket_sysfs.h"
27
28
29#define APEX_DEVICE_NAME "Apex"
30#define APEX_DRIVER_VERSION "1.0"
31
32
33#define APEX_BAR_INDEX 2
34
35#define APEX_PCI_VENDOR_ID 0x1ac1
36#define APEX_PCI_DEVICE_ID 0x089a
37
38
39#define APEX_BAR_OFFSET 0
40#define APEX_CM_OFFSET 0x1000000
41
42
43#define APEX_BAR_BYTES 0x100000
44#define APEX_CH_MEM_BYTES (PAGE_SIZE * MAX_NUM_COHERENT_PAGES)
45
46
47#define NUM_REGIONS 3
48
49
50#define NUM_NODES 1
51
52
53
54
55
56#define APEX_PAGE_TABLE_TOTAL_ENTRIES 8192
57
58#define APEX_EXTENDED_SHIFT 63
59
60
61#define APEX_RESET_RETRY 120
62
63#define APEX_RESET_DELAY 100
64
65
66enum sysfs_attribute_type {
67 ATTR_KERNEL_HIB_PAGE_TABLE_SIZE,
68 ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE,
69 ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES,
70};
71
72
73
74
75
76enum apex_bar2_regs {
77 APEX_BAR2_REG_SCU_BASE = 0x1A300,
78 APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE = 0x46000,
79 APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE = 0x46008,
80 APEX_BAR2_REG_KERNEL_HIB_TRANSLATION_ENABLE = 0x46010,
81 APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL = 0x46018,
82 APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL = 0x46020,
83 APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL = 0x46028,
84 APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL = 0x46030,
85 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL = 0x46038,
86 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL = 0x46040,
87 APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL = 0x46048,
88 APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE = 0x46050,
89 APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE_MASK = 0x46058,
90 APEX_BAR2_REG_KERNEL_HIB_STATUS_BLOCK_DELAY = 0x46060,
91 APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY0 = 0x46068,
92 APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY1 = 0x46070,
93 APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT = 0x46078,
94 APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT = 0x46080,
95 APEX_BAR2_REG_KERNEL_WIRE_INT_PENDING_BIT_ARRAY = 0x48778,
96 APEX_BAR2_REG_KERNEL_WIRE_INT_MASK_ARRAY = 0x48780,
97 APEX_BAR2_REG_USER_HIB_DMA_PAUSE = 0x486D8,
98 APEX_BAR2_REG_USER_HIB_DMA_PAUSED = 0x486E0,
99 APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000,
100 APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000,
101
102
103 APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0,
104 APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS = 0x41a0,
105};
106
107
108#define APEX_BAR2_REG_AXI_QUIESCE (APEX_BAR2_REG_SCU_BASE + 0x2C)
109#define APEX_BAR2_REG_GCB_CLOCK_GATE (APEX_BAR2_REG_SCU_BASE + 0x14)
110#define APEX_BAR2_REG_SCU_0 (APEX_BAR2_REG_SCU_BASE + 0xc)
111#define APEX_BAR2_REG_SCU_1 (APEX_BAR2_REG_SCU_BASE + 0x10)
112#define APEX_BAR2_REG_SCU_2 (APEX_BAR2_REG_SCU_BASE + 0x14)
113#define APEX_BAR2_REG_SCU_3 (APEX_BAR2_REG_SCU_BASE + 0x18)
114#define APEX_BAR2_REG_SCU_4 (APEX_BAR2_REG_SCU_BASE + 0x1c)
115#define APEX_BAR2_REG_SCU_5 (APEX_BAR2_REG_SCU_BASE + 0x20)
116
117#define SCU3_RG_PWR_STATE_OVR_BIT_OFFSET 26
118#define SCU3_RG_PWR_STATE_OVR_MASK_WIDTH 2
119#define SCU3_CUR_RST_GCB_BIT_MASK 0x10
120#define SCU2_RG_RST_GCB_BIT_MASK 0xc
121
122
123static struct gasket_page_table_config apex_page_table_configs[NUM_NODES] = {
124 {
125 .id = 0,
126 .mode = GASKET_PAGE_TABLE_MODE_NORMAL,
127 .total_entries = APEX_PAGE_TABLE_TOTAL_ENTRIES,
128 .base_reg = APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE,
129 .extended_reg = APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE,
130 .extended_bit = APEX_EXTENDED_SHIFT,
131 },
132};
133
134
135static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = {
136 { 0x40000, 0x1000 },
137 { 0x44000, 0x1000 },
138 { 0x48000, 0x1000 },
139};
140
141
142enum apex_interrupt {
143 APEX_INTERRUPT_INSTR_QUEUE = 0,
144 APEX_INTERRUPT_INPUT_ACTV_QUEUE = 1,
145 APEX_INTERRUPT_PARAM_QUEUE = 2,
146 APEX_INTERRUPT_OUTPUT_ACTV_QUEUE = 3,
147 APEX_INTERRUPT_SC_HOST_0 = 4,
148 APEX_INTERRUPT_SC_HOST_1 = 5,
149 APEX_INTERRUPT_SC_HOST_2 = 6,
150 APEX_INTERRUPT_SC_HOST_3 = 7,
151 APEX_INTERRUPT_TOP_LEVEL_0 = 8,
152 APEX_INTERRUPT_TOP_LEVEL_1 = 9,
153 APEX_INTERRUPT_TOP_LEVEL_2 = 10,
154 APEX_INTERRUPT_TOP_LEVEL_3 = 11,
155 APEX_INTERRUPT_FATAL_ERR = 12,
156 APEX_INTERRUPT_COUNT = 13,
157};
158
159
160static struct gasket_interrupt_desc apex_interrupts[] = {
161 {
162 APEX_INTERRUPT_INSTR_QUEUE,
163 APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL,
164 UNPACKED,
165 },
166 {
167 APEX_INTERRUPT_INPUT_ACTV_QUEUE,
168 APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL,
169 UNPACKED
170 },
171 {
172 APEX_INTERRUPT_PARAM_QUEUE,
173 APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL,
174 UNPACKED
175 },
176 {
177 APEX_INTERRUPT_OUTPUT_ACTV_QUEUE,
178 APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL,
179 UNPACKED
180 },
181 {
182 APEX_INTERRUPT_SC_HOST_0,
183 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
184 PACK_0
185 },
186 {
187 APEX_INTERRUPT_SC_HOST_1,
188 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
189 PACK_1
190 },
191 {
192 APEX_INTERRUPT_SC_HOST_2,
193 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
194 PACK_2
195 },
196 {
197 APEX_INTERRUPT_SC_HOST_3,
198 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
199 PACK_3
200 },
201 {
202 APEX_INTERRUPT_TOP_LEVEL_0,
203 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
204 PACK_0
205 },
206 {
207 APEX_INTERRUPT_TOP_LEVEL_1,
208 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
209 PACK_1
210 },
211 {
212 APEX_INTERRUPT_TOP_LEVEL_2,
213 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
214 PACK_2
215 },
216 {
217 APEX_INTERRUPT_TOP_LEVEL_3,
218 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
219 PACK_3
220 },
221 {
222 APEX_INTERRUPT_FATAL_ERR,
223 APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL,
224 UNPACKED
225 },
226};
227
228
229static int allow_power_save = 1;
230
231
232static int allow_sw_clock_gating;
233
234
235
236static int allow_hw_clock_gating = 1;
237
238
239static int bypass_top_level;
240
241module_param(allow_power_save, int, 0644);
242module_param(allow_sw_clock_gating, int, 0644);
243module_param(allow_hw_clock_gating, int, 0644);
244module_param(bypass_top_level, int, 0644);
245
246
247static int apex_get_status(struct gasket_dev *gasket_dev)
248{
249
250 return GASKET_STATUS_ALIVE;
251}
252
253
254static int apex_enter_reset(struct gasket_dev *gasket_dev)
255{
256 if (bypass_top_level)
257 return 0;
258
259
260
261
262
263
264
265 gasket_read_modify_write_64(gasket_dev, APEX_BAR_INDEX,
266 APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER,
267 0x0, 1, 32);
268
269
270 gasket_dev_write_64(gasket_dev, 1, APEX_BAR_INDEX,
271 APEX_BAR2_REG_USER_HIB_DMA_PAUSE);
272
273
274 if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
275 APEX_BAR2_REG_USER_HIB_DMA_PAUSED, 1, 1,
276 APEX_RESET_DELAY, APEX_RESET_RETRY)) {
277 dev_err(gasket_dev->dev,
278 "DMAs did not quiesce within timeout (%d ms)\n",
279 APEX_RESET_RETRY * APEX_RESET_DELAY);
280 return -ETIMEDOUT;
281 }
282
283
284 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
285 APEX_BAR2_REG_SCU_2, 0x1, 2, 2);
286
287
288 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
289 APEX_BAR2_REG_SCU_2, 0x1, 2, 18);
290
291
292 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
293 APEX_BAR2_REG_SCU_3, 0x3, 2, 14);
294
295
296 if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
297 APEX_BAR2_REG_SCU_3, BIT(6), BIT(6),
298 APEX_RESET_DELAY, APEX_RESET_RETRY)) {
299 dev_err(gasket_dev->dev,
300 "RAM did not shut down within timeout (%d ms)\n",
301 APEX_RESET_RETRY * APEX_RESET_DELAY);
302 return -ETIMEDOUT;
303 }
304
305 return 0;
306}
307
308
309static int apex_quit_reset(struct gasket_dev *gasket_dev)
310{
311 u32 val0, val1;
312
313 if (bypass_top_level)
314 return 0;
315
316
317
318
319
320
321
322 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
323 APEX_BAR2_REG_SCU_3, 0x0, 2, 14);
324
325
326
327
328
329
330 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
331 APEX_BAR2_REG_SCU_2, 0x0, 2, 18);
332
333
334
335
336
337
338 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
339 APEX_BAR2_REG_SCU_2, 0x2, 2, 2);
340
341
342 if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
343 APEX_BAR2_REG_SCU_3, BIT(6), 0,
344 APEX_RESET_DELAY, APEX_RESET_RETRY)) {
345 dev_err(gasket_dev->dev,
346 "RAM did not enable within timeout (%d ms)\n",
347 APEX_RESET_RETRY * APEX_RESET_DELAY);
348 return -ETIMEDOUT;
349 }
350
351
352 if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
353 APEX_BAR2_REG_SCU_3,
354 SCU3_CUR_RST_GCB_BIT_MASK, 0,
355 APEX_RESET_DELAY, APEX_RESET_RETRY)) {
356 dev_err(gasket_dev->dev,
357 "GCB did not leave reset within timeout (%d ms)\n",
358 APEX_RESET_RETRY * APEX_RESET_DELAY);
359 return -ETIMEDOUT;
360 }
361
362 if (!allow_hw_clock_gating) {
363 val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
364 APEX_BAR2_REG_SCU_3);
365
366 gasket_read_modify_write_32(gasket_dev,
367 APEX_BAR_INDEX,
368 APEX_BAR2_REG_SCU_3, 0x3,
369 SCU3_RG_PWR_STATE_OVR_MASK_WIDTH,
370 SCU3_RG_PWR_STATE_OVR_BIT_OFFSET);
371 val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
372 APEX_BAR2_REG_SCU_3);
373 dev_dbg(gasket_dev->dev,
374 "Disallow HW clock gating 0x%x -> 0x%x\n", val0, val1);
375 } else {
376 val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
377 APEX_BAR2_REG_SCU_3);
378
379 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
380 APEX_BAR2_REG_SCU_3, 2,
381 SCU3_RG_PWR_STATE_OVR_MASK_WIDTH,
382 SCU3_RG_PWR_STATE_OVR_BIT_OFFSET);
383 val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
384 APEX_BAR2_REG_SCU_3);
385 dev_dbg(gasket_dev->dev, "Allow HW clock gating 0x%x -> 0x%x\n",
386 val0, val1);
387 }
388
389 return 0;
390}
391
392
393static int apex_device_cleanup(struct gasket_dev *gasket_dev)
394{
395 u64 scalar_error;
396 u64 hib_error;
397 int ret = 0;
398
399 hib_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
400 APEX_BAR2_REG_USER_HIB_ERROR_STATUS);
401 scalar_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
402 APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS);
403
404 dev_dbg(gasket_dev->dev,
405 "%s 0x%p hib_error 0x%llx scalar_error 0x%llx\n",
406 __func__, gasket_dev, hib_error, scalar_error);
407
408 if (allow_power_save)
409 ret = apex_enter_reset(gasket_dev);
410
411 return ret;
412}
413
414
415static bool is_gcb_in_reset(struct gasket_dev *gasket_dev)
416{
417 u32 val = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
418 APEX_BAR2_REG_SCU_3);
419
420
421 return (val & SCU3_CUR_RST_GCB_BIT_MASK);
422}
423
424
425static int apex_reset(struct gasket_dev *gasket_dev)
426{
427 int ret;
428
429 if (bypass_top_level)
430 return 0;
431
432 if (!is_gcb_in_reset(gasket_dev)) {
433
434
435
436 dev_dbg(gasket_dev->dev, "%s: toggle reset\n", __func__);
437
438 ret = apex_enter_reset(gasket_dev);
439 if (ret)
440 return ret;
441 }
442 return apex_quit_reset(gasket_dev);
443}
444
445
446
447
448
449static bool apex_ioctl_check_permissions(struct file *filp, uint cmd)
450{
451 return !!(filp->f_mode & FMODE_WRITE);
452}
453
454
455static long apex_clock_gating(struct gasket_dev *gasket_dev,
456 struct apex_gate_clock_ioctl __user *argp)
457{
458 struct apex_gate_clock_ioctl ibuf;
459
460 if (bypass_top_level || !allow_sw_clock_gating)
461 return 0;
462
463 if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
464 return -EFAULT;
465
466 dev_dbg(gasket_dev->dev, "%s %llu\n", __func__, ibuf.enable);
467
468 if (ibuf.enable) {
469
470 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
471 APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1,
472 16);
473 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
474 APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1,
475 2, 18);
476 } else {
477
478 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
479 APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0,
480 2, 18);
481 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
482 APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1,
483 16);
484 }
485 return 0;
486}
487
488
489static long apex_ioctl(struct file *filp, uint cmd, void __user *argp)
490{
491 struct gasket_dev *gasket_dev = filp->private_data;
492
493 if (!apex_ioctl_check_permissions(filp, cmd))
494 return -EPERM;
495
496 switch (cmd) {
497 case APEX_IOCTL_GATE_CLOCK:
498 return apex_clock_gating(gasket_dev, argp);
499 default:
500 return -ENOTTY;
501 }
502}
503
504
505static ssize_t sysfs_show(struct device *device, struct device_attribute *attr,
506 char *buf)
507{
508 int ret;
509 struct gasket_dev *gasket_dev;
510 struct gasket_sysfs_attribute *gasket_attr;
511 enum sysfs_attribute_type type;
512 struct gasket_page_table *gpt;
513 uint val;
514
515 gasket_dev = gasket_sysfs_get_device_data(device);
516 if (!gasket_dev) {
517 dev_err(device, "No Apex device sysfs mapping found\n");
518 return -ENODEV;
519 }
520
521 gasket_attr = gasket_sysfs_get_attr(device, attr);
522 if (!gasket_attr) {
523 dev_err(device, "No Apex device sysfs attr data found\n");
524 gasket_sysfs_put_device_data(device, gasket_dev);
525 return -ENODEV;
526 }
527
528 type = (enum sysfs_attribute_type)gasket_attr->data.attr_type;
529 gpt = gasket_dev->page_table[0];
530 switch (type) {
531 case ATTR_KERNEL_HIB_PAGE_TABLE_SIZE:
532 val = gasket_page_table_num_entries(gpt);
533 break;
534 case ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE:
535 val = gasket_page_table_num_simple_entries(gpt);
536 break;
537 case ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES:
538 val = gasket_page_table_num_active_pages(gpt);
539 break;
540 default:
541 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
542 attr->attr.name);
543 ret = 0;
544 goto exit;
545 }
546 ret = scnprintf(buf, PAGE_SIZE, "%u\n", val);
547exit:
548 gasket_sysfs_put_attr(device, gasket_attr);
549 gasket_sysfs_put_device_data(device, gasket_dev);
550 return ret;
551}
552
553static struct gasket_sysfs_attribute apex_sysfs_attrs[] = {
554 GASKET_SYSFS_RO(node_0_page_table_entries, sysfs_show,
555 ATTR_KERNEL_HIB_PAGE_TABLE_SIZE),
556 GASKET_SYSFS_RO(node_0_simple_page_table_entries, sysfs_show,
557 ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE),
558 GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show,
559 ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES),
560 GASKET_END_OF_ATTR_ARRAY
561};
562
563
564static int apex_device_open_cb(struct gasket_dev *gasket_dev)
565{
566 return gasket_reset_nolock(gasket_dev);
567}
568
569static const struct pci_device_id apex_pci_ids[] = {
570 { PCI_DEVICE(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID) }, { 0 }
571};
572
573static int apex_pci_probe(struct pci_dev *pci_dev,
574 const struct pci_device_id *id)
575{
576 int ret;
577 ulong page_table_ready, msix_table_ready;
578 int retries = 0;
579 struct gasket_dev *gasket_dev;
580
581 ret = pci_enable_device(pci_dev);
582 if (ret) {
583 dev_err(&pci_dev->dev, "error enabling PCI device\n");
584 return ret;
585 }
586
587 pci_set_master(pci_dev);
588
589 ret = gasket_pci_add_device(pci_dev, &gasket_dev);
590 if (ret) {
591 dev_err(&pci_dev->dev, "error adding gasket device\n");
592 pci_disable_device(pci_dev);
593 return ret;
594 }
595
596 pci_set_drvdata(pci_dev, gasket_dev);
597 apex_reset(gasket_dev);
598
599 while (retries < APEX_RESET_RETRY) {
600 page_table_ready =
601 gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
602 APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT);
603 msix_table_ready =
604 gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
605 APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT);
606 if (page_table_ready && msix_table_ready)
607 break;
608 schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY));
609 retries++;
610 }
611
612 if (retries == APEX_RESET_RETRY) {
613 if (!page_table_ready)
614 dev_err(gasket_dev->dev, "Page table init timed out\n");
615 if (!msix_table_ready)
616 dev_err(gasket_dev->dev, "MSI-X table init timed out\n");
617 ret = -ETIMEDOUT;
618 goto remove_device;
619 }
620
621 ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device,
622 apex_sysfs_attrs);
623 if (ret)
624 dev_err(&pci_dev->dev, "error creating device sysfs entries\n");
625
626 ret = gasket_enable_device(gasket_dev);
627 if (ret) {
628 dev_err(&pci_dev->dev, "error enabling gasket device\n");
629 goto remove_device;
630 }
631
632
633 if (allow_power_save)
634 apex_enter_reset(gasket_dev);
635
636 return 0;
637
638remove_device:
639 gasket_pci_remove_device(pci_dev);
640 pci_disable_device(pci_dev);
641 return ret;
642}
643
644static void apex_pci_remove(struct pci_dev *pci_dev)
645{
646 struct gasket_dev *gasket_dev = pci_get_drvdata(pci_dev);
647
648 gasket_disable_device(gasket_dev);
649 gasket_pci_remove_device(pci_dev);
650 pci_disable_device(pci_dev);
651}
652
653static const struct gasket_driver_desc apex_desc = {
654 .name = "apex",
655 .driver_version = APEX_DRIVER_VERSION,
656 .major = 120,
657 .minor = 0,
658 .module = THIS_MODULE,
659 .pci_id_table = apex_pci_ids,
660
661 .num_page_tables = NUM_NODES,
662 .page_table_bar_index = APEX_BAR_INDEX,
663 .page_table_configs = apex_page_table_configs,
664 .page_table_extended_bit = APEX_EXTENDED_SHIFT,
665
666 .bar_descriptions = {
667 GASKET_UNUSED_BAR,
668 GASKET_UNUSED_BAR,
669 { APEX_BAR_BYTES, (VM_WRITE | VM_READ), APEX_BAR_OFFSET,
670 NUM_REGIONS, mappable_regions, PCI_BAR },
671 GASKET_UNUSED_BAR,
672 GASKET_UNUSED_BAR,
673 GASKET_UNUSED_BAR,
674 },
675 .coherent_buffer_description = {
676 APEX_CH_MEM_BYTES,
677 (VM_WRITE | VM_READ),
678 APEX_CM_OFFSET,
679 },
680 .interrupt_type = PCI_MSIX,
681 .interrupt_bar_index = APEX_BAR_INDEX,
682 .num_interrupts = APEX_INTERRUPT_COUNT,
683 .interrupts = apex_interrupts,
684 .interrupt_pack_width = 7,
685
686 .device_open_cb = apex_device_open_cb,
687 .device_close_cb = apex_device_cleanup,
688
689 .ioctl_handler_cb = apex_ioctl,
690 .device_status_cb = apex_get_status,
691 .hardware_revision_cb = NULL,
692 .device_reset_cb = apex_reset,
693};
694
695static struct pci_driver apex_pci_driver = {
696 .name = "apex",
697 .probe = apex_pci_probe,
698 .remove = apex_pci_remove,
699 .id_table = apex_pci_ids,
700};
701
702static int __init apex_init(void)
703{
704 int ret;
705
706 ret = gasket_register_device(&apex_desc);
707 if (ret)
708 return ret;
709 ret = pci_register_driver(&apex_pci_driver);
710 if (ret)
711 gasket_unregister_device(&apex_desc);
712 return ret;
713}
714
715static void apex_exit(void)
716{
717 pci_unregister_driver(&apex_pci_driver);
718 gasket_unregister_device(&apex_desc);
719}
720MODULE_DESCRIPTION("Google Apex driver");
721MODULE_VERSION(APEX_DRIVER_VERSION);
722MODULE_LICENSE("GPL v2");
723MODULE_AUTHOR("John Joseph <jnjoseph@google.com>");
724MODULE_DEVICE_TABLE(pci, apex_pci_ids);
725module_init(apex_init);
726module_exit(apex_exit);
727
