1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/mm.h>
24#include <linux/interrupt.h>
25#include <linux/init.h>
26#include <linux/delay.h>
27#include <linux/sched.h>
28#include <linux/pci.h>
29#include <linux/mc146818rtc.h>
30#include <linux/compiler.h>
31#include <linux/acpi.h>
32#include <linux/module.h>
33#include <linux/syscore_ops.h>
34#include <linux/msi.h>
35#include <linux/htirq.h>
36#include <linux/freezer.h>
37#include <linux/kthread.h>
38#include <linux/jiffies.h>
39#include <linux/slab.h>
40#ifdef CONFIG_ACPI
41#include <acpi/acpi_bus.h>
42#endif
43#include <linux/bootmem.h>
44#include <linux/dmar.h>
45#include <linux/hpet.h>
46
47#include <asm/idle.h>
48#include <asm/io.h>
49#include <asm/smp.h>
50#include <asm/cpu.h>
51#include <asm/desc.h>
52#include <asm/proto.h>
53#include <asm/acpi.h>
54#include <asm/dma.h>
55#include <asm/timer.h>
56#include <asm/i8259.h>
57#include <asm/msidef.h>
58#include <asm/hypertransport.h>
59#include <asm/setup.h>
60#include <asm/irq_remapping.h>
61#include <asm/hpet.h>
62#include <asm/hw_irq.h>
63
64#include <asm/apic.h>
65
66#define __apicdebuginit(type) static type __init
67
68#define for_each_irq_pin(entry, head) \
69 for (entry = head; entry; entry = entry->next)
70
71
72
73
74
75int sis_apic_bug = -1;
76
77static DEFINE_RAW_SPINLOCK(ioapic_lock);
78static DEFINE_RAW_SPINLOCK(vector_lock);
79
80static struct ioapic {
81
82
83
84 int nr_registers;
85
86
87
88 struct IO_APIC_route_entry *saved_registers;
89
90 struct mpc_ioapic mp_config;
91
92 struct mp_ioapic_gsi gsi_config;
93 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
94} ioapics[MAX_IO_APICS];
95
96#define mpc_ioapic_ver(ioapic_idx) ioapics[ioapic_idx].mp_config.apicver
97
98int mpc_ioapic_id(int ioapic_idx)
99{
100 return ioapics[ioapic_idx].mp_config.apicid;
101}
102
103unsigned int mpc_ioapic_addr(int ioapic_idx)
104{
105 return ioapics[ioapic_idx].mp_config.apicaddr;
106}
107
108struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx)
109{
110 return &ioapics[ioapic_idx].gsi_config;
111}
112
113int nr_ioapics;
114
115
116u32 gsi_top;
117
118
119struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
120
121
122int mp_irq_entries;
123
124
125static int nr_irqs_gsi = NR_IRQS_LEGACY;
126
127#ifdef CONFIG_EISA
128int mp_bus_id_to_type[MAX_MP_BUSSES];
129#endif
130
131DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
132
133int skip_ioapic_setup;
134
135
136
137
138void disable_ioapic_support(void)
139{
140#ifdef CONFIG_PCI
141 noioapicquirk = 1;
142 noioapicreroute = -1;
143#endif
144 skip_ioapic_setup = 1;
145}
146
147static int __init parse_noapic(char *str)
148{
149
150 disable_ioapic_support();
151 return 0;
152}
153early_param("noapic", parse_noapic);
154
155static int io_apic_setup_irq_pin(unsigned int irq, int node,
156 struct io_apic_irq_attr *attr);
157
158
159void mp_save_irq(struct mpc_intsrc *m)
160{
161 int i;
162
163 apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
164 " IRQ %02x, APIC ID %x, APIC INT %02x\n",
165 m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus,
166 m->srcbusirq, m->dstapic, m->dstirq);
167
168 for (i = 0; i < mp_irq_entries; i++) {
169 if (!memcmp(&mp_irqs[i], m, sizeof(*m)))
170 return;
171 }
172
173 memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m));
174 if (++mp_irq_entries == MAX_IRQ_SOURCES)
175 panic("Max # of irq sources exceeded!!\n");
176}
177
178struct irq_pin_list {
179 int apic, pin;
180 struct irq_pin_list *next;
181};
182
183static struct irq_pin_list *alloc_irq_pin_list(int node)
184{
185 return kzalloc_node(sizeof(struct irq_pin_list), GFP_KERNEL, node);
186}
187
188
189
190static struct irq_cfg irq_cfgx[NR_IRQS_LEGACY];
191
192int __init arch_early_irq_init(void)
193{
194 struct irq_cfg *cfg;
195 int count, node, i;
196
197 if (!legacy_pic->nr_legacy_irqs)
198 io_apic_irqs = ~0UL;
199
200 for (i = 0; i < nr_ioapics; i++) {
201 ioapics[i].saved_registers =
202 kzalloc(sizeof(struct IO_APIC_route_entry) *
203 ioapics[i].nr_registers, GFP_KERNEL);
204 if (!ioapics[i].saved_registers)
205 pr_err("IOAPIC %d: suspend/resume impossible!\n", i);
206 }
207
208 cfg = irq_cfgx;
209 count = ARRAY_SIZE(irq_cfgx);
210 node = cpu_to_node(0);
211
212
213 irq_reserve_irqs(0, legacy_pic->nr_legacy_irqs);
214
215 for (i = 0; i < count; i++) {
216 irq_set_chip_data(i, &cfg[i]);
217 zalloc_cpumask_var_node(&cfg[i].domain, GFP_KERNEL, node);
218 zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_KERNEL, node);
219
220
221
222
223 if (i < legacy_pic->nr_legacy_irqs) {
224 cfg[i].vector = IRQ0_VECTOR + i;
225 cpumask_setall(cfg[i].domain);
226 }
227 }
228
229 return 0;
230}
231
232static struct irq_cfg *irq_cfg(unsigned int irq)
233{
234 return irq_get_chip_data(irq);
235}
236
237static struct irq_cfg *alloc_irq_cfg(unsigned int irq, int node)
238{
239 struct irq_cfg *cfg;
240
241 cfg = kzalloc_node(sizeof(*cfg), GFP_KERNEL, node);
242 if (!cfg)
243 return NULL;
244 if (!zalloc_cpumask_var_node(&cfg->domain, GFP_KERNEL, node))
245 goto out_cfg;
246 if (!zalloc_cpumask_var_node(&cfg->old_domain, GFP_KERNEL, node))
247 goto out_domain;
248 return cfg;
249out_domain:
250 free_cpumask_var(cfg->domain);
251out_cfg:
252 kfree(cfg);
253 return NULL;
254}
255
256static void free_irq_cfg(unsigned int at, struct irq_cfg *cfg)
257{
258 if (!cfg)
259 return;
260 irq_set_chip_data(at, NULL);
261 free_cpumask_var(cfg->domain);
262 free_cpumask_var(cfg->old_domain);
263 kfree(cfg);
264}
265
266static struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node)
267{
268 int res = irq_alloc_desc_at(at, node);
269 struct irq_cfg *cfg;
270
271 if (res < 0) {
272 if (res != -EEXIST)
273 return NULL;
274 cfg = irq_get_chip_data(at);
275 if (cfg)
276 return cfg;
277 }
278
279 cfg = alloc_irq_cfg(at, node);
280 if (cfg)
281 irq_set_chip_data(at, cfg);
282 else
283 irq_free_desc(at);
284 return cfg;
285}
286
287static int alloc_irqs_from(unsigned int from, unsigned int count, int node)
288{
289 return irq_alloc_descs_from(from, count, node);
290}
291
292static void free_irq_at(unsigned int at, struct irq_cfg *cfg)
293{
294 free_irq_cfg(at, cfg);
295 irq_free_desc(at);
296}
297
298
299struct io_apic {
300 unsigned int index;
301 unsigned int unused[3];
302 unsigned int data;
303 unsigned int unused2[11];
304 unsigned int eoi;
305};
306
307static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
308{
309 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
310 + (mpc_ioapic_addr(idx) & ~PAGE_MASK);
311}
312
313void io_apic_eoi(unsigned int apic, unsigned int vector)
314{
315 struct io_apic __iomem *io_apic = io_apic_base(apic);
316 writel(vector, &io_apic->eoi);
317}
318
319unsigned int native_io_apic_read(unsigned int apic, unsigned int reg)
320{
321 struct io_apic __iomem *io_apic = io_apic_base(apic);
322 writel(reg, &io_apic->index);
323 return readl(&io_apic->data);
324}
325
326void native_io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
327{
328 struct io_apic __iomem *io_apic = io_apic_base(apic);
329
330 writel(reg, &io_apic->index);
331 writel(value, &io_apic->data);
332}
333
334
335
336
337
338
339
340void native_io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
341{
342 struct io_apic __iomem *io_apic = io_apic_base(apic);
343
344 if (sis_apic_bug)
345 writel(reg, &io_apic->index);
346 writel(value, &io_apic->data);
347}
348
349union entry_union {
350 struct { u32 w1, w2; };
351 struct IO_APIC_route_entry entry;
352};
353
354static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin)
355{
356 union entry_union eu;
357
358 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
359 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
360
361 return eu.entry;
362}
363
364static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
365{
366 union entry_union eu;
367 unsigned long flags;
368
369 raw_spin_lock_irqsave(&ioapic_lock, flags);
370 eu.entry = __ioapic_read_entry(apic, pin);
371 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
372
373 return eu.entry;
374}
375
376
377
378
379
380
381
382static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
383{
384 union entry_union eu = {{0, 0}};
385
386 eu.entry = e;
387 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
388 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
389}
390
391static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
392{
393 unsigned long flags;
394
395 raw_spin_lock_irqsave(&ioapic_lock, flags);
396 __ioapic_write_entry(apic, pin, e);
397 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
398}
399
400
401
402
403
404
405static void ioapic_mask_entry(int apic, int pin)
406{
407 unsigned long flags;
408 union entry_union eu = { .entry.mask = 1 };
409
410 raw_spin_lock_irqsave(&ioapic_lock, flags);
411 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
412 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
413 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
414}
415
416
417
418
419
420
421static int __add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
422{
423 struct irq_pin_list **last, *entry;
424
425
426 last = &cfg->irq_2_pin;
427 for_each_irq_pin(entry, cfg->irq_2_pin) {
428 if (entry->apic == apic && entry->pin == pin)
429 return 0;
430 last = &entry->next;
431 }
432
433 entry = alloc_irq_pin_list(node);
434 if (!entry) {
435 pr_err("can not alloc irq_pin_list (%d,%d,%d)\n",
436 node, apic, pin);
437 return -ENOMEM;
438 }
439 entry->apic = apic;
440 entry->pin = pin;
441
442 *last = entry;
443 return 0;
444}
445
446static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
447{
448 if (__add_pin_to_irq_node(cfg, node, apic, pin))
449 panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
450}
451
452
453
454
455static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
456 int oldapic, int oldpin,
457 int newapic, int newpin)
458{
459 struct irq_pin_list *entry;
460
461 for_each_irq_pin(entry, cfg->irq_2_pin) {
462 if (entry->apic == oldapic && entry->pin == oldpin) {
463 entry->apic = newapic;
464 entry->pin = newpin;
465
466 return;
467 }
468 }
469
470
471 add_pin_to_irq_node(cfg, node, newapic, newpin);
472}
473
474static void __io_apic_modify_irq(struct irq_pin_list *entry,
475 int mask_and, int mask_or,
476 void (*final)(struct irq_pin_list *entry))
477{
478 unsigned int reg, pin;
479
480 pin = entry->pin;
481 reg = io_apic_read(entry->apic, 0x10 + pin * 2);
482 reg &= mask_and;
483 reg |= mask_or;
484 io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
485 if (final)
486 final(entry);
487}
488
489static void io_apic_modify_irq(struct irq_cfg *cfg,
490 int mask_and, int mask_or,
491 void (*final)(struct irq_pin_list *entry))
492{
493 struct irq_pin_list *entry;
494
495 for_each_irq_pin(entry, cfg->irq_2_pin)
496 __io_apic_modify_irq(entry, mask_and, mask_or, final);
497}
498
499static void io_apic_sync(struct irq_pin_list *entry)
500{
501
502
503
504
505 struct io_apic __iomem *io_apic;
506
507 io_apic = io_apic_base(entry->apic);
508 readl(&io_apic->data);
509}
510
511static void mask_ioapic(struct irq_cfg *cfg)
512{
513 unsigned long flags;
514
515 raw_spin_lock_irqsave(&ioapic_lock, flags);
516 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
517 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
518}
519
520static void mask_ioapic_irq(struct irq_data *data)
521{
522 mask_ioapic(data->chip_data);
523}
524
525static void __unmask_ioapic(struct irq_cfg *cfg)
526{
527 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
528}
529
530static void unmask_ioapic(struct irq_cfg *cfg)
531{
532 unsigned long flags;
533
534 raw_spin_lock_irqsave(&ioapic_lock, flags);
535 __unmask_ioapic(cfg);
536 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
537}
538
539static void unmask_ioapic_irq(struct irq_data *data)
540{
541 unmask_ioapic(data->chip_data);
542}
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560void native_eoi_ioapic_pin(int apic, int pin, int vector)
561{
562 if (mpc_ioapic_ver(apic) >= 0x20) {
563 io_apic_eoi(apic, vector);
564 } else {
565 struct IO_APIC_route_entry entry, entry1;
566
567 entry = entry1 = __ioapic_read_entry(apic, pin);
568
569
570
571
572 entry1.mask = 1;
573 entry1.trigger = IOAPIC_EDGE;
574
575 __ioapic_write_entry(apic, pin, entry1);
576
577
578
579
580 __ioapic_write_entry(apic, pin, entry);
581 }
582}
583
584void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
585{
586 struct irq_pin_list *entry;
587 unsigned long flags;
588
589 raw_spin_lock_irqsave(&ioapic_lock, flags);
590 for_each_irq_pin(entry, cfg->irq_2_pin)
591 x86_io_apic_ops.eoi_ioapic_pin(entry->apic, entry->pin,
592 cfg->vector);
593 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
594}
595
596static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
597{
598 struct IO_APIC_route_entry entry;
599
600
601 entry = ioapic_read_entry(apic, pin);
602 if (entry.delivery_mode == dest_SMI)
603 return;
604
605
606
607
608
609 if (!entry.mask) {
610 entry.mask = 1;
611 ioapic_write_entry(apic, pin, entry);
612 entry = ioapic_read_entry(apic, pin);
613 }
614
615 if (entry.irr) {
616 unsigned long flags;
617
618
619
620
621
622
623 if (!entry.trigger) {
624 entry.trigger = IOAPIC_LEVEL;
625 ioapic_write_entry(apic, pin, entry);
626 }
627
628 raw_spin_lock_irqsave(&ioapic_lock, flags);
629 x86_io_apic_ops.eoi_ioapic_pin(apic, pin, entry.vector);
630 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
631 }
632
633
634
635
636
637 ioapic_mask_entry(apic, pin);
638 entry = ioapic_read_entry(apic, pin);
639 if (entry.irr)
640 pr_err("Unable to reset IRR for apic: %d, pin :%d\n",
641 mpc_ioapic_id(apic), pin);
642}
643
644static void clear_IO_APIC (void)
645{
646 int apic, pin;
647
648 for (apic = 0; apic < nr_ioapics; apic++)
649 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
650 clear_IO_APIC_pin(apic, pin);
651}
652
653#ifdef CONFIG_X86_32
654
655
656
657
658
659#define MAX_PIRQS 8
660static int pirq_entries[MAX_PIRQS] = {
661 [0 ... MAX_PIRQS - 1] = -1
662};
663
664static int __init ioapic_pirq_setup(char *str)
665{
666 int i, max;
667 int ints[MAX_PIRQS+1];
668
669 get_options(str, ARRAY_SIZE(ints), ints);
670
671 apic_printk(APIC_VERBOSE, KERN_INFO
672 "PIRQ redirection, working around broken MP-BIOS.\n");
673 max = MAX_PIRQS;
674 if (ints[0] < MAX_PIRQS)
675 max = ints[0];
676
677 for (i = 0; i < max; i++) {
678 apic_printk(APIC_VERBOSE, KERN_DEBUG
679 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
680
681
682
683 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
684 }
685 return 1;
686}
687
688__setup("pirq=", ioapic_pirq_setup);
689#endif
690
691
692
693
694int save_ioapic_entries(void)
695{
696 int apic, pin;
697 int err = 0;
698
699 for (apic = 0; apic < nr_ioapics; apic++) {
700 if (!ioapics[apic].saved_registers) {
701 err = -ENOMEM;
702 continue;
703 }
704
705 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
706 ioapics[apic].saved_registers[pin] =
707 ioapic_read_entry(apic, pin);
708 }
709
710 return err;
711}
712
713
714
715
716void mask_ioapic_entries(void)
717{
718 int apic, pin;
719
720 for (apic = 0; apic < nr_ioapics; apic++) {
721 if (!ioapics[apic].saved_registers)
722 continue;
723
724 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
725 struct IO_APIC_route_entry entry;
726
727 entry = ioapics[apic].saved_registers[pin];
728 if (!entry.mask) {
729 entry.mask = 1;
730 ioapic_write_entry(apic, pin, entry);
731 }
732 }
733 }
734}
735
736
737
738
739int restore_ioapic_entries(void)
740{
741 int apic, pin;
742
743 for (apic = 0; apic < nr_ioapics; apic++) {
744 if (!ioapics[apic].saved_registers)
745 continue;
746
747 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
748 ioapic_write_entry(apic, pin,
749 ioapics[apic].saved_registers[pin]);
750 }
751 return 0;
752}
753
754
755
756
757static int find_irq_entry(int ioapic_idx, int pin, int type)
758{
759 int i;
760
761 for (i = 0; i < mp_irq_entries; i++)
762 if (mp_irqs[i].irqtype == type &&
763 (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) ||
764 mp_irqs[i].dstapic == MP_APIC_ALL) &&
765 mp_irqs[i].dstirq == pin)
766 return i;
767
768 return -1;
769}
770
771
772
773
774static int __init find_isa_irq_pin(int irq, int type)
775{
776 int i;
777
778 for (i = 0; i < mp_irq_entries; i++) {
779 int lbus = mp_irqs[i].srcbus;
780
781 if (test_bit(lbus, mp_bus_not_pci) &&
782 (mp_irqs[i].irqtype == type) &&
783 (mp_irqs[i].srcbusirq == irq))
784
785 return mp_irqs[i].dstirq;
786 }
787 return -1;
788}
789
790static int __init find_isa_irq_apic(int irq, int type)
791{
792 int i;
793
794 for (i = 0; i < mp_irq_entries; i++) {
795 int lbus = mp_irqs[i].srcbus;
796
797 if (test_bit(lbus, mp_bus_not_pci) &&
798 (mp_irqs[i].irqtype == type) &&
799 (mp_irqs[i].srcbusirq == irq))
800 break;
801 }
802
803 if (i < mp_irq_entries) {
804 int ioapic_idx;
805
806 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
807 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic)
808 return ioapic_idx;
809 }
810
811 return -1;
812}
813
814#ifdef CONFIG_EISA
815
816
817
818static int EISA_ELCR(unsigned int irq)
819{
820 if (irq < legacy_pic->nr_legacy_irqs) {
821 unsigned int port = 0x4d0 + (irq >> 3);
822 return (inb(port) >> (irq & 7)) & 1;
823 }
824 apic_printk(APIC_VERBOSE, KERN_INFO
825 "Broken MPtable reports ISA irq %d\n", irq);
826 return 0;
827}
828
829#endif
830
831
832
833
834#define default_ISA_trigger(idx) (0)
835#define default_ISA_polarity(idx) (0)
836
837
838
839
840
841
842#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].srcbusirq))
843#define default_EISA_polarity(idx) default_ISA_polarity(idx)
844
845
846
847
848#define default_PCI_trigger(idx) (1)
849#define default_PCI_polarity(idx) (1)
850
851static int irq_polarity(int idx)
852{
853 int bus = mp_irqs[idx].srcbus;
854 int polarity;
855
856
857
858
859 switch (mp_irqs[idx].irqflag & 3)
860 {
861 case 0:
862 if (test_bit(bus, mp_bus_not_pci))
863 polarity = default_ISA_polarity(idx);
864 else
865 polarity = default_PCI_polarity(idx);
866 break;
867 case 1:
868 {
869 polarity = 0;
870 break;
871 }
872 case 2:
873 {
874 pr_warn("broken BIOS!!\n");
875 polarity = 1;
876 break;
877 }
878 case 3:
879 {
880 polarity = 1;
881 break;
882 }
883 default:
884 {
885 pr_warn("broken BIOS!!\n");
886 polarity = 1;
887 break;
888 }
889 }
890 return polarity;
891}
892
893static int irq_trigger(int idx)
894{
895 int bus = mp_irqs[idx].srcbus;
896 int trigger;
897
898
899
900
901 switch ((mp_irqs[idx].irqflag>>2) & 3)
902 {
903 case 0:
904 if (test_bit(bus, mp_bus_not_pci))
905 trigger = default_ISA_trigger(idx);
906 else
907 trigger = default_PCI_trigger(idx);
908#ifdef CONFIG_EISA
909 switch (mp_bus_id_to_type[bus]) {
910 case MP_BUS_ISA:
911 {
912
913 break;
914 }
915 case MP_BUS_EISA:
916 {
917 trigger = default_EISA_trigger(idx);
918 break;
919 }
920 case MP_BUS_PCI:
921 {
922
923 break;
924 }
925 default:
926 {
927 pr_warn("broken BIOS!!\n");
928 trigger = 1;
929 break;
930 }
931 }
932#endif
933 break;
934 case 1:
935 {
936 trigger = 0;
937 break;
938 }
939 case 2:
940 {
941 pr_warn("broken BIOS!!\n");
942 trigger = 1;
943 break;
944 }
945 case 3:
946 {
947 trigger = 1;
948 break;
949 }
950 default:
951 {
952 pr_warn("broken BIOS!!\n");
953 trigger = 0;
954 break;
955 }
956 }
957 return trigger;
958}
959
960static int pin_2_irq(int idx, int apic, int pin)
961{
962 int irq;
963 int bus = mp_irqs[idx].srcbus;
964 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(apic);
965
966
967
968
969 if (mp_irqs[idx].dstirq != pin)
970 pr_err("broken BIOS or MPTABLE parser, ayiee!!\n");
971
972 if (test_bit(bus, mp_bus_not_pci)) {
973 irq = mp_irqs[idx].srcbusirq;
974 } else {
975 u32 gsi = gsi_cfg->gsi_base + pin;
976
977 if (gsi >= NR_IRQS_LEGACY)
978 irq = gsi;
979 else
980 irq = gsi_top + gsi;
981 }
982
983#ifdef CONFIG_X86_32
984
985
986
987 if ((pin >= 16) && (pin <= 23)) {
988 if (pirq_entries[pin-16] != -1) {
989 if (!pirq_entries[pin-16]) {
990 apic_printk(APIC_VERBOSE, KERN_DEBUG
991 "disabling PIRQ%d\n", pin-16);
992 } else {
993 irq = pirq_entries[pin-16];
994 apic_printk(APIC_VERBOSE, KERN_DEBUG
995 "using PIRQ%d -> IRQ %d\n",
996 pin-16, irq);
997 }
998 }
999 }
1000#endif
1001
1002 return irq;
1003}
1004
1005
1006
1007
1008
1009int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1010 struct io_apic_irq_attr *irq_attr)
1011{
1012 int ioapic_idx, i, best_guess = -1;
1013
1014 apic_printk(APIC_DEBUG,
1015 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1016 bus, slot, pin);
1017 if (test_bit(bus, mp_bus_not_pci)) {
1018 apic_printk(APIC_VERBOSE,
1019 "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1020 return -1;
1021 }
1022 for (i = 0; i < mp_irq_entries; i++) {
1023 int lbus = mp_irqs[i].srcbus;
1024
1025 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1026 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic ||
1027 mp_irqs[i].dstapic == MP_APIC_ALL)
1028 break;
1029
1030 if (!test_bit(lbus, mp_bus_not_pci) &&
1031 !mp_irqs[i].irqtype &&
1032 (bus == lbus) &&
1033 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1034 int irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq);
1035
1036 if (!(ioapic_idx || IO_APIC_IRQ(irq)))
1037 continue;
1038
1039 if (pin == (mp_irqs[i].srcbusirq & 3)) {
1040 set_io_apic_irq_attr(irq_attr, ioapic_idx,
1041 mp_irqs[i].dstirq,
1042 irq_trigger(i),
1043 irq_polarity(i));
1044 return irq;
1045 }
1046
1047
1048
1049
1050 if (best_guess < 0) {
1051 set_io_apic_irq_attr(irq_attr, ioapic_idx,
1052 mp_irqs[i].dstirq,
1053 irq_trigger(i),
1054 irq_polarity(i));
1055 best_guess = irq;
1056 }
1057 }
1058 }
1059 return best_guess;
1060}
1061EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1062
1063void lock_vector_lock(void)
1064{
1065
1066
1067
1068 raw_spin_lock(&vector_lock);
1069}
1070
1071void unlock_vector_lock(void)
1072{
1073 raw_spin_unlock(&vector_lock);
1074}
1075
1076static int
1077__assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1078{
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090 static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START;
1091 static int current_offset = VECTOR_OFFSET_START % 16;
1092 int cpu, err;
1093 cpumask_var_t tmp_mask;
1094
1095 if (cfg->move_in_progress)
1096 return -EBUSY;
1097
1098 if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
1099 return -ENOMEM;
1100
1101
1102 err = -ENOSPC;
1103 cpumask_clear(cfg->old_domain);
1104 cpu = cpumask_first_and(mask, cpu_online_mask);
1105 while (cpu < nr_cpu_ids) {
1106 int new_cpu, vector, offset;
1107
1108 apic->vector_allocation_domain(cpu, tmp_mask, mask);
1109
1110 if (cpumask_subset(tmp_mask, cfg->domain)) {
1111 err = 0;
1112 if (cpumask_equal(tmp_mask, cfg->domain))
1113 break;
1114
1115
1116
1117
1118
1119 cpumask_andnot(cfg->old_domain, cfg->domain, tmp_mask);
1120 cfg->move_in_progress =
1121 cpumask_intersects(cfg->old_domain, cpu_online_mask);
1122 cpumask_and(cfg->domain, cfg->domain, tmp_mask);
1123 break;
1124 }
1125
1126 vector = current_vector;
1127 offset = current_offset;
1128next:
1129 vector += 16;
1130 if (vector >= first_system_vector) {
1131 offset = (offset + 1) % 16;
1132 vector = FIRST_EXTERNAL_VECTOR + offset;
1133 }
1134
1135 if (unlikely(current_vector == vector)) {
1136 cpumask_or(cfg->old_domain, cfg->old_domain, tmp_mask);
1137 cpumask_andnot(tmp_mask, mask, cfg->old_domain);
1138 cpu = cpumask_first_and(tmp_mask, cpu_online_mask);
1139 continue;
1140 }
1141
1142 if (test_bit(vector, used_vectors))
1143 goto next;
1144
1145 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1146 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1147 goto next;
1148
1149 current_vector = vector;
1150 current_offset = offset;
1151 if (cfg->vector) {
1152 cpumask_copy(cfg->old_domain, cfg->domain);
1153 cfg->move_in_progress =
1154 cpumask_intersects(cfg->old_domain, cpu_online_mask);
1155 }
1156 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1157 per_cpu(vector_irq, new_cpu)[vector] = irq;
1158 cfg->vector = vector;
1159 cpumask_copy(cfg->domain, tmp_mask);
1160 err = 0;
1161 break;
1162 }
1163 free_cpumask_var(tmp_mask);
1164 return err;
1165}
1166
1167int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1168{
1169 int err;
1170 unsigned long flags;
1171
1172 raw_spin_lock_irqsave(&vector_lock, flags);
1173 err = __assign_irq_vector(irq, cfg, mask);
1174 raw_spin_unlock_irqrestore(&vector_lock, flags);
1175 return err;
1176}
1177
1178static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1179{
1180 int cpu, vector;
1181
1182 BUG_ON(!cfg->vector);
1183
1184 vector = cfg->vector;
1185 for_each_cpu_and(cpu, cfg->domain, cpu_online_mask)
1186 per_cpu(vector_irq, cpu)[vector] = -1;
1187
1188 cfg->vector = 0;
1189 cpumask_clear(cfg->domain);
1190
1191 if (likely(!cfg->move_in_progress))
1192 return;
1193 for_each_cpu_and(cpu, cfg->old_domain, cpu_online_mask) {
1194 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
1195 vector++) {
1196 if (per_cpu(vector_irq, cpu)[vector] != irq)
1197 continue;
1198 per_cpu(vector_irq, cpu)[vector] = -1;
1199 break;
1200 }
1201 }
1202 cfg->move_in_progress = 0;
1203}
1204
1205void __setup_vector_irq(int cpu)
1206{
1207
1208 int irq, vector;
1209 struct irq_cfg *cfg;
1210
1211
1212
1213
1214
1215
1216 raw_spin_lock(&vector_lock);
1217
1218 for_each_active_irq(irq) {
1219 cfg = irq_get_chip_data(irq);
1220 if (!cfg)
1221 continue;
1222
1223 if (!cpumask_test_cpu(cpu, cfg->domain))
1224 continue;
1225 vector = cfg->vector;
1226 per_cpu(vector_irq, cpu)[vector] = irq;
1227 }
1228
1229 for (vector = 0; vector < NR_VECTORS; ++vector) {
1230 irq = per_cpu(vector_irq, cpu)[vector];
1231 if (irq < 0)
1232 continue;
1233
1234 cfg = irq_cfg(irq);
1235 if (!cpumask_test_cpu(cpu, cfg->domain))
1236 per_cpu(vector_irq, cpu)[vector] = -1;
1237 }
1238 raw_spin_unlock(&vector_lock);
1239}
1240
1241static struct irq_chip ioapic_chip;
1242
1243#ifdef CONFIG_X86_32
1244static inline int IO_APIC_irq_trigger(int irq)
1245{
1246 int apic, idx, pin;
1247
1248 for (apic = 0; apic < nr_ioapics; apic++) {
1249 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1250 idx = find_irq_entry(apic, pin, mp_INT);
1251 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1252 return irq_trigger(idx);
1253 }
1254 }
1255
1256
1257
1258 return 0;
1259}
1260#else
1261static inline int IO_APIC_irq_trigger(int irq)
1262{
1263 return 1;
1264}
1265#endif
1266
1267static void ioapic_register_intr(unsigned int irq, struct irq_cfg *cfg,
1268 unsigned long trigger)
1269{
1270 struct irq_chip *chip = &ioapic_chip;
1271 irq_flow_handler_t hdl;
1272 bool fasteoi;
1273
1274 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1275 trigger == IOAPIC_LEVEL) {
1276 irq_set_status_flags(irq, IRQ_LEVEL);
1277 fasteoi = true;
1278 } else {
1279 irq_clear_status_flags(irq, IRQ_LEVEL);
1280 fasteoi = false;
1281 }
1282
1283 if (setup_remapped_irq(irq, cfg, chip))
1284 fasteoi = trigger != 0;
1285
1286 hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
1287 irq_set_chip_and_handler_name(irq, chip, hdl,
1288 fasteoi ? "fasteoi" : "edge");
1289}
1290
1291int native_setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
1292 unsigned int destination, int vector,
1293 struct io_apic_irq_attr *attr)
1294{
1295 memset(entry, 0, sizeof(*entry));
1296
1297 entry->delivery_mode = apic->irq_delivery_mode;
1298 entry->dest_mode = apic->irq_dest_mode;
1299 entry->dest = destination;
1300 entry->vector = vector;
1301 entry->mask = 0;
1302 entry->trigger = attr->trigger;
1303 entry->polarity = attr->polarity;
1304
1305
1306
1307
1308
1309 if (attr->trigger)
1310 entry->mask = 1;
1311
1312 return 0;
1313}
1314
1315static void setup_ioapic_irq(unsigned int irq, struct irq_cfg *cfg,
1316 struct io_apic_irq_attr *attr)
1317{
1318 struct IO_APIC_route_entry entry;
1319 unsigned int dest;
1320
1321 if (!IO_APIC_IRQ(irq))
1322 return;
1323
1324 if (assign_irq_vector(irq, cfg, apic->target_cpus()))
1325 return;
1326
1327 if (apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus(),
1328 &dest)) {
1329 pr_warn("Failed to obtain apicid for ioapic %d, pin %d\n",
1330 mpc_ioapic_id(attr->ioapic), attr->ioapic_pin);
1331 __clear_irq_vector(irq, cfg);
1332
1333 return;
1334 }
1335
1336 apic_printk(APIC_VERBOSE,KERN_DEBUG
1337 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1338 "IRQ %d Mode:%i Active:%i Dest:%d)\n",
1339 attr->ioapic, mpc_ioapic_id(attr->ioapic), attr->ioapic_pin,
1340 cfg->vector, irq, attr->trigger, attr->polarity, dest);
1341
1342 if (x86_io_apic_ops.setup_entry(irq, &entry, dest, cfg->vector, attr)) {
1343 pr_warn("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1344 mpc_ioapic_id(attr->ioapic), attr->ioapic_pin);
1345 __clear_irq_vector(irq, cfg);
1346
1347 return;
1348 }
1349
1350 ioapic_register_intr(irq, cfg, attr->trigger);
1351 if (irq < legacy_pic->nr_legacy_irqs)
1352 legacy_pic->mask(irq);
1353
1354 ioapic_write_entry(attr->ioapic, attr->ioapic_pin, entry);
1355}
1356
1357static bool __init io_apic_pin_not_connected(int idx, int ioapic_idx, int pin)
1358{
1359 if (idx != -1)
1360 return false;
1361
1362 apic_printk(APIC_VERBOSE, KERN_DEBUG " apic %d pin %d not connected\n",
1363 mpc_ioapic_id(ioapic_idx), pin);
1364 return true;
1365}
1366
1367static void __init __io_apic_setup_irqs(unsigned int ioapic_idx)
1368{
1369 int idx, node = cpu_to_node(0);
1370 struct io_apic_irq_attr attr;
1371 unsigned int pin, irq;
1372
1373 for (pin = 0; pin < ioapics[ioapic_idx].nr_registers; pin++) {
1374 idx = find_irq_entry(ioapic_idx, pin, mp_INT);
1375 if (io_apic_pin_not_connected(idx, ioapic_idx, pin))
1376 continue;
1377
1378 irq = pin_2_irq(idx, ioapic_idx, pin);
1379
1380 if ((ioapic_idx > 0) && (irq > 16))
1381 continue;
1382
1383
1384
1385
1386
1387 if (apic->multi_timer_check &&
1388 apic->multi_timer_check(ioapic_idx, irq))
1389 continue;
1390
1391 set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1392 irq_polarity(idx));
1393
1394 io_apic_setup_irq_pin(irq, node, &attr);
1395 }
1396}
1397
1398static void __init setup_IO_APIC_irqs(void)
1399{
1400 unsigned int ioapic_idx;
1401
1402 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1403
1404 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1405 __io_apic_setup_irqs(ioapic_idx);
1406}
1407
1408
1409
1410
1411
1412
1413void setup_IO_APIC_irq_extra(u32 gsi)
1414{
1415 int ioapic_idx = 0, pin, idx, irq, node = cpu_to_node(0);
1416 struct io_apic_irq_attr attr;
1417
1418
1419
1420
1421 ioapic_idx = mp_find_ioapic(gsi);
1422 if (ioapic_idx < 0)
1423 return;
1424
1425 pin = mp_find_ioapic_pin(ioapic_idx, gsi);
1426 idx = find_irq_entry(ioapic_idx, pin, mp_INT);
1427 if (idx == -1)
1428 return;
1429
1430 irq = pin_2_irq(idx, ioapic_idx, pin);
1431
1432
1433 if (ioapic_idx == 0 || irq < NR_IRQS_LEGACY)
1434 return;
1435
1436 set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1437 irq_polarity(idx));
1438
1439 io_apic_setup_irq_pin_once(irq, node, &attr);
1440}
1441
1442
1443
1444
1445static void __init setup_timer_IRQ0_pin(unsigned int ioapic_idx,
1446 unsigned int pin, int vector)
1447{
1448 struct IO_APIC_route_entry entry;
1449 unsigned int dest;
1450
1451 memset(&entry, 0, sizeof(entry));
1452
1453
1454
1455
1456
1457 if (unlikely(apic->cpu_mask_to_apicid_and(apic->target_cpus(),
1458 apic->target_cpus(), &dest)))
1459 dest = BAD_APICID;
1460
1461 entry.dest_mode = apic->irq_dest_mode;
1462 entry.mask = 0;
1463 entry.dest = dest;
1464 entry.delivery_mode = apic->irq_delivery_mode;
1465 entry.polarity = 0;
1466 entry.trigger = 0;
1467 entry.vector = vector;
1468
1469
1470
1471
1472
1473 irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
1474 "edge");
1475
1476
1477
1478
1479 ioapic_write_entry(ioapic_idx, pin, entry);
1480}
1481
1482void native_io_apic_print_entries(unsigned int apic, unsigned int nr_entries)
1483{
1484 int i;
1485
1486 pr_debug(" NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect:\n");
1487
1488 for (i = 0; i <= nr_entries; i++) {
1489 struct IO_APIC_route_entry entry;
1490
1491 entry = ioapic_read_entry(apic, i);
1492
1493 pr_debug(" %02x %02X ", i, entry.dest);
1494 pr_cont("%1d %1d %1d %1d %1d "
1495 "%1d %1d %02X\n",
1496 entry.mask,
1497 entry.trigger,
1498 entry.irr,
1499 entry.polarity,
1500 entry.delivery_status,
1501 entry.dest_mode,
1502 entry.delivery_mode,
1503 entry.vector);
1504 }
1505}
1506
1507void intel_ir_io_apic_print_entries(unsigned int apic,
1508 unsigned int nr_entries)
1509{
1510 int i;
1511
1512 pr_debug(" NR Indx Fmt Mask Trig IRR Pol Stat Indx2 Zero Vect:\n");
1513
1514 for (i = 0; i <= nr_entries; i++) {
1515 struct IR_IO_APIC_route_entry *ir_entry;
1516 struct IO_APIC_route_entry entry;
1517
1518 entry = ioapic_read_entry(apic, i);
1519
1520 ir_entry = (struct IR_IO_APIC_route_entry *)&entry;
1521
1522 pr_debug(" %02x %04X ", i, ir_entry->index);
1523 pr_cont("%1d %1d %1d %1d %1d "
1524 "%1d %1d %X %02X\n",
1525 ir_entry->format,
1526 ir_entry->mask,
1527 ir_entry->trigger,
1528 ir_entry->irr,
1529 ir_entry->polarity,
1530 ir_entry->delivery_status,
1531 ir_entry->index2,
1532 ir_entry->zero,
1533 ir_entry->vector);
1534 }
1535}
1536
1537__apicdebuginit(void) print_IO_APIC(int ioapic_idx)
1538{
1539 union IO_APIC_reg_00 reg_00;
1540 union IO_APIC_reg_01 reg_01;
1541 union IO_APIC_reg_02 reg_02;
1542 union IO_APIC_reg_03 reg_03;
1543 unsigned long flags;
1544
1545 raw_spin_lock_irqsave(&ioapic_lock, flags);
1546 reg_00.raw = io_apic_read(ioapic_idx, 0);
1547 reg_01.raw = io_apic_read(ioapic_idx, 1);
1548 if (reg_01.bits.version >= 0x10)
1549 reg_02.raw = io_apic_read(ioapic_idx, 2);
1550 if (reg_01.bits.version >= 0x20)
1551 reg_03.raw = io_apic_read(ioapic_idx, 3);
1552 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1553
1554 printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx));
1555 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1556 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1557 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1558 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
1559
1560 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01);
1561 printk(KERN_DEBUG "....... : max redirection entries: %02X\n",
1562 reg_01.bits.entries);
1563
1564 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1565 printk(KERN_DEBUG "....... : IO APIC version: %02X\n",
1566 reg_01.bits.version);
1567
1568
1569
1570
1571
1572
1573 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1574 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1575 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
1576 }
1577
1578
1579
1580
1581
1582
1583 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1584 reg_03.raw != reg_01.raw) {
1585 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1586 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
1587 }
1588
1589 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1590
1591 x86_io_apic_ops.print_entries(ioapic_idx, reg_01.bits.entries);
1592}
1593
1594__apicdebuginit(void) print_IO_APICs(void)
1595{
1596 int ioapic_idx;
1597 struct irq_cfg *cfg;
1598 unsigned int irq;
1599 struct irq_chip *chip;
1600
1601 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1602 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1603 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1604 mpc_ioapic_id(ioapic_idx),
1605 ioapics[ioapic_idx].nr_registers);
1606
1607
1608
1609
1610
1611 printk(KERN_INFO "testing the IO APIC.......................\n");
1612
1613 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1614 print_IO_APIC(ioapic_idx);
1615
1616 printk(KERN_DEBUG "IRQ to pin mappings:\n");
1617 for_each_active_irq(irq) {
1618 struct irq_pin_list *entry;
1619
1620 chip = irq_get_chip(irq);
1621 if (chip != &ioapic_chip)
1622 continue;
1623
1624 cfg = irq_get_chip_data(irq);
1625 if (!cfg)
1626 continue;
1627 entry = cfg->irq_2_pin;
1628 if (!entry)
1629 continue;
1630 printk(KERN_DEBUG "IRQ%d ", irq);
1631 for_each_irq_pin(entry, cfg->irq_2_pin)
1632 pr_cont("-> %d:%d", entry->apic, entry->pin);
1633 pr_cont("\n");
1634 }
1635
1636 printk(KERN_INFO ".................................... done.\n");
1637}
1638
1639__apicdebuginit(void) print_APIC_field(int base)
1640{
1641 int i;
1642
1643 printk(KERN_DEBUG);
1644
1645 for (i = 0; i < 8; i++)
1646 pr_cont("%08x", apic_read(base + i*0x10));
1647
1648 pr_cont("\n");
1649}
1650
1651__apicdebuginit(void) print_local_APIC(void *dummy)
1652{
1653 unsigned int i, v, ver, maxlvt;
1654 u64 icr;
1655
1656 printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1657 smp_processor_id(), hard_smp_processor_id());
1658 v = apic_read(APIC_ID);
1659 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
1660 v = apic_read(APIC_LVR);
1661 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
1662 ver = GET_APIC_VERSION(v);
1663 maxlvt = lapic_get_maxlvt();
1664
1665 v = apic_read(APIC_TASKPRI);
1666 printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1667
1668 if (APIC_INTEGRATED(ver)) {
1669 if (!APIC_XAPIC(ver)) {
1670 v = apic_read(APIC_ARBPRI);
1671 printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
1672 v & APIC_ARBPRI_MASK);
1673 }
1674 v = apic_read(APIC_PROCPRI);
1675 printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
1676 }
1677
1678
1679
1680
1681
1682 if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
1683 v = apic_read(APIC_RRR);
1684 printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
1685 }
1686
1687 v = apic_read(APIC_LDR);
1688 printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1689 if (!x2apic_enabled()) {
1690 v = apic_read(APIC_DFR);
1691 printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
1692 }
1693 v = apic_read(APIC_SPIV);
1694 printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
1695
1696 printk(KERN_DEBUG "... APIC ISR field:\n");
1697 print_APIC_field(APIC_ISR);
1698 printk(KERN_DEBUG "... APIC TMR field:\n");
1699 print_APIC_field(APIC_TMR);
1700 printk(KERN_DEBUG "... APIC IRR field:\n");
1701 print_APIC_field(APIC_IRR);
1702
1703 if (APIC_INTEGRATED(ver)) {
1704 if (maxlvt > 3)
1705 apic_write(APIC_ESR, 0);
1706
1707 v = apic_read(APIC_ESR);
1708 printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
1709 }
1710
1711 icr = apic_icr_read();
1712 printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
1713 printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
1714
1715 v = apic_read(APIC_LVTT);
1716 printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
1717
1718 if (maxlvt > 3) {
1719 v = apic_read(APIC_LVTPC);
1720 printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
1721 }
1722 v = apic_read(APIC_LVT0);
1723 printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
1724 v = apic_read(APIC_LVT1);
1725 printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
1726
1727 if (maxlvt > 2) {
1728 v = apic_read(APIC_LVTERR);
1729 printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
1730 }
1731
1732 v = apic_read(APIC_TMICT);
1733 printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
1734 v = apic_read(APIC_TMCCT);
1735 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1736 v = apic_read(APIC_TDCR);
1737 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1738
1739 if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
1740 v = apic_read(APIC_EFEAT);
1741 maxlvt = (v >> 16) & 0xff;
1742 printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
1743 v = apic_read(APIC_ECTRL);
1744 printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
1745 for (i = 0; i < maxlvt; i++) {
1746 v = apic_read(APIC_EILVTn(i));
1747 printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
1748 }
1749 }
1750 pr_cont("\n");
1751}
1752
1753__apicdebuginit(void) print_local_APICs(int maxcpu)
1754{
1755 int cpu;
1756
1757 if (!maxcpu)
1758 return;
1759
1760 preempt_disable();
1761 for_each_online_cpu(cpu) {
1762 if (cpu >= maxcpu)
1763 break;
1764 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1765 }
1766 preempt_enable();
1767}
1768
1769__apicdebuginit(void) print_PIC(void)
1770{
1771 unsigned int v;
1772 unsigned long flags;
1773
1774 if (!legacy_pic->nr_legacy_irqs)
1775 return;
1776
1777 printk(KERN_DEBUG "\nprinting PIC contents\n");
1778
1779 raw_spin_lock_irqsave(&i8259A_lock, flags);
1780
1781 v = inb(0xa1) << 8 | inb(0x21);
1782 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
1783
1784 v = inb(0xa0) << 8 | inb(0x20);
1785 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1786
1787 outb(0x0b,0xa0);
1788 outb(0x0b,0x20);
1789 v = inb(0xa0) << 8 | inb(0x20);
1790 outb(0x0a,0xa0);
1791 outb(0x0a,0x20);
1792
1793 raw_spin_unlock_irqrestore(&i8259A_lock, flags);
1794
1795 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1796
1797 v = inb(0x4d1) << 8 | inb(0x4d0);
1798 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1799}
1800
1801static int __initdata show_lapic = 1;
1802static __init int setup_show_lapic(char *arg)
1803{
1804 int num = -1;
1805
1806 if (strcmp(arg, "all") == 0) {
1807 show_lapic = CONFIG_NR_CPUS;
1808 } else {
1809 get_option(&arg, &num);
1810 if (num >= 0)
1811 show_lapic = num;
1812 }
1813
1814 return 1;
1815}
1816__setup("show_lapic=", setup_show_lapic);
1817
1818__apicdebuginit(int) print_ICs(void)
1819{
1820 if (apic_verbosity == APIC_QUIET)
1821 return 0;
1822
1823 print_PIC();
1824
1825
1826 if (!cpu_has_apic && !apic_from_smp_config())
1827 return 0;
1828
1829 print_local_APICs(show_lapic);
1830 print_IO_APICs();
1831
1832 return 0;
1833}
1834
1835late_initcall(print_ICs);
1836
1837
1838
1839static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1840
1841void __init enable_IO_APIC(void)
1842{
1843 int i8259_apic, i8259_pin;
1844 int apic;
1845
1846 if (!legacy_pic->nr_legacy_irqs)
1847 return;
1848
1849 for(apic = 0; apic < nr_ioapics; apic++) {
1850 int pin;
1851
1852 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1853 struct IO_APIC_route_entry entry;
1854 entry = ioapic_read_entry(apic, pin);
1855
1856
1857
1858
1859 if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
1860 ioapic_i8259.apic = apic;
1861 ioapic_i8259.pin = pin;
1862 goto found_i8259;
1863 }
1864 }
1865 }
1866 found_i8259:
1867
1868
1869
1870
1871
1872 i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1873 i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1874
1875 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1876 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1877 ioapic_i8259.pin = i8259_pin;
1878 ioapic_i8259.apic = i8259_apic;
1879 }
1880
1881 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1882 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1883 {
1884 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1885 }
1886
1887
1888
1889
1890 clear_IO_APIC();
1891}
1892
1893void native_disable_io_apic(void)
1894{
1895
1896
1897
1898
1899
1900 if (ioapic_i8259.pin != -1) {
1901 struct IO_APIC_route_entry entry;
1902
1903 memset(&entry, 0, sizeof(entry));
1904 entry.mask = 0;
1905 entry.trigger = 0;
1906 entry.irr = 0;
1907 entry.polarity = 0;
1908 entry.delivery_status = 0;
1909 entry.dest_mode = 0;
1910 entry.delivery_mode = dest_ExtINT;
1911 entry.vector = 0;
1912 entry.dest = read_apic_id();
1913
1914
1915
1916
1917 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1918 }
1919
1920 if (cpu_has_apic || apic_from_smp_config())
1921 disconnect_bsp_APIC(ioapic_i8259.pin != -1);
1922
1923}
1924
1925
1926
1927
1928void disable_IO_APIC(void)
1929{
1930
1931
1932
1933 clear_IO_APIC();
1934
1935 if (!legacy_pic->nr_legacy_irqs)
1936 return;
1937
1938 x86_io_apic_ops.disable();
1939}
1940
1941#ifdef CONFIG_X86_32
1942
1943
1944
1945
1946
1947
1948void __init setup_ioapic_ids_from_mpc_nocheck(void)
1949{
1950 union IO_APIC_reg_00 reg_00;
1951 physid_mask_t phys_id_present_map;
1952 int ioapic_idx;
1953 int i;
1954 unsigned char old_id;
1955 unsigned long flags;
1956
1957
1958
1959
1960
1961 apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map);
1962
1963
1964
1965
1966 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) {
1967
1968 raw_spin_lock_irqsave(&ioapic_lock, flags);
1969 reg_00.raw = io_apic_read(ioapic_idx, 0);
1970 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1971
1972 old_id = mpc_ioapic_id(ioapic_idx);
1973
1974 if (mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) {
1975 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
1976 ioapic_idx, mpc_ioapic_id(ioapic_idx));
1977 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1978 reg_00.bits.ID);
1979 ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID;
1980 }
1981
1982
1983
1984
1985
1986
1987 if (apic->check_apicid_used(&phys_id_present_map,
1988 mpc_ioapic_id(ioapic_idx))) {
1989 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
1990 ioapic_idx, mpc_ioapic_id(ioapic_idx));
1991 for (i = 0; i < get_physical_broadcast(); i++)
1992 if (!physid_isset(i, phys_id_present_map))
1993 break;
1994 if (i >= get_physical_broadcast())
1995 panic("Max APIC ID exceeded!\n");
1996 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1997 i);
1998 physid_set(i, phys_id_present_map);
1999 ioapics[ioapic_idx].mp_config.apicid = i;
2000 } else {
2001 physid_mask_t tmp;
2002 apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx),
2003 &tmp);
2004 apic_printk(APIC_VERBOSE, "Setting %d in the "
2005 "phys_id_present_map\n",
2006 mpc_ioapic_id(ioapic_idx));
2007 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2008 }
2009
2010
2011
2012
2013
2014 if (old_id != mpc_ioapic_id(ioapic_idx))
2015 for (i = 0; i < mp_irq_entries; i++)
2016 if (mp_irqs[i].dstapic == old_id)
2017 mp_irqs[i].dstapic
2018 = mpc_ioapic_id(ioapic_idx);
2019
2020
2021
2022
2023
2024 if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID)
2025 continue;
2026
2027 apic_printk(APIC_VERBOSE, KERN_INFO
2028 "...changing IO-APIC physical APIC ID to %d ...",
2029 mpc_ioapic_id(ioapic_idx));
2030
2031 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2032 raw_spin_lock_irqsave(&ioapic_lock, flags);
2033 io_apic_write(ioapic_idx, 0, reg_00.raw);
2034 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2035
2036
2037
2038
2039 raw_spin_lock_irqsave(&ioapic_lock, flags);
2040 reg_00.raw = io_apic_read(ioapic_idx, 0);
2041 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2042 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx))
2043 pr_cont("could not set ID!\n");
2044 else
2045 apic_printk(APIC_VERBOSE, " ok.\n");
2046 }
2047}
2048
2049void __init setup_ioapic_ids_from_mpc(void)
2050{
2051
2052 if (acpi_ioapic)
2053 return;
2054
2055
2056
2057
2058 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2059 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
2060 return;
2061 setup_ioapic_ids_from_mpc_nocheck();
2062}
2063#endif
2064
2065int no_timer_check __initdata;
2066
2067static int __init notimercheck(char *s)
2068{
2069 no_timer_check = 1;
2070 return 1;
2071}
2072__setup("no_timer_check", notimercheck);
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082static int __init timer_irq_works(void)
2083{
2084 unsigned long t1 = jiffies;
2085 unsigned long flags;
2086
2087 if (no_timer_check)
2088 return 1;
2089
2090 local_save_flags(flags);
2091 local_irq_enable();
2092
2093 mdelay((10 * 1000) / HZ);
2094 local_irq_restore(flags);
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105 if (time_after(jiffies, t1 + 4))
2106 return 1;
2107 return 0;
2108}
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133static unsigned int startup_ioapic_irq(struct irq_data *data)
2134{
2135 int was_pending = 0, irq = data->irq;
2136 unsigned long flags;
2137
2138 raw_spin_lock_irqsave(&ioapic_lock, flags);
2139 if (irq < legacy_pic->nr_legacy_irqs) {
2140 legacy_pic->mask(irq);
2141 if (legacy_pic->irq_pending(irq))
2142 was_pending = 1;
2143 }
2144 __unmask_ioapic(data->chip_data);
2145 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2146
2147 return was_pending;
2148}
2149
2150static int ioapic_retrigger_irq(struct irq_data *data)
2151{
2152 struct irq_cfg *cfg = data->chip_data;
2153 unsigned long flags;
2154 int cpu;
2155
2156 raw_spin_lock_irqsave(&vector_lock, flags);
2157 cpu = cpumask_first_and(cfg->domain, cpu_online_mask);
2158 apic->send_IPI_mask(cpumask_of(cpu), cfg->vector);
2159 raw_spin_unlock_irqrestore(&vector_lock, flags);
2160
2161 return 1;
2162}
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173#ifdef CONFIG_SMP
2174void send_cleanup_vector(struct irq_cfg *cfg)
2175{
2176 cpumask_var_t cleanup_mask;
2177
2178 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
2179 unsigned int i;
2180 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2181 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
2182 } else {
2183 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
2184 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2185 free_cpumask_var(cleanup_mask);
2186 }
2187 cfg->move_in_progress = 0;
2188}
2189
2190asmlinkage void smp_irq_move_cleanup_interrupt(void)
2191{
2192 unsigned vector, me;
2193
2194 ack_APIC_irq();
2195 irq_enter();
2196 exit_idle();
2197
2198 me = smp_processor_id();
2199 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
2200 unsigned int irq;
2201 unsigned int irr;
2202 struct irq_desc *desc;
2203 struct irq_cfg *cfg;
2204 irq = __this_cpu_read(vector_irq[vector]);
2205
2206 if (irq == -1)
2207 continue;
2208
2209 desc = irq_to_desc(irq);
2210 if (!desc)
2211 continue;
2212
2213 cfg = irq_cfg(irq);
2214 if (!cfg)
2215 continue;
2216
2217 raw_spin_lock(&desc->lock);
2218
2219
2220
2221
2222
2223 if (cfg->move_in_progress)
2224 goto unlock;
2225
2226 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2227 goto unlock;
2228
2229 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
2230
2231
2232
2233
2234
2235
2236
2237 if (irr & (1 << (vector % 32))) {
2238 apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
2239 goto unlock;
2240 }
2241 __this_cpu_write(vector_irq[vector], -1);
2242unlock:
2243 raw_spin_unlock(&desc->lock);
2244 }
2245
2246 irq_exit();
2247}
2248
2249static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
2250{
2251 unsigned me;
2252
2253 if (likely(!cfg->move_in_progress))
2254 return;
2255
2256 me = smp_processor_id();
2257
2258 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2259 send_cleanup_vector(cfg);
2260}
2261
2262static void irq_complete_move(struct irq_cfg *cfg)
2263{
2264 __irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
2265}
2266
2267void irq_force_complete_move(int irq)
2268{
2269 struct irq_cfg *cfg = irq_get_chip_data(irq);
2270
2271 if (!cfg)
2272 return;
2273
2274 __irq_complete_move(cfg, cfg->vector);
2275}
2276#else
2277static inline void irq_complete_move(struct irq_cfg *cfg) { }
2278#endif
2279
2280static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2281{
2282 int apic, pin;
2283 struct irq_pin_list *entry;
2284 u8 vector = cfg->vector;
2285
2286 for_each_irq_pin(entry, cfg->irq_2_pin) {
2287 unsigned int reg;
2288
2289 apic = entry->apic;
2290 pin = entry->pin;
2291
2292 io_apic_write(apic, 0x11 + pin*2, dest);
2293 reg = io_apic_read(apic, 0x10 + pin*2);
2294 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
2295 reg |= vector;
2296 io_apic_modify(apic, 0x10 + pin*2, reg);
2297 }
2298}
2299
2300
2301
2302
2303
2304
2305int __ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
2306 unsigned int *dest_id)
2307{
2308 struct irq_cfg *cfg = data->chip_data;
2309 unsigned int irq = data->irq;
2310 int err;
2311
2312 if (!config_enabled(CONFIG_SMP))
2313 return -1;
2314
2315 if (!cpumask_intersects(mask, cpu_online_mask))
2316 return -EINVAL;
2317
2318 err = assign_irq_vector(irq, cfg, mask);
2319 if (err)
2320 return err;
2321
2322 err = apic->cpu_mask_to_apicid_and(mask, cfg->domain, dest_id);
2323 if (err) {
2324 if (assign_irq_vector(irq, cfg, data->affinity))
2325 pr_err("Failed to recover vector for irq %d\n", irq);
2326 return err;
2327 }
2328
2329 cpumask_copy(data->affinity, mask);
2330
2331 return 0;
2332}
2333
2334
2335int native_ioapic_set_affinity(struct irq_data *data,
2336 const struct cpumask *mask,
2337 bool force)
2338{
2339 unsigned int dest, irq = data->irq;
2340 unsigned long flags;
2341 int ret;
2342
2343 if (!config_enabled(CONFIG_SMP))
2344 return -1;
2345
2346 raw_spin_lock_irqsave(&ioapic_lock, flags);
2347 ret = __ioapic_set_affinity(data, mask, &dest);
2348 if (!ret) {
2349
2350 dest = SET_APIC_LOGICAL_ID(dest);
2351 __target_IO_APIC_irq(irq, dest, data->chip_data);
2352 ret = IRQ_SET_MASK_OK_NOCOPY;
2353 }
2354 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2355 return ret;
2356}
2357
2358static void ack_apic_edge(struct irq_data *data)
2359{
2360 irq_complete_move(data->chip_data);
2361 irq_move_irq(data);
2362 ack_APIC_irq();
2363}
2364
2365atomic_t irq_mis_count;
2366
2367#ifdef CONFIG_GENERIC_PENDING_IRQ
2368static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
2369{
2370 struct irq_pin_list *entry;
2371 unsigned long flags;
2372
2373 raw_spin_lock_irqsave(&ioapic_lock, flags);
2374 for_each_irq_pin(entry, cfg->irq_2_pin) {
2375 unsigned int reg;
2376 int pin;
2377
2378 pin = entry->pin;
2379 reg = io_apic_read(entry->apic, 0x10 + pin*2);
2380
2381 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
2382 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2383 return true;
2384 }
2385 }
2386 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2387
2388 return false;
2389}
2390
2391static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
2392{
2393
2394 if (unlikely(irqd_is_setaffinity_pending(data))) {
2395 mask_ioapic(cfg);
2396 return true;
2397 }
2398 return false;
2399}
2400
2401static inline void ioapic_irqd_unmask(struct irq_data *data,
2402 struct irq_cfg *cfg, bool masked)
2403{
2404 if (unlikely(masked)) {
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431 if (!io_apic_level_ack_pending(cfg))
2432 irq_move_masked_irq(data);
2433 unmask_ioapic(cfg);
2434 }
2435}
2436#else
2437static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
2438{
2439 return false;
2440}
2441static inline void ioapic_irqd_unmask(struct irq_data *data,
2442 struct irq_cfg *cfg, bool masked)
2443{
2444}
2445#endif
2446
2447static void ack_apic_level(struct irq_data *data)
2448{
2449 struct irq_cfg *cfg = data->chip_data;
2450 int i, irq = data->irq;
2451 unsigned long v;
2452 bool masked;
2453
2454 irq_complete_move(cfg);
2455 masked = ioapic_irqd_mask(data, cfg);
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489 i = cfg->vector;
2490 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2491
2492
2493
2494
2495
2496 ack_APIC_irq();
2497
2498
2499
2500
2501
2502
2503
2504
2505 if (!(v & (1 << (i & 0x1f)))) {
2506 atomic_inc(&irq_mis_count);
2507
2508 eoi_ioapic_irq(irq, cfg);
2509 }
2510
2511 ioapic_irqd_unmask(data, cfg, masked);
2512}
2513
2514static struct irq_chip ioapic_chip __read_mostly = {
2515 .name = "IO-APIC",
2516 .irq_startup = startup_ioapic_irq,
2517 .irq_mask = mask_ioapic_irq,
2518 .irq_unmask = unmask_ioapic_irq,
2519 .irq_ack = ack_apic_edge,
2520 .irq_eoi = ack_apic_level,
2521 .irq_set_affinity = native_ioapic_set_affinity,
2522 .irq_retrigger = ioapic_retrigger_irq,
2523};
2524
2525static inline void init_IO_APIC_traps(void)
2526{
2527 struct irq_cfg *cfg;
2528 unsigned int irq;
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541 for_each_active_irq(irq) {
2542 cfg = irq_get_chip_data(irq);
2543 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
2544
2545
2546
2547
2548
2549 if (irq < legacy_pic->nr_legacy_irqs)
2550 legacy_pic->make_irq(irq);
2551 else
2552
2553 irq_set_chip(irq, &no_irq_chip);
2554 }
2555 }
2556}
2557
2558
2559
2560
2561
2562static void mask_lapic_irq(struct irq_data *data)
2563{
2564 unsigned long v;
2565
2566 v = apic_read(APIC_LVT0);
2567 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
2568}
2569
2570static void unmask_lapic_irq(struct irq_data *data)
2571{
2572 unsigned long v;
2573
2574 v = apic_read(APIC_LVT0);
2575 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2576}
2577
2578static void ack_lapic_irq(struct irq_data *data)
2579{
2580 ack_APIC_irq();
2581}
2582
2583static struct irq_chip lapic_chip __read_mostly = {
2584 .name = "local-APIC",
2585 .irq_mask = mask_lapic_irq,
2586 .irq_unmask = unmask_lapic_irq,
2587 .irq_ack = ack_lapic_irq,
2588};
2589
2590static void lapic_register_intr(int irq)
2591{
2592 irq_clear_status_flags(irq, IRQ_LEVEL);
2593 irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2594 "edge");
2595}
2596
2597
2598
2599
2600
2601
2602
2603
2604static inline void __init unlock_ExtINT_logic(void)
2605{
2606 int apic, pin, i;
2607 struct IO_APIC_route_entry entry0, entry1;
2608 unsigned char save_control, save_freq_select;
2609
2610 pin = find_isa_irq_pin(8, mp_INT);
2611 if (pin == -1) {
2612 WARN_ON_ONCE(1);
2613 return;
2614 }
2615 apic = find_isa_irq_apic(8, mp_INT);
2616 if (apic == -1) {
2617 WARN_ON_ONCE(1);
2618 return;
2619 }
2620
2621 entry0 = ioapic_read_entry(apic, pin);
2622 clear_IO_APIC_pin(apic, pin);
2623
2624 memset(&entry1, 0, sizeof(entry1));
2625
2626 entry1.dest_mode = 0;
2627 entry1.mask = 0;
2628 entry1.dest = hard_smp_processor_id();
2629 entry1.delivery_mode = dest_ExtINT;
2630 entry1.polarity = entry0.polarity;
2631 entry1.trigger = 0;
2632 entry1.vector = 0;
2633
2634 ioapic_write_entry(apic, pin, entry1);
2635
2636 save_control = CMOS_READ(RTC_CONTROL);
2637 save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2638 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2639 RTC_FREQ_SELECT);
2640 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2641
2642 i = 100;
2643 while (i-- > 0) {
2644 mdelay(10);
2645 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2646 i -= 10;
2647 }
2648
2649 CMOS_WRITE(save_control, RTC_CONTROL);
2650 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2651 clear_IO_APIC_pin(apic, pin);
2652
2653 ioapic_write_entry(apic, pin, entry0);
2654}
2655
2656static int disable_timer_pin_1 __initdata;
2657
2658static int __init disable_timer_pin_setup(char *arg)
2659{
2660 disable_timer_pin_1 = 1;
2661 return 0;
2662}
2663early_param("disable_timer_pin_1", disable_timer_pin_setup);
2664
2665int timer_through_8259 __initdata;
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675static inline void __init check_timer(void)
2676{
2677 struct irq_cfg *cfg = irq_get_chip_data(0);
2678 int node = cpu_to_node(0);
2679 int apic1, pin1, apic2, pin2;
2680 unsigned long flags;
2681 int no_pin1 = 0;
2682
2683 local_irq_save(flags);
2684
2685
2686
2687
2688 legacy_pic->mask(0);
2689 assign_irq_vector(0, cfg, apic->target_cpus());
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2701 legacy_pic->init(1);
2702
2703 pin1 = find_isa_irq_pin(0, mp_INT);
2704 apic1 = find_isa_irq_apic(0, mp_INT);
2705 pin2 = ioapic_i8259.pin;
2706 apic2 = ioapic_i8259.apic;
2707
2708 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2709 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2710 cfg->vector, apic1, pin1, apic2, pin2);
2711
2712
2713
2714
2715
2716
2717
2718
2719 if (pin1 == -1) {
2720 panic_if_irq_remap("BIOS bug: timer not connected to IO-APIC");
2721 pin1 = pin2;
2722 apic1 = apic2;
2723 no_pin1 = 1;
2724 } else if (pin2 == -1) {
2725 pin2 = pin1;
2726 apic2 = apic1;
2727 }
2728
2729 if (pin1 != -1) {
2730
2731
2732
2733 if (no_pin1) {
2734 add_pin_to_irq_node(cfg, node, apic1, pin1);
2735 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2736 } else {
2737
2738
2739
2740
2741
2742 int idx;
2743 idx = find_irq_entry(apic1, pin1, mp_INT);
2744 if (idx != -1 && irq_trigger(idx))
2745 unmask_ioapic(cfg);
2746 }
2747 if (timer_irq_works()) {
2748 if (disable_timer_pin_1 > 0)
2749 clear_IO_APIC_pin(0, pin1);
2750 goto out;
2751 }
2752 panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC");
2753 local_irq_disable();
2754 clear_IO_APIC_pin(apic1, pin1);
2755 if (!no_pin1)
2756 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2757 "8254 timer not connected to IO-APIC\n");
2758
2759 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2760 "(IRQ0) through the 8259A ...\n");
2761 apic_printk(APIC_QUIET, KERN_INFO
2762 "..... (found apic %d pin %d) ...\n", apic2, pin2);
2763
2764
2765
2766 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2767 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2768 legacy_pic->unmask(0);
2769 if (timer_irq_works()) {
2770 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2771 timer_through_8259 = 1;
2772 goto out;
2773 }
2774
2775
2776
2777 local_irq_disable();
2778 legacy_pic->mask(0);
2779 clear_IO_APIC_pin(apic2, pin2);
2780 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
2781 }
2782
2783 apic_printk(APIC_QUIET, KERN_INFO
2784 "...trying to set up timer as Virtual Wire IRQ...\n");
2785
2786 lapic_register_intr(0);
2787 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);
2788 legacy_pic->unmask(0);
2789
2790 if (timer_irq_works()) {
2791 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2792 goto out;
2793 }
2794 local_irq_disable();
2795 legacy_pic->mask(0);
2796 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2797 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
2798
2799 apic_printk(APIC_QUIET, KERN_INFO
2800 "...trying to set up timer as ExtINT IRQ...\n");
2801
2802 legacy_pic->init(0);
2803 legacy_pic->make_irq(0);
2804 apic_write(APIC_LVT0, APIC_DM_EXTINT);
2805
2806 unlock_ExtINT_logic();
2807
2808 if (timer_irq_works()) {
2809 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2810 goto out;
2811 }
2812 local_irq_disable();
2813 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
2814 if (x2apic_preenabled)
2815 apic_printk(APIC_QUIET, KERN_INFO
2816 "Perhaps problem with the pre-enabled x2apic mode\n"
2817 "Try booting with x2apic and interrupt-remapping disabled in the bios.\n");
2818 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
2819 "report. Then try booting with the 'noapic' option.\n");
2820out:
2821 local_irq_restore(flags);
2822}
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841#define PIC_IRQS (1UL << PIC_CASCADE_IR)
2842
2843void __init setup_IO_APIC(void)
2844{
2845
2846
2847
2848
2849 io_apic_irqs = legacy_pic->nr_legacy_irqs ? ~PIC_IRQS : ~0UL;
2850
2851 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2852
2853
2854
2855 x86_init.mpparse.setup_ioapic_ids();
2856
2857 sync_Arb_IDs();
2858 setup_IO_APIC_irqs();
2859 init_IO_APIC_traps();
2860 if (legacy_pic->nr_legacy_irqs)
2861 check_timer();
2862}
2863
2864
2865
2866
2867
2868
2869static int __init io_apic_bug_finalize(void)
2870{
2871 if (sis_apic_bug == -1)
2872 sis_apic_bug = 0;
2873 return 0;
2874}
2875
2876late_initcall(io_apic_bug_finalize);
2877
2878static void resume_ioapic_id(int ioapic_idx)
2879{
2880 unsigned long flags;
2881 union IO_APIC_reg_00 reg_00;
2882
2883 raw_spin_lock_irqsave(&ioapic_lock, flags);
2884 reg_00.raw = io_apic_read(ioapic_idx, 0);
2885 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) {
2886 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2887 io_apic_write(ioapic_idx, 0, reg_00.raw);
2888 }
2889 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2890}
2891
2892static void ioapic_resume(void)
2893{
2894 int ioapic_idx;
2895
2896 for (ioapic_idx = nr_ioapics - 1; ioapic_idx >= 0; ioapic_idx--)
2897 resume_ioapic_id(ioapic_idx);
2898
2899 restore_ioapic_entries();
2900}
2901
2902static struct syscore_ops ioapic_syscore_ops = {
2903 .suspend = save_ioapic_entries,
2904 .resume = ioapic_resume,
2905};
2906
2907static int __init ioapic_init_ops(void)
2908{
2909 register_syscore_ops(&ioapic_syscore_ops);
2910
2911 return 0;
2912}
2913
2914device_initcall(ioapic_init_ops);
2915
2916
2917
2918
2919unsigned int __create_irqs(unsigned int from, unsigned int count, int node)
2920{
2921 struct irq_cfg **cfg;
2922 unsigned long flags;
2923 int irq, i;
2924
2925 if (from < nr_irqs_gsi)
2926 from = nr_irqs_gsi;
2927
2928 cfg = kzalloc_node(count * sizeof(cfg[0]), GFP_KERNEL, node);
2929 if (!cfg)
2930 return 0;
2931
2932 irq = alloc_irqs_from(from, count, node);
2933 if (irq < 0)
2934 goto out_cfgs;
2935
2936 for (i = 0; i < count; i++) {
2937 cfg[i] = alloc_irq_cfg(irq + i, node);
2938 if (!cfg[i])
2939 goto out_irqs;
2940 }
2941
2942 raw_spin_lock_irqsave(&vector_lock, flags);
2943 for (i = 0; i < count; i++)
2944 if (__assign_irq_vector(irq + i, cfg[i], apic->target_cpus()))
2945 goto out_vecs;
2946 raw_spin_unlock_irqrestore(&vector_lock, flags);
2947
2948 for (i = 0; i < count; i++) {
2949 irq_set_chip_data(irq + i, cfg[i]);
2950 irq_clear_status_flags(irq + i, IRQ_NOREQUEST);
2951 }
2952
2953 kfree(cfg);
2954 return irq;
2955
2956out_vecs:
2957 for (i--; i >= 0; i--)
2958 __clear_irq_vector(irq + i, cfg[i]);
2959 raw_spin_unlock_irqrestore(&vector_lock, flags);
2960out_irqs:
2961 for (i = 0; i < count; i++)
2962 free_irq_at(irq + i, cfg[i]);
2963out_cfgs:
2964 kfree(cfg);
2965 return 0;
2966}
2967
2968unsigned int create_irq_nr(unsigned int from, int node)
2969{
2970 return __create_irqs(from, 1, node);
2971}
2972
2973int create_irq(void)
2974{
2975 int node = cpu_to_node(0);
2976 unsigned int irq_want;
2977 int irq;
2978
2979 irq_want = nr_irqs_gsi;
2980 irq = create_irq_nr(irq_want, node);
2981
2982 if (irq == 0)
2983 irq = -1;
2984
2985 return irq;
2986}
2987
2988void destroy_irq(unsigned int irq)
2989{
2990 struct irq_cfg *cfg = irq_get_chip_data(irq);
2991 unsigned long flags;
2992
2993 irq_set_status_flags(irq, IRQ_NOREQUEST|IRQ_NOPROBE);
2994
2995 free_remapped_irq(irq);
2996
2997 raw_spin_lock_irqsave(&vector_lock, flags);
2998 __clear_irq_vector(irq, cfg);
2999 raw_spin_unlock_irqrestore(&vector_lock, flags);
3000 free_irq_at(irq, cfg);
3001}
3002
3003void destroy_irqs(unsigned int irq, unsigned int count)
3004{
3005 unsigned int i;
3006
3007 for (i = 0; i < count; i++)
3008 destroy_irq(irq + i);
3009}
3010
3011
3012
3013
3014void native_compose_msi_msg(struct pci_dev *pdev,
3015 unsigned int irq, unsigned int dest,
3016 struct msi_msg *msg, u8 hpet_id)
3017{
3018 struct irq_cfg *cfg = irq_cfg(irq);
3019
3020 msg->address_hi = MSI_ADDR_BASE_HI;
3021
3022 if (x2apic_enabled())
3023 msg->address_hi |= MSI_ADDR_EXT_DEST_ID(dest);
3024
3025 msg->address_lo =
3026 MSI_ADDR_BASE_LO |
3027 ((apic->irq_dest_mode == 0) ?
3028 MSI_ADDR_DEST_MODE_PHYSICAL:
3029 MSI_ADDR_DEST_MODE_LOGICAL) |
3030 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3031 MSI_ADDR_REDIRECTION_CPU:
3032 MSI_ADDR_REDIRECTION_LOWPRI) |
3033 MSI_ADDR_DEST_ID(dest);
3034
3035 msg->data =
3036 MSI_DATA_TRIGGER_EDGE |
3037 MSI_DATA_LEVEL_ASSERT |
3038 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3039 MSI_DATA_DELIVERY_FIXED:
3040 MSI_DATA_DELIVERY_LOWPRI) |
3041 MSI_DATA_VECTOR(cfg->vector);
3042}
3043
3044#ifdef CONFIG_PCI_MSI
3045static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
3046 struct msi_msg *msg, u8 hpet_id)
3047{
3048 struct irq_cfg *cfg;
3049 int err;
3050 unsigned dest;
3051
3052 if (disable_apic)
3053 return -ENXIO;
3054
3055 cfg = irq_cfg(irq);
3056 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3057 if (err)
3058 return err;
3059
3060 err = apic->cpu_mask_to_apicid_and(cfg->domain,
3061 apic->target_cpus(), &dest);
3062 if (err)
3063 return err;
3064
3065 x86_msi.compose_msi_msg(pdev, irq, dest, msg, hpet_id);
3066
3067 return 0;
3068}
3069
3070static int
3071msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3072{
3073 struct irq_cfg *cfg = data->chip_data;
3074 struct msi_msg msg;
3075 unsigned int dest;
3076
3077 if (__ioapic_set_affinity(data, mask, &dest))
3078 return -1;
3079
3080 __get_cached_msi_msg(data->msi_desc, &msg);
3081
3082 msg.data &= ~MSI_DATA_VECTOR_MASK;
3083 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3084 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3085 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3086
3087 __write_msi_msg(data->msi_desc, &msg);
3088
3089 return IRQ_SET_MASK_OK_NOCOPY;
3090}
3091
3092
3093
3094
3095
3096static struct irq_chip msi_chip = {
3097 .name = "PCI-MSI",
3098 .irq_unmask = unmask_msi_irq,
3099 .irq_mask = mask_msi_irq,
3100 .irq_ack = ack_apic_edge,
3101 .irq_set_affinity = msi_set_affinity,
3102 .irq_retrigger = ioapic_retrigger_irq,
3103};
3104
3105int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc,
3106 unsigned int irq_base, unsigned int irq_offset)
3107{
3108 struct irq_chip *chip = &msi_chip;
3109 struct msi_msg msg;
3110 unsigned int irq = irq_base + irq_offset;
3111 int ret;
3112
3113 ret = msi_compose_msg(dev, irq, &msg, -1);
3114 if (ret < 0)
3115 return ret;
3116
3117 irq_set_msi_desc_off(irq_base, irq_offset, msidesc);
3118
3119
3120
3121
3122
3123 if (!irq_offset)
3124 write_msi_msg(irq, &msg);
3125
3126 setup_remapped_irq(irq, irq_get_chip_data(irq), chip);
3127
3128 irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3129
3130 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);
3131
3132 return 0;
3133}
3134
3135int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3136{
3137 unsigned int irq, irq_want;
3138 struct msi_desc *msidesc;
3139 int node, ret;
3140
3141
3142 if (type == PCI_CAP_ID_MSI && nvec > 1)
3143 return 1;
3144
3145 node = dev_to_node(&dev->dev);
3146 irq_want = nr_irqs_gsi;
3147 list_for_each_entry(msidesc, &dev->msi_list, list) {
3148 irq = create_irq_nr(irq_want, node);
3149 if (irq == 0)
3150 return -ENOSPC;
3151
3152 irq_want = irq + 1;
3153
3154 ret = setup_msi_irq(dev, msidesc, irq, 0);
3155 if (ret < 0)
3156 goto error;
3157 }
3158 return 0;
3159
3160error:
3161 destroy_irq(irq);
3162 return ret;
3163}
3164
3165void native_teardown_msi_irq(unsigned int irq)
3166{
3167 destroy_irq(irq);
3168}
3169
3170#ifdef CONFIG_DMAR_TABLE
3171static int
3172dmar_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
3173 bool force)
3174{
3175 struct irq_cfg *cfg = data->chip_data;
3176 unsigned int dest, irq = data->irq;
3177 struct msi_msg msg;
3178
3179 if (__ioapic_set_affinity(data, mask, &dest))
3180 return -1;
3181
3182 dmar_msi_read(irq, &msg);
3183
3184 msg.data &= ~MSI_DATA_VECTOR_MASK;
3185 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3186 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3187 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3188 msg.address_hi = MSI_ADDR_BASE_HI | MSI_ADDR_EXT_DEST_ID(dest);
3189
3190 dmar_msi_write(irq, &msg);
3191
3192 return IRQ_SET_MASK_OK_NOCOPY;
3193}
3194
3195static struct irq_chip dmar_msi_type = {
3196 .name = "DMAR_MSI",
3197 .irq_unmask = dmar_msi_unmask,
3198 .irq_mask = dmar_msi_mask,
3199 .irq_ack = ack_apic_edge,
3200 .irq_set_affinity = dmar_msi_set_affinity,
3201 .irq_retrigger = ioapic_retrigger_irq,
3202};
3203
3204int arch_setup_dmar_msi(unsigned int irq)
3205{
3206 int ret;
3207 struct msi_msg msg;
3208
3209 ret = msi_compose_msg(NULL, irq, &msg, -1);
3210 if (ret < 0)
3211 return ret;
3212 dmar_msi_write(irq, &msg);
3213 irq_set_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
3214 "edge");
3215 return 0;
3216}
3217#endif
3218
3219#ifdef CONFIG_HPET_TIMER
3220
3221static int hpet_msi_set_affinity(struct irq_data *data,
3222 const struct cpumask *mask, bool force)
3223{
3224 struct irq_cfg *cfg = data->chip_data;
3225 struct msi_msg msg;
3226 unsigned int dest;
3227
3228 if (__ioapic_set_affinity(data, mask, &dest))
3229 return -1;
3230
3231 hpet_msi_read(data->handler_data, &msg);
3232
3233 msg.data &= ~MSI_DATA_VECTOR_MASK;
3234 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3235 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3236 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3237
3238 hpet_msi_write(data->handler_data, &msg);
3239
3240 return IRQ_SET_MASK_OK_NOCOPY;
3241}
3242
3243static struct irq_chip hpet_msi_type = {
3244 .name = "HPET_MSI",
3245 .irq_unmask = hpet_msi_unmask,
3246 .irq_mask = hpet_msi_mask,
3247 .irq_ack = ack_apic_edge,
3248 .irq_set_affinity = hpet_msi_set_affinity,
3249 .irq_retrigger = ioapic_retrigger_irq,
3250};
3251
3252int default_setup_hpet_msi(unsigned int irq, unsigned int id)
3253{
3254 struct irq_chip *chip = &hpet_msi_type;
3255 struct msi_msg msg;
3256 int ret;
3257
3258 ret = msi_compose_msg(NULL, irq, &msg, id);
3259 if (ret < 0)
3260 return ret;
3261
3262 hpet_msi_write(irq_get_handler_data(irq), &msg);
3263 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
3264 setup_remapped_irq(irq, irq_get_chip_data(irq), chip);
3265
3266 irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3267 return 0;
3268}
3269#endif
3270
3271#endif
3272
3273
3274
3275#ifdef CONFIG_HT_IRQ
3276
3277static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3278{
3279 struct ht_irq_msg msg;
3280 fetch_ht_irq_msg(irq, &msg);
3281
3282 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
3283 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3284
3285 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
3286 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3287
3288 write_ht_irq_msg(irq, &msg);
3289}
3290
3291static int
3292ht_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3293{
3294 struct irq_cfg *cfg = data->chip_data;
3295 unsigned int dest;
3296
3297 if (__ioapic_set_affinity(data, mask, &dest))
3298 return -1;
3299
3300 target_ht_irq(data->irq, dest, cfg->vector);
3301 return IRQ_SET_MASK_OK_NOCOPY;
3302}
3303
3304static struct irq_chip ht_irq_chip = {
3305 .name = "PCI-HT",
3306 .irq_mask = mask_ht_irq,
3307 .irq_unmask = unmask_ht_irq,
3308 .irq_ack = ack_apic_edge,
3309 .irq_set_affinity = ht_set_affinity,
3310 .irq_retrigger = ioapic_retrigger_irq,
3311};
3312
3313int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3314{
3315 struct irq_cfg *cfg;
3316 struct ht_irq_msg msg;
3317 unsigned dest;
3318 int err;
3319
3320 if (disable_apic)
3321 return -ENXIO;
3322
3323 cfg = irq_cfg(irq);
3324 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3325 if (err)
3326 return err;
3327
3328 err = apic->cpu_mask_to_apicid_and(cfg->domain,
3329 apic->target_cpus(), &dest);
3330 if (err)
3331 return err;
3332
3333 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3334
3335 msg.address_lo =
3336 HT_IRQ_LOW_BASE |
3337 HT_IRQ_LOW_DEST_ID(dest) |
3338 HT_IRQ_LOW_VECTOR(cfg->vector) |
3339 ((apic->irq_dest_mode == 0) ?
3340 HT_IRQ_LOW_DM_PHYSICAL :
3341 HT_IRQ_LOW_DM_LOGICAL) |
3342 HT_IRQ_LOW_RQEOI_EDGE |
3343 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3344 HT_IRQ_LOW_MT_FIXED :
3345 HT_IRQ_LOW_MT_ARBITRATED) |
3346 HT_IRQ_LOW_IRQ_MASKED;
3347
3348 write_ht_irq_msg(irq, &msg);
3349
3350 irq_set_chip_and_handler_name(irq, &ht_irq_chip,
3351 handle_edge_irq, "edge");
3352
3353 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
3354
3355 return 0;
3356}
3357#endif
3358
3359static int
3360io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
3361{
3362 struct irq_cfg *cfg = alloc_irq_and_cfg_at(irq, node);
3363 int ret;
3364
3365 if (!cfg)
3366 return -EINVAL;
3367 ret = __add_pin_to_irq_node(cfg, node, attr->ioapic, attr->ioapic_pin);
3368 if (!ret)
3369 setup_ioapic_irq(irq, cfg, attr);
3370 return ret;
3371}
3372
3373int io_apic_setup_irq_pin_once(unsigned int irq, int node,
3374 struct io_apic_irq_attr *attr)
3375{
3376 unsigned int ioapic_idx = attr->ioapic, pin = attr->ioapic_pin;
3377 int ret;
3378
3379
3380 if (test_bit(pin, ioapics[ioapic_idx].pin_programmed)) {
3381 pr_debug("Pin %d-%d already programmed\n",
3382 mpc_ioapic_id(ioapic_idx), pin);
3383 return 0;
3384 }
3385 ret = io_apic_setup_irq_pin(irq, node, attr);
3386 if (!ret)
3387 set_bit(pin, ioapics[ioapic_idx].pin_programmed);
3388 return ret;
3389}
3390
3391static int __init io_apic_get_redir_entries(int ioapic)
3392{
3393 union IO_APIC_reg_01 reg_01;
3394 unsigned long flags;
3395
3396 raw_spin_lock_irqsave(&ioapic_lock, flags);
3397 reg_01.raw = io_apic_read(ioapic, 1);
3398 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3399
3400
3401
3402
3403
3404 return reg_01.bits.entries + 1;
3405}
3406
3407static void __init probe_nr_irqs_gsi(void)
3408{
3409 int nr;
3410
3411 nr = gsi_top + NR_IRQS_LEGACY;
3412 if (nr > nr_irqs_gsi)
3413 nr_irqs_gsi = nr;
3414
3415 printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
3416}
3417
3418int get_nr_irqs_gsi(void)
3419{
3420 return nr_irqs_gsi;
3421}
3422
3423int __init arch_probe_nr_irqs(void)
3424{
3425 int nr;
3426
3427 if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
3428 nr_irqs = NR_VECTORS * nr_cpu_ids;
3429
3430 nr = nr_irqs_gsi + 8 * nr_cpu_ids;
3431#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
3432
3433
3434
3435 nr += nr_irqs_gsi * 16;
3436#endif
3437 if (nr < nr_irqs)
3438 nr_irqs = nr;
3439
3440 return NR_IRQS_LEGACY;
3441}
3442
3443int io_apic_set_pci_routing(struct device *dev, int irq,
3444 struct io_apic_irq_attr *irq_attr)
3445{
3446 int node;
3447
3448 if (!IO_APIC_IRQ(irq)) {
3449 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3450 irq_attr->ioapic);
3451 return -EINVAL;
3452 }
3453
3454 node = dev ? dev_to_node(dev) : cpu_to_node(0);
3455
3456 return io_apic_setup_irq_pin_once(irq, node, irq_attr);
3457}
3458
3459#ifdef CONFIG_X86_32
3460static int __init io_apic_get_unique_id(int ioapic, int apic_id)
3461{
3462 union IO_APIC_reg_00 reg_00;
3463 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
3464 physid_mask_t tmp;
3465 unsigned long flags;
3466 int i = 0;
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477 if (physids_empty(apic_id_map))
3478 apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map);
3479
3480 raw_spin_lock_irqsave(&ioapic_lock, flags);
3481 reg_00.raw = io_apic_read(ioapic, 0);
3482 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3483
3484 if (apic_id >= get_physical_broadcast()) {
3485 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
3486 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3487 apic_id = reg_00.bits.ID;
3488 }
3489
3490
3491
3492
3493
3494 if (apic->check_apicid_used(&apic_id_map, apic_id)) {
3495
3496 for (i = 0; i < get_physical_broadcast(); i++) {
3497 if (!apic->check_apicid_used(&apic_id_map, i))
3498 break;
3499 }
3500
3501 if (i == get_physical_broadcast())
3502 panic("Max apic_id exceeded!\n");
3503
3504 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
3505 "trying %d\n", ioapic, apic_id, i);
3506
3507 apic_id = i;
3508 }
3509
3510 apic->apicid_to_cpu_present(apic_id, &tmp);
3511 physids_or(apic_id_map, apic_id_map, tmp);
3512
3513 if (reg_00.bits.ID != apic_id) {
3514 reg_00.bits.ID = apic_id;
3515
3516 raw_spin_lock_irqsave(&ioapic_lock, flags);
3517 io_apic_write(ioapic, 0, reg_00.raw);
3518 reg_00.raw = io_apic_read(ioapic, 0);
3519 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3520
3521
3522 if (reg_00.bits.ID != apic_id) {
3523 pr_err("IOAPIC[%d]: Unable to change apic_id!\n",
3524 ioapic);
3525 return -1;
3526 }
3527 }
3528
3529 apic_printk(APIC_VERBOSE, KERN_INFO
3530 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3531
3532 return apic_id;
3533}
3534
3535static u8 __init io_apic_unique_id(u8 id)
3536{
3537 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
3538 !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
3539 return io_apic_get_unique_id(nr_ioapics, id);
3540 else
3541 return id;
3542}
3543#else
3544static u8 __init io_apic_unique_id(u8 id)
3545{
3546 int i;
3547 DECLARE_BITMAP(used, 256);
3548
3549 bitmap_zero(used, 256);
3550 for (i = 0; i < nr_ioapics; i++) {
3551 __set_bit(mpc_ioapic_id(i), used);
3552 }
3553 if (!test_bit(id, used))
3554 return id;
3555 return find_first_zero_bit(used, 256);
3556}
3557#endif
3558
3559static int __init io_apic_get_version(int ioapic)
3560{
3561 union IO_APIC_reg_01 reg_01;
3562 unsigned long flags;
3563
3564 raw_spin_lock_irqsave(&ioapic_lock, flags);
3565 reg_01.raw = io_apic_read(ioapic, 1);
3566 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3567
3568 return reg_01.bits.version;
3569}
3570
3571int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity)
3572{
3573 int ioapic, pin, idx;
3574
3575 if (skip_ioapic_setup)
3576 return -1;
3577
3578 ioapic = mp_find_ioapic(gsi);
3579 if (ioapic < 0)
3580 return -1;
3581
3582 pin = mp_find_ioapic_pin(ioapic, gsi);
3583 if (pin < 0)
3584 return -1;
3585
3586 idx = find_irq_entry(ioapic, pin, mp_INT);
3587 if (idx < 0)
3588 return -1;
3589
3590 *trigger = irq_trigger(idx);
3591 *polarity = irq_polarity(idx);
3592 return 0;
3593}
3594
3595
3596
3597
3598
3599
3600#ifdef CONFIG_SMP
3601void __init setup_ioapic_dest(void)
3602{
3603 int pin, ioapic, irq, irq_entry;
3604 const struct cpumask *mask;
3605 struct irq_data *idata;
3606
3607 if (skip_ioapic_setup == 1)
3608 return;
3609
3610 for (ioapic = 0; ioapic < nr_ioapics; ioapic++)
3611 for (pin = 0; pin < ioapics[ioapic].nr_registers; pin++) {
3612 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
3613 if (irq_entry == -1)
3614 continue;
3615 irq = pin_2_irq(irq_entry, ioapic, pin);
3616
3617 if ((ioapic > 0) && (irq > 16))
3618 continue;
3619
3620 idata = irq_get_irq_data(irq);
3621
3622
3623
3624
3625 if (!irqd_can_balance(idata) || irqd_affinity_was_set(idata))
3626 mask = idata->affinity;
3627 else
3628 mask = apic->target_cpus();
3629
3630 x86_io_apic_ops.set_affinity(idata, mask, false);
3631 }
3632
3633}
3634#endif
3635
3636#define IOAPIC_RESOURCE_NAME_SIZE 11
3637
3638static struct resource *ioapic_resources;
3639
3640static struct resource * __init ioapic_setup_resources(int nr_ioapics)
3641{
3642 unsigned long n;
3643 struct resource *res;
3644 char *mem;
3645 int i;
3646
3647 if (nr_ioapics <= 0)
3648 return NULL;
3649
3650 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
3651 n *= nr_ioapics;
3652
3653 mem = alloc_bootmem(n);
3654 res = (void *)mem;
3655
3656 mem += sizeof(struct resource) * nr_ioapics;
3657
3658 for (i = 0; i < nr_ioapics; i++) {
3659 res[i].name = mem;
3660 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
3661 snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
3662 mem += IOAPIC_RESOURCE_NAME_SIZE;
3663 }
3664
3665 ioapic_resources = res;
3666
3667 return res;
3668}
3669
3670void __init native_io_apic_init_mappings(void)
3671{
3672 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3673 struct resource *ioapic_res;
3674 int i;
3675
3676 ioapic_res = ioapic_setup_resources(nr_ioapics);
3677 for (i = 0; i < nr_ioapics; i++) {
3678 if (smp_found_config) {
3679 ioapic_phys = mpc_ioapic_addr(i);
3680#ifdef CONFIG_X86_32
3681 if (!ioapic_phys) {
3682 printk(KERN_ERR
3683 "WARNING: bogus zero IO-APIC "
3684 "address found in MPTABLE, "
3685 "disabling IO/APIC support!\n");
3686 smp_found_config = 0;
3687 skip_ioapic_setup = 1;
3688 goto fake_ioapic_page;
3689 }
3690#endif
3691 } else {
3692#ifdef CONFIG_X86_32
3693fake_ioapic_page:
3694#endif
3695 ioapic_phys = (unsigned long)alloc_bootmem_pages(PAGE_SIZE);
3696 ioapic_phys = __pa(ioapic_phys);
3697 }
3698 set_fixmap_nocache(idx, ioapic_phys);
3699 apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n",
3700 __fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK),
3701 ioapic_phys);
3702 idx++;
3703
3704 ioapic_res->start = ioapic_phys;
3705 ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
3706 ioapic_res++;
3707 }
3708
3709 probe_nr_irqs_gsi();
3710}
3711
3712void __init ioapic_insert_resources(void)
3713{
3714 int i;
3715 struct resource *r = ioapic_resources;
3716
3717 if (!r) {
3718 if (nr_ioapics > 0)
3719 printk(KERN_ERR
3720 "IO APIC resources couldn't be allocated.\n");
3721 return;
3722 }
3723
3724 for (i = 0; i < nr_ioapics; i++) {
3725 insert_resource(&iomem_resource, r);
3726 r++;
3727 }
3728}
3729
3730int mp_find_ioapic(u32 gsi)
3731{
3732 int i = 0;
3733
3734 if (nr_ioapics == 0)
3735 return -1;
3736
3737
3738 for (i = 0; i < nr_ioapics; i++) {
3739 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
3740 if ((gsi >= gsi_cfg->gsi_base)
3741 && (gsi <= gsi_cfg->gsi_end))
3742 return i;
3743 }
3744
3745 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
3746 return -1;
3747}
3748
3749int mp_find_ioapic_pin(int ioapic, u32 gsi)
3750{
3751 struct mp_ioapic_gsi *gsi_cfg;
3752
3753 if (WARN_ON(ioapic == -1))
3754 return -1;
3755
3756 gsi_cfg = mp_ioapic_gsi_routing(ioapic);
3757 if (WARN_ON(gsi > gsi_cfg->gsi_end))
3758 return -1;
3759
3760 return gsi - gsi_cfg->gsi_base;
3761}
3762
3763static __init int bad_ioapic(unsigned long address)
3764{
3765 if (nr_ioapics >= MAX_IO_APICS) {
3766 pr_warn("WARNING: Max # of I/O APICs (%d) exceeded (found %d), skipping\n",
3767 MAX_IO_APICS, nr_ioapics);
3768 return 1;
3769 }
3770 if (!address) {
3771 pr_warn("WARNING: Bogus (zero) I/O APIC address found in table, skipping!\n");
3772 return 1;
3773 }
3774 return 0;
3775}
3776
3777static __init int bad_ioapic_register(int idx)
3778{
3779 union IO_APIC_reg_00 reg_00;
3780 union IO_APIC_reg_01 reg_01;
3781 union IO_APIC_reg_02 reg_02;
3782
3783 reg_00.raw = io_apic_read(idx, 0);
3784 reg_01.raw = io_apic_read(idx, 1);
3785 reg_02.raw = io_apic_read(idx, 2);
3786
3787 if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) {
3788 pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n",
3789 mpc_ioapic_addr(idx));
3790 return 1;
3791 }
3792
3793 return 0;
3794}
3795
3796void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
3797{
3798 int idx = 0;
3799 int entries;
3800 struct mp_ioapic_gsi *gsi_cfg;
3801
3802 if (bad_ioapic(address))
3803 return;
3804
3805 idx = nr_ioapics;
3806
3807 ioapics[idx].mp_config.type = MP_IOAPIC;
3808 ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
3809 ioapics[idx].mp_config.apicaddr = address;
3810
3811 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
3812
3813 if (bad_ioapic_register(idx)) {
3814 clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
3815 return;
3816 }
3817
3818 ioapics[idx].mp_config.apicid = io_apic_unique_id(id);
3819 ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
3820
3821
3822
3823
3824
3825 entries = io_apic_get_redir_entries(idx);
3826 gsi_cfg = mp_ioapic_gsi_routing(idx);
3827 gsi_cfg->gsi_base = gsi_base;
3828 gsi_cfg->gsi_end = gsi_base + entries - 1;
3829
3830
3831
3832
3833 ioapics[idx].nr_registers = entries;
3834
3835 if (gsi_cfg->gsi_end >= gsi_top)
3836 gsi_top = gsi_cfg->gsi_end + 1;
3837
3838 pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n",
3839 idx, mpc_ioapic_id(idx),
3840 mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
3841 gsi_cfg->gsi_base, gsi_cfg->gsi_end);
3842
3843 nr_ioapics++;
3844}
3845
3846
3847void __init pre_init_apic_IRQ0(void)
3848{
3849 struct io_apic_irq_attr attr = { 0, 0, 0, 0 };
3850
3851 printk(KERN_INFO "Early APIC setup for system timer0\n");
3852#ifndef CONFIG_SMP
3853 physid_set_mask_of_physid(boot_cpu_physical_apicid,
3854 &phys_cpu_present_map);
3855#endif
3856 setup_local_APIC();
3857
3858 io_apic_setup_irq_pin(0, 0, &attr);
3859 irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
3860 "edge");
3861}
3862