1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/mm.h>
24#include <linux/interrupt.h>
25#include <linux/init.h>
26#include <linux/delay.h>
27#include <linux/sched.h>
28#include <linux/pci.h>
29#include <linux/mc146818rtc.h>
30#include <linux/compiler.h>
31#include <linux/acpi.h>
32#include <linux/module.h>
33#include <linux/syscore_ops.h>
34#include <linux/msi.h>
35#include <linux/htirq.h>
36#include <linux/freezer.h>
37#include <linux/kthread.h>
38#include <linux/jiffies.h>
39#include <linux/slab.h>
40#ifdef CONFIG_ACPI
41#include <acpi/acpi_bus.h>
42#endif
43#include <linux/bootmem.h>
44#include <linux/dmar.h>
45#include <linux/hpet.h>
46
47#include <asm/idle.h>
48#include <asm/io.h>
49#include <asm/smp.h>
50#include <asm/cpu.h>
51#include <asm/desc.h>
52#include <asm/proto.h>
53#include <asm/acpi.h>
54#include <asm/dma.h>
55#include <asm/timer.h>
56#include <asm/i8259.h>
57#include <asm/msidef.h>
58#include <asm/hypertransport.h>
59#include <asm/setup.h>
60#include <asm/irq_remapping.h>
61#include <asm/hpet.h>
62#include <asm/hw_irq.h>
63
64#include <asm/apic.h>
65
66#define __apicdebuginit(type) static type __init
67
68#define for_each_irq_pin(entry, head) \
69 for (entry = head; entry; entry = entry->next)
70
71
72
73
74
75int sis_apic_bug = -1;
76
77static DEFINE_RAW_SPINLOCK(ioapic_lock);
78static DEFINE_RAW_SPINLOCK(vector_lock);
79
80static struct ioapic {
81
82
83
84 int nr_registers;
85
86
87
88 struct IO_APIC_route_entry *saved_registers;
89
90 struct mpc_ioapic mp_config;
91
92 struct mp_ioapic_gsi gsi_config;
93 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
94} ioapics[MAX_IO_APICS];
95
96#define mpc_ioapic_ver(ioapic_idx) ioapics[ioapic_idx].mp_config.apicver
97
98int mpc_ioapic_id(int ioapic_idx)
99{
100 return ioapics[ioapic_idx].mp_config.apicid;
101}
102
103unsigned int mpc_ioapic_addr(int ioapic_idx)
104{
105 return ioapics[ioapic_idx].mp_config.apicaddr;
106}
107
108struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx)
109{
110 return &ioapics[ioapic_idx].gsi_config;
111}
112
113int nr_ioapics;
114
115
116u32 gsi_top;
117
118
119struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
120
121
122int mp_irq_entries;
123
124
125static int nr_irqs_gsi = NR_IRQS_LEGACY;
126
127#ifdef CONFIG_EISA
128int mp_bus_id_to_type[MAX_MP_BUSSES];
129#endif
130
131DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
132
133int skip_ioapic_setup;
134
135
136
137
138void disable_ioapic_support(void)
139{
140#ifdef CONFIG_PCI
141 noioapicquirk = 1;
142 noioapicreroute = -1;
143#endif
144 skip_ioapic_setup = 1;
145}
146
147static int __init parse_noapic(char *str)
148{
149
150 disable_ioapic_support();
151 return 0;
152}
153early_param("noapic", parse_noapic);
154
155static int io_apic_setup_irq_pin(unsigned int irq, int node,
156 struct io_apic_irq_attr *attr);
157
158
159void mp_save_irq(struct mpc_intsrc *m)
160{
161 int i;
162
163 apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
164 " IRQ %02x, APIC ID %x, APIC INT %02x\n",
165 m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus,
166 m->srcbusirq, m->dstapic, m->dstirq);
167
168 for (i = 0; i < mp_irq_entries; i++) {
169 if (!memcmp(&mp_irqs[i], m, sizeof(*m)))
170 return;
171 }
172
173 memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m));
174 if (++mp_irq_entries == MAX_IRQ_SOURCES)
175 panic("Max # of irq sources exceeded!!\n");
176}
177
178struct irq_pin_list {
179 int apic, pin;
180 struct irq_pin_list *next;
181};
182
183static struct irq_pin_list *alloc_irq_pin_list(int node)
184{
185 return kzalloc_node(sizeof(struct irq_pin_list), GFP_KERNEL, node);
186}
187
188
189
190static struct irq_cfg irq_cfgx[NR_IRQS_LEGACY];
191
192int __init arch_early_irq_init(void)
193{
194 struct irq_cfg *cfg;
195 int count, node, i;
196
197 if (!legacy_pic->nr_legacy_irqs)
198 io_apic_irqs = ~0UL;
199
200 for (i = 0; i < nr_ioapics; i++) {
201 ioapics[i].saved_registers =
202 kzalloc(sizeof(struct IO_APIC_route_entry) *
203 ioapics[i].nr_registers, GFP_KERNEL);
204 if (!ioapics[i].saved_registers)
205 pr_err("IOAPIC %d: suspend/resume impossible!\n", i);
206 }
207
208 cfg = irq_cfgx;
209 count = ARRAY_SIZE(irq_cfgx);
210 node = cpu_to_node(0);
211
212
213 irq_reserve_irqs(0, legacy_pic->nr_legacy_irqs);
214
215 for (i = 0; i < count; i++) {
216 irq_set_chip_data(i, &cfg[i]);
217 zalloc_cpumask_var_node(&cfg[i].domain, GFP_KERNEL, node);
218 zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_KERNEL, node);
219
220
221
222
223 if (i < legacy_pic->nr_legacy_irqs) {
224 cfg[i].vector = IRQ0_VECTOR + i;
225 cpumask_setall(cfg[i].domain);
226 }
227 }
228
229 return 0;
230}
231
232static struct irq_cfg *irq_cfg(unsigned int irq)
233{
234 return irq_get_chip_data(irq);
235}
236
237static struct irq_cfg *alloc_irq_cfg(unsigned int irq, int node)
238{
239 struct irq_cfg *cfg;
240
241 cfg = kzalloc_node(sizeof(*cfg), GFP_KERNEL, node);
242 if (!cfg)
243 return NULL;
244 if (!zalloc_cpumask_var_node(&cfg->domain, GFP_KERNEL, node))
245 goto out_cfg;
246 if (!zalloc_cpumask_var_node(&cfg->old_domain, GFP_KERNEL, node))
247 goto out_domain;
248 return cfg;
249out_domain:
250 free_cpumask_var(cfg->domain);
251out_cfg:
252 kfree(cfg);
253 return NULL;
254}
255
256static void free_irq_cfg(unsigned int at, struct irq_cfg *cfg)
257{
258 if (!cfg)
259 return;
260 irq_set_chip_data(at, NULL);
261 free_cpumask_var(cfg->domain);
262 free_cpumask_var(cfg->old_domain);
263 kfree(cfg);
264}
265
266static struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node)
267{
268 int res = irq_alloc_desc_at(at, node);
269 struct irq_cfg *cfg;
270
271 if (res < 0) {
272 if (res != -EEXIST)
273 return NULL;
274 cfg = irq_get_chip_data(at);
275 if (cfg)
276 return cfg;
277 }
278
279 cfg = alloc_irq_cfg(at, node);
280 if (cfg)
281 irq_set_chip_data(at, cfg);
282 else
283 irq_free_desc(at);
284 return cfg;
285}
286
287static int alloc_irqs_from(unsigned int from, unsigned int count, int node)
288{
289 return irq_alloc_descs_from(from, count, node);
290}
291
292static void free_irq_at(unsigned int at, struct irq_cfg *cfg)
293{
294 free_irq_cfg(at, cfg);
295 irq_free_desc(at);
296}
297
298
299struct io_apic {
300 unsigned int index;
301 unsigned int unused[3];
302 unsigned int data;
303 unsigned int unused2[11];
304 unsigned int eoi;
305};
306
307static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
308{
309 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
310 + (mpc_ioapic_addr(idx) & ~PAGE_MASK);
311}
312
313void io_apic_eoi(unsigned int apic, unsigned int vector)
314{
315 struct io_apic __iomem *io_apic = io_apic_base(apic);
316 writel(vector, &io_apic->eoi);
317}
318
319unsigned int native_io_apic_read(unsigned int apic, unsigned int reg)
320{
321 struct io_apic __iomem *io_apic = io_apic_base(apic);
322 writel(reg, &io_apic->index);
323 return readl(&io_apic->data);
324}
325
326void native_io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
327{
328 struct io_apic __iomem *io_apic = io_apic_base(apic);
329
330 writel(reg, &io_apic->index);
331 writel(value, &io_apic->data);
332}
333
334
335
336
337
338
339
340void native_io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
341{
342 struct io_apic __iomem *io_apic = io_apic_base(apic);
343
344 if (sis_apic_bug)
345 writel(reg, &io_apic->index);
346 writel(value, &io_apic->data);
347}
348
349union entry_union {
350 struct { u32 w1, w2; };
351 struct IO_APIC_route_entry entry;
352};
353
354static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin)
355{
356 union entry_union eu;
357
358 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
359 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
360
361 return eu.entry;
362}
363
364static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
365{
366 union entry_union eu;
367 unsigned long flags;
368
369 raw_spin_lock_irqsave(&ioapic_lock, flags);
370 eu.entry = __ioapic_read_entry(apic, pin);
371 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
372
373 return eu.entry;
374}
375
376
377
378
379
380
381
382static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
383{
384 union entry_union eu = {{0, 0}};
385
386 eu.entry = e;
387 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
388 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
389}
390
391static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
392{
393 unsigned long flags;
394
395 raw_spin_lock_irqsave(&ioapic_lock, flags);
396 __ioapic_write_entry(apic, pin, e);
397 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
398}
399
400
401
402
403
404
405static void ioapic_mask_entry(int apic, int pin)
406{
407 unsigned long flags;
408 union entry_union eu = { .entry.mask = 1 };
409
410 raw_spin_lock_irqsave(&ioapic_lock, flags);
411 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
412 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
413 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
414}
415
416
417
418
419
420
421static int __add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
422{
423 struct irq_pin_list **last, *entry;
424
425
426 last = &cfg->irq_2_pin;
427 for_each_irq_pin(entry, cfg->irq_2_pin) {
428 if (entry->apic == apic && entry->pin == pin)
429 return 0;
430 last = &entry->next;
431 }
432
433 entry = alloc_irq_pin_list(node);
434 if (!entry) {
435 pr_err("can not alloc irq_pin_list (%d,%d,%d)\n",
436 node, apic, pin);
437 return -ENOMEM;
438 }
439 entry->apic = apic;
440 entry->pin = pin;
441
442 *last = entry;
443 return 0;
444}
445
446static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
447{
448 if (__add_pin_to_irq_node(cfg, node, apic, pin))
449 panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
450}
451
452
453
454
455static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
456 int oldapic, int oldpin,
457 int newapic, int newpin)
458{
459 struct irq_pin_list *entry;
460
461 for_each_irq_pin(entry, cfg->irq_2_pin) {
462 if (entry->apic == oldapic && entry->pin == oldpin) {
463 entry->apic = newapic;
464 entry->pin = newpin;
465
466 return;
467 }
468 }
469
470
471 add_pin_to_irq_node(cfg, node, newapic, newpin);
472}
473
474static void __io_apic_modify_irq(struct irq_pin_list *entry,
475 int mask_and, int mask_or,
476 void (*final)(struct irq_pin_list *entry))
477{
478 unsigned int reg, pin;
479
480 pin = entry->pin;
481 reg = io_apic_read(entry->apic, 0x10 + pin * 2);
482 reg &= mask_and;
483 reg |= mask_or;
484 io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
485 if (final)
486 final(entry);
487}
488
489static void io_apic_modify_irq(struct irq_cfg *cfg,
490 int mask_and, int mask_or,
491 void (*final)(struct irq_pin_list *entry))
492{
493 struct irq_pin_list *entry;
494
495 for_each_irq_pin(entry, cfg->irq_2_pin)
496 __io_apic_modify_irq(entry, mask_and, mask_or, final);
497}
498
499static void io_apic_sync(struct irq_pin_list *entry)
500{
501
502
503
504
505 struct io_apic __iomem *io_apic;
506
507 io_apic = io_apic_base(entry->apic);
508 readl(&io_apic->data);
509}
510
511static void mask_ioapic(struct irq_cfg *cfg)
512{
513 unsigned long flags;
514
515 raw_spin_lock_irqsave(&ioapic_lock, flags);
516 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
517 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
518}
519
520static void mask_ioapic_irq(struct irq_data *data)
521{
522 mask_ioapic(data->chip_data);
523}
524
525static void __unmask_ioapic(struct irq_cfg *cfg)
526{
527 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
528}
529
530static void unmask_ioapic(struct irq_cfg *cfg)
531{
532 unsigned long flags;
533
534 raw_spin_lock_irqsave(&ioapic_lock, flags);
535 __unmask_ioapic(cfg);
536 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
537}
538
539static void unmask_ioapic_irq(struct irq_data *data)
540{
541 unmask_ioapic(data->chip_data);
542}
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560void native_eoi_ioapic_pin(int apic, int pin, int vector)
561{
562 if (mpc_ioapic_ver(apic) >= 0x20) {
563 io_apic_eoi(apic, vector);
564 } else {
565 struct IO_APIC_route_entry entry, entry1;
566
567 entry = entry1 = __ioapic_read_entry(apic, pin);
568
569
570
571
572 entry1.mask = 1;
573 entry1.trigger = IOAPIC_EDGE;
574
575 __ioapic_write_entry(apic, pin, entry1);
576
577
578
579
580 __ioapic_write_entry(apic, pin, entry);
581 }
582}
583
584void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
585{
586 struct irq_pin_list *entry;
587 unsigned long flags;
588
589 raw_spin_lock_irqsave(&ioapic_lock, flags);
590 for_each_irq_pin(entry, cfg->irq_2_pin)
591 x86_io_apic_ops.eoi_ioapic_pin(entry->apic, entry->pin,
592 cfg->vector);
593 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
594}
595
596static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
597{
598 struct IO_APIC_route_entry entry;
599
600
601 entry = ioapic_read_entry(apic, pin);
602 if (entry.delivery_mode == dest_SMI)
603 return;
604
605
606
607
608
609 if (!entry.mask) {
610 entry.mask = 1;
611 ioapic_write_entry(apic, pin, entry);
612 entry = ioapic_read_entry(apic, pin);
613 }
614
615 if (entry.irr) {
616 unsigned long flags;
617
618
619
620
621
622
623 if (!entry.trigger) {
624 entry.trigger = IOAPIC_LEVEL;
625 ioapic_write_entry(apic, pin, entry);
626 }
627
628 raw_spin_lock_irqsave(&ioapic_lock, flags);
629 x86_io_apic_ops.eoi_ioapic_pin(apic, pin, entry.vector);
630 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
631 }
632
633
634
635
636
637 ioapic_mask_entry(apic, pin);
638 entry = ioapic_read_entry(apic, pin);
639 if (entry.irr)
640 pr_err("Unable to reset IRR for apic: %d, pin :%d\n",
641 mpc_ioapic_id(apic), pin);
642}
643
644static void clear_IO_APIC (void)
645{
646 int apic, pin;
647
648 for (apic = 0; apic < nr_ioapics; apic++)
649 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
650 clear_IO_APIC_pin(apic, pin);
651}
652
653#ifdef CONFIG_X86_32
654
655
656
657
658
659#define MAX_PIRQS 8
660static int pirq_entries[MAX_PIRQS] = {
661 [0 ... MAX_PIRQS - 1] = -1
662};
663
664static int __init ioapic_pirq_setup(char *str)
665{
666 int i, max;
667 int ints[MAX_PIRQS+1];
668
669 get_options(str, ARRAY_SIZE(ints), ints);
670
671 apic_printk(APIC_VERBOSE, KERN_INFO
672 "PIRQ redirection, working around broken MP-BIOS.\n");
673 max = MAX_PIRQS;
674 if (ints[0] < MAX_PIRQS)
675 max = ints[0];
676
677 for (i = 0; i < max; i++) {
678 apic_printk(APIC_VERBOSE, KERN_DEBUG
679 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
680
681
682
683 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
684 }
685 return 1;
686}
687
688__setup("pirq=", ioapic_pirq_setup);
689#endif
690
691
692
693
694int save_ioapic_entries(void)
695{
696 int apic, pin;
697 int err = 0;
698
699 for (apic = 0; apic < nr_ioapics; apic++) {
700 if (!ioapics[apic].saved_registers) {
701 err = -ENOMEM;
702 continue;
703 }
704
705 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
706 ioapics[apic].saved_registers[pin] =
707 ioapic_read_entry(apic, pin);
708 }
709
710 return err;
711}
712
713
714
715
716void mask_ioapic_entries(void)
717{
718 int apic, pin;
719
720 for (apic = 0; apic < nr_ioapics; apic++) {
721 if (!ioapics[apic].saved_registers)
722 continue;
723
724 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
725 struct IO_APIC_route_entry entry;
726
727 entry = ioapics[apic].saved_registers[pin];
728 if (!entry.mask) {
729 entry.mask = 1;
730 ioapic_write_entry(apic, pin, entry);
731 }
732 }
733 }
734}
735
736
737
738
739int restore_ioapic_entries(void)
740{
741 int apic, pin;
742
743 for (apic = 0; apic < nr_ioapics; apic++) {
744 if (!ioapics[apic].saved_registers)
745 continue;
746
747 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
748 ioapic_write_entry(apic, pin,
749 ioapics[apic].saved_registers[pin]);
750 }
751 return 0;
752}
753
754
755
756
757static int find_irq_entry(int ioapic_idx, int pin, int type)
758{
759 int i;
760
761 for (i = 0; i < mp_irq_entries; i++)
762 if (mp_irqs[i].irqtype == type &&
763 (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) ||
764 mp_irqs[i].dstapic == MP_APIC_ALL) &&
765 mp_irqs[i].dstirq == pin)
766 return i;
767
768 return -1;
769}
770
771
772
773
774static int __init find_isa_irq_pin(int irq, int type)
775{
776 int i;
777
778 for (i = 0; i < mp_irq_entries; i++) {
779 int lbus = mp_irqs[i].srcbus;
780
781 if (test_bit(lbus, mp_bus_not_pci) &&
782 (mp_irqs[i].irqtype == type) &&
783 (mp_irqs[i].srcbusirq == irq))
784
785 return mp_irqs[i].dstirq;
786 }
787 return -1;
788}
789
790static int __init find_isa_irq_apic(int irq, int type)
791{
792 int i;
793
794 for (i = 0; i < mp_irq_entries; i++) {
795 int lbus = mp_irqs[i].srcbus;
796
797 if (test_bit(lbus, mp_bus_not_pci) &&
798 (mp_irqs[i].irqtype == type) &&
799 (mp_irqs[i].srcbusirq == irq))
800 break;
801 }
802
803 if (i < mp_irq_entries) {
804 int ioapic_idx;
805
806 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
807 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic)
808 return ioapic_idx;
809 }
810
811 return -1;
812}
813
814#ifdef CONFIG_EISA
815
816
817
818static int EISA_ELCR(unsigned int irq)
819{
820 if (irq < legacy_pic->nr_legacy_irqs) {
821 unsigned int port = 0x4d0 + (irq >> 3);
822 return (inb(port) >> (irq & 7)) & 1;
823 }
824 apic_printk(APIC_VERBOSE, KERN_INFO
825 "Broken MPtable reports ISA irq %d\n", irq);
826 return 0;
827}
828
829#endif
830
831
832
833
834#define default_ISA_trigger(idx) (0)
835#define default_ISA_polarity(idx) (0)
836
837
838
839
840
841
842#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].srcbusirq))
843#define default_EISA_polarity(idx) default_ISA_polarity(idx)
844
845
846
847
848#define default_PCI_trigger(idx) (1)
849#define default_PCI_polarity(idx) (1)
850
851static int irq_polarity(int idx)
852{
853 int bus = mp_irqs[idx].srcbus;
854 int polarity;
855
856
857
858
859 switch (mp_irqs[idx].irqflag & 3)
860 {
861 case 0:
862 if (test_bit(bus, mp_bus_not_pci))
863 polarity = default_ISA_polarity(idx);
864 else
865 polarity = default_PCI_polarity(idx);
866 break;
867 case 1:
868 {
869 polarity = 0;
870 break;
871 }
872 case 2:
873 {
874 pr_warn("broken BIOS!!\n");
875 polarity = 1;
876 break;
877 }
878 case 3:
879 {
880 polarity = 1;
881 break;
882 }
883 default:
884 {
885 pr_warn("broken BIOS!!\n");
886 polarity = 1;
887 break;
888 }
889 }
890 return polarity;
891}
892
893static int irq_trigger(int idx)
894{
895 int bus = mp_irqs[idx].srcbus;
896 int trigger;
897
898
899
900
901 switch ((mp_irqs[idx].irqflag>>2) & 3)
902 {
903 case 0:
904 if (test_bit(bus, mp_bus_not_pci))
905 trigger = default_ISA_trigger(idx);
906 else
907 trigger = default_PCI_trigger(idx);
908#ifdef CONFIG_EISA
909 switch (mp_bus_id_to_type[bus]) {
910 case MP_BUS_ISA:
911 {
912
913 break;
914 }
915 case MP_BUS_EISA:
916 {
917 trigger = default_EISA_trigger(idx);
918 break;
919 }
920 case MP_BUS_PCI:
921 {
922
923 break;
924 }
925 default:
926 {
927 pr_warn("broken BIOS!!\n");
928 trigger = 1;
929 break;
930 }
931 }
932#endif
933 break;
934 case 1:
935 {
936 trigger = 0;
937 break;
938 }
939 case 2:
940 {
941 pr_warn("broken BIOS!!\n");
942 trigger = 1;
943 break;
944 }
945 case 3:
946 {
947 trigger = 1;
948 break;
949 }
950 default:
951 {
952 pr_warn("broken BIOS!!\n");
953 trigger = 0;
954 break;
955 }
956 }
957 return trigger;
958}
959
960static int pin_2_irq(int idx, int apic, int pin)
961{
962 int irq;
963 int bus = mp_irqs[idx].srcbus;
964 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(apic);
965
966
967
968
969 if (mp_irqs[idx].dstirq != pin)
970 pr_err("broken BIOS or MPTABLE parser, ayiee!!\n");
971
972 if (test_bit(bus, mp_bus_not_pci)) {
973 irq = mp_irqs[idx].srcbusirq;
974 } else {
975 u32 gsi = gsi_cfg->gsi_base + pin;
976
977 if (gsi >= NR_IRQS_LEGACY)
978 irq = gsi;
979 else
980 irq = gsi_top + gsi;
981 }
982
983#ifdef CONFIG_X86_32
984
985
986
987 if ((pin >= 16) && (pin <= 23)) {
988 if (pirq_entries[pin-16] != -1) {
989 if (!pirq_entries[pin-16]) {
990 apic_printk(APIC_VERBOSE, KERN_DEBUG
991 "disabling PIRQ%d\n", pin-16);
992 } else {
993 irq = pirq_entries[pin-16];
994 apic_printk(APIC_VERBOSE, KERN_DEBUG
995 "using PIRQ%d -> IRQ %d\n",
996 pin-16, irq);
997 }
998 }
999 }
1000#endif
1001
1002 return irq;
1003}
1004
1005
1006
1007
1008
1009int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1010 struct io_apic_irq_attr *irq_attr)
1011{
1012 int ioapic_idx, i, best_guess = -1;
1013
1014 apic_printk(APIC_DEBUG,
1015 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1016 bus, slot, pin);
1017 if (test_bit(bus, mp_bus_not_pci)) {
1018 apic_printk(APIC_VERBOSE,
1019 "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1020 return -1;
1021 }
1022 for (i = 0; i < mp_irq_entries; i++) {
1023 int lbus = mp_irqs[i].srcbus;
1024
1025 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1026 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic ||
1027 mp_irqs[i].dstapic == MP_APIC_ALL)
1028 break;
1029
1030 if (!test_bit(lbus, mp_bus_not_pci) &&
1031 !mp_irqs[i].irqtype &&
1032 (bus == lbus) &&
1033 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1034 int irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq);
1035
1036 if (!(ioapic_idx || IO_APIC_IRQ(irq)))
1037 continue;
1038
1039 if (pin == (mp_irqs[i].srcbusirq & 3)) {
1040 set_io_apic_irq_attr(irq_attr, ioapic_idx,
1041 mp_irqs[i].dstirq,
1042 irq_trigger(i),
1043 irq_polarity(i));
1044 return irq;
1045 }
1046
1047
1048
1049
1050 if (best_guess < 0) {
1051 set_io_apic_irq_attr(irq_attr, ioapic_idx,
1052 mp_irqs[i].dstirq,
1053 irq_trigger(i),
1054 irq_polarity(i));
1055 best_guess = irq;
1056 }
1057 }
1058 }
1059 return best_guess;
1060}
1061EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1062
1063void lock_vector_lock(void)
1064{
1065
1066
1067
1068 raw_spin_lock(&vector_lock);
1069}
1070
1071void unlock_vector_lock(void)
1072{
1073 raw_spin_unlock(&vector_lock);
1074}
1075
1076static int
1077__assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1078{
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090 static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START;
1091 static int current_offset = VECTOR_OFFSET_START % 16;
1092 int cpu, err;
1093 cpumask_var_t tmp_mask;
1094
1095 if (cfg->move_in_progress)
1096 return -EBUSY;
1097
1098 if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
1099 return -ENOMEM;
1100
1101
1102 err = -ENOSPC;
1103 cpumask_clear(cfg->old_domain);
1104 cpu = cpumask_first_and(mask, cpu_online_mask);
1105 while (cpu < nr_cpu_ids) {
1106 int new_cpu, vector, offset;
1107
1108 apic->vector_allocation_domain(cpu, tmp_mask, mask);
1109
1110 if (cpumask_subset(tmp_mask, cfg->domain)) {
1111 err = 0;
1112 if (cpumask_equal(tmp_mask, cfg->domain))
1113 break;
1114
1115
1116
1117
1118
1119 cpumask_andnot(cfg->old_domain, cfg->domain, tmp_mask);
1120 cfg->move_in_progress =
1121 cpumask_intersects(cfg->old_domain, cpu_online_mask);
1122 cpumask_and(cfg->domain, cfg->domain, tmp_mask);
1123 break;
1124 }
1125
1126 vector = current_vector;
1127 offset = current_offset;
1128next:
1129 vector += 16;
1130 if (vector >= first_system_vector) {
1131 offset = (offset + 1) % 16;
1132 vector = FIRST_EXTERNAL_VECTOR + offset;
1133 }
1134
1135 if (unlikely(current_vector == vector)) {
1136 cpumask_or(cfg->old_domain, cfg->old_domain, tmp_mask);
1137 cpumask_andnot(tmp_mask, mask, cfg->old_domain);
1138 cpu = cpumask_first_and(tmp_mask, cpu_online_mask);
1139 continue;
1140 }
1141
1142 if (test_bit(vector, used_vectors))
1143 goto next;
1144
1145 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1146 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1147 goto next;
1148
1149 current_vector = vector;
1150 current_offset = offset;
1151 if (cfg->vector) {
1152 cpumask_copy(cfg->old_domain, cfg->domain);
1153 cfg->move_in_progress =
1154 cpumask_intersects(cfg->old_domain, cpu_online_mask);
1155 }
1156 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1157 per_cpu(vector_irq, new_cpu)[vector] = irq;
1158 cfg->vector = vector;
1159 cpumask_copy(cfg->domain, tmp_mask);
1160 err = 0;
1161 break;
1162 }
1163 free_cpumask_var(tmp_mask);
1164 return err;
1165}
1166
1167int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1168{
1169 int err;
1170 unsigned long flags;
1171
1172 raw_spin_lock_irqsave(&vector_lock, flags);
1173 err = __assign_irq_vector(irq, cfg, mask);
1174 raw_spin_unlock_irqrestore(&vector_lock, flags);
1175 return err;
1176}
1177
1178static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1179{
1180 int cpu, vector;
1181
1182 BUG_ON(!cfg->vector);
1183
1184 vector = cfg->vector;
1185 for_each_cpu_and(cpu, cfg->domain, cpu_online_mask)
1186 per_cpu(vector_irq, cpu)[vector] = -1;
1187
1188 cfg->vector = 0;
1189 cpumask_clear(cfg->domain);
1190
1191 if (likely(!cfg->move_in_progress))
1192 return;
1193 for_each_cpu_and(cpu, cfg->old_domain, cpu_online_mask) {
1194 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
1195 vector++) {
1196 if (per_cpu(vector_irq, cpu)[vector] != irq)
1197 continue;
1198 per_cpu(vector_irq, cpu)[vector] = -1;
1199 break;
1200 }
1201 }
1202 cfg->move_in_progress = 0;
1203}
1204
1205void __setup_vector_irq(int cpu)
1206{
1207
1208 int irq, vector;
1209 struct irq_cfg *cfg;
1210
1211
1212
1213
1214
1215
1216 raw_spin_lock(&vector_lock);
1217
1218 for_each_active_irq(irq) {
1219 cfg = irq_get_chip_data(irq);
1220 if (!cfg)
1221 continue;
1222
1223 if (!cpumask_test_cpu(cpu, cfg->domain))
1224 continue;
1225 vector = cfg->vector;
1226 per_cpu(vector_irq, cpu)[vector] = irq;
1227 }
1228
1229 for (vector = 0; vector < NR_VECTORS; ++vector) {
1230 irq = per_cpu(vector_irq, cpu)[vector];
1231 if (irq < 0)
1232 continue;
1233
1234 cfg = irq_cfg(irq);
1235 if (!cpumask_test_cpu(cpu, cfg->domain))
1236 per_cpu(vector_irq, cpu)[vector] = -1;
1237 }
1238 raw_spin_unlock(&vector_lock);
1239}
1240
1241static struct irq_chip ioapic_chip;
1242
1243#ifdef CONFIG_X86_32
1244static inline int IO_APIC_irq_trigger(int irq)
1245{
1246 int apic, idx, pin;
1247
1248 for (apic = 0; apic < nr_ioapics; apic++) {
1249 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1250 idx = find_irq_entry(apic, pin, mp_INT);
1251 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1252 return irq_trigger(idx);
1253 }
1254 }
1255
1256
1257
1258 return 0;
1259}
1260#else
1261static inline int IO_APIC_irq_trigger(int irq)
1262{
1263 return 1;
1264}
1265#endif
1266
1267static void ioapic_register_intr(unsigned int irq, struct irq_cfg *cfg,
1268 unsigned long trigger)
1269{
1270 struct irq_chip *chip = &ioapic_chip;
1271 irq_flow_handler_t hdl;
1272 bool fasteoi;
1273
1274 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1275 trigger == IOAPIC_LEVEL) {
1276 irq_set_status_flags(irq, IRQ_LEVEL);
1277 fasteoi = true;
1278 } else {
1279 irq_clear_status_flags(irq, IRQ_LEVEL);
1280 fasteoi = false;
1281 }
1282
1283 if (setup_remapped_irq(irq, cfg, chip))
1284 fasteoi = trigger != 0;
1285
1286 hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
1287 irq_set_chip_and_handler_name(irq, chip, hdl,
1288 fasteoi ? "fasteoi" : "edge");
1289}
1290
1291int native_setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
1292 unsigned int destination, int vector,
1293 struct io_apic_irq_attr *attr)
1294{
1295 memset(entry, 0, sizeof(*entry));
1296
1297 entry->delivery_mode = apic->irq_delivery_mode;
1298 entry->dest_mode = apic->irq_dest_mode;
1299 entry->dest = destination;
1300 entry->vector = vector;
1301 entry->mask = 0;
1302 entry->trigger = attr->trigger;
1303 entry->polarity = attr->polarity;
1304
1305
1306
1307
1308
1309 if (attr->trigger)
1310 entry->mask = 1;
1311
1312 return 0;
1313}
1314
1315static void setup_ioapic_irq(unsigned int irq, struct irq_cfg *cfg,
1316 struct io_apic_irq_attr *attr)
1317{
1318 struct IO_APIC_route_entry entry;
1319 unsigned int dest;
1320
1321 if (!IO_APIC_IRQ(irq))
1322 return;
1323
1324 if (assign_irq_vector(irq, cfg, apic->target_cpus()))
1325 return;
1326
1327 if (apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus(),
1328 &dest)) {
1329 pr_warn("Failed to obtain apicid for ioapic %d, pin %d\n",
1330 mpc_ioapic_id(attr->ioapic), attr->ioapic_pin);
1331 __clear_irq_vector(irq, cfg);
1332
1333 return;
1334 }
1335
1336 apic_printk(APIC_VERBOSE,KERN_DEBUG
1337 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1338 "IRQ %d Mode:%i Active:%i Dest:%d)\n",
1339 attr->ioapic, mpc_ioapic_id(attr->ioapic), attr->ioapic_pin,
1340 cfg->vector, irq, attr->trigger, attr->polarity, dest);
1341
1342 if (x86_io_apic_ops.setup_entry(irq, &entry, dest, cfg->vector, attr)) {
1343 pr_warn("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1344 mpc_ioapic_id(attr->ioapic), attr->ioapic_pin);
1345 __clear_irq_vector(irq, cfg);
1346
1347 return;
1348 }
1349
1350 ioapic_register_intr(irq, cfg, attr->trigger);
1351 if (irq < legacy_pic->nr_legacy_irqs)
1352 legacy_pic->mask(irq);
1353
1354 ioapic_write_entry(attr->ioapic, attr->ioapic_pin, entry);
1355}
1356
1357static bool __init io_apic_pin_not_connected(int idx, int ioapic_idx, int pin)
1358{
1359 if (idx != -1)
1360 return false;
1361
1362 apic_printk(APIC_VERBOSE, KERN_DEBUG " apic %d pin %d not connected\n",
1363 mpc_ioapic_id(ioapic_idx), pin);
1364 return true;
1365}
1366
1367static void __init __io_apic_setup_irqs(unsigned int ioapic_idx)
1368{
1369 int idx, node = cpu_to_node(0);
1370 struct io_apic_irq_attr attr;
1371 unsigned int pin, irq;
1372
1373 for (pin = 0; pin < ioapics[ioapic_idx].nr_registers; pin++) {
1374 idx = find_irq_entry(ioapic_idx, pin, mp_INT);
1375 if (io_apic_pin_not_connected(idx, ioapic_idx, pin))
1376 continue;
1377
1378 irq = pin_2_irq(idx, ioapic_idx, pin);
1379
1380 if ((ioapic_idx > 0) && (irq > 16))
1381 continue;
1382
1383
1384
1385
1386
1387 if (apic->multi_timer_check &&
1388 apic->multi_timer_check(ioapic_idx, irq))
1389 continue;
1390
1391 set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1392 irq_polarity(idx));
1393
1394 io_apic_setup_irq_pin(irq, node, &attr);
1395 }
1396}
1397
1398static void __init setup_IO_APIC_irqs(void)
1399{
1400 unsigned int ioapic_idx;
1401
1402 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1403
1404 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1405 __io_apic_setup_irqs(ioapic_idx);
1406}
1407
1408
1409
1410
1411
1412
1413void setup_IO_APIC_irq_extra(u32 gsi)
1414{
1415 int ioapic_idx = 0, pin, idx, irq, node = cpu_to_node(0);
1416 struct io_apic_irq_attr attr;
1417
1418
1419
1420
1421 ioapic_idx = mp_find_ioapic(gsi);
1422 if (ioapic_idx < 0)
1423 return;
1424
1425 pin = mp_find_ioapic_pin(ioapic_idx, gsi);
1426 idx = find_irq_entry(ioapic_idx, pin, mp_INT);
1427 if (idx == -1)
1428 return;
1429
1430 irq = pin_2_irq(idx, ioapic_idx, pin);
1431
1432
1433 if (ioapic_idx == 0 || irq < NR_IRQS_LEGACY)
1434 return;
1435
1436 set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1437 irq_polarity(idx));
1438
1439 io_apic_setup_irq_pin_once(irq, node, &attr);
1440}
1441
1442
1443
1444
1445static void __init setup_timer_IRQ0_pin(unsigned int ioapic_idx,
1446 unsigned int pin, int vector)
1447{
1448 struct IO_APIC_route_entry entry;
1449 unsigned int dest;
1450
1451 memset(&entry, 0, sizeof(entry));
1452
1453
1454
1455
1456
1457 if (unlikely(apic->cpu_mask_to_apicid_and(apic->target_cpus(),
1458 apic->target_cpus(), &dest)))
1459 dest = BAD_APICID;
1460
1461 entry.dest_mode = apic->irq_dest_mode;
1462 entry.mask = 0;
1463 entry.dest = dest;
1464 entry.delivery_mode = apic->irq_delivery_mode;
1465 entry.polarity = 0;
1466 entry.trigger = 0;
1467 entry.vector = vector;
1468
1469
1470
1471
1472
1473 irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
1474 "edge");
1475
1476
1477
1478
1479 ioapic_write_entry(ioapic_idx, pin, entry);
1480}
1481
1482void native_io_apic_print_entries(unsigned int apic, unsigned int nr_entries)
1483{
1484 int i;
1485
1486 pr_debug(" NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect:\n");
1487
1488 for (i = 0; i <= nr_entries; i++) {
1489 struct IO_APIC_route_entry entry;
1490
1491 entry = ioapic_read_entry(apic, i);
1492
1493 pr_debug(" %02x %02X ", i, entry.dest);
1494 pr_cont("%1d %1d %1d %1d %1d "
1495 "%1d %1d %02X\n",
1496 entry.mask,
1497 entry.trigger,
1498 entry.irr,
1499 entry.polarity,
1500 entry.delivery_status,
1501 entry.dest_mode,
1502 entry.delivery_mode,
1503 entry.vector);
1504 }
1505}
1506
1507void intel_ir_io_apic_print_entries(unsigned int apic,
1508 unsigned int nr_entries)
1509{
1510 int i;
1511
1512 pr_debug(" NR Indx Fmt Mask Trig IRR Pol Stat Indx2 Zero Vect:\n");
1513
1514 for (i = 0; i <= nr_entries; i++) {
1515 struct IR_IO_APIC_route_entry *ir_entry;
1516 struct IO_APIC_route_entry entry;
1517
1518 entry = ioapic_read_entry(apic, i);
1519
1520 ir_entry = (struct IR_IO_APIC_route_entry *)&entry;
1521
1522 pr_debug(" %02x %04X ", i, ir_entry->index);
1523 pr_cont("%1d %1d %1d %1d %1d "
1524 "%1d %1d %X %02X\n",
1525 ir_entry->format,
1526 ir_entry->mask,
1527 ir_entry->trigger,
1528 ir_entry->irr,
1529 ir_entry->polarity,
1530 ir_entry->delivery_status,
1531 ir_entry->index2,
1532 ir_entry->zero,
1533 ir_entry->vector);
1534 }
1535}
1536
1537void ioapic_zap_locks(void)
1538{
1539 raw_spin_lock_init(&ioapic_lock);
1540}
1541
1542__apicdebuginit(void) print_IO_APIC(int ioapic_idx)
1543{
1544 union IO_APIC_reg_00 reg_00;
1545 union IO_APIC_reg_01 reg_01;
1546 union IO_APIC_reg_02 reg_02;
1547 union IO_APIC_reg_03 reg_03;
1548 unsigned long flags;
1549
1550 raw_spin_lock_irqsave(&ioapic_lock, flags);
1551 reg_00.raw = io_apic_read(ioapic_idx, 0);
1552 reg_01.raw = io_apic_read(ioapic_idx, 1);
1553 if (reg_01.bits.version >= 0x10)
1554 reg_02.raw = io_apic_read(ioapic_idx, 2);
1555 if (reg_01.bits.version >= 0x20)
1556 reg_03.raw = io_apic_read(ioapic_idx, 3);
1557 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1558
1559 printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx));
1560 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1561 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1562 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1563 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
1564
1565 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01);
1566 printk(KERN_DEBUG "....... : max redirection entries: %02X\n",
1567 reg_01.bits.entries);
1568
1569 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1570 printk(KERN_DEBUG "....... : IO APIC version: %02X\n",
1571 reg_01.bits.version);
1572
1573
1574
1575
1576
1577
1578 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1579 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1580 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
1581 }
1582
1583
1584
1585
1586
1587
1588 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1589 reg_03.raw != reg_01.raw) {
1590 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1591 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
1592 }
1593
1594 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1595
1596 x86_io_apic_ops.print_entries(ioapic_idx, reg_01.bits.entries);
1597}
1598
1599__apicdebuginit(void) print_IO_APICs(void)
1600{
1601 int ioapic_idx;
1602 struct irq_cfg *cfg;
1603 unsigned int irq;
1604 struct irq_chip *chip;
1605
1606 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1607 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1608 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1609 mpc_ioapic_id(ioapic_idx),
1610 ioapics[ioapic_idx].nr_registers);
1611
1612
1613
1614
1615
1616 printk(KERN_INFO "testing the IO APIC.......................\n");
1617
1618 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1619 print_IO_APIC(ioapic_idx);
1620
1621 printk(KERN_DEBUG "IRQ to pin mappings:\n");
1622 for_each_active_irq(irq) {
1623 struct irq_pin_list *entry;
1624
1625 chip = irq_get_chip(irq);
1626 if (chip != &ioapic_chip)
1627 continue;
1628
1629 cfg = irq_get_chip_data(irq);
1630 if (!cfg)
1631 continue;
1632 entry = cfg->irq_2_pin;
1633 if (!entry)
1634 continue;
1635 printk(KERN_DEBUG "IRQ%d ", irq);
1636 for_each_irq_pin(entry, cfg->irq_2_pin)
1637 pr_cont("-> %d:%d", entry->apic, entry->pin);
1638 pr_cont("\n");
1639 }
1640
1641 printk(KERN_INFO ".................................... done.\n");
1642}
1643
1644__apicdebuginit(void) print_APIC_field(int base)
1645{
1646 int i;
1647
1648 printk(KERN_DEBUG);
1649
1650 for (i = 0; i < 8; i++)
1651 pr_cont("%08x", apic_read(base + i*0x10));
1652
1653 pr_cont("\n");
1654}
1655
1656__apicdebuginit(void) print_local_APIC(void *dummy)
1657{
1658 unsigned int i, v, ver, maxlvt;
1659 u64 icr;
1660
1661 printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1662 smp_processor_id(), hard_smp_processor_id());
1663 v = apic_read(APIC_ID);
1664 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
1665 v = apic_read(APIC_LVR);
1666 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
1667 ver = GET_APIC_VERSION(v);
1668 maxlvt = lapic_get_maxlvt();
1669
1670 v = apic_read(APIC_TASKPRI);
1671 printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1672
1673 if (APIC_INTEGRATED(ver)) {
1674 if (!APIC_XAPIC(ver)) {
1675 v = apic_read(APIC_ARBPRI);
1676 printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
1677 v & APIC_ARBPRI_MASK);
1678 }
1679 v = apic_read(APIC_PROCPRI);
1680 printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
1681 }
1682
1683
1684
1685
1686
1687 if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
1688 v = apic_read(APIC_RRR);
1689 printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
1690 }
1691
1692 v = apic_read(APIC_LDR);
1693 printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1694 if (!x2apic_enabled()) {
1695 v = apic_read(APIC_DFR);
1696 printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
1697 }
1698 v = apic_read(APIC_SPIV);
1699 printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
1700
1701 printk(KERN_DEBUG "... APIC ISR field:\n");
1702 print_APIC_field(APIC_ISR);
1703 printk(KERN_DEBUG "... APIC TMR field:\n");
1704 print_APIC_field(APIC_TMR);
1705 printk(KERN_DEBUG "... APIC IRR field:\n");
1706 print_APIC_field(APIC_IRR);
1707
1708 if (APIC_INTEGRATED(ver)) {
1709 if (maxlvt > 3)
1710 apic_write(APIC_ESR, 0);
1711
1712 v = apic_read(APIC_ESR);
1713 printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
1714 }
1715
1716 icr = apic_icr_read();
1717 printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
1718 printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
1719
1720 v = apic_read(APIC_LVTT);
1721 printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
1722
1723 if (maxlvt > 3) {
1724 v = apic_read(APIC_LVTPC);
1725 printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
1726 }
1727 v = apic_read(APIC_LVT0);
1728 printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
1729 v = apic_read(APIC_LVT1);
1730 printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
1731
1732 if (maxlvt > 2) {
1733 v = apic_read(APIC_LVTERR);
1734 printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
1735 }
1736
1737 v = apic_read(APIC_TMICT);
1738 printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
1739 v = apic_read(APIC_TMCCT);
1740 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1741 v = apic_read(APIC_TDCR);
1742 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1743
1744 if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
1745 v = apic_read(APIC_EFEAT);
1746 maxlvt = (v >> 16) & 0xff;
1747 printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
1748 v = apic_read(APIC_ECTRL);
1749 printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
1750 for (i = 0; i < maxlvt; i++) {
1751 v = apic_read(APIC_EILVTn(i));
1752 printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
1753 }
1754 }
1755 pr_cont("\n");
1756}
1757
1758__apicdebuginit(void) print_local_APICs(int maxcpu)
1759{
1760 int cpu;
1761
1762 if (!maxcpu)
1763 return;
1764
1765 preempt_disable();
1766 for_each_online_cpu(cpu) {
1767 if (cpu >= maxcpu)
1768 break;
1769 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1770 }
1771 preempt_enable();
1772}
1773
1774__apicdebuginit(void) print_PIC(void)
1775{
1776 unsigned int v;
1777 unsigned long flags;
1778
1779 if (!legacy_pic->nr_legacy_irqs)
1780 return;
1781
1782 printk(KERN_DEBUG "\nprinting PIC contents\n");
1783
1784 raw_spin_lock_irqsave(&i8259A_lock, flags);
1785
1786 v = inb(0xa1) << 8 | inb(0x21);
1787 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
1788
1789 v = inb(0xa0) << 8 | inb(0x20);
1790 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1791
1792 outb(0x0b,0xa0);
1793 outb(0x0b,0x20);
1794 v = inb(0xa0) << 8 | inb(0x20);
1795 outb(0x0a,0xa0);
1796 outb(0x0a,0x20);
1797
1798 raw_spin_unlock_irqrestore(&i8259A_lock, flags);
1799
1800 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1801
1802 v = inb(0x4d1) << 8 | inb(0x4d0);
1803 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1804}
1805
1806static int __initdata show_lapic = 1;
1807static __init int setup_show_lapic(char *arg)
1808{
1809 int num = -1;
1810
1811 if (strcmp(arg, "all") == 0) {
1812 show_lapic = CONFIG_NR_CPUS;
1813 } else {
1814 get_option(&arg, &num);
1815 if (num >= 0)
1816 show_lapic = num;
1817 }
1818
1819 return 1;
1820}
1821__setup("show_lapic=", setup_show_lapic);
1822
1823__apicdebuginit(int) print_ICs(void)
1824{
1825 if (apic_verbosity == APIC_QUIET)
1826 return 0;
1827
1828 print_PIC();
1829
1830
1831 if (!cpu_has_apic && !apic_from_smp_config())
1832 return 0;
1833
1834 print_local_APICs(show_lapic);
1835 print_IO_APICs();
1836
1837 return 0;
1838}
1839
1840late_initcall(print_ICs);
1841
1842
1843
1844static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1845
1846void __init enable_IO_APIC(void)
1847{
1848 int i8259_apic, i8259_pin;
1849 int apic;
1850
1851 if (!legacy_pic->nr_legacy_irqs)
1852 return;
1853
1854 for(apic = 0; apic < nr_ioapics; apic++) {
1855 int pin;
1856
1857 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1858 struct IO_APIC_route_entry entry;
1859 entry = ioapic_read_entry(apic, pin);
1860
1861
1862
1863
1864 if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
1865 ioapic_i8259.apic = apic;
1866 ioapic_i8259.pin = pin;
1867 goto found_i8259;
1868 }
1869 }
1870 }
1871 found_i8259:
1872
1873
1874
1875
1876
1877 i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1878 i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1879
1880 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1881 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1882 ioapic_i8259.pin = i8259_pin;
1883 ioapic_i8259.apic = i8259_apic;
1884 }
1885
1886 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1887 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1888 {
1889 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1890 }
1891
1892
1893
1894
1895 clear_IO_APIC();
1896}
1897
1898void native_disable_io_apic(void)
1899{
1900
1901
1902
1903
1904
1905 if (ioapic_i8259.pin != -1) {
1906 struct IO_APIC_route_entry entry;
1907
1908 memset(&entry, 0, sizeof(entry));
1909 entry.mask = 0;
1910 entry.trigger = 0;
1911 entry.irr = 0;
1912 entry.polarity = 0;
1913 entry.delivery_status = 0;
1914 entry.dest_mode = 0;
1915 entry.delivery_mode = dest_ExtINT;
1916 entry.vector = 0;
1917 entry.dest = read_apic_id();
1918
1919
1920
1921
1922 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1923 }
1924
1925 if (cpu_has_apic || apic_from_smp_config())
1926 disconnect_bsp_APIC(ioapic_i8259.pin != -1);
1927
1928}
1929
1930
1931
1932
1933void disable_IO_APIC(void)
1934{
1935
1936
1937
1938 clear_IO_APIC();
1939
1940 if (!legacy_pic->nr_legacy_irqs)
1941 return;
1942
1943 x86_io_apic_ops.disable();
1944}
1945
1946#ifdef CONFIG_X86_32
1947
1948
1949
1950
1951
1952
1953void __init setup_ioapic_ids_from_mpc_nocheck(void)
1954{
1955 union IO_APIC_reg_00 reg_00;
1956 physid_mask_t phys_id_present_map;
1957 int ioapic_idx;
1958 int i;
1959 unsigned char old_id;
1960 unsigned long flags;
1961
1962
1963
1964
1965
1966 apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map);
1967
1968
1969
1970
1971 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) {
1972
1973 raw_spin_lock_irqsave(&ioapic_lock, flags);
1974 reg_00.raw = io_apic_read(ioapic_idx, 0);
1975 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1976
1977 old_id = mpc_ioapic_id(ioapic_idx);
1978
1979 if (mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) {
1980 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
1981 ioapic_idx, mpc_ioapic_id(ioapic_idx));
1982 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1983 reg_00.bits.ID);
1984 ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID;
1985 }
1986
1987
1988
1989
1990
1991
1992 if (apic->check_apicid_used(&phys_id_present_map,
1993 mpc_ioapic_id(ioapic_idx))) {
1994 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
1995 ioapic_idx, mpc_ioapic_id(ioapic_idx));
1996 for (i = 0; i < get_physical_broadcast(); i++)
1997 if (!physid_isset(i, phys_id_present_map))
1998 break;
1999 if (i >= get_physical_broadcast())
2000 panic("Max APIC ID exceeded!\n");
2001 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2002 i);
2003 physid_set(i, phys_id_present_map);
2004 ioapics[ioapic_idx].mp_config.apicid = i;
2005 } else {
2006 physid_mask_t tmp;
2007 apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx),
2008 &tmp);
2009 apic_printk(APIC_VERBOSE, "Setting %d in the "
2010 "phys_id_present_map\n",
2011 mpc_ioapic_id(ioapic_idx));
2012 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2013 }
2014
2015
2016
2017
2018
2019 if (old_id != mpc_ioapic_id(ioapic_idx))
2020 for (i = 0; i < mp_irq_entries; i++)
2021 if (mp_irqs[i].dstapic == old_id)
2022 mp_irqs[i].dstapic
2023 = mpc_ioapic_id(ioapic_idx);
2024
2025
2026
2027
2028
2029 if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID)
2030 continue;
2031
2032 apic_printk(APIC_VERBOSE, KERN_INFO
2033 "...changing IO-APIC physical APIC ID to %d ...",
2034 mpc_ioapic_id(ioapic_idx));
2035
2036 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2037 raw_spin_lock_irqsave(&ioapic_lock, flags);
2038 io_apic_write(ioapic_idx, 0, reg_00.raw);
2039 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2040
2041
2042
2043
2044 raw_spin_lock_irqsave(&ioapic_lock, flags);
2045 reg_00.raw = io_apic_read(ioapic_idx, 0);
2046 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2047 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx))
2048 pr_cont("could not set ID!\n");
2049 else
2050 apic_printk(APIC_VERBOSE, " ok.\n");
2051 }
2052}
2053
2054void __init setup_ioapic_ids_from_mpc(void)
2055{
2056
2057 if (acpi_ioapic)
2058 return;
2059
2060
2061
2062
2063 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2064 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
2065 return;
2066 setup_ioapic_ids_from_mpc_nocheck();
2067}
2068#endif
2069
2070int no_timer_check __initdata;
2071
2072static int __init notimercheck(char *s)
2073{
2074 no_timer_check = 1;
2075 return 1;
2076}
2077__setup("no_timer_check", notimercheck);
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087static int __init timer_irq_works(void)
2088{
2089 unsigned long t1 = jiffies;
2090 unsigned long flags;
2091
2092 if (no_timer_check)
2093 return 1;
2094
2095 local_save_flags(flags);
2096 local_irq_enable();
2097
2098 mdelay((10 * 1000) / HZ);
2099 local_irq_restore(flags);
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110 if (time_after(jiffies, t1 + 4))
2111 return 1;
2112 return 0;
2113}
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138static unsigned int startup_ioapic_irq(struct irq_data *data)
2139{
2140 int was_pending = 0, irq = data->irq;
2141 unsigned long flags;
2142
2143 raw_spin_lock_irqsave(&ioapic_lock, flags);
2144 if (irq < legacy_pic->nr_legacy_irqs) {
2145 legacy_pic->mask(irq);
2146 if (legacy_pic->irq_pending(irq))
2147 was_pending = 1;
2148 }
2149 __unmask_ioapic(data->chip_data);
2150 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2151
2152 return was_pending;
2153}
2154
2155static int ioapic_retrigger_irq(struct irq_data *data)
2156{
2157 struct irq_cfg *cfg = data->chip_data;
2158 unsigned long flags;
2159 int cpu;
2160
2161 raw_spin_lock_irqsave(&vector_lock, flags);
2162 cpu = cpumask_first_and(cfg->domain, cpu_online_mask);
2163 apic->send_IPI_mask(cpumask_of(cpu), cfg->vector);
2164 raw_spin_unlock_irqrestore(&vector_lock, flags);
2165
2166 return 1;
2167}
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178#ifdef CONFIG_SMP
2179void send_cleanup_vector(struct irq_cfg *cfg)
2180{
2181 cpumask_var_t cleanup_mask;
2182
2183 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
2184 unsigned int i;
2185 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2186 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
2187 } else {
2188 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
2189 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2190 free_cpumask_var(cleanup_mask);
2191 }
2192 cfg->move_in_progress = 0;
2193}
2194
2195asmlinkage void smp_irq_move_cleanup_interrupt(void)
2196{
2197 unsigned vector, me;
2198
2199 ack_APIC_irq();
2200 irq_enter();
2201 exit_idle();
2202
2203 me = smp_processor_id();
2204 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
2205 unsigned int irq;
2206 unsigned int irr;
2207 struct irq_desc *desc;
2208 struct irq_cfg *cfg;
2209 irq = __this_cpu_read(vector_irq[vector]);
2210
2211 if (irq == -1)
2212 continue;
2213
2214 desc = irq_to_desc(irq);
2215 if (!desc)
2216 continue;
2217
2218 cfg = irq_cfg(irq);
2219 if (!cfg)
2220 continue;
2221
2222 raw_spin_lock(&desc->lock);
2223
2224
2225
2226
2227
2228 if (cfg->move_in_progress)
2229 goto unlock;
2230
2231 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2232 goto unlock;
2233
2234 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
2235
2236
2237
2238
2239
2240
2241
2242 if (irr & (1 << (vector % 32))) {
2243 apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
2244 goto unlock;
2245 }
2246 __this_cpu_write(vector_irq[vector], -1);
2247unlock:
2248 raw_spin_unlock(&desc->lock);
2249 }
2250
2251 irq_exit();
2252}
2253
2254static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
2255{
2256 unsigned me;
2257
2258 if (likely(!cfg->move_in_progress))
2259 return;
2260
2261 me = smp_processor_id();
2262
2263 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2264 send_cleanup_vector(cfg);
2265}
2266
2267static void irq_complete_move(struct irq_cfg *cfg)
2268{
2269 __irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
2270}
2271
2272void irq_force_complete_move(int irq)
2273{
2274 struct irq_cfg *cfg = irq_get_chip_data(irq);
2275
2276 if (!cfg)
2277 return;
2278
2279 __irq_complete_move(cfg, cfg->vector);
2280}
2281#else
2282static inline void irq_complete_move(struct irq_cfg *cfg) { }
2283#endif
2284
2285static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2286{
2287 int apic, pin;
2288 struct irq_pin_list *entry;
2289 u8 vector = cfg->vector;
2290
2291 for_each_irq_pin(entry, cfg->irq_2_pin) {
2292 unsigned int reg;
2293
2294 apic = entry->apic;
2295 pin = entry->pin;
2296
2297 io_apic_write(apic, 0x11 + pin*2, dest);
2298 reg = io_apic_read(apic, 0x10 + pin*2);
2299 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
2300 reg |= vector;
2301 io_apic_modify(apic, 0x10 + pin*2, reg);
2302 }
2303}
2304
2305
2306
2307
2308
2309
2310int __ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
2311 unsigned int *dest_id)
2312{
2313 struct irq_cfg *cfg = data->chip_data;
2314 unsigned int irq = data->irq;
2315 int err;
2316
2317 if (!config_enabled(CONFIG_SMP))
2318 return -1;
2319
2320 if (!cpumask_intersects(mask, cpu_online_mask))
2321 return -EINVAL;
2322
2323 err = assign_irq_vector(irq, cfg, mask);
2324 if (err)
2325 return err;
2326
2327 err = apic->cpu_mask_to_apicid_and(mask, cfg->domain, dest_id);
2328 if (err) {
2329 if (assign_irq_vector(irq, cfg, data->affinity))
2330 pr_err("Failed to recover vector for irq %d\n", irq);
2331 return err;
2332 }
2333
2334 cpumask_copy(data->affinity, mask);
2335
2336 return 0;
2337}
2338
2339
2340int native_ioapic_set_affinity(struct irq_data *data,
2341 const struct cpumask *mask,
2342 bool force)
2343{
2344 unsigned int dest, irq = data->irq;
2345 unsigned long flags;
2346 int ret;
2347
2348 if (!config_enabled(CONFIG_SMP))
2349 return -1;
2350
2351 raw_spin_lock_irqsave(&ioapic_lock, flags);
2352 ret = __ioapic_set_affinity(data, mask, &dest);
2353 if (!ret) {
2354
2355 dest = SET_APIC_LOGICAL_ID(dest);
2356 __target_IO_APIC_irq(irq, dest, data->chip_data);
2357 ret = IRQ_SET_MASK_OK_NOCOPY;
2358 }
2359 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2360 return ret;
2361}
2362
2363static void ack_apic_edge(struct irq_data *data)
2364{
2365 irq_complete_move(data->chip_data);
2366 irq_move_irq(data);
2367 ack_APIC_irq();
2368}
2369
2370atomic_t irq_mis_count;
2371
2372#ifdef CONFIG_GENERIC_PENDING_IRQ
2373static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
2374{
2375 struct irq_pin_list *entry;
2376 unsigned long flags;
2377
2378 raw_spin_lock_irqsave(&ioapic_lock, flags);
2379 for_each_irq_pin(entry, cfg->irq_2_pin) {
2380 unsigned int reg;
2381 int pin;
2382
2383 pin = entry->pin;
2384 reg = io_apic_read(entry->apic, 0x10 + pin*2);
2385
2386 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
2387 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2388 return true;
2389 }
2390 }
2391 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2392
2393 return false;
2394}
2395
2396static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
2397{
2398
2399 if (unlikely(irqd_is_setaffinity_pending(data))) {
2400 mask_ioapic(cfg);
2401 return true;
2402 }
2403 return false;
2404}
2405
2406static inline void ioapic_irqd_unmask(struct irq_data *data,
2407 struct irq_cfg *cfg, bool masked)
2408{
2409 if (unlikely(masked)) {
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436 if (!io_apic_level_ack_pending(cfg))
2437 irq_move_masked_irq(data);
2438 unmask_ioapic(cfg);
2439 }
2440}
2441#else
2442static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
2443{
2444 return false;
2445}
2446static inline void ioapic_irqd_unmask(struct irq_data *data,
2447 struct irq_cfg *cfg, bool masked)
2448{
2449}
2450#endif
2451
2452static void ack_apic_level(struct irq_data *data)
2453{
2454 struct irq_cfg *cfg = data->chip_data;
2455 int i, irq = data->irq;
2456 unsigned long v;
2457 bool masked;
2458
2459 irq_complete_move(cfg);
2460 masked = ioapic_irqd_mask(data, cfg);
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494 i = cfg->vector;
2495 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2496
2497
2498
2499
2500
2501 ack_APIC_irq();
2502
2503
2504
2505
2506
2507
2508
2509
2510 if (!(v & (1 << (i & 0x1f)))) {
2511 atomic_inc(&irq_mis_count);
2512
2513 eoi_ioapic_irq(irq, cfg);
2514 }
2515
2516 ioapic_irqd_unmask(data, cfg, masked);
2517}
2518
2519static struct irq_chip ioapic_chip __read_mostly = {
2520 .name = "IO-APIC",
2521 .irq_startup = startup_ioapic_irq,
2522 .irq_mask = mask_ioapic_irq,
2523 .irq_unmask = unmask_ioapic_irq,
2524 .irq_ack = ack_apic_edge,
2525 .irq_eoi = ack_apic_level,
2526 .irq_set_affinity = native_ioapic_set_affinity,
2527 .irq_retrigger = ioapic_retrigger_irq,
2528};
2529
2530static inline void init_IO_APIC_traps(void)
2531{
2532 struct irq_cfg *cfg;
2533 unsigned int irq;
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546 for_each_active_irq(irq) {
2547 cfg = irq_get_chip_data(irq);
2548 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
2549
2550
2551
2552
2553
2554 if (irq < legacy_pic->nr_legacy_irqs)
2555 legacy_pic->make_irq(irq);
2556 else
2557
2558 irq_set_chip(irq, &no_irq_chip);
2559 }
2560 }
2561}
2562
2563
2564
2565
2566
2567static void mask_lapic_irq(struct irq_data *data)
2568{
2569 unsigned long v;
2570
2571 v = apic_read(APIC_LVT0);
2572 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
2573}
2574
2575static void unmask_lapic_irq(struct irq_data *data)
2576{
2577 unsigned long v;
2578
2579 v = apic_read(APIC_LVT0);
2580 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2581}
2582
2583static void ack_lapic_irq(struct irq_data *data)
2584{
2585 ack_APIC_irq();
2586}
2587
2588static struct irq_chip lapic_chip __read_mostly = {
2589 .name = "local-APIC",
2590 .irq_mask = mask_lapic_irq,
2591 .irq_unmask = unmask_lapic_irq,
2592 .irq_ack = ack_lapic_irq,
2593};
2594
2595static void lapic_register_intr(int irq)
2596{
2597 irq_clear_status_flags(irq, IRQ_LEVEL);
2598 irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2599 "edge");
2600}
2601
2602
2603
2604
2605
2606
2607
2608
2609static inline void __init unlock_ExtINT_logic(void)
2610{
2611 int apic, pin, i;
2612 struct IO_APIC_route_entry entry0, entry1;
2613 unsigned char save_control, save_freq_select;
2614
2615 pin = find_isa_irq_pin(8, mp_INT);
2616 if (pin == -1) {
2617 WARN_ON_ONCE(1);
2618 return;
2619 }
2620 apic = find_isa_irq_apic(8, mp_INT);
2621 if (apic == -1) {
2622 WARN_ON_ONCE(1);
2623 return;
2624 }
2625
2626 entry0 = ioapic_read_entry(apic, pin);
2627 clear_IO_APIC_pin(apic, pin);
2628
2629 memset(&entry1, 0, sizeof(entry1));
2630
2631 entry1.dest_mode = 0;
2632 entry1.mask = 0;
2633 entry1.dest = hard_smp_processor_id();
2634 entry1.delivery_mode = dest_ExtINT;
2635 entry1.polarity = entry0.polarity;
2636 entry1.trigger = 0;
2637 entry1.vector = 0;
2638
2639 ioapic_write_entry(apic, pin, entry1);
2640
2641 save_control = CMOS_READ(RTC_CONTROL);
2642 save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2643 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2644 RTC_FREQ_SELECT);
2645 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2646
2647 i = 100;
2648 while (i-- > 0) {
2649 mdelay(10);
2650 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2651 i -= 10;
2652 }
2653
2654 CMOS_WRITE(save_control, RTC_CONTROL);
2655 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2656 clear_IO_APIC_pin(apic, pin);
2657
2658 ioapic_write_entry(apic, pin, entry0);
2659}
2660
2661static int disable_timer_pin_1 __initdata;
2662
2663static int __init disable_timer_pin_setup(char *arg)
2664{
2665 disable_timer_pin_1 = 1;
2666 return 0;
2667}
2668early_param("disable_timer_pin_1", disable_timer_pin_setup);
2669
2670int timer_through_8259 __initdata;
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680static inline void __init check_timer(void)
2681{
2682 struct irq_cfg *cfg = irq_get_chip_data(0);
2683 int node = cpu_to_node(0);
2684 int apic1, pin1, apic2, pin2;
2685 unsigned long flags;
2686 int no_pin1 = 0;
2687
2688 local_irq_save(flags);
2689
2690
2691
2692
2693 legacy_pic->mask(0);
2694 assign_irq_vector(0, cfg, apic->target_cpus());
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2706 legacy_pic->init(1);
2707
2708 pin1 = find_isa_irq_pin(0, mp_INT);
2709 apic1 = find_isa_irq_apic(0, mp_INT);
2710 pin2 = ioapic_i8259.pin;
2711 apic2 = ioapic_i8259.apic;
2712
2713 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2714 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2715 cfg->vector, apic1, pin1, apic2, pin2);
2716
2717
2718
2719
2720
2721
2722
2723
2724 if (pin1 == -1) {
2725 panic_if_irq_remap("BIOS bug: timer not connected to IO-APIC");
2726 pin1 = pin2;
2727 apic1 = apic2;
2728 no_pin1 = 1;
2729 } else if (pin2 == -1) {
2730 pin2 = pin1;
2731 apic2 = apic1;
2732 }
2733
2734 if (pin1 != -1) {
2735
2736
2737
2738 if (no_pin1) {
2739 add_pin_to_irq_node(cfg, node, apic1, pin1);
2740 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2741 } else {
2742
2743
2744
2745
2746
2747 int idx;
2748 idx = find_irq_entry(apic1, pin1, mp_INT);
2749 if (idx != -1 && irq_trigger(idx))
2750 unmask_ioapic(cfg);
2751 }
2752 if (timer_irq_works()) {
2753 if (disable_timer_pin_1 > 0)
2754 clear_IO_APIC_pin(0, pin1);
2755 goto out;
2756 }
2757 panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC");
2758 local_irq_disable();
2759 clear_IO_APIC_pin(apic1, pin1);
2760 if (!no_pin1)
2761 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2762 "8254 timer not connected to IO-APIC\n");
2763
2764 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2765 "(IRQ0) through the 8259A ...\n");
2766 apic_printk(APIC_QUIET, KERN_INFO
2767 "..... (found apic %d pin %d) ...\n", apic2, pin2);
2768
2769
2770
2771 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2772 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2773 legacy_pic->unmask(0);
2774 if (timer_irq_works()) {
2775 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2776 timer_through_8259 = 1;
2777 goto out;
2778 }
2779
2780
2781
2782 local_irq_disable();
2783 legacy_pic->mask(0);
2784 clear_IO_APIC_pin(apic2, pin2);
2785 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
2786 }
2787
2788 apic_printk(APIC_QUIET, KERN_INFO
2789 "...trying to set up timer as Virtual Wire IRQ...\n");
2790
2791 lapic_register_intr(0);
2792 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);
2793 legacy_pic->unmask(0);
2794
2795 if (timer_irq_works()) {
2796 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2797 goto out;
2798 }
2799 local_irq_disable();
2800 legacy_pic->mask(0);
2801 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2802 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
2803
2804 apic_printk(APIC_QUIET, KERN_INFO
2805 "...trying to set up timer as ExtINT IRQ...\n");
2806
2807 legacy_pic->init(0);
2808 legacy_pic->make_irq(0);
2809 apic_write(APIC_LVT0, APIC_DM_EXTINT);
2810
2811 unlock_ExtINT_logic();
2812
2813 if (timer_irq_works()) {
2814 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2815 goto out;
2816 }
2817 local_irq_disable();
2818 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
2819 if (x2apic_preenabled)
2820 apic_printk(APIC_QUIET, KERN_INFO
2821 "Perhaps problem with the pre-enabled x2apic mode\n"
2822 "Try booting with x2apic and interrupt-remapping disabled in the bios.\n");
2823 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
2824 "report. Then try booting with the 'noapic' option.\n");
2825out:
2826 local_irq_restore(flags);
2827}
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846#define PIC_IRQS (1UL << PIC_CASCADE_IR)
2847
2848void __init setup_IO_APIC(void)
2849{
2850
2851
2852
2853
2854 io_apic_irqs = legacy_pic->nr_legacy_irqs ? ~PIC_IRQS : ~0UL;
2855
2856 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2857
2858
2859
2860 x86_init.mpparse.setup_ioapic_ids();
2861
2862 sync_Arb_IDs();
2863 setup_IO_APIC_irqs();
2864 init_IO_APIC_traps();
2865 if (legacy_pic->nr_legacy_irqs)
2866 check_timer();
2867}
2868
2869
2870
2871
2872
2873
2874static int __init io_apic_bug_finalize(void)
2875{
2876 if (sis_apic_bug == -1)
2877 sis_apic_bug = 0;
2878 return 0;
2879}
2880
2881late_initcall(io_apic_bug_finalize);
2882
2883static void resume_ioapic_id(int ioapic_idx)
2884{
2885 unsigned long flags;
2886 union IO_APIC_reg_00 reg_00;
2887
2888 raw_spin_lock_irqsave(&ioapic_lock, flags);
2889 reg_00.raw = io_apic_read(ioapic_idx, 0);
2890 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) {
2891 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2892 io_apic_write(ioapic_idx, 0, reg_00.raw);
2893 }
2894 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2895}
2896
2897static void ioapic_resume(void)
2898{
2899 int ioapic_idx;
2900
2901 for (ioapic_idx = nr_ioapics - 1; ioapic_idx >= 0; ioapic_idx--)
2902 resume_ioapic_id(ioapic_idx);
2903
2904 restore_ioapic_entries();
2905}
2906
2907static struct syscore_ops ioapic_syscore_ops = {
2908 .suspend = save_ioapic_entries,
2909 .resume = ioapic_resume,
2910};
2911
2912static int __init ioapic_init_ops(void)
2913{
2914 register_syscore_ops(&ioapic_syscore_ops);
2915
2916 return 0;
2917}
2918
2919device_initcall(ioapic_init_ops);
2920
2921
2922
2923
2924unsigned int __create_irqs(unsigned int from, unsigned int count, int node)
2925{
2926 struct irq_cfg **cfg;
2927 unsigned long flags;
2928 int irq, i;
2929
2930 if (from < nr_irqs_gsi)
2931 from = nr_irqs_gsi;
2932
2933 cfg = kzalloc_node(count * sizeof(cfg[0]), GFP_KERNEL, node);
2934 if (!cfg)
2935 return 0;
2936
2937 irq = alloc_irqs_from(from, count, node);
2938 if (irq < 0)
2939 goto out_cfgs;
2940
2941 for (i = 0; i < count; i++) {
2942 cfg[i] = alloc_irq_cfg(irq + i, node);
2943 if (!cfg[i])
2944 goto out_irqs;
2945 }
2946
2947 raw_spin_lock_irqsave(&vector_lock, flags);
2948 for (i = 0; i < count; i++)
2949 if (__assign_irq_vector(irq + i, cfg[i], apic->target_cpus()))
2950 goto out_vecs;
2951 raw_spin_unlock_irqrestore(&vector_lock, flags);
2952
2953 for (i = 0; i < count; i++) {
2954 irq_set_chip_data(irq + i, cfg[i]);
2955 irq_clear_status_flags(irq + i, IRQ_NOREQUEST);
2956 }
2957
2958 kfree(cfg);
2959 return irq;
2960
2961out_vecs:
2962 for (i--; i >= 0; i--)
2963 __clear_irq_vector(irq + i, cfg[i]);
2964 raw_spin_unlock_irqrestore(&vector_lock, flags);
2965out_irqs:
2966 for (i = 0; i < count; i++)
2967 free_irq_at(irq + i, cfg[i]);
2968out_cfgs:
2969 kfree(cfg);
2970 return 0;
2971}
2972
2973unsigned int create_irq_nr(unsigned int from, int node)
2974{
2975 return __create_irqs(from, 1, node);
2976}
2977
2978int create_irq(void)
2979{
2980 int node = cpu_to_node(0);
2981 unsigned int irq_want;
2982 int irq;
2983
2984 irq_want = nr_irqs_gsi;
2985 irq = create_irq_nr(irq_want, node);
2986
2987 if (irq == 0)
2988 irq = -1;
2989
2990 return irq;
2991}
2992
2993void destroy_irq(unsigned int irq)
2994{
2995 struct irq_cfg *cfg = irq_get_chip_data(irq);
2996 unsigned long flags;
2997
2998 irq_set_status_flags(irq, IRQ_NOREQUEST|IRQ_NOPROBE);
2999
3000 free_remapped_irq(irq);
3001
3002 raw_spin_lock_irqsave(&vector_lock, flags);
3003 __clear_irq_vector(irq, cfg);
3004 raw_spin_unlock_irqrestore(&vector_lock, flags);
3005 free_irq_at(irq, cfg);
3006}
3007
3008void destroy_irqs(unsigned int irq, unsigned int count)
3009{
3010 unsigned int i;
3011
3012 for (i = 0; i < count; i++)
3013 destroy_irq(irq + i);
3014}
3015
3016
3017
3018
3019void native_compose_msi_msg(struct pci_dev *pdev,
3020 unsigned int irq, unsigned int dest,
3021 struct msi_msg *msg, u8 hpet_id)
3022{
3023 struct irq_cfg *cfg = irq_cfg(irq);
3024
3025 msg->address_hi = MSI_ADDR_BASE_HI;
3026
3027 if (x2apic_enabled())
3028 msg->address_hi |= MSI_ADDR_EXT_DEST_ID(dest);
3029
3030 msg->address_lo =
3031 MSI_ADDR_BASE_LO |
3032 ((apic->irq_dest_mode == 0) ?
3033 MSI_ADDR_DEST_MODE_PHYSICAL:
3034 MSI_ADDR_DEST_MODE_LOGICAL) |
3035 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3036 MSI_ADDR_REDIRECTION_CPU:
3037 MSI_ADDR_REDIRECTION_LOWPRI) |
3038 MSI_ADDR_DEST_ID(dest);
3039
3040 msg->data =
3041 MSI_DATA_TRIGGER_EDGE |
3042 MSI_DATA_LEVEL_ASSERT |
3043 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3044 MSI_DATA_DELIVERY_FIXED:
3045 MSI_DATA_DELIVERY_LOWPRI) |
3046 MSI_DATA_VECTOR(cfg->vector);
3047}
3048
3049#ifdef CONFIG_PCI_MSI
3050static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
3051 struct msi_msg *msg, u8 hpet_id)
3052{
3053 struct irq_cfg *cfg;
3054 int err;
3055 unsigned dest;
3056
3057 if (disable_apic)
3058 return -ENXIO;
3059
3060 cfg = irq_cfg(irq);
3061 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3062 if (err)
3063 return err;
3064
3065 err = apic->cpu_mask_to_apicid_and(cfg->domain,
3066 apic->target_cpus(), &dest);
3067 if (err)
3068 return err;
3069
3070 x86_msi.compose_msi_msg(pdev, irq, dest, msg, hpet_id);
3071
3072 return 0;
3073}
3074
3075static int
3076msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3077{
3078 struct irq_cfg *cfg = data->chip_data;
3079 struct msi_msg msg;
3080 unsigned int dest;
3081
3082 if (__ioapic_set_affinity(data, mask, &dest))
3083 return -1;
3084
3085 __get_cached_msi_msg(data->msi_desc, &msg);
3086
3087 msg.data &= ~MSI_DATA_VECTOR_MASK;
3088 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3089 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3090 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3091
3092 __write_msi_msg(data->msi_desc, &msg);
3093
3094 return IRQ_SET_MASK_OK_NOCOPY;
3095}
3096
3097
3098
3099
3100
3101static struct irq_chip msi_chip = {
3102 .name = "PCI-MSI",
3103 .irq_unmask = unmask_msi_irq,
3104 .irq_mask = mask_msi_irq,
3105 .irq_ack = ack_apic_edge,
3106 .irq_set_affinity = msi_set_affinity,
3107 .irq_retrigger = ioapic_retrigger_irq,
3108};
3109
3110int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc,
3111 unsigned int irq_base, unsigned int irq_offset)
3112{
3113 struct irq_chip *chip = &msi_chip;
3114 struct msi_msg msg;
3115 unsigned int irq = irq_base + irq_offset;
3116 int ret;
3117
3118 ret = msi_compose_msg(dev, irq, &msg, -1);
3119 if (ret < 0)
3120 return ret;
3121
3122 irq_set_msi_desc_off(irq_base, irq_offset, msidesc);
3123
3124
3125
3126
3127
3128 if (!irq_offset)
3129 write_msi_msg(irq, &msg);
3130
3131 setup_remapped_irq(irq, irq_get_chip_data(irq), chip);
3132
3133 irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3134
3135 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);
3136
3137 return 0;
3138}
3139
3140int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3141{
3142 unsigned int irq, irq_want;
3143 struct msi_desc *msidesc;
3144 int node, ret;
3145
3146
3147 if (type == PCI_CAP_ID_MSI && nvec > 1)
3148 return 1;
3149
3150 node = dev_to_node(&dev->dev);
3151 irq_want = nr_irqs_gsi;
3152 list_for_each_entry(msidesc, &dev->msi_list, list) {
3153 irq = create_irq_nr(irq_want, node);
3154 if (irq == 0)
3155 return -ENOSPC;
3156
3157 irq_want = irq + 1;
3158
3159 ret = setup_msi_irq(dev, msidesc, irq, 0);
3160 if (ret < 0)
3161 goto error;
3162 }
3163 return 0;
3164
3165error:
3166 destroy_irq(irq);
3167 return ret;
3168}
3169
3170void native_teardown_msi_irq(unsigned int irq)
3171{
3172 destroy_irq(irq);
3173}
3174
3175#ifdef CONFIG_DMAR_TABLE
3176static int
3177dmar_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
3178 bool force)
3179{
3180 struct irq_cfg *cfg = data->chip_data;
3181 unsigned int dest, irq = data->irq;
3182 struct msi_msg msg;
3183
3184 if (__ioapic_set_affinity(data, mask, &dest))
3185 return -1;
3186
3187 dmar_msi_read(irq, &msg);
3188
3189 msg.data &= ~MSI_DATA_VECTOR_MASK;
3190 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3191 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3192 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3193 msg.address_hi = MSI_ADDR_BASE_HI | MSI_ADDR_EXT_DEST_ID(dest);
3194
3195 dmar_msi_write(irq, &msg);
3196
3197 return IRQ_SET_MASK_OK_NOCOPY;
3198}
3199
3200static struct irq_chip dmar_msi_type = {
3201 .name = "DMAR_MSI",
3202 .irq_unmask = dmar_msi_unmask,
3203 .irq_mask = dmar_msi_mask,
3204 .irq_ack = ack_apic_edge,
3205 .irq_set_affinity = dmar_msi_set_affinity,
3206 .irq_retrigger = ioapic_retrigger_irq,
3207};
3208
3209int arch_setup_dmar_msi(unsigned int irq)
3210{
3211 int ret;
3212 struct msi_msg msg;
3213
3214 ret = msi_compose_msg(NULL, irq, &msg, -1);
3215 if (ret < 0)
3216 return ret;
3217 dmar_msi_write(irq, &msg);
3218 irq_set_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
3219 "edge");
3220 return 0;
3221}
3222#endif
3223
3224#ifdef CONFIG_HPET_TIMER
3225
3226static int hpet_msi_set_affinity(struct irq_data *data,
3227 const struct cpumask *mask, bool force)
3228{
3229 struct irq_cfg *cfg = data->chip_data;
3230 struct msi_msg msg;
3231 unsigned int dest;
3232
3233 if (__ioapic_set_affinity(data, mask, &dest))
3234 return -1;
3235
3236 hpet_msi_read(data->handler_data, &msg);
3237
3238 msg.data &= ~MSI_DATA_VECTOR_MASK;
3239 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3240 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3241 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3242
3243 hpet_msi_write(data->handler_data, &msg);
3244
3245 return IRQ_SET_MASK_OK_NOCOPY;
3246}
3247
3248static struct irq_chip hpet_msi_type = {
3249 .name = "HPET_MSI",
3250 .irq_unmask = hpet_msi_unmask,
3251 .irq_mask = hpet_msi_mask,
3252 .irq_ack = ack_apic_edge,
3253 .irq_set_affinity = hpet_msi_set_affinity,
3254 .irq_retrigger = ioapic_retrigger_irq,
3255};
3256
3257int default_setup_hpet_msi(unsigned int irq, unsigned int id)
3258{
3259 struct irq_chip *chip = &hpet_msi_type;
3260 struct msi_msg msg;
3261 int ret;
3262
3263 ret = msi_compose_msg(NULL, irq, &msg, id);
3264 if (ret < 0)
3265 return ret;
3266
3267 hpet_msi_write(irq_get_handler_data(irq), &msg);
3268 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
3269 setup_remapped_irq(irq, irq_get_chip_data(irq), chip);
3270
3271 irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3272 return 0;
3273}
3274#endif
3275
3276#endif
3277
3278
3279
3280#ifdef CONFIG_HT_IRQ
3281
3282static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3283{
3284 struct ht_irq_msg msg;
3285 fetch_ht_irq_msg(irq, &msg);
3286
3287 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
3288 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3289
3290 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
3291 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3292
3293 write_ht_irq_msg(irq, &msg);
3294}
3295
3296static int
3297ht_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3298{
3299 struct irq_cfg *cfg = data->chip_data;
3300 unsigned int dest;
3301
3302 if (__ioapic_set_affinity(data, mask, &dest))
3303 return -1;
3304
3305 target_ht_irq(data->irq, dest, cfg->vector);
3306 return IRQ_SET_MASK_OK_NOCOPY;
3307}
3308
3309static struct irq_chip ht_irq_chip = {
3310 .name = "PCI-HT",
3311 .irq_mask = mask_ht_irq,
3312 .irq_unmask = unmask_ht_irq,
3313 .irq_ack = ack_apic_edge,
3314 .irq_set_affinity = ht_set_affinity,
3315 .irq_retrigger = ioapic_retrigger_irq,
3316};
3317
3318int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3319{
3320 struct irq_cfg *cfg;
3321 struct ht_irq_msg msg;
3322 unsigned dest;
3323 int err;
3324
3325 if (disable_apic)
3326 return -ENXIO;
3327
3328 cfg = irq_cfg(irq);
3329 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3330 if (err)
3331 return err;
3332
3333 err = apic->cpu_mask_to_apicid_and(cfg->domain,
3334 apic->target_cpus(), &dest);
3335 if (err)
3336 return err;
3337
3338 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3339
3340 msg.address_lo =
3341 HT_IRQ_LOW_BASE |
3342 HT_IRQ_LOW_DEST_ID(dest) |
3343 HT_IRQ_LOW_VECTOR(cfg->vector) |
3344 ((apic->irq_dest_mode == 0) ?
3345 HT_IRQ_LOW_DM_PHYSICAL :
3346 HT_IRQ_LOW_DM_LOGICAL) |
3347 HT_IRQ_LOW_RQEOI_EDGE |
3348 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3349 HT_IRQ_LOW_MT_FIXED :
3350 HT_IRQ_LOW_MT_ARBITRATED) |
3351 HT_IRQ_LOW_IRQ_MASKED;
3352
3353 write_ht_irq_msg(irq, &msg);
3354
3355 irq_set_chip_and_handler_name(irq, &ht_irq_chip,
3356 handle_edge_irq, "edge");
3357
3358 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
3359
3360 return 0;
3361}
3362#endif
3363
3364static int
3365io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
3366{
3367 struct irq_cfg *cfg = alloc_irq_and_cfg_at(irq, node);
3368 int ret;
3369
3370 if (!cfg)
3371 return -EINVAL;
3372 ret = __add_pin_to_irq_node(cfg, node, attr->ioapic, attr->ioapic_pin);
3373 if (!ret)
3374 setup_ioapic_irq(irq, cfg, attr);
3375 return ret;
3376}
3377
3378int io_apic_setup_irq_pin_once(unsigned int irq, int node,
3379 struct io_apic_irq_attr *attr)
3380{
3381 unsigned int ioapic_idx = attr->ioapic, pin = attr->ioapic_pin;
3382 int ret;
3383 struct IO_APIC_route_entry orig_entry;
3384
3385
3386 if (test_bit(pin, ioapics[ioapic_idx].pin_programmed)) {
3387 pr_debug("Pin %d-%d already programmed\n", mpc_ioapic_id(ioapic_idx), pin);
3388 orig_entry = ioapic_read_entry(attr->ioapic, pin);
3389 if (attr->trigger == orig_entry.trigger && attr->polarity == orig_entry.polarity)
3390 return 0;
3391 return -EBUSY;
3392 }
3393 ret = io_apic_setup_irq_pin(irq, node, attr);
3394 if (!ret)
3395 set_bit(pin, ioapics[ioapic_idx].pin_programmed);
3396 return ret;
3397}
3398
3399static int __init io_apic_get_redir_entries(int ioapic)
3400{
3401 union IO_APIC_reg_01 reg_01;
3402 unsigned long flags;
3403
3404 raw_spin_lock_irqsave(&ioapic_lock, flags);
3405 reg_01.raw = io_apic_read(ioapic, 1);
3406 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3407
3408
3409
3410
3411
3412 return reg_01.bits.entries + 1;
3413}
3414
3415static void __init probe_nr_irqs_gsi(void)
3416{
3417 int nr;
3418
3419 nr = gsi_top + NR_IRQS_LEGACY;
3420 if (nr > nr_irqs_gsi)
3421 nr_irqs_gsi = nr;
3422
3423 printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
3424}
3425
3426int get_nr_irqs_gsi(void)
3427{
3428 return nr_irqs_gsi;
3429}
3430
3431int __init arch_probe_nr_irqs(void)
3432{
3433 int nr;
3434
3435 if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
3436 nr_irqs = NR_VECTORS * nr_cpu_ids;
3437
3438 nr = nr_irqs_gsi + 8 * nr_cpu_ids;
3439#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
3440
3441
3442
3443 nr += nr_irqs_gsi * 16;
3444#endif
3445 if (nr < nr_irqs)
3446 nr_irqs = nr;
3447
3448 return NR_IRQS_LEGACY;
3449}
3450
3451int io_apic_set_pci_routing(struct device *dev, int irq,
3452 struct io_apic_irq_attr *irq_attr)
3453{
3454 int node;
3455
3456 if (!IO_APIC_IRQ(irq)) {
3457 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3458 irq_attr->ioapic);
3459 return -EINVAL;
3460 }
3461
3462 node = dev ? dev_to_node(dev) : cpu_to_node(0);
3463
3464 return io_apic_setup_irq_pin_once(irq, node, irq_attr);
3465}
3466
3467#ifdef CONFIG_X86_32
3468static int __init io_apic_get_unique_id(int ioapic, int apic_id)
3469{
3470 union IO_APIC_reg_00 reg_00;
3471 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
3472 physid_mask_t tmp;
3473 unsigned long flags;
3474 int i = 0;
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485 if (physids_empty(apic_id_map))
3486 apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map);
3487
3488 raw_spin_lock_irqsave(&ioapic_lock, flags);
3489 reg_00.raw = io_apic_read(ioapic, 0);
3490 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3491
3492 if (apic_id >= get_physical_broadcast()) {
3493 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
3494 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3495 apic_id = reg_00.bits.ID;
3496 }
3497
3498
3499
3500
3501
3502 if (apic->check_apicid_used(&apic_id_map, apic_id)) {
3503
3504 for (i = 0; i < get_physical_broadcast(); i++) {
3505 if (!apic->check_apicid_used(&apic_id_map, i))
3506 break;
3507 }
3508
3509 if (i == get_physical_broadcast())
3510 panic("Max apic_id exceeded!\n");
3511
3512 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
3513 "trying %d\n", ioapic, apic_id, i);
3514
3515 apic_id = i;
3516 }
3517
3518 apic->apicid_to_cpu_present(apic_id, &tmp);
3519 physids_or(apic_id_map, apic_id_map, tmp);
3520
3521 if (reg_00.bits.ID != apic_id) {
3522 reg_00.bits.ID = apic_id;
3523
3524 raw_spin_lock_irqsave(&ioapic_lock, flags);
3525 io_apic_write(ioapic, 0, reg_00.raw);
3526 reg_00.raw = io_apic_read(ioapic, 0);
3527 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3528
3529
3530 if (reg_00.bits.ID != apic_id) {
3531 pr_err("IOAPIC[%d]: Unable to change apic_id!\n",
3532 ioapic);
3533 return -1;
3534 }
3535 }
3536
3537 apic_printk(APIC_VERBOSE, KERN_INFO
3538 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3539
3540 return apic_id;
3541}
3542
3543static u8 __init io_apic_unique_id(u8 id)
3544{
3545 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
3546 !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
3547 return io_apic_get_unique_id(nr_ioapics, id);
3548 else
3549 return id;
3550}
3551#else
3552static u8 __init io_apic_unique_id(u8 id)
3553{
3554 int i;
3555 DECLARE_BITMAP(used, 256);
3556
3557 bitmap_zero(used, 256);
3558 for (i = 0; i < nr_ioapics; i++) {
3559 __set_bit(mpc_ioapic_id(i), used);
3560 }
3561 if (!test_bit(id, used))
3562 return id;
3563 return find_first_zero_bit(used, 256);
3564}
3565#endif
3566
3567static int __init io_apic_get_version(int ioapic)
3568{
3569 union IO_APIC_reg_01 reg_01;
3570 unsigned long flags;
3571
3572 raw_spin_lock_irqsave(&ioapic_lock, flags);
3573 reg_01.raw = io_apic_read(ioapic, 1);
3574 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3575
3576 return reg_01.bits.version;
3577}
3578
3579int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity)
3580{
3581 int ioapic, pin, idx;
3582
3583 if (skip_ioapic_setup)
3584 return -1;
3585
3586 ioapic = mp_find_ioapic(gsi);
3587 if (ioapic < 0)
3588 return -1;
3589
3590 pin = mp_find_ioapic_pin(ioapic, gsi);
3591 if (pin < 0)
3592 return -1;
3593
3594 idx = find_irq_entry(ioapic, pin, mp_INT);
3595 if (idx < 0)
3596 return -1;
3597
3598 *trigger = irq_trigger(idx);
3599 *polarity = irq_polarity(idx);
3600 return 0;
3601}
3602
3603
3604
3605
3606
3607
3608#ifdef CONFIG_SMP
3609void __init setup_ioapic_dest(void)
3610{
3611 int pin, ioapic, irq, irq_entry;
3612 const struct cpumask *mask;
3613 struct irq_data *idata;
3614
3615 if (skip_ioapic_setup == 1)
3616 return;
3617
3618 for (ioapic = 0; ioapic < nr_ioapics; ioapic++)
3619 for (pin = 0; pin < ioapics[ioapic].nr_registers; pin++) {
3620 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
3621 if (irq_entry == -1)
3622 continue;
3623 irq = pin_2_irq(irq_entry, ioapic, pin);
3624
3625 if ((ioapic > 0) && (irq > 16))
3626 continue;
3627
3628 idata = irq_get_irq_data(irq);
3629
3630
3631
3632
3633 if (!irqd_can_balance(idata) || irqd_affinity_was_set(idata))
3634 mask = idata->affinity;
3635 else
3636 mask = apic->target_cpus();
3637
3638 x86_io_apic_ops.set_affinity(idata, mask, false);
3639 }
3640
3641}
3642#endif
3643
3644#define IOAPIC_RESOURCE_NAME_SIZE 11
3645
3646static struct resource *ioapic_resources;
3647
3648static struct resource * __init ioapic_setup_resources(int nr_ioapics)
3649{
3650 unsigned long n;
3651 struct resource *res;
3652 char *mem;
3653 int i;
3654
3655 if (nr_ioapics <= 0)
3656 return NULL;
3657
3658 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
3659 n *= nr_ioapics;
3660
3661 mem = alloc_bootmem(n);
3662 res = (void *)mem;
3663
3664 mem += sizeof(struct resource) * nr_ioapics;
3665
3666 for (i = 0; i < nr_ioapics; i++) {
3667 res[i].name = mem;
3668 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
3669 snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
3670 mem += IOAPIC_RESOURCE_NAME_SIZE;
3671 }
3672
3673 ioapic_resources = res;
3674
3675 return res;
3676}
3677
3678void __init native_io_apic_init_mappings(void)
3679{
3680 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3681 struct resource *ioapic_res;
3682 int i;
3683
3684 ioapic_res = ioapic_setup_resources(nr_ioapics);
3685 for (i = 0; i < nr_ioapics; i++) {
3686 if (smp_found_config) {
3687 ioapic_phys = mpc_ioapic_addr(i);
3688#ifdef CONFIG_X86_32
3689 if (!ioapic_phys) {
3690 printk(KERN_ERR
3691 "WARNING: bogus zero IO-APIC "
3692 "address found in MPTABLE, "
3693 "disabling IO/APIC support!\n");
3694 smp_found_config = 0;
3695 skip_ioapic_setup = 1;
3696 goto fake_ioapic_page;
3697 }
3698#endif
3699 } else {
3700#ifdef CONFIG_X86_32
3701fake_ioapic_page:
3702#endif
3703 ioapic_phys = (unsigned long)alloc_bootmem_pages(PAGE_SIZE);
3704 ioapic_phys = __pa(ioapic_phys);
3705 }
3706 set_fixmap_nocache(idx, ioapic_phys);
3707 apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n",
3708 __fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK),
3709 ioapic_phys);
3710 idx++;
3711
3712 ioapic_res->start = ioapic_phys;
3713 ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
3714 ioapic_res++;
3715 }
3716
3717 probe_nr_irqs_gsi();
3718}
3719
3720void __init ioapic_insert_resources(void)
3721{
3722 int i;
3723 struct resource *r = ioapic_resources;
3724
3725 if (!r) {
3726 if (nr_ioapics > 0)
3727 printk(KERN_ERR
3728 "IO APIC resources couldn't be allocated.\n");
3729 return;
3730 }
3731
3732 for (i = 0; i < nr_ioapics; i++) {
3733 insert_resource(&iomem_resource, r);
3734 r++;
3735 }
3736}
3737
3738int mp_find_ioapic(u32 gsi)
3739{
3740 int i = 0;
3741
3742 if (nr_ioapics == 0)
3743 return -1;
3744
3745
3746 for (i = 0; i < nr_ioapics; i++) {
3747 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
3748 if ((gsi >= gsi_cfg->gsi_base)
3749 && (gsi <= gsi_cfg->gsi_end))
3750 return i;
3751 }
3752
3753 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
3754 return -1;
3755}
3756
3757int mp_find_ioapic_pin(int ioapic, u32 gsi)
3758{
3759 struct mp_ioapic_gsi *gsi_cfg;
3760
3761 if (WARN_ON(ioapic == -1))
3762 return -1;
3763
3764 gsi_cfg = mp_ioapic_gsi_routing(ioapic);
3765 if (WARN_ON(gsi > gsi_cfg->gsi_end))
3766 return -1;
3767
3768 return gsi - gsi_cfg->gsi_base;
3769}
3770
3771static __init int bad_ioapic(unsigned long address)
3772{
3773 if (nr_ioapics >= MAX_IO_APICS) {
3774 pr_warn("WARNING: Max # of I/O APICs (%d) exceeded (found %d), skipping\n",
3775 MAX_IO_APICS, nr_ioapics);
3776 return 1;
3777 }
3778 if (!address) {
3779 pr_warn("WARNING: Bogus (zero) I/O APIC address found in table, skipping!\n");
3780 return 1;
3781 }
3782 return 0;
3783}
3784
3785static __init int bad_ioapic_register(int idx)
3786{
3787 union IO_APIC_reg_00 reg_00;
3788 union IO_APIC_reg_01 reg_01;
3789 union IO_APIC_reg_02 reg_02;
3790
3791 reg_00.raw = io_apic_read(idx, 0);
3792 reg_01.raw = io_apic_read(idx, 1);
3793 reg_02.raw = io_apic_read(idx, 2);
3794
3795 if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) {
3796 pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n",
3797 mpc_ioapic_addr(idx));
3798 return 1;
3799 }
3800
3801 return 0;
3802}
3803
3804void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
3805{
3806 int idx = 0;
3807 int entries;
3808 struct mp_ioapic_gsi *gsi_cfg;
3809
3810 if (bad_ioapic(address))
3811 return;
3812
3813 idx = nr_ioapics;
3814
3815 ioapics[idx].mp_config.type = MP_IOAPIC;
3816 ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
3817 ioapics[idx].mp_config.apicaddr = address;
3818
3819 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
3820
3821 if (bad_ioapic_register(idx)) {
3822 clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
3823 return;
3824 }
3825
3826 ioapics[idx].mp_config.apicid = io_apic_unique_id(id);
3827 ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
3828
3829
3830
3831
3832
3833 entries = io_apic_get_redir_entries(idx);
3834 gsi_cfg = mp_ioapic_gsi_routing(idx);
3835 gsi_cfg->gsi_base = gsi_base;
3836 gsi_cfg->gsi_end = gsi_base + entries - 1;
3837
3838
3839
3840
3841 ioapics[idx].nr_registers = entries;
3842
3843 if (gsi_cfg->gsi_end >= gsi_top)
3844 gsi_top = gsi_cfg->gsi_end + 1;
3845
3846 pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n",
3847 idx, mpc_ioapic_id(idx),
3848 mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
3849 gsi_cfg->gsi_base, gsi_cfg->gsi_end);
3850
3851 nr_ioapics++;
3852}
3853
3854
3855void __init pre_init_apic_IRQ0(void)
3856{
3857 struct io_apic_irq_attr attr = { 0, 0, 0, 0 };
3858
3859 printk(KERN_INFO "Early APIC setup for system timer0\n");
3860#ifndef CONFIG_SMP
3861 physid_set_mask_of_physid(boot_cpu_physical_apicid,
3862 &phys_cpu_present_map);
3863#endif
3864 setup_local_APIC();
3865
3866 io_apic_setup_irq_pin(0, 0, &attr);
3867 irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
3868 "edge");
3869}
3870