blob: 68f8859139272420fa88417804cae535d2d10fdd [file] [log] [blame]
Jiang Liu74afab72014-10-27 16:12:00 +08001/*
2 * Local APIC related interfaces to support IOAPIC, MSI, HT_IRQ etc.
3 *
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 * Moved from arch/x86/kernel/apic/io_apic.c.
Jiang Liub5dc8e62015-04-13 14:11:24 +08006 * Jiang Liu <jiang.liu@linux.intel.com>
7 * Enable support of hierarchical irqdomains
Jiang Liu74afab72014-10-27 16:12:00 +08008 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13#include <linux/interrupt.h>
14#include <linux/init.h>
15#include <linux/compiler.h>
Jiang Liu74afab72014-10-27 16:12:00 +080016#include <linux/slab.h>
Jiang Liud746d1e2015-04-14 10:30:09 +080017#include <asm/irqdomain.h>
Jiang Liu74afab72014-10-27 16:12:00 +080018#include <asm/hw_irq.h>
19#include <asm/apic.h>
20#include <asm/i8259.h>
21#include <asm/desc.h>
22#include <asm/irq_remapping.h>
23
Jiang Liu7f3262e2015-04-14 10:30:03 +080024struct apic_chip_data {
25 struct irq_cfg cfg;
Thomas Gleixner029c6e12017-09-13 23:29:31 +020026 unsigned int cpu;
27 unsigned int prev_cpu;
Thomas Gleixnerdccfe312017-09-13 23:29:32 +020028 struct hlist_node clist;
Jiang Liu7f3262e2015-04-14 10:30:03 +080029 cpumask_var_t domain;
30 cpumask_var_t old_domain;
31 u8 move_in_progress : 1;
32};
33
Jiang Liub5dc8e62015-04-13 14:11:24 +080034struct irq_domain *x86_vector_domain;
Jake Oshinsc8f3e512015-12-10 17:52:59 +000035EXPORT_SYMBOL_GPL(x86_vector_domain);
Jiang Liu74afab72014-10-27 16:12:00 +080036static DEFINE_RAW_SPINLOCK(vector_lock);
Thomas Gleixner3716fd22015-12-31 16:30:48 +000037static cpumask_var_t vector_cpumask, vector_searchmask, searched_cpumask;
Jiang Liub5dc8e62015-04-13 14:11:24 +080038static struct irq_chip lapic_controller;
Jiang Liu13315322015-04-13 14:11:56 +080039#ifdef CONFIG_X86_IO_APIC
Jiang Liu7f3262e2015-04-14 10:30:03 +080040static struct apic_chip_data *legacy_irq_data[NR_IRQS_LEGACY];
Jiang Liu13315322015-04-13 14:11:56 +080041#endif
Thomas Gleixnerdccfe312017-09-13 23:29:32 +020042#ifdef CONFIG_SMP
43static DEFINE_PER_CPU(struct hlist_head, cleanup_list);
44#endif
Jiang Liu74afab72014-10-27 16:12:00 +080045
46void lock_vector_lock(void)
47{
48 /* Used to the online set of cpus does not change
49 * during assign_irq_vector.
50 */
51 raw_spin_lock(&vector_lock);
52}
53
54void unlock_vector_lock(void)
55{
56 raw_spin_unlock(&vector_lock);
57}
58
Thomas Gleixner86ba6552017-09-13 23:29:30 +020059static struct apic_chip_data *apic_chip_data(struct irq_data *irqd)
Jiang Liu74afab72014-10-27 16:12:00 +080060{
Thomas Gleixner86ba6552017-09-13 23:29:30 +020061 if (!irqd)
Jiang Liub5dc8e62015-04-13 14:11:24 +080062 return NULL;
63
Thomas Gleixner86ba6552017-09-13 23:29:30 +020064 while (irqd->parent_data)
65 irqd = irqd->parent_data;
Jiang Liub5dc8e62015-04-13 14:11:24 +080066
Thomas Gleixner86ba6552017-09-13 23:29:30 +020067 return irqd->chip_data;
Jiang Liu74afab72014-10-27 16:12:00 +080068}
69
Thomas Gleixner86ba6552017-09-13 23:29:30 +020070struct irq_cfg *irqd_cfg(struct irq_data *irqd)
Jiang Liu74afab72014-10-27 16:12:00 +080071{
Thomas Gleixner86ba6552017-09-13 23:29:30 +020072 struct apic_chip_data *apicd = apic_chip_data(irqd);
Jiang Liu74afab72014-10-27 16:12:00 +080073
Thomas Gleixner86ba6552017-09-13 23:29:30 +020074 return apicd ? &apicd->cfg : NULL;
Jiang Liu7f3262e2015-04-14 10:30:03 +080075}
Jake Oshinsc8f3e512015-12-10 17:52:59 +000076EXPORT_SYMBOL_GPL(irqd_cfg);
Jiang Liu7f3262e2015-04-14 10:30:03 +080077
78struct irq_cfg *irq_cfg(unsigned int irq)
79{
80 return irqd_cfg(irq_get_irq_data(irq));
81}
82
83static struct apic_chip_data *alloc_apic_chip_data(int node)
84{
Thomas Gleixner86ba6552017-09-13 23:29:30 +020085 struct apic_chip_data *apicd;
Jiang Liu7f3262e2015-04-14 10:30:03 +080086
Thomas Gleixner86ba6552017-09-13 23:29:30 +020087 apicd = kzalloc_node(sizeof(*apicd), GFP_KERNEL, node);
88 if (!apicd)
Jiang Liu74afab72014-10-27 16:12:00 +080089 return NULL;
Thomas Gleixner86ba6552017-09-13 23:29:30 +020090 if (!zalloc_cpumask_var_node(&apicd->domain, GFP_KERNEL, node))
Jiang Liu7f3262e2015-04-14 10:30:03 +080091 goto out_data;
Thomas Gleixner86ba6552017-09-13 23:29:30 +020092 if (!zalloc_cpumask_var_node(&apicd->old_domain, GFP_KERNEL, node))
Jiang Liu74afab72014-10-27 16:12:00 +080093 goto out_domain;
Thomas Gleixnerdccfe312017-09-13 23:29:32 +020094 INIT_HLIST_NODE(&apicd->clist);
Thomas Gleixner86ba6552017-09-13 23:29:30 +020095 return apicd;
Jiang Liu74afab72014-10-27 16:12:00 +080096out_domain:
Thomas Gleixner86ba6552017-09-13 23:29:30 +020097 free_cpumask_var(apicd->domain);
Jiang Liu7f3262e2015-04-14 10:30:03 +080098out_data:
Thomas Gleixner86ba6552017-09-13 23:29:30 +020099 kfree(apicd);
Jiang Liu74afab72014-10-27 16:12:00 +0800100 return NULL;
101}
102
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200103static void free_apic_chip_data(struct apic_chip_data *apicd)
Jiang Liu74afab72014-10-27 16:12:00 +0800104{
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200105 if (apicd) {
106 free_cpumask_var(apicd->domain);
107 free_cpumask_var(apicd->old_domain);
108 kfree(apicd);
Jiang Liub5dc8e62015-04-13 14:11:24 +0800109 }
Jiang Liu74afab72014-10-27 16:12:00 +0800110}
111
Jiang Liu7f3262e2015-04-14 10:30:03 +0800112static int __assign_irq_vector(int irq, struct apic_chip_data *d,
Thomas Gleixner0e24f7c2017-06-20 01:37:44 +0200113 const struct cpumask *mask,
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200114 struct irq_data *irqd)
Jiang Liu74afab72014-10-27 16:12:00 +0800115{
116 /*
117 * NOTE! The local APIC isn't very good at handling
118 * multiple interrupts at the same interrupt level.
119 * As the interrupt level is determined by taking the
120 * vector number and shifting that right by 4, we
121 * want to spread these out a bit so that they don't
122 * all fall in the same interrupt level.
123 *
124 * Also, we've got to be careful not to trash gate
125 * 0x80, because int 0x80 is hm, kind of importantish. ;)
126 */
127 static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START;
128 static int current_offset = VECTOR_OFFSET_START % 16;
Thomas Gleixnerab25ac02015-12-31 16:30:49 +0000129 int cpu, vector;
Jiang Liu74afab72014-10-27 16:12:00 +0800130
Thomas Gleixner98229aa2015-12-31 16:30:54 +0000131 /*
132 * If there is still a move in progress or the previous move has not
133 * been cleaned up completely, tell the caller to come back later.
134 */
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200135 if (d->cfg.old_vector)
Jiang Liu74afab72014-10-27 16:12:00 +0800136 return -EBUSY;
137
Jiang Liu74afab72014-10-27 16:12:00 +0800138 /* Only try and allocate irqs on cpus that are present */
Jiang Liu7f3262e2015-04-14 10:30:03 +0800139 cpumask_clear(d->old_domain);
Jiang Liu8a580f72015-12-31 16:30:46 +0000140 cpumask_clear(searched_cpumask);
Jiang Liu74afab72014-10-27 16:12:00 +0800141 cpu = cpumask_first_and(mask, cpu_online_mask);
142 while (cpu < nr_cpu_ids) {
Thomas Gleixnerab25ac02015-12-31 16:30:49 +0000143 int new_cpu, offset;
Jiang Liu74afab72014-10-27 16:12:00 +0800144
Thomas Gleixnerfdba46f2017-09-13 23:29:27 +0200145 cpumask_copy(vector_cpumask, cpumask_of(cpu));
Jiang Liu74afab72014-10-27 16:12:00 +0800146
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000147 /*
148 * Clear the offline cpus from @vector_cpumask for searching
149 * and verify whether the result overlaps with @mask. If true,
Thomas Gleixner91cd9cb2017-06-20 01:37:43 +0200150 * then the call to apic->cpu_mask_to_apicid() will
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000151 * succeed as well. If not, no point in trying to find a
152 * vector in this mask.
153 */
154 cpumask_and(vector_searchmask, vector_cpumask, cpu_online_mask);
155 if (!cpumask_intersects(vector_searchmask, mask))
156 goto next_cpu;
157
Jiang Liuf7fa7ae2015-04-14 10:30:10 +0800158 if (cpumask_subset(vector_cpumask, d->domain)) {
Jiang Liuf7fa7ae2015-04-14 10:30:10 +0800159 if (cpumask_equal(vector_cpumask, d->domain))
Thomas Gleixner433cbd52015-12-31 16:30:46 +0000160 goto success;
Jiang Liu74afab72014-10-27 16:12:00 +0800161 /*
Thomas Gleixnerab25ac02015-12-31 16:30:49 +0000162 * Mark the cpus which are not longer in the mask for
163 * cleanup.
Jiang Liu74afab72014-10-27 16:12:00 +0800164 */
Thomas Gleixnerab25ac02015-12-31 16:30:49 +0000165 cpumask_andnot(d->old_domain, d->domain, vector_cpumask);
166 vector = d->cfg.vector;
167 goto update;
Jiang Liu74afab72014-10-27 16:12:00 +0800168 }
169
170 vector = current_vector;
171 offset = current_offset;
172next:
173 vector += 16;
Thomas Gleixner05161b92017-08-28 08:47:18 +0200174 if (vector >= FIRST_SYSTEM_VECTOR) {
Jiang Liu74afab72014-10-27 16:12:00 +0800175 offset = (offset + 1) % 16;
176 vector = FIRST_EXTERNAL_VECTOR + offset;
177 }
178
Thomas Gleixner95ffeb42015-12-31 16:30:47 +0000179 /* If the search wrapped around, try the next cpu */
180 if (unlikely(current_vector == vector))
181 goto next_cpu;
Jiang Liu74afab72014-10-27 16:12:00 +0800182
Thomas Gleixner7854f822017-09-13 23:29:26 +0200183 if (test_bit(vector, system_vectors))
Jiang Liu74afab72014-10-27 16:12:00 +0800184 goto next;
185
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000186 for_each_cpu(new_cpu, vector_searchmask) {
Thomas Gleixnera782a7e2015-08-02 20:38:27 +0000187 if (!IS_ERR_OR_NULL(per_cpu(vector_irq, new_cpu)[vector]))
Jiang Liu74afab72014-10-27 16:12:00 +0800188 goto next;
189 }
190 /* Found one! */
191 current_vector = vector;
192 current_offset = offset;
Thomas Gleixnerab25ac02015-12-31 16:30:49 +0000193 /* Schedule the old vector for cleanup on all cpus */
194 if (d->cfg.vector)
Jiang Liu7f3262e2015-04-14 10:30:03 +0800195 cpumask_copy(d->old_domain, d->domain);
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000196 for_each_cpu(new_cpu, vector_searchmask)
Thomas Gleixnera782a7e2015-08-02 20:38:27 +0000197 per_cpu(vector_irq, new_cpu)[vector] = irq_to_desc(irq);
Thomas Gleixnerab25ac02015-12-31 16:30:49 +0000198 goto update;
Thomas Gleixner95ffeb42015-12-31 16:30:47 +0000199
200next_cpu:
201 /*
202 * We exclude the current @vector_cpumask from the requested
203 * @mask and try again with the next online cpu in the
204 * result. We cannot modify @mask, so we use @vector_cpumask
205 * as a temporary buffer here as it will be reassigned when
206 * calling apic->vector_allocation_domain() above.
207 */
208 cpumask_or(searched_cpumask, searched_cpumask, vector_cpumask);
209 cpumask_andnot(vector_cpumask, mask, searched_cpumask);
210 cpu = cpumask_first_and(vector_cpumask, cpu_online_mask);
211 continue;
Jiang Liu74afab72014-10-27 16:12:00 +0800212 }
Thomas Gleixner433cbd52015-12-31 16:30:46 +0000213 return -ENOSPC;
Jiang Liu74afab72014-10-27 16:12:00 +0800214
Thomas Gleixnerab25ac02015-12-31 16:30:49 +0000215update:
Thomas Gleixner847667e2015-12-31 16:30:50 +0000216 /*
217 * Exclude offline cpus from the cleanup mask and set the
218 * move_in_progress flag when the result is not empty.
219 */
220 cpumask_and(d->old_domain, d->old_domain, cpu_online_mask);
221 d->move_in_progress = !cpumask_empty(d->old_domain);
Thomas Gleixner551adc62016-03-14 09:40:46 +0100222 d->cfg.old_vector = d->move_in_progress ? d->cfg.vector : 0;
Thomas Gleixner029c6e12017-09-13 23:29:31 +0200223 d->prev_cpu = d->cpu;
Thomas Gleixnerab25ac02015-12-31 16:30:49 +0000224 d->cfg.vector = vector;
225 cpumask_copy(d->domain, vector_cpumask);
Thomas Gleixner433cbd52015-12-31 16:30:46 +0000226success:
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000227 /*
228 * Cache destination APIC IDs into cfg->dest_apicid. This cannot fail
229 * as we already established, that mask & d->domain & cpu_online_mask
230 * is not empty.
Thomas Gleixner52b166a2017-06-20 01:37:42 +0200231 *
232 * vector_searchmask is a subset of d->domain and has the offline
233 * cpus masked out.
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000234 */
Thomas Gleixner91cd9cb2017-06-20 01:37:43 +0200235 cpumask_and(vector_searchmask, vector_searchmask, mask);
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200236 BUG_ON(apic->cpu_mask_to_apicid(vector_searchmask, irqd,
Thomas Gleixner0e24f7c2017-06-20 01:37:44 +0200237 &d->cfg.dest_apicid));
Thomas Gleixner029c6e12017-09-13 23:29:31 +0200238 d->cpu = cpumask_first(vector_searchmask);
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000239 return 0;
Jiang Liu74afab72014-10-27 16:12:00 +0800240}
241
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200242static int assign_irq_vector(int irq, struct apic_chip_data *apicd,
Thomas Gleixner0e24f7c2017-06-20 01:37:44 +0200243 const struct cpumask *mask,
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200244 struct irq_data *irqd)
Jiang Liu74afab72014-10-27 16:12:00 +0800245{
246 int err;
247 unsigned long flags;
248
249 raw_spin_lock_irqsave(&vector_lock, flags);
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200250 err = __assign_irq_vector(irq, apicd, mask, irqd);
Jiang Liu74afab72014-10-27 16:12:00 +0800251 raw_spin_unlock_irqrestore(&vector_lock, flags);
252 return err;
253}
254
Jiang Liu486ca532015-05-07 10:53:56 +0800255static int assign_irq_vector_policy(int irq, int node,
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200256 struct apic_chip_data *apicd,
Thomas Gleixner0e24f7c2017-06-20 01:37:44 +0200257 struct irq_alloc_info *info,
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200258 struct irq_data *irqd)
Jiang Liu486ca532015-05-07 10:53:56 +0800259{
260 if (info && info->mask)
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200261 return assign_irq_vector(irq, apicd, info->mask, irqd);
Jiang Liu486ca532015-05-07 10:53:56 +0800262 if (node != NUMA_NO_NODE &&
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200263 assign_irq_vector(irq, apicd, cpumask_of_node(node), irqd) == 0)
Jiang Liu486ca532015-05-07 10:53:56 +0800264 return 0;
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200265 return assign_irq_vector(irq, apicd, cpu_online_mask, irqd);
Jiang Liu486ca532015-05-07 10:53:56 +0800266}
267
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200268static void clear_irq_vector(int irq, struct apic_chip_data *apicd)
Jiang Liu74afab72014-10-27 16:12:00 +0800269{
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200270 unsigned int vector = apicd->cfg.vector;
Jiang Liu74afab72014-10-27 16:12:00 +0800271
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200272 if (!vector)
Keith Busch1bdb8972016-04-27 14:22:32 -0600273 return;
Jiang Liu74afab72014-10-27 16:12:00 +0800274
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200275 per_cpu(vector_irq, apicd->cpu)[vector] = VECTOR_UNUSED;
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200276 apicd->cfg.vector = 0;
Jiang Liu74afab72014-10-27 16:12:00 +0800277
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200278 /* Clean up move in progress */
279 vector = apicd->cfg.old_vector;
280 if (!vector)
Jiang Liu74afab72014-10-27 16:12:00 +0800281 return;
Jiang Liu74afab72014-10-27 16:12:00 +0800282
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200283 per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_UNUSED;
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200284 apicd->move_in_progress = 0;
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200285 hlist_del_init(&apicd->clist);
Jiang Liu74afab72014-10-27 16:12:00 +0800286}
287
Jiang Liub5dc8e62015-04-13 14:11:24 +0800288void init_irq_alloc_info(struct irq_alloc_info *info,
289 const struct cpumask *mask)
290{
291 memset(info, 0, sizeof(*info));
292 info->mask = mask;
293}
294
295void copy_irq_alloc_info(struct irq_alloc_info *dst, struct irq_alloc_info *src)
296{
297 if (src)
298 *dst = *src;
299 else
300 memset(dst, 0, sizeof(*dst));
301}
302
Jiang Liub5dc8e62015-04-13 14:11:24 +0800303static void x86_vector_free_irqs(struct irq_domain *domain,
304 unsigned int virq, unsigned int nr_irqs)
305{
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200306 struct apic_chip_data *apicd;
307 struct irq_data *irqd;
Jiang Liu111abeb2015-12-31 16:30:44 +0000308 unsigned long flags;
Jiang Liub5dc8e62015-04-13 14:11:24 +0800309 int i;
310
311 for (i = 0; i < nr_irqs; i++) {
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200312 irqd = irq_domain_get_irq_data(x86_vector_domain, virq + i);
313 if (irqd && irqd->chip_data) {
Jiang Liu111abeb2015-12-31 16:30:44 +0000314 raw_spin_lock_irqsave(&vector_lock, flags);
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200315 clear_irq_vector(virq + i, irqd->chip_data);
316 apicd = irqd->chip_data;
317 irq_domain_reset_irq_data(irqd);
Jiang Liu111abeb2015-12-31 16:30:44 +0000318 raw_spin_unlock_irqrestore(&vector_lock, flags);
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200319 free_apic_chip_data(apicd);
Jiang Liu13315322015-04-13 14:11:56 +0800320#ifdef CONFIG_X86_IO_APIC
321 if (virq + i < nr_legacy_irqs())
Jiang Liu7f3262e2015-04-14 10:30:03 +0800322 legacy_irq_data[virq + i] = NULL;
Jiang Liu13315322015-04-13 14:11:56 +0800323#endif
Jiang Liub5dc8e62015-04-13 14:11:24 +0800324 }
325 }
326}
327
328static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq,
329 unsigned int nr_irqs, void *arg)
330{
331 struct irq_alloc_info *info = arg;
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200332 struct apic_chip_data *apicd;
333 struct irq_data *irqd;
Jiang Liu5f2dbbc2015-06-01 16:05:14 +0800334 int i, err, node;
Jiang Liub5dc8e62015-04-13 14:11:24 +0800335
336 if (disable_apic)
337 return -ENXIO;
338
339 /* Currently vector allocator can't guarantee contiguous allocations */
340 if ((info->flags & X86_IRQ_ALLOC_CONTIGUOUS_VECTORS) && nr_irqs > 1)
341 return -ENOSYS;
342
Jiang Liub5dc8e62015-04-13 14:11:24 +0800343 for (i = 0; i < nr_irqs; i++) {
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200344 irqd = irq_domain_get_irq_data(domain, virq + i);
345 BUG_ON(!irqd);
346 node = irq_data_get_node(irqd);
Jiang Liu13315322015-04-13 14:11:56 +0800347#ifdef CONFIG_X86_IO_APIC
Jiang Liu7f3262e2015-04-14 10:30:03 +0800348 if (virq + i < nr_legacy_irqs() && legacy_irq_data[virq + i])
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200349 apicd = legacy_irq_data[virq + i];
Jiang Liu13315322015-04-13 14:11:56 +0800350 else
351#endif
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200352 apicd = alloc_apic_chip_data(node);
353 if (!apicd) {
Jiang Liub5dc8e62015-04-13 14:11:24 +0800354 err = -ENOMEM;
355 goto error;
356 }
357
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200358 irqd->chip = &lapic_controller;
359 irqd->chip_data = apicd;
360 irqd->hwirq = virq + i;
361 irqd_set_single_target(irqd);
362 err = assign_irq_vector_policy(virq + i, node, apicd, info,
363 irqd);
Jiang Liub5dc8e62015-04-13 14:11:24 +0800364 if (err)
365 goto error;
366 }
367
368 return 0;
369
370error:
371 x86_vector_free_irqs(domain, virq, i + 1);
372 return err;
373}
374
Thomas Gleixnereb18cf52015-05-05 11:10:11 +0200375static const struct irq_domain_ops x86_vector_domain_ops = {
376 .alloc = x86_vector_alloc_irqs,
377 .free = x86_vector_free_irqs,
Jiang Liub5dc8e62015-04-13 14:11:24 +0800378};
379
Jiang Liu11d686e2014-10-27 16:12:05 +0800380int __init arch_probe_nr_irqs(void)
381{
382 int nr;
383
384 if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
385 nr_irqs = NR_VECTORS * nr_cpu_ids;
386
387 nr = (gsi_top + nr_legacy_irqs()) + 8 * nr_cpu_ids;
388#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
389 /*
390 * for MSI and HT dyn irq
391 */
392 if (gsi_top <= NR_IRQS_LEGACY)
393 nr += 8 * nr_cpu_ids;
394 else
395 nr += gsi_top * 16;
396#endif
397 if (nr < nr_irqs)
398 nr_irqs = nr;
399
Vitaly Kuznetsov8c058b02015-11-03 10:40:14 +0100400 /*
401 * We don't know if PIC is present at this point so we need to do
402 * probe() to get the right number of legacy IRQs.
403 */
404 return legacy_pic->probe();
Jiang Liu11d686e2014-10-27 16:12:05 +0800405}
406
Jiang Liu13315322015-04-13 14:11:56 +0800407#ifdef CONFIG_X86_IO_APIC
Dou Liyanga884d252017-06-21 18:14:21 +0800408static void __init init_legacy_irqs(void)
Jiang Liu13315322015-04-13 14:11:56 +0800409{
410 int i, node = cpu_to_node(0);
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200411 struct apic_chip_data *apicd;
Jiang Liu13315322015-04-13 14:11:56 +0800412
413 /*
414 * For legacy IRQ's, start with assigning irq0 to irq15 to
Ingo Molnar191a66352015-05-11 16:05:09 +0200415 * ISA_IRQ_VECTOR(i) for all cpu's.
Jiang Liu13315322015-04-13 14:11:56 +0800416 */
417 for (i = 0; i < nr_legacy_irqs(); i++) {
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200418 apicd = legacy_irq_data[i] = alloc_apic_chip_data(node);
419 BUG_ON(!apicd);
Ingo Molnar191a66352015-05-11 16:05:09 +0200420
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200421 apicd->cfg.vector = ISA_IRQ_VECTOR(i);
422 cpumask_copy(apicd->domain, cpumask_of(0));
Thomas Gleixner029c6e12017-09-13 23:29:31 +0200423 apicd->cpu = 0;
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200424 irq_set_chip_data(i, apicd);
Jiang Liu13315322015-04-13 14:11:56 +0800425 }
426}
427#else
Dou Liyanga884d252017-06-21 18:14:21 +0800428static inline void init_legacy_irqs(void) { }
Jiang Liu13315322015-04-13 14:11:56 +0800429#endif
430
Jiang Liu11d686e2014-10-27 16:12:05 +0800431int __init arch_early_irq_init(void)
432{
Thomas Gleixner9d35f852017-06-20 01:37:06 +0200433 struct fwnode_handle *fn;
434
Jiang Liu13315322015-04-13 14:11:56 +0800435 init_legacy_irqs();
436
Thomas Gleixner9d35f852017-06-20 01:37:06 +0200437 fn = irq_domain_alloc_named_fwnode("VECTOR");
438 BUG_ON(!fn);
439 x86_vector_domain = irq_domain_create_tree(fn, &x86_vector_domain_ops,
440 NULL);
Jiang Liub5dc8e62015-04-13 14:11:24 +0800441 BUG_ON(x86_vector_domain == NULL);
Thomas Gleixner9d35f852017-06-20 01:37:06 +0200442 irq_domain_free_fwnode(fn);
Jiang Liub5dc8e62015-04-13 14:11:24 +0800443 irq_set_default_host(x86_vector_domain);
444
Jiang Liu52f518a2015-04-13 14:11:35 +0800445 arch_init_msi_domain(x86_vector_domain);
Jiang Liu49e07d82015-04-13 14:11:43 +0800446 arch_init_htirq_domain(x86_vector_domain);
Jiang Liu52f518a2015-04-13 14:11:35 +0800447
Jiang Liuf7fa7ae2015-04-14 10:30:10 +0800448 BUG_ON(!alloc_cpumask_var(&vector_cpumask, GFP_KERNEL));
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000449 BUG_ON(!alloc_cpumask_var(&vector_searchmask, GFP_KERNEL));
Jiang Liu8a580f72015-12-31 16:30:46 +0000450 BUG_ON(!alloc_cpumask_var(&searched_cpumask, GFP_KERNEL));
Jiang Liuf7fa7ae2015-04-14 10:30:10 +0800451
Jiang Liu11d686e2014-10-27 16:12:05 +0800452 return arch_early_ioapic_init();
453}
454
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200455/* Temporary hack to keep things working */
456static void vector_update_shutdown_irqs(void)
Jiang Liu74afab72014-10-27 16:12:00 +0800457{
Thomas Gleixnera782a7e2015-08-02 20:38:27 +0000458 struct irq_desc *desc;
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200459 int irq;
Jiang Liu74afab72014-10-27 16:12:00 +0800460
Thomas Gleixnera782a7e2015-08-02 20:38:27 +0000461 for_each_irq_desc(irq, desc) {
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200462 struct irq_data *irqd = irq_desc_get_irq_data(desc);
463 struct apic_chip_data *ad = apic_chip_data(irqd);
Jiang Liu74afab72014-10-27 16:12:00 +0800464
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200465 if (ad && ad->cfg.vector && ad->cpu == smp_processor_id())
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200466 this_cpu_write(vector_irq[ad->cfg.vector], desc);
Jiang Liu74afab72014-10-27 16:12:00 +0800467 }
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200468}
Jiang Liu74afab72014-10-27 16:12:00 +0800469
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200470static struct irq_desc *__setup_vector_irq(int vector)
471{
472 int isairq = vector - ISA_IRQ_VECTOR(0);
473
474 /* Check whether the irq is in the legacy space */
475 if (isairq < 0 || isairq >= nr_legacy_irqs())
476 return VECTOR_UNUSED;
477 /* Check whether the irq is handled by the IOAPIC */
478 if (test_bit(isairq, &io_apic_irqs))
479 return VECTOR_UNUSED;
480 return irq_to_desc(isairq);
Jiang Liu74afab72014-10-27 16:12:00 +0800481}
482
483/*
Thomas Gleixner5a3f75e2015-07-05 17:12:32 +0000484 * Setup the vector to irq mappings. Must be called with vector_lock held.
Jiang Liu74afab72014-10-27 16:12:00 +0800485 */
486void setup_vector_irq(int cpu)
487{
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200488 unsigned int vector;
Jiang Liu74afab72014-10-27 16:12:00 +0800489
Thomas Gleixner5a3f75e2015-07-05 17:12:32 +0000490 lockdep_assert_held(&vector_lock);
Jiang Liu74afab72014-10-27 16:12:00 +0800491 /*
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200492 * The interrupt affinity logic never targets interrupts to offline
493 * CPUs. The exception are the legacy PIC interrupts. In general
494 * they are only targeted to CPU0, but depending on the platform
495 * they can be distributed to any online CPU in hardware. The
496 * kernel has no influence on that. So all active legacy vectors
497 * must be installed on all CPUs. All non legacy interrupts can be
498 * cleared.
Jiang Liu74afab72014-10-27 16:12:00 +0800499 */
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200500 for (vector = 0; vector < NR_VECTORS; vector++)
501 this_cpu_write(vector_irq[vector], __setup_vector_irq(vector));
Jiang Liu74afab72014-10-27 16:12:00 +0800502
Thomas Gleixnerf0cc6cc2017-09-13 23:29:29 +0200503 /*
504 * Until the rewrite of the managed interrupt management is in
505 * place it's necessary to walk the irq descriptors and check for
506 * interrupts which are targeted at this CPU.
507 */
508 vector_update_shutdown_irqs();
Jiang Liu74afab72014-10-27 16:12:00 +0800509}
510
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200511static int apic_retrigger_irq(struct irq_data *irqd)
Jiang Liu74afab72014-10-27 16:12:00 +0800512{
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200513 struct apic_chip_data *apicd = apic_chip_data(irqd);
Jiang Liu74afab72014-10-27 16:12:00 +0800514 unsigned long flags;
Jiang Liu74afab72014-10-27 16:12:00 +0800515
516 raw_spin_lock_irqsave(&vector_lock, flags);
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200517 apic->send_IPI(apicd->cpu, apicd->cfg.vector);
Jiang Liu74afab72014-10-27 16:12:00 +0800518 raw_spin_unlock_irqrestore(&vector_lock, flags);
519
520 return 1;
521}
522
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200523void apic_ack_edge(struct irq_data *irqd)
Jiang Liu74afab72014-10-27 16:12:00 +0800524{
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200525 irq_complete_move(irqd_cfg(irqd));
526 irq_move_irq(irqd);
Jiang Liu74afab72014-10-27 16:12:00 +0800527 ack_APIC_irq();
528}
529
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200530static int apic_set_affinity(struct irq_data *irqd,
Jiang Liu68f9f442015-04-14 10:30:01 +0800531 const struct cpumask *dest, bool force)
Jiang Liub5dc8e62015-04-13 14:11:24 +0800532{
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200533 struct apic_chip_data *apicd = irqd->chip_data;
534 int err, irq = irqd->irq;
Jiang Liub5dc8e62015-04-13 14:11:24 +0800535
Masahiro Yamada97f26452016-08-03 13:45:50 -0700536 if (!IS_ENABLED(CONFIG_SMP))
Jiang Liub5dc8e62015-04-13 14:11:24 +0800537 return -EPERM;
538
539 if (!cpumask_intersects(dest, cpu_online_mask))
540 return -EINVAL;
541
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200542 err = assign_irq_vector(irq, apicd, dest, irqd);
Thomas Gleixner3716fd22015-12-31 16:30:48 +0000543 return err ? err : IRQ_SET_MASK_OK;
Jiang Liub5dc8e62015-04-13 14:11:24 +0800544}
545
546static struct irq_chip lapic_controller = {
Thomas Gleixner8947dfb2017-06-20 01:37:01 +0200547 .name = "APIC",
Jiang Liub5dc8e62015-04-13 14:11:24 +0800548 .irq_ack = apic_ack_edge,
Jiang Liu68f9f442015-04-14 10:30:01 +0800549 .irq_set_affinity = apic_set_affinity,
Jiang Liub5dc8e62015-04-13 14:11:24 +0800550 .irq_retrigger = apic_retrigger_irq,
551};
552
Jiang Liu74afab72014-10-27 16:12:00 +0800553#ifdef CONFIG_SMP
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200554
555asmlinkage __visible void __irq_entry smp_irq_move_cleanup_interrupt(void)
556{
557 struct hlist_head *clhead = this_cpu_ptr(&cleanup_list);
558 struct apic_chip_data *apicd;
559 struct hlist_node *tmp;
560
561 entering_ack_irq();
562 /* Prevent vectors vanishing under us */
563 raw_spin_lock(&vector_lock);
564
565 hlist_for_each_entry_safe(apicd, tmp, clhead, clist) {
566 unsigned int irr, vector = apicd->cfg.old_vector;
567
568 /*
569 * Paranoia: Check if the vector that needs to be cleaned
570 * up is registered at the APICs IRR. If so, then this is
571 * not the best time to clean it up. Clean it up in the
572 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
573 * to this CPU. IRQ_MOVE_CLEANUP_VECTOR is the lowest
574 * priority external vector, so on return from this
575 * interrupt the device interrupt will happen first.
576 */
577 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
578 if (irr & (1U << (vector % 32))) {
579 apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
580 continue;
581 }
582 hlist_del_init(&apicd->clist);
583 __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
584 apicd->cfg.old_vector = 0;
585 }
586
587 raw_spin_unlock(&vector_lock);
588 exiting_irq();
589}
590
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200591static void __send_cleanup_vector(struct apic_chip_data *apicd)
Jiang Liu74afab72014-10-27 16:12:00 +0800592{
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200593 unsigned int cpu;
594
Thomas Gleixnerc1684f52015-12-31 16:30:51 +0000595 raw_spin_lock(&vector_lock);
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200596 apicd->move_in_progress = 0;
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200597 cpu = apicd->prev_cpu;
598 if (cpu_online(cpu)) {
599 hlist_add_head(&apicd->clist, per_cpu_ptr(&cleanup_list, cpu));
600 apic->send_IPI(cpu, IRQ_MOVE_CLEANUP_VECTOR);
601 } else {
602 apicd->cfg.old_vector = 0;
603 }
Thomas Gleixnerc1684f52015-12-31 16:30:51 +0000604 raw_spin_unlock(&vector_lock);
Jiang Liu74afab72014-10-27 16:12:00 +0800605}
606
Jiang Liuc6c20022015-04-14 10:30:02 +0800607void send_cleanup_vector(struct irq_cfg *cfg)
608{
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200609 struct apic_chip_data *apicd;
Jiang Liu7f3262e2015-04-14 10:30:03 +0800610
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200611 apicd = container_of(cfg, struct apic_chip_data, cfg);
612 if (apicd->move_in_progress)
613 __send_cleanup_vector(apicd);
Jiang Liuc6c20022015-04-14 10:30:02 +0800614}
615
Jiang Liu74afab72014-10-27 16:12:00 +0800616static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
617{
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200618 struct apic_chip_data *apicd;
Jiang Liu74afab72014-10-27 16:12:00 +0800619
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200620 apicd = container_of(cfg, struct apic_chip_data, cfg);
621 if (likely(!apicd->move_in_progress))
Jiang Liu74afab72014-10-27 16:12:00 +0800622 return;
623
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200624 if (vector == apicd->cfg.vector && apicd->cpu == smp_processor_id())
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200625 __send_cleanup_vector(apicd);
Jiang Liu74afab72014-10-27 16:12:00 +0800626}
627
628void irq_complete_move(struct irq_cfg *cfg)
629{
630 __irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
631}
632
Thomas Gleixner90a22822015-12-31 16:30:53 +0000633/*
Thomas Gleixner551adc62016-03-14 09:40:46 +0100634 * Called from fixup_irqs() with @desc->lock held and interrupts disabled.
Thomas Gleixner90a22822015-12-31 16:30:53 +0000635 */
636void irq_force_complete_move(struct irq_desc *desc)
Jiang Liu74afab72014-10-27 16:12:00 +0800637{
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200638 struct apic_chip_data *apicd;
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200639 struct irq_data *irqd;
640 unsigned int vector;
Jiang Liu74afab72014-10-27 16:12:00 +0800641
Mika Westerbergdb91aa72016-10-03 13:17:08 +0300642 /*
643 * The function is called for all descriptors regardless of which
644 * irqdomain they belong to. For example if an IRQ is provided by
645 * an irq_chip as part of a GPIO driver, the chip data for that
646 * descriptor is specific to the irq_chip in question.
647 *
648 * Check first that the chip_data is what we expect
649 * (apic_chip_data) before touching it any further.
650 */
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200651 irqd = irq_domain_get_irq_data(x86_vector_domain,
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200652 irq_desc_get_irq(desc));
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200653 if (!irqd)
Mika Westerbergdb91aa72016-10-03 13:17:08 +0300654 return;
655
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200656 raw_spin_lock(&vector_lock);
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200657 apicd = apic_chip_data(irqd);
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200658 if (!apicd)
659 goto unlock;
Thomas Gleixner56d7d2f2015-12-31 16:30:52 +0000660
Thomas Gleixner56d7d2f2015-12-31 16:30:52 +0000661 /*
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200662 * If old_vector is empty, no action required.
663 */
664 vector = apicd->cfg.old_vector;
665 if (!vector)
666 goto unlock;
667
668 /*
669 * This is tricky. If the cleanup of the old vector has not been
Thomas Gleixner98229aa2015-12-31 16:30:54 +0000670 * done yet, then the following setaffinity call will fail with
671 * -EBUSY. This can leave the interrupt in a stale state.
672 *
Thomas Gleixner551adc62016-03-14 09:40:46 +0100673 * All CPUs are stuck in stop machine with interrupts disabled so
674 * calling __irq_complete_move() would be completely pointless.
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200675 *
Thomas Gleixner551adc62016-03-14 09:40:46 +0100676 * 1) The interrupt is in move_in_progress state. That means that we
677 * have not seen an interrupt since the io_apic was reprogrammed to
678 * the new vector.
679 *
680 * 2) The interrupt has fired on the new vector, but the cleanup IPIs
681 * have not been processed yet.
682 */
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200683 if (apicd->move_in_progress) {
Thomas Gleixner551adc62016-03-14 09:40:46 +0100684 /*
685 * In theory there is a race:
686 *
687 * set_ioapic(new_vector) <-- Interrupt is raised before update
688 * is effective, i.e. it's raised on
689 * the old vector.
690 *
691 * So if the target cpu cannot handle that interrupt before
692 * the old vector is cleaned up, we get a spurious interrupt
693 * and in the worst case the ioapic irq line becomes stale.
694 *
695 * But in case of cpu hotplug this should be a non issue
696 * because if the affinity update happens right before all
697 * cpus rendevouz in stop machine, there is no way that the
698 * interrupt can be blocked on the target cpu because all cpus
699 * loops first with interrupts enabled in stop machine, so the
700 * old vector is not yet cleaned up when the interrupt fires.
701 *
702 * So the only way to run into this issue is if the delivery
703 * of the interrupt on the apic/system bus would be delayed
704 * beyond the point where the target cpu disables interrupts
705 * in stop machine. I doubt that it can happen, but at least
706 * there is a theroretical chance. Virtualization might be
707 * able to expose this, but AFAICT the IOAPIC emulation is not
708 * as stupid as the real hardware.
709 *
710 * Anyway, there is nothing we can do about that at this point
711 * w/o refactoring the whole fixup_irq() business completely.
712 * We print at least the irq number and the old vector number,
713 * so we have the necessary information when a problem in that
714 * area arises.
715 */
716 pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n",
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200717 irqd->irq, vector);
Thomas Gleixner551adc62016-03-14 09:40:46 +0100718 }
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200719 per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_UNUSED;
Thomas Gleixner551adc62016-03-14 09:40:46 +0100720 /* Cleanup the left overs of the (half finished) move */
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200721 cpumask_clear(apicd->old_domain);
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200722 apicd->cfg.old_vector = 0;
Thomas Gleixner86ba6552017-09-13 23:29:30 +0200723 apicd->move_in_progress = 0;
Thomas Gleixnerdccfe312017-09-13 23:29:32 +0200724 hlist_del_init(&apicd->clist);
725unlock:
Thomas Gleixner56d7d2f2015-12-31 16:30:52 +0000726 raw_spin_unlock(&vector_lock);
Jiang Liu74afab72014-10-27 16:12:00 +0800727}
Jiang Liu74afab72014-10-27 16:12:00 +0800728#endif
729
Jiang Liu74afab72014-10-27 16:12:00 +0800730static void __init print_APIC_field(int base)
731{
732 int i;
733
734 printk(KERN_DEBUG);
735
736 for (i = 0; i < 8; i++)
737 pr_cont("%08x", apic_read(base + i*0x10));
738
739 pr_cont("\n");
740}
741
742static void __init print_local_APIC(void *dummy)
743{
744 unsigned int i, v, ver, maxlvt;
745 u64 icr;
746
Jiang Liu849d3562014-10-27 16:12:01 +0800747 pr_debug("printing local APIC contents on CPU#%d/%d:\n",
748 smp_processor_id(), hard_smp_processor_id());
Jiang Liu74afab72014-10-27 16:12:00 +0800749 v = apic_read(APIC_ID);
Jiang Liu849d3562014-10-27 16:12:01 +0800750 pr_info("... APIC ID: %08x (%01x)\n", v, read_apic_id());
Jiang Liu74afab72014-10-27 16:12:00 +0800751 v = apic_read(APIC_LVR);
Jiang Liu849d3562014-10-27 16:12:01 +0800752 pr_info("... APIC VERSION: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800753 ver = GET_APIC_VERSION(v);
754 maxlvt = lapic_get_maxlvt();
755
756 v = apic_read(APIC_TASKPRI);
Jiang Liu849d3562014-10-27 16:12:01 +0800757 pr_debug("... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
Jiang Liu74afab72014-10-27 16:12:00 +0800758
759 /* !82489DX */
760 if (APIC_INTEGRATED(ver)) {
761 if (!APIC_XAPIC(ver)) {
762 v = apic_read(APIC_ARBPRI);
Jiang Liu849d3562014-10-27 16:12:01 +0800763 pr_debug("... APIC ARBPRI: %08x (%02x)\n",
764 v, v & APIC_ARBPRI_MASK);
Jiang Liu74afab72014-10-27 16:12:00 +0800765 }
766 v = apic_read(APIC_PROCPRI);
Jiang Liu849d3562014-10-27 16:12:01 +0800767 pr_debug("... APIC PROCPRI: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800768 }
769
770 /*
771 * Remote read supported only in the 82489DX and local APIC for
772 * Pentium processors.
773 */
774 if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
775 v = apic_read(APIC_RRR);
Jiang Liu849d3562014-10-27 16:12:01 +0800776 pr_debug("... APIC RRR: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800777 }
778
779 v = apic_read(APIC_LDR);
Jiang Liu849d3562014-10-27 16:12:01 +0800780 pr_debug("... APIC LDR: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800781 if (!x2apic_enabled()) {
782 v = apic_read(APIC_DFR);
Jiang Liu849d3562014-10-27 16:12:01 +0800783 pr_debug("... APIC DFR: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800784 }
785 v = apic_read(APIC_SPIV);
Jiang Liu849d3562014-10-27 16:12:01 +0800786 pr_debug("... APIC SPIV: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800787
Jiang Liu849d3562014-10-27 16:12:01 +0800788 pr_debug("... APIC ISR field:\n");
Jiang Liu74afab72014-10-27 16:12:00 +0800789 print_APIC_field(APIC_ISR);
Jiang Liu849d3562014-10-27 16:12:01 +0800790 pr_debug("... APIC TMR field:\n");
Jiang Liu74afab72014-10-27 16:12:00 +0800791 print_APIC_field(APIC_TMR);
Jiang Liu849d3562014-10-27 16:12:01 +0800792 pr_debug("... APIC IRR field:\n");
Jiang Liu74afab72014-10-27 16:12:00 +0800793 print_APIC_field(APIC_IRR);
794
795 /* !82489DX */
796 if (APIC_INTEGRATED(ver)) {
797 /* Due to the Pentium erratum 3AP. */
798 if (maxlvt > 3)
799 apic_write(APIC_ESR, 0);
800
801 v = apic_read(APIC_ESR);
Jiang Liu849d3562014-10-27 16:12:01 +0800802 pr_debug("... APIC ESR: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800803 }
804
805 icr = apic_icr_read();
Jiang Liu849d3562014-10-27 16:12:01 +0800806 pr_debug("... APIC ICR: %08x\n", (u32)icr);
807 pr_debug("... APIC ICR2: %08x\n", (u32)(icr >> 32));
Jiang Liu74afab72014-10-27 16:12:00 +0800808
809 v = apic_read(APIC_LVTT);
Jiang Liu849d3562014-10-27 16:12:01 +0800810 pr_debug("... APIC LVTT: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800811
812 if (maxlvt > 3) {
813 /* PC is LVT#4. */
814 v = apic_read(APIC_LVTPC);
Jiang Liu849d3562014-10-27 16:12:01 +0800815 pr_debug("... APIC LVTPC: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800816 }
817 v = apic_read(APIC_LVT0);
Jiang Liu849d3562014-10-27 16:12:01 +0800818 pr_debug("... APIC LVT0: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800819 v = apic_read(APIC_LVT1);
Jiang Liu849d3562014-10-27 16:12:01 +0800820 pr_debug("... APIC LVT1: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800821
822 if (maxlvt > 2) {
823 /* ERR is LVT#3. */
824 v = apic_read(APIC_LVTERR);
Jiang Liu849d3562014-10-27 16:12:01 +0800825 pr_debug("... APIC LVTERR: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800826 }
827
828 v = apic_read(APIC_TMICT);
Jiang Liu849d3562014-10-27 16:12:01 +0800829 pr_debug("... APIC TMICT: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800830 v = apic_read(APIC_TMCCT);
Jiang Liu849d3562014-10-27 16:12:01 +0800831 pr_debug("... APIC TMCCT: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800832 v = apic_read(APIC_TDCR);
Jiang Liu849d3562014-10-27 16:12:01 +0800833 pr_debug("... APIC TDCR: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800834
835 if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
836 v = apic_read(APIC_EFEAT);
837 maxlvt = (v >> 16) & 0xff;
Jiang Liu849d3562014-10-27 16:12:01 +0800838 pr_debug("... APIC EFEAT: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800839 v = apic_read(APIC_ECTRL);
Jiang Liu849d3562014-10-27 16:12:01 +0800840 pr_debug("... APIC ECTRL: %08x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800841 for (i = 0; i < maxlvt; i++) {
842 v = apic_read(APIC_EILVTn(i));
Jiang Liu849d3562014-10-27 16:12:01 +0800843 pr_debug("... APIC EILVT%d: %08x\n", i, v);
Jiang Liu74afab72014-10-27 16:12:00 +0800844 }
845 }
846 pr_cont("\n");
847}
848
849static void __init print_local_APICs(int maxcpu)
850{
851 int cpu;
852
853 if (!maxcpu)
854 return;
855
856 preempt_disable();
857 for_each_online_cpu(cpu) {
858 if (cpu >= maxcpu)
859 break;
860 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
861 }
862 preempt_enable();
863}
864
865static void __init print_PIC(void)
866{
867 unsigned int v;
868 unsigned long flags;
869
870 if (!nr_legacy_irqs())
871 return;
872
Jiang Liu849d3562014-10-27 16:12:01 +0800873 pr_debug("\nprinting PIC contents\n");
Jiang Liu74afab72014-10-27 16:12:00 +0800874
875 raw_spin_lock_irqsave(&i8259A_lock, flags);
876
877 v = inb(0xa1) << 8 | inb(0x21);
Jiang Liu849d3562014-10-27 16:12:01 +0800878 pr_debug("... PIC IMR: %04x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800879
880 v = inb(0xa0) << 8 | inb(0x20);
Jiang Liu849d3562014-10-27 16:12:01 +0800881 pr_debug("... PIC IRR: %04x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800882
883 outb(0x0b, 0xa0);
884 outb(0x0b, 0x20);
885 v = inb(0xa0) << 8 | inb(0x20);
886 outb(0x0a, 0xa0);
887 outb(0x0a, 0x20);
888
889 raw_spin_unlock_irqrestore(&i8259A_lock, flags);
890
Jiang Liu849d3562014-10-27 16:12:01 +0800891 pr_debug("... PIC ISR: %04x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800892
893 v = inb(0x4d1) << 8 | inb(0x4d0);
Jiang Liu849d3562014-10-27 16:12:01 +0800894 pr_debug("... PIC ELCR: %04x\n", v);
Jiang Liu74afab72014-10-27 16:12:00 +0800895}
896
897static int show_lapic __initdata = 1;
898static __init int setup_show_lapic(char *arg)
899{
900 int num = -1;
901
902 if (strcmp(arg, "all") == 0) {
903 show_lapic = CONFIG_NR_CPUS;
904 } else {
905 get_option(&arg, &num);
906 if (num >= 0)
907 show_lapic = num;
908 }
909
910 return 1;
911}
912__setup("show_lapic=", setup_show_lapic);
913
914static int __init print_ICs(void)
915{
916 if (apic_verbosity == APIC_QUIET)
917 return 0;
918
919 print_PIC();
920
921 /* don't print out if apic is not there */
Borislav Petkov93984fb2016-04-04 22:25:00 +0200922 if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
Jiang Liu74afab72014-10-27 16:12:00 +0800923 return 0;
924
925 print_local_APICs(show_lapic);
926 print_IO_APICs();
927
928 return 0;
929}
930
931late_initcall(print_ICs);