linux/arch/x86/kernel/apic/ipi.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2
   3#include <linux/cpumask.h>
   4#include <linux/smp.h>
   5#include <asm/io_apic.h>
   6
   7#include "local.h"
   8
   9DEFINE_STATIC_KEY_FALSE(apic_use_ipi_shorthand);
  10
  11#ifdef CONFIG_SMP
  12static int apic_ipi_shorthand_off __ro_after_init;
  13
  14static __init int apic_ipi_shorthand(char *str)
  15{
  16        get_option(&str, &apic_ipi_shorthand_off);
  17        return 1;
  18}
  19__setup("no_ipi_broadcast=", apic_ipi_shorthand);
  20
  21static int __init print_ipi_mode(void)
  22{
  23        pr_info("IPI shorthand broadcast: %s\n",
  24                apic_ipi_shorthand_off ? "disabled" : "enabled");
  25        return 0;
  26}
  27late_initcall(print_ipi_mode);
  28
  29void apic_smt_update(void)
  30{
  31        /*
  32         * Do not switch to broadcast mode if:
  33         * - Disabled on the command line
  34         * - Only a single CPU is online
  35         * - Not all present CPUs have been at least booted once
  36         *
  37         * The latter is important as the local APIC might be in some
  38         * random state and a broadcast might cause havoc. That's
  39         * especially true for NMI broadcasting.
  40         */
  41        if (apic_ipi_shorthand_off || num_online_cpus() == 1 ||
  42            !cpumask_equal(cpu_present_mask, &cpus_booted_once_mask)) {
  43                static_branch_disable(&apic_use_ipi_shorthand);
  44        } else {
  45                static_branch_enable(&apic_use_ipi_shorthand);
  46        }
  47}
  48
  49void apic_send_IPI_allbutself(unsigned int vector)
  50{
  51        if (num_online_cpus() < 2)
  52                return;
  53
  54        if (static_branch_likely(&apic_use_ipi_shorthand))
  55                apic->send_IPI_allbutself(vector);
  56        else
  57                apic->send_IPI_mask_allbutself(cpu_online_mask, vector);
  58}
  59
  60/*
  61 * Send a 'reschedule' IPI to another CPU. It goes straight through and
  62 * wastes no time serializing anything. Worst case is that we lose a
  63 * reschedule ...
  64 */
  65void native_smp_send_reschedule(int cpu)
  66{
  67        if (unlikely(cpu_is_offline(cpu))) {
  68                WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu);
  69                return;
  70        }
  71        apic->send_IPI(cpu, RESCHEDULE_VECTOR);
  72}
  73
  74void native_send_call_func_single_ipi(int cpu)
  75{
  76        apic->send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR);
  77}
  78
  79void native_send_call_func_ipi(const struct cpumask *mask)
  80{
  81        if (static_branch_likely(&apic_use_ipi_shorthand)) {
  82                unsigned int cpu = smp_processor_id();
  83
  84                if (!cpumask_or_equal(mask, cpumask_of(cpu), cpu_online_mask))
  85                        goto sendmask;
  86
  87                if (cpumask_test_cpu(cpu, mask))
  88                        apic->send_IPI_all(CALL_FUNCTION_VECTOR);
  89                else if (num_online_cpus() > 1)
  90                        apic->send_IPI_allbutself(CALL_FUNCTION_VECTOR);
  91                return;
  92        }
  93
  94sendmask:
  95        apic->send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
  96}
  97
  98#endif /* CONFIG_SMP */
  99
 100static inline int __prepare_ICR2(unsigned int mask)
 101{
 102        return SET_APIC_DEST_FIELD(mask);
 103}
 104
 105static inline void __xapic_wait_icr_idle(void)
 106{
 107        while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY)
 108                cpu_relax();
 109}
 110
 111void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
 112{
 113        /*
 114         * Subtle. In the case of the 'never do double writes' workaround
 115         * we have to lock out interrupts to be safe.  As we don't care
 116         * of the value read we use an atomic rmw access to avoid costly
 117         * cli/sti.  Otherwise we use an even cheaper single atomic write
 118         * to the APIC.
 119         */
 120        unsigned int cfg;
 121
 122        /*
 123         * Wait for idle.
 124         */
 125        if (unlikely(vector == NMI_VECTOR))
 126                safe_apic_wait_icr_idle();
 127        else
 128                __xapic_wait_icr_idle();
 129
 130        /*
 131         * No need to touch the target chip field. Also the destination
 132         * mode is ignored when a shorthand is used.
 133         */
 134        cfg = __prepare_ICR(shortcut, vector, 0);
 135
 136        /*
 137         * Send the IPI. The write to APIC_ICR fires this off.
 138         */
 139        native_apic_mem_write(APIC_ICR, cfg);
 140}
 141
 142/*
 143 * This is used to send an IPI with no shorthand notation (the destination is
 144 * specified in bits 56 to 63 of the ICR).
 145 */
 146void __default_send_IPI_dest_field(unsigned int mask, int vector, unsigned int dest)
 147{
 148        unsigned long cfg;
 149
 150        /*
 151         * Wait for idle.
 152         */
 153        if (unlikely(vector == NMI_VECTOR))
 154                safe_apic_wait_icr_idle();
 155        else
 156                __xapic_wait_icr_idle();
 157
 158        /*
 159         * prepare target chip field
 160         */
 161        cfg = __prepare_ICR2(mask);
 162        native_apic_mem_write(APIC_ICR2, cfg);
 163
 164        /*
 165         * program the ICR
 166         */
 167        cfg = __prepare_ICR(0, vector, dest);
 168
 169        /*
 170         * Send the IPI. The write to APIC_ICR fires this off.
 171         */
 172        native_apic_mem_write(APIC_ICR, cfg);
 173}
 174
 175void default_send_IPI_single_phys(int cpu, int vector)
 176{
 177        unsigned long flags;
 178
 179        local_irq_save(flags);
 180        __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu),
 181                                      vector, APIC_DEST_PHYSICAL);
 182        local_irq_restore(flags);
 183}
 184
 185void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
 186{
 187        unsigned long query_cpu;
 188        unsigned long flags;
 189
 190        /*
 191         * Hack. The clustered APIC addressing mode doesn't allow us to send
 192         * to an arbitrary mask, so I do a unicast to each CPU instead.
 193         * - mbligh
 194         */
 195        local_irq_save(flags);
 196        for_each_cpu(query_cpu, mask) {
 197                __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
 198                                query_cpu), vector, APIC_DEST_PHYSICAL);
 199        }
 200        local_irq_restore(flags);
 201}
 202
 203void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
 204                                                 int vector)
 205{
 206        unsigned int this_cpu = smp_processor_id();
 207        unsigned int query_cpu;
 208        unsigned long flags;
 209
 210        /* See Hack comment above */
 211
 212        local_irq_save(flags);
 213        for_each_cpu(query_cpu, mask) {
 214                if (query_cpu == this_cpu)
 215                        continue;
 216                __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
 217                                 query_cpu), vector, APIC_DEST_PHYSICAL);
 218        }
 219        local_irq_restore(flags);
 220}
 221
 222/*
 223 * Helper function for APICs which insist on cpumasks
 224 */
 225void default_send_IPI_single(int cpu, int vector)
 226{
 227        apic->send_IPI_mask(cpumask_of(cpu), vector);
 228}
 229
 230void default_send_IPI_allbutself(int vector)
 231{
 232        __default_send_IPI_shortcut(APIC_DEST_ALLBUT, vector);
 233}
 234
 235void default_send_IPI_all(int vector)
 236{
 237        __default_send_IPI_shortcut(APIC_DEST_ALLINC, vector);
 238}
 239
 240void default_send_IPI_self(int vector)
 241{
 242        __default_send_IPI_shortcut(APIC_DEST_SELF, vector);
 243}
 244
 245#ifdef CONFIG_X86_32
 246
 247void default_send_IPI_mask_sequence_logical(const struct cpumask *mask,
 248                                                 int vector)
 249{
 250        unsigned long flags;
 251        unsigned int query_cpu;
 252
 253        /*
 254         * Hack. The clustered APIC addressing mode doesn't allow us to send
 255         * to an arbitrary mask, so I do a unicasts to each CPU instead. This
 256         * should be modified to do 1 message per cluster ID - mbligh
 257         */
 258
 259        local_irq_save(flags);
 260        for_each_cpu(query_cpu, mask)
 261                __default_send_IPI_dest_field(
 262                        early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
 263                        vector, apic->dest_logical);
 264        local_irq_restore(flags);
 265}
 266
 267void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
 268                                                 int vector)
 269{
 270        unsigned long flags;
 271        unsigned int query_cpu;
 272        unsigned int this_cpu = smp_processor_id();
 273
 274        /* See Hack comment above */
 275
 276        local_irq_save(flags);
 277        for_each_cpu(query_cpu, mask) {
 278                if (query_cpu == this_cpu)
 279                        continue;
 280                __default_send_IPI_dest_field(
 281                        early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
 282                        vector, apic->dest_logical);
 283                }
 284        local_irq_restore(flags);
 285}
 286
 287/*
 288 * This is only used on smaller machines.
 289 */
 290void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
 291{
 292        unsigned long mask = cpumask_bits(cpumask)[0];
 293        unsigned long flags;
 294
 295        if (!mask)
 296                return;
 297
 298        local_irq_save(flags);
 299        WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
 300        __default_send_IPI_dest_field(mask, vector, apic->dest_logical);
 301        local_irq_restore(flags);
 302}
 303
 304/* must come after the send_IPI functions above for inlining */
 305static int convert_apicid_to_cpu(int apic_id)
 306{
 307        int i;
 308
 309        for_each_possible_cpu(i) {
 310                if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
 311                        return i;
 312        }
 313        return -1;
 314}
 315
 316int safe_smp_processor_id(void)
 317{
 318        int apicid, cpuid;
 319
 320        if (!boot_cpu_has(X86_FEATURE_APIC))
 321                return 0;
 322
 323        apicid = hard_smp_processor_id();
 324        if (apicid == BAD_APICID)
 325                return 0;
 326
 327        cpuid = convert_apicid_to_cpu(apicid);
 328
 329        return cpuid >= 0 ? cpuid : 0;
 330}
 331#endif
 332