linux/arch/x86/kernel/fpu/init.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * x86 FPU boot time init code:
   4 */
   5#include <asm/fpu/internal.h>
   6#include <asm/tlbflush.h>
   7#include <asm/setup.h>
   8#include <asm/cmdline.h>
   9
  10#include <linux/sched.h>
  11#include <linux/sched/task.h>
  12#include <linux/init.h>
  13
  14/*
  15 * Initialize the registers found in all CPUs, CR0 and CR4:
  16 */
  17static void fpu__init_cpu_generic(void)
  18{
  19        unsigned long cr0;
  20        unsigned long cr4_mask = 0;
  21
  22        if (boot_cpu_has(X86_FEATURE_FXSR))
  23                cr4_mask |= X86_CR4_OSFXSR;
  24        if (boot_cpu_has(X86_FEATURE_XMM))
  25                cr4_mask |= X86_CR4_OSXMMEXCPT;
  26        if (cr4_mask)
  27                cr4_set_bits(cr4_mask);
  28
  29        cr0 = read_cr0();
  30        cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
  31        if (!boot_cpu_has(X86_FEATURE_FPU))
  32                cr0 |= X86_CR0_EM;
  33        write_cr0(cr0);
  34
  35        /* Flush out any pending x87 state: */
  36#ifdef CONFIG_MATH_EMULATION
  37        if (!boot_cpu_has(X86_FEATURE_FPU))
  38                fpstate_init_soft(&current->thread.fpu.state.soft);
  39        else
  40#endif
  41                asm volatile ("fninit");
  42}
  43
  44/*
  45 * Enable all supported FPU features. Called when a CPU is brought online:
  46 */
  47void fpu__init_cpu(void)
  48{
  49        fpu__init_cpu_generic();
  50        fpu__init_cpu_xstate();
  51}
  52
  53static bool fpu__probe_without_cpuid(void)
  54{
  55        unsigned long cr0;
  56        u16 fsw, fcw;
  57
  58        fsw = fcw = 0xffff;
  59
  60        cr0 = read_cr0();
  61        cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
  62        write_cr0(cr0);
  63
  64        asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw));
  65
  66        pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw);
  67
  68        return fsw == 0 && (fcw & 0x103f) == 0x003f;
  69}
  70
  71static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
  72{
  73        if (!boot_cpu_has(X86_FEATURE_CPUID) &&
  74            !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
  75                if (fpu__probe_without_cpuid())
  76                        setup_force_cpu_cap(X86_FEATURE_FPU);
  77                else
  78                        setup_clear_cpu_cap(X86_FEATURE_FPU);
  79        }
  80
  81#ifndef CONFIG_MATH_EMULATION
  82        if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
  83                pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
  84                for (;;)
  85                        asm volatile("hlt");
  86        }
  87#endif
  88}
  89
  90/*
  91 * Boot time FPU feature detection code:
  92 */
  93unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
  94EXPORT_SYMBOL_GPL(mxcsr_feature_mask);
  95
  96static void __init fpu__init_system_mxcsr(void)
  97{
  98        unsigned int mask = 0;
  99
 100        if (boot_cpu_has(X86_FEATURE_FXSR)) {
 101                /* Static because GCC does not get 16-byte stack alignment right: */
 102                static struct fxregs_state fxregs __initdata;
 103
 104                asm volatile("fxsave %0" : "+m" (fxregs));
 105
 106                mask = fxregs.mxcsr_mask;
 107
 108                /*
 109                 * If zero then use the default features mask,
 110                 * which has all features set, except the
 111                 * denormals-are-zero feature bit:
 112                 */
 113                if (mask == 0)
 114                        mask = 0x0000ffbf;
 115        }
 116        mxcsr_feature_mask &= mask;
 117}
 118
 119/*
 120 * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
 121 */
 122static void __init fpu__init_system_generic(void)
 123{
 124        /*
 125         * Set up the legacy init FPU context. (xstate init might overwrite this
 126         * with a more modern format, if the CPU supports it.)
 127         */
 128        fpstate_init(&init_fpstate);
 129
 130        fpu__init_system_mxcsr();
 131}
 132
 133/*
 134 * Size of the FPU context state. All tasks in the system use the
 135 * same context size, regardless of what portion they use.
 136 * This is inherent to the XSAVE architecture which puts all state
 137 * components into a single, continuous memory block:
 138 */
 139unsigned int fpu_kernel_xstate_size;
 140EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size);
 141
 142/* Get alignment of the TYPE. */
 143#define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
 144
 145/*
 146 * Enforce that 'MEMBER' is the last field of 'TYPE'.
 147 *
 148 * Align the computed size with alignment of the TYPE,
 149 * because that's how C aligns structs.
 150 */
 151#define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
 152        BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
 153                                           TYPE_ALIGN(TYPE)))
 154
 155/*
 156 * We append the 'struct fpu' to the task_struct:
 157 */
 158static void __init fpu__init_task_struct_size(void)
 159{
 160        int task_size = sizeof(struct task_struct);
 161
 162        /*
 163         * Subtract off the static size of the register state.
 164         * It potentially has a bunch of padding.
 165         */
 166        task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state);
 167
 168        /*
 169         * Add back the dynamically-calculated register state
 170         * size.
 171         */
 172        task_size += fpu_kernel_xstate_size;
 173
 174        /*
 175         * We dynamically size 'struct fpu', so we require that
 176         * it be at the end of 'thread_struct' and that
 177         * 'thread_struct' be at the end of 'task_struct'.  If
 178         * you hit a compile error here, check the structure to
 179         * see if something got added to the end.
 180         */
 181        CHECK_MEMBER_AT_END_OF(struct fpu, state);
 182        CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
 183        CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
 184
 185        arch_task_struct_size = task_size;
 186}
 187
 188/*
 189 * Set up the user and kernel xstate sizes based on the legacy FPU context size.
 190 *
 191 * We set this up first, and later it will be overwritten by
 192 * fpu__init_system_xstate() if the CPU knows about xstates.
 193 */
 194static void __init fpu__init_system_xstate_size_legacy(void)
 195{
 196        static int on_boot_cpu __initdata = 1;
 197
 198        WARN_ON_FPU(!on_boot_cpu);
 199        on_boot_cpu = 0;
 200
 201        /*
 202         * Note that xstate sizes might be overwritten later during
 203         * fpu__init_system_xstate().
 204         */
 205
 206        if (!boot_cpu_has(X86_FEATURE_FPU)) {
 207                fpu_kernel_xstate_size = sizeof(struct swregs_state);
 208        } else {
 209                if (boot_cpu_has(X86_FEATURE_FXSR))
 210                        fpu_kernel_xstate_size =
 211                                sizeof(struct fxregs_state);
 212                else
 213                        fpu_kernel_xstate_size =
 214                                sizeof(struct fregs_state);
 215        }
 216
 217        fpu_user_xstate_size = fpu_kernel_xstate_size;
 218}
 219
 220/*
 221 * Find supported xfeatures based on cpu features and command-line input.
 222 * This must be called after fpu__init_parse_early_param() is called and
 223 * xfeatures_mask is enumerated.
 224 */
 225u64 __init fpu__get_supported_xfeatures_mask(void)
 226{
 227        return XFEATURE_MASK_USER_SUPPORTED |
 228               XFEATURE_MASK_SUPERVISOR_SUPPORTED;
 229}
 230
 231/* Legacy code to initialize eager fpu mode. */
 232static void __init fpu__init_system_ctx_switch(void)
 233{
 234        static bool on_boot_cpu __initdata = 1;
 235
 236        WARN_ON_FPU(!on_boot_cpu);
 237        on_boot_cpu = 0;
 238}
 239
 240/*
 241 * We parse fpu parameters early because fpu__init_system() is executed
 242 * before parse_early_param().
 243 */
 244static void __init fpu__init_parse_early_param(void)
 245{
 246        char arg[32];
 247        char *argptr = arg;
 248        int bit;
 249
 250#ifdef CONFIG_X86_32
 251        if (cmdline_find_option_bool(boot_command_line, "no387"))
 252#ifdef CONFIG_MATH_EMULATION
 253                setup_clear_cpu_cap(X86_FEATURE_FPU);
 254#else
 255                pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
 256#endif
 257
 258        if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
 259                setup_clear_cpu_cap(X86_FEATURE_FXSR);
 260#endif
 261
 262        if (cmdline_find_option_bool(boot_command_line, "noxsave"))
 263                setup_clear_cpu_cap(X86_FEATURE_XSAVE);
 264
 265        if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
 266                setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
 267
 268        if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
 269                setup_clear_cpu_cap(X86_FEATURE_XSAVES);
 270
 271        if (cmdline_find_option(boot_command_line, "clearcpuid", arg,
 272                                sizeof(arg)) &&
 273            get_option(&argptr, &bit) &&
 274            bit >= 0 &&
 275            bit < NCAPINTS * 32)
 276                setup_clear_cpu_cap(bit);
 277}
 278
 279/*
 280 * Called on the boot CPU once per system bootup, to set up the initial
 281 * FPU state that is later cloned into all processes:
 282 */
 283void __init fpu__init_system(struct cpuinfo_x86 *c)
 284{
 285        fpu__init_parse_early_param();
 286        fpu__init_system_early_generic(c);
 287
 288        /*
 289         * The FPU has to be operational for some of the
 290         * later FPU init activities:
 291         */
 292        fpu__init_cpu();
 293
 294        fpu__init_system_generic();
 295        fpu__init_system_xstate_size_legacy();
 296        fpu__init_system_xstate();
 297        fpu__init_task_struct_size();
 298
 299        fpu__init_system_ctx_switch();
 300}
 301