linux/arch/x86/include/asm/msr.h
<<
>>
Prefs
   1#ifndef _ASM_X86_MSR_H
   2#define _ASM_X86_MSR_H
   3
   4#include "msr-index.h"
   5
   6#ifndef __ASSEMBLY__
   7
   8#include <asm/asm.h>
   9#include <asm/errno.h>
  10#include <asm/cpumask.h>
  11#include <uapi/asm/msr.h>
  12
  13struct msr {
  14        union {
  15                struct {
  16                        u32 l;
  17                        u32 h;
  18                };
  19                u64 q;
  20        };
  21};
  22
  23struct msr_info {
  24        u32 msr_no;
  25        struct msr reg;
  26        struct msr *msrs;
  27        int err;
  28};
  29
  30struct msr_regs_info {
  31        u32 *regs;
  32        int err;
  33};
  34
  35struct saved_msr {
  36        bool valid;
  37        struct msr_info info;
  38};
  39
  40struct saved_msrs {
  41        unsigned int num;
  42        struct saved_msr *array;
  43};
  44
  45/*
  46 * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A"
  47 * constraint has different meanings. For i386, "A" means exactly
  48 * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead,
  49 * it means rax *or* rdx.
  50 */
  51#ifdef CONFIG_X86_64
  52/* Using 64-bit values saves one instruction clearing the high half of low */
  53#define DECLARE_ARGS(val, low, high)    unsigned long low, high
  54#define EAX_EDX_VAL(val, low, high)     ((low) | (high) << 32)
  55#define EAX_EDX_RET(val, low, high)     "=a" (low), "=d" (high)
  56#else
  57#define DECLARE_ARGS(val, low, high)    unsigned long long val
  58#define EAX_EDX_VAL(val, low, high)     (val)
  59#define EAX_EDX_RET(val, low, high)     "=A" (val)
  60#endif
  61
  62#ifdef CONFIG_TRACEPOINTS
  63/*
  64 * Be very careful with includes. This header is prone to include loops.
  65 */
  66#include <asm/atomic.h>
  67#include <linux/tracepoint-defs.h>
  68
  69extern struct tracepoint __tracepoint_read_msr;
  70extern struct tracepoint __tracepoint_write_msr;
  71extern struct tracepoint __tracepoint_rdpmc;
  72#define msr_tracepoint_active(t) static_key_false(&(t).key)
  73extern void do_trace_write_msr(unsigned int msr, u64 val, int failed);
  74extern void do_trace_read_msr(unsigned int msr, u64 val, int failed);
  75extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed);
  76#else
  77#define msr_tracepoint_active(t) false
  78static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {}
  79static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {}
  80static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
  81#endif
  82
  83/*
  84 * __rdmsr() and __wrmsr() are the two primitives which are the bare minimum MSR
  85 * accessors and should not have any tracing or other functionality piggybacking
  86 * on them - those are *purely* for accessing MSRs and nothing more. So don't even
  87 * think of extending them - you will be slapped with a stinking trout or a frozen
  88 * shark will reach you, wherever you are! You've been warned.
  89 */
  90static inline unsigned long long notrace __rdmsr(unsigned int msr)
  91{
  92        DECLARE_ARGS(val, low, high);
  93
  94        asm volatile("1: rdmsr\n"
  95                     "2:\n"
  96                     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_unsafe)
  97                     : EAX_EDX_RET(val, low, high) : "c" (msr));
  98
  99        return EAX_EDX_VAL(val, low, high);
 100}
 101
 102static inline void notrace __wrmsr(unsigned int msr, u32 low, u32 high)
 103{
 104        asm volatile("1: wrmsr\n"
 105                     "2:\n"
 106                     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_unsafe)
 107                     : : "c" (msr), "a"(low), "d" (high) : "memory");
 108}
 109
 110static inline unsigned long long native_read_msr(unsigned int msr)
 111{
 112        unsigned long long val;
 113
 114        val = __rdmsr(msr);
 115
 116        if (msr_tracepoint_active(__tracepoint_read_msr))
 117                do_trace_read_msr(msr, val, 0);
 118
 119        return val;
 120}
 121
 122static inline unsigned long long native_read_msr_safe(unsigned int msr,
 123                                                      int *err)
 124{
 125        DECLARE_ARGS(val, low, high);
 126
 127        asm volatile("2: rdmsr ; xor %[err],%[err]\n"
 128                     "1:\n\t"
 129                     ".section .fixup,\"ax\"\n\t"
 130                     "3: mov %[fault],%[err]\n\t"
 131                     "xorl %%eax, %%eax\n\t"
 132                     "xorl %%edx, %%edx\n\t"
 133                     "jmp 1b\n\t"
 134                     ".previous\n\t"
 135                     _ASM_EXTABLE(2b, 3b)
 136                     : [err] "=r" (*err), EAX_EDX_RET(val, low, high)
 137                     : "c" (msr), [fault] "i" (-EIO));
 138        if (msr_tracepoint_active(__tracepoint_read_msr))
 139                do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err);
 140        return EAX_EDX_VAL(val, low, high);
 141}
 142
 143/* Can be uninlined because referenced by paravirt */
 144static inline void notrace
 145native_write_msr(unsigned int msr, u32 low, u32 high)
 146{
 147        __wrmsr(msr, low, high);
 148
 149        if (msr_tracepoint_active(__tracepoint_write_msr))
 150                do_trace_write_msr(msr, ((u64)high << 32 | low), 0);
 151}
 152
 153/* Can be uninlined because referenced by paravirt */
 154static inline int notrace
 155native_write_msr_safe(unsigned int msr, u32 low, u32 high)
 156{
 157        int err;
 158
 159        asm volatile("2: wrmsr ; xor %[err],%[err]\n"
 160                     "1:\n\t"
 161                     ".section .fixup,\"ax\"\n\t"
 162                     "3:  mov %[fault],%[err] ; jmp 1b\n\t"
 163                     ".previous\n\t"
 164                     _ASM_EXTABLE(2b, 3b)
 165                     : [err] "=a" (err)
 166                     : "c" (msr), "0" (low), "d" (high),
 167                       [fault] "i" (-EIO)
 168                     : "memory");
 169        if (msr_tracepoint_active(__tracepoint_write_msr))
 170                do_trace_write_msr(msr, ((u64)high << 32 | low), err);
 171        return err;
 172}
 173
 174extern int rdmsr_safe_regs(u32 regs[8]);
 175extern int wrmsr_safe_regs(u32 regs[8]);
 176
 177/**
 178 * rdtsc() - returns the current TSC without ordering constraints
 179 *
 180 * rdtsc() returns the result of RDTSC as a 64-bit integer.  The
 181 * only ordering constraint it supplies is the ordering implied by
 182 * "asm volatile": it will put the RDTSC in the place you expect.  The
 183 * CPU can and will speculatively execute that RDTSC, though, so the
 184 * results can be non-monotonic if compared on different CPUs.
 185 */
 186static __always_inline unsigned long long rdtsc(void)
 187{
 188        DECLARE_ARGS(val, low, high);
 189
 190        asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
 191
 192        return EAX_EDX_VAL(val, low, high);
 193}
 194
 195/**
 196 * rdtsc_ordered() - read the current TSC in program order
 197 *
 198 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
 199 * It is ordered like a load to a global in-memory counter.  It should
 200 * be impossible to observe non-monotonic rdtsc_unordered() behavior
 201 * across multiple CPUs as long as the TSC is synced.
 202 */
 203static __always_inline unsigned long long rdtsc_ordered(void)
 204{
 205        /*
 206         * The RDTSC instruction is not ordered relative to memory
 207         * access.  The Intel SDM and the AMD APM are both vague on this
 208         * point, but empirically an RDTSC instruction can be
 209         * speculatively executed before prior loads.  An RDTSC
 210         * immediately after an appropriate barrier appears to be
 211         * ordered as a normal load, that is, it provides the same
 212         * ordering guarantees as reading from a global memory location
 213         * that some other imaginary CPU is updating continuously with a
 214         * time stamp.
 215         */
 216        alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC,
 217                          "lfence", X86_FEATURE_LFENCE_RDTSC);
 218        return rdtsc();
 219}
 220
 221/* Deprecated, keep it for a cycle for easier merging: */
 222#define rdtscll(now)    do { (now) = rdtsc_ordered(); } while (0)
 223
 224static inline unsigned long long native_read_pmc(int counter)
 225{
 226        DECLARE_ARGS(val, low, high);
 227
 228        asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter));
 229        if (msr_tracepoint_active(__tracepoint_rdpmc))
 230                do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0);
 231        return EAX_EDX_VAL(val, low, high);
 232}
 233
 234#ifdef CONFIG_PARAVIRT
 235#include <asm/paravirt.h>
 236#else
 237#include <linux/errno.h>
 238/*
 239 * Access to machine-specific registers (available on 586 and better only)
 240 * Note: the rd* operations modify the parameters directly (without using
 241 * pointer indirection), this allows gcc to optimize better
 242 */
 243
 244#define rdmsr(msr, low, high)                                   \
 245do {                                                            \
 246        u64 __val = native_read_msr((msr));                     \
 247        (void)((low) = (u32)__val);                             \
 248        (void)((high) = (u32)(__val >> 32));                    \
 249} while (0)
 250
 251static inline void wrmsr(unsigned int msr, u32 low, u32 high)
 252{
 253        native_write_msr(msr, low, high);
 254}
 255
 256#define rdmsrl(msr, val)                        \
 257        ((val) = native_read_msr((msr)))
 258
 259static inline void wrmsrl(unsigned int msr, u64 val)
 260{
 261        native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
 262}
 263
 264/* wrmsr with exception handling */
 265static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high)
 266{
 267        return native_write_msr_safe(msr, low, high);
 268}
 269
 270/* rdmsr with exception handling */
 271#define rdmsr_safe(msr, low, high)                              \
 272({                                                              \
 273        int __err;                                              \
 274        u64 __val = native_read_msr_safe((msr), &__err);        \
 275        (*low) = (u32)__val;                                    \
 276        (*high) = (u32)(__val >> 32);                           \
 277        __err;                                                  \
 278})
 279
 280static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p)
 281{
 282        int err;
 283
 284        *p = native_read_msr_safe(msr, &err);
 285        return err;
 286}
 287
 288#define rdpmc(counter, low, high)                       \
 289do {                                                    \
 290        u64 _l = native_read_pmc((counter));            \
 291        (low)  = (u32)_l;                               \
 292        (high) = (u32)(_l >> 32);                       \
 293} while (0)
 294
 295#define rdpmcl(counter, val) ((val) = native_read_pmc(counter))
 296
 297#endif  /* !CONFIG_PARAVIRT */
 298
 299/*
 300 * 64-bit version of wrmsr_safe():
 301 */
 302static inline int wrmsrl_safe(u32 msr, u64 val)
 303{
 304        return wrmsr_safe(msr, (u32)val,  (u32)(val >> 32));
 305}
 306
 307#define write_tsc(low, high) wrmsr(MSR_IA32_TSC, (low), (high))
 308
 309#define write_rdtscp_aux(val) wrmsr(MSR_TSC_AUX, (val), 0)
 310
 311struct msr *msrs_alloc(void);
 312void msrs_free(struct msr *msrs);
 313int msr_set_bit(u32 msr, u8 bit);
 314int msr_clear_bit(u32 msr, u8 bit);
 315
 316#ifdef CONFIG_SMP
 317int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
 318int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
 319int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
 320int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
 321void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
 322void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
 323int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
 324int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
 325int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
 326int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
 327int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
 328int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
 329#else  /*  CONFIG_SMP  */
 330static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
 331{
 332        rdmsr(msr_no, *l, *h);
 333        return 0;
 334}
 335static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
 336{
 337        wrmsr(msr_no, l, h);
 338        return 0;
 339}
 340static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
 341{
 342        rdmsrl(msr_no, *q);
 343        return 0;
 344}
 345static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
 346{
 347        wrmsrl(msr_no, q);
 348        return 0;
 349}
 350static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no,
 351                                struct msr *msrs)
 352{
 353        rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h));
 354}
 355static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no,
 356                                struct msr *msrs)
 357{
 358        wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h);
 359}
 360static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no,
 361                                    u32 *l, u32 *h)
 362{
 363        return rdmsr_safe(msr_no, l, h);
 364}
 365static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
 366{
 367        return wrmsr_safe(msr_no, l, h);
 368}
 369static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
 370{
 371        return rdmsrl_safe(msr_no, q);
 372}
 373static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
 374{
 375        return wrmsrl_safe(msr_no, q);
 376}
 377static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
 378{
 379        return rdmsr_safe_regs(regs);
 380}
 381static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
 382{
 383        return wrmsr_safe_regs(regs);
 384}
 385#endif  /* CONFIG_SMP */
 386#endif /* __ASSEMBLY__ */
 387#endif /* _ASM_X86_MSR_H */
 388