linux/kernel/trace/trace_osnoise.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
   4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
   5 *
   6 * Based on "hwlat_detector" tracer by:
   7 *   Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
   8 *   Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
   9 *   With feedback from Clark Williams <williams@redhat.com>
  10 *
  11 * And also based on the rtsl tracer presented on:
  12 *  DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
  13 *  scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
  14 *  (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
  15 *
  16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
  17 */
  18
  19#include <linux/kthread.h>
  20#include <linux/tracefs.h>
  21#include <linux/uaccess.h>
  22#include <linux/cpumask.h>
  23#include <linux/delay.h>
  24#include <linux/sched/clock.h>
  25#include <uapi/linux/sched/types.h>
  26#include <linux/sched.h>
  27#include "trace.h"
  28
  29#ifdef CONFIG_X86_LOCAL_APIC
  30#include <asm/trace/irq_vectors.h>
  31#undef TRACE_INCLUDE_PATH
  32#undef TRACE_INCLUDE_FILE
  33#endif /* CONFIG_X86_LOCAL_APIC */
  34
  35#include <trace/events/irq.h>
  36#include <trace/events/sched.h>
  37
  38#define CREATE_TRACE_POINTS
  39#include <trace/events/osnoise.h>
  40
  41static struct trace_array       *osnoise_trace;
  42
  43/*
  44 * Default values.
  45 */
  46#define BANNER                  "osnoise: "
  47#define DEFAULT_SAMPLE_PERIOD   1000000                 /* 1s */
  48#define DEFAULT_SAMPLE_RUNTIME  1000000                 /* 1s */
  49
  50#define DEFAULT_TIMERLAT_PERIOD 1000                    /* 1ms */
  51#define DEFAULT_TIMERLAT_PRIO   95                      /* FIFO 95 */
  52
  53/*
  54 * NMI runtime info.
  55 */
  56struct osn_nmi {
  57        u64     count;
  58        u64     delta_start;
  59};
  60
  61/*
  62 * IRQ runtime info.
  63 */
  64struct osn_irq {
  65        u64     count;
  66        u64     arrival_time;
  67        u64     delta_start;
  68};
  69
  70#define IRQ_CONTEXT     0
  71#define THREAD_CONTEXT  1
  72/*
  73 * sofirq runtime info.
  74 */
  75struct osn_softirq {
  76        u64     count;
  77        u64     arrival_time;
  78        u64     delta_start;
  79};
  80
  81/*
  82 * thread runtime info.
  83 */
  84struct osn_thread {
  85        u64     count;
  86        u64     arrival_time;
  87        u64     delta_start;
  88};
  89
  90/*
  91 * Runtime information: this structure saves the runtime information used by
  92 * one sampling thread.
  93 */
  94struct osnoise_variables {
  95        struct task_struct      *kthread;
  96        bool                    sampling;
  97        pid_t                   pid;
  98        struct osn_nmi          nmi;
  99        struct osn_irq          irq;
 100        struct osn_softirq      softirq;
 101        struct osn_thread       thread;
 102        local_t                 int_counter;
 103};
 104
 105/*
 106 * Per-cpu runtime information.
 107 */
 108DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
 109
 110/*
 111 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
 112 */
 113static inline struct osnoise_variables *this_cpu_osn_var(void)
 114{
 115        return this_cpu_ptr(&per_cpu_osnoise_var);
 116}
 117
 118#ifdef CONFIG_TIMERLAT_TRACER
 119/*
 120 * Runtime information for the timer mode.
 121 */
 122struct timerlat_variables {
 123        struct task_struct      *kthread;
 124        struct hrtimer          timer;
 125        u64                     rel_period;
 126        u64                     abs_period;
 127        bool                    tracing_thread;
 128        u64                     count;
 129};
 130
 131DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
 132
 133/*
 134 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
 135 */
 136static inline struct timerlat_variables *this_cpu_tmr_var(void)
 137{
 138        return this_cpu_ptr(&per_cpu_timerlat_var);
 139}
 140
 141/*
 142 * tlat_var_reset - Reset the values of the given timerlat_variables
 143 */
 144static inline void tlat_var_reset(void)
 145{
 146        struct timerlat_variables *tlat_var;
 147        int cpu;
 148        /*
 149         * So far, all the values are initialized as 0, so
 150         * zeroing the structure is perfect.
 151         */
 152        for_each_cpu(cpu, cpu_online_mask) {
 153                tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
 154                memset(tlat_var, 0, sizeof(*tlat_var));
 155        }
 156}
 157#else /* CONFIG_TIMERLAT_TRACER */
 158#define tlat_var_reset()        do {} while (0)
 159#endif /* CONFIG_TIMERLAT_TRACER */
 160
 161/*
 162 * osn_var_reset - Reset the values of the given osnoise_variables
 163 */
 164static inline void osn_var_reset(void)
 165{
 166        struct osnoise_variables *osn_var;
 167        int cpu;
 168
 169        /*
 170         * So far, all the values are initialized as 0, so
 171         * zeroing the structure is perfect.
 172         */
 173        for_each_cpu(cpu, cpu_online_mask) {
 174                osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
 175                memset(osn_var, 0, sizeof(*osn_var));
 176        }
 177}
 178
 179/*
 180 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
 181 */
 182static inline void osn_var_reset_all(void)
 183{
 184        osn_var_reset();
 185        tlat_var_reset();
 186}
 187
 188/*
 189 * Tells NMIs to call back to the osnoise tracer to record timestamps.
 190 */
 191bool trace_osnoise_callback_enabled;
 192
 193/*
 194 * osnoise sample structure definition. Used to store the statistics of a
 195 * sample run.
 196 */
 197struct osnoise_sample {
 198        u64                     runtime;        /* runtime */
 199        u64                     noise;          /* noise */
 200        u64                     max_sample;     /* max single noise sample */
 201        int                     hw_count;       /* # HW (incl. hypervisor) interference */
 202        int                     nmi_count;      /* # NMIs during this sample */
 203        int                     irq_count;      /* # IRQs during this sample */
 204        int                     softirq_count;  /* # softirqs during this sample */
 205        int                     thread_count;   /* # threads during this sample */
 206};
 207
 208#ifdef CONFIG_TIMERLAT_TRACER
 209/*
 210 * timerlat sample structure definition. Used to store the statistics of
 211 * a sample run.
 212 */
 213struct timerlat_sample {
 214        u64                     timer_latency;  /* timer_latency */
 215        unsigned int            seqnum;         /* unique sequence */
 216        int                     context;        /* timer context */
 217};
 218#endif
 219
 220/*
 221 * Protect the interface.
 222 */
 223struct mutex interface_lock;
 224
 225/*
 226 * Tracer data.
 227 */
 228static struct osnoise_data {
 229        u64     sample_period;          /* total sampling period */
 230        u64     sample_runtime;         /* active sampling portion of period */
 231        u64     stop_tracing;           /* stop trace in the internal operation (loop/irq) */
 232        u64     stop_tracing_total;     /* stop trace in the final operation (report/thread) */
 233#ifdef CONFIG_TIMERLAT_TRACER
 234        u64     timerlat_period;        /* timerlat period */
 235        u64     print_stack;            /* print IRQ stack if total > */
 236        int     timerlat_tracer;        /* timerlat tracer */
 237#endif
 238        bool    tainted;                /* infor users and developers about a problem */
 239} osnoise_data = {
 240        .sample_period                  = DEFAULT_SAMPLE_PERIOD,
 241        .sample_runtime                 = DEFAULT_SAMPLE_RUNTIME,
 242        .stop_tracing                   = 0,
 243        .stop_tracing_total             = 0,
 244#ifdef CONFIG_TIMERLAT_TRACER
 245        .print_stack                    = 0,
 246        .timerlat_period                = DEFAULT_TIMERLAT_PERIOD,
 247        .timerlat_tracer                = 0,
 248#endif
 249};
 250
 251/*
 252 * Boolean variable used to inform that the tracer is currently sampling.
 253 */
 254static bool osnoise_busy;
 255
 256#ifdef CONFIG_PREEMPT_RT
 257/*
 258 * Print the osnoise header info.
 259 */
 260static void print_osnoise_headers(struct seq_file *s)
 261{
 262        if (osnoise_data.tainted)
 263                seq_puts(s, "# osnoise is tainted!\n");
 264
 265        seq_puts(s, "#                                _-------=> irqs-off\n");
 266        seq_puts(s, "#                               / _------=> need-resched\n");
 267        seq_puts(s, "#                              | / _-----=> need-resched-lazy\n");
 268        seq_puts(s, "#                              || / _----=> hardirq/softirq\n");
 269        seq_puts(s, "#                              ||| / _---=> preempt-depth\n");
 270        seq_puts(s, "#                              |||| / _--=> preempt-lazy-depth\n");
 271        seq_puts(s, "#                              ||||| / _-=> migrate-disable\n");
 272
 273        seq_puts(s, "#                              |||||| /          ");
 274        seq_puts(s, "                                     MAX\n");
 275
 276        seq_puts(s, "#                              ||||| /                         ");
 277        seq_puts(s, "                    SINGLE      Interference counters:\n");
 278
 279        seq_puts(s, "#                              |||||||               RUNTIME   ");
 280        seq_puts(s, "   NOISE  %% OF CPU  NOISE    +-----------------------------+\n");
 281
 282        seq_puts(s, "#           TASK-PID      CPU# |||||||   TIMESTAMP    IN US    ");
 283        seq_puts(s, "   IN US  AVAILABLE  IN US     HW    NMI    IRQ   SIRQ THREAD\n");
 284
 285        seq_puts(s, "#              | |         |   |||||||      |           |      ");
 286        seq_puts(s, "       |    |            |      |      |      |      |      |\n");
 287}
 288#else /* CONFIG_PREEMPT_RT */
 289static void print_osnoise_headers(struct seq_file *s)
 290{
 291        if (osnoise_data.tainted)
 292                seq_puts(s, "# osnoise is tainted!\n");
 293
 294        seq_puts(s, "#                                _-----=> irqs-off\n");
 295        seq_puts(s, "#                               / _----=> need-resched\n");
 296        seq_puts(s, "#                              | / _---=> hardirq/softirq\n");
 297        seq_puts(s, "#                              || / _--=> preempt-depth     ");
 298        seq_puts(s, "                       MAX\n");
 299
 300        seq_puts(s, "#                              || /                         ");
 301        seq_puts(s, "                    SINGLE      Interference counters:\n");
 302
 303        seq_puts(s, "#                              ||||               RUNTIME   ");
 304        seq_puts(s, "   NOISE  %% OF CPU  NOISE    +-----------------------------+\n");
 305
 306        seq_puts(s, "#           TASK-PID      CPU# ||||   TIMESTAMP    IN US    ");
 307        seq_puts(s, "   IN US  AVAILABLE  IN US     HW    NMI    IRQ   SIRQ THREAD\n");
 308
 309        seq_puts(s, "#              | |         |   ||||      |           |      ");
 310        seq_puts(s, "       |    |            |      |      |      |      |      |\n");
 311}
 312#endif /* CONFIG_PREEMPT_RT */
 313
 314/*
 315 * osnoise_taint - report an osnoise error.
 316 */
 317#define osnoise_taint(msg) ({                                                   \
 318        struct trace_array *tr = osnoise_trace;                                 \
 319                                                                                \
 320        trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, msg);        \
 321        osnoise_data.tainted = true;                                            \
 322})
 323
 324/*
 325 * Record an osnoise_sample into the tracer buffer.
 326 */
 327static void trace_osnoise_sample(struct osnoise_sample *sample)
 328{
 329        struct trace_array *tr = osnoise_trace;
 330        struct trace_buffer *buffer = tr->array_buffer.buffer;
 331        struct trace_event_call *call = &event_osnoise;
 332        struct ring_buffer_event *event;
 333        struct osnoise_entry *entry;
 334
 335        event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
 336                                          tracing_gen_ctx());
 337        if (!event)
 338                return;
 339        entry   = ring_buffer_event_data(event);
 340        entry->runtime          = sample->runtime;
 341        entry->noise            = sample->noise;
 342        entry->max_sample       = sample->max_sample;
 343        entry->hw_count         = sample->hw_count;
 344        entry->nmi_count        = sample->nmi_count;
 345        entry->irq_count        = sample->irq_count;
 346        entry->softirq_count    = sample->softirq_count;
 347        entry->thread_count     = sample->thread_count;
 348
 349        if (!call_filter_check_discard(call, entry, buffer, event))
 350                trace_buffer_unlock_commit_nostack(buffer, event);
 351}
 352
 353#ifdef CONFIG_TIMERLAT_TRACER
 354/*
 355 * Print the timerlat header info.
 356 */
 357#ifdef CONFIG_PREEMPT_RT
 358static void print_timerlat_headers(struct seq_file *s)
 359{
 360        seq_puts(s, "#                                _-------=> irqs-off\n");
 361        seq_puts(s, "#                               / _------=> need-resched\n");
 362        seq_puts(s, "#                              | / _-----=> need-resched-lazy\n");
 363        seq_puts(s, "#                              || / _----=> hardirq/softirq\n");
 364        seq_puts(s, "#                              ||| / _---=> preempt-depth\n");
 365        seq_puts(s, "#                              |||| / _--=> preempt-lazy-depth\n");
 366        seq_puts(s, "#                              ||||| / _-=> migrate-disable\n");
 367        seq_puts(s, "#                              |||||| /\n");
 368        seq_puts(s, "#                              |||||||             ACTIVATION\n");
 369        seq_puts(s, "#           TASK-PID      CPU# |||||||   TIMESTAMP    ID     ");
 370        seq_puts(s, "       CONTEXT                LATENCY\n");
 371        seq_puts(s, "#              | |         |   |||||||      |         |      ");
 372        seq_puts(s, "            |                       |\n");
 373}
 374#else /* CONFIG_PREEMPT_RT */
 375static void print_timerlat_headers(struct seq_file *s)
 376{
 377        seq_puts(s, "#                                _-----=> irqs-off\n");
 378        seq_puts(s, "#                               / _----=> need-resched\n");
 379        seq_puts(s, "#                              | / _---=> hardirq/softirq\n");
 380        seq_puts(s, "#                              || / _--=> preempt-depth\n");
 381        seq_puts(s, "#                              || /\n");
 382        seq_puts(s, "#                              ||||             ACTIVATION\n");
 383        seq_puts(s, "#           TASK-PID      CPU# ||||   TIMESTAMP    ID     ");
 384        seq_puts(s, "       CONTEXT                LATENCY\n");
 385        seq_puts(s, "#              | |         |   ||||      |         |      ");
 386        seq_puts(s, "            |                       |\n");
 387}
 388#endif /* CONFIG_PREEMPT_RT */
 389
 390/*
 391 * Record an timerlat_sample into the tracer buffer.
 392 */
 393static void trace_timerlat_sample(struct timerlat_sample *sample)
 394{
 395        struct trace_array *tr = osnoise_trace;
 396        struct trace_event_call *call = &event_osnoise;
 397        struct trace_buffer *buffer = tr->array_buffer.buffer;
 398        struct ring_buffer_event *event;
 399        struct timerlat_entry *entry;
 400
 401        event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
 402                                          tracing_gen_ctx());
 403        if (!event)
 404                return;
 405        entry   = ring_buffer_event_data(event);
 406        entry->seqnum                   = sample->seqnum;
 407        entry->context                  = sample->context;
 408        entry->timer_latency            = sample->timer_latency;
 409
 410        if (!call_filter_check_discard(call, entry, buffer, event))
 411                trace_buffer_unlock_commit_nostack(buffer, event);
 412}
 413
 414#ifdef CONFIG_STACKTRACE
 415
 416#define MAX_CALLS       256
 417
 418/*
 419 * Stack trace will take place only at IRQ level, so, no need
 420 * to control nesting here.
 421 */
 422struct trace_stack {
 423        int             stack_size;
 424        int             nr_entries;
 425        unsigned long   calls[MAX_CALLS];
 426};
 427
 428static DEFINE_PER_CPU(struct trace_stack, trace_stack);
 429
 430/*
 431 * timerlat_save_stack - save a stack trace without printing
 432 *
 433 * Save the current stack trace without printing. The
 434 * stack will be printed later, after the end of the measurement.
 435 */
 436static void timerlat_save_stack(int skip)
 437{
 438        unsigned int size, nr_entries;
 439        struct trace_stack *fstack;
 440
 441        fstack = this_cpu_ptr(&trace_stack);
 442
 443        size = ARRAY_SIZE(fstack->calls);
 444
 445        nr_entries = stack_trace_save(fstack->calls, size, skip);
 446
 447        fstack->stack_size = nr_entries * sizeof(unsigned long);
 448        fstack->nr_entries = nr_entries;
 449
 450        return;
 451
 452}
 453/*
 454 * timerlat_dump_stack - dump a stack trace previously saved
 455 *
 456 * Dump a saved stack trace into the trace buffer.
 457 */
 458static void timerlat_dump_stack(void)
 459{
 460        struct trace_event_call *call = &event_osnoise;
 461        struct trace_array *tr = osnoise_trace;
 462        struct trace_buffer *buffer = tr->array_buffer.buffer;
 463        struct ring_buffer_event *event;
 464        struct trace_stack *fstack;
 465        struct stack_entry *entry;
 466        unsigned int size;
 467
 468        preempt_disable_notrace();
 469        fstack = this_cpu_ptr(&trace_stack);
 470        size = fstack->stack_size;
 471
 472        event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
 473                                          tracing_gen_ctx());
 474        if (!event)
 475                goto out;
 476
 477        entry = ring_buffer_event_data(event);
 478
 479        memcpy(&entry->caller, fstack->calls, size);
 480        entry->size = fstack->nr_entries;
 481
 482        if (!call_filter_check_discard(call, entry, buffer, event))
 483                trace_buffer_unlock_commit_nostack(buffer, event);
 484
 485out:
 486        preempt_enable_notrace();
 487}
 488#else
 489#define timerlat_dump_stack() do {} while (0)
 490#define timerlat_save_stack(a) do {} while (0)
 491#endif /* CONFIG_STACKTRACE */
 492#endif /* CONFIG_TIMERLAT_TRACER */
 493
 494/*
 495 * Macros to encapsulate the time capturing infrastructure.
 496 */
 497#define time_get()      trace_clock_local()
 498#define time_to_us(x)   div_u64(x, 1000)
 499#define time_sub(a, b)  ((a) - (b))
 500
 501/*
 502 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
 503 *
 504 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
 505 * to discount the NMI interference.
 506 *
 507 * See get_int_safe_duration().
 508 */
 509static inline void
 510cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
 511{
 512        if (osn_var->irq.delta_start)
 513                osn_var->irq.delta_start += duration;
 514}
 515
 516#ifndef CONFIG_PREEMPT_RT
 517/*
 518 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
 519 *
 520 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
 521 * forward to discount the interference.
 522 *
 523 * See get_int_safe_duration().
 524 */
 525static inline void
 526cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
 527{
 528        if (osn_var->softirq.delta_start)
 529                osn_var->softirq.delta_start += duration;
 530}
 531#else /* CONFIG_PREEMPT_RT */
 532#define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
 533#endif
 534
 535/*
 536 * cond_move_thread_delta_start - Forward the delta_start of a running thread
 537 *
 538 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
 539 * is pushed forward to discount the interference.
 540 *
 541 * See get_int_safe_duration().
 542 */
 543static inline void
 544cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
 545{
 546        if (osn_var->thread.delta_start)
 547                osn_var->thread.delta_start += duration;
 548}
 549
 550/*
 551 * get_int_safe_duration - Get the duration of a window
 552 *
 553 * The irq, softirq and thread varaibles need to have its duration without
 554 * the interference from higher priority interrupts. Instead of keeping a
 555 * variable to discount the interrupt interference from these variables, the
 556 * starting time of these variables are pushed forward with the interrupt's
 557 * duration. In this way, a single variable is used to:
 558 *
 559 *   - Know if a given window is being measured.
 560 *   - Account its duration.
 561 *   - Discount the interference.
 562 *
 563 * To avoid getting inconsistent values, e.g.,:
 564 *
 565 *      now = time_get()
 566 *              --->    interrupt!
 567 *                      delta_start -= int duration;
 568 *              <---
 569 *      duration = now - delta_start;
 570 *
 571 *      result: negative duration if the variable duration before the
 572 *      interrupt was smaller than the interrupt execution.
 573 *
 574 * A counter of interrupts is used. If the counter increased, try
 575 * to capture an interference safe duration.
 576 */
 577static inline s64
 578get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
 579{
 580        u64 int_counter, now;
 581        s64 duration;
 582
 583        do {
 584                int_counter = local_read(&osn_var->int_counter);
 585                /* synchronize with interrupts */
 586                barrier();
 587
 588                now = time_get();
 589                duration = (now - *delta_start);
 590
 591                /* synchronize with interrupts */
 592                barrier();
 593        } while (int_counter != local_read(&osn_var->int_counter));
 594
 595        /*
 596         * This is an evidence of race conditions that cause
 597         * a value to be "discounted" too much.
 598         */
 599        if (duration < 0)
 600                osnoise_taint("Negative duration!\n");
 601
 602        *delta_start = 0;
 603
 604        return duration;
 605}
 606
 607/*
 608 *
 609 * set_int_safe_time - Save the current time on *time, aware of interference
 610 *
 611 * Get the time, taking into consideration a possible interference from
 612 * higher priority interrupts.
 613 *
 614 * See get_int_safe_duration() for an explanation.
 615 */
 616static u64
 617set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
 618{
 619        u64 int_counter;
 620
 621        do {
 622                int_counter = local_read(&osn_var->int_counter);
 623                /* synchronize with interrupts */
 624                barrier();
 625
 626                *time = time_get();
 627
 628                /* synchronize with interrupts */
 629                barrier();
 630        } while (int_counter != local_read(&osn_var->int_counter));
 631
 632        return int_counter;
 633}
 634
 635#ifdef CONFIG_TIMERLAT_TRACER
 636/*
 637 * copy_int_safe_time - Copy *src into *desc aware of interference
 638 */
 639static u64
 640copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
 641{
 642        u64 int_counter;
 643
 644        do {
 645                int_counter = local_read(&osn_var->int_counter);
 646                /* synchronize with interrupts */
 647                barrier();
 648
 649                *dst = *src;
 650
 651                /* synchronize with interrupts */
 652                barrier();
 653        } while (int_counter != local_read(&osn_var->int_counter));
 654
 655        return int_counter;
 656}
 657#endif /* CONFIG_TIMERLAT_TRACER */
 658
 659/*
 660 * trace_osnoise_callback - NMI entry/exit callback
 661 *
 662 * This function is called at the entry and exit NMI code. The bool enter
 663 * distinguishes between either case. This function is used to note a NMI
 664 * occurrence, compute the noise caused by the NMI, and to remove the noise
 665 * it is potentially causing on other interference variables.
 666 */
 667void trace_osnoise_callback(bool enter)
 668{
 669        struct osnoise_variables *osn_var = this_cpu_osn_var();
 670        u64 duration;
 671
 672        if (!osn_var->sampling)
 673                return;
 674
 675        /*
 676         * Currently trace_clock_local() calls sched_clock() and the
 677         * generic version is not NMI safe.
 678         */
 679        if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
 680                if (enter) {
 681                        osn_var->nmi.delta_start = time_get();
 682                        local_inc(&osn_var->int_counter);
 683                } else {
 684                        duration = time_get() - osn_var->nmi.delta_start;
 685
 686                        trace_nmi_noise(osn_var->nmi.delta_start, duration);
 687
 688                        cond_move_irq_delta_start(osn_var, duration);
 689                        cond_move_softirq_delta_start(osn_var, duration);
 690                        cond_move_thread_delta_start(osn_var, duration);
 691                }
 692        }
 693
 694        if (enter)
 695                osn_var->nmi.count++;
 696}
 697
 698/*
 699 * osnoise_trace_irq_entry - Note the starting of an IRQ
 700 *
 701 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
 702 * it is safe to use a single variable (ons_var->irq) to save the statistics.
 703 * The arrival_time is used to report... the arrival time. The delta_start
 704 * is used to compute the duration at the IRQ exit handler. See
 705 * cond_move_irq_delta_start().
 706 */
 707void osnoise_trace_irq_entry(int id)
 708{
 709        struct osnoise_variables *osn_var = this_cpu_osn_var();
 710
 711        if (!osn_var->sampling)
 712                return;
 713        /*
 714         * This value will be used in the report, but not to compute
 715         * the execution time, so it is safe to get it unsafe.
 716         */
 717        osn_var->irq.arrival_time = time_get();
 718        set_int_safe_time(osn_var, &osn_var->irq.delta_start);
 719        osn_var->irq.count++;
 720
 721        local_inc(&osn_var->int_counter);
 722}
 723
 724/*
 725 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
 726 *
 727 * Computes the duration of the IRQ noise, and trace it. Also discounts the
 728 * interference from other sources of noise could be currently being accounted.
 729 */
 730void osnoise_trace_irq_exit(int id, const char *desc)
 731{
 732        struct osnoise_variables *osn_var = this_cpu_osn_var();
 733        int duration;
 734
 735        if (!osn_var->sampling)
 736                return;
 737
 738        duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
 739        trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
 740        osn_var->irq.arrival_time = 0;
 741        cond_move_softirq_delta_start(osn_var, duration);
 742        cond_move_thread_delta_start(osn_var, duration);
 743}
 744
 745/*
 746 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
 747 *
 748 * Used to note the starting of an IRQ occurece.
 749 */
 750static void trace_irqentry_callback(void *data, int irq,
 751                                    struct irqaction *action)
 752{
 753        osnoise_trace_irq_entry(irq);
 754}
 755
 756/*
 757 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
 758 *
 759 * Used to note the end of an IRQ occurece.
 760 */
 761static void trace_irqexit_callback(void *data, int irq,
 762                                   struct irqaction *action, int ret)
 763{
 764        osnoise_trace_irq_exit(irq, action->name);
 765}
 766
 767/*
 768 * arch specific register function.
 769 */
 770int __weak osnoise_arch_register(void)
 771{
 772        return 0;
 773}
 774
 775/*
 776 * arch specific unregister function.
 777 */
 778void __weak osnoise_arch_unregister(void)
 779{
 780        return;
 781}
 782
 783/*
 784 * hook_irq_events - Hook IRQ handling events
 785 *
 786 * This function hooks the IRQ related callbacks to the respective trace
 787 * events.
 788 */
 789static int hook_irq_events(void)
 790{
 791        int ret;
 792
 793        ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
 794        if (ret)
 795                goto out_err;
 796
 797        ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
 798        if (ret)
 799                goto out_unregister_entry;
 800
 801        ret = osnoise_arch_register();
 802        if (ret)
 803                goto out_irq_exit;
 804
 805        return 0;
 806
 807out_irq_exit:
 808        unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
 809out_unregister_entry:
 810        unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
 811out_err:
 812        return -EINVAL;
 813}
 814
 815/*
 816 * unhook_irq_events - Unhook IRQ handling events
 817 *
 818 * This function unhooks the IRQ related callbacks to the respective trace
 819 * events.
 820 */
 821static void unhook_irq_events(void)
 822{
 823        osnoise_arch_unregister();
 824        unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
 825        unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
 826}
 827
 828#ifndef CONFIG_PREEMPT_RT
 829/*
 830 * trace_softirq_entry_callback - Note the starting of a softirq
 831 *
 832 * Save the starting time of a softirq. As softirqs are non-preemptive to
 833 * other softirqs, it is safe to use a single variable (ons_var->softirq)
 834 * to save the statistics. The arrival_time is used to report... the
 835 * arrival time. The delta_start is used to compute the duration at the
 836 * softirq exit handler. See cond_move_softirq_delta_start().
 837 */
 838static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
 839{
 840        struct osnoise_variables *osn_var = this_cpu_osn_var();
 841
 842        if (!osn_var->sampling)
 843                return;
 844        /*
 845         * This value will be used in the report, but not to compute
 846         * the execution time, so it is safe to get it unsafe.
 847         */
 848        osn_var->softirq.arrival_time = time_get();
 849        set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
 850        osn_var->softirq.count++;
 851
 852        local_inc(&osn_var->int_counter);
 853}
 854
 855/*
 856 * trace_softirq_exit_callback - Note the end of an softirq
 857 *
 858 * Computes the duration of the softirq noise, and trace it. Also discounts the
 859 * interference from other sources of noise could be currently being accounted.
 860 */
 861static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
 862{
 863        struct osnoise_variables *osn_var = this_cpu_osn_var();
 864        int duration;
 865
 866        if (!osn_var->sampling)
 867                return;
 868
 869#ifdef CONFIG_TIMERLAT_TRACER
 870        /*
 871         * If the timerlat is enabled, but the irq handler did
 872         * not run yet enabling timerlat_tracer, do not trace.
 873         */
 874        if (unlikely(osnoise_data.timerlat_tracer)) {
 875                struct timerlat_variables *tlat_var;
 876                tlat_var = this_cpu_tmr_var();
 877                if (!tlat_var->tracing_thread) {
 878                        osn_var->softirq.arrival_time = 0;
 879                        osn_var->softirq.delta_start = 0;
 880                        return;
 881                }
 882        }
 883#endif
 884
 885        duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
 886        trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
 887        cond_move_thread_delta_start(osn_var, duration);
 888        osn_var->softirq.arrival_time = 0;
 889}
 890
 891/*
 892 * hook_softirq_events - Hook softirq handling events
 893 *
 894 * This function hooks the softirq related callbacks to the respective trace
 895 * events.
 896 */
 897static int hook_softirq_events(void)
 898{
 899        int ret;
 900
 901        ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
 902        if (ret)
 903                goto out_err;
 904
 905        ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
 906        if (ret)
 907                goto out_unreg_entry;
 908
 909        return 0;
 910
 911out_unreg_entry:
 912        unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
 913out_err:
 914        return -EINVAL;
 915}
 916
 917/*
 918 * unhook_softirq_events - Unhook softirq handling events
 919 *
 920 * This function hooks the softirq related callbacks to the respective trace
 921 * events.
 922 */
 923static void unhook_softirq_events(void)
 924{
 925        unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
 926        unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
 927}
 928#else /* CONFIG_PREEMPT_RT */
 929/*
 930 * softirq are threads on the PREEMPT_RT mode.
 931 */
 932static int hook_softirq_events(void)
 933{
 934        return 0;
 935}
 936static void unhook_softirq_events(void)
 937{
 938}
 939#endif
 940
 941/*
 942 * thread_entry - Record the starting of a thread noise window
 943 *
 944 * It saves the context switch time for a noisy thread, and increments
 945 * the interference counters.
 946 */
 947static void
 948thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
 949{
 950        if (!osn_var->sampling)
 951                return;
 952        /*
 953         * The arrival time will be used in the report, but not to compute
 954         * the execution time, so it is safe to get it unsafe.
 955         */
 956        osn_var->thread.arrival_time = time_get();
 957
 958        set_int_safe_time(osn_var, &osn_var->thread.delta_start);
 959
 960        osn_var->thread.count++;
 961        local_inc(&osn_var->int_counter);
 962}
 963
 964/*
 965 * thread_exit - Report the end of a thread noise window
 966 *
 967 * It computes the total noise from a thread, tracing if needed.
 968 */
 969static void
 970thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
 971{
 972        int duration;
 973
 974        if (!osn_var->sampling)
 975                return;
 976
 977#ifdef CONFIG_TIMERLAT_TRACER
 978        if (osnoise_data.timerlat_tracer) {
 979                struct timerlat_variables *tlat_var;
 980                tlat_var = this_cpu_tmr_var();
 981                if (!tlat_var->tracing_thread) {
 982                        osn_var->thread.delta_start = 0;
 983                        osn_var->thread.arrival_time = 0;
 984                        return;
 985                }
 986        }
 987#endif
 988
 989        duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
 990
 991        trace_thread_noise(t, osn_var->thread.arrival_time, duration);
 992
 993        osn_var->thread.arrival_time = 0;
 994}
 995
 996/*
 997 * trace_sched_switch - sched:sched_switch trace event handler
 998 *
 999 * This function is hooked to the sched:sched_switch trace event, and it is
1000 * used to record the beginning and to report the end of a thread noise window.
1001 */
1002static void
1003trace_sched_switch_callback(void *data, bool preempt, struct task_struct *p,
1004                            struct task_struct *n)
1005{
1006        struct osnoise_variables *osn_var = this_cpu_osn_var();
1007
1008        if (p->pid != osn_var->pid)
1009                thread_exit(osn_var, p);
1010
1011        if (n->pid != osn_var->pid)
1012                thread_entry(osn_var, n);
1013}
1014
1015/*
1016 * hook_thread_events - Hook the insturmentation for thread noise
1017 *
1018 * Hook the osnoise tracer callbacks to handle the noise from other
1019 * threads on the necessary kernel events.
1020 */
1021static int hook_thread_events(void)
1022{
1023        int ret;
1024
1025        ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1026        if (ret)
1027                return -EINVAL;
1028
1029        return 0;
1030}
1031
1032/*
1033 * unhook_thread_events - *nhook the insturmentation for thread noise
1034 *
1035 * Unook the osnoise tracer callbacks to handle the noise from other
1036 * threads on the necessary kernel events.
1037 */
1038static void unhook_thread_events(void)
1039{
1040        unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1041}
1042
1043/*
1044 * save_osn_sample_stats - Save the osnoise_sample statistics
1045 *
1046 * Save the osnoise_sample statistics before the sampling phase. These
1047 * values will be used later to compute the diff betwneen the statistics
1048 * before and after the osnoise sampling.
1049 */
1050static void
1051save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1052{
1053        s->nmi_count = osn_var->nmi.count;
1054        s->irq_count = osn_var->irq.count;
1055        s->softirq_count = osn_var->softirq.count;
1056        s->thread_count = osn_var->thread.count;
1057}
1058
1059/*
1060 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1061 *
1062 * After a sample period, compute the difference on the osnoise_sample
1063 * statistics. The struct osnoise_sample *s contains the statistics saved via
1064 * save_osn_sample_stats() before the osnoise sampling.
1065 */
1066static void
1067diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1068{
1069        s->nmi_count = osn_var->nmi.count - s->nmi_count;
1070        s->irq_count = osn_var->irq.count - s->irq_count;
1071        s->softirq_count = osn_var->softirq.count - s->softirq_count;
1072        s->thread_count = osn_var->thread.count - s->thread_count;
1073}
1074
1075/*
1076 * osnoise_stop_tracing - Stop tracing and the tracer.
1077 */
1078static __always_inline void osnoise_stop_tracing(void)
1079{
1080        struct trace_array *tr = osnoise_trace;
1081
1082        trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1083                        "stop tracing hit on cpu %d\n", smp_processor_id());
1084
1085        tracer_tracing_off(tr);
1086}
1087
1088/*
1089 * run_osnoise - Sample the time and look for osnoise
1090 *
1091 * Used to capture the time, looking for potential osnoise latency repeatedly.
1092 * Different from hwlat_detector, it is called with preemption and interrupts
1093 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1094 * osnoise sampling thread, as they would do with a regular thread.
1095 */
1096static int run_osnoise(void)
1097{
1098        struct osnoise_variables *osn_var = this_cpu_osn_var();
1099        struct trace_array *tr = osnoise_trace;
1100        u64 start, sample, last_sample;
1101        u64 last_int_count, int_count;
1102        s64 noise = 0, max_noise = 0;
1103        s64 total, last_total = 0;
1104        struct osnoise_sample s;
1105        unsigned int threshold;
1106        u64 runtime, stop_in;
1107        u64 sum_noise = 0;
1108        int hw_count = 0;
1109        int ret = -1;
1110
1111        /*
1112         * Considers the current thread as the workload.
1113         */
1114        osn_var->pid = current->pid;
1115
1116        /*
1117         * Save the current stats for the diff
1118         */
1119        save_osn_sample_stats(osn_var, &s);
1120
1121        /*
1122         * if threshold is 0, use the default value of 5 us.
1123         */
1124        threshold = tracing_thresh ? : 5000;
1125
1126        /*
1127         * Make sure NMIs see sampling first
1128         */
1129        osn_var->sampling = true;
1130        barrier();
1131
1132        /*
1133         * Transform the *_us config to nanoseconds to avoid the
1134         * division on the main loop.
1135         */
1136        runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1137        stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1138
1139        /*
1140         * Start timestemp
1141         */
1142        start = time_get();
1143
1144        /*
1145         * "previous" loop.
1146         */
1147        last_int_count = set_int_safe_time(osn_var, &last_sample);
1148
1149        do {
1150                /*
1151                 * Get sample!
1152                 */
1153                int_count = set_int_safe_time(osn_var, &sample);
1154
1155                noise = time_sub(sample, last_sample);
1156
1157                /*
1158                 * This shouldn't happen.
1159                 */
1160                if (noise < 0) {
1161                        osnoise_taint("negative noise!");
1162                        goto out;
1163                }
1164
1165                /*
1166                 * Sample runtime.
1167                 */
1168                total = time_sub(sample, start);
1169
1170                /*
1171                 * Check for possible overflows.
1172                 */
1173                if (total < last_total) {
1174                        osnoise_taint("total overflow!");
1175                        break;
1176                }
1177
1178                last_total = total;
1179
1180                if (noise >= threshold) {
1181                        int interference = int_count - last_int_count;
1182
1183                        if (noise > max_noise)
1184                                max_noise = noise;
1185
1186                        if (!interference)
1187                                hw_count++;
1188
1189                        sum_noise += noise;
1190
1191                        trace_sample_threshold(last_sample, noise, interference);
1192
1193                        if (osnoise_data.stop_tracing)
1194                                if (noise > stop_in)
1195                                        osnoise_stop_tracing();
1196                }
1197
1198                /*
1199                 * For the non-preemptive kernel config: let threads runs, if
1200                 * they so wish.
1201                 */
1202                cond_resched();
1203
1204                last_sample = sample;
1205                last_int_count = int_count;
1206
1207        } while (total < runtime && !kthread_should_stop());
1208
1209        /*
1210         * Finish the above in the view for interrupts.
1211         */
1212        barrier();
1213
1214        osn_var->sampling = false;
1215
1216        /*
1217         * Make sure sampling data is no longer updated.
1218         */
1219        barrier();
1220
1221        /*
1222         * Save noise info.
1223         */
1224        s.noise = time_to_us(sum_noise);
1225        s.runtime = time_to_us(total);
1226        s.max_sample = time_to_us(max_noise);
1227        s.hw_count = hw_count;
1228
1229        /* Save interference stats info */
1230        diff_osn_sample_stats(osn_var, &s);
1231
1232        trace_osnoise_sample(&s);
1233
1234        /* Keep a running maximum ever recorded osnoise "latency" */
1235        if (max_noise > tr->max_latency) {
1236                tr->max_latency = max_noise;
1237                latency_fsnotify(tr);
1238        }
1239
1240        if (osnoise_data.stop_tracing_total)
1241                if (s.noise > osnoise_data.stop_tracing_total)
1242                        osnoise_stop_tracing();
1243
1244        return 0;
1245out:
1246        return ret;
1247}
1248
1249static struct cpumask osnoise_cpumask;
1250static struct cpumask save_cpumask;
1251
1252/*
1253 * osnoise_main - The osnoise detection kernel thread
1254 *
1255 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1256 * every period.
1257 */
1258static int osnoise_main(void *data)
1259{
1260        u64 interval;
1261
1262        while (!kthread_should_stop()) {
1263
1264                run_osnoise();
1265
1266                mutex_lock(&interface_lock);
1267                interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1268                mutex_unlock(&interface_lock);
1269
1270                do_div(interval, USEC_PER_MSEC);
1271
1272                /*
1273                 * differently from hwlat_detector, the osnoise tracer can run
1274                 * without a pause because preemption is on.
1275                 */
1276                if (interval < 1) {
1277                        /* Let synchronize_rcu_tasks() make progress */
1278                        cond_resched_tasks_rcu_qs();
1279                        continue;
1280                }
1281
1282                if (msleep_interruptible(interval))
1283                        break;
1284        }
1285
1286        return 0;
1287}
1288
1289#ifdef CONFIG_TIMERLAT_TRACER
1290/*
1291 * timerlat_irq - hrtimer handler for timerlat.
1292 */
1293static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1294{
1295        struct osnoise_variables *osn_var = this_cpu_osn_var();
1296        struct trace_array *tr = osnoise_trace;
1297        struct timerlat_variables *tlat;
1298        struct timerlat_sample s;
1299        u64 now;
1300        u64 diff;
1301
1302        /*
1303         * I am not sure if the timer was armed for this CPU. So, get
1304         * the timerlat struct from the timer itself, not from this
1305         * CPU.
1306         */
1307        tlat = container_of(timer, struct timerlat_variables, timer);
1308
1309        now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1310
1311        /*
1312         * Enable the osnoise: events for thread an softirq.
1313         */
1314        tlat->tracing_thread = true;
1315
1316        osn_var->thread.arrival_time = time_get();
1317
1318        /*
1319         * A hardirq is running: the timer IRQ. It is for sure preempting
1320         * a thread, and potentially preempting a softirq.
1321         *
1322         * At this point, it is not interesting to know the duration of the
1323         * preempted thread (and maybe softirq), but how much time they will
1324         * delay the beginning of the execution of the timer thread.
1325         *
1326         * To get the correct (net) delay added by the softirq, its delta_start
1327         * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1328         * start of the sofitrq will be zeroed, accounting then only the time
1329         * after that.
1330         *
1331         * The thread follows the same principle. However, if a softirq is
1332         * running, the thread needs to receive the softirq delta_start. The
1333         * reason being is that the softirq will be the last to be unfolded,
1334         * resseting the thread delay to zero.
1335         */
1336#ifndef CONFIG_PREEMPT_RT
1337        if (osn_var->softirq.delta_start) {
1338                copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1339                                   &osn_var->softirq.delta_start);
1340
1341                copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1342                                    &osn_var->irq.delta_start);
1343        } else {
1344                copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1345                                    &osn_var->irq.delta_start);
1346        }
1347#else /* CONFIG_PREEMPT_RT */
1348        /*
1349         * The sofirqs run as threads on RT, so there is not need
1350         * to keep track of it.
1351         */
1352        copy_int_safe_time(osn_var, &osn_var->thread.delta_start, &osn_var->irq.delta_start);
1353#endif /* CONFIG_PREEMPT_RT */
1354
1355        /*
1356         * Compute the current time with the expected time.
1357         */
1358        diff = now - tlat->abs_period;
1359
1360        tlat->count++;
1361        s.seqnum = tlat->count;
1362        s.timer_latency = diff;
1363        s.context = IRQ_CONTEXT;
1364
1365        trace_timerlat_sample(&s);
1366
1367        /* Keep a running maximum ever recorded os noise "latency" */
1368        if (diff > tr->max_latency) {
1369                tr->max_latency = diff;
1370                latency_fsnotify(tr);
1371        }
1372
1373        if (osnoise_data.stop_tracing)
1374                if (time_to_us(diff) >= osnoise_data.stop_tracing)
1375                        osnoise_stop_tracing();
1376
1377        wake_up_process(tlat->kthread);
1378
1379        if (osnoise_data.print_stack)
1380                timerlat_save_stack(0);
1381
1382        return HRTIMER_NORESTART;
1383}
1384
1385/*
1386 * wait_next_period - Wait for the next period for timerlat
1387 */
1388static int wait_next_period(struct timerlat_variables *tlat)
1389{
1390        ktime_t next_abs_period, now;
1391        u64 rel_period = osnoise_data.timerlat_period * 1000;
1392
1393        now = hrtimer_cb_get_time(&tlat->timer);
1394        next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1395
1396        /*
1397         * Save the next abs_period.
1398         */
1399        tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1400
1401        /*
1402         * If the new abs_period is in the past, skip the activation.
1403         */
1404        while (ktime_compare(now, next_abs_period) > 0) {
1405                next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1406                tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1407        }
1408
1409        set_current_state(TASK_INTERRUPTIBLE);
1410
1411        hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1412        schedule();
1413        return 1;
1414}
1415
1416/*
1417 * timerlat_main- Timerlat main
1418 */
1419static int timerlat_main(void *data)
1420{
1421        struct osnoise_variables *osn_var = this_cpu_osn_var();
1422        struct timerlat_variables *tlat = this_cpu_tmr_var();
1423        struct timerlat_sample s;
1424        struct sched_param sp;
1425        u64 now, diff;
1426
1427        /*
1428         * Make the thread RT, that is how cyclictest is usually used.
1429         */
1430        sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1431        sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1432
1433        tlat->count = 0;
1434        tlat->tracing_thread = false;
1435
1436        hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1437        tlat->timer.function = timerlat_irq;
1438        tlat->kthread = current;
1439        osn_var->pid = current->pid;
1440        /*
1441         * Anotate the arrival time.
1442         */
1443        tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1444
1445        wait_next_period(tlat);
1446
1447        osn_var->sampling = 1;
1448
1449        while (!kthread_should_stop()) {
1450                now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1451                diff = now - tlat->abs_period;
1452
1453                s.seqnum = tlat->count;
1454                s.timer_latency = diff;
1455                s.context = THREAD_CONTEXT;
1456
1457                trace_timerlat_sample(&s);
1458
1459#ifdef CONFIG_STACKTRACE
1460                if (osnoise_data.print_stack)
1461                        if (osnoise_data.print_stack <= time_to_us(diff))
1462                                timerlat_dump_stack();
1463#endif /* CONFIG_STACKTRACE */
1464
1465                tlat->tracing_thread = false;
1466                if (osnoise_data.stop_tracing_total)
1467                        if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1468                                osnoise_stop_tracing();
1469
1470                wait_next_period(tlat);
1471        }
1472
1473        hrtimer_cancel(&tlat->timer);
1474        return 0;
1475}
1476#endif /* CONFIG_TIMERLAT_TRACER */
1477
1478/*
1479 * stop_kthread - stop a workload thread
1480 */
1481static void stop_kthread(unsigned int cpu)
1482{
1483        struct task_struct *kthread;
1484
1485        kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread;
1486        if (kthread)
1487                kthread_stop(kthread);
1488        per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1489}
1490
1491/*
1492 * stop_per_cpu_kthread - Stop per-cpu threads
1493 *
1494 * Stop the osnoise sampling htread. Use this on unload and at system
1495 * shutdown.
1496 */
1497static void stop_per_cpu_kthreads(void)
1498{
1499        int cpu;
1500
1501        get_online_cpus();
1502
1503        for_each_online_cpu(cpu)
1504                stop_kthread(cpu);
1505
1506        put_online_cpus();
1507}
1508
1509/*
1510 * start_kthread - Start a workload tread
1511 */
1512static int start_kthread(unsigned int cpu)
1513{
1514        struct task_struct *kthread;
1515        void *main = osnoise_main;
1516        char comm[24];
1517
1518#ifdef CONFIG_TIMERLAT_TRACER
1519        if (osnoise_data.timerlat_tracer) {
1520                snprintf(comm, 24, "timerlat/%d", cpu);
1521                main = timerlat_main;
1522        } else {
1523                snprintf(comm, 24, "osnoise/%d", cpu);
1524        }
1525#else
1526        snprintf(comm, 24, "osnoise/%d", cpu);
1527#endif
1528        kthread = kthread_create_on_cpu(main, NULL, cpu, comm);
1529
1530        if (IS_ERR(kthread)) {
1531                pr_err(BANNER "could not start sampling thread\n");
1532                stop_per_cpu_kthreads();
1533                return -ENOMEM;
1534        }
1535
1536        per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
1537        wake_up_process(kthread);
1538
1539        return 0;
1540}
1541
1542/*
1543 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
1544 *
1545 * This starts the kernel thread that will look for osnoise on many
1546 * cpus.
1547 */
1548static int start_per_cpu_kthreads(struct trace_array *tr)
1549{
1550        struct cpumask *current_mask = &save_cpumask;
1551        int retval;
1552        int cpu;
1553
1554        get_online_cpus();
1555        /*
1556         * Run only on CPUs in which trace and osnoise are allowed to run.
1557         */
1558        cpumask_and(current_mask, tr->tracing_cpumask, &osnoise_cpumask);
1559        /*
1560         * And the CPU is online.
1561         */
1562        cpumask_and(current_mask, cpu_online_mask, current_mask);
1563
1564        for_each_possible_cpu(cpu)
1565                per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1566
1567        for_each_cpu(cpu, current_mask) {
1568                retval = start_kthread(cpu);
1569                if (retval) {
1570                        stop_per_cpu_kthreads();
1571                        return retval;
1572                }
1573        }
1574
1575        put_online_cpus();
1576
1577        return 0;
1578}
1579
1580#ifdef CONFIG_HOTPLUG_CPU
1581static void osnoise_hotplug_workfn(struct work_struct *dummy)
1582{
1583        struct trace_array *tr = osnoise_trace;
1584        unsigned int cpu = smp_processor_id();
1585
1586
1587        mutex_lock(&trace_types_lock);
1588
1589        if (!osnoise_busy)
1590                goto out_unlock_trace;
1591
1592        mutex_lock(&interface_lock);
1593        get_online_cpus();
1594
1595        if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
1596                goto out_unlock;
1597
1598        if (!cpumask_test_cpu(cpu, tr->tracing_cpumask))
1599                goto out_unlock;
1600
1601        start_kthread(cpu);
1602
1603out_unlock:
1604        put_online_cpus();
1605        mutex_unlock(&interface_lock);
1606out_unlock_trace:
1607        mutex_unlock(&trace_types_lock);
1608}
1609
1610static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
1611
1612/*
1613 * osnoise_cpu_init - CPU hotplug online callback function
1614 */
1615static int osnoise_cpu_init(unsigned int cpu)
1616{
1617        schedule_work_on(cpu, &osnoise_hotplug_work);
1618        return 0;
1619}
1620
1621/*
1622 * osnoise_cpu_die - CPU hotplug offline callback function
1623 */
1624static int osnoise_cpu_die(unsigned int cpu)
1625{
1626        stop_kthread(cpu);
1627        return 0;
1628}
1629
1630static void osnoise_init_hotplug_support(void)
1631{
1632        int ret;
1633
1634        ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
1635                                osnoise_cpu_init, osnoise_cpu_die);
1636        if (ret < 0)
1637                pr_warn(BANNER "Error to init cpu hotplug support\n");
1638
1639        return;
1640}
1641#else /* CONFIG_HOTPLUG_CPU */
1642static void osnoise_init_hotplug_support(void)
1643{
1644        return;
1645}
1646#endif /* CONFIG_HOTPLUG_CPU */
1647
1648/*
1649 * osnoise_cpus_read - Read function for reading the "cpus" file
1650 * @filp: The active open file structure
1651 * @ubuf: The userspace provided buffer to read value into
1652 * @cnt: The maximum number of bytes to read
1653 * @ppos: The current "file" position
1654 *
1655 * Prints the "cpus" output into the user-provided buffer.
1656 */
1657static ssize_t
1658osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
1659                  loff_t *ppos)
1660{
1661        char *mask_str;
1662        int len;
1663
1664        mutex_lock(&interface_lock);
1665
1666        len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
1667        mask_str = kmalloc(len, GFP_KERNEL);
1668        if (!mask_str) {
1669                count = -ENOMEM;
1670                goto out_unlock;
1671        }
1672
1673        len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
1674        if (len >= count) {
1675                count = -EINVAL;
1676                goto out_free;
1677        }
1678
1679        count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
1680
1681out_free:
1682        kfree(mask_str);
1683out_unlock:
1684        mutex_unlock(&interface_lock);
1685
1686        return count;
1687}
1688
1689static void osnoise_tracer_start(struct trace_array *tr);
1690static void osnoise_tracer_stop(struct trace_array *tr);
1691
1692/*
1693 * osnoise_cpus_write - Write function for "cpus" entry
1694 * @filp: The active open file structure
1695 * @ubuf: The user buffer that contains the value to write
1696 * @cnt: The maximum number of bytes to write to "file"
1697 * @ppos: The current position in @file
1698 *
1699 * This function provides a write implementation for the "cpus"
1700 * interface to the osnoise trace. By default, it lists all  CPUs,
1701 * in this way, allowing osnoise threads to run on any online CPU
1702 * of the system. It serves to restrict the execution of osnoise to the
1703 * set of CPUs writing via this interface. Note that osnoise also
1704 * respects the "tracing_cpumask." Hence, osnoise threads will run only
1705 * on the set of CPUs allowed here AND on "tracing_cpumask." Why not
1706 * have just "tracing_cpumask?" Because the user might be interested
1707 * in tracing what is running on other CPUs. For instance, one might
1708 * run osnoise in one HT CPU while observing what is running on the
1709 * sibling HT CPU.
1710 */
1711static ssize_t
1712osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
1713                   loff_t *ppos)
1714{
1715        struct trace_array *tr = osnoise_trace;
1716        cpumask_var_t osnoise_cpumask_new;
1717        int running, err;
1718        char buf[256];
1719
1720        if (count >= 256)
1721                return -EINVAL;
1722
1723        if (copy_from_user(buf, ubuf, count))
1724                return -EFAULT;
1725
1726        if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
1727                return -ENOMEM;
1728
1729        err = cpulist_parse(buf, osnoise_cpumask_new);
1730        if (err)
1731                goto err_free;
1732
1733        /*
1734         * trace_types_lock is taken to avoid concurrency on start/stop
1735         * and osnoise_busy.
1736         */
1737        mutex_lock(&trace_types_lock);
1738        running = osnoise_busy;
1739        if (running)
1740                osnoise_tracer_stop(tr);
1741
1742        mutex_lock(&interface_lock);
1743        /*
1744         * osnoise_cpumask is read by CPU hotplug operations.
1745         */
1746        get_online_cpus();
1747
1748        cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
1749
1750        put_online_cpus();
1751        mutex_unlock(&interface_lock);
1752
1753        if (running)
1754                osnoise_tracer_start(tr);
1755        mutex_unlock(&trace_types_lock);
1756
1757        free_cpumask_var(osnoise_cpumask_new);
1758        return count;
1759
1760err_free:
1761        free_cpumask_var(osnoise_cpumask_new);
1762
1763        return err;
1764}
1765
1766/*
1767 * osnoise/runtime_us: cannot be greater than the period.
1768 */
1769static struct trace_min_max_param osnoise_runtime = {
1770        .lock   = &interface_lock,
1771        .val    = &osnoise_data.sample_runtime,
1772        .max    = &osnoise_data.sample_period,
1773        .min    = NULL,
1774};
1775
1776/*
1777 * osnoise/period_us: cannot be smaller than the runtime.
1778 */
1779static struct trace_min_max_param osnoise_period = {
1780        .lock   = &interface_lock,
1781        .val    = &osnoise_data.sample_period,
1782        .max    = NULL,
1783        .min    = &osnoise_data.sample_runtime,
1784};
1785
1786/*
1787 * osnoise/stop_tracing_us: no limit.
1788 */
1789static struct trace_min_max_param osnoise_stop_tracing_in = {
1790        .lock   = &interface_lock,
1791        .val    = &osnoise_data.stop_tracing,
1792        .max    = NULL,
1793        .min    = NULL,
1794};
1795
1796/*
1797 * osnoise/stop_tracing_total_us: no limit.
1798 */
1799static struct trace_min_max_param osnoise_stop_tracing_total = {
1800        .lock   = &interface_lock,
1801        .val    = &osnoise_data.stop_tracing_total,
1802        .max    = NULL,
1803        .min    = NULL,
1804};
1805
1806#ifdef CONFIG_TIMERLAT_TRACER
1807/*
1808 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
1809 * latency is higher than val.
1810 */
1811static struct trace_min_max_param osnoise_print_stack = {
1812        .lock   = &interface_lock,
1813        .val    = &osnoise_data.print_stack,
1814        .max    = NULL,
1815        .min    = NULL,
1816};
1817
1818/*
1819 * osnoise/timerlat_period: min 100 us, max 1 s
1820 */
1821u64 timerlat_min_period = 100;
1822u64 timerlat_max_period = 1000000;
1823static struct trace_min_max_param timerlat_period = {
1824        .lock   = &interface_lock,
1825        .val    = &osnoise_data.timerlat_period,
1826        .max    = &timerlat_max_period,
1827        .min    = &timerlat_min_period,
1828};
1829#endif
1830
1831static const struct file_operations cpus_fops = {
1832        .open           = tracing_open_generic,
1833        .read           = osnoise_cpus_read,
1834        .write          = osnoise_cpus_write,
1835        .llseek         = generic_file_llseek,
1836};
1837
1838/*
1839 * init_tracefs - A function to initialize the tracefs interface files
1840 *
1841 * This function creates entries in tracefs for "osnoise" and "timerlat".
1842 * It creates these directories in the tracing directory, and within that
1843 * directory the use can change and view the configs.
1844 */
1845static int init_tracefs(void)
1846{
1847        struct dentry *top_dir;
1848        struct dentry *tmp;
1849        int ret;
1850
1851        ret = tracing_init_dentry();
1852        if (ret)
1853                return -ENOMEM;
1854
1855        top_dir = tracefs_create_dir("osnoise", NULL);
1856        if (!top_dir)
1857                return 0;
1858
1859        tmp = tracefs_create_file("period_us", 0640, top_dir,
1860                                  &osnoise_period, &trace_min_max_fops);
1861        if (!tmp)
1862                goto err;
1863
1864        tmp = tracefs_create_file("runtime_us", 0644, top_dir,
1865                                  &osnoise_runtime, &trace_min_max_fops);
1866        if (!tmp)
1867                goto err;
1868
1869        tmp = tracefs_create_file("stop_tracing_us", 0640, top_dir,
1870                                  &osnoise_stop_tracing_in, &trace_min_max_fops);
1871        if (!tmp)
1872                goto err;
1873
1874        tmp = tracefs_create_file("stop_tracing_total_us", 0640, top_dir,
1875                                  &osnoise_stop_tracing_total, &trace_min_max_fops);
1876        if (!tmp)
1877                goto err;
1878
1879        tmp = trace_create_file("cpus", 0644, top_dir, NULL, &cpus_fops);
1880        if (!tmp)
1881                goto err;
1882#ifdef CONFIG_TIMERLAT_TRACER
1883#ifdef CONFIG_STACKTRACE
1884        tmp = tracefs_create_file("print_stack", 0640, top_dir,
1885                                  &osnoise_print_stack, &trace_min_max_fops);
1886        if (!tmp)
1887                goto err;
1888#endif
1889
1890        tmp = tracefs_create_file("timerlat_period_us", 0640, top_dir,
1891                                  &timerlat_period, &trace_min_max_fops);
1892        if (!tmp)
1893                goto err;
1894#endif
1895
1896        return 0;
1897
1898err:
1899        tracefs_remove(top_dir);
1900        return -ENOMEM;
1901}
1902
1903static int osnoise_hook_events(void)
1904{
1905        int retval;
1906
1907        /*
1908         * Trace is already hooked, we are re-enabling from
1909         * a stop_tracing_*.
1910         */
1911        if (trace_osnoise_callback_enabled)
1912                return 0;
1913
1914        retval = hook_irq_events();
1915        if (retval)
1916                return -EINVAL;
1917
1918        retval = hook_softirq_events();
1919        if (retval)
1920                goto out_unhook_irq;
1921
1922        retval = hook_thread_events();
1923        /*
1924         * All fine!
1925         */
1926        if (!retval)
1927                return 0;
1928
1929        unhook_softirq_events();
1930out_unhook_irq:
1931        unhook_irq_events();
1932        return -EINVAL;
1933}
1934
1935static int __osnoise_tracer_start(struct trace_array *tr)
1936{
1937        int retval;
1938
1939        osn_var_reset_all();
1940
1941        retval = osnoise_hook_events();
1942        if (retval)
1943                return retval;
1944        /*
1945         * Make sure NMIs see reseted values.
1946         */
1947        barrier();
1948        trace_osnoise_callback_enabled = true;
1949
1950        retval = start_per_cpu_kthreads(tr);
1951        if (retval) {
1952                unhook_irq_events();
1953                return retval;
1954        }
1955
1956        osnoise_busy = true;
1957
1958        return 0;
1959}
1960
1961static void osnoise_tracer_start(struct trace_array *tr)
1962{
1963        int retval;
1964
1965        if (osnoise_busy)
1966                return;
1967
1968        retval = __osnoise_tracer_start(tr);
1969        if (retval)
1970                pr_err(BANNER "Error starting osnoise tracer\n");
1971
1972}
1973
1974static void osnoise_tracer_stop(struct trace_array *tr)
1975{
1976        if (!osnoise_busy)
1977                return;
1978
1979        trace_osnoise_callback_enabled = false;
1980        barrier();
1981
1982        stop_per_cpu_kthreads();
1983
1984        unhook_irq_events();
1985        unhook_softirq_events();
1986        unhook_thread_events();
1987
1988        osnoise_busy = false;
1989}
1990
1991static int osnoise_tracer_init(struct trace_array *tr)
1992{
1993
1994        /* Only allow one instance to enable this */
1995        if (osnoise_busy)
1996                return -EBUSY;
1997
1998        osnoise_trace = tr;
1999        tr->max_latency = 0;
2000
2001        osnoise_tracer_start(tr);
2002
2003        return 0;
2004}
2005
2006static void osnoise_tracer_reset(struct trace_array *tr)
2007{
2008        osnoise_tracer_stop(tr);
2009}
2010
2011static struct tracer osnoise_tracer __read_mostly = {
2012        .name           = "osnoise",
2013        .init           = osnoise_tracer_init,
2014        .reset          = osnoise_tracer_reset,
2015        .start          = osnoise_tracer_start,
2016        .stop           = osnoise_tracer_stop,
2017        .print_header   = print_osnoise_headers,
2018        .allow_instances = true,
2019};
2020
2021#ifdef CONFIG_TIMERLAT_TRACER
2022static void timerlat_tracer_start(struct trace_array *tr)
2023{
2024        int retval;
2025
2026        if (osnoise_busy)
2027                return;
2028
2029        osnoise_data.timerlat_tracer = 1;
2030
2031        retval = __osnoise_tracer_start(tr);
2032        if (retval)
2033                goto out_err;
2034
2035        return;
2036out_err:
2037        pr_err(BANNER "Error starting timerlat tracer\n");
2038}
2039
2040static void timerlat_tracer_stop(struct trace_array *tr)
2041{
2042        int cpu;
2043
2044        if (!osnoise_busy)
2045                return;
2046
2047        for_each_online_cpu(cpu)
2048                per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
2049
2050        osnoise_tracer_stop(tr);
2051
2052        osnoise_data.timerlat_tracer = 0;
2053}
2054
2055static int timerlat_tracer_init(struct trace_array *tr)
2056{
2057        /* Only allow one instance to enable this */
2058        if (osnoise_busy)
2059                return -EBUSY;
2060
2061        osnoise_trace = tr;
2062
2063        tr->max_latency = 0;
2064
2065        timerlat_tracer_start(tr);
2066
2067        return 0;
2068}
2069
2070static void timerlat_tracer_reset(struct trace_array *tr)
2071{
2072        timerlat_tracer_stop(tr);
2073}
2074
2075static struct tracer timerlat_tracer __read_mostly = {
2076        .name           = "timerlat",
2077        .init           = timerlat_tracer_init,
2078        .reset          = timerlat_tracer_reset,
2079        .start          = timerlat_tracer_start,
2080        .stop           = timerlat_tracer_stop,
2081        .print_header   = print_timerlat_headers,
2082        .allow_instances = true,
2083};
2084#endif /* CONFIG_TIMERLAT_TRACER */
2085
2086__init static int init_osnoise_tracer(void)
2087{
2088        int ret;
2089
2090        mutex_init(&interface_lock);
2091
2092        cpumask_copy(&osnoise_cpumask, cpu_all_mask);
2093
2094        ret = register_tracer(&osnoise_tracer);
2095        if (ret) {
2096                pr_err(BANNER "Error registering osnoise!\n");
2097                return ret;
2098        }
2099
2100#ifdef CONFIG_TIMERLAT_TRACER
2101        ret = register_tracer(&timerlat_tracer);
2102        if (ret) {
2103                pr_err(BANNER "Error registering timerlat\n");
2104                return ret;
2105        }
2106#endif
2107        osnoise_init_hotplug_support();
2108
2109        init_tracefs();
2110
2111        return 0;
2112}
2113late_initcall(init_osnoise_tracer);
2114