linux/drivers/thermal/intel/intel_powerclamp.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * intel_powerclamp.c - package c-state idle injection
   4 *
   5 * Copyright (c) 2012, Intel Corporation.
   6 *
   7 * Authors:
   8 *     Arjan van de Ven <arjan@linux.intel.com>
   9 *     Jacob Pan <jacob.jun.pan@linux.intel.com>
  10 *
  11 *      TODO:
  12 *           1. better handle wakeup from external interrupts, currently a fixed
  13 *              compensation is added to clamping duration when excessive amount
  14 *              of wakeups are observed during idle time. the reason is that in
  15 *              case of external interrupts without need for ack, clamping down
  16 *              cpu in non-irq context does not reduce irq. for majority of the
  17 *              cases, clamping down cpu does help reduce irq as well, we should
  18 *              be able to differentiate the two cases and give a quantitative
  19 *              solution for the irqs that we can control. perhaps based on
  20 *              get_cpu_iowait_time_us()
  21 *
  22 *           2. synchronization with other hw blocks
  23 */
  24
  25#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  26
  27#include <linux/module.h>
  28#include <linux/kernel.h>
  29#include <linux/delay.h>
  30#include <linux/kthread.h>
  31#include <linux/cpu.h>
  32#include <linux/thermal.h>
  33#include <linux/slab.h>
  34#include <linux/tick.h>
  35#include <linux/debugfs.h>
  36#include <linux/seq_file.h>
  37#include <linux/sched/rt.h>
  38#include <uapi/linux/sched/types.h>
  39
  40#include <asm/nmi.h>
  41#include <asm/msr.h>
  42#include <asm/mwait.h>
  43#include <asm/cpu_device_id.h>
  44#include <asm/hardirq.h>
  45
  46#define MAX_TARGET_RATIO (50U)
  47/* For each undisturbed clamping period (no extra wake ups during idle time),
  48 * we increment the confidence counter for the given target ratio.
  49 * CONFIDENCE_OK defines the level where runtime calibration results are
  50 * valid.
  51 */
  52#define CONFIDENCE_OK (3)
  53/* Default idle injection duration, driver adjust sleep time to meet target
  54 * idle ratio. Similar to frequency modulation.
  55 */
  56#define DEFAULT_DURATION_JIFFIES (6)
  57
  58static unsigned int target_mwait;
  59static struct dentry *debug_dir;
  60
  61/* user selected target */
  62static unsigned int set_target_ratio;
  63static unsigned int current_ratio;
  64static bool should_skip;
  65static bool reduce_irq;
  66static atomic_t idle_wakeup_counter;
  67static unsigned int control_cpu; /* The cpu assigned to collect stat and update
  68                                  * control parameters. default to BSP but BSP
  69                                  * can be offlined.
  70                                  */
  71static bool clamping;
  72
  73struct powerclamp_worker_data {
  74        struct kthread_worker *worker;
  75        struct kthread_work balancing_work;
  76        struct kthread_delayed_work idle_injection_work;
  77        unsigned int cpu;
  78        unsigned int count;
  79        unsigned int guard;
  80        unsigned int window_size_now;
  81        unsigned int target_ratio;
  82        unsigned int duration_jiffies;
  83        bool clamping;
  84};
  85
  86static struct powerclamp_worker_data __percpu *worker_data;
  87static struct thermal_cooling_device *cooling_dev;
  88static unsigned long *cpu_clamping_mask;  /* bit map for tracking per cpu
  89                                           * clamping kthread worker
  90                                           */
  91
  92static unsigned int duration;
  93static unsigned int pkg_cstate_ratio_cur;
  94static unsigned int window_size;
  95
  96static int duration_set(const char *arg, const struct kernel_param *kp)
  97{
  98        int ret = 0;
  99        unsigned long new_duration;
 100
 101        ret = kstrtoul(arg, 10, &new_duration);
 102        if (ret)
 103                goto exit;
 104        if (new_duration > 25 || new_duration < 6) {
 105                pr_err("Out of recommended range %lu, between 6-25ms\n",
 106                        new_duration);
 107                ret = -EINVAL;
 108        }
 109
 110        duration = clamp(new_duration, 6ul, 25ul);
 111        smp_mb();
 112
 113exit:
 114
 115        return ret;
 116}
 117
 118static const struct kernel_param_ops duration_ops = {
 119        .set = duration_set,
 120        .get = param_get_int,
 121};
 122
 123
 124module_param_cb(duration, &duration_ops, &duration, 0644);
 125MODULE_PARM_DESC(duration, "forced idle time for each attempt in msec.");
 126
 127struct powerclamp_calibration_data {
 128        unsigned long confidence;  /* used for calibration, basically a counter
 129                                    * gets incremented each time a clamping
 130                                    * period is completed without extra wakeups
 131                                    * once that counter is reached given level,
 132                                    * compensation is deemed usable.
 133                                    */
 134        unsigned long steady_comp; /* steady state compensation used when
 135                                    * no extra wakeups occurred.
 136                                    */
 137        unsigned long dynamic_comp; /* compensate excessive wakeup from idle
 138                                     * mostly from external interrupts.
 139                                     */
 140};
 141
 142static struct powerclamp_calibration_data cal_data[MAX_TARGET_RATIO];
 143
 144static int window_size_set(const char *arg, const struct kernel_param *kp)
 145{
 146        int ret = 0;
 147        unsigned long new_window_size;
 148
 149        ret = kstrtoul(arg, 10, &new_window_size);
 150        if (ret)
 151                goto exit_win;
 152        if (new_window_size > 10 || new_window_size < 2) {
 153                pr_err("Out of recommended window size %lu, between 2-10\n",
 154                        new_window_size);
 155                ret = -EINVAL;
 156        }
 157
 158        window_size = clamp(new_window_size, 2ul, 10ul);
 159        smp_mb();
 160
 161exit_win:
 162
 163        return ret;
 164}
 165
 166static const struct kernel_param_ops window_size_ops = {
 167        .set = window_size_set,
 168        .get = param_get_int,
 169};
 170
 171module_param_cb(window_size, &window_size_ops, &window_size, 0644);
 172MODULE_PARM_DESC(window_size, "sliding window in number of clamping cycles\n"
 173        "\tpowerclamp controls idle ratio within this window. larger\n"
 174        "\twindow size results in slower response time but more smooth\n"
 175        "\tclamping results. default to 2.");
 176
 177static void find_target_mwait(void)
 178{
 179        unsigned int eax, ebx, ecx, edx;
 180        unsigned int highest_cstate = 0;
 181        unsigned int highest_subcstate = 0;
 182        int i;
 183
 184        if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
 185                return;
 186
 187        cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
 188
 189        if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
 190            !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
 191                return;
 192
 193        edx >>= MWAIT_SUBSTATE_SIZE;
 194        for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
 195                if (edx & MWAIT_SUBSTATE_MASK) {
 196                        highest_cstate = i;
 197                        highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
 198                }
 199        }
 200        target_mwait = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
 201                (highest_subcstate - 1);
 202
 203}
 204
 205struct pkg_cstate_info {
 206        bool skip;
 207        int msr_index;
 208        int cstate_id;
 209};
 210
 211#define PKG_CSTATE_INIT(id) {                           \
 212                .msr_index = MSR_PKG_C##id##_RESIDENCY, \
 213                .cstate_id = id                         \
 214                        }
 215
 216static struct pkg_cstate_info pkg_cstates[] = {
 217        PKG_CSTATE_INIT(2),
 218        PKG_CSTATE_INIT(3),
 219        PKG_CSTATE_INIT(6),
 220        PKG_CSTATE_INIT(7),
 221        PKG_CSTATE_INIT(8),
 222        PKG_CSTATE_INIT(9),
 223        PKG_CSTATE_INIT(10),
 224        {NULL},
 225};
 226
 227static bool has_pkg_state_counter(void)
 228{
 229        u64 val;
 230        struct pkg_cstate_info *info = pkg_cstates;
 231
 232        /* check if any one of the counter msrs exists */
 233        while (info->msr_index) {
 234                if (!rdmsrl_safe(info->msr_index, &val))
 235                        return true;
 236                info++;
 237        }
 238
 239        return false;
 240}
 241
 242static u64 pkg_state_counter(void)
 243{
 244        u64 val;
 245        u64 count = 0;
 246        struct pkg_cstate_info *info = pkg_cstates;
 247
 248        while (info->msr_index) {
 249                if (!info->skip) {
 250                        if (!rdmsrl_safe(info->msr_index, &val))
 251                                count += val;
 252                        else
 253                                info->skip = true;
 254                }
 255                info++;
 256        }
 257
 258        return count;
 259}
 260
 261static unsigned int get_compensation(int ratio)
 262{
 263        unsigned int comp = 0;
 264
 265        /* we only use compensation if all adjacent ones are good */
 266        if (ratio == 1 &&
 267                cal_data[ratio].confidence >= CONFIDENCE_OK &&
 268                cal_data[ratio + 1].confidence >= CONFIDENCE_OK &&
 269                cal_data[ratio + 2].confidence >= CONFIDENCE_OK) {
 270                comp = (cal_data[ratio].steady_comp +
 271                        cal_data[ratio + 1].steady_comp +
 272                        cal_data[ratio + 2].steady_comp) / 3;
 273        } else if (ratio == MAX_TARGET_RATIO - 1 &&
 274                cal_data[ratio].confidence >= CONFIDENCE_OK &&
 275                cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
 276                cal_data[ratio - 2].confidence >= CONFIDENCE_OK) {
 277                comp = (cal_data[ratio].steady_comp +
 278                        cal_data[ratio - 1].steady_comp +
 279                        cal_data[ratio - 2].steady_comp) / 3;
 280        } else if (cal_data[ratio].confidence >= CONFIDENCE_OK &&
 281                cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
 282                cal_data[ratio + 1].confidence >= CONFIDENCE_OK) {
 283                comp = (cal_data[ratio].steady_comp +
 284                        cal_data[ratio - 1].steady_comp +
 285                        cal_data[ratio + 1].steady_comp) / 3;
 286        }
 287
 288        /* REVISIT: simple penalty of double idle injection */
 289        if (reduce_irq)
 290                comp = ratio;
 291        /* do not exceed limit */
 292        if (comp + ratio >= MAX_TARGET_RATIO)
 293                comp = MAX_TARGET_RATIO - ratio - 1;
 294
 295        return comp;
 296}
 297
 298static void adjust_compensation(int target_ratio, unsigned int win)
 299{
 300        int delta;
 301        struct powerclamp_calibration_data *d = &cal_data[target_ratio];
 302
 303        /*
 304         * adjust compensations if confidence level has not been reached or
 305         * there are too many wakeups during the last idle injection period, we
 306         * cannot trust the data for compensation.
 307         */
 308        if (d->confidence >= CONFIDENCE_OK ||
 309                atomic_read(&idle_wakeup_counter) >
 310                win * num_online_cpus())
 311                return;
 312
 313        delta = set_target_ratio - current_ratio;
 314        /* filter out bad data */
 315        if (delta >= 0 && delta <= (1+target_ratio/10)) {
 316                if (d->steady_comp)
 317                        d->steady_comp =
 318                                roundup(delta+d->steady_comp, 2)/2;
 319                else
 320                        d->steady_comp = delta;
 321                d->confidence++;
 322        }
 323}
 324
 325static bool powerclamp_adjust_controls(unsigned int target_ratio,
 326                                unsigned int guard, unsigned int win)
 327{
 328        static u64 msr_last, tsc_last;
 329        u64 msr_now, tsc_now;
 330        u64 val64;
 331
 332        /* check result for the last window */
 333        msr_now = pkg_state_counter();
 334        tsc_now = rdtsc();
 335
 336        /* calculate pkg cstate vs tsc ratio */
 337        if (!msr_last || !tsc_last)
 338                current_ratio = 1;
 339        else if (tsc_now-tsc_last) {
 340                val64 = 100*(msr_now-msr_last);
 341                do_div(val64, (tsc_now-tsc_last));
 342                current_ratio = val64;
 343        }
 344
 345        /* update record */
 346        msr_last = msr_now;
 347        tsc_last = tsc_now;
 348
 349        adjust_compensation(target_ratio, win);
 350        /*
 351         * too many external interrupts, set flag such
 352         * that we can take measure later.
 353         */
 354        reduce_irq = atomic_read(&idle_wakeup_counter) >=
 355                2 * win * num_online_cpus();
 356
 357        atomic_set(&idle_wakeup_counter, 0);
 358        /* if we are above target+guard, skip */
 359        return set_target_ratio + guard <= current_ratio;
 360}
 361
 362static void clamp_balancing_func(struct kthread_work *work)
 363{
 364        struct powerclamp_worker_data *w_data;
 365        int sleeptime;
 366        unsigned long target_jiffies;
 367        unsigned int compensated_ratio;
 368        int interval; /* jiffies to sleep for each attempt */
 369
 370        w_data = container_of(work, struct powerclamp_worker_data,
 371                              balancing_work);
 372
 373        /*
 374         * make sure user selected ratio does not take effect until
 375         * the next round. adjust target_ratio if user has changed
 376         * target such that we can converge quickly.
 377         */
 378        w_data->target_ratio = READ_ONCE(set_target_ratio);
 379        w_data->guard = 1 + w_data->target_ratio / 20;
 380        w_data->window_size_now = window_size;
 381        w_data->duration_jiffies = msecs_to_jiffies(duration);
 382        w_data->count++;
 383
 384        /*
 385         * systems may have different ability to enter package level
 386         * c-states, thus we need to compensate the injected idle ratio
 387         * to achieve the actual target reported by the HW.
 388         */
 389        compensated_ratio = w_data->target_ratio +
 390                get_compensation(w_data->target_ratio);
 391        if (compensated_ratio <= 0)
 392                compensated_ratio = 1;
 393        interval = w_data->duration_jiffies * 100 / compensated_ratio;
 394
 395        /* align idle time */
 396        target_jiffies = roundup(jiffies, interval);
 397        sleeptime = target_jiffies - jiffies;
 398        if (sleeptime <= 0)
 399                sleeptime = 1;
 400
 401        if (clamping && w_data->clamping && cpu_online(w_data->cpu))
 402                kthread_queue_delayed_work(w_data->worker,
 403                                           &w_data->idle_injection_work,
 404                                           sleeptime);
 405}
 406
 407static void clamp_idle_injection_func(struct kthread_work *work)
 408{
 409        struct powerclamp_worker_data *w_data;
 410
 411        w_data = container_of(work, struct powerclamp_worker_data,
 412                              idle_injection_work.work);
 413
 414        /*
 415         * only elected controlling cpu can collect stats and update
 416         * control parameters.
 417         */
 418        if (w_data->cpu == control_cpu &&
 419            !(w_data->count % w_data->window_size_now)) {
 420                should_skip =
 421                        powerclamp_adjust_controls(w_data->target_ratio,
 422                                                   w_data->guard,
 423                                                   w_data->window_size_now);
 424                smp_mb();
 425        }
 426
 427        if (should_skip)
 428                goto balance;
 429
 430        play_idle(jiffies_to_usecs(w_data->duration_jiffies));
 431
 432balance:
 433        if (clamping && w_data->clamping && cpu_online(w_data->cpu))
 434                kthread_queue_work(w_data->worker, &w_data->balancing_work);
 435}
 436
 437/*
 438 * 1 HZ polling while clamping is active, useful for userspace
 439 * to monitor actual idle ratio.
 440 */
 441static void poll_pkg_cstate(struct work_struct *dummy);
 442static DECLARE_DELAYED_WORK(poll_pkg_cstate_work, poll_pkg_cstate);
 443static void poll_pkg_cstate(struct work_struct *dummy)
 444{
 445        static u64 msr_last;
 446        static u64 tsc_last;
 447
 448        u64 msr_now;
 449        u64 tsc_now;
 450        u64 val64;
 451
 452        msr_now = pkg_state_counter();
 453        tsc_now = rdtsc();
 454
 455        /* calculate pkg cstate vs tsc ratio */
 456        if (!msr_last || !tsc_last)
 457                pkg_cstate_ratio_cur = 1;
 458        else {
 459                if (tsc_now - tsc_last) {
 460                        val64 = 100 * (msr_now - msr_last);
 461                        do_div(val64, (tsc_now - tsc_last));
 462                        pkg_cstate_ratio_cur = val64;
 463                }
 464        }
 465
 466        /* update record */
 467        msr_last = msr_now;
 468        tsc_last = tsc_now;
 469
 470        if (true == clamping)
 471                schedule_delayed_work(&poll_pkg_cstate_work, HZ);
 472}
 473
 474static void start_power_clamp_worker(unsigned long cpu)
 475{
 476        struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
 477        struct kthread_worker *worker;
 478
 479        worker = kthread_create_worker_on_cpu(cpu, 0, "kidle_inj/%ld", cpu);
 480        if (IS_ERR(worker))
 481                return;
 482
 483        w_data->worker = worker;
 484        w_data->count = 0;
 485        w_data->cpu = cpu;
 486        w_data->clamping = true;
 487        set_bit(cpu, cpu_clamping_mask);
 488        sched_set_fifo(worker->task);
 489        kthread_init_work(&w_data->balancing_work, clamp_balancing_func);
 490        kthread_init_delayed_work(&w_data->idle_injection_work,
 491                                  clamp_idle_injection_func);
 492        kthread_queue_work(w_data->worker, &w_data->balancing_work);
 493}
 494
 495static void stop_power_clamp_worker(unsigned long cpu)
 496{
 497        struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
 498
 499        if (!w_data->worker)
 500                return;
 501
 502        w_data->clamping = false;
 503        /*
 504         * Make sure that all works that get queued after this point see
 505         * the clamping disabled. The counter part is not needed because
 506         * there is an implicit memory barrier when the queued work
 507         * is proceed.
 508         */
 509        smp_wmb();
 510        kthread_cancel_work_sync(&w_data->balancing_work);
 511        kthread_cancel_delayed_work_sync(&w_data->idle_injection_work);
 512        /*
 513         * The balancing work still might be queued here because
 514         * the handling of the "clapming" variable, cancel, and queue
 515         * operations are not synchronized via a lock. But it is not
 516         * a big deal. The balancing work is fast and destroy kthread
 517         * will wait for it.
 518         */
 519        clear_bit(w_data->cpu, cpu_clamping_mask);
 520        kthread_destroy_worker(w_data->worker);
 521
 522        w_data->worker = NULL;
 523}
 524
 525static int start_power_clamp(void)
 526{
 527        unsigned long cpu;
 528
 529        set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1);
 530        /* prevent cpu hotplug */
 531        get_online_cpus();
 532
 533        /* prefer BSP */
 534        control_cpu = 0;
 535        if (!cpu_online(control_cpu))
 536                control_cpu = smp_processor_id();
 537
 538        clamping = true;
 539        schedule_delayed_work(&poll_pkg_cstate_work, 0);
 540
 541        /* start one kthread worker per online cpu */
 542        for_each_online_cpu(cpu) {
 543                start_power_clamp_worker(cpu);
 544        }
 545        put_online_cpus();
 546
 547        return 0;
 548}
 549
 550static void end_power_clamp(void)
 551{
 552        int i;
 553
 554        /*
 555         * Block requeuing in all the kthread workers. They will flush and
 556         * stop faster.
 557         */
 558        clamping = false;
 559        if (bitmap_weight(cpu_clamping_mask, num_possible_cpus())) {
 560                for_each_set_bit(i, cpu_clamping_mask, num_possible_cpus()) {
 561                        pr_debug("clamping worker for cpu %d alive, destroy\n",
 562                                 i);
 563                        stop_power_clamp_worker(i);
 564                }
 565        }
 566}
 567
 568static int powerclamp_cpu_online(unsigned int cpu)
 569{
 570        if (clamping == false)
 571                return 0;
 572        start_power_clamp_worker(cpu);
 573        /* prefer BSP as controlling CPU */
 574        if (cpu == 0) {
 575                control_cpu = 0;
 576                smp_mb();
 577        }
 578        return 0;
 579}
 580
 581static int powerclamp_cpu_predown(unsigned int cpu)
 582{
 583        if (clamping == false)
 584                return 0;
 585
 586        stop_power_clamp_worker(cpu);
 587        if (cpu != control_cpu)
 588                return 0;
 589
 590        control_cpu = cpumask_first(cpu_online_mask);
 591        if (control_cpu == cpu)
 592                control_cpu = cpumask_next(cpu, cpu_online_mask);
 593        smp_mb();
 594        return 0;
 595}
 596
 597static int powerclamp_get_max_state(struct thermal_cooling_device *cdev,
 598                                 unsigned long *state)
 599{
 600        *state = MAX_TARGET_RATIO;
 601
 602        return 0;
 603}
 604
 605static int powerclamp_get_cur_state(struct thermal_cooling_device *cdev,
 606                                 unsigned long *state)
 607{
 608        if (true == clamping)
 609                *state = pkg_cstate_ratio_cur;
 610        else
 611                /* to save power, do not poll idle ratio while not clamping */
 612                *state = -1; /* indicates invalid state */
 613
 614        return 0;
 615}
 616
 617static int powerclamp_set_cur_state(struct thermal_cooling_device *cdev,
 618                                 unsigned long new_target_ratio)
 619{
 620        int ret = 0;
 621
 622        new_target_ratio = clamp(new_target_ratio, 0UL,
 623                                (unsigned long) (MAX_TARGET_RATIO-1));
 624        if (set_target_ratio == 0 && new_target_ratio > 0) {
 625                pr_info("Start idle injection to reduce power\n");
 626                set_target_ratio = new_target_ratio;
 627                ret = start_power_clamp();
 628                goto exit_set;
 629        } else  if (set_target_ratio > 0 && new_target_ratio == 0) {
 630                pr_info("Stop forced idle injection\n");
 631                end_power_clamp();
 632                set_target_ratio = 0;
 633        } else  /* adjust currently running */ {
 634                set_target_ratio = new_target_ratio;
 635                /* make new set_target_ratio visible to other cpus */
 636                smp_mb();
 637        }
 638
 639exit_set:
 640        return ret;
 641}
 642
 643/* bind to generic thermal layer as cooling device*/
 644static struct thermal_cooling_device_ops powerclamp_cooling_ops = {
 645        .get_max_state = powerclamp_get_max_state,
 646        .get_cur_state = powerclamp_get_cur_state,
 647        .set_cur_state = powerclamp_set_cur_state,
 648};
 649
 650static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = {
 651        X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL),
 652        {}
 653};
 654MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids);
 655
 656static int __init powerclamp_probe(void)
 657{
 658
 659        if (!x86_match_cpu(intel_powerclamp_ids)) {
 660                pr_err("CPU does not support MWAIT\n");
 661                return -ENODEV;
 662        }
 663
 664        /* The goal for idle time alignment is to achieve package cstate. */
 665        if (!has_pkg_state_counter()) {
 666                pr_info("No package C-state available\n");
 667                return -ENODEV;
 668        }
 669
 670        /* find the deepest mwait value */
 671        find_target_mwait();
 672
 673        return 0;
 674}
 675
 676static int powerclamp_debug_show(struct seq_file *m, void *unused)
 677{
 678        int i = 0;
 679
 680        seq_printf(m, "controlling cpu: %d\n", control_cpu);
 681        seq_printf(m, "pct confidence steady dynamic (compensation)\n");
 682        for (i = 0; i < MAX_TARGET_RATIO; i++) {
 683                seq_printf(m, "%d\t%lu\t%lu\t%lu\n",
 684                        i,
 685                        cal_data[i].confidence,
 686                        cal_data[i].steady_comp,
 687                        cal_data[i].dynamic_comp);
 688        }
 689
 690        return 0;
 691}
 692
 693DEFINE_SHOW_ATTRIBUTE(powerclamp_debug);
 694
 695static inline void powerclamp_create_debug_files(void)
 696{
 697        debug_dir = debugfs_create_dir("intel_powerclamp", NULL);
 698
 699        debugfs_create_file("powerclamp_calib", S_IRUGO, debug_dir, cal_data,
 700                            &powerclamp_debug_fops);
 701}
 702
 703static enum cpuhp_state hp_state;
 704
 705static int __init powerclamp_init(void)
 706{
 707        int retval;
 708        int bitmap_size;
 709
 710        bitmap_size = BITS_TO_LONGS(num_possible_cpus()) * sizeof(long);
 711        cpu_clamping_mask = kzalloc(bitmap_size, GFP_KERNEL);
 712        if (!cpu_clamping_mask)
 713                return -ENOMEM;
 714
 715        /* probe cpu features and ids here */
 716        retval = powerclamp_probe();
 717        if (retval)
 718                goto exit_free;
 719
 720        /* set default limit, maybe adjusted during runtime based on feedback */
 721        window_size = 2;
 722        retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 723                                           "thermal/intel_powerclamp:online",
 724                                           powerclamp_cpu_online,
 725                                           powerclamp_cpu_predown);
 726        if (retval < 0)
 727                goto exit_free;
 728
 729        hp_state = retval;
 730
 731        worker_data = alloc_percpu(struct powerclamp_worker_data);
 732        if (!worker_data) {
 733                retval = -ENOMEM;
 734                goto exit_unregister;
 735        }
 736
 737        cooling_dev = thermal_cooling_device_register("intel_powerclamp", NULL,
 738                                                &powerclamp_cooling_ops);
 739        if (IS_ERR(cooling_dev)) {
 740                retval = -ENODEV;
 741                goto exit_free_thread;
 742        }
 743
 744        if (!duration)
 745                duration = jiffies_to_msecs(DEFAULT_DURATION_JIFFIES);
 746
 747        powerclamp_create_debug_files();
 748
 749        return 0;
 750
 751exit_free_thread:
 752        free_percpu(worker_data);
 753exit_unregister:
 754        cpuhp_remove_state_nocalls(hp_state);
 755exit_free:
 756        kfree(cpu_clamping_mask);
 757        return retval;
 758}
 759module_init(powerclamp_init);
 760
 761static void __exit powerclamp_exit(void)
 762{
 763        end_power_clamp();
 764        cpuhp_remove_state_nocalls(hp_state);
 765        free_percpu(worker_data);
 766        thermal_cooling_device_unregister(cooling_dev);
 767        kfree(cpu_clamping_mask);
 768
 769        cancel_delayed_work_sync(&poll_pkg_cstate_work);
 770        debugfs_remove_recursive(debug_dir);
 771}
 772module_exit(powerclamp_exit);
 773
 774MODULE_LICENSE("GPL");
 775MODULE_AUTHOR("Arjan van de Ven <arjan@linux.intel.com>");
 776MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@linux.intel.com>");
 777MODULE_DESCRIPTION("Package Level C-state Idle Injection for Intel CPUs");
 778