LXR linux/drivers/gpu/drm/i915/i915

   1/*
   2 * SPDX-License-Identifier: MIT
   3 *
   4 * Copyright © 2019 Intel Corporation
   5 */
   6
   7#include <linux/slab.h>
   8#include <linux/workqueue.h>
   9
  10#include "i915_active.h"
  11#include "gem/i915_gem_context.h"
  12#include "gem/i915_gem_object.h"
  13#include "i915_globals.h"
  14#include "i915_request.h"
  15#include "i915_scheduler.h"
  16#include "i915_vma.h"
  17
  18static LIST_HEAD(globals);
  19
  20static atomic_t active;
  21static atomic_t epoch;
  22static struct park_work {
  23        struct delayed_work work;
  24        struct rcu_head rcu;
  25        unsigned long flags;
  26#define PENDING 0
  27        int epoch;
  28} park;
  29
  30static void i915_globals_shrink(void)
  31{
  32        struct i915_global *global;
  33
  34        /*
  35         * kmem_cache_shrink() discards empty slabs and reorders partially
  36         * filled slabs to prioritise allocating from the mostly full slabs,
  37         * with the aim of reducing fragmentation.
  38         */
  39        list_for_each_entry(global, &globals, link)
  40                global->shrink();
  41}
  42
  43static void __i915_globals_grace(struct rcu_head *rcu)
  44{
  45        /* Ratelimit parking as shrinking is quite slow */
  46        schedule_delayed_work(&park.work, round_jiffies_up_relative(2 * HZ));
  47}
  48
  49static void __i915_globals_queue_rcu(void)
  50{
  51        park.epoch = atomic_inc_return(&epoch);
  52        if (!atomic_read(&active)) {
  53                init_rcu_head(&park.rcu);
  54                call_rcu(&park.rcu, __i915_globals_grace);
  55        }
  56}
  57
  58static void __i915_globals_park(struct work_struct *work)
  59{
  60        destroy_rcu_head(&park.rcu);
  61
  62        /* Confirm nothing woke up in the last grace period */
  63        if (park.epoch != atomic_read(&epoch)) {
  64                __i915_globals_queue_rcu();
  65                return;
  66        }
  67
  68        clear_bit(PENDING, &park.flags);
  69        i915_globals_shrink();
  70}
  71
  72void __init i915_global_register(struct i915_global *global)
  73{
  74        GEM_BUG_ON(!global->shrink);
  75        GEM_BUG_ON(!global->exit);
  76
  77        list_add_tail(&global->link, &globals);
  78}
  79
  80static void __i915_globals_cleanup(void)
  81{
  82        struct i915_global *global, *next;
  83
  84        list_for_each_entry_safe_reverse(global, next, &globals, link)
  85                global->exit();
  86}
  87
  88static __initconst int (* const initfn[])(void) = {
  89        i915_global_active_init,
  90        i915_global_buddy_init,
  91        i915_global_context_init,
  92        i915_global_gem_context_init,
  93        i915_global_objects_init,
  94        i915_global_request_init,
  95        i915_global_scheduler_init,
  96        i915_global_vma_init,
  97};
  98
  99int __init i915_globals_init(void)
 100{
 101        int i;
 102
 103        for (i = 0; i < ARRAY_SIZE(initfn); i++) {
 104                int err;
 105
 106                err = initfn[i]();
 107                if (err) {
 108                        __i915_globals_cleanup();
 109                        return err;
 110                }
 111        }
 112
 113        INIT_DELAYED_WORK(&park.work, __i915_globals_park);
 114        return 0;
 115}
 116
 117void i915_globals_park(void)
 118{
 119        /*
 120         * Defer shrinking the global slab caches (and other work) until
 121         * after a RCU grace period has completed with no activity. This
 122         * is to try and reduce the latency impact on the consumers caused
 123         * by us shrinking the caches the same time as they are trying to
 124         * allocate, with the assumption being that if we idle long enough
 125         * for an RCU grace period to elapse since the last use, it is likely
 126         * to be longer until we need the caches again.
 127         */
 128        if (!atomic_dec_and_test(&active))
 129                return;
 130
 131        /* Queue cleanup after the next RCU grace period has freed slabs */
 132        if (!test_and_set_bit(PENDING, &park.flags))
 133                __i915_globals_queue_rcu();
 134}
 135
 136void i915_globals_unpark(void)
 137{
 138        atomic_inc(&epoch);
 139        atomic_inc(&active);
 140}
 141
 142static void __exit __i915_globals_flush(void)
 143{
 144        atomic_inc(&active); /* skip shrinking */
 145
 146        rcu_barrier(); /* wait for the work to be queued */
 147        flush_delayed_work(&park.work);
 148
 149        atomic_dec(&active);
 150}
 151
 152void __exit i915_globals_exit(void)
 153{
 154        GEM_BUG_ON(atomic_read(&active));
 155
 156        __i915_globals_flush();
 157        __i915_globals_cleanup();
 158
 159        /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
 160        rcu_barrier();
 161}
 162