LXR linux/mm/swap

   1#include <linux/swap_cgroup.h>
   2#include <linux/vmalloc.h>
   3#include <linux/mm.h>
   4
   5#include <linux/swapops.h> /* depends on mm.h include */
   6
   7static DEFINE_MUTEX(swap_cgroup_mutex);
   8struct swap_cgroup_ctrl {
   9        struct page **map;
  10        unsigned long length;
  11        spinlock_t      lock;
  12};
  13
  14static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
  15
  16struct swap_cgroup {
  17        unsigned short          id;
  18};
  19#define SC_PER_PAGE     (PAGE_SIZE/sizeof(struct swap_cgroup))
  20
  21/*
  22 * SwapCgroup implements "lookup" and "exchange" operations.
  23 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
  24 * against SwapCache. At swap_free(), this is accessed directly from swap.
  25 *
  26 * This means,
  27 *  - we have no race in "exchange" when we're accessed via SwapCache because
  28 *    SwapCache(and its swp_entry) is under lock.
  29 *  - When called via swap_free(), there is no user of this entry and no race.
  30 * Then, we don't need lock around "exchange".
  31 *
  32 * TODO: we can push these buffers out to HIGHMEM.
  33 */
  34
  35/*
  36 * allocate buffer for swap_cgroup.
  37 */
  38static int swap_cgroup_prepare(int type)
  39{
  40        struct page *page;
  41        struct swap_cgroup_ctrl *ctrl;
  42        unsigned long idx, max;
  43
  44        ctrl = &swap_cgroup_ctrl[type];
  45
  46        for (idx = 0; idx < ctrl->length; idx++) {
  47                page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  48                if (!page)
  49                        goto not_enough_page;
  50                ctrl->map[idx] = page;
  51        }
  52        return 0;
  53not_enough_page:
  54        max = idx;
  55        for (idx = 0; idx < max; idx++)
  56                __free_page(ctrl->map[idx]);
  57
  58        return -ENOMEM;
  59}
  60
  61static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
  62                                        struct swap_cgroup_ctrl **ctrlp)
  63{
  64        pgoff_t offset = swp_offset(ent);
  65        struct swap_cgroup_ctrl *ctrl;
  66        struct page *mappage;
  67        struct swap_cgroup *sc;
  68
  69        ctrl = &swap_cgroup_ctrl[swp_type(ent)];
  70        if (ctrlp)
  71                *ctrlp = ctrl;
  72
  73        mappage = ctrl->map[offset / SC_PER_PAGE];
  74        sc = page_address(mappage);
  75        return sc + offset % SC_PER_PAGE;
  76}
  77
  78/**
  79 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
  80 * @ent: swap entry to be cmpxchged
  81 * @old: old id
  82 * @new: new id
  83 *
  84 * Returns old id at success, 0 at failure.
  85 * (There is no mem_cgroup using 0 as its id)
  86 */
  87unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
  88                                        unsigned short old, unsigned short new)
  89{
  90        struct swap_cgroup_ctrl *ctrl;
  91        struct swap_cgroup *sc;
  92        unsigned long flags;
  93        unsigned short retval;
  94
  95        sc = lookup_swap_cgroup(ent, &ctrl);
  96
  97        spin_lock_irqsave(&ctrl->lock, flags);
  98        retval = sc->id;
  99        if (retval == old)
 100                sc->id = new;
 101        else
 102                retval = 0;
 103        spin_unlock_irqrestore(&ctrl->lock, flags);
 104        return retval;
 105}
 106
 107/**
 108 * swap_cgroup_record - record mem_cgroup for this swp_entry.
 109 * @ent: swap entry to be recorded into
 110 * @id: mem_cgroup to be recorded
 111 *
 112 * Returns old value at success, 0 at failure.
 113 * (Of course, old value can be 0.)
 114 */
 115unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
 116{
 117        struct swap_cgroup_ctrl *ctrl;
 118        struct swap_cgroup *sc;
 119        unsigned short old;
 120        unsigned long flags;
 121
 122        sc = lookup_swap_cgroup(ent, &ctrl);
 123
 124        spin_lock_irqsave(&ctrl->lock, flags);
 125        old = sc->id;
 126        sc->id = id;
 127        spin_unlock_irqrestore(&ctrl->lock, flags);
 128
 129        return old;
 130}
 131
 132/**
 133 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
 134 * @ent: swap entry to be looked up.
 135 *
 136 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 137 */
 138unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
 139{
 140        return lookup_swap_cgroup(ent, NULL)->id;
 141}
 142
 143int swap_cgroup_swapon(int type, unsigned long max_pages)
 144{
 145        void *array;
 146        unsigned long array_size;
 147        unsigned long length;
 148        struct swap_cgroup_ctrl *ctrl;
 149
 150        if (!do_swap_account)
 151                return 0;
 152
 153        length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
 154        array_size = length * sizeof(void *);
 155
 156        array = vzalloc(array_size);
 157        if (!array)
 158                goto nomem;
 159
 160        ctrl = &swap_cgroup_ctrl[type];
 161        mutex_lock(&swap_cgroup_mutex);
 162        ctrl->length = length;
 163        ctrl->map = array;
 164        spin_lock_init(&ctrl->lock);
 165        if (swap_cgroup_prepare(type)) {
 166                /* memory shortage */
 167                ctrl->map = NULL;
 168                ctrl->length = 0;
 169                mutex_unlock(&swap_cgroup_mutex);
 170                vfree(array);
 171                goto nomem;
 172        }
 173        mutex_unlock(&swap_cgroup_mutex);
 174
 175        return 0;
 176nomem:
 177        pr_info("couldn't allocate enough memory for swap_cgroup\n");
 178        pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
 179        return -ENOMEM;
 180}
 181
 182void swap_cgroup_swapoff(int type)
 183{
 184        struct page **map;
 185        unsigned long i, length;
 186        struct swap_cgroup_ctrl *ctrl;
 187
 188        if (!do_swap_account)
 189                return;
 190
 191        mutex_lock(&swap_cgroup_mutex);
 192        ctrl = &swap_cgroup_ctrl[type];
 193        map = ctrl->map;
 194        length = ctrl->length;
 195        ctrl->map = NULL;
 196        ctrl->length = 0;
 197        mutex_unlock(&swap_cgroup_mutex);
 198
 199        if (map) {
 200                for (i = 0; i < length; i++) {
 201                        struct page *page = map[i];
 202                        if (page)
 203                                __free_page(page);
 204                }
 205                vfree(map);
 206        }
 207}
 208