LXR linux/include/linux/cookie.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef __LINUX_COOKIE_H
   3#define __LINUX_COOKIE_H
   4
   5#include <linux/atomic.h>
   6#include <linux/percpu.h>
   7#include <asm/local.h>
   8
   9struct pcpu_gen_cookie {
  10        local_t nesting;
  11        u64 last;
  12} __aligned(16);
  13
  14struct gen_cookie {
  15        struct pcpu_gen_cookie __percpu *local;
  16        atomic64_t forward_last ____cacheline_aligned_in_smp;
  17        atomic64_t reverse_last;
  18};
  19
  20#define COOKIE_LOCAL_BATCH      4096
  21
  22#define DEFINE_COOKIE(name)                                             \
  23        static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name);        \
  24        static struct gen_cookie name = {                               \
  25                .local          = &__##name,                            \
  26                .forward_last   = ATOMIC64_INIT(0),                     \
  27                .reverse_last   = ATOMIC64_INIT(0),                     \
  28        }
  29
  30static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)
  31{
  32        struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);
  33        u64 val;
  34
  35        if (likely(local_inc_return(&local->nesting) == 1)) {
  36                val = local->last;
  37                if (__is_defined(CONFIG_SMP) &&
  38                    unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {
  39                        s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,
  40                                                       &gc->forward_last);
  41                        val = next - COOKIE_LOCAL_BATCH;
  42                }
  43                local->last = ++val;
  44        } else {
  45                val = atomic64_dec_return(&gc->reverse_last);
  46        }
  47        local_dec(&local->nesting);
  48        return val;
  49}
  50
  51#endif /* __LINUX_COOKIE_H */
  52