LXR linux/kernel/bpf/percpu

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (c) 2016 Facebook
   3 */
   4#include "percpu_freelist.h"
   5
   6int pcpu_freelist_init(struct pcpu_freelist *s)
   7{
   8        int cpu;
   9
  10        s->freelist = alloc_percpu(struct pcpu_freelist_head);
  11        if (!s->freelist)
  12                return -ENOMEM;
  13
  14        for_each_possible_cpu(cpu) {
  15                struct pcpu_freelist_head *head = per_cpu_ptr(s->freelist, cpu);
  16
  17                raw_spin_lock_init(&head->lock);
  18                head->first = NULL;
  19        }
  20        return 0;
  21}
  22
  23void pcpu_freelist_destroy(struct pcpu_freelist *s)
  24{
  25        free_percpu(s->freelist);
  26}
  27
  28static inline void pcpu_freelist_push_node(struct pcpu_freelist_head *head,
  29                                           struct pcpu_freelist_node *node)
  30{
  31        node->next = head->first;
  32        head->first = node;
  33}
  34
  35static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head,
  36                                         struct pcpu_freelist_node *node)
  37{
  38        raw_spin_lock(&head->lock);
  39        pcpu_freelist_push_node(head, node);
  40        raw_spin_unlock(&head->lock);
  41}
  42
  43void __pcpu_freelist_push(struct pcpu_freelist *s,
  44                        struct pcpu_freelist_node *node)
  45{
  46        struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist);
  47
  48        ___pcpu_freelist_push(head, node);
  49}
  50
  51void pcpu_freelist_push(struct pcpu_freelist *s,
  52                        struct pcpu_freelist_node *node)
  53{
  54        unsigned long flags;
  55
  56        local_irq_save(flags);
  57        __pcpu_freelist_push(s, node);
  58        local_irq_restore(flags);
  59}
  60
  61void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
  62                            u32 nr_elems)
  63{
  64        struct pcpu_freelist_head *head;
  65        int i, cpu, pcpu_entries;
  66
  67        pcpu_entries = nr_elems / num_possible_cpus() + 1;
  68        i = 0;
  69
  70        for_each_possible_cpu(cpu) {
  71again:
  72                head = per_cpu_ptr(s->freelist, cpu);
  73                /* No locking required as this is not visible yet. */
  74                pcpu_freelist_push_node(head, buf);
  75                i++;
  76                buf += elem_size;
  77                if (i == nr_elems)
  78                        break;
  79                if (i % pcpu_entries)
  80                        goto again;
  81        }
  82}
  83
  84struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s)
  85{
  86        struct pcpu_freelist_head *head;
  87        struct pcpu_freelist_node *node;
  88        int orig_cpu, cpu;
  89
  90        orig_cpu = cpu = raw_smp_processor_id();
  91        while (1) {
  92                head = per_cpu_ptr(s->freelist, cpu);
  93                raw_spin_lock(&head->lock);
  94                node = head->first;
  95                if (node) {
  96                        head->first = node->next;
  97                        raw_spin_unlock(&head->lock);
  98                        return node;
  99                }
 100                raw_spin_unlock(&head->lock);
 101                cpu = cpumask_next(cpu, cpu_possible_mask);
 102                if (cpu >= nr_cpu_ids)
 103                        cpu = 0;
 104                if (cpu == orig_cpu)
 105                        return NULL;
 106        }
 107}
 108
 109struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
 110{
 111        struct pcpu_freelist_node *ret;
 112        unsigned long flags;
 113
 114        local_irq_save(flags);
 115        ret = __pcpu_freelist_pop(s);
 116        local_irq_restore(flags);
 117        return ret;
 118}
 119