LXR linux/include/vdso/helpers.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef __VDSO_HELPERS_H
   3#define __VDSO_HELPERS_H
   4
   5#ifndef __ASSEMBLY__
   6
   7#include <vdso/datapage.h>
   8
   9static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
  10{
  11        u32 seq;
  12
  13        while (unlikely((seq = READ_ONCE(vd->seq)) & 1))
  14                cpu_relax();
  15
  16        smp_rmb();
  17        return seq;
  18}
  19
  20static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
  21                                           u32 start)
  22{
  23        u32 seq;
  24
  25        smp_rmb();
  26        seq = READ_ONCE(vd->seq);
  27        return seq != start;
  28}
  29
  30static __always_inline void vdso_write_begin(struct vdso_data *vd)
  31{
  32        /*
  33         * WRITE_ONCE it is required otherwise the compiler can validly tear
  34         * updates to vd[x].seq and it is possible that the value seen by the
  35         * reader it is inconsistent.
  36         */
  37        WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
  38        WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
  39        smp_wmb();
  40}
  41
  42static __always_inline void vdso_write_end(struct vdso_data *vd)
  43{
  44        smp_wmb();
  45        /*
  46         * WRITE_ONCE it is required otherwise the compiler can validly tear
  47         * updates to vd[x].seq and it is possible that the value seen by the
  48         * reader it is inconsistent.
  49         */
  50        WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
  51        WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
  52}
  53
  54#endif /* !__ASSEMBLY__ */
  55
  56#endif /* __VDSO_HELPERS_H */
  57