// SPDX-License-Identifier: GPL-2.0
/*
 * Generic userspace implementations of gettimeofday() and similar.
 */
#include <vdso/datapage.h>
#include <vdso/helpers.h>

#ifndef vdso_calc_delta
/*
 * Default implementation which works for all sane clocksources. That
 * obviously excludes x86/TSC.
 */
static __always_inline
u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
{
        return ((cycles - last) & mask) * mult;
}
#endif

#ifndef vdso_shift_ns
static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
{
        return ns >> shift;
}
#endif

#ifndef __arch_vdso_hres_capable
static inline bool __arch_vdso_hres_capable(void)
{
        return true;
}
#endif

#ifndef vdso_clocksource_ok
static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
{
        return vd->clock_mode != VDSO_CLOCKMODE_NONE;
}
#endif

#ifndef vdso_cycles_ok
static inline bool vdso_cycles_ok(u64 cycles)
{
        return true;
}
#endif

#ifdef CONFIG_TIME_NS
static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
                                          struct __kernel_timespec *ts)
{
        const struct vdso_data *vd;
        const struct timens_offset *offs = &vdns->offset[clk];
        const struct vdso_timestamp *vdso_ts;
        u64 cycles, last, ns;
        u32 seq;
        s64 sec;

        vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
        vd = __arch_get_timens_vdso_data(vd);
        if (clk != CLOCK_MONOTONIC_RAW)
                vd = &vd[CS_HRES_COARSE];
        else
                vd = &vd[CS_RAW];
        vdso_ts = &vd->basetime[clk];

        do {
                seq = vdso_read_begin(vd);

                if (unlikely(!vdso_clocksource_ok(vd)))
                        return -1;

                cycles = __arch_get_hw_counter(vd->clock_mode, vd);
                if (unlikely(!vdso_cycles_ok(cycles)))
                        return -1;
                ns = vdso_ts->nsec;
                last = vd->cycle_last;
                ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
                ns = vdso_shift_ns(ns, vd->shift);
                sec = vdso_ts->sec;
        } while (unlikely(vdso_read_retry(vd, seq)));

        /* Add the namespace offset */
        sec += offs->sec;
        ns += offs->nsec;

        /*
         * Do this outside the loop: a race inside the loop could result
         * in __iter_div_u64_rem() being extremely slow.
         */
        ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
        ts->tv_nsec = ns;

        return 0;
}
#else
static __always_inline
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
{
        return NULL;
}

static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
                                          struct __kernel_timespec *ts)
{
        return -EINVAL;
}
#endif

static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
                                   struct __kernel_timespec *ts)
{
        const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
        u64 cycles, last, sec, ns;
        u32 seq;

        /* Allows to compile the high resolution parts out */
        if (!__arch_vdso_hres_capable())
                return -1;

        do {
                /*
                 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
                 * enabled tasks have a special VVAR page installed which
                 * has vd->seq set to 1 and vd->clock_mode set to
                 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
                 * this does not affect performance because if vd->seq is
                 * odd, i.e. a concurrent update is in progress the extra
                 * check for vd->clock_mode is just a few extra
                 * instructions while spin waiting for vd->seq to become
                 * even again.
                 */
                while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
                        if (IS_ENABLED(CONFIG_TIME_NS) &&
                            vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
                                return do_hres_timens(vd, clk, ts);
                        cpu_relax();
                }
                smp_rmb();

                if (unlikely(!vdso_clocksource_ok(vd)))
                        return -1;

                cycles = __arch_get_hw_counter(vd->clock_mode, vd);
                if (unlikely(!vdso_cycles_ok(cycles)))
                        return -1;
                ns = vdso_ts->nsec;
                last = vd->cycle_last;
                ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
                ns = vdso_shift_ns(ns, vd->shift);
                sec = vdso_ts->sec;
        } while (unlikely(vdso_read_retry(vd, seq)));

        /*
         * Do this outside the loop: a race inside the loop could result
         * in __iter_div_u64_rem() being extremely slow.
         */
        ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
        ts->tv_nsec = ns;

        return 0;
}

#ifdef CONFIG_TIME_NS
static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
                                            struct __kernel_timespec *ts)
{
        const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);
        const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
        const struct timens_offset *offs = &vdns->offset[clk];
        u64 nsec;
        s64 sec;
        s32 seq;

        do {
                seq = vdso_read_begin(vd);
                sec = vdso_ts->sec;
                nsec = vdso_ts->nsec;
        } while (unlikely(vdso_read_retry(vd, seq)));

        /* Add the namespace offset */
        sec += offs->sec;
        nsec += offs->nsec;

        /*
         * Do this outside the loop: a race inside the loop could result
         * in __iter_div_u64_rem() being extremely slow.
         */
        ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
        ts->tv_nsec = nsec;
        return 0;
}
#else
static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
                                            struct __kernel_timespec *ts)
{
        return -1;
}
#endif

static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
                                     struct __kernel_timespec *ts)
{
        const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
        u32 seq;

        do {
                /*
                 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in
                 * do_hres().
                 */
                while ((seq = READ_ONCE(vd->seq)) & 1) {
                        if (IS_ENABLED(CONFIG_TIME_NS) &&
                            vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
                                return do_coarse_timens(vd, clk, ts);
                        cpu_relax();
                }
                smp_rmb();

                ts->tv_sec = vdso_ts->sec;
                ts->tv_nsec = vdso_ts->nsec;
        } while (unlikely(vdso_read_retry(vd, seq)));

        return 0;
}

static __always_inline int
__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
                             struct __kernel_timespec *ts)
{
        u32 msk;

        /* Check for negative values or invalid clocks */
        if (unlikely((u32) clock >= MAX_CLOCKS))
                return -1;

        /*
         * Convert the clockid to a bitmask and use it to check which
         * clocks are handled in the VDSO directly.
         */
        msk = 1U << clock;
        if (likely(msk & VDSO_HRES))
                vd = &vd[CS_HRES_COARSE];
        else if (msk & VDSO_COARSE)
                return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
        else if (msk & VDSO_RAW)
                vd = &vd[CS_RAW];
        else
                return -1;

        return do_hres(vd, clock, ts);
}

static __maybe_unused int
__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
                           struct __kernel_timespec *ts)
{
        int ret = __cvdso_clock_gettime_common(vd, clock, ts);

        if (unlikely(ret))
                return clock_gettime_fallback(clock, ts);
        return 0;
}

static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
        return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
}

#ifdef BUILD_VDSO32
static __maybe_unused int
__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
                             struct old_timespec32 *res)
{
        struct __kernel_timespec ts;
        int ret;

        ret = __cvdso_clock_gettime_common(vd, clock, &ts);

        if (unlikely(ret))
                return clock_gettime32_fallback(clock, res);

        /* For ret == 0 */
        res->tv_sec = ts.tv_sec;
        res->tv_nsec = ts.tv_nsec;

        return ret;
}

static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
{
        return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
}
#endif /* BUILD_VDSO32 */

static __maybe_unused int
__cvdso_gettimeofday_data(const struct vdso_data *vd,
                          struct __kernel_old_timeval *tv, struct timezone *tz)
{

        if (likely(tv != NULL)) {
                struct __kernel_timespec ts;

                if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
                        return gettimeofday_fallback(tv, tz);

                tv->tv_sec = ts.tv_sec;
                tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
        }

        if (unlikely(tz != NULL)) {
                if (IS_ENABLED(CONFIG_TIME_NS) &&
                    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
                        vd = __arch_get_timens_vdso_data(vd);

                tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
                tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
        }

        return 0;
}

static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
        return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
}

#ifdef VDSO_HAS_TIME
static __maybe_unused __kernel_old_time_t
__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
{
        __kernel_old_time_t t;

        if (IS_ENABLED(CONFIG_TIME_NS) &&
            vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
                vd = __arch_get_timens_vdso_data(vd);

        t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);

        if (time)
                *time = t;

        return t;
}

static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
{
        return __cvdso_time_data(__arch_get_vdso_data(), time);
}
#endif /* VDSO_HAS_TIME */

#ifdef VDSO_HAS_CLOCK_GETRES
static __maybe_unused
int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
                                struct __kernel_timespec *res)
{
        u32 msk;
        u64 ns;

        /* Check for negative values or invalid clocks */
        if (unlikely((u32) clock >= MAX_CLOCKS))
                return -1;

        if (IS_ENABLED(CONFIG_TIME_NS) &&
            vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
                vd = __arch_get_timens_vdso_data(vd);

        /*
         * Convert the clockid to a bitmask and use it to check which
         * clocks are handled in the VDSO directly.
         */
        msk = 1U << clock;
        if (msk & (VDSO_HRES | VDSO_RAW)) {
                /*
                 * Preserves the behaviour of posix_get_hrtimer_res().
                 */
                ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
        } else if (msk & VDSO_COARSE) {
                /*
                 * Preserves the behaviour of posix_get_coarse_res().
                 */
                ns = LOW_RES_NSEC;
        } else {
                return -1;
        }

        if (likely(res)) {
                res->tv_sec = 0;
                res->tv_nsec = ns;
        }
        return 0;
}

static __maybe_unused
int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
                              struct __kernel_timespec *res)
{
        int ret = __cvdso_clock_getres_common(vd, clock, res);

        if (unlikely(ret))
                return clock_getres_fallback(clock, res);
        return 0;
}

static __maybe_unused
int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
{
        return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
}

#ifdef BUILD_VDSO32
static __maybe_unused int
__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
                                 struct old_timespec32 *res)
{
        struct __kernel_timespec ts;
        int ret;

        ret = __cvdso_clock_getres_common(vd, clock, &ts);

        if (unlikely(ret))
                return clock_getres32_fallback(clock, res);

        if (likely(res)) {
                res->tv_sec = ts.tv_sec;
                res->tv_nsec = ts.tv_nsec;
        }
        return ret;
}

static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
{
        return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
                                                clock, res);
}
#endif /* BUILD_VDSO32 */
#endif /* VDSO_HAS_CLOCK_GETRES */