/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 * Connection Data Control (CDC)
 *
 * Copyright IBM Corp. 2016
 *
 * Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 */

#ifndef SMC_CDC_H
#define SMC_CDC_H

#include <linux/kernel.h> /* max_t */
#include <linux/atomic.h>
#include <linux/in.h>
#include <linux/compiler.h>

#include "smc.h"
#include "smc_core.h"
#include "smc_wr.h"

#define SMC_CDC_MSG_TYPE                0xFE

/* in network byte order */
union smc_cdc_cursor {          /* SMC cursor */
        struct {
                __be16  reserved;
                __be16  wrap;
                __be32  count;
        };
#ifdef KERNEL_HAS_ATOMIC64
        atomic64_t      acurs;          /* for atomic processing */
#else
        u64             acurs;          /* for atomic processing */
#endif
} __aligned(8);

/* in network byte order */
struct smc_cdc_msg {
        struct smc_wr_rx_hdr            common; /* .type = 0xFE */
        u8                              len;    /* 44 */
        __be16                          seqno;
        __be32                          token;
        union smc_cdc_cursor            prod;
        union smc_cdc_cursor            cons;   /* piggy backed "ack" */
        struct smc_cdc_producer_flags   prod_flags;
        struct smc_cdc_conn_state_flags conn_state_flags;
        u8                              reserved[18];
};

/* SMC-D cursor format */
union smcd_cdc_cursor {
        struct {
                u16     wrap;
                u32     count;
                struct smc_cdc_producer_flags   prod_flags;
                struct smc_cdc_conn_state_flags conn_state_flags;
        } __packed;
#ifdef KERNEL_HAS_ATOMIC64
        atomic64_t              acurs;          /* for atomic processing */
#else
        u64                     acurs;          /* for atomic processing */
#endif
} __aligned(8);

/* CDC message for SMC-D */
struct smcd_cdc_msg {
        struct smc_wr_rx_hdr common;    /* Type = 0xFE */
        u8 res1[7];
        union smcd_cdc_cursor   prod;
        union smcd_cdc_cursor   cons;
        u8 res3[8];
} __aligned(8);

static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
{
        return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
               conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
}

static inline bool smc_cdc_rxed_any_close_or_senddone(
        struct smc_connection *conn)
{
        return smc_cdc_rxed_any_close(conn) ||
               conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
}

static inline void smc_curs_add(int size, union smc_host_cursor *curs,
                                int value)
{
        curs->count += value;
        if (curs->count >= size) {
                curs->wrap++;
                curs->count -= size;
        }
}

/* SMC cursors are 8 bytes long and require atomic reading and writing */
static inline u64 smc_curs_read(union smc_host_cursor *curs,
                                struct smc_connection *conn)
{
#ifndef KERNEL_HAS_ATOMIC64
        unsigned long flags;
        u64 ret;

        spin_lock_irqsave(&conn->acurs_lock, flags);
        ret = curs->acurs;
        spin_unlock_irqrestore(&conn->acurs_lock, flags);
        return ret;
#else
        return atomic64_read(&curs->acurs);
#endif
}

/* Copy cursor src into tgt */
static inline void smc_curs_copy(union smc_host_cursor *tgt,
                                 union smc_host_cursor *src,
                                 struct smc_connection *conn)
{
#ifndef KERNEL_HAS_ATOMIC64
        unsigned long flags;

        spin_lock_irqsave(&conn->acurs_lock, flags);
        tgt->acurs = src->acurs;
        spin_unlock_irqrestore(&conn->acurs_lock, flags);
#else
        atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
#endif
}

static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
                                     union smc_cdc_cursor *src,
                                     struct smc_connection *conn)
{
#ifndef KERNEL_HAS_ATOMIC64
        unsigned long flags;

        spin_lock_irqsave(&conn->acurs_lock, flags);
        tgt->acurs = src->acurs;
        spin_unlock_irqrestore(&conn->acurs_lock, flags);
#else
        atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
#endif
}

static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
                                  union smcd_cdc_cursor *src,
                                  struct smc_connection *conn)
{
#ifndef KERNEL_HAS_ATOMIC64
        unsigned long flags;

        spin_lock_irqsave(&conn->acurs_lock, flags);
        tgt->acurs = src->acurs;
        spin_unlock_irqrestore(&conn->acurs_lock, flags);
#else
        atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
#endif
}

/* calculate cursor difference between old and new, where old <= new and
 * difference cannot exceed size
 */
static inline int smc_curs_diff(unsigned int size,
                                union smc_host_cursor *old,
                                union smc_host_cursor *new)
{
        if (old->wrap != new->wrap)
                return max_t(int, 0,
                             ((size - old->count) + new->count));

        return max_t(int, 0, (new->count - old->count));
}

/* calculate cursor difference between old and new - returns negative
 * value in case old > new
 */
static inline int smc_curs_comp(unsigned int size,
                                union smc_host_cursor *old,
                                union smc_host_cursor *new)
{
        if (old->wrap > new->wrap ||
            (old->wrap == new->wrap && old->count > new->count))
                return -smc_curs_diff(size, new, old);
        return smc_curs_diff(size, old, new);
}

/* calculate cursor difference between old and new, where old <= new and
 * difference may exceed size
 */
static inline int smc_curs_diff_large(unsigned int size,
                                      union smc_host_cursor *old,
                                      union smc_host_cursor *new)
{
        if (old->wrap < new->wrap)
                return min_t(int,
                             (size - old->count) + new->count +
                             (new->wrap - old->wrap - 1) * size,
                             size);

        if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
                return min_t(int,
                             (size - old->count) + new->count +
                             (new->wrap + 0xffff - old->wrap) * size,
                             size);

        return max_t(int, 0, (new->count - old->count));
}

static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
                                          union smc_host_cursor *local,
                                          union smc_host_cursor *save,
                                          struct smc_connection *conn)
{
        smc_curs_copy(save, local, conn);
        peer->count = htonl(save->count);
        peer->wrap = htons(save->wrap);
        /* peer->reserved = htons(0); must be ensured by caller */
}

static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
                                       struct smc_connection *conn,
                                       union smc_host_cursor *save)
{
        struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;

        peer->common.type = local->common.type;
        peer->len = local->len;
        peer->seqno = htons(local->seqno);
        peer->token = htonl(local->token);
        smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
        smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
        peer->prod_flags = local->prod_flags;
        peer->conn_state_flags = local->conn_state_flags;
}

static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
                                          union smc_cdc_cursor *peer,
                                          struct smc_connection *conn)
{
        union smc_host_cursor temp, old;
        union smc_cdc_cursor net;

        smc_curs_copy(&old, local, conn);
        smc_curs_copy_net(&net, peer, conn);
        temp.count = ntohl(net.count);
        temp.wrap = ntohs(net.wrap);
        if ((old.wrap > temp.wrap) && temp.wrap)
                return;
        if ((old.wrap == temp.wrap) &&
            (old.count > temp.count))
                return;
        smc_curs_copy(local, &temp, conn);
}

static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
                                        struct smc_cdc_msg *peer,
                                        struct smc_connection *conn)
{
        local->common.type = peer->common.type;
        local->len = peer->len;
        local->seqno = ntohs(peer->seqno);
        local->token = ntohl(peer->token);
        smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
        smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
        local->prod_flags = peer->prod_flags;
        local->conn_state_flags = peer->conn_state_flags;
}

static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
                                        struct smcd_cdc_msg *peer,
                                        struct smc_connection *conn)
{
        union smc_host_cursor temp;

        temp.wrap = peer->prod.wrap;
        temp.count = peer->prod.count;
        smc_curs_copy(&local->prod, &temp, conn);

        temp.wrap = peer->cons.wrap;
        temp.count = peer->cons.count;
        smc_curs_copy(&local->cons, &temp, conn);
        local->prod_flags = peer->cons.prod_flags;
        local->conn_state_flags = peer->cons.conn_state_flags;
}

static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
                                       struct smc_cdc_msg *peer,
                                       struct smc_connection *conn)
{
        if (conn->lgr->is_smcd)
                smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
        else
                smcr_cdc_msg_to_host(local, peer, conn);
}

struct smc_cdc_tx_pend {
        struct smc_connection   *conn;          /* socket connection */
        union smc_host_cursor   cursor;         /* tx sndbuf cursor sent */
        union smc_host_cursor   p_cursor;       /* rx RMBE cursor produced */
        u16                     ctrl_seq;       /* conn. tx sequence # */
};

int smc_cdc_get_free_slot(struct smc_connection *conn,
                          struct smc_wr_buf **wr_buf,
                          struct smc_rdma_wr **wr_rdma_buf,
                          struct smc_cdc_tx_pend **pend);
void smc_cdc_tx_dismiss_slots(struct smc_connection *conn);
int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
                     struct smc_cdc_tx_pend *pend);
int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
int smcd_cdc_msg_send(struct smc_connection *conn);
int smc_cdc_init(void) __init;
void smcd_cdc_rx_init(struct smc_connection *conn);

#endif /* SMC_CDC_H */