// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) Marvell International Ltd. and its affiliates
 */

#include <common.h>
#include <i2c.h>
#include <spl.h>
#include <asm/io.h>
#include <asm/arch/cpu.h>
#include <asm/arch/soc.h>

#include "ddr3_hw_training.h"
#include "xor.h"
#include "xor_regs.h"

static void ddr3_flush_l1_line(u32 line);

extern u32 pbs_pattern[2][LEN_16BIT_PBS_PATTERN];
extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
#if defined(MV88F78X60)
extern u32 pbs_pattern_64b[2][LEN_PBS_PATTERN];
#endif
extern u32 pbs_dq_mapping[PUP_NUM_64BIT][DQ_NUM];

#if defined(MV88F78X60) || defined(MV88F672X)
/* PBS locked dq (per pup) */
u32 pbs_locked_dq[MAX_PUP_NUM][DQ_NUM] = { { 0 } };
u32 pbs_locked_dm[MAX_PUP_NUM] = { 0 };
u32 pbs_locked_value[MAX_PUP_NUM][DQ_NUM] = { { 0 } };

int per_bit_data[MAX_PUP_NUM][DQ_NUM];
#endif

static u32 sdram_data[LEN_KILLER_PATTERN] __aligned(32) = { 0 };

static struct crc_dma_desc dma_desc __aligned(32) = { 0 };

#define XOR_TIMEOUT 0x8000000

struct xor_channel_t {
        struct crc_dma_desc *desc;
        unsigned long desc_phys_addr;
};

#define XOR_CAUSE_DONE_MASK(chan)       ((0x1 | 0x2) << (chan * 16))

void xor_waiton_eng(int chan)
{
        int timeout;

        timeout = 0;
        while (!(reg_read(XOR_CAUSE_REG(XOR_UNIT(chan))) &
                 XOR_CAUSE_DONE_MASK(XOR_CHAN(chan)))) {
                if (timeout > XOR_TIMEOUT)
                        goto timeout;

                timeout++;
        }

        timeout = 0;
        while (mv_xor_state_get(chan) != MV_IDLE) {
                if (timeout > XOR_TIMEOUT)
                        goto timeout;

                timeout++;
        }

        /* Clear int */
        reg_write(XOR_CAUSE_REG(XOR_UNIT(chan)),
                  ~(XOR_CAUSE_DONE_MASK(XOR_CHAN(chan))));

timeout:
        return;
}

static int special_compare_pattern(u32 uj)
{
        if ((uj == 30) || (uj == 31) || (uj == 61) || (uj == 62) ||
            (uj == 93) || (uj == 94) || (uj == 126) || (uj == 127))
                return 1;

        return 0;
}

/*
 * Compare code extracted as its used by multiple functions. This
 * reduces code-size and makes it easier to maintain it. Additionally
 * the code is not indented that much and therefore easier to read.
 */
static void compare_pattern_v1(u32 uj, u32 *pup, u32 *pattern,
                               u32 pup_groups, int debug_dqs)
{
        u32 val;
        u32 uk;
        u32 var1;
        u32 var2;
        __maybe_unused u32 dq;

        if (((sdram_data[uj]) != (pattern[uj])) && (*pup != 0xFF)) {
                for (uk = 0; uk < PUP_NUM_32BIT; uk++) {
                        val = CMP_BYTE_SHIFT * uk;
                        var1 = ((sdram_data[uj] >> val) & CMP_BYTE_MASK);
                        var2 = ((pattern[uj] >> val) & CMP_BYTE_MASK);

                        if (var1 != var2) {
                                *pup |= (1 << (uk + (PUP_NUM_32BIT *
                                                     (uj % pup_groups))));

#ifdef MV_DEBUG_DQS
                                if (!debug_dqs)
                                        continue;

                                for (dq = 0; dq < DQ_NUM; dq++) {
                                        val = uk + (PUP_NUM_32BIT *
                                                    (uj % pup_groups));
                                        if (((var1 >> dq) & 0x1) !=
                                            ((var2 >> dq) & 0x1))
                                                per_bit_data[val][dq] = 1;
                                        else
                                                per_bit_data[val][dq] = 0;
                                }
#endif
                        }
                }
        }
}

static void compare_pattern_v2(u32 uj, u32 *pup, u32 *pattern)
{
        u32 val;
        u32 uk;
        u32 var1;
        u32 var2;

        if (((sdram_data[uj]) != (pattern[uj])) && (*pup != 0x3)) {
                /* Found error */
                for (uk = 0; uk < PUP_NUM_32BIT; uk++) {
                        val = CMP_BYTE_SHIFT * uk;
                        var1 = (sdram_data[uj] >> val) & CMP_BYTE_MASK;
                        var2 = (pattern[uj] >> val) & CMP_BYTE_MASK;
                        if (var1 != var2)
                                *pup |= (1 << (uk % PUP_NUM_16BIT));
                }
        }
}

/*
 * Name:     ddr3_sdram_compare
 * Desc:     Execute compare per PUP
 * Args:     unlock_pup      Bit array of the unlock pups
 *           new_locked_pup  Output  bit array of the pups with failed compare
 *           pattern         Pattern to compare
 *           pattern_len     Length of pattern (in bytes)
 *           sdram_offset    offset address to the SDRAM
 *           write           write to the SDRAM before read
 *           mask            compare pattern with mask;
 *           mask_pattern    Mask to compare pattern
 *
 * Notes:
 * Returns:  MV_OK if success, other error code if fail.
 */
int ddr3_sdram_compare(MV_DRAM_INFO *dram_info, u32 unlock_pup,
                       u32 *new_locked_pup, u32 *pattern,
                       u32 pattern_len, u32 sdram_offset, int write,
                       int mask, u32 *mask_pattern,
                       int special_compare)
{
        u32 uj;
        __maybe_unused u32 pup_groups;
        __maybe_unused u32 dq;

#if !defined(MV88F67XX)
        if (dram_info->num_of_std_pups == PUP_NUM_64BIT)
                pup_groups = 2;
        else
                pup_groups = 1;
#endif

        ddr3_reset_phy_read_fifo();

        /* Check if need to write to sdram before read */
        if (write == 1)
                ddr3_dram_sram_burst((u32)pattern, sdram_offset, pattern_len);

        ddr3_dram_sram_burst(sdram_offset, (u32)sdram_data, pattern_len);

        /* Compare read result to write */
        for (uj = 0; uj < pattern_len; uj++) {
                if (special_compare && special_compare_pattern(uj))
                        continue;

#if defined(MV88F78X60) || defined(MV88F672X)
                compare_pattern_v1(uj, new_locked_pup, pattern, pup_groups, 1);
#elif defined(MV88F67XX)
                compare_pattern_v2(uj, new_locked_pup, pattern);
#endif
        }

        return MV_OK;
}

#if defined(MV88F78X60) || defined(MV88F672X)
/*
 * Name:     ddr3_sdram_dm_compare
 * Desc:     Execute compare per PUP
 * Args:     unlock_pup      Bit array of the unlock pups
 *           new_locked_pup  Output  bit array of the pups with failed compare
 *           pattern         Pattern to compare
 *           pattern_len     Length of pattern (in bytes)
 *           sdram_offset    offset address to the SDRAM
 *           write           write to the SDRAM before read
 *           mask            compare pattern with mask;
 *           mask_pattern    Mask to compare pattern
 *
 * Notes:
 * Returns:  MV_OK if success, other error code if fail.
 */
int ddr3_sdram_dm_compare(MV_DRAM_INFO *dram_info, u32 unlock_pup,
                          u32 *new_locked_pup, u32 *pattern,
                          u32 sdram_offset)
{
        u32 uj, uk, var1, var2, pup_groups;
        u32 val;
        u32 pup = 0;

        if (dram_info->num_of_std_pups == PUP_NUM_64BIT)
                pup_groups = 2;
        else
                pup_groups = 1;

        ddr3_dram_sram_burst((u32)pattern, SDRAM_PBS_TX_OFFS,
                             LEN_PBS_PATTERN);
        ddr3_dram_sram_burst(SDRAM_PBS_TX_OFFS, (u32)sdram_data,
                             LEN_PBS_PATTERN);

        /* Validate the correctness of the results */
        for (uj = 0; uj < LEN_PBS_PATTERN; uj++)
                compare_pattern_v1(uj, &pup, pattern, pup_groups, 0);

        /* Test the DM Signals */
        *(u32 *)(SDRAM_PBS_TX_OFFS + 0x10) = 0x12345678;
        *(u32 *)(SDRAM_PBS_TX_OFFS + 0x14) = 0x12345678;

        sdram_data[0] = *(u32 *)(SDRAM_PBS_TX_OFFS + 0x10);
        sdram_data[1] = *(u32 *)(SDRAM_PBS_TX_OFFS + 0x14);

        for (uj = 0; uj < 2; uj++) {
                if (((sdram_data[uj]) != (pattern[uj])) &&
                    (*new_locked_pup != 0xFF)) {
                        for (uk = 0; uk < PUP_NUM_32BIT; uk++) {
                                val = CMP_BYTE_SHIFT * uk;
                                var1 = ((sdram_data[uj] >> val) & CMP_BYTE_MASK);
                                var2 = ((pattern[uj] >> val) & CMP_BYTE_MASK);
                                if (var1 != var2) {
                                        *new_locked_pup |= (1 << (uk +
                                                (PUP_NUM_32BIT * (uj % pup_groups))));
                                        *new_locked_pup |= pup;
                                }
                        }
                }
        }

        return MV_OK;
}

/*
 * Name:     ddr3_sdram_pbs_compare
 * Desc:     Execute SRAM compare per PUP and DQ.
 * Args:     pup_locked             bit array of locked pups
 *           is_tx                  Indicate whether Rx or Tx
 *           pbs_pattern_idx        Index of PBS pattern
 *           pbs_curr_val           The PBS value
 *           pbs_lock_val           The value to set to locked PBS
 *           skew_array             Global array to update with the compare results
 *           ai_unlock_pup_dq_array bit array of the locked / unlocked pups per dq.
 * Notes:
 * Returns:  MV_OK if success, other error code if fail.
 */
int ddr3_sdram_pbs_compare(MV_DRAM_INFO *dram_info, u32 pup_locked,
                           int is_tx, u32 pbs_pattern_idx,
                           u32 pbs_curr_val, u32 pbs_lock_val,
                           u32 *skew_array, u8 *unlock_pup_dq_array,
                           u32 ecc)
{
        /* bit array failed dq per pup for current compare */
        u32 pbs_write_pup[DQ_NUM] = { 0 };
        u32 update_pup; /* pup as HW convention */
        u32 max_pup;    /* maximal pup index */
        u32 pup_addr;
        u32 ui, dq, pup;
        int var1, var2;
        u32 sdram_offset, pup_groups, tmp_pup;
        u32 *pattern_ptr;
        u32 val;

        /* Choose pattern */
        switch (dram_info->ddr_width) {
#if defined(MV88F672X)
        case 16:
                pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
                break;
#endif
        case 32:
                pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
                break;
#if defined(MV88F78X60)
        case 64:
                pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
                break;
#endif
        default:
                return MV_FAIL;
        }

        max_pup = dram_info->num_of_std_pups;

        sdram_offset = SDRAM_PBS_I_OFFS + pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS;

        if (dram_info->num_of_std_pups == PUP_NUM_64BIT)
                pup_groups = 2;
        else
                pup_groups = 1;

        ddr3_reset_phy_read_fifo();

        /* Check if need to write to sdram before read */
        if (is_tx == 1) {
                ddr3_dram_sram_burst((u32)pattern_ptr, sdram_offset,
                                     LEN_PBS_PATTERN);
        }

        ddr3_dram_sram_read(sdram_offset, (u32)sdram_data, LEN_PBS_PATTERN);

        /* Compare read result to write */
        for (ui = 0; ui < LEN_PBS_PATTERN; ui++) {
                if ((sdram_data[ui]) != (pattern_ptr[ui])) {
                        /* found error */
                        /* error in low pup group */
                        for (pup = 0; pup < PUP_NUM_32BIT; pup++) {
                                val = CMP_BYTE_SHIFT * pup;
                                var1 = ((sdram_data[ui] >> val) &
                                        CMP_BYTE_MASK);
                                var2 = ((pattern_ptr[ui] >> val) &
                                        CMP_BYTE_MASK);

                                if (var1 != var2) {
                                        if (dram_info->ddr_width > 16) {
                                                tmp_pup = (pup + PUP_NUM_32BIT *
                                                           (ui % pup_groups));
                                        } else {
                                                tmp_pup = (pup % PUP_NUM_16BIT);
                                        }

                                        update_pup = (1 << tmp_pup);
                                        if (ecc && (update_pup != 0x1))
                                                continue;

                                        /*
                                         * Pup is failed - Go over all DQs and
                                         * look for failures
                                         */
                                        for (dq = 0; dq < DQ_NUM; dq++) {
                                                val = tmp_pup * (1 - ecc) +
                                                        ecc * ECC_PUP;
                                                if (((var1 >> dq) & 0x1) !=
                                                    ((var2 >> dq) & 0x1)) {
                                                        if (pbs_locked_dq[val][dq] == 1 &&
                                                            pbs_locked_value[val][dq] != pbs_curr_val)
                                                                continue;

                                                        /*
                                                         * Activate write to
                                                         * update PBS to
                                                         * pbs_lock_val
                                                         */
                                                        pbs_write_pup[dq] |=
                                                                update_pup;

                                                        /*
                                                         * Update the
                                                         * unlock_pup_dq_array
                                                         */
                                                        unlock_pup_dq_array[dq] &=
                                                                ~update_pup;

                                                        /*
                                                         * Lock PBS value for
                                                         * failed bits in
                                                         * compare operation
                                                         */
                                                        skew_array[tmp_pup * DQ_NUM + dq] =
                                                                pbs_curr_val;
                                                }
                                        }
                                }
                        }
                }
        }

        pup_addr = (is_tx == 1) ? PUP_PBS_TX : PUP_PBS_RX;

        /* Set last failed bits PBS to min / max pbs value */
        for (dq = 0; dq < DQ_NUM; dq++) {
                for (pup = 0; pup < max_pup; pup++) {
                        if (pbs_write_pup[dq] & (1 << pup)) {
                                val = pup * (1 - ecc) + ecc * ECC_PUP;
                                if (pbs_locked_dq[val][dq] == 1 &&
                                    pbs_locked_value[val][dq] != pbs_curr_val)
                                        continue;

                                /* Mark the dq as locked */
                                pbs_locked_dq[val][dq] = 1;
                                pbs_locked_value[val][dq] = pbs_curr_val;
                                ddr3_write_pup_reg(pup_addr +
                                                   pbs_dq_mapping[val][dq],
                                                   CS0, val, 0, pbs_lock_val);
                        }
                }
        }

        return MV_OK;
}
#endif

/*
 * Name:     ddr3_sdram_direct_compare
 * Desc:     Execute compare  per PUP without DMA (no burst mode)
 * Args:     unlock_pup       Bit array of the unlock pups
 *           new_locked_pup   Output  bit array of the pups with failed compare
 *           pattern          Pattern to compare
 *           pattern_len      Length of pattern (in bytes)
 *           sdram_offset     offset address to the SDRAM
 *           write            write to the SDRAM before read
 *           mask             compare pattern with mask;
 *           auiMaskPatter    Mask to compare pattern
 *
 * Notes:
 * Returns:  MV_OK if success, other error code if fail.
 */
int ddr3_sdram_direct_compare(MV_DRAM_INFO *dram_info, u32 unlock_pup,
                              u32 *new_locked_pup, u32 *pattern,
                              u32 pattern_len, u32 sdram_offset,
                              int write, int mask, u32 *mask_pattern)
{
        u32 uj, uk, pup_groups;
        u32 *sdram_addr;        /* used to read from SDRAM */

        sdram_addr = (u32 *)sdram_offset;

        if (dram_info->num_of_std_pups == PUP_NUM_64BIT)
                pup_groups = 2;
        else
                pup_groups = 1;

        /* Check if need to write before read */
        if (write == 1) {
                for (uk = 0; uk < pattern_len; uk++) {
                        *sdram_addr = pattern[uk];
                        sdram_addr++;
                }
        }

        sdram_addr = (u32 *)sdram_offset;

        for (uk = 0; uk < pattern_len; uk++) {
                sdram_data[uk] = *sdram_addr;
                sdram_addr++;
        }

        /* Compare read result to write */
        for (uj = 0; uj < pattern_len; uj++) {
                if (dram_info->ddr_width > 16) {
                        compare_pattern_v1(uj, new_locked_pup, pattern,
                                           pup_groups, 0);
                } else {
                        compare_pattern_v2(uj, new_locked_pup, pattern);
                }
        }

        return MV_OK;
}

/*
 * Name:     ddr3_dram_sram_burst
 * Desc:     Read from the SDRAM in burst of 64 bytes
 * Args:     src
 *           dst
 * Notes:    Using the XOR mechanism
 * Returns:  MV_OK if success, other error code if fail.
 */
int ddr3_dram_sram_burst(u32 src, u32 dst, u32 len)
{
        u32 chan, byte_count, cs_num, byte;
        struct xor_channel_t channel;

        chan = 0;
        byte_count = len * 4;

        /* Wait for previous transfer completion */
        while (mv_xor_state_get(chan) != MV_IDLE)
                ;

        /* Build the channel descriptor */
        channel.desc = &dma_desc;

        /* Enable Address Override and set correct src and dst */
        if (src < SRAM_BASE) {
                /* src is DRAM CS, dst is SRAM */
                cs_num = (src / (1 + SDRAM_CS_SIZE));
                reg_write(XOR_ADDR_OVRD_REG(0, 0),
                          ((cs_num << 1) | (1 << 0)));
                channel.desc->src_addr0 = (src % (1 + SDRAM_CS_SIZE));
                channel.desc->dst_addr = dst;
        } else {
                /* src is SRAM, dst is DRAM CS */
                cs_num = (dst / (1 + SDRAM_CS_SIZE));
                reg_write(XOR_ADDR_OVRD_REG(0, 0),
                          ((cs_num << 25) | (1 << 24)));
                channel.desc->src_addr0 = (src);
                channel.desc->dst_addr = (dst % (1 + SDRAM_CS_SIZE));
                channel.desc->src_addr0 = src;
                channel.desc->dst_addr = (dst % (1 + SDRAM_CS_SIZE));
        }

        channel.desc->src_addr1 = 0;
        channel.desc->byte_cnt = byte_count;
        channel.desc->next_desc_ptr = 0;
        channel.desc->status = 1 << 31;
        channel.desc->desc_cmd = 0x0;
        channel.desc_phys_addr = (unsigned long)&dma_desc;

        ddr3_flush_l1_line((u32)&dma_desc);

        /* Issue the transfer */
        if (mv_xor_transfer(chan, MV_DMA, channel.desc_phys_addr) != MV_OK)
                return MV_FAIL;

        /* Wait for completion */
        xor_waiton_eng(chan);

        if (dst > SRAM_BASE) {
                for (byte = 0; byte < byte_count; byte += 0x20)
                        cache_inv(dst + byte);
        }

        return MV_OK;
}

/*
 * Name:     ddr3_flush_l1_line
 * Desc:
 * Args:
 * Notes:
 * Returns:  MV_OK if success, other error code if fail.
 */
static void ddr3_flush_l1_line(u32 line)
{
        u32 reg;

#if defined(MV88F672X)
        reg = 1;
#else
        reg = reg_read(REG_SAMPLE_RESET_LOW_ADDR) &
                (1 << REG_SAMPLE_RESET_CPU_ARCH_OFFS);
#ifdef MV88F67XX
        reg = ~reg & (1 << REG_SAMPLE_RESET_CPU_ARCH_OFFS);
#endif
#endif

        if (reg) {
                /* V7 Arch mode */
                flush_l1_v7(line);
                flush_l1_v7(line + CACHE_LINE_SIZE);
        } else {
                /* V6 Arch mode */
                flush_l1_v6(line);
                flush_l1_v6(line + CACHE_LINE_SIZE);
        }
}

int ddr3_dram_sram_read(u32 src, u32 dst, u32 len)
{
        u32 ui;
        u32 *dst_ptr, *src_ptr;

        dst_ptr = (u32 *)dst;
        src_ptr = (u32 *)src;

        for (ui = 0; ui < len; ui++) {
                *dst_ptr = *src_ptr;
                dst_ptr++;
                src_ptr++;
        }

        return MV_OK;
}

int ddr3_sdram_dqs_compare(MV_DRAM_INFO *dram_info, u32 unlock_pup,
                           u32 *new_locked_pup, u32 *pattern,
                           u32 pattern_len, u32 sdram_offset, int write,
                           int mask, u32 *mask_pattern,
                           int special_compare)
{
        u32 uj, pup_groups;

        if (dram_info->num_of_std_pups == PUP_NUM_64BIT)
                pup_groups = 2;
        else
                pup_groups = 1;

        ddr3_reset_phy_read_fifo();

        /* Check if need to write to sdram before read */
        if (write == 1)
                ddr3_dram_sram_burst((u32)pattern, sdram_offset, pattern_len);

        ddr3_dram_sram_burst(sdram_offset, (u32)sdram_data, pattern_len);

        /* Compare read result to write */
        for (uj = 0; uj < pattern_len; uj++) {
                if (special_compare && special_compare_pattern(uj))
                        continue;

                if (dram_info->ddr_width > 16) {
                        compare_pattern_v1(uj, new_locked_pup, pattern,
                                           pup_groups, 1);
                } else {
                        compare_pattern_v2(uj, new_locked_pup, pattern);
                }
        }

        return MV_OK;
}

void ddr3_reset_phy_read_fifo(void)
{
        u32 reg;

        /* reset read FIFO */
        reg = reg_read(REG_DRAM_TRAINING_ADDR);
        /* Start Auto Read Leveling procedure */
        reg |= (1 << REG_DRAM_TRAINING_RL_OFFS);

        /* 0x15B0 - Training Register */
        reg_write(REG_DRAM_TRAINING_ADDR, reg);

        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
        reg |= ((1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS) +
                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS));

        /* [0] = 1 - Enable SW override, [4] = 1 - FIFO reset  */
        /* 0x15B8 - Training SW 2 Register */
        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);

        do {
                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
                        (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
        } while (reg);  /* Wait for '0' */

        reg = reg_read(REG_DRAM_TRAINING_ADDR);

        /* Clear Auto Read Leveling procedure */
        reg &= ~(1 << REG_DRAM_TRAINING_RL_OFFS);

        /* 0x15B0 - Training Register */
        reg_write(REG_DRAM_TRAINING_ADDR, reg);
}