// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2020 Marvell International Ltd.
 */

#include <command.h>
#include <dm.h>
#include <hang.h>
#include <i2c.h>
#include <ram.h>
#include <time.h>

#include <linux/bitops.h>
#include <linux/io.h>

#include <mach/octeon_ddr.h>

/* Random number generator stuff */

#define CVMX_OCT_DID_RNG        8ULL

static u64 cvmx_rng_get_random64(void)
{
        return csr_rd(cvmx_build_io_address(CVMX_OCT_DID_RNG, 0));
}

static void cvmx_rng_enable(void)
{
        u64 val;

        val = csr_rd(CVMX_RNM_CTL_STATUS);
        val |= BIT(0) | BIT(1);
        csr_wr(CVMX_RNM_CTL_STATUS, val);
}

#define RLEVEL_PRINTALL_DEFAULT         1
#define WLEVEL_PRINTALL_DEFAULT         1

/*
 * Define how many HW WL samples to take for majority voting.
 * MUST BE odd!!
 * Assume there should only be 2 possible values that will show up,
 * so treat ties as a problem!!!
 * NOTE: Do not change this without checking the code!!!
 */
#define WLEVEL_LOOPS_DEFAULT            5

#define ENABLE_COMPUTED_VREF_ADJUSTMENT 1
#define SW_WLEVEL_HW_DEFAULT            1
#define DEFAULT_BEST_RANK_SCORE         9999999
#define MAX_RANK_SCORE_LIMIT            99

/*
 * Define how many HW RL samples per rank to take multiple samples will
 * allow looking for the best sample score
 */
#define RLEVEL_SAMPLES_DEFAULT          3

#define ddr_seq_print(format, ...) do {} while (0)

struct wlevel_bitcnt {
        int bitcnt[4];
};

static void display_dac_dbi_settings(int lmc, int dac_or_dbi,
                                     int ecc_ena, int *settings, char *title);

static unsigned short load_dac_override(struct ddr_priv *priv, int if_num,
                                        int dac_value, int byte);

/* "mode" arg */
#define DBTRAIN_TEST 0
#define DBTRAIN_DBI  1
#define DBTRAIN_LFSR 2

static int run_best_hw_patterns(struct ddr_priv *priv, int lmc, u64 phys_addr,
                                int mode, u64 *xor_data);

#define LMC_DDR3_RESET_ASSERT   0
#define LMC_DDR3_RESET_DEASSERT 1

static void cn7xxx_lmc_ddr3_reset(struct ddr_priv *priv, int if_num, int reset)
{
        union cvmx_lmcx_reset_ctl reset_ctl;

        /*
         * 4. Deassert DDRn_RESET_L pin by writing
         *    LMC(0..3)_RESET_CTL[DDR3RST] = 1
         *    without modifying any other LMC(0..3)_RESET_CTL fields.
         * 5. Read LMC(0..3)_RESET_CTL and wait for the result.
         * 6. Wait a minimum of 500us. This guarantees the necessary T = 500us
         *    delay between DDRn_RESET_L deassertion and DDRn_DIMM*_CKE*
         *    assertion.
         */
        debug("LMC%d %s DDR_RESET_L\n", if_num,
              (reset ==
               LMC_DDR3_RESET_DEASSERT) ? "De-asserting" : "Asserting");

        reset_ctl.u64 = lmc_rd(priv, CVMX_LMCX_RESET_CTL(if_num));
        reset_ctl.cn78xx.ddr3rst = reset;
        lmc_wr(priv, CVMX_LMCX_RESET_CTL(if_num), reset_ctl.u64);

        lmc_rd(priv, CVMX_LMCX_RESET_CTL(if_num));

        udelay(500);
}

static void perform_lmc_reset(struct ddr_priv *priv, int node, int if_num)
{
        /*
         * 5.9.6 LMC RESET Initialization
         *
         * The purpose of this step is to assert/deassert the RESET# pin at the
         * DDR3/DDR4 parts.
         *
         * This LMC RESET step is done for all enabled LMCs.
         *
         * It may be appropriate to skip this step if the DDR3/DDR4 DRAM parts
         * are in self refresh and are currently preserving their
         * contents. (Software can determine this via
         * LMC(0..3)_RESET_CTL[DDR3PSV] in some circumstances.) The remainder of
         * this section assumes that the DRAM contents need not be preserved.
         *
         * The remainder of this section assumes that the CN78XX DDRn_RESET_L
         * pin is attached to the RESET# pin of the attached DDR3/DDR4 parts,
         * as will be appropriate in many systems.
         *
         * (In other systems, such as ones that can preserve DDR3/DDR4 part
         * contents while CN78XX is powered down, it will not be appropriate to
         * directly attach the CN78XX DDRn_RESET_L pin to DRESET# of the
         * DDR3/DDR4 parts, and this section may not apply.)
         *
         * The remainder of this section describes the sequence for LMCn.
         *
         * Perform the following six substeps for LMC reset initialization:
         *
         * 1. If not done already, assert DDRn_RESET_L pin by writing
         * LMC(0..3)_RESET_ CTL[DDR3RST] = 0 without modifying any other
         * LMC(0..3)_RESET_CTL fields.
         */

        if (!ddr_memory_preserved(priv)) {
                /*
                 * 2. Read LMC(0..3)_RESET_CTL and wait for the result.
                 */

                lmc_rd(priv, CVMX_LMCX_RESET_CTL(if_num));

                /*
                 * 3. Wait until RESET# assertion-time requirement from JEDEC
                 * DDR3/DDR4 specification is satisfied (200 us during a
                 * power-on ramp, 100ns when power is already stable).
                 */

                udelay(200);

                /*
                 * 4. Deassert DDRn_RESET_L pin by writing
                 *    LMC(0..3)_RESET_CTL[DDR3RST] = 1
                 *    without modifying any other LMC(0..3)_RESET_CTL fields.
                 * 5. Read LMC(0..3)_RESET_CTL and wait for the result.
                 * 6. Wait a minimum of 500us. This guarantees the necessary
                 *    T = 500us delay between DDRn_RESET_L deassertion and
                 *    DDRn_DIMM*_CKE* assertion.
                 */
                cn7xxx_lmc_ddr3_reset(priv, if_num, LMC_DDR3_RESET_DEASSERT);

                /* Toggle Reset Again */
                /* That is, assert, then de-assert, one more time */
                cn7xxx_lmc_ddr3_reset(priv, if_num, LMC_DDR3_RESET_ASSERT);
                cn7xxx_lmc_ddr3_reset(priv, if_num, LMC_DDR3_RESET_DEASSERT);
        }
}

void oct3_ddr3_seq(struct ddr_priv *priv, int rank_mask, int if_num,
                   int sequence)
{
        /*
         * 3. Without changing any other fields in LMC(0)_CONFIG, write
         *    LMC(0)_CONFIG[RANKMASK] then write both
         *    LMC(0)_SEQ_CTL[SEQ_SEL,INIT_START] = 1 with a single CSR write
         *    operation. LMC(0)_CONFIG[RANKMASK] bits should be set to indicate
         *    the ranks that will participate in the sequence.
         *
         *    The LMC(0)_SEQ_CTL[SEQ_SEL] value should select power-up/init or
         *    selfrefresh exit, depending on whether the DRAM parts are in
         *    self-refresh and whether their contents should be preserved. While
         *    LMC performs these sequences, it will not perform any other DDR3
         *    transactions. When the sequence is complete, hardware sets the
         *    LMC(0)_CONFIG[INIT_STATUS] bits for the ranks that have been
         *    initialized.
         *
         *    If power-up/init is selected immediately following a DRESET
         *    assertion, LMC executes the sequence described in the "Reset and
         *    Initialization Procedure" section of the JEDEC DDR3
         *    specification. This includes activating CKE, writing all four DDR3
         *    mode registers on all selected ranks, and issuing the required
         *    ZQCL
         *    command. The LMC(0)_CONFIG[RANKMASK] value should select all ranks
         *    with attached DRAM in this case. If LMC(0)_CONTROL[RDIMM_ENA] = 1,
         *    LMC writes the JEDEC standard SSTE32882 control words selected by
         *    LMC(0)_DIMM_CTL[DIMM*_WMASK] between DDR_CKE* signal assertion and
         *    the first DDR3 mode register write operation.
         *    LMC(0)_DIMM_CTL[DIMM*_WMASK] should be cleared to 0 if the
         *    corresponding DIMM is not present.
         *
         *    If self-refresh exit is selected, LMC executes the required SRX
         *    command followed by a refresh and ZQ calibration. Section 4.5
         *    describes behavior of a REF + ZQCS.  LMC does not write the DDR3
         *    mode registers as part of this sequence, and the mode register
         *    parameters must match at self-refresh entry and exit times.
         *
         * 4. Read LMC(0)_SEQ_CTL and wait for LMC(0)_SEQ_CTL[SEQ_COMPLETE]
         *    to be set.
         *
         * 5. Read LMC(0)_CONFIG[INIT_STATUS] and confirm that all ranks have
         *    been initialized.
         */

        union cvmx_lmcx_seq_ctl seq_ctl;
        union cvmx_lmcx_config lmc_config;
        int timeout;

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        lmc_config.s.rankmask = rank_mask;
        lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), lmc_config.u64);

        seq_ctl.u64 = 0;

        seq_ctl.s.init_start = 1;
        seq_ctl.s.seq_sel = sequence;

        ddr_seq_print
            ("Performing LMC sequence: rank_mask=0x%02x, sequence=0x%x, %s\n",
             rank_mask, sequence, sequence_str[sequence]);

        if (seq_ctl.s.seq_sel == 3)
                debug("LMC%d: Exiting Self-refresh Rank_mask:%x\n", if_num,
                      rank_mask);

        lmc_wr(priv, CVMX_LMCX_SEQ_CTL(if_num), seq_ctl.u64);
        lmc_rd(priv, CVMX_LMCX_SEQ_CTL(if_num));

        timeout = 100;
        do {
                udelay(100);    /* Wait a while */
                seq_ctl.u64 = lmc_rd(priv, CVMX_LMCX_SEQ_CTL(if_num));
                if (--timeout == 0) {
                        printf("Sequence %d timed out\n", sequence);
                        break;
                }
        } while (seq_ctl.s.seq_complete != 1);

        ddr_seq_print("           LMC sequence=%x: Completed.\n", sequence);
}

#define bdk_numa_get_address(n, p)      ((p) | ((u64)n) << CVMX_NODE_MEM_SHIFT)
#define AREA_BASE_OFFSET                BIT_ULL(26)

static int test_dram_byte64(struct ddr_priv *priv, int lmc, u64 p,
                            u64 bitmask, u64 *xor_data)
{
        u64 p1, p2, d1, d2;
        u64 v, v1;
        u64 p2offset = (1ULL << 26);    // offset to area 2
        u64 datamask;
        u64 xor;
        u64 i, j, k;
        u64 ii;
        int errors = 0;
        //u64 index;
        u64 pattern1 = cvmx_rng_get_random64();
        u64 pattern2 = 0;
        u64 bad_bits[2] = { 0, 0 };
        int kbitno = (octeon_is_cpuid(OCTEON_CN7XXX)) ? 20 : 18;
        union cvmx_l2c_ctl l2c_ctl;
        int burst;
        int saved_dissblkdty;
        int node = 0;

        // Force full cacheline write-backs to boost traffic
        l2c_ctl.u64 = l2c_rd(priv, CVMX_L2C_CTL_REL);
        saved_dissblkdty = l2c_ctl.cn78xx.dissblkdty;
        l2c_ctl.cn78xx.dissblkdty = 1;
        l2c_wr(priv, CVMX_L2C_CTL_REL, l2c_ctl.u64);

        if (octeon_is_cpuid(OCTEON_CN73XX) || octeon_is_cpuid(OCTEON_CNF75XX))
                kbitno = 18;

        // Byte lanes may be clear in the mask to indicate no testing on that
        //lane.
        datamask = bitmask;

        /*
         * Add offset to both test regions to not clobber boot stuff
         * when running from L2 for NAND boot.
         */
        p += AREA_BASE_OFFSET;  // make sure base is out of the way of boot

        // final address must include LMC and node
        p |= (lmc << 7);        /* Map address into proper interface */
        p = bdk_numa_get_address(node, p);      /* Map to node */
        p |= 1ull << 63;

#define II_INC BIT_ULL(22)
#define II_MAX BIT_ULL(22)
#define K_INC  BIT_ULL(14)
#define K_MAX  BIT_ULL(kbitno)
#define J_INC  BIT_ULL(9)
#define J_MAX  BIT_ULL(12)
#define I_INC  BIT_ULL(3)
#define I_MAX  BIT_ULL(7)

        debug("N%d.LMC%d: %s: phys_addr=0x%llx/0x%llx (0x%llx)\n",
              node, lmc, __func__, p, p + p2offset, 1ULL << kbitno);

        // loops are ordered so that only a single 64-bit slot is written to
        // each cacheline at one time, then the cachelines are forced out;
        // this should maximize read/write traffic

        // FIXME? extend the range of memory tested!!
        for (ii = 0; ii < II_MAX; ii += II_INC) {
                for (i = 0; i < I_MAX; i += I_INC) {
                        for (k = 0; k < K_MAX; k += K_INC) {
                                for (j = 0; j < J_MAX; j += J_INC) {
                                        p1 = p + ii + k + j;
                                        p2 = p1 + p2offset;

                                        v = pattern1 * (p1 + i);
                                        // write the same thing to both areas
                                        v1 = v;

                                        cvmx_write64_uint64(p1 + i, v);
                                        cvmx_write64_uint64(p2 + i, v1);

                                        CVMX_CACHE_WBIL2(p1, 0);
                                        CVMX_CACHE_WBIL2(p2, 0);
                                }
                        }
                }
        }

        CVMX_DCACHE_INVALIDATE;

        debug("N%d.LMC%d: dram_tuning_mem_xor: done INIT loop\n", node, lmc);

        /* Make a series of passes over the memory areas. */

        for (burst = 0; burst < 1 /* was: dram_tune_use_bursts */ ; burst++) {
                u64 this_pattern = cvmx_rng_get_random64();

                pattern2 ^= this_pattern;

                /*
                 * XOR the data with a random value, applying the change to both
                 * memory areas.
                 */

                // FIXME? extend the range of memory tested!!
                for (ii = 0; ii < II_MAX; ii += II_INC) {
                        // FIXME: rearranged, did not make much difference?
                        for (i = 0; i < I_MAX; i += I_INC) {
                                for (k = 0; k < K_MAX; k += K_INC) {
                                        for (j = 0; j < J_MAX; j += J_INC) {
                                                p1 = p + ii + k + j;
                                                p2 = p1 + p2offset;

                                                v = cvmx_read64_uint64(p1 +
                                                                      i) ^
                                                    this_pattern;
                                                v1 = cvmx_read64_uint64(p2 +
                                                                       i) ^
                                                    this_pattern;

                                                cvmx_write64_uint64(p1 + i, v);
                                                cvmx_write64_uint64(p2 + i, v1);

                                                CVMX_CACHE_WBIL2(p1, 0);
                                                CVMX_CACHE_WBIL2(p2, 0);
                                        }
                                }
                        }
                }

                CVMX_DCACHE_INVALIDATE;

                debug("N%d.LMC%d: dram_tuning_mem_xor: done MODIFY loop\n",
                      node, lmc);

                /*
                 * Look for differences in the areas. If there is a mismatch,
                 * reset both memory locations with the same pattern. Failing
                 * to do so means that on all subsequent passes the pair of
                 * locations remain out of sync giving spurious errors.
                 */

                // FIXME: Change the loop order so that an entire cache line
                //        is compared at one time. This is so that a read
                //        error that occurs *anywhere* on the cacheline will
                //        be caught, rather than comparing only 1 cacheline
                //        slot at a time, where an error on a different
                //        slot will be missed that time around
                // Does the above make sense?

                // FIXME? extend the range of memory tested!!
                for (ii = 0; ii < II_MAX; ii += II_INC) {
                        for (k = 0; k < K_MAX; k += K_INC) {
                                for (j = 0; j < J_MAX; j += J_INC) {
                                        p1 = p + ii + k + j;
                                        p2 = p1 + p2offset;

                                        // process entire cachelines in the
                                        //innermost loop
                                        for (i = 0; i < I_MAX; i += I_INC) {
                                                int bybit = 1;
                                                // start in byte lane 0
                                                u64 bymsk = 0xffULL;

                                                // FIXME: this should predict
                                                // what we find...???
                                                v = ((p1 + i) * pattern1) ^
                                                        pattern2;
                                                d1 = cvmx_read64_uint64(p1 + i);
                                                d2 = cvmx_read64_uint64(p2 + i);

                                                // union of error bits only in
                                                // active byte lanes
                                                xor = ((d1 ^ v) | (d2 ^ v)) &
                                                        datamask;

                                                if (!xor)
                                                        continue;

                                                // accumulate bad bits
                                                bad_bits[0] |= xor;

                                                while (xor != 0) {
                                                        debug("ERROR(%03d): [0x%016llX] [0x%016llX]  expected 0x%016llX d1 %016llX d2 %016llX\n",
                                                              burst, p1, p2, v,
                                                              d1, d2);
                                                        // error(s) in this lane
                                                        if (xor & bymsk) {
                                                                // set the byte
                                                                // error bit
                                                                errors |= bybit;
                                                                // clear byte
                                                                // lane in
                                                                // error bits
                                                                xor &= ~bymsk;
                                                                // clear the
                                                                // byte lane in
                                                                // the mask
                                                                datamask &= ~bymsk;
#if EXIT_WHEN_ALL_LANES_HAVE_ERRORS
                                                                // nothing
                                                                // left to do
                                                                if (datamask == 0) {
                                                                        return errors;
                                                                }
#endif /* EXIT_WHEN_ALL_LANES_HAVE_ERRORS */
                                                        }
                                                        // move mask into
                                                        // next byte lane
                                                        bymsk <<= 8;
                                                        // move bit into next
                                                        // byte position
                                                        bybit <<= 1;
                                                }
                                        }
                                        CVMX_CACHE_WBIL2(p1, 0);
                                        CVMX_CACHE_WBIL2(p2, 0);
                                }
                        }
                }

                debug("N%d.LMC%d: dram_tuning_mem_xor: done TEST loop\n",
                      node, lmc);
        }

        if (xor_data) {         // send the bad bits back...
                xor_data[0] = bad_bits[0];
                xor_data[1] = bad_bits[1];      // let it be zeroed
        }

        // Restore original setting that could enable partial cacheline writes
        l2c_ctl.u64 = l2c_rd(priv, CVMX_L2C_CTL_REL);
        l2c_ctl.cn78xx.dissblkdty = saved_dissblkdty;
        l2c_wr(priv, CVMX_L2C_CTL_REL, l2c_ctl.u64);

        return errors;
}

static void ddr4_mrw(struct ddr_priv *priv, int if_num, int rank,
                     int mr_wr_addr, int mr_wr_sel, int mr_wr_bg1)
{
        union cvmx_lmcx_mr_mpr_ctl lmc_mr_mpr_ctl;

        lmc_mr_mpr_ctl.u64 = 0;
        lmc_mr_mpr_ctl.cn78xx.mr_wr_addr = (mr_wr_addr == -1) ? 0 : mr_wr_addr;
        lmc_mr_mpr_ctl.cn78xx.mr_wr_sel = mr_wr_sel;
        lmc_mr_mpr_ctl.cn78xx.mr_wr_rank = rank;
        lmc_mr_mpr_ctl.cn78xx.mr_wr_use_default_value =
                (mr_wr_addr == -1) ? 1 : 0;
        lmc_mr_mpr_ctl.cn78xx.mr_wr_bg1 = mr_wr_bg1;
        lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64);

        /* Mode Register Write */
        oct3_ddr3_seq(priv, 1 << rank, if_num, 0x8);
}

#define INV_A0_17(x)    ((x) ^ 0x22bf8)

static void set_mpr_mode(struct ddr_priv *priv, int rank_mask,
                         int if_num, int dimm_count, int mpr, int bg1)
{
        int rankx;

        debug("All Ranks: Set mpr mode = %x %c-side\n",
              mpr, (bg1 == 0) ? 'A' : 'B');

        for (rankx = 0; rankx < dimm_count * 4; rankx++) {
                if (!(rank_mask & (1 << rankx)))
                        continue;
                if (bg1 == 0) {
                        /* MR3 A-side */
                        ddr4_mrw(priv, if_num, rankx, mpr << 2, 3, bg1);
                } else {
                        /* MR3 B-side */
                        ddr4_mrw(priv, if_num, rankx, INV_A0_17(mpr << 2), ~3,
                                 bg1);
                }
        }
}

static void do_ddr4_mpr_read(struct ddr_priv *priv, int if_num,
                             int rank, int page, int location)
{
        union cvmx_lmcx_mr_mpr_ctl lmc_mr_mpr_ctl;

        lmc_mr_mpr_ctl.u64 = lmc_rd(priv, CVMX_LMCX_MR_MPR_CTL(if_num));
        lmc_mr_mpr_ctl.cn70xx.mr_wr_addr = 0;
        lmc_mr_mpr_ctl.cn70xx.mr_wr_sel = page; /* Page */
        lmc_mr_mpr_ctl.cn70xx.mr_wr_rank = rank;
        lmc_mr_mpr_ctl.cn70xx.mpr_loc = location;
        lmc_mr_mpr_ctl.cn70xx.mpr_wr = 0;       /* Read=0, Write=1 */
        lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64);

        /* MPR register access sequence */
        oct3_ddr3_seq(priv, 1 << rank, if_num, 0x9);

        debug("LMC_MR_MPR_CTL                  : 0x%016llx\n",
              lmc_mr_mpr_ctl.u64);
        debug("lmc_mr_mpr_ctl.cn70xx.mr_wr_addr: 0x%02x\n",
              lmc_mr_mpr_ctl.cn70xx.mr_wr_addr);
        debug("lmc_mr_mpr_ctl.cn70xx.mr_wr_sel : 0x%02x\n",
              lmc_mr_mpr_ctl.cn70xx.mr_wr_sel);
        debug("lmc_mr_mpr_ctl.cn70xx.mpr_loc   : 0x%02x\n",
              lmc_mr_mpr_ctl.cn70xx.mpr_loc);
        debug("lmc_mr_mpr_ctl.cn70xx.mpr_wr    : 0x%02x\n",
              lmc_mr_mpr_ctl.cn70xx.mpr_wr);
}

static int set_rdimm_mode(struct ddr_priv *priv, int if_num, int enable)
{
        union cvmx_lmcx_control lmc_control;
        int save_rdimm_mode;

        lmc_control.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num));
        save_rdimm_mode = lmc_control.s.rdimm_ena;
        lmc_control.s.rdimm_ena = enable;
        debug("Setting RDIMM_ENA = %x\n", enable);
        lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), lmc_control.u64);

        return save_rdimm_mode;
}

static void ddr4_mpr_read(struct ddr_priv *priv, int if_num, int rank,
                          int page, int location, u64 *mpr_data)
{
        do_ddr4_mpr_read(priv, if_num, rank, page, location);

        mpr_data[0] = lmc_rd(priv, CVMX_LMCX_MPR_DATA0(if_num));
}

/* Display MPR values for Page */
static void display_mpr_page(struct ddr_priv *priv, int rank_mask,
                             int if_num, int page)
{
        int rankx, location;
        u64 mpr_data[3];

        for (rankx = 0; rankx < 4; rankx++) {
                if (!(rank_mask & (1 << rankx)))
                        continue;

                debug("N0.LMC%d.R%d: MPR Page %d loc [0:3]: ",
                      if_num, rankx, page);
                for (location = 0; location < 4; location++) {
                        ddr4_mpr_read(priv, if_num, rankx, page, location,
                                      mpr_data);
                        debug("0x%02llx ", mpr_data[0] & 0xFF);
                }
                debug("\n");

        }                       /* for (rankx = 0; rankx < 4; rankx++) */
}

static void ddr4_mpr_write(struct ddr_priv *priv, int if_num, int rank,
                           int page, int location, u8 mpr_data)
{
        union cvmx_lmcx_mr_mpr_ctl lmc_mr_mpr_ctl;

        lmc_mr_mpr_ctl.u64 = 0;
        lmc_mr_mpr_ctl.cn70xx.mr_wr_addr = mpr_data;
        lmc_mr_mpr_ctl.cn70xx.mr_wr_sel = page; /* Page */
        lmc_mr_mpr_ctl.cn70xx.mr_wr_rank = rank;
        lmc_mr_mpr_ctl.cn70xx.mpr_loc = location;
        lmc_mr_mpr_ctl.cn70xx.mpr_wr = 1;       /* Read=0, Write=1 */
        lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64);

        /* MPR register access sequence */
        oct3_ddr3_seq(priv, 1 << rank, if_num, 0x9);

        debug("LMC_MR_MPR_CTL                  : 0x%016llx\n",
              lmc_mr_mpr_ctl.u64);
        debug("lmc_mr_mpr_ctl.cn70xx.mr_wr_addr: 0x%02x\n",
              lmc_mr_mpr_ctl.cn70xx.mr_wr_addr);
        debug("lmc_mr_mpr_ctl.cn70xx.mr_wr_sel : 0x%02x\n",
              lmc_mr_mpr_ctl.cn70xx.mr_wr_sel);
        debug("lmc_mr_mpr_ctl.cn70xx.mpr_loc   : 0x%02x\n",
              lmc_mr_mpr_ctl.cn70xx.mpr_loc);
        debug("lmc_mr_mpr_ctl.cn70xx.mpr_wr    : 0x%02x\n",
              lmc_mr_mpr_ctl.cn70xx.mpr_wr);
}

static void set_vref(struct ddr_priv *priv, int if_num, int rank,
                     int range, int value)
{
        union cvmx_lmcx_mr_mpr_ctl lmc_mr_mpr_ctl;
        union cvmx_lmcx_modereg_params3 lmc_modereg_params3;
        int mr_wr_addr = 0;

        lmc_mr_mpr_ctl.u64 = 0;
        lmc_modereg_params3.u64 = lmc_rd(priv,
                                         CVMX_LMCX_MODEREG_PARAMS3(if_num));

        /* A12:A10 tCCD_L */
        mr_wr_addr |= lmc_modereg_params3.s.tccd_l << 10;
        mr_wr_addr |= 1 << 7;   /* A7 1 = Enable(Training Mode) */
        mr_wr_addr |= range << 6;       /* A6 vrefDQ Training Range */
        mr_wr_addr |= value << 0;       /* A5:A0 vrefDQ Training Value */

        lmc_mr_mpr_ctl.cn70xx.mr_wr_addr = mr_wr_addr;
        lmc_mr_mpr_ctl.cn70xx.mr_wr_sel = 6;    /* Write MR6 */
        lmc_mr_mpr_ctl.cn70xx.mr_wr_rank = rank;
        lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64);

        /* 0x8 = Mode Register Write */
        oct3_ddr3_seq(priv, 1 << rank, if_num, 0x8);

        /*
         * It is vendor specific whether vref_value is captured with A7=1.
         * A subsequent MRS might be necessary.
         */
        oct3_ddr3_seq(priv, 1 << rank, if_num, 0x8);

        mr_wr_addr &= ~(1 << 7);        /* A7 0 = Disable(Training Mode) */
        lmc_mr_mpr_ctl.cn70xx.mr_wr_addr = mr_wr_addr;
        lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64);
}

static void set_dram_output_inversion(struct ddr_priv *priv, int if_num,
                                      int dimm_count, int rank_mask,
                                      int inversion)
{
        union cvmx_lmcx_ddr4_dimm_ctl lmc_ddr4_dimm_ctl;
        union cvmx_lmcx_dimmx_params lmc_dimmx_params;
        union cvmx_lmcx_dimm_ctl lmc_dimm_ctl;
        int dimm_no;

        /* Don't touch extenced register control words */
        lmc_ddr4_dimm_ctl.u64 = 0;
        lmc_wr(priv, CVMX_LMCX_DDR4_DIMM_CTL(if_num), lmc_ddr4_dimm_ctl.u64);

        debug("All DIMMs: Register Control Word          RC0 : %x\n",
              (inversion & 1));

        for (dimm_no = 0; dimm_no < dimm_count; ++dimm_no) {
                lmc_dimmx_params.u64 =
                    lmc_rd(priv, CVMX_LMCX_DIMMX_PARAMS(dimm_no, if_num));
                lmc_dimmx_params.s.rc0 =
                    (lmc_dimmx_params.s.rc0 & ~1) | (inversion & 1);

                lmc_wr(priv,
                       CVMX_LMCX_DIMMX_PARAMS(dimm_no, if_num),
                       lmc_dimmx_params.u64);
        }

        /* LMC0_DIMM_CTL */
        lmc_dimm_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DIMM_CTL(if_num));
        lmc_dimm_ctl.s.dimm0_wmask = 0x1;
        lmc_dimm_ctl.s.dimm1_wmask = (dimm_count > 1) ? 0x0001 : 0x0000;

        debug("LMC DIMM_CTL                                  : 0x%016llx\n",
              lmc_dimm_ctl.u64);
        lmc_wr(priv, CVMX_LMCX_DIMM_CTL(if_num), lmc_dimm_ctl.u64);

        oct3_ddr3_seq(priv, rank_mask, if_num, 0x7);    /* Init RCW */
}

static void write_mpr_page0_pattern(struct ddr_priv *priv, int rank_mask,
                                    int if_num, int dimm_count, int pattern,
                                    int location_mask)
{
        int rankx;
        int location;

        for (rankx = 0; rankx < dimm_count * 4; rankx++) {
                if (!(rank_mask & (1 << rankx)))
                        continue;
                for (location = 0; location < 4; ++location) {
                        if (!(location_mask & (1 << location)))
                                continue;

                        ddr4_mpr_write(priv, if_num, rankx,
                                       /* page */ 0, /* location */ location,
                                       pattern);
                }
        }
}

static void change_rdimm_mpr_pattern(struct ddr_priv *priv, int rank_mask,
                                     int if_num, int dimm_count)
{
        int save_ref_zqcs_int;
        union cvmx_lmcx_config lmc_config;

        /*
         * Okay, here is the latest sequence.  This should work for all
         * chips and passes (78,88,73,etc).  This sequence should be run
         * immediately after DRAM INIT.  The basic idea is to write the
         * same pattern into each of the 4 MPR locations in the DRAM, so
         * that the same value is returned when doing MPR reads regardless
         * of the inversion state.  My advice is to put this into a
         * function, change_rdimm_mpr_pattern or something like that, so
         * that it can be called multiple times, as I think David wants a
         * clock-like pattern for OFFSET training, but does not want a
         * clock pattern for Bit-Deskew.  You should then be able to call
         * this at any point in the init sequence (after DRAM init) to
         * change the pattern to a new value.
         * Mike
         *
         * A correction: PHY doesn't need any pattern during offset
         * training, but needs clock like pattern for internal vref and
         * bit-dskew training.  So for that reason, these steps below have
         * to be conducted before those trainings to pre-condition
         * the pattern.  David
         *
         * Note: Step 3, 4, 8 and 9 have to be done through RDIMM
         * sequence. If you issue MRW sequence to do RCW write (in o78 pass
         * 1 at least), LMC will still do two commands because
         * CONTROL[RDIMM_ENA] is still set high. We don't want it to have
         * any unintentional mode register write so it's best to do what
         * Mike is doing here.
         * Andrew
         */

        /* 1) Disable refresh (REF_ZQCS_INT = 0) */

        debug("1) Disable refresh (REF_ZQCS_INT = 0)\n");

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        save_ref_zqcs_int = lmc_config.cn78xx.ref_zqcs_int;
        lmc_config.cn78xx.ref_zqcs_int = 0;
        lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), lmc_config.u64);

        /*
         * 2) Put all devices in MPR mode (Run MRW sequence (sequence=8)
         * with MODEREG_PARAMS0[MPRLOC]=0,
         * MODEREG_PARAMS0[MPR]=1, MR_MPR_CTL[MR_WR_SEL]=3, and
         * MR_MPR_CTL[MR_WR_USE_DEFAULT_VALUE]=1)
         */

        debug("2) Put all devices in MPR mode (Run MRW sequence (sequence=8)\n");

        /* A-side */
        set_mpr_mode(priv, rank_mask, if_num, dimm_count, 1, 0);
        /* B-side */
        set_mpr_mode(priv, rank_mask, if_num, dimm_count, 1, 1);

        /*
         * a. Or you can set MR_MPR_CTL[MR_WR_USE_DEFAULT_VALUE]=0 and set
         * the value you would like directly into
         * MR_MPR_CTL[MR_WR_ADDR]
         */

        /*
         * 3) Disable RCD Parity (if previously enabled) - parity does not
         * work if inversion disabled
         */

        debug("3) Disable RCD Parity\n");

        /*
         * 4) Disable Inversion in the RCD.
         * a. I did (3&4) via the RDIMM sequence (seq_sel=7), but it
         * may be easier to use the MRW sequence (seq_sel=8).  Just set
         * MR_MPR_CTL[MR_WR_SEL]=7, MR_MPR_CTL[MR_WR_ADDR][3:0]=data,
         * MR_MPR_CTL[MR_WR_ADDR][7:4]=RCD reg
         */

        debug("4) Disable Inversion in the RCD.\n");

        set_dram_output_inversion(priv, if_num, dimm_count, rank_mask, 1);

        /*
         * 5) Disable CONTROL[RDIMM_ENA] so that MR sequence goes out
         * non-inverted.
         */

        debug("5) Disable CONTROL[RDIMM_ENA]\n");

        set_rdimm_mode(priv, if_num, 0);

        /*
         * 6) Write all 4 MPR registers with the desired pattern (have to
         * do this for all enabled ranks)
         * a. MR_MPR_CTL.MPR_WR=1, MR_MPR_CTL.MPR_LOC=0..3,
         * MR_MPR_CTL.MR_WR_SEL=0, MR_MPR_CTL.MR_WR_ADDR[7:0]=pattern
         */

        debug("6) Write all 4 MPR page 0 Training Patterns\n");

        write_mpr_page0_pattern(priv, rank_mask, if_num, dimm_count, 0x55, 0x8);

        /* 7) Re-enable RDIMM_ENA */

        debug("7) Re-enable RDIMM_ENA\n");

        set_rdimm_mode(priv, if_num, 1);

        /* 8) Re-enable RDIMM inversion */

        debug("8) Re-enable RDIMM inversion\n");

        set_dram_output_inversion(priv, if_num, dimm_count, rank_mask, 0);

        /* 9) Re-enable RDIMM parity (if desired) */

        debug("9) Re-enable RDIMM parity (if desired)\n");

        /*
         * 10)Take B-side devices out of MPR mode (Run MRW sequence
         * (sequence=8) with MODEREG_PARAMS0[MPRLOC]=0,
         * MODEREG_PARAMS0[MPR]=0, MR_MPR_CTL[MR_WR_SEL]=3, and
         * MR_MPR_CTL[MR_WR_USE_DEFAULT_VALUE]=1)
         */

        debug("10)Take B-side devices out of MPR mode\n");

        set_mpr_mode(priv, rank_mask, if_num, dimm_count,
                     /* mpr */ 0, /* bg1 */ 1);

        /*
         * a. Or you can set MR_MPR_CTL[MR_WR_USE_DEFAULT_VALUE]=0 and
         * set the value you would like directly into MR_MPR_CTL[MR_WR_ADDR]
         */

        /* 11)Re-enable refresh (REF_ZQCS_INT=previous value) */

        debug("11)Re-enable refresh (REF_ZQCS_INT=previous value)\n");

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        lmc_config.cn78xx.ref_zqcs_int = save_ref_zqcs_int;
        lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), lmc_config.u64);
}

static int validate_hwl_seq(int *wl, int *seq)
{
        // sequence index, step through the sequence array
        int seqx;
        int bitnum;

        seqx = 0;

        while (seq[seqx + 1] >= 0) {    // stop on next seq entry == -1
                // but now, check current versus next
                bitnum = (wl[seq[seqx]] << 2) | wl[seq[seqx + 1]];
                // magic validity number (see matrix above)
                if (!((1 << bitnum) & 0xBDE7))
                        return 1;
                seqx++;
        }

        return 0;
}

static int validate_hw_wl_settings(int if_num,
                                   union cvmx_lmcx_wlevel_rankx
                                   *lmc_wlevel_rank, int is_rdimm, int ecc_ena)
{
        int wl[9], byte, errors;

        // arrange the sequences so
        // index 0 has byte 0, etc, ECC in middle
        int useq[] = { 0, 1, 2, 3, 8, 4, 5, 6, 7, -1 };
        // index 0 is ECC, then go down
        int rseq1[] = { 8, 3, 2, 1, 0, -1 };
        // index 0 has byte 4, then go up
        int rseq2[] = { 4, 5, 6, 7, -1 };
        // index 0 has byte 0, etc, no ECC
        int useqno[] = { 0, 1, 2, 3, 4, 5, 6, 7, -1 };
        // index 0 is byte 3, then go down, no ECC
        int rseq1no[] = { 3, 2, 1, 0, -1 };

        // in the CSR, bytes 0-7 are always data, byte 8 is ECC
        for (byte = 0; byte < (8 + ecc_ena); byte++) {
                // preprocess :-)
                wl[byte] = (get_wl_rank(lmc_wlevel_rank, byte) >>
                            1) & 3;
        }

        errors = 0;
        if (is_rdimm) {         // RDIMM order
                errors = validate_hwl_seq(wl, (ecc_ena) ? rseq1 : rseq1no);
                errors += validate_hwl_seq(wl, rseq2);
        } else {                // UDIMM order
                errors = validate_hwl_seq(wl, (ecc_ena) ? useq : useqno);
        }

        return errors;
}

static unsigned int extr_wr(u64 u, int x)
{
        return (unsigned int)(((u >> (x * 12 + 5)) & 0x3ULL) |
                              ((u >> (51 + x - 2)) & 0x4ULL));
}

static void insrt_wr(u64 *up, int x, int v)
{
        u64 u = *up;

        u &= ~(((0x3ULL) << (x * 12 + 5)) | ((0x1ULL) << (51 + x)));
        *up = (u | ((v & 0x3ULL) << (x * 12 + 5)) |
               ((v & 0x4ULL) << (51 + x - 2)));
}

/* Read out Deskew Settings for DDR */

struct deskew_bytes {
        u16 bits[8];
};

struct deskew_data {
        struct deskew_bytes bytes[9];
};

struct dac_data {
        int bytes[9];
};

// T88 pass 1, skip 4=DAC
static const u8 dsk_bit_seq_p1[8] = { 0, 1, 2, 3, 5, 6, 7, 8 };
// T88 Pass 2, skip 4=DAC and 5=DBI
static const u8 dsk_bit_seq_p2[8] = { 0, 1, 2, 3, 6, 7, 8, 9 };

static void get_deskew_settings(struct ddr_priv *priv, int if_num,
                                struct deskew_data *dskdat)
{
        union cvmx_lmcx_phy_ctl phy_ctl;
        union cvmx_lmcx_config lmc_config;
        int bit_index;
        int byte_lane, byte_limit;
        // NOTE: these are for pass 2.x
        int is_o78p2 = !octeon_is_cpuid(OCTEON_CN78XX_PASS1_X);
        const u8 *bit_seq = (is_o78p2) ? dsk_bit_seq_p2 : dsk_bit_seq_p1;

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        byte_limit = ((!lmc_config.s.mode32b) ? 8 : 4) + lmc_config.s.ecc_ena;

        memset(dskdat, 0, sizeof(*dskdat));

        phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
        phy_ctl.s.dsk_dbg_clk_scaler = 3;

        for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) {
                phy_ctl.s.dsk_dbg_byte_sel = byte_lane; // set byte lane

                for (bit_index = 0; bit_index < 8; ++bit_index) {
                        // set bit number and start read sequence
                        phy_ctl.s.dsk_dbg_bit_sel = bit_seq[bit_index];
                        phy_ctl.s.dsk_dbg_rd_start = 1;
                        lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);

                        // poll for read sequence to complete
                        do {
                                phy_ctl.u64 =
                                        lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
                        } while (phy_ctl.s.dsk_dbg_rd_complete != 1);

                        // record the data
                        dskdat->bytes[byte_lane].bits[bit_index] =
                                phy_ctl.s.dsk_dbg_rd_data & 0x3ff;
                }
        }
}

static void display_deskew_settings(struct ddr_priv *priv, int if_num,
                                    struct deskew_data *dskdat,
                                    int print_enable)
{
        int byte_lane;
        int bit_num;
        u16 flags, deskew;
        union cvmx_lmcx_config lmc_config;
        int byte_limit;
        const char *fc = " ?-=+*#&";

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        byte_limit = ((lmc_config.s.mode32b) ? 4 : 8) + lmc_config.s.ecc_ena;

        if (print_enable) {
                debug("N0.LMC%d: Deskew Data:              Bit =>      :",
                      if_num);
                for (bit_num = 7; bit_num >= 0; --bit_num)
                        debug(" %3d  ", bit_num);
                debug("\n");
        }

        for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) {
                if (print_enable)
                        debug("N0.LMC%d: Bit Deskew Byte %d %s               :",
                              if_num, byte_lane,
                              (print_enable >= 3) ? "FINAL" : "     ");

                for (bit_num = 7; bit_num >= 0; --bit_num) {
                        flags = dskdat->bytes[byte_lane].bits[bit_num] & 7;
                        deskew = dskdat->bytes[byte_lane].bits[bit_num] >> 3;

                        if (print_enable)
                                debug(" %3d %c", deskew, fc[flags ^ 1]);

                }               /* for (bit_num = 7; bit_num >= 0; --bit_num) */

                if (print_enable)
                        debug("\n");
        }
}

static void override_deskew_settings(struct ddr_priv *priv, int if_num,
                                     struct deskew_data *dskdat)
{
        union cvmx_lmcx_phy_ctl phy_ctl;
        union cvmx_lmcx_config lmc_config;

        int bit, byte_lane, byte_limit;
        u64 csr_data;

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        byte_limit = ((lmc_config.s.mode32b) ? 4 : 8) + lmc_config.s.ecc_ena;

        phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));

        phy_ctl.s.phy_reset = 0;
        phy_ctl.s.dsk_dbg_num_bits_sel = 1;
        phy_ctl.s.dsk_dbg_offset = 0;
        phy_ctl.s.dsk_dbg_clk_scaler = 3;

        phy_ctl.s.dsk_dbg_wr_mode = 1;
        phy_ctl.s.dsk_dbg_load_dis = 0;
        phy_ctl.s.dsk_dbg_overwrt_ena = 0;

        phy_ctl.s.phy_dsk_reset = 0;

        lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);
        lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));

        for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) {
                csr_data = 0;
                // FIXME: can we ignore DBI?
                for (bit = 0; bit < 8; ++bit) {
                        // fetch input and adjust
                        u64 bits = (dskdat->bytes[byte_lane].bits[bit] >> 3) &
                                0x7F;

                        /*
                         * lmc_general_purpose0.data[6:0]    // DQ0
                         * lmc_general_purpose0.data[13:7]   // DQ1
                         * lmc_general_purpose0.data[20:14]  // DQ2
                         * lmc_general_purpose0.data[27:21]  // DQ3
                         * lmc_general_purpose0.data[34:28]  // DQ4
                         * lmc_general_purpose0.data[41:35]  // DQ5
                         * lmc_general_purpose0.data[48:42]  // DQ6
                         * lmc_general_purpose0.data[55:49]  // DQ7
                         * lmc_general_purpose0.data[62:56]  // DBI
                         */
                        csr_data |= (bits << (7 * bit));

                } /* for (bit = 0; bit < 8; ++bit) */

                // update GP0 with the bit data for this byte lane
                lmc_wr(priv, CVMX_LMCX_GENERAL_PURPOSE0(if_num), csr_data);
                lmc_rd(priv, CVMX_LMCX_GENERAL_PURPOSE0(if_num));

                // start the deskew load sequence
                phy_ctl.s.dsk_dbg_byte_sel = byte_lane;
                phy_ctl.s.dsk_dbg_rd_start = 1;
                lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);

                // poll for read sequence to complete
                do {
                        udelay(100);
                        phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
                } while (phy_ctl.s.dsk_dbg_rd_complete != 1);
        }

        // tell phy to use the new settings
        phy_ctl.s.dsk_dbg_overwrt_ena = 1;
        phy_ctl.s.dsk_dbg_rd_start = 0;
        lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);

        phy_ctl.s.dsk_dbg_wr_mode = 0;
        lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);
}

static void process_by_rank_dac(struct ddr_priv *priv, int if_num,
                                int rank_mask, struct dac_data *dacdat)
{
        union cvmx_lmcx_config lmc_config;
        int rankx, byte_lane;
        int byte_limit;
        int rank_count;
        struct dac_data dacsum;
        int lane_probs;

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        byte_limit = ((lmc_config.s.mode32b) ? 4 : 8) + lmc_config.s.ecc_ena;

        memset((void *)&dacsum, 0, sizeof(dacsum));
        rank_count = 0;
        lane_probs = 0;

        for (rankx = 0; rankx < 4; rankx++) {
                if (!(rank_mask & (1 << rankx)))
                        continue;
                rank_count++;

                display_dac_dbi_settings(if_num, /*dac */ 1,
                                         lmc_config.s.ecc_ena,
                                         &dacdat[rankx].bytes[0],
                                         "By-Ranks VREF");
                // sum
                for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) {
                        if (rank_count == 2) {
                                int ranks_diff =
                                    abs((dacsum.bytes[byte_lane] -
                                         dacdat[rankx].bytes[byte_lane]));

                                // FIXME: is 19 a good number?
                                if (ranks_diff > 19)
                                        lane_probs |= (1 << byte_lane);
                        }
                        dacsum.bytes[byte_lane] +=
                            dacdat[rankx].bytes[byte_lane];
                }
        }

        // average
        for (byte_lane = 0; byte_lane < byte_limit; byte_lane++)
                dacsum.bytes[byte_lane] /= rank_count;  // FIXME: nint?

        display_dac_dbi_settings(if_num, /*dac */ 1, lmc_config.s.ecc_ena,
                                 &dacsum.bytes[0], "All-Rank VREF");

        if (lane_probs) {
                debug("N0.LMC%d: All-Rank VREF DAC Problem Bytelane(s): 0x%03x\n",
                      if_num, lane_probs);
        }

        // finally, write the averaged DAC values
        for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) {
                load_dac_override(priv, if_num, dacsum.bytes[byte_lane],
                                  byte_lane);
        }
}

static void process_by_rank_dsk(struct ddr_priv *priv, int if_num,
                                int rank_mask, struct deskew_data *dskdat)
{
        union cvmx_lmcx_config lmc_config;
        int rankx, lane, bit;
        int byte_limit;
        struct deskew_data dsksum, dskcnt;
        u16 deskew;

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        byte_limit = ((lmc_config.s.mode32b) ? 4 : 8) + lmc_config.s.ecc_ena;

        memset((void *)&dsksum, 0, sizeof(dsksum));
        memset((void *)&dskcnt, 0, sizeof(dskcnt));

        for (rankx = 0; rankx < 4; rankx++) {
                if (!(rank_mask & (1 << rankx)))
                        continue;

                // sum ranks
                for (lane = 0; lane < byte_limit; lane++) {
                        for (bit = 0; bit < 8; ++bit) {
                                deskew = dskdat[rankx].bytes[lane].bits[bit];
                                // if flags indicate sat hi or lo, skip it
                                if (deskew & 6)
                                        continue;

                                // clear flags
                                dsksum.bytes[lane].bits[bit] +=
                                        deskew & ~7;
                                // count entries
                                dskcnt.bytes[lane].bits[bit] += 1;
                        }
                }
        }

        // average ranks
        for (lane = 0; lane < byte_limit; lane++) {
                for (bit = 0; bit < 8; ++bit) {
                        int div = dskcnt.bytes[lane].bits[bit];

                        if (div > 0) {
                                dsksum.bytes[lane].bits[bit] /= div;
                                // clear flags
                                dsksum.bytes[lane].bits[bit] &= ~7;
                                // set LOCK
                                dsksum.bytes[lane].bits[bit] |= 1;
                        } else {
                                // FIXME? use reset value?
                                dsksum.bytes[lane].bits[bit] =
                                        (64 << 3) | 1;
                        }
                }
        }

        // TME for FINAL version
        display_deskew_settings(priv, if_num, &dsksum, /*VBL_TME */ 3);

        // finally, write the averaged DESKEW values
        override_deskew_settings(priv, if_num, &dsksum);
}

struct deskew_counts {
        int saturated;          // number saturated
        int unlocked;           // number unlocked
        int nibrng_errs;        // nibble range errors
        int nibunl_errs;        // nibble unlocked errors
        int bitval_errs;        // bit value errors
};

#define MIN_BITVAL  17
#define MAX_BITVAL 110

static void validate_deskew_training(struct ddr_priv *priv, int rank_mask,
                                     int if_num, struct deskew_counts *counts,
                                     int print_flags)
{
        int byte_lane, bit_index, nib_num;
        int nibrng_errs, nibunl_errs, bitval_errs;
        union cvmx_lmcx_config lmc_config;
        s16 nib_min[2], nib_max[2], nib_unl[2];
        int byte_limit;
        int print_enable = print_flags & 1;
        struct deskew_data dskdat;
        s16 flags, deskew;
        const char *fc = " ?-=+*#&";
        int bit_last;

        lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
        byte_limit = ((!lmc_config.s.mode32b) ? 8 : 4) + lmc_config.s.ecc_ena;

        memset(counts, 0, sizeof(struct deskew_counts));

        get_deskew_settings(priv, if_num, &dskdat);

        if (print_enable) {
                debug("N0.LMC%d: Deskew Settings:          Bit =>      :",
                      if_num);
                for (bit_index = 7; bit_index >= 0; --bit_index)
                        debug(" %3d  ", bit_index);
                debug("\n");
        }

        for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) {
                if (print_enable)
                        debug("N0.LMC%d: Bit Deskew Byte %d %s               :",
                              if_num, byte_lane,
                              (print_flags & 2) ? "FINAL" : "     ");

                nib_min[0] = 127;
                nib_min[1] = 127;
                nib_max[0] = 0;
                nib_max[1] = 0;
                nib_unl[0] = 0;
                nib_unl[1] = 0;

                if (lmc_config.s.mode32b == 1 && byte_lane == 4) {
                        bit_last = 3;
                        if (print_enable)
                                debug("                        ");
                } else {
                        bit_last = 7;
                }

                for (bit_index = bit_last; bit_index >= 0; --bit_index) {
                        nib_num = (bit_index > 3) ? 1 : 0;

                        flags = dskdat.bytes[byte_lane].bits[bit_index] & 7;
                        deskew = dskdat.bytes[byte_lane].bits[bit_index] >> 3;

                        counts->saturated += !!(flags & 6);

                        // Do range calc even when locked; it could happen
                        // that a bit is still unlocked after final retry,
                        // and we want to have an external retry if a RANGE
                        // error is present at exit...
                        nib_min[nib_num] = min(nib_min[nib_num], deskew);
                        nib_max[nib_num] = max(nib_max[nib_num], deskew);

                        if (!(flags & 1)) {     // only when not locked
                                counts->unlocked += 1;
                                nib_unl[nib_num] += 1;
                        }

                        if (print_enable)
                                debug(" %3d %c", deskew, fc[flags ^ 1]);
                }

                /*
                 * Now look for nibble errors
                 *
                 * For bit 55, it looks like a bit deskew problem. When the
                 * upper nibble of byte 6 needs to go to saturation, bit 7
                 * of byte 6 locks prematurely at 64. For DIMMs with raw
                 * card A and B, can we reset the deskew training when we
                 * encounter this case? The reset criteria should be looking
                 * at one nibble at a time for raw card A and B; if the
                 * bit-deskew setting within a nibble is different by > 33,
                 * we'll issue a reset to the bit deskew training.
                 *
                 * LMC0 Bit Deskew Byte(6): 64 0 - 0 - 0 - 26 61 35 64
                 */
                // upper nibble range, then lower nibble range
                nibrng_errs = ((nib_max[1] - nib_min[1]) > 33) ? 1 : 0;
                nibrng_errs |= ((nib_max[0] - nib_min[0]) > 33) ? 1 : 0;

                // check for nibble all unlocked
                nibunl_errs = ((nib_unl[0] == 4) || (nib_unl[1] == 4)) ? 1 : 0;

                // check for bit value errors, ie < 17 or > 110
                // FIXME? assume max always > MIN_BITVAL and min < MAX_BITVAL
                bitval_errs = ((nib_max[1] > MAX_BITVAL) ||
                               (nib_max[0] > MAX_BITVAL)) ? 1 : 0;
                bitval_errs |= ((nib_min[1] < MIN_BITVAL) ||
                                (nib_min[0] < MIN_BITVAL)) ? 1 : 0;

                if ((nibrng_errs != 0 || nibunl_errs != 0 ||
                     bitval_errs != 0) && print_enable) {
                        debug(" %c%c%c",
                              (nibrng_errs) ? 'R' : ' ',
                              (nibunl_errs) ? 'U' : ' ',
                              (bitval_errs) ? 'V' : ' ');
                }

                if (print_enable)
                        debug("\n");

                counts->nibrng_errs |= (nibrng_errs << byte_lane);
                counts->nibunl_errs |= (nibunl_errs << byte_lane);
                counts->bitval_errs |= (bitval_errs << byte_lane);
        }
}

static unsigned short load_dac_override(struct ddr_priv *priv, int if_num,
                                        int dac_value, int byte)
{
        union cvmx_lmcx_dll_ctl3 ddr_dll_ctl3;
        // single bytelanes incr by 1; A is for ALL
        int bytex = (byte == 0x0A) ? byte : byte + 1;

        ddr_dll_ctl3.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num));

        SET_DDR_DLL_CTL3(byte_sel, bytex);
        SET_DDR_DLL_CTL3(offset, dac_value >> 1);

        ddr_dll_ctl3.cn73xx.bit_select = 0x9;   /* No-op */
        lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64);

        ddr_dll_ctl3.cn73xx.bit_select = 0xC;   /* vref bypass setting load */
        lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64);

        ddr_dll_ctl3.cn73xx.bit_select = 0xD;   /* vref bypass on. */
        lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64);

        ddr_dll_ctl3.cn73xx.bit_select = 0x9;   /* No-op */
        lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64);

        lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num));       // flush writes

        return (unsigned short)GET_DDR_DLL_CTL3(offset);
}

// arg dac_or_dbi is 1 for DAC, 0 for DBI
// returns 9 entries (bytelanes 0 through 8) in settings[]
// returns 0 if OK, -1 if a problem
static int read_dac_dbi_settings(struct ddr_priv *priv, int if_num,
                                 int dac_or_dbi, int *settings)
{
        union cvmx_lmcx_phy_ctl phy_ctl;
        int byte_lane, bit_num;
        int deskew;
        int dac_value;
        int new_deskew_layout = 0;

        new_deskew_layout = octeon_is_cpuid(OCTEON_CN73XX) ||
                octeon_is_cpuid(OCTEON_CNF75XX);
        new_deskew_layout |= (octeon_is_cpuid(OCTEON_CN78XX) &&
                              !octeon_is_cpuid(OCTEON_CN78XX_PASS1_X));

        phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
        phy_ctl.s.dsk_dbg_clk_scaler = 3;
        lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);

        bit_num = (dac_or_dbi) ? 4 : 5;
        // DBI not available
        if (bit_num == 5 && !new_deskew_layout)
                return -1;

        // FIXME: always assume ECC is available
        for (byte_lane = 8; byte_lane >= 0; --byte_lane) {
                //set byte lane and bit to read
                phy_ctl.s.dsk_dbg_bit_sel = bit_num;
                phy_ctl.s.dsk_dbg_byte_sel = byte_lane;
                lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);

                //start read sequence
                phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
                phy_ctl.s.dsk_dbg_rd_start = 1;
                lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);

                //poll for read sequence to complete
                do {
                        phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
                } while (phy_ctl.s.dsk_dbg_rd_complete != 1);

                // keep the flag bits where they are for DBI
                deskew = phy_ctl.s.dsk_dbg_rd_data; /* >> 3 */
                dac_value = phy_ctl.s.dsk_dbg_rd_data & 0xff;

                settings[byte_lane] = (dac_or_dbi) ? dac_value : deskew;
        }

        return 0;
}

// print out the DBI settings array
// arg dac_or_dbi is 1 for DAC, 0 for DBI
static void display_dac_dbi_settings(int lmc, int dac_or_dbi,
                                     int ecc_ena, int *settings, char *title)
{
        int byte;
        int flags;
        int deskew;
        const char *fc = " ?-=+*#&";

        debug("N0.LMC%d: %s %s Settings %d:0 :",
              lmc, title, (dac_or_dbi) ? "DAC" : "DBI", 7 + ecc_ena);
        // FIXME: what about 32-bit mode?
        for (byte = (7 + ecc_ena); byte >= 0; --byte) {
                if (dac_or_dbi) {       // DAC
                        flags = 1;      // say its locked to get blank
                        deskew = settings[byte] & 0xff;
                } else {        // DBI
                        flags = settings[byte] & 7;
                        deskew = (settings[byte] >> 3) & 0x7f;
                }
                debug(" %3d %c", deskew, fc[flags ^ 1]);
        }
        debug("\n");
}

// Find a HWL majority
static int find_wl_majority(struct wlevel_bitcnt *bc, int *mx, int *mc,
                            int *xc, int *cc)
{
        int ix, ic;

        *mx = -1;
        *mc = 0;
        *xc = 0;
        *cc = 0;

        for (ix = 0; ix < 4; ix++) {
                ic = bc->bitcnt[ix];

                // make a bitmask of the ones with a count
                if (ic > 0) {
                        *mc |= (1 << ix);
                        *cc += 1;       // count how many had non-zero counts
                }

                // find the majority
                if (ic > *xc) { // new max?
                        *xc = ic;       // yes
                        *mx = ix;       // set its index
                }
        }

        return (*mx << 1);
}

// Evaluate the DAC settings array
static int evaluate_dac_settings(int if_64b, int ecc_ena, int *settings)
{
        int byte, lane, dac, comp;
        int last = (if_64b) ? 7 : 3;

        // FIXME: change the check...???
        // this looks only for sets of DAC values whose max/min differ by a lot
        // let any EVEN go so long as it is within range...
        for (byte = (last + ecc_ena); byte >= 0; --byte) {
                dac = settings[byte] & 0xff;

                for (lane = (last + ecc_ena); lane >= 0; --lane) {
                        comp = settings[lane] & 0xff;
                        if (abs((dac - comp)) > 25)
                                return 1;
                }
        }

        return 0;
}

static void perform_offset_training(struct ddr_priv *priv, int rank_mask,
                                    int if_num)
{
        union cvmx_lmcx_phy_ctl lmc_phy_ctl;
        u64 orig_phy_ctl;
        const char *s;

        /*
         * 4.8.6 LMC Offset Training
         *
         * LMC requires input-receiver offset training.
         *
         * 1. Write LMC(0)_PHY_CTL[DAC_ON] = 1
         */
        lmc_phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
        orig_phy_ctl = lmc_phy_ctl.u64;
        lmc_phy_ctl.s.dac_on = 1;

        // allow full CSR override
        s = lookup_env_ull(priv, "ddr_phy_ctl");
        if (s)
                lmc_phy_ctl.u64 = strtoull(s, NULL, 0);

        // do not print or write if CSR does not change...
        if (lmc_phy_ctl.u64 != orig_phy_ctl) {
                debug("PHY_CTL                                       : 0x%016llx\n",
                      lmc_phy_ctl.u64);
                lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), lmc_phy_ctl.u64);
        }

        /*
         * 2. Write LMC(0)_SEQ_CTL[SEQ_SEL] = 0x0B and
         *    LMC(0)_SEQ_CTL[INIT_START] = 1.
         *
         * 3. Wait for LMC(0)_SEQ_CTL[SEQ_COMPLETE] to be set to 1.
         */
        /* Start Offset training sequence */
        oct3_ddr3_seq(priv, rank_mask, if_num, 0x0B);
}

static void perform_internal_vref_training(struct ddr_priv *priv,
                                           int rank_mask, int if_num)
{
        union cvmx_lmcx_ext_config ext_config;
        union cvmx_lmcx_dll_ctl3 ddr_dll_ctl3;

        // First, make sure all byte-lanes are out of VREF bypass mode
        ddr_dll_ctl3.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num));

        ddr_dll_ctl3.cn78xx.byte_sel = 0x0A;    /* all byte-lanes */
        ddr_dll_ctl3.cn78xx.bit_select = 0x09;  /* No-op */
        lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64);

        ddr_dll_ctl3.cn78xx.bit_select = 0x0E;  /* vref bypass off. */
        lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64);

        ddr_dll_ctl3.cn78xx.bit_select = 0x09;  /* No-op */
        lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64);

        /*
         * 4.8.7 LMC Internal vref Training
         *
         * LMC requires input-reference-voltage training.
         *
         * 1. Write LMC(0)_EXT_CONFIG[VREFINT_SEQ_DESKEW] = 0.
         */
        ext_config.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG(if_num));
        ext_config.s.vrefint_seq_deskew = 0;

        ddr_seq_print("Performing LMC sequence: vrefint_seq_deskew = %d\n",
                      ext_config.s.vrefint_seq_deskew);

        lmc_wr(priv, CVMX_LMCX_EXT_CONFIG(if_num), ext_config.u64);

        /*
         * 2. Write LMC(0)_SEQ_CTL[SEQ_SEL] = 0x0a and
         *    LMC(0)_SEQ_CTL[INIT_START] = 1.
         *
         * 3. Wait for LMC(0)_SEQ_CTL[SEQ_COMPLETE] to be set to 1.
         */
        /* Start LMC Internal vref Training */
        oct3_ddr3_seq(priv, rank_mask, if_num, 0x0A);
}

#define dbg_avg(format, ...)    // debug(format, ##__VA_ARGS__)

static int process_samples_average(s16 *bytes, int num_samples,
                                   int lmc, int lane_no)
{
        int i, sadj, sum = 0, ret, asum, trunc;
        s16 smin = 32767, smax = -32768;
        int nmin, nmax;
        //int rng;

        dbg_avg("DBG_AVG%d.%d: ", lmc, lane_no);

        for (i = 0; i < num_samples; i++) {
                sum += bytes[i];
                if (bytes[i] < smin)
                        smin = bytes[i];
                if (bytes[i] > smax)
                        smax = bytes[i];
                dbg_avg(" %3d", bytes[i]);
        }

        nmin = 0;
        nmax = 0;
        for (i = 0; i < num_samples; i++) {
                if (bytes[i] == smin)
                        nmin += 1;
                if (bytes[i] == smax)
                        nmax += 1;
        }
        dbg_avg(" (min=%3d/%d, max=%3d/%d, range=%2d, samples=%2d)",
                smin, nmin, smax, nmax, rng, num_samples);

        asum = sum - smin - smax;

        sadj = divide_nint(asum * 10, (num_samples - 2));

        trunc = asum / (num_samples - 2);

        dbg_avg(" [%3d.%d, %3d]", sadj / 10, sadj % 10, trunc);

        sadj = divide_nint(sadj, 10);
        if (trunc & 1)
                ret = trunc;
        else if (sadj & 1)
                ret = sadj;
        else
                ret = trunc + 1;

        dbg_avg(" -> %3d\n", ret);

        return ret;
}

#define DEFAULT_SAT_RETRY_LIMIT    11   // 1 + 10 retries

#define default_lock_retry_limit   20   // 20 retries
#define deskew_validation_delay    10000        // 10 millisecs

static int perform_deskew_training(struct ddr_priv *priv, int rank_mask,
                                   int if_num, int spd_rawcard_aorb)
{
        int unsaturated, locked;
        int sat_retries, sat_retries_limit;
        int lock_retries, lock_retries_total, lock_retries_limit;
        int print_first;
        int print_them_all;
        struct deskew_counts dsk_counts;
        union cvmx_lmcx_phy_ctl phy_ctl;
        char *s;
        int has_no_sat = octeon_is_cpuid(OCTEON_CN78XX_PASS2_X) ||
                octeon_is_cpuid(OCTEON_CNF75XX);
        int disable_bitval_retries = 1; // default to disabled

        debug("N0.LMC%d: Performing Deskew Training.\n", if_num);

        sat_retries = 0;
        sat_retries_limit = (has_no_sat) ? 5 : DEFAULT_SAT_RETRY_LIMIT;

        lock_retries_total = 0;
        unsaturated = 0;
        print_first = 1;        // print the first one
        // set to true for printing all normal deskew attempts
        print_them_all = 0;

        // provide override for bitval_errs causing internal VREF retries
        s = env_get("ddr_disable_bitval_retries");
        if (s)
                disable_bitval_retries = !!simple_strtoul(s, NULL, 0);

        lock_retries_limit = default_lock_retry_limit;
        if ((octeon_is_cpuid(OCTEON_CN78XX_PASS2_X)) ||
            (octeon_is_cpuid(OCTEON_CN73XX)) ||
            (octeon_is_cpuid(OCTEON_CNF75XX)))
                lock_retries_limit *= 2;        // give new chips twice as many

        do {                    /* while (sat_retries < sat_retry_limit) */
                /*
                 * 4.8.8 LMC Deskew Training
                 *
                 * LMC requires input-read-data deskew training.
                 *
                 * 1. Write LMC(0)_EXT_CONFIG[VREFINT_SEQ_DESKEW] = 1.
                 */

                union cvmx_lmcx_ext_config ext_config;

                ext_config.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG(if_num));
                ext_config.s.vrefint_seq_deskew = 1;

                ddr_seq_print
                    ("Performing LMC sequence: vrefint_seq_deskew = %d\n",
                     ext_config.s.vrefint_seq_deskew);

                lmc_wr(priv, CVMX_LMCX_EXT_CONFIG(if_num), ext_config.u64);

                /*
                 * 2. Write LMC(0)_SEQ_CTL[SEQ_SEL] = 0x0A and
                 *    LMC(0)_SEQ_CTL[INIT_START] = 1.
                 *
                 * 3. Wait for LMC(0)_SEQ_CTL[SEQ_COMPLETE] to be set to 1.
                 */

                phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
                phy_ctl.s.phy_dsk_reset = 1;    /* RESET Deskew sequence */
                lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);

                /* LMC Deskew Training */
                oct3_ddr3_seq(priv, rank_mask, if_num, 0x0A);

                lock_retries = 0;

perform_deskew_training:

                phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num));
                phy_ctl.s.phy_dsk_reset = 0;    /* Normal Deskew sequence */
                lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64);

                /* LMC Deskew Training */
                oct3_ddr3_seq(priv, rank_mask, if_num, 0x0A);

                // Moved this from validate_deskew_training
                /* Allow deskew results to stabilize before evaluating them. */
                udelay(deskew_validation_delay);

                // Now go look at lock and saturation status...
                validate_deskew_training(priv, rank_mask, if_num, &dsk_counts,
                                         print_first);
                // after printing the first and not doing them all, no more
                if (print_first && !print_them_all)
                        print_first = 0;

                unsaturated = (dsk_counts.saturated == 0);
                locked = (dsk_counts.unlocked == 0);

                // only do locking retries if unsaturated or rawcard A or B,
                // otherwise full SAT retry
                if (unsaturated || (spd_rawcard_aorb && !has_no_sat)) {
                        if (!locked) {  // and not locked
                                lock_retries++;
                                lock_retries_total++;
                                if (lock_retries <= lock_retries_limit) {
                                        goto perform_deskew_training;
                                } else {
                                        debug("N0.LMC%d: LOCK RETRIES failed after %d retries\n",
                                              if_num, lock_retries_limit);
                                }
                        } else {
                                // only print if we did try
                                if (lock_retries_total > 0)
                                        debug("N0.LMC%d: LOCK RETRIES successful after %d retries\n",
                                              if_num, lock_retries);
                        }
                }               /* if (unsaturated || spd_rawcard_aorb) */

                ++sat_retries;

                /*
                 * At this point, check for a DDR4 RDIMM that will not
                 * benefit from SAT retries; if so, exit
                 */
                if (spd_rawcard_aorb && !has_no_sat) {
                        debug("N0.LMC%d: Deskew Training Loop: Exiting for RAWCARD == A or B.\n",
                              if_num);
                        break;  // no sat or lock retries
                }

        } while (!unsaturated && (sat_retries < sat_retries_limit));

        debug("N0.LMC%d: Deskew Training %s. %d sat-retries, %d lock-retries\n",
              if_num, (sat_retries >= DEFAULT_SAT_RETRY_LIMIT) ?
              "Timed Out" : "Completed", sat_retries - 1, lock_retries_total);

        // FIXME? add saturation to reasons for fault return - give it a
        // chance via Internal VREF
        // FIXME? add OPTIONAL bit value to reasons for fault return -
        // give it a chance via Internal VREF
        if (dsk_counts.nibrng_errs != 0 || dsk_counts.nibunl_errs != 0 ||
            (dsk_counts.bitval_errs != 0 && !disable_bitval_retries) ||
            !unsaturated) {
                debug("N0.LMC%d: Nibble or Saturation Error(s) found, returning FAULT\n",
                      if_num);
                // FIXME: do we want this output always for errors?
                validate_deskew_training(priv, rank_mask, if_num,
                                         &dsk_counts, 1);
                return -1;      // we did retry locally, they did not help
        }

        // NOTE: we (currently) always print one last training validation
        // before starting Read Leveling...

        return 0;
}

#define SCALING_FACTOR (1000)

// NOTE: this gets called for 1-rank and 2-rank DIMMs in single-slot config
static int compute_vref_1slot_2rank(int rtt_wr, int rtt_park, int dqx_ctl,
                                    int rank_count, int dram_connection)
{
        u64 reff_s;
        u64 rser_s = (dram_connection) ? 0 : 15;
        u64 vdd = 1200;
        u64 vref;
        // 99 == HiZ
        u64 rtt_wr_s = (((rtt_wr == 0) || rtt_wr == 99) ?
                        1 * 1024 * 1024 : rtt_wr);
        u64 rtt_park_s = (((rtt_park == 0) || ((rank_count == 1) &&
                                               (rtt_wr != 0))) ?
                          1 * 1024 * 1024 : rtt_park);
        u64 dqx_ctl_s = (dqx_ctl == 0 ? 1 * 1024 * 1024 : dqx_ctl);
        int vref_value;
        u64 rangepc = 6000;     // range1 base
        u64 vrefpc;
        int vref_range = 0;

        reff_s = divide_nint((rtt_wr_s * rtt_park_s), (rtt_wr_s + rtt_park_s));

        vref = (((rser_s + dqx_ctl_s) * SCALING_FACTOR) /
                (rser_s + dqx_ctl_s + reff_s)) + SCALING_FACTOR;

        vref = (vref * vdd) / 2 / SCALING_FACTOR;

        vrefpc = (vref * 100 * 100) / vdd;

        if (vrefpc < rangepc) { // < range1 base, use range2
                vref_range = 1 << 6;    // set bit A6 for range2
                rangepc = 4500; // range2 base is 45%
        }

        vref_value = divide_nint(vrefpc - rangepc, 65);
        if (vref_value < 0)
                vref_value = vref_range;        // set to base of range
        else
                vref_value |= vref_range;

        debug("rtt_wr: %d, rtt_park: %d, dqx_ctl: %d, rank_count: %d\n",
              rtt_wr, rtt_park, dqx_ctl, rank_count);
        debug("rtt_wr_s: %lld, rtt_park_s: %lld, dqx_ctl_s: %lld, vref_value: 0x%x, range: %d\n",
              rtt_wr_s, rtt_park_s, dqx_ctl_s, vref_value ^ vref_range,
              vref_range ? 2 : 1);

        return vref_value;
}

// NOTE: this gets called for 1-rank and 2-rank DIMMs in two-slot configs
static int compute_vref_2slot_2rank(int rtt_wr, int rtt_park_00,
                                    int rtt_park_01,
                                    int dqx_ctl, int rtt_nom,
                                    int dram_connection)
{
        u64 rser = (dram_connection) ? 0 : 15;
        u64 vdd = 1200;
        u64 vl, vlp, vcm;
        u64 rd0, rd1, rpullup;
        // 99 == HiZ
        u64 rtt_wr_s = (((rtt_wr == 0) || rtt_wr == 99) ?
                        1 * 1024 * 1024 : rtt_wr);
        u64 rtt_park_00_s = (rtt_park_00 == 0 ? 1 * 1024 * 1024 : rtt_park_00);
        u64 rtt_park_01_s = (rtt_park_01 == 0 ? 1 * 1024 * 1024 : rtt_park_01);
        u64 dqx_ctl_s = (dqx_ctl == 0 ? 1 * 1024 * 1024 : dqx_ctl);
        u64 rtt_nom_s = (rtt_nom == 0 ? 1 * 1024 * 1024 : rtt_nom);
        int vref_value;
        u64 rangepc = 6000;     // range1 base
        u64 vrefpc;
        int vref_range = 0;

        // rd0 = (RTT_NOM (parallel) RTT_WR) +  =
        // ((RTT_NOM * RTT_WR) / (RTT_NOM + RTT_WR)) + RSER
        rd0 = divide_nint((rtt_nom_s * rtt_wr_s),
                          (rtt_nom_s + rtt_wr_s)) + rser;

        // rd1 = (RTT_PARK_00 (parallel) RTT_PARK_01) + RSER =
        // ((RTT_PARK_00 * RTT_PARK_01) / (RTT_PARK_00 + RTT_PARK_01)) + RSER
        rd1 = divide_nint((rtt_park_00_s * rtt_park_01_s),
                          (rtt_park_00_s + rtt_park_01_s)) + rser;

        // rpullup = rd0 (parallel) rd1 = (rd0 * rd1) / (rd0 + rd1)
        rpullup = divide_nint((rd0 * rd1), (rd0 + rd1));

        // vl = (DQX_CTL / (DQX_CTL + rpullup)) * 1.2
        vl = divide_nint((dqx_ctl_s * vdd), (dqx_ctl_s + rpullup));

        // vlp = ((RSER / rd0) * (1.2 - vl)) + vl
        vlp = divide_nint((rser * (vdd - vl)), rd0) + vl;

        // vcm = (vlp + 1.2) / 2
        vcm = divide_nint((vlp + vdd), 2);

        // vrefpc = (vcm / 1.2) * 100
        vrefpc = divide_nint((vcm * 100 * 100), vdd);

        if (vrefpc < rangepc) { // < range1 base, use range2
                vref_range = 1 << 6;    // set bit A6 for range2
                rangepc = 4500; // range2 base is 45%
        }

        vref_value = divide_nint(vrefpc - rangepc, 65);
        if (vref_value < 0)
                vref_value = vref_range;        // set to base of range
        else
                vref_value |= vref_range;

        debug("rtt_wr:%d, rtt_park_00:%d, rtt_park_01:%d, dqx_ctl:%d, rtt_nom:%d, vref_value:%d (0x%x)\n",
              rtt_wr, rtt_park_00, rtt_park_01, dqx_ctl, rtt_nom, vref_value,
              vref_value);

        return vref_value;
}

// NOTE: only call this for DIMMs with 1 or 2 ranks, not 4.
static int compute_vref_val(struct ddr_priv *priv, int if_num, int rankx,
                            int dimm_count, int rank_count,
                            struct impedence_values *imp_values,
                            int is_stacked_die, int dram_connection)
{
        int computed_final_vref_value = 0;
        int enable_adjust = ENABLE_COMPUTED_VREF_ADJUSTMENT;
        const char *s;
        int rtt_wr, dqx_ctl, rtt_nom, index;
        union cvmx_lmcx_modereg_params1 lmc_modereg_params1;
        union cvmx_lmcx_modereg_params2 lmc_modereg_params2;
        union cvmx_lmcx_comp_ctl2 comp_ctl2;
        int rtt_park;
        int rtt_park_00;
        int rtt_park_01;

        debug("N0.LMC%d.R%d: %s(...dram_connection = %d)\n",
              if_num, rankx, __func__, dram_connection);

        // allow some overrides...
        s = env_get("ddr_adjust_computed_vref");
        if (s) {
                enable_adjust = !!simple_strtoul(s, NULL, 0);
                if (!enable_adjust) {
                        debug("N0.LMC%d.R%d: DISABLE adjustment of computed VREF\n",
                              if_num, rankx);
                }
        }

        s = env_get("ddr_set_computed_vref");
        if (s) {
                int new_vref = simple_strtoul(s, NULL, 0);

                debug("N0.LMC%d.R%d: OVERRIDE computed VREF to 0x%x (%d)\n",
                      if_num, rankx, new_vref, new_vref);
                return new_vref;
        }

        /*
         * Calculate an alternative to the measured vref value
         * but only for configurations we know how to...
         */
        // We have code for 2-rank DIMMs in both 1-slot or 2-slot configs,
        // and can use the 2-rank 1-slot code for 1-rank DIMMs in 1-slot
        // configs, and can use the 2-rank 2-slot code for 1-rank DIMMs
        // in 2-slot configs.

        lmc_modereg_params1.u64 =
            lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS1(if_num));
        lmc_modereg_params2.u64 =
            lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS2(if_num));
        comp_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num));
        dqx_ctl = imp_values->dqx_strength[comp_ctl2.s.dqx_ctl];

        // WR always comes from the current rank
        index = (lmc_modereg_params1.u64 >> (rankx * 12 + 5)) & 0x03;
        if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X))
                index |= lmc_modereg_params1.u64 >> (51 + rankx - 2) & 0x04;
        rtt_wr = imp_values->rtt_wr_ohms[index];

        // separate calculations for 1 vs 2 DIMMs per LMC
        if (dimm_count == 1) {
                // PARK comes from this rank if 1-rank, otherwise other rank
                index =
                    (lmc_modereg_params2.u64 >>
                     ((rankx ^ (rank_count - 1)) * 10 + 0)) & 0x07;
                rtt_park = imp_values->rtt_nom_ohms[index];
                computed_final_vref_value =
                    compute_vref_1slot_2rank(rtt_wr, rtt_park, dqx_ctl,
                                             rank_count, dram_connection);
        } else {
                // get both PARK values from the other DIMM
                index =
                    (lmc_modereg_params2.u64 >> ((rankx ^ 0x02) * 10 + 0)) &
                    0x07;
                rtt_park_00 = imp_values->rtt_nom_ohms[index];
                index =
                    (lmc_modereg_params2.u64 >> ((rankx ^ 0x03) * 10 + 0)) &
                    0x07;
                rtt_park_01 = imp_values->rtt_nom_ohms[index];
                // NOM comes from this rank if 1-rank, otherwise other rank
                index =
                    (lmc_modereg_params1.u64 >>
                     ((rankx ^ (rank_count - 1)) * 12 + 9)) & 0x07;
                rtt_nom = imp_values->rtt_nom_ohms[index];
                computed_final_vref_value =
                    compute_vref_2slot_2rank(rtt_wr, rtt_park_00, rtt_park_01,
                                             dqx_ctl, rtt_nom, dram_connection);
        }

        if (enable_adjust) {
                union cvmx_lmcx_config lmc_config;
                union cvmx_lmcx_control lmc_control;

                lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num));
                lmc_control.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num));

                /*
                 *  New computed vref = existing computed vref – X
                 *
                 * The value of X is depending on different conditions.
                 * Both #122 and #139 are 2Rx4 RDIMM, while #124 is stacked
                 * die 2Rx4, so I conclude the results into two conditions:
                 *
                 * 1. Stacked Die: 2Rx4
                 * 1-slot: offset = 7. i, e New computed vref = existing
                 * computed vref – 7
                 * 2-slot: offset = 6
                 *
                 * 2. Regular: 2Rx4
                 * 1-slot: offset = 3
                 * 2-slot:  offset = 2
                 */
                // we know we never get called unless DDR4, so test just
                // the other conditions
                if (lmc_control.s.rdimm_ena == 1 &&
                    rank_count == 2 && lmc_config.s.mode_x4dev) {
                        // it must first be RDIMM and 2-rank and x4
                        int adj;

                        // now do according to stacked die or not...
                        if (is_stacked_die)
                                adj = (dimm_count == 1) ? -7 : -6;
                        else
                                adj = (dimm_count == 1) ? -3 : -2;

                        // we must have adjusted it, so print it out if
                        // verbosity is right
                        debug("N0.LMC%d.R%d: adjusting computed vref from %2d (0x%02x) to %2d (0x%02x)\n",
                              if_num, rankx, computed_final_vref_value,
                              computed_final_vref_value,
                              computed_final_vref_value + adj,
                              computed_final_vref_value + adj);
                        computed_final_vref_value += adj;
                }
        }

        return computed_final_vref_value;
}

static void unpack_rlevel_settings(int if_bytemask, int ecc_ena,
                                   struct rlevel_byte_data *rlevel_byte,
                                   union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank)
{
        if ((if_bytemask & 0xff) == 0xff) {
                if (ecc_ena) {
                        rlevel_byte[8].delay = lmc_rlevel_rank.s.byte7;
                        rlevel_byte[7].delay = lmc_rlevel_rank.s.byte6;
                        rlevel_byte[6].delay = lmc_rlevel_rank.s.byte5;
                        rlevel_byte[5].delay = lmc_rlevel_rank.s.byte4;
                        /* ECC */
                        rlevel_byte[4].delay = lmc_rlevel_rank.s.byte8;
                } else {
                        rlevel_byte[7].delay = lmc_rlevel_rank.s.byte7;
                        rlevel_byte[6].delay = lmc_rlevel_rank.s.byte6;
                        rlevel_byte[5].delay = lmc_rlevel_rank.s.byte5;
                        rlevel_byte[4].delay = lmc_rlevel_rank.s.byte4;
                }
        } else {
                rlevel_byte[8].delay = lmc_rlevel_rank.s.byte8; /* unused */
                rlevel_byte[7].delay = lmc_rlevel_rank.s.byte7; /* unused */
                rlevel_byte[6].delay = lmc_rlevel_rank.s.byte6; /* unused */
                rlevel_byte[5].delay = lmc_rlevel_rank.s.byte5; /* unused */
                rlevel_byte[4].delay = lmc_rlevel_rank.s.byte4; /* ECC */
        }

        rlevel_byte[3].delay = lmc_rlevel_rank.s.byte3;
        rlevel_byte[2].delay = lmc_rlevel_rank.s.byte2;
        rlevel_byte[1].delay = lmc_rlevel_rank.s.byte1;
        rlevel_byte[0].delay = lmc_rlevel_rank.s.byte0;
}

static void pack_rlevel_settings(int if_bytemask, int ecc_ena,
                                 struct rlevel_byte_data *rlevel_byte,
                                 union cvmx_lmcx_rlevel_rankx
                                 *final_rlevel_rank)
{
        union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank = *final_rlevel_rank;

        if ((if_bytemask & 0xff) == 0xff) {
                if (ecc_ena) {
                        lmc_rlevel_rank.s.byte7 = rlevel_byte[8].delay;
                        lmc_rlevel_rank.s.byte6 = rlevel_byte[7].delay;
                        lmc_rlevel_rank.s.byte5 = rlevel_byte[6].delay;
                        lmc_rlevel_rank.s.byte4 = rlevel_byte[5].delay;
                        /* ECC */
                        lmc_rlevel_rank.s.byte8 = rlevel_byte[4].delay;
                } else {
                        lmc_rlevel_rank.s.byte7 = rlevel_byte[7].delay;
                        lmc_rlevel_rank.s.byte6 = rlevel_byte[6].delay;
                        lmc_rlevel_rank.s.byte5 = rlevel_byte[5].delay;
                        lmc_rlevel_rank.s.byte4 = rlevel_byte[4].delay;
                }
        } else {
                lmc_rlevel_rank.s.byte8 = rlevel_byte[8].delay;
                lmc_rlevel_rank.s.byte7 = rlevel_byte[7].delay;
                lmc_rlevel_rank.s.byte6 = rlevel_byte[6].delay;
                lmc_rlevel_rank.s.byte5 = rlevel_byte[5].delay;
                lmc_rlevel_rank.s.byte4 = rlevel_byte[4].delay;
        }

        lmc_rlevel_rank.s.byte3 = rlevel_byte[3].delay;
        lmc_rlevel_rank.s.byte2 = rlevel_byte[2].delay;
        lmc_rlevel_rank.s.byte1 = rlevel_byte[1].delay;
        lmc_rlevel_rank.s.byte0 = rlevel_byte[0].delay;

        *final_rlevel_rank = lmc_rlevel_rank;
}

/////////////////// These are the RLEVEL settings display routines

// flags
#define WITH_NOTHING 0
#define WITH_SCORE   1
#define WITH_AVERAGE 2
#define WITH_FINAL   4
#define WITH_COMPUTE 8

static void do_display_rl(int if_num,
                          union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank,
                          int rank, int flags, int score)
{
        char score_buf[16];
        char *msg_buf;
        char hex_buf[20];

        if (flags & WITH_SCORE) {
                snprintf(score_buf, sizeof(score_buf), "(%d)", score);
        } else {
                score_buf[0] = ' ';
                score_buf[1] = 0;
        }

        if (flags & WITH_AVERAGE) {
                msg_buf = "  DELAY AVERAGES  ";
        } else if (flags & WITH_FINAL) {
                msg_buf = "  FINAL SETTINGS  ";
        } else if (flags & WITH_COMPUTE) {
                msg_buf = "  COMPUTED DELAYS ";
        } else {
                snprintf(hex_buf, sizeof(hex_buf), "0x%016llX",
                         (unsigned long long)lmc_rlevel_rank.u64);
                msg_buf = hex_buf;
        }

        debug("N0.LMC%d.R%d: Rlevel Rank %#4x, %s  : %5d %5d %5d %5d %5d %5d %5d %5d %5d %s\n",
              if_num, rank, lmc_rlevel_rank.s.status, msg_buf,
              lmc_rlevel_rank.s.byte8, lmc_rlevel_rank.s.byte7,
              lmc_rlevel_rank.s.byte6, lmc_rlevel_rank.s.byte5,
              lmc_rlevel_rank.s.byte4, lmc_rlevel_rank.s.byte3,
              lmc_rlevel_rank.s.byte2, lmc_rlevel_rank.s.byte1,
              lmc_rlevel_rank.s.byte0, score_buf);
}

static void display_rl(int if_num,
                       union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank, int rank)
{
        do_display_rl(if_num, lmc_rlevel_rank, rank, 0, 0);
}

static void display_rl_with_score(int if_num,
                                  union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank,
                                  int rank, int score)
{
        do_display_rl(if_num, lmc_rlevel_rank, rank, 1, score);
}

static void display_rl_with_final(int if_num,
                                  union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank,
                                  int rank)
{
        do_display_rl(if_num, lmc_rlevel_rank, rank, 4, 0);
}

static void display_rl_with_computed(int if_num,
                                     union cvmx_lmcx_rlevel_rankx
                                     lmc_rlevel_rank, int rank, int score)
{
        do_display_rl(if_num, lmc_rlevel_rank, rank, 9, score);
}

// flag values
#define WITH_RODT_BLANK      0
#define WITH_RODT_SKIPPING   1
#define WITH_RODT_BESTROW    2
#define WITH_RODT_BESTSCORE  3
// control
#define SKIP_SKIPPING 1

static const char *with_rodt_canned_msgs[4] = {
        "          ", "SKIPPING  ", "BEST ROW  ", "BEST SCORE"
};

static void display_rl_with_rodt(int if_num,
                                 union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank,
                                 int rank, int score,
                                 int nom_ohms, int rodt_ohms, int flag)
{
        const char *msg_buf;
        char set_buf[20];

#if SKIP_SKIPPING
        if (flag == WITH_RODT_SKIPPING)
                return;
#endif

        msg_buf = with_rodt_canned_msgs[flag];
        if (nom_ohms < 0) {
                snprintf(set_buf, sizeof(set_buf), "    RODT %3d    ",
                         rodt_ohms);
        } else {
                snprintf(set_buf, sizeof(set_buf), "NOM %3d RODT %3d", nom_ohms,
                         rodt_ohms);
        }

        debug("N0.LMC%d.R%d: Rlevel %s   %s  : %5d %5d %5d %5d %5d %5d %5d %5d %5d (%d)\n",
              if_num, rank, set_buf, msg_buf, lmc_rlevel_rank.s.byte8,
              lmc_rlevel_rank.s.byte7, lmc_rlevel_rank.s.byte6,
              lmc_rlevel_rank.s.byte5, lmc_rlevel_rank.s.byte4,
              lmc_rlevel_rank.s.byte3, lmc_rlevel_rank.s.byte2,
              lmc_rlevel_rank.s.byte1, lmc_rlevel_rank.s.byte0, score);
}

static void do_display_wl(int if_num,
                          union cvmx_lmcx_wlevel_rankx lmc_wlevel_rank,
                          int rank, int flags)
{
        char *msg_buf;
        char hex_buf[20];

        if (flags & WITH_FINAL) {
                msg_buf = "  FINAL SETTINGS  ";
        } else {
                snprintf(hex_buf, sizeof(hex_buf), "0x%016llX",
                         (unsigned long long)lmc_wlevel_rank.u64);
                msg_buf = hex_buf;
        }

        debug("N0.LMC%d.R%d: Wlevel Rank %#4x, %s  : %5d %5d %5d %5d %5d %5d %5d %5d %5d\n",
              if_num, rank, lmc_wlevel_rank.s.status, msg_buf,
              lmc_wlevel_rank.s.byte8, lmc_wlevel_rank.s.byte7,
              lmc_wlevel_rank.s.byte6, lmc_wlevel_rank.s.byte5,
              lmc_wlevel_rank.s.byte4, lmc_wlevel_rank.s.byte3,
              lmc_wlevel_rank.s.byte2, lmc_wlevel_rank.s.byte1,
              lmc_wlevel_rank.s.byte0);
}

static void display_wl(int if_num,
                       union cvmx_lmcx_wlevel_rankx lmc_wlevel_rank, int rank)
{
        do_display_wl(if_num, lmc_wlevel_rank, rank, WITH_NOTHING);
}

static void display_wl_with_final(int if_num,
                                  union cvmx_lmcx_wlevel_rankx lmc_wlevel_rank,
                                  int rank)
{
        do_display_wl(if_num, lmc_wlevel_rank, rank, WITH_FINAL);
}

// pretty-print bitmask adjuster
static u64 ppbm(u64 bm)
{
        if (bm != 0ul) {
                while ((bm & 0x0fful) == 0ul)
                        bm >>= 4;
        }

        return bm;
}

// xlate PACKED index to UNPACKED index to use with rlevel_byte
#define XPU(i, e) (((i) < 4) ? (i) : (((i) < 8) ? (i) + (e) : 4))
// xlate UNPACKED index to PACKED index to use with rlevel_bitmask
#define XUP(i, e) (((i) < 4) ? (i) : (e) ? (((i) > 4) ? (i) - 1 : 8) : (i))

// flag values
#define WITH_WL_BITMASKS      0
#define WITH_RL_BITMASKS      1
#define WITH_RL_MASK_SCORES   2
#define WITH_RL_SEQ_SCORES    3

static void do_display_bm(int if_num, int rank, void *bm,
                          int flags, int ecc)
{
        if (flags == WITH_WL_BITMASKS) {
                // wlevel_bitmask array in PACKED index order, so just
                // print them
                int *bitmasks = (int *)bm;

                debug("N0.LMC%d.R%d: Wlevel Debug Bitmasks                 : %05x %05x %05x %05x %05x %05x %05x %05x %05x\n",
                      if_num, rank, bitmasks[8], bitmasks[7], bitmasks[6],
                      bitmasks[5], bitmasks[4], bitmasks[3], bitmasks[2],
                      bitmasks[1], bitmasks[0]
                        );
        } else if (flags == WITH_RL_BITMASKS) {
                // rlevel_bitmask array in PACKED index order, so just
                // print them
                struct rlevel_bitmask *rlevel_bitmask =
                        (struct rlevel_bitmask *)bm;

                debug("N0.LMC%d.R%d: Rlevel Debug Bitmasks        8:0      : %05llx %05llx %05llx %05llx %05llx %05llx %05llx %05llx %05llx\n",
                      if_num, rank, ppbm(rlevel_bitmask[8].bm),
                      ppbm(rlevel_bitmask[7].bm), ppbm(rlevel_bitmask[6].bm),
                      ppbm(rlevel_bitmask[5].bm), ppbm(rlevel_bitmask[4].bm),
                      ppbm(rlevel_bitmask[3].bm), ppbm(rlevel_bitmask[2].bm),
                      ppbm(rlevel_bitmask[1].bm), ppbm(rlevel_bitmask[0].bm)
                        );
        } else if (flags == WITH_RL_MASK_SCORES) {
                // rlevel_bitmask array in PACKED index order, so just
                // print them
                struct rlevel_bitmask *rlevel_bitmask =
                        (struct rlevel_bitmask *)bm;

                debug("N0.LMC%d.R%d: Rlevel Debug Bitmask Scores  8:0      : %5d %5d %5d %5d %5d %5d %5d %5d %5d\n",
                      if_num, rank, rlevel_bitmask[8].errs,
                      rlevel_bitmask[7].errs, rlevel_bitmask[6].errs,
                      rlevel_bitmask[5].errs, rlevel_bitmask[4].errs,
                      rlevel_bitmask[3].errs, rlevel_bitmask[2].errs,
                      rlevel_bitmask[1].errs, rlevel_bitmask[0].errs);
        } else if (flags == WITH_RL_SEQ_SCORES) {
                // rlevel_byte array in UNPACKED index order, so xlate
                // and print them
                struct rlevel_byte_data *rlevel_byte =
                        (struct rlevel_byte_data *)bm;

                debug("N0.LMC%d.R%d: Rlevel Debug Non-seq Scores  8:0      : %5d %5d %5d %5d %5d %5d %5d %5d %5d\n",
                      if_num, rank, rlevel_byte[XPU(8, ecc)].sqerrs,
                      rlevel_byte[XPU(7, ecc)].sqerrs,
                      rlevel_byte[XPU(6, ecc)].sqerrs,
                      rlevel_byte[XPU(5, ecc)].sqerrs,
                      rlevel_byte[XPU(4, ecc)].sqerrs,
                      rlevel_byte[XPU(3, ecc)].sqerrs,
                      rlevel_byte[XPU(2, ecc)].sqerrs,
                      rlevel_byte[XPU(1, ecc)].sqerrs,
                      rlevel_byte[XPU(0, ecc)].sqerrs);
        }
}

static void display_wl_bm(int if_num, int rank, int *bitmasks)
{
        do_display_bm(if_num, rank, (void *)bitmasks, WITH_WL_BITMASKS, 0);
}

static void display_rl_bm(int if_num, int rank,
                          struct rlevel_bitmask *bitmasks, int ecc_ena)
{
        do_display_bm(if_num, rank, (void *)bitmasks, WITH_RL_BITMASKS,
                      ecc_ena);
}

static void display_rl_bm_scores(int if_num, int rank,
                                 struct rlevel_bitmask *bitmasks, int ecc_ena)
{
        do_display_bm(if_num, rank, (void *)bitmasks, WITH_RL_MASK_SCORES,
                      ecc_ena);
}

static void display_rl_seq_scores(int if_num, int rank,
                                  struct rlevel_byte_data *bytes, int ecc_ena)
{
        do_display_bm(if_num, rank, (void *)bytes, WITH_RL_SEQ_SCORES, ecc_ena);
}

#define RODT_OHMS_COUNT        8
#define RTT_NOM_OHMS_COUNT     8
#define RTT_NOM_TABLE_COUNT    8
#define RTT_WR_OHMS_COUNT      8
#define DIC_OHMS_COUNT         3
#define DRIVE_STRENGTH_COUNT  15

static unsigned char ddr4_rodt_ohms[RODT_OHMS_COUNT] = {
        0, 40, 60, 80, 120, 240, 34, 48 };
static unsigned char ddr4_rtt_nom_ohms[RTT_NOM_OHMS_COUNT] = {
        0, 60, 120, 40, 240, 48, 80, 34 };
static unsigned char ddr4_rtt_nom_table[RTT_NOM_TABLE_COUNT] = {
        0, 4, 2, 6, 1, 5, 3, 7 };
// setting HiZ ohms to 99 for computed vref
static unsigned char ddr4_rtt_wr_ohms[RTT_WR_OHMS_COUNT] = {
        0, 120, 240, 99, 80 };
static unsigned char ddr4_dic_ohms[DIC_OHMS_COUNT] = { 34, 48 };
static short ddr4_drive_strength[DRIVE_STRENGTH_COUNT] = {
        0, 0, 26, 30, 34, 40, 48, 68, 0, 0, 0, 0, 0, 0, 0 };
static short ddr4_dqx_strength[DRIVE_STRENGTH_COUNT] = {
        0, 24, 27, 30, 34, 40, 48, 60, 0, 0, 0, 0, 0, 0, 0 };
struct impedence_values ddr4_impedence_val = {
        .rodt_ohms = ddr4_rodt_ohms,
        .rtt_nom_ohms = ddr4_rtt_nom_ohms,
        .rtt_nom_table = ddr4_rtt_nom_table,
        .rtt_wr_ohms = ddr4_rtt_wr_ohms,
        .dic_ohms = ddr4_dic_ohms,
        .drive_strength = ddr4_drive_strength,
        .dqx_strength = ddr4_dqx_strength,
};

static unsigned char ddr3_rodt_ohms[RODT_OHMS_COUNT] = {
        0, 20, 30, 40, 60, 120, 0, 0 };
static unsigned char ddr3_rtt_nom_ohms[RTT_NOM_OHMS_COUNT] = {
        0, 60, 120, 40, 20, 30, 0, 0 };
static unsigned char ddr3_rtt_nom_table[RTT_NOM_TABLE_COUNT] = {
        0, 2, 1, 3, 5, 4, 0, 0 };
static unsigned char ddr3_rtt_wr_ohms[RTT_WR_OHMS_COUNT] = { 0, 60, 120 };
static unsigned char ddr3_dic_ohms[DIC_OHMS_COUNT] = { 40, 34 };
static short ddr3_drive_strength[DRIVE_STRENGTH_COUNT] = {
        0, 24, 27, 30, 34, 40, 48, 60, 0, 0, 0, 0, 0, 0, 0 };
static struct impedence_values ddr3_impedence_val = {
        .rodt_ohms = ddr3_rodt_ohms,
        .rtt_nom_ohms = ddr3_rtt_nom_ohms,
        .rtt_nom_table = ddr3_rtt_nom_table,
        .rtt_wr_ohms = ddr3_rtt_wr_ohms,
        .dic_ohms = ddr3_dic_ohms,
        .drive_strength = ddr3_drive_strength,
        .dqx_strength = ddr3_drive_strength,
};

static u64 hertz_to_psecs(u64 hertz)
{
        /* Clock in psecs */
        return divide_nint((u64)1000 * 1000 * 1000 * 1000, hertz);
}

#define DIVIDEND_SCALE 1000     /* Scale to avoid rounding error. */

static u64 psecs_to_mts(u64 psecs)
{
        return divide_nint(divide_nint((u64)(2 * 1000000 * DIVIDEND_SCALE),
                                       psecs), DIVIDEND_SCALE);
}

#define WITHIN(v, b, m) (((v) >= ((b) - (m))) && ((v) <= ((b) + (m))))

static unsigned long pretty_psecs_to_mts(u64 psecs)
{
        u64 ret = 0;            // default to error

        if (WITHIN(psecs, 2500, 1))
                ret = 800;
        else if (WITHIN(psecs, 1875, 1))
                ret = 1066;
        else if (WITHIN(psecs, 1500, 1))
                ret = 1333;
        else if (WITHIN(psecs, 1250, 1))
                ret = 1600;
        else if (WITHIN(psecs, 1071, 1))
                ret = 1866;
        else if (WITHIN(psecs, 937, 1))
                ret = 2133;
        else if (WITHIN(psecs, 833, 1))
                ret = 2400;
        else if (WITHIN(psecs, 750, 1))
                ret = 2666;
        return ret;
}

static u64 mts_to_hertz(u64 mts)
{
        return ((mts * 1000 * 1000) / 2);
}

static int compute_rc3x(int64_t tclk_psecs)
{
        long speed;
        long tclk_psecs_min, tclk_psecs_max;
        long data_rate_mhz, data_rate_mhz_min, data_rate_mhz_max;
        int rc3x;

#define ENCODING_BASE 1240

        data_rate_mhz = psecs_to_mts(tclk_psecs);

        /*
         * 2400 MT/s is a special case. Using integer arithmetic it rounds
         * from 833 psecs to 2401 MT/s. Force it to 2400 to pick the
         * proper setting from the table.
         */
        if (tclk_psecs == 833)
                data_rate_mhz = 2400;

        for (speed = ENCODING_BASE; speed < 3200; speed += 20) {
                int error = 0;

                /* Clock in psecs */
                tclk_psecs_min = hertz_to_psecs(mts_to_hertz(speed + 00));
                /* Clock in psecs */
                tclk_psecs_max = hertz_to_psecs(mts_to_hertz(speed + 18));

                data_rate_mhz_min = psecs_to_mts(tclk_psecs_min);
                data_rate_mhz_max = psecs_to_mts(tclk_psecs_max);

                /* Force alingment to multiple to avound rounding errors. */
                data_rate_mhz_min = ((data_rate_mhz_min + 18) / 20) * 20;
                data_rate_mhz_max = ((data_rate_mhz_max + 18) / 20) * 20;

                error += (speed + 00 != data_rate_mhz_min);
                error += (speed + 20 != data_rate_mhz_max);

                rc3x = (speed - ENCODING_BASE) / 20;

                if (data_rate_mhz <= (speed + 20))
                        break;
        }

        return rc3x;
}

/*
 * static global variables needed, so that functions (loops) can be
 * restructured from the main huge function. Its not elegant, but the
 * only way to break the original functions like init_octeon3_ddr3_interface()
 * into separate logical smaller functions with less indentation levels.
 */
static int if_num __section(".data");
static u32 if_mask __section(".data");
static int ddr_hertz __section(".data");

static struct ddr_conf *ddr_conf __section(".data");
static const struct dimm_odt_config *odt_1rank_config __section(".data");
static const struct dimm_odt_config *odt_2rank_config __section(".data");
static const struct dimm_odt_config *odt_4rank_config __section(".data");
static struct dimm_config *dimm_config_table __section(".data");
static const struct dimm_odt_config *odt_config __section(".data");
static const struct ddr3_custom_config *c_cfg __section(".data");

static int odt_idx __section(".data");

static ulong tclk_psecs __section(".data");
static ulong eclk_psecs __section(".data");

static int row_bits __section(".data");
static int col_bits __section(".data");
static int num_banks __section(".data");
static int num_ranks __section(".data");
static int dram_width __section(".data");
static int dimm_count __section(".data");
/* Accumulate and report all the errors before giving up */
static int fatal_error __section(".data");
/* Flag that indicates safe DDR settings should be used */
static int safe_ddr_flag __section(".data");
/* Octeon II Default: 64bit interface width */
static int if_64b __section(".data");
static int if_bytemask __section(".data");
static u32 mem_size_mbytes __section(".data");
static unsigned int didx __section(".data");
static int bank_bits __section(".data");
static int bunk_enable __section(".data");
static int rank_mask __section(".data");
static int column_bits_start __section(".data");
static int row_lsb __section(".data");
static int pbank_lsb __section(".data");
static int use_ecc __section(".data");
static int mtb_psec __section(".data");
static short ftb_dividend __section(".data");
static short ftb_divisor __section(".data");
static int taamin __section(".data");
static int tckmin __section(".data");
static int cl __section(".data");
static int min_cas_latency __section(".data");
static int max_cas_latency __section(".data");
static int override_cas_latency __section(".data");
static int ddr_rtt_nom_auto __section(".data");
static int ddr_rodt_ctl_auto __section(".data");

static int spd_addr __section(".data");
static int spd_org __section(".data");
static int spd_banks __section(".data");
static int spd_rdimm __section(".data");
static int spd_dimm_type __section(".data");
static int spd_ecc __section(".data");
static u32 spd_cas_latency __section(".data");
static int spd_mtb_dividend __section(".data");
static int spd_mtb_divisor __section(".data");
static int spd_tck_min __section(".data");
static int spd_taa_min __section(".data");
static int spd_twr __section(".data");
static int spd_trcd __section(".data");
static int spd_trrd __section(".data");
static int spd_trp __section(".data");
static int spd_tras __section(".data");
static int spd_trc __section(".data");
static int spd_trfc __section(".data");
static int spd_twtr __section(".data");
static int spd_trtp __section(".data");
static int spd_tfaw __section(".data");
static int spd_addr_mirror __section(".data");
static int spd_package __section(".data");
static int spd_rawcard __section(".data");
static int spd_rawcard_aorb __section(".data");
static int spd_rdimm_registers __section(".data");
static int spd_thermal_sensor __section(".data");

static int is_stacked_die __section(".data");
static int is_3ds_dimm __section(".data");
// 3DS: logical ranks per package rank
static int lranks_per_prank __section(".data");
// 3DS: logical ranks bits
static int lranks_bits __section(".data");
// in Mbits; only used for 3DS
static int die_capacity __section(".data");

static enum ddr_type ddr_type __section(".data");

static int twr __section(".data");
static int trcd __section(".data");
static int trrd __section(".data");
static int trp __section(".data");
static int tras __section(".data");
static int trc __section(".data");
static int trfc __section(".data");
static int twtr __section(".data");
static int trtp __section(".data");
static int tfaw __section(".data");

static int ddr4_tckavgmin __section(".data");
static int ddr4_tckavgmax __section(".data");
static int ddr4_trdcmin __section(".data");
static int ddr4_trpmin __section(".data");
static int ddr4_trasmin __section(".data");
static int ddr4_trcmin __section(".data");
static int ddr4_trfc1min __section(".data");
static int ddr4_trfc2min __section(".data");
static int ddr4_trfc4min __section(".data");
static int ddr4_tfawmin __section(".data");
static int ddr4_trrd_smin __section(".data");
static int ddr4_trrd_lmin __section(".data");
static int ddr4_tccd_lmin __section(".data");

static int wl_mask_err __section(".data");
static int wl_loops __section(".data");
static int default_rtt_nom[4] __section(".data");
static int dyn_rtt_nom_mask __section(".data");
static struct impedence_values *imp_val __section(".data");
static char default_rodt_ctl __section(".data");
// default to disabled (ie, try LMC restart, not chip reset)
static int ddr_disable_chip_reset __section(".data");
static const char *dimm_type_name __section(".data");
static int match_wl_rtt_nom __section(".data");

struct hwl_alt_by_rank {
        u16 hwl_alt_mask;       // mask of bytelanes with alternate
        u16 hwl_alt_delay[9];   // bytelane alternate avail if mask=1
};

static struct hwl_alt_by_rank hwl_alts[4] __section(".data");

#define DEFAULT_INTERNAL_VREF_TRAINING_LIMIT 3  // was: 5
static int internal_retries __section(".data");

static int deskew_training_errors __section(".data");
static struct deskew_counts deskew_training_results __section(".data");
static int disable_deskew_training __section(".data");
static int restart_if_dsk_incomplete __section(".data");
static int dac_eval_retries __section(".data");
static int dac_settings[9] __section(".data");
static int num_samples __section(".data");
static int sample __section(".data");
static int lane __section(".data");
static int last_lane __section(".data");
static int total_dac_eval_retries __section(".data");
static int dac_eval_exhausted __section(".data");

#define DEFAULT_DAC_SAMPLES 7   // originally was 5
#define DAC_RETRIES_LIMIT   2

struct bytelane_sample {
        s16 bytes[DEFAULT_DAC_SAMPLES];
};

static struct bytelane_sample lanes[9] __section(".data");

static char disable_sequential_delay_check __section(".data");
static int wl_print __section(".data");

static int enable_by_rank_init __section(".data");
static int saved_rank_mask __section(".data");
static int by_rank __section(".data");
static struct deskew_data rank_dsk[4] __section(".data");
static struct dac_data rank_dac[4] __section(".data");

// todo: perhaps remove node at some time completely?
static int node __section(".data");
static int base_cl __section(".data");

/* Parameters from DDR3 Specifications */
#define DDR3_TREFI         7800000      /* 7.8 us */
#define DDR3_ZQCS          80000ull     /* 80 ns */
#define DDR3_ZQCS_INTERNAL 1280000000ull        /* 128ms/100 */
#define DDR3_TCKE          5000 /* 5 ns */
#define DDR3_TMRD          4    /* 4 nCK */
#define DDR3_TDLLK         512  /* 512 nCK */
#define DDR3_TMPRR         1    /* 1 nCK */
#define DDR3_TWLMRD        40   /* 40 nCK */
#define DDR3_TWLDQSEN      25   /* 25 nCK */

/* Parameters from DDR4 Specifications */
#define DDR4_TMRD          8    /* 8 nCK */
#define DDR4_TDLLK         768  /* 768 nCK */

static void lmc_config(struct ddr_priv *priv)
{
        union cvmx_lmcx_config cfg;
        char *s;

        cfg.u64 = 0;

        cfg.cn78xx.ecc_ena = use_ecc;
        cfg.cn78xx.row_lsb = encode_row_lsb_ddr3(row_lsb);
        cfg.cn78xx.pbank_lsb = encode_pbank_lsb_ddr3(pbank_lsb);

        cfg.cn78xx.idlepower = 0;       /* Disabled */

        s = lookup_env(priv, "ddr_idlepower");
        if (s)
                cfg.cn78xx.idlepower = simple_strtoul(s, NULL, 0);

        cfg.cn78xx.forcewrite = 0;      /* Disabled */
        /* Include memory reference address in the ECC */
        cfg.cn78xx.ecc_adr = 1;

        s = lookup_env(priv, "ddr_ecc_adr");
        if (s)
                cfg.cn78xx.ecc_adr = simple_strtoul(s, NULL, 0);

        cfg.cn78xx.reset = 0;

        /*
         * Program LMC0_CONFIG[24:18], ref_zqcs_int(6:0) to
         * RND-DN(tREFI/clkPeriod/512) Program LMC0_CONFIG[36:25],
         * ref_zqcs_int(18:7) to
         * RND-DN(ZQCS_Interval/clkPeriod/(512*128)). Note that this
         * value should always be greater than 32, to account for
         * resistor calibration delays.
         */

        cfg.cn78xx.ref_zqcs_int = ((DDR3_TREFI / tclk_psecs / 512) & 0x7f);
        cfg.cn78xx.ref_zqcs_int |=
                ((max(33ull, (DDR3_ZQCS_INTERNAL / (tclk_psecs / 100) /
                              (512 * 128))) & 0xfff) << 7);

        cfg.cn78xx.early_dqx = 1;       /* Default to enabled */

        s = lookup_env(priv, "ddr_early_dqx");
        if (!s)
                s = lookup_env(priv, "ddr%d_early_dqx", if_num);

        if (s)
                cfg.cn78xx.early_dqx = simple_strtoul(s, NULL, 0);

        cfg.cn78xx.sref_with_dll = 0;

        cfg.cn78xx.rank_ena = bunk_enable;
        cfg.cn78xx.rankmask = rank_mask;        /* Set later */
        cfg.cn78xx.mirrmask = (spd_addr_mirror << 1 | spd_addr_mirror << 3) &
                rank_mask;
        /* Set once and don't change it. */
        cfg.cn78xx.init_status = rank_mask;
        cfg.cn78xx.early_unload_d0_r0 = 0;
        cfg.cn78xx.early_unload_d0_r1 = 0;
        cfg.cn78xx.early_unload_d1_r0 = 0;
        cfg.cn78xx.early_unload_d1_r1 = 0;
        cfg.cn78xx.scrz = 0;
        if (octeon_is_cpuid(OCTEON_CN70XX))
                cfg.cn78xx.mode32b = 1; /* Read-only. Always 1. */
        cfg.cn78xx.mode_x4dev = (dram_width == 4) ? 1 : 0;
        cfg.cn78xx.bg2_enable = ((ddr_type == DDR4_DRAM) &&
                                 (dram_width == 16)) ? 0 : 1;

        s = lookup_env_ull(priv, "ddr_config");
        if (s)
                cfg.u64 = simple_strtoull(s, NULL, 0);
        debug("LMC_CONFIG                                    : 0x%016llx\n",
              cfg.u64);
        lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), cfg.u64);
}

static void lmc_control(struct ddr_priv *priv)
{
        union cvmx_lmcx_control ctrl;
        char *s;

        ctrl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num));
        ctrl.s.rdimm_ena = spd_rdimm;
        ctrl.s.bwcnt = 0;       /* Clear counter later */
        if (spd_rdimm)
                ctrl.s.ddr2t = (safe_ddr_flag ? 1 : c_cfg->ddr2t_rdimm);
        else
                ctrl.s.ddr2t = (safe_ddr_flag ? 1 : c_cfg->ddr2t_udimm);
        ctrl.s.pocas = 0;
        ctrl.s.fprch2 = (safe_ddr_flag ? 2 : c_cfg->fprch2);
        ctrl.s.throttle_rd = safe_ddr_flag ? 1 : 0;
        ctrl.s.throttle_wr = safe_ddr_flag ? 1 : 0;
        ctrl.s.inorder_rd = safe_ddr_flag ? 1 : 0;
        ctrl.s.inorder_wr = safe_ddr_flag ? 1 : 0;
        ctrl.s.elev_prio_dis = safe_ddr_flag ? 1 : 0;
        /* discards writes to addresses that don't exist in the DRAM */
        ctrl.s.nxm_write_en = 0;
        ctrl.s.max_write_batch = 8;
        ctrl.s.xor_bank = 1;
        ctrl.s.auto_dclkdis = 1;
        ctrl.s.int_zqcs_dis = 0;
        ctrl.s.ext_zqcs_dis = 0;
        ctrl.s.bprch = 1;
        ctrl.s.wodt_bprch = 1;
        ctrl.s.rodt_bprch = 1;

        s = lookup_env(priv, "ddr_xor_bank");
        if (s)
                ctrl.s.xor_bank = simple_strtoul(s, NULL, 0);

        s = lookup_env(priv, "ddr_2t");
        if (s)
                ctrl.s.ddr2t = simple_strtoul(s, NULL, 0);

        s = lookup_env(priv, "ddr_fprch2");
        if (s)
                ctrl.s.fprch2 = simple_strtoul(s, NULL, 0);

        s = lookup_env(priv, "ddr_bprch");
        if (s)
                ctrl.s.bprch = simple_strtoul(s, NULL, 0);

        s = lookup_env(priv, "ddr_wodt_bprch");
        if (s)
                ctrl.s.wodt_bprch = simple_strtoul(s, NULL, 0);

        s = lookup_env(priv, "ddr_rodt_bprch");
        if (s)
                ctrl.s.rodt_bprch = simple_strtoul(s, NULL, 0);

        s = lookup_env(priv, "ddr_int_zqcs_dis");
        if (s)
                ctrl.s.int_zqcs_dis = simple_strtoul(s, NULL, 0);

        s = lookup_env(priv, "ddr_ext_zqcs_dis");
        if (s)
                ctrl.s.ext_zqcs_dis = simple_strtoul(s, NULL, 0);

        s = lookup_env_ull(priv, "ddr_control");
        if (s)
                ctrl.u64 = simple_strtoull(s, NULL, 0);

        debug("LMC_CONTROL                                   : 0x%016llx\n",
              ctrl.u64);
        lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctrl.u64);
}

static void lmc_timing_params0(struct ddr_priv *priv)
{
        union cvmx_lmcx_timing_params0 tp0;
        unsigned int trp_value;
        char *s;

        tp0.u64 = lmc_rd(priv, CVMX_LMCX_TIMING_PARAMS0(if_num));

        trp_value = divide_roundup(trp, tclk_psecs) - 1;
        debug("TIMING_PARAMS0[TRP]: NEW 0x%x, OLD 0x%x\n", trp_value,
              trp_value +
              (unsigned int)(divide_roundup(max(4ull * tclk_psecs, 7500ull),
                                            tclk_psecs)) - 4);
        s = lookup_env_ull(priv, "ddr_use_old_trp");
        if (s) {
                if (!!simple_strtoull(s, NULL, 0)) {
                        trp_value +=
                            divide_roundup(max(4ull * tclk_psecs, 7500ull),
                                           tclk_psecs) - 4;
                        debug("TIMING_PARAMS0[trp]: USING OLD 0x%x\n",
                              trp_value);
                }
        }

        tp0.cn78xx.txpr =
            divide_roundup(max(5ull * tclk_psecs, trfc + 10000ull),
                           16 * tclk_psecs);
        tp0.cn78xx.trp = trp_value & 0x1f;
        tp0.cn78xx.tcksre =
            divide_roundup(max(5ull * tclk_psecs, 10000ull), tclk_psecs) - 1;

        if (ddr_type == DDR4_DRAM) {
                int tzqinit = 4;        // Default to 4, for all DDR4 speed bins

                s = lookup_env(priv, "ddr_tzqinit");
                if (s)
                        tzqinit = simple_strtoul(s, NULL, 0);

                tp0.cn78xx.tzqinit = tzqinit;
                /* Always 8. */
                tp0.cn78xx.tzqcs = divide_roundup(128 * tclk_psecs,
                                                  (16 * tclk_psecs));
                tp0.cn78xx.tcke =
                    divide_roundup(max(3 * tclk_psecs, (ulong)DDR3_TCKE),
                                   tclk_psecs) - 1;
                tp0.cn78xx.tmrd =
                    divide_roundup((DDR4_TMRD * tclk_psecs), tclk_psecs) - 1;
                tp0.cn78xx.tmod = 25;   /* 25 is the max allowed */
                tp0.cn78xx.tdllk = divide_roundup(DDR4_TDLLK, 256);
        } else {
                tp0.cn78xx.tzqinit =
                    divide_roundup(max(512ull * tclk_psecs, 640000ull),
                                   (256 * tclk_psecs));
                tp0.cn78xx.tzqcs =
                    divide_roundup(max(64ull * tclk_psecs, DDR3_ZQCS),
                                   (16 * tclk_psecs));
                tp0.cn78xx.tcke = divide_roundup(DDR3_TCKE, tclk_psecs) - 1;
                tp0.cn78xx.tmrd =
                    divide_roundup((DDR3_TMRD * tclk_psecs), tclk_psecs) - 1;
                tp0.cn78xx.tmod =
                    divide_roundup(max(12ull * tclk_psecs, 15000ull),
                                   tclk_psecs) - 1;
                tp0.cn78xx.tdllk = divide_roundup(DDR3_TDLLK, 256);
        }

        s = lookup_env_ull(priv, "ddr_timing_params0");
        if (s)
                tp0.u64 = simple_strtoull(s, NULL, 0);
        debug("TIMING_PARAMS0                                : 0x%016llx\n",
              tp0.u64);
        lmc_wr(priv, CVMX_LMCX_TIMING_PARAMS0(if_num), tp0.u64);
}

static void lmc_timing_params1(struct ddr_priv *priv)
{
        union cvmx_lmcx_timing_params1 tp1;
        unsigned int txp, temp_trcd, trfc_dlr;
        char *s;

        tp1.u64 = lmc_rd(priv, CVMX_LMCX_TIMING_PARAMS1(if_num));

        /* .cn70xx. */
        tp1.s.tmprr = divide_roundup(DDR3_TMPRR * tclk_psecs, tclk_psecs) - 1;

        tp1.cn78xx.tras = divide_roundup(tras, tclk_psecs) - 1;

        temp_trcd = divide_roundup(trcd, tclk_psecs);
        if (temp_trcd > 15) {
                debug("TIMING_PARAMS1[trcd]: need extension bit for 0x%x\n",
                      temp_trcd);
        }
        if (octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) && temp_trcd > 15) {
                /*
                 * Let .trcd=0 serve as a flag that the field has
                 * overflowed. Must use Additive Latency mode as a
                 * workaround.
                 */
                temp_trcd = 0;
        }
        tp1.cn78xx.trcd = (temp_trcd >> 0) & 0xf;
        tp1.cn78xx.trcd_ext = (temp_trcd >> 4) & 0x1;