LXR uboot/drivers/ddr/marvell/axp/ddr3_read

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) Marvell International Ltd. and its affiliates
   4 */
   5
   6#include <common.h>
   7#include <i2c.h>
   8#include <spl.h>
   9#include <asm/io.h>
  10#include <asm/arch/cpu.h>
  11#include <asm/arch/soc.h>
  12
  13#include "ddr3_hw_training.h"
  14
  15/*
  16 * Debug
  17 */
  18#define DEBUG_RL_C(s, d, l) \
  19        DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n")
  20#define DEBUG_RL_FULL_C(s, d, l) \
  21        DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n")
  22
  23#ifdef MV_DEBUG_RL
  24#define DEBUG_RL_S(s) \
  25        debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  26#define DEBUG_RL_D(d, l) \
  27        debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  28#else
  29#define DEBUG_RL_S(s)
  30#define DEBUG_RL_D(d, l)
  31#endif
  32
  33#ifdef MV_DEBUG_RL_FULL
  34#define DEBUG_RL_FULL_S(s)              puts(s)
  35#define DEBUG_RL_FULL_D(d, l)           printf("%x", d)
  36#else
  37#define DEBUG_RL_FULL_S(s)
  38#define DEBUG_RL_FULL_D(d, l)
  39#endif
  40
  41extern u32 rl_pattern[LEN_STD_PATTERN];
  42
  43#ifdef RL_MODE
  44static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
  45                                                int ratio_2to1, u32 ecc,
  46                                                MV_DRAM_INFO *dram_info);
  47#else
  48static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
  49                                                    int ratio_2to1, u32 ecc,
  50                                                    MV_DRAM_INFO *dram_info);
  51#endif
  52
  53/*
  54 * Name:     ddr3_read_leveling_hw
  55 * Desc:     Execute the Read leveling phase by HW
  56 * Args:     dram_info - main struct
  57 *           freq      - current sequence frequency
  58 * Notes:
  59 * Returns:  MV_OK if success, MV_FAIL if fail.
  60 */
  61int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  62{
  63        u32 reg;
  64
  65        /* Debug message - Start Read leveling procedure */
  66        DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n");
  67
  68        /* Start Auto Read Leveling procedure */
  69        reg = 1 << REG_DRAM_TRAINING_RL_OFFS;
  70        /* Config the retest number */
  71        reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS);
  72
  73        /* Enable CS in the automatic process */
  74        reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS);
  75
  76        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  77
  78        reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  79                (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  80        reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  81
  82        /* Wait */
  83        do {
  84                reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  85                        (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  86        } while (reg);          /* Wait for '0' */
  87
  88        /* Check if Successful */
  89        if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  90            (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
  91                u32 delay, phase, pup, cs;
  92
  93                dram_info->rl_max_phase = 0;
  94                dram_info->rl_min_phase = 10;
  95
  96                /* Read results to arrays */
  97                for (cs = 0; cs < MAX_CS; cs++) {
  98                        if (dram_info->cs_ena & (1 << cs)) {
  99                                for (pup = 0;
 100                                     pup < dram_info->num_of_total_pups;
 101                                     pup++) {
 102                                        if (pup == dram_info->num_of_std_pups
 103                                            && dram_info->ecc_ena)
 104                                                pup = ECC_PUP;
 105                                        reg =
 106                                            ddr3_read_pup_reg(PUP_RL_MODE, cs,
 107                                                              pup);
 108                                        phase = (reg >> REG_PHY_PHASE_OFFS) &
 109                                                PUP_PHASE_MASK;
 110                                        delay = reg & PUP_DELAY_MASK;
 111                                        dram_info->rl_val[cs][pup][P] = phase;
 112                                        if (phase > dram_info->rl_max_phase)
 113                                                dram_info->rl_max_phase = phase;
 114                                        if (phase < dram_info->rl_min_phase)
 115                                                dram_info->rl_min_phase = phase;
 116                                        dram_info->rl_val[cs][pup][D] = delay;
 117                                        dram_info->rl_val[cs][pup][S] =
 118                                            RL_FINAL_STATE;
 119                                        reg =
 120                                            ddr3_read_pup_reg(PUP_RL_MODE + 0x1,
 121                                                              cs, pup);
 122                                        dram_info->rl_val[cs][pup][DQS] =
 123                                            (reg & 0x3F);
 124                                }
 125#ifdef MV_DEBUG_RL
 126                                /* Print results */
 127                                DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ",
 128                                           (u32) cs, 1);
 129
 130                                for (pup = 0;
 131                                     pup < (dram_info->num_of_total_pups);
 132                                     pup++) {
 133                                        if (pup == dram_info->num_of_std_pups
 134                                            && dram_info->ecc_ena)
 135                                                pup = ECC_PUP;
 136                                        DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
 137                                        DEBUG_RL_D((u32) pup, 1);
 138                                        DEBUG_RL_S(", Phase: ");
 139                                        DEBUG_RL_D((u32) dram_info->
 140                                                   rl_val[cs][pup][P], 1);
 141                                        DEBUG_RL_S(", Delay: ");
 142                                        DEBUG_RL_D((u32) dram_info->
 143                                                   rl_val[cs][pup][D], 2);
 144                                        DEBUG_RL_S("\n");
 145                                }
 146#endif
 147                        }
 148                }
 149
 150                dram_info->rd_rdy_dly =
 151                        reg_read(REG_READ_DATA_READY_DELAYS_ADDR) &
 152                        REG_READ_DATA_SAMPLE_DELAYS_MASK;
 153                dram_info->rd_smpl_dly =
 154                        reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) &
 155                        REG_READ_DATA_READY_DELAYS_MASK;
 156#ifdef MV_DEBUG_RL
 157                DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
 158                           dram_info->rd_smpl_dly, 2);
 159                DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
 160                           dram_info->rd_rdy_dly, 2);
 161                DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n");
 162#endif
 163                return MV_OK;
 164
 165        } else {
 166                DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n");
 167                return MV_FAIL;
 168        }
 169}
 170
 171/*
 172 * Name:     ddr3_read_leveling_sw
 173 * Desc:     Execute the Read leveling phase by SW
 174 * Args:     dram_info - main struct
 175 *           freq      - current sequence frequency
 176 * Notes:
 177 * Returns:  MV_OK if success, MV_FAIL if fail.
 178 */
 179int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
 180{
 181        u32 reg, cs, ecc, pup_num, phase, delay, pup;
 182        int status;
 183
 184        /* Debug message - Start Read leveling procedure */
 185        DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n");
 186
 187        /* Enable SW Read Leveling */
 188        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 189                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 190        reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS);
 191        /* [0]=1 - Enable SW override  */
 192        /* 0x15B8 - Training SW 2 Register */
 193        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 194
 195#ifdef RL_MODE
 196        reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
 197                (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 198        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 199#endif
 200
 201        /* Loop for each CS */
 202        for (cs = 0; cs < dram_info->num_cs; cs++) {
 203                DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1);
 204
 205                for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) {
 206                        /* ECC Support - Switch ECC Mux on ecc=1 */
 207                        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 208                                ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 209                        reg |= (dram_info->ecc_ena *
 210                                ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 211                        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 212
 213                        if (ecc)
 214                                DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n");
 215                        else
 216                                DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n");
 217
 218                        /* Set current sample delays */
 219                        reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
 220                        reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
 221                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
 222                        reg |= (dram_info->cl <<
 223                                (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
 224                        reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg);
 225
 226                        /* Set current Ready delay */
 227                        reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 228                        reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 229                                 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 230                        if (!ratio_2to1) {
 231                                /* 1:1 mode */
 232                                reg |= ((dram_info->cl + 1) <<
 233                                        (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 234                        } else {
 235                                /* 2:1 mode */
 236                                reg |= ((dram_info->cl + 2) <<
 237                                        (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 238                        }
 239                        reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 240
 241                        /* Read leveling Single CS[cs] */
 242#ifdef RL_MODE
 243                        status =
 244                            ddr3_read_leveling_single_cs_rl_mode(cs, freq,
 245                                                                 ratio_2to1,
 246                                                                 ecc,
 247                                                                 dram_info);
 248                        if (MV_OK != status)
 249                                return status;
 250#else
 251                        status =
 252                            ddr3_read_leveling_single_cs_window_mode(cs, freq,
 253                                                                     ratio_2to1,
 254                                                                     ecc,
 255                                                                     dram_info)
 256                            if (MV_OK != status)
 257                                return status;
 258#endif
 259                }
 260
 261                /* Print results */
 262                DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs,
 263                           1);
 264
 265                for (pup = 0;
 266                     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
 267                     pup++) {
 268                        DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
 269                        DEBUG_RL_D((u32) pup, 1);
 270                        DEBUG_RL_S(", Phase: ");
 271                        DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1);
 272                        DEBUG_RL_S(", Delay: ");
 273                        DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2);
 274                        DEBUG_RL_S("\n");
 275                }
 276
 277                DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
 278                           dram_info->rd_smpl_dly, 2);
 279                DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
 280                           dram_info->rd_rdy_dly, 2);
 281
 282                /* Configure PHY with average of 3 locked leveling settings */
 283                for (pup = 0;
 284                     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
 285                     pup++) {
 286                        /* ECC support - bit 8 */
 287                        pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
 288
 289                        /* For now, set last cnt result */
 290                        phase = dram_info->rl_val[cs][pup][P];
 291                        delay = dram_info->rl_val[cs][pup][D];
 292                        ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,
 293                                           delay);
 294                }
 295        }
 296
 297        /* Reset PHY read FIFO */
 298        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 299                (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 300        /* 0x15B8 - Training SW 2 Register */
 301        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 302
 303        do {
 304                reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 305                        (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 306        } while (reg);          /* Wait for '0' */
 307
 308        /* ECC Support - Switch ECC Mux off ecc=0 */
 309        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 310                ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 311        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 312
 313#ifdef RL_MODE
 314        reg_write(REG_DRAM_TRAINING_ADDR, 0);   /* 0x15B0 - Training Register */
 315#endif
 316
 317        /* Disable SW Read Leveling */
 318        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 319                ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 320        /* [0] = 0 - Disable SW override  */
 321        reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS));
 322        /* [3] = 1 - Disable RL MODE */
 323        /* 0x15B8 - Training SW 2 Register */
 324        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 325
 326        DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n");
 327        return MV_OK;
 328}
 329
 330#ifdef RL_MODE
 331/*
 332 * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode().
 333 * This just got too much indented making it hard to read / edit.
 334 */
 335static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups,
 336                    u32 *locked_sum, u32 ecc, int *first_octet_locked,
 337                    int *counter_in_progress, int final_delay, u32 delay,
 338                    u32 phase)
 339{
 340        /* If no OverRun */
 341        if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) {
 342                int idx;
 343
 344                idx = pup + ecc * ECC_BIT;
 345
 346                /* PUP passed, start examining */
 347                if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) {
 348                        /* Must be RL_UNLOCK_STATE */
 349                        /* Match expected value ? - Update State Machine */
 350                        if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) {
 351                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
 352                                                (u32)pup, 1);
 353                                info->rl_val[cs][idx][C]++;
 354
 355                                /* If pup got to last state - lock the delays */
 356                                if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) {
 357                                        info->rl_val[cs][idx][C] = 0;
 358                                        info->rl_val[cs][idx][DS] = delay;
 359                                        info->rl_val[cs][idx][PS] = phase;
 360
 361                                        /* Go to Final State */
 362                                        info->rl_val[cs][idx][S] = RL_FINAL_STATE;
 363                                        *locked_sum = *locked_sum + 1;
 364                                        DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ",
 365                                                        (u32)pup, 1);
 366
 367                                        /*
 368                                         * If first lock - need to lock delays
 369                                         */
 370                                        if (*first_octet_locked == 0) {
 371                                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
 372                                                                (u32)pup, 1);
 373                                                *first_octet_locked = 1;
 374                                        }
 375
 376                                        /*
 377                                         * If pup is in not in final state but
 378                                         * there was match - dont increment
 379                                         * counter
 380                                         */
 381                                } else {
 382                                        *counter_in_progress = 1;
 383                                }
 384                        }
 385                }
 386        }
 387}
 388
 389/*
 390 * Name:     ddr3_read_leveling_single_cs_rl_mode
 391 * Desc:     Execute Read leveling for single Chip select
 392 * Args:     cs        - current chip select
 393 *           freq      - current sequence frequency
 394 *           ecc       - ecc iteration indication
 395 *           dram_info - main struct
 396 * Notes:
 397 * Returns:  MV_OK if success, MV_FAIL if fail.
 398 */
 399static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
 400                                                int ratio_2to1, u32 ecc,
 401                                                MV_DRAM_INFO *dram_info)
 402{
 403        u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,
 404                repeat_max_cnt, sdram_offset, locked_sum;
 405        u32 phase_min, ui_max_delay;
 406        int all_locked, first_octet_locked, counter_in_progress;
 407        int final_delay = 0;
 408
 409        DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
 410
 411        /* Init values */
 412        phase = 0;
 413        delay = 0;
 414        rd_sample_delay = dram_info->cl;
 415        all_locked = 0;
 416        first_octet_locked = 0;
 417        repeat_max_cnt = 0;
 418        locked_sum = 0;
 419
 420        for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 421             pup++)
 422                dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
 423
 424        /* Main loop */
 425        while (!all_locked) {
 426                counter_in_progress = 0;
 427
 428                DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
 429                DEBUG_RL_FULL_D(rd_sample_delay, 2);
 430                DEBUG_RL_FULL_S(", RdRdyDly = ");
 431                DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
 432                DEBUG_RL_FULL_S(", Phase = ");
 433                DEBUG_RL_FULL_D(phase, 1);
 434                DEBUG_RL_FULL_S(", Delay = ");
 435                DEBUG_RL_FULL_D(delay, 2);
 436                DEBUG_RL_FULL_S("\n");
 437
 438                /*
 439                 * Broadcast to all PUPs current RL delays: DQS phase,
 440                 * leveling delay
 441                 */
 442                ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
 443
 444                /* Reset PHY read FIFO */
 445                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 446                        (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 447                /* 0x15B8 - Training SW 2 Register */
 448                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 449
 450                do {
 451                        reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 452                                (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 453                } while (reg);  /* Wait for '0' */
 454
 455                /* Read pattern from SDRAM */
 456                sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
 457                locked_pups = 0;
 458                if (MV_OK !=
 459                    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
 460                                       rl_pattern, LEN_STD_PATTERN,
 461                                       sdram_offset, 0, 0, NULL, 0))
 462                        return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN;
 463
 464                /* Octet evaluation */
 465                /* pup_num = Q or 1 for ECC */
 466                for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 467                        /* Check Overrun */
 468                        if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
 469                               (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) {
 470                                overrun(cs, dram_info, pup, locked_pups,
 471                                        &locked_sum, ecc, &first_octet_locked,
 472                                        &counter_in_progress, final_delay,
 473                                        delay, phase);
 474                        } else {
 475                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
 476                                                (u32)pup, 1);
 477                        }
 478                }
 479
 480                if (locked_sum == (dram_info->num_of_std_pups *
 481                                   (1 - ecc) + ecc)) {
 482                        all_locked = 1;
 483                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
 484                }
 485
 486                /*
 487                 * This is a fix for unstable condition where pups are
 488                 * toggling between match and no match
 489                 */
 490                /*
 491                 * If some of the pups is >1 <3, check if we did it too
 492                 * many times
 493                 */
 494                if (counter_in_progress == 1) {
 495                        /* Notify at least one Counter is >=1 and < 3 */
 496                        if (repeat_max_cnt < RL_RETRY_COUNT) {
 497                                repeat_max_cnt++;
 498                                counter_in_progress = 1;
 499                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
 500                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
 501                        } else {
 502                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
 503                                counter_in_progress = 0;
 504                        }
 505                }
 506
 507                /*
 508                 * Check some of the pups are in the middle of state machine
 509                 * and don't increment the delays
 510                 */
 511                if (!counter_in_progress && !all_locked) {
 512                        int idx;
 513
 514                        idx = pup + ecc * ECC_BIT;
 515
 516                        repeat_max_cnt = 0;
 517                        /* if 1:1 mode */
 518                        if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))
 519                                ui_max_delay = MAX_DELAY_INV;
 520                        else
 521                                ui_max_delay = MAX_DELAY;
 522
 523                        /* Increment Delay */
 524                        if (delay < ui_max_delay) {
 525                                delay++;
 526                                /*
 527                                 * Mark the last delay/pahse place for
 528                                 * window final place
 529                                 */
 530                                if (delay == ui_max_delay) {
 531                                        if ((!ratio_2to1 && phase ==
 532                                             MAX_PHASE_RL_L_1TO1)
 533                                            || (ratio_2to1 && phase ==
 534                                                MAX_PHASE_RL_L_2TO1))
 535                                                final_delay = 1;
 536                                }
 537                        } else {
 538                                /* Phase+CL Incrementation */
 539                                delay = 0;
 540
 541                                if (!ratio_2to1) {
 542                                        /* 1:1 mode */
 543                                        if (first_octet_locked) {
 544                                                /* some Pup was Locked */
 545                                                if (phase < MAX_PHASE_RL_L_1TO1) {
 546                                                        if (phase == 1) {
 547                                                                phase = 4;
 548                                                        } else {
 549                                                                phase++;
 550                                                                delay = MIN_DELAY_PHASE_1_LIMIT;
 551                                                        }
 552                                                } else {
 553                                                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 554                                                        DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n");
 555                                                        return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
 556                                                }
 557                                        } else {
 558                                                /* NO Pup was Locked */
 559                                                if (phase < MAX_PHASE_RL_UL_1TO1) {
 560                                                        phase++;
 561                                                        delay =
 562                                                            MIN_DELAY_PHASE_1_LIMIT;
 563                                                } else {
 564                                                        phase = 0;
 565                                                }
 566                                        }
 567                                } else {
 568                                        /* 2:1 mode */
 569                                        if (first_octet_locked) {
 570                                                /* some Pup was Locked */
 571                                                if (phase < MAX_PHASE_RL_L_2TO1) {
 572                                                        phase++;
 573                                                } else {
 574                                                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 575                                                        DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 576                                                        for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 577                                                                /* pup_num = Q or 1 for ECC */
 578                                                                if (dram_info->rl_val[cs][idx][S]
 579                                                                    == 0) {
 580                                                                        DEBUG_RL_C("Failed byte is = ",
 581                                                                                   pup, 1);
 582                                                                }
 583                                                        }
 584                                                        return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
 585                                                }
 586                                        } else {
 587                                                /* No Pup was Locked */
 588                                                if (phase < MAX_PHASE_RL_UL_2TO1)
 589                                                        phase++;
 590                                                else
 591                                                        phase = 0;
 592                                        }
 593                                }
 594
 595                                /*
 596                                 * If we finished a full Phases cycle (so now
 597                                 * phase = 0, need to increment rd_sample_dly
 598                                 */
 599                                if (phase == 0 && first_octet_locked == 0) {
 600                                        rd_sample_delay++;
 601                                        if (rd_sample_delay == 0x10) {
 602                                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 603                                                DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 604                                                for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 605                                                        /* pup_num = Q or 1 for ECC */
 606                                                        if (dram_info->
 607                                                            rl_val[cs][idx][S] == 0) {
 608                                                                DEBUG_RL_C("Failed byte is = ",
 609                                                                           pup, 1);
 610                                                        }
 611                                                }
 612                                                return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK;
 613                                        }
 614
 615                                        /* Set current rd_sample_delay  */
 616                                        reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
 617                                        reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK
 618                                              << (REG_READ_DATA_SAMPLE_DELAYS_OFFS
 619                                                  * cs));
 620                                        reg |= (rd_sample_delay <<
 621                                                (REG_READ_DATA_SAMPLE_DELAYS_OFFS *
 622                                                 cs));
 623                                        reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
 624                                                  reg);
 625                                }
 626
 627                                /*
 628                                 * Set current rdReadyDelay according to the
 629                                 * hash table (Need to do this in every phase
 630                                 * change)
 631                                 */
 632                                if (!ratio_2to1) {
 633                                        /* 1:1 mode */
 634                                        add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
 635                                        switch (phase) {
 636                                        case 0:
 637                                                add = (add >>
 638                                                       REG_TRAINING_DEBUG_2_OFFS);
 639                                                break;
 640                                        case 1:
 641                                                add = (add >>
 642                                                       (REG_TRAINING_DEBUG_2_OFFS
 643                                                        + 3));
 644                                                break;
 645                                        case 4:
 646                                                add = (add >>
 647                                                       (REG_TRAINING_DEBUG_2_OFFS
 648                                                        + 6));
 649                                                break;
 650                                        case 5:
 651                                                add = (add >>
 652                                                       (REG_TRAINING_DEBUG_2_OFFS
 653                                                        + 9));
 654                                                break;
 655                                        }
 656                                        add &= REG_TRAINING_DEBUG_2_MASK;
 657                                } else {
 658                                        /* 2:1 mode */
 659                                        add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
 660                                        add = (add >>
 661                                               (phase *
 662                                                REG_TRAINING_DEBUG_3_OFFS));
 663                                        add &= REG_TRAINING_DEBUG_3_MASK;
 664                                }
 665
 666                                reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 667                                reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 668                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 669                                reg |= ((rd_sample_delay + add) <<
 670                                        (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 671                                reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 672                                dram_info->rd_smpl_dly = rd_sample_delay;
 673                                dram_info->rd_rdy_dly = rd_sample_delay + add;
 674                        }
 675
 676                        /* Reset counters for pups with states<RD_STATE_COUNT */
 677                        for (pup = 0; pup <
 678                                     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 679                             pup++) {
 680                                if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
 681                                        dram_info->rl_val[cs][idx][C] = 0;
 682                        }
 683                }
 684        }
 685
 686        phase_min = 10;
 687
 688        for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
 689                if (dram_info->rl_val[cs][pup][PS] < phase_min)
 690                        phase_min = dram_info->rl_val[cs][pup][PS];
 691        }
 692
 693        /*
 694         * Set current rdReadyDelay according to the hash table (Need to
 695         * do this in every phase change)
 696         */
 697        if (!ratio_2to1) {
 698                /* 1:1 mode */
 699                add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
 700                switch (phase_min) {
 701                case 0:
 702                        add = (add >> REG_TRAINING_DEBUG_2_OFFS);
 703                        break;
 704                case 1:
 705                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
 706                        break;
 707                case 4:
 708                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
 709                        break;
 710                case 5:
 711                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
 712                        break;
 713                }
 714                add &= REG_TRAINING_DEBUG_2_MASK;
 715        } else {
 716                /* 2:1 mode */
 717                add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
 718                add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS));
 719                add &= REG_TRAINING_DEBUG_3_MASK;
 720        }
 721
 722        reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 723        reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 724                 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 725        reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 726        reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 727        dram_info->rd_rdy_dly = rd_sample_delay + add;
 728
 729        for (cs = 0; cs < dram_info->num_cs; cs++) {
 730                for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
 731                        reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
 732                        dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
 733                }
 734        }
 735
 736        return MV_OK;
 737}
 738
 739#else
 740
 741/*
 742 * Name:     ddr3_read_leveling_single_cs_window_mode
 743 * Desc:     Execute Read leveling for single Chip select
 744 * Args:     cs        - current chip select
 745 *           freq      - current sequence frequency
 746 *           ecc       - ecc iteration indication
 747 *           dram_info - main struct
 748 * Notes:
 749 * Returns:  MV_OK if success, MV_FAIL if fail.
 750 */
 751static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
 752                                                    int ratio_2to1, u32 ecc,
 753                                                    MV_DRAM_INFO *dram_info)
 754{
 755        u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,
 756            repeat_max_cnt, sdram_offset, final_sum, locked_sum;
 757        u32 delay_s, delay_e, tmp, phase_min, ui_max_delay;
 758        int all_locked, first_octet_locked, counter_in_progress;
 759        int final_delay = 0;
 760
 761        DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
 762
 763        /* Init values */
 764        phase = 0;
 765        delay = 0;
 766        rd_sample_delay = dram_info->cl;
 767        all_locked = 0;
 768        first_octet_locked = 0;
 769        repeat_max_cnt = 0;
 770        sum = 0;
 771        final_sum = 0;
 772        locked_sum = 0;
 773
 774        for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 775             pup++)
 776                dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
 777
 778        /* Main loop */
 779        while (!all_locked) {
 780                counter_in_progress = 0;
 781
 782                DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
 783                DEBUG_RL_FULL_D(rd_sample_delay, 2);
 784                DEBUG_RL_FULL_S(", RdRdyDly = ");
 785                DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
 786                DEBUG_RL_FULL_S(", Phase = ");
 787                DEBUG_RL_FULL_D(phase, 1);
 788                DEBUG_RL_FULL_S(", Delay = ");
 789                DEBUG_RL_FULL_D(delay, 2);
 790                DEBUG_RL_FULL_S("\n");
 791
 792                /*
 793                 * Broadcast to all PUPs current RL delays: DQS phase,leveling
 794                 * delay
 795                 */
 796                ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
 797
 798                /* Reset PHY read FIFO */
 799                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 800                        (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 801                /* 0x15B8 - Training SW 2 Register */
 802                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 803
 804                do {
 805                        reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 806                                (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 807                } while (reg);  /* Wait for '0' */
 808
 809                /* Read pattern from SDRAM */
 810                sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
 811                locked_pups = 0;
 812                if (MV_OK !=
 813                    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
 814                                       rl_pattern, LEN_STD_PATTERN,
 815                                       sdram_offset, 0, 0, NULL, 0))
 816                        return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN;
 817
 818                /* Octet evaluation */
 819                for (pup = 0; pup < (dram_info->num_of_std_pups *
 820                                     (1 - ecc) + ecc); pup++) {
 821                        /* pup_num = Q or 1 for ECC */
 822                        int idx;
 823
 824                        idx = pup + ecc * ECC_BIT;
 825
 826                        /* Check Overrun */
 827                        if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
 828                              (REG_DRAM_TRAINING_2_OVERRUN_OFFS +
 829                               pup)) & 0x1)) {
 830                                /* If no OverRun */
 831
 832                                /* Inside the window */
 833                                if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) {
 834                                        /*
 835                                         * Match expected value ? - Update
 836                                         * State Machine
 837                                         */
 838                                        if (((~locked_pups >> pup) & 0x1)
 839                                            && (final_delay == 0)) {
 840                                                /* Match - Still inside the Window */
 841                                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window  for pup: ",
 842                                                                (u32)pup, 1);
 843
 844                                        } else {
 845                                                /* We got fail -> this is the end of the window */
 846                                                dram_info->rl_val[cs][idx][DE] = delay;
 847                                                dram_info->rl_val[cs][idx][PE] = phase;
 848                                                /* Go to Final State */
 849                                                dram_info->rl_val[cs][idx][S]++;
 850                                                final_sum++;
 851                                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ",
 852                                                                (u32)pup, 1);
 853                                        }
 854
 855                                        /* Before the start of the window */
 856                                } else if (dram_info->rl_val[cs][idx][S] ==
 857                                           RL_UNLOCK_STATE) {
 858                                        /* Must be RL_UNLOCK_STATE */
 859                                        /*
 860                                         * Match expected value ? - Update
 861                                         * State Machine
 862                                         */
 863                                        if (dram_info->rl_val[cs][idx][C] <
 864                                            RL_RETRY_COUNT) {
 865                                                if (((~locked_pups >> pup) & 0x1)) {
 866                                                        /* Match */
 867                                                        DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
 868                                                                        (u32)pup, 1);
 869                                                        dram_info->rl_val[cs][idx][C]++;
 870
 871                                                        /* If pup got to last state - lock the delays */
 872                                                        if (dram_info->rl_val[cs][idx][C] ==
 873                                                            RL_RETRY_COUNT) {
 874                                                                dram_info->rl_val[cs][idx][C] = 0;
 875                                                                dram_info->rl_val[cs][idx][DS] =
 876                                                                        delay;
 877                                                                dram_info->rl_val[cs][idx][PS] =
 878                                                                        phase;
 879                                                                dram_info->rl_val[cs][idx][S]++;        /* Go to Window State */
 880                                                                locked_sum++;
 881                                                                /* Will count the pups that got locked */
 882
 883                                                                /* IF First lock - need to lock delays */
 884                                                                if (first_octet_locked == 0) {
 885                                                                        DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
 886                                                                                        (u32)pup, 1);
 887                                                                        first_octet_locked
 888                                                                            =
 889                                                                            1;
 890                                                                }
 891                                                        }
 892
 893                                                        /* if pup is in not in final state but there was match - dont increment counter */
 894                                                        else {
 895                                                                counter_in_progress
 896                                                                    = 1;
 897                                                        }
 898                                                }
 899                                        }
 900                                }
 901                        } else {
 902                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
 903                                                (u32)pup, 1);
 904                                counter_in_progress = 1;
 905                        }
 906                }
 907
 908                if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) {
 909                        all_locked = 1;
 910                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
 911                }
 912
 913                /*
 914                 * This is a fix for unstable condition where pups are
 915                 * toggling between match and no match
 916                 */
 917                /*
 918                 * If some of the pups is >1 <3, check if we did it too many
 919                 * times
 920                 */
 921                if (counter_in_progress == 1) {
 922                        if (repeat_max_cnt < RL_RETRY_COUNT) {
 923                                /* Notify at least one Counter is >=1 and < 3 */
 924                                repeat_max_cnt++;
 925                                counter_in_progress = 1;
 926                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
 927                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
 928                        } else {
 929                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
 930                                counter_in_progress = 0;
 931                        }
 932                }
 933
 934                /*
 935                 * Check some of the pups are in the middle of state machine
 936                 * and don't increment the delays
 937                 */
 938                if (!counter_in_progress && !all_locked) {
 939                        repeat_max_cnt = 0;
 940                        if (!ratio_2to1)
 941                                ui_max_delay = MAX_DELAY_INV;
 942                        else
 943                                ui_max_delay = MAX_DELAY;
 944
 945                        /* Increment Delay */
 946                        if (delay < ui_max_delay) {
 947                                /* Delay Incrementation */
 948                                delay++;
 949                                if (delay == ui_max_delay) {
 950                                        /*
 951                                         * Mark the last delay/pahse place
 952                                         * for window final place
 953                                         */
 954                                        if ((!ratio_2to1
 955                                             && phase == MAX_PHASE_RL_L_1TO1)
 956                                            || (ratio_2to1
 957                                                && phase ==
 958                                                MAX_PHASE_RL_L_2TO1))
 959                                                final_delay = 1;
 960                                }
 961                        } else {
 962                                /* Phase+CL Incrementation */
 963                                delay = 0;
 964                                if (!ratio_2to1) {
 965                                        /* 1:1 mode */
 966                                        if (first_octet_locked) {
 967                                                /* some pupet was Locked */
 968                                                if (phase < MAX_PHASE_RL_L_1TO1) {
 969#ifdef RL_WINDOW_WA
 970                                                        if (phase == 0)
 971#else
 972                                                        if (phase == 1)
 973#endif
 974                                                                phase = 4;
 975                                                        else
 976                                                                phase++;
 977                                                } else {
 978                                                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 979                                                        return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
 980                                                }
 981                                        } else {
 982                                                /* No Pup was Locked */
 983                                                if (phase < MAX_PHASE_RL_UL_1TO1) {
 984#ifdef RL_WINDOW_WA
 985                                                        if (phase == 0)
 986                                                                phase = 4;
 987#else
 988                                                        phase++;
 989#endif
 990                                                } else
 991                                                        phase = 0;
 992                                        }
 993                                } else {
 994                                        /* 2:1 mode */
 995                                        if (first_octet_locked) {
 996                                                /* Some Pup was Locked */
 997                                                if (phase < MAX_PHASE_RL_L_2TO1) {
 998                                                        phase++;
 999                                                } else {
1000                                                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");

1001                                                        return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
1002                                                }
1003                                        } else {
1004                                                /* No Pup was Locked */
1005                                                if (phase < MAX_PHASE_RL_UL_2TO1)
1006                                                        phase++;
1007                                                else
1008                                                        phase = 0;
1009                                        }
1010                                }
1011
1012                                /*
1013                                 * If we finished a full Phases cycle (so
1014                                 * now phase = 0, need to increment
1015                                 * rd_sample_dly
1016                                 */
1017                                if (phase == 0 && first_octet_locked == 0) {
1018                                        rd_sample_delay++;
1019
1020                                        /* Set current rd_sample_delay  */
1021                                        reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
1022                                        reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
1023                                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS
1024                                                  * cs));
1025                                        reg |= (rd_sample_delay <<
1026                                                (REG_READ_DATA_SAMPLE_DELAYS_OFFS *
1027                                                 cs));
1028                                        reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
1029                                                  reg);
1030                                }
1031
1032                                /*
1033                                 * Set current rdReadyDelay according to the
1034                                 * hash table (Need to do this in every phase
1035                                 * change)
1036                                 */
1037                                if (!ratio_2to1) {
1038                                        /* 1:1 mode */
1039                                        add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1040                                        switch (phase) {
1041                                        case 0:
1042                                                add = add >>
1043                                                        REG_TRAINING_DEBUG_2_OFFS;
1044                                                break;
1045                                        case 1:
1046                                                add = add >>
1047                                                        (REG_TRAINING_DEBUG_2_OFFS
1048                                                         + 3);
1049                                                break;
1050                                        case 4:
1051                                                add = add >>
1052                                                        (REG_TRAINING_DEBUG_2_OFFS
1053                                                         + 6);
1054                                                break;
1055                                        case 5:
1056                                                add = add >>
1057                                                        (REG_TRAINING_DEBUG_2_OFFS
1058                                                         + 9);
1059                                                break;
1060                                        }
1061                                } else {
1062                                        /* 2:1 mode */
1063                                        add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1064                                        add = (add >> phase *
1065                                               REG_TRAINING_DEBUG_3_OFFS);
1066                                }
1067                                add &= REG_TRAINING_DEBUG_2_MASK;
1068                                reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1069                                reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
1070                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1071                                reg |= ((rd_sample_delay + add) <<
1072                                        (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1073                                reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1074                                dram_info->rd_smpl_dly = rd_sample_delay;
1075                                dram_info->rd_rdy_dly = rd_sample_delay + add;
1076                        }
1077
1078                        /* Reset counters for pups with states<RD_STATE_COUNT */
1079                        for (pup = 0;
1080                             pup <
1081                             (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1082                             pup++) {
1083                                if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
1084                                        dram_info->rl_val[cs][idx][C] = 0;
1085                        }
1086                }
1087        }
1088
1089        phase_min = 10;
1090
1091        for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
1092                DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: ");
1093                DEBUG_RL_D((u32) pup, 1);
1094                DEBUG_RL_S(", PS: ");
1095                DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1);
1096                DEBUG_RL_S(", DS: ");
1097                DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2);
1098                DEBUG_RL_S(", PE: ");
1099                DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1);
1100                DEBUG_RL_S(", DE: ");
1101                DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2);
1102                DEBUG_RL_S("\n");
1103        }
1104
1105        /* Find center of the window procedure */
1106        for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1107             pup++) {
1108#ifdef RL_WINDOW_WA
1109                if (!ratio_2to1) {      /* 1:1 mode */
1110                        if (dram_info->rl_val[cs][idx][PS] == 4)
1111                                dram_info->rl_val[cs][idx][PS] = 1;
1112                        if (dram_info->rl_val[cs][idx][PE] == 4)
1113                                dram_info->rl_val[cs][idx][PE] = 1;
1114
1115                        delay_s = dram_info->rl_val[cs][idx][PS] *
1116                                MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS];
1117                        delay_e = dram_info->rl_val[cs][idx][PE] *
1118                                MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE];
1119
1120                        tmp = (delay_e - delay_s) / 2 + delay_s;
1121                        phase = tmp / MAX_DELAY_INV;
1122                        if (phase == 1) /* 1:1 mode */
1123                                phase = 4;
1124
1125                        if (phase < phase_min)  /* for the read ready delay */
1126                                phase_min = phase;
1127
1128                        dram_info->rl_val[cs][idx][P] = phase;
1129                        dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV;
1130
1131                } else {
1132                        delay_s = dram_info->rl_val[cs][idx][PS] *
1133                                MAX_DELAY + dram_info->rl_val[cs][idx][DS];
1134                        delay_e = dram_info->rl_val[cs][idx][PE] *
1135                                MAX_DELAY + dram_info->rl_val[cs][idx][DE];
1136
1137                        tmp = (delay_e - delay_s) / 2 + delay_s;
1138                        phase = tmp / MAX_DELAY;
1139
1140                        if (phase < phase_min)  /* for the read ready delay */
1141                                phase_min = phase;
1142
1143                        dram_info->rl_val[cs][idx][P] = phase;
1144                        dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1145                }
1146#else
1147                if (!ratio_2to1) {      /* 1:1 mode */
1148                        if (dram_info->rl_val[cs][idx][PS] > 1)
1149                                dram_info->rl_val[cs][idx][PS] -= 2;
1150                        if (dram_info->rl_val[cs][idx][PE] > 1)
1151                                dram_info->rl_val[cs][idx][PE] -= 2;
1152                }
1153
1154                delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY +
1155                        dram_info->rl_val[cs][idx][DS];
1156                delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY +
1157                        dram_info->rl_val[cs][idx][DE];
1158
1159                tmp = (delay_e - delay_s) / 2 + delay_s;
1160                phase = tmp / MAX_DELAY;
1161                if (!ratio_2to1 && phase > 1)   /* 1:1 mode */
1162                        phase += 2;
1163
1164                if (phase < phase_min)  /* for the read ready delay */
1165                        phase_min = phase;
1166
1167                dram_info->rl_val[cs][idx][P] = phase;
1168                dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1169#endif
1170        }
1171
1172        /* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */
1173        if (!ratio_2to1) {      /* 1:1 mode */
1174                add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1175                switch (phase_min) {
1176                case 0:
1177                        add = (add >> REG_TRAINING_DEBUG_2_OFFS);
1178                        break;
1179                case 1:
1180                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
1181                        break;
1182                case 4:
1183                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
1184                        break;
1185                case 5:
1186                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
1187                        break;
1188                }
1189        } else {                /* 2:1 mode */
1190                add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1191                add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS);
1192        }
1193
1194        add &= REG_TRAINING_DEBUG_2_MASK;
1195        reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1196        reg &=
1197            ~(REG_READ_DATA_READY_DELAYS_MASK <<
1198              (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1199        reg |=
1200            ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1201        reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1202        dram_info->rd_rdy_dly = rd_sample_delay + add;
1203
1204        for (cs = 0; cs < dram_info->num_cs; cs++) {
1205                for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
1206                        reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
1207                        dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
1208                }
1209        }
1210
1211        return MV_OK;
1212}
1213#endif
1214