LXR uboot/drivers/ddr/marvell/axp/ddr3_read

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) Marvell International Ltd. and its affiliates
   4 */
   5
   6#include <common.h>
   7#include <i2c.h>
   8#include <log.h>
   9#include <spl.h>
  10#include <asm/io.h>
  11#include <asm/arch/cpu.h>
  12#include <asm/arch/soc.h>
  13
  14#include "ddr3_hw_training.h"
  15
  16/*
  17 * Debug
  18 */
  19#define DEBUG_RL_C(s, d, l) \
  20        DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n")
  21#define DEBUG_RL_FULL_C(s, d, l) \
  22        DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n")
  23
  24#ifdef MV_DEBUG_RL
  25#define DEBUG_RL_S(s) \
  26        debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  27#define DEBUG_RL_D(d, l) \
  28        debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  29#else
  30#define DEBUG_RL_S(s)
  31#define DEBUG_RL_D(d, l)
  32#endif
  33
  34#ifdef MV_DEBUG_RL_FULL
  35#define DEBUG_RL_FULL_S(s)              puts(s)
  36#define DEBUG_RL_FULL_D(d, l)           printf("%x", d)
  37#else
  38#define DEBUG_RL_FULL_S(s)
  39#define DEBUG_RL_FULL_D(d, l)
  40#endif
  41
  42extern u32 rl_pattern[LEN_STD_PATTERN];
  43
  44#ifdef RL_MODE
  45static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
  46                                                int ratio_2to1, u32 ecc,
  47                                                MV_DRAM_INFO *dram_info);
  48#else
  49static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
  50                                                    int ratio_2to1, u32 ecc,
  51                                                    MV_DRAM_INFO *dram_info);
  52#endif
  53
  54/*
  55 * Name:     ddr3_read_leveling_hw
  56 * Desc:     Execute the Read leveling phase by HW
  57 * Args:     dram_info - main struct
  58 *           freq      - current sequence frequency
  59 * Notes:
  60 * Returns:  MV_OK if success, MV_FAIL if fail.
  61 */
  62int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  63{
  64        u32 reg;
  65
  66        /* Debug message - Start Read leveling procedure */
  67        DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n");
  68
  69        /* Start Auto Read Leveling procedure */
  70        reg = 1 << REG_DRAM_TRAINING_RL_OFFS;
  71        /* Config the retest number */
  72        reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS);
  73
  74        /* Enable CS in the automatic process */
  75        reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS);
  76
  77        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  78
  79        reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  80                (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  81        reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  82
  83        /* Wait */
  84        do {
  85                reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  86                        (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  87        } while (reg);          /* Wait for '0' */
  88
  89        /* Check if Successful */
  90        if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  91            (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
  92                u32 delay, phase, pup, cs;
  93
  94                dram_info->rl_max_phase = 0;
  95                dram_info->rl_min_phase = 10;
  96
  97                /* Read results to arrays */
  98                for (cs = 0; cs < MAX_CS; cs++) {
  99                        if (dram_info->cs_ena & (1 << cs)) {
 100                                for (pup = 0;
 101                                     pup < dram_info->num_of_total_pups;
 102                                     pup++) {
 103                                        if (pup == dram_info->num_of_std_pups
 104                                            && dram_info->ecc_ena)
 105                                                pup = ECC_PUP;
 106                                        reg =
 107                                            ddr3_read_pup_reg(PUP_RL_MODE, cs,
 108                                                              pup);
 109                                        phase = (reg >> REG_PHY_PHASE_OFFS) &
 110                                                PUP_PHASE_MASK;
 111                                        delay = reg & PUP_DELAY_MASK;
 112                                        dram_info->rl_val[cs][pup][P] = phase;
 113                                        if (phase > dram_info->rl_max_phase)
 114                                                dram_info->rl_max_phase = phase;
 115                                        if (phase < dram_info->rl_min_phase)
 116                                                dram_info->rl_min_phase = phase;
 117                                        dram_info->rl_val[cs][pup][D] = delay;
 118                                        dram_info->rl_val[cs][pup][S] =
 119                                            RL_FINAL_STATE;
 120                                        reg =
 121                                            ddr3_read_pup_reg(PUP_RL_MODE + 0x1,
 122                                                              cs, pup);
 123                                        dram_info->rl_val[cs][pup][DQS] =
 124                                            (reg & 0x3F);
 125                                }
 126#ifdef MV_DEBUG_RL
 127                                /* Print results */
 128                                DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ",
 129                                           (u32) cs, 1);
 130
 131                                for (pup = 0;
 132                                     pup < (dram_info->num_of_total_pups);
 133                                     pup++) {
 134                                        if (pup == dram_info->num_of_std_pups
 135                                            && dram_info->ecc_ena)
 136                                                pup = ECC_PUP;
 137                                        DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
 138                                        DEBUG_RL_D((u32) pup, 1);
 139                                        DEBUG_RL_S(", Phase: ");
 140                                        DEBUG_RL_D((u32) dram_info->
 141                                                   rl_val[cs][pup][P], 1);
 142                                        DEBUG_RL_S(", Delay: ");
 143                                        DEBUG_RL_D((u32) dram_info->
 144                                                   rl_val[cs][pup][D], 2);
 145                                        DEBUG_RL_S("\n");
 146                                }
 147#endif
 148                        }
 149                }
 150
 151                dram_info->rd_rdy_dly =
 152                        reg_read(REG_READ_DATA_READY_DELAYS_ADDR) &
 153                        REG_READ_DATA_SAMPLE_DELAYS_MASK;
 154                dram_info->rd_smpl_dly =
 155                        reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) &
 156                        REG_READ_DATA_READY_DELAYS_MASK;
 157#ifdef MV_DEBUG_RL
 158                DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
 159                           dram_info->rd_smpl_dly, 2);
 160                DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
 161                           dram_info->rd_rdy_dly, 2);
 162                DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n");
 163#endif
 164                return MV_OK;
 165
 166        } else {
 167                DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n");
 168                return MV_FAIL;
 169        }
 170}
 171
 172/*
 173 * Name:     ddr3_read_leveling_sw
 174 * Desc:     Execute the Read leveling phase by SW
 175 * Args:     dram_info - main struct
 176 *           freq      - current sequence frequency
 177 * Notes:
 178 * Returns:  MV_OK if success, MV_FAIL if fail.
 179 */
 180int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
 181{
 182        u32 reg, cs, ecc, pup_num, phase, delay, pup;
 183        int status;
 184
 185        /* Debug message - Start Read leveling procedure */
 186        DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n");
 187
 188        /* Enable SW Read Leveling */
 189        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 190                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 191        reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS);
 192        /* [0]=1 - Enable SW override  */
 193        /* 0x15B8 - Training SW 2 Register */
 194        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 195
 196#ifdef RL_MODE
 197        reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
 198                (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 199        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 200#endif
 201
 202        /* Loop for each CS */
 203        for (cs = 0; cs < dram_info->num_cs; cs++) {
 204                DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1);
 205
 206                for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) {
 207                        /* ECC Support - Switch ECC Mux on ecc=1 */
 208                        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 209                                ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 210                        reg |= (dram_info->ecc_ena *
 211                                ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 212                        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 213
 214                        if (ecc)
 215                                DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n");
 216                        else
 217                                DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n");
 218
 219                        /* Set current sample delays */
 220                        reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
 221                        reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
 222                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
 223                        reg |= (dram_info->cl <<
 224                                (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
 225                        reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg);
 226
 227                        /* Set current Ready delay */
 228                        reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 229                        reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 230                                 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 231                        if (!ratio_2to1) {
 232                                /* 1:1 mode */
 233                                reg |= ((dram_info->cl + 1) <<
 234                                        (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 235                        } else {
 236                                /* 2:1 mode */
 237                                reg |= ((dram_info->cl + 2) <<
 238                                        (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 239                        }
 240                        reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 241
 242                        /* Read leveling Single CS[cs] */
 243#ifdef RL_MODE
 244                        status =
 245                            ddr3_read_leveling_single_cs_rl_mode(cs, freq,
 246                                                                 ratio_2to1,
 247                                                                 ecc,
 248                                                                 dram_info);
 249                        if (MV_OK != status)
 250                                return status;
 251#else
 252                        status =
 253                            ddr3_read_leveling_single_cs_window_mode(cs, freq,
 254                                                                     ratio_2to1,
 255                                                                     ecc,
 256                                                                     dram_info)
 257                            if (MV_OK != status)
 258                                return status;
 259#endif
 260                }
 261
 262                /* Print results */
 263                DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs,
 264                           1);
 265
 266                for (pup = 0;
 267                     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
 268                     pup++) {
 269                        DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
 270                        DEBUG_RL_D((u32) pup, 1);
 271                        DEBUG_RL_S(", Phase: ");
 272                        DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1);
 273                        DEBUG_RL_S(", Delay: ");
 274                        DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2);
 275                        DEBUG_RL_S("\n");
 276                }
 277
 278                DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
 279                           dram_info->rd_smpl_dly, 2);
 280                DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
 281                           dram_info->rd_rdy_dly, 2);
 282
 283                /* Configure PHY with average of 3 locked leveling settings */
 284                for (pup = 0;
 285                     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
 286                     pup++) {
 287                        /* ECC support - bit 8 */
 288                        pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
 289
 290                        /* For now, set last cnt result */
 291                        phase = dram_info->rl_val[cs][pup][P];
 292                        delay = dram_info->rl_val[cs][pup][D];
 293                        ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,
 294                                           delay);
 295                }
 296        }
 297
 298        /* Reset PHY read FIFO */
 299        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 300                (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 301        /* 0x15B8 - Training SW 2 Register */
 302        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 303
 304        do {
 305                reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 306                        (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 307        } while (reg);          /* Wait for '0' */
 308
 309        /* ECC Support - Switch ECC Mux off ecc=0 */
 310        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 311                ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 312        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 313
 314#ifdef RL_MODE
 315        reg_write(REG_DRAM_TRAINING_ADDR, 0);   /* 0x15B0 - Training Register */
 316#endif
 317
 318        /* Disable SW Read Leveling */
 319        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 320                ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 321        /* [0] = 0 - Disable SW override  */
 322        reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS));
 323        /* [3] = 1 - Disable RL MODE */
 324        /* 0x15B8 - Training SW 2 Register */
 325        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 326
 327        DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n");
 328        return MV_OK;
 329}
 330
 331#ifdef RL_MODE
 332/*
 333 * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode().
 334 * This just got too much indented making it hard to read / edit.
 335 */
 336static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups,
 337                    u32 *locked_sum, u32 ecc, int *first_octet_locked,
 338                    int *counter_in_progress, int final_delay, u32 delay,
 339                    u32 phase)
 340{
 341        /* If no OverRun */
 342        if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) {
 343                int idx;
 344
 345                idx = pup + ecc * ECC_BIT;
 346
 347                /* PUP passed, start examining */
 348                if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) {
 349                        /* Must be RL_UNLOCK_STATE */
 350                        /* Match expected value ? - Update State Machine */
 351                        if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) {
 352                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
 353                                                (u32)pup, 1);
 354                                info->rl_val[cs][idx][C]++;
 355
 356                                /* If pup got to last state - lock the delays */
 357                                if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) {
 358                                        info->rl_val[cs][idx][C] = 0;
 359                                        info->rl_val[cs][idx][DS] = delay;
 360                                        info->rl_val[cs][idx][PS] = phase;
 361
 362                                        /* Go to Final State */
 363                                        info->rl_val[cs][idx][S] = RL_FINAL_STATE;
 364                                        *locked_sum = *locked_sum + 1;
 365                                        DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ",
 366                                                        (u32)pup, 1);
 367
 368                                        /*
 369                                         * If first lock - need to lock delays
 370                                         */
 371                                        if (*first_octet_locked == 0) {
 372                                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
 373                                                                (u32)pup, 1);
 374                                                *first_octet_locked = 1;
 375                                        }
 376
 377                                        /*
 378                                         * If pup is in not in final state but
 379                                         * there was match - dont increment
 380                                         * counter
 381                                         */
 382                                } else {
 383                                        *counter_in_progress = 1;
 384                                }
 385                        }
 386                }
 387        }
 388}
 389
 390/*
 391 * Name:     ddr3_read_leveling_single_cs_rl_mode
 392 * Desc:     Execute Read leveling for single Chip select
 393 * Args:     cs        - current chip select
 394 *           freq      - current sequence frequency
 395 *           ecc       - ecc iteration indication
 396 *           dram_info - main struct
 397 * Notes:
 398 * Returns:  MV_OK if success, MV_FAIL if fail.
 399 */
 400static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
 401                                                int ratio_2to1, u32 ecc,
 402                                                MV_DRAM_INFO *dram_info)
 403{
 404        u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,
 405                repeat_max_cnt, sdram_offset, locked_sum;
 406        u32 phase_min, ui_max_delay;
 407        int all_locked, first_octet_locked, counter_in_progress;
 408        int final_delay = 0;
 409
 410        DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
 411
 412        /* Init values */
 413        phase = 0;
 414        delay = 0;
 415        rd_sample_delay = dram_info->cl;
 416        all_locked = 0;
 417        first_octet_locked = 0;
 418        repeat_max_cnt = 0;
 419        locked_sum = 0;
 420
 421        for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 422             pup++)
 423                dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
 424
 425        /* Main loop */
 426        while (!all_locked) {
 427                counter_in_progress = 0;
 428
 429                DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
 430                DEBUG_RL_FULL_D(rd_sample_delay, 2);
 431                DEBUG_RL_FULL_S(", RdRdyDly = ");
 432                DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
 433                DEBUG_RL_FULL_S(", Phase = ");
 434                DEBUG_RL_FULL_D(phase, 1);
 435                DEBUG_RL_FULL_S(", Delay = ");
 436                DEBUG_RL_FULL_D(delay, 2);
 437                DEBUG_RL_FULL_S("\n");
 438
 439                /*
 440                 * Broadcast to all PUPs current RL delays: DQS phase,
 441                 * leveling delay
 442                 */
 443                ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
 444
 445                /* Reset PHY read FIFO */
 446                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 447                        (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 448                /* 0x15B8 - Training SW 2 Register */
 449                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 450
 451                do {
 452                        reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 453                                (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 454                } while (reg);  /* Wait for '0' */
 455
 456                /* Read pattern from SDRAM */
 457                sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
 458                locked_pups = 0;
 459                if (MV_OK !=
 460                    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
 461                                       rl_pattern, LEN_STD_PATTERN,
 462                                       sdram_offset, 0, 0, NULL, 0))
 463                        return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN;
 464
 465                /* Octet evaluation */
 466                /* pup_num = Q or 1 for ECC */
 467                for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 468                        /* Check Overrun */
 469                        if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
 470                               (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) {
 471                                overrun(cs, dram_info, pup, locked_pups,
 472                                        &locked_sum, ecc, &first_octet_locked,
 473                                        &counter_in_progress, final_delay,
 474                                        delay, phase);
 475                        } else {
 476                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
 477                                                (u32)pup, 1);
 478                        }
 479                }
 480
 481                if (locked_sum == (dram_info->num_of_std_pups *
 482                                   (1 - ecc) + ecc)) {
 483                        all_locked = 1;
 484                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
 485                }
 486
 487                /*
 488                 * This is a fix for unstable condition where pups are
 489                 * toggling between match and no match
 490                 */
 491                /*
 492                 * If some of the pups is >1 <3, check if we did it too
 493                 * many times
 494                 */
 495                if (counter_in_progress == 1) {
 496                        /* Notify at least one Counter is >=1 and < 3 */
 497                        if (repeat_max_cnt < RL_RETRY_COUNT) {
 498                                repeat_max_cnt++;
 499                                counter_in_progress = 1;
 500                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
 501                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
 502                        } else {
 503                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
 504                                counter_in_progress = 0;
 505                        }
 506                }
 507
 508                /*
 509                 * Check some of the pups are in the middle of state machine
 510                 * and don't increment the delays
 511                 */
 512                if (!counter_in_progress && !all_locked) {
 513                        int idx;
 514
 515                        idx = pup + ecc * ECC_BIT;
 516
 517                        repeat_max_cnt = 0;
 518                        /* if 1:1 mode */
 519                        if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))
 520                                ui_max_delay = MAX_DELAY_INV;
 521                        else
 522                                ui_max_delay = MAX_DELAY;
 523
 524                        /* Increment Delay */
 525                        if (delay < ui_max_delay) {
 526                                delay++;
 527                                /*
 528                                 * Mark the last delay/pahse place for
 529                                 * window final place
 530                                 */
 531                                if (delay == ui_max_delay) {
 532                                        if ((!ratio_2to1 && phase ==
 533                                             MAX_PHASE_RL_L_1TO1)
 534                                            || (ratio_2to1 && phase ==
 535                                                MAX_PHASE_RL_L_2TO1))
 536                                                final_delay = 1;
 537                                }
 538                        } else {
 539                                /* Phase+CL Incrementation */
 540                                delay = 0;
 541
 542                                if (!ratio_2to1) {
 543                                        /* 1:1 mode */
 544                                        if (first_octet_locked) {
 545                                                /* some Pup was Locked */
 546                                                if (phase < MAX_PHASE_RL_L_1TO1) {
 547                                                        if (phase == 1) {
 548                                                                phase = 4;
 549                                                        } else {
 550                                                                phase++;
 551                                                                delay = MIN_DELAY_PHASE_1_LIMIT;
 552                                                        }
 553                                                } else {
 554                                                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 555                                                        DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n");
 556                                                        return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
 557                                                }
 558                                        } else {
 559                                                /* NO Pup was Locked */
 560                                                if (phase < MAX_PHASE_RL_UL_1TO1) {
 561                                                        phase++;
 562                                                        delay =
 563                                                            MIN_DELAY_PHASE_1_LIMIT;
 564                                                } else {
 565                                                        phase = 0;
 566                                                }
 567                                        }
 568                                } else {
 569                                        /* 2:1 mode */
 570                                        if (first_octet_locked) {
 571                                                /* some Pup was Locked */
 572                                                if (phase < MAX_PHASE_RL_L_2TO1) {
 573                                                        phase++;
 574                                                } else {
 575                                                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 576                                                        DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 577                                                        for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 578                                                                /* pup_num = Q or 1 for ECC */
 579                                                                if (dram_info->rl_val[cs][idx][S]
 580                                                                    == 0) {
 581                                                                        DEBUG_RL_C("Failed byte is = ",
 582                                                                                   pup, 1);
 583                                                                }
 584                                                        }
 585                                                        return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
 586                                                }
 587                                        } else {
 588                                                /* No Pup was Locked */
 589                                                if (phase < MAX_PHASE_RL_UL_2TO1)
 590                                                        phase++;
 591                                                else
 592                                                        phase = 0;
 593                                        }
 594                                }
 595
 596                                /*
 597                                 * If we finished a full Phases cycle (so now
 598                                 * phase = 0, need to increment rd_sample_dly
 599                                 */
 600                                if (phase == 0 && first_octet_locked == 0) {
 601                                        rd_sample_delay++;
 602                                        if (rd_sample_delay == 0x10) {
 603                                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 604                                                DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 605                                                for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 606                                                        /* pup_num = Q or 1 for ECC */
 607                                                        if (dram_info->
 608                                                            rl_val[cs][idx][S] == 0) {
 609                                                                DEBUG_RL_C("Failed byte is = ",
 610                                                                           pup, 1);
 611                                                        }
 612                                                }
 613                                                return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK;
 614                                        }
 615
 616                                        /* Set current rd_sample_delay  */
 617                                        reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
 618                                        reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK
 619                                              << (REG_READ_DATA_SAMPLE_DELAYS_OFFS
 620                                                  * cs));
 621                                        reg |= (rd_sample_delay <<
 622                                                (REG_READ_DATA_SAMPLE_DELAYS_OFFS *
 623                                                 cs));
 624                                        reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
 625                                                  reg);
 626                                }
 627
 628                                /*
 629                                 * Set current rdReadyDelay according to the
 630                                 * hash table (Need to do this in every phase
 631                                 * change)
 632                                 */
 633                                if (!ratio_2to1) {
 634                                        /* 1:1 mode */
 635                                        add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
 636                                        switch (phase) {
 637                                        case 0:
 638                                                add = (add >>
 639                                                       REG_TRAINING_DEBUG_2_OFFS);
 640                                                break;
 641                                        case 1:
 642                                                add = (add >>
 643                                                       (REG_TRAINING_DEBUG_2_OFFS
 644                                                        + 3));
 645                                                break;
 646                                        case 4:
 647                                                add = (add >>
 648                                                       (REG_TRAINING_DEBUG_2_OFFS
 649                                                        + 6));
 650                                                break;
 651                                        case 5:
 652                                                add = (add >>
 653                                                       (REG_TRAINING_DEBUG_2_OFFS
 654                                                        + 9));
 655                                                break;
 656                                        }
 657                                        add &= REG_TRAINING_DEBUG_2_MASK;
 658                                } else {
 659                                        /* 2:1 mode */
 660                                        add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
 661                                        add = (add >>
 662                                               (phase *
 663                                                REG_TRAINING_DEBUG_3_OFFS));
 664                                        add &= REG_TRAINING_DEBUG_3_MASK;
 665                                }
 666
 667                                reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 668                                reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 669                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 670                                reg |= ((rd_sample_delay + add) <<
 671                                        (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 672                                reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 673                                dram_info->rd_smpl_dly = rd_sample_delay;
 674                                dram_info->rd_rdy_dly = rd_sample_delay + add;
 675                        }
 676
 677                        /* Reset counters for pups with states<RD_STATE_COUNT */
 678                        for (pup = 0; pup <
 679                                     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 680                             pup++) {
 681                                if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
 682                                        dram_info->rl_val[cs][idx][C] = 0;
 683                        }
 684                }
 685        }
 686
 687        phase_min = 10;
 688
 689        for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
 690                if (dram_info->rl_val[cs][pup][PS] < phase_min)
 691                        phase_min = dram_info->rl_val[cs][pup][PS];
 692        }
 693
 694        /*
 695         * Set current rdReadyDelay according to the hash table (Need to
 696         * do this in every phase change)
 697         */
 698        if (!ratio_2to1) {
 699                /* 1:1 mode */
 700                add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
 701                switch (phase_min) {
 702                case 0:
 703                        add = (add >> REG_TRAINING_DEBUG_2_OFFS);
 704                        break;
 705                case 1:
 706                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
 707                        break;
 708                case 4:
 709                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
 710                        break;
 711                case 5:
 712                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
 713                        break;
 714                }
 715                add &= REG_TRAINING_DEBUG_2_MASK;
 716        } else {
 717                /* 2:1 mode */
 718                add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
 719                add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS));
 720                add &= REG_TRAINING_DEBUG_3_MASK;
 721        }
 722
 723        reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 724        reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 725                 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 726        reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 727        reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 728        dram_info->rd_rdy_dly = rd_sample_delay + add;
 729
 730        for (cs = 0; cs < dram_info->num_cs; cs++) {
 731                for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
 732                        reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
 733                        dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
 734                }
 735        }
 736
 737        return MV_OK;
 738}
 739
 740#else
 741
 742/*
 743 * Name:     ddr3_read_leveling_single_cs_window_mode
 744 * Desc:     Execute Read leveling for single Chip select
 745 * Args:     cs        - current chip select
 746 *           freq      - current sequence frequency
 747 *           ecc       - ecc iteration indication
 748 *           dram_info - main struct
 749 * Notes:
 750 * Returns:  MV_OK if success, MV_FAIL if fail.
 751 */
 752static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
 753                                                    int ratio_2to1, u32 ecc,
 754                                                    MV_DRAM_INFO *dram_info)
 755{
 756        u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,
 757            repeat_max_cnt, sdram_offset, final_sum, locked_sum;
 758        u32 delay_s, delay_e, tmp, phase_min, ui_max_delay;
 759        int all_locked, first_octet_locked, counter_in_progress;
 760        int final_delay = 0;
 761
 762        DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
 763
 764        /* Init values */
 765        phase = 0;
 766        delay = 0;
 767        rd_sample_delay = dram_info->cl;
 768        all_locked = 0;
 769        first_octet_locked = 0;
 770        repeat_max_cnt = 0;
 771        sum = 0;
 772        final_sum = 0;
 773        locked_sum = 0;
 774
 775        for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 776             pup++)
 777                dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
 778
 779        /* Main loop */
 780        while (!all_locked) {
 781                counter_in_progress = 0;
 782
 783                DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
 784                DEBUG_RL_FULL_D(rd_sample_delay, 2);
 785                DEBUG_RL_FULL_S(", RdRdyDly = ");
 786                DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
 787                DEBUG_RL_FULL_S(", Phase = ");
 788                DEBUG_RL_FULL_D(phase, 1);
 789                DEBUG_RL_FULL_S(", Delay = ");
 790                DEBUG_RL_FULL_D(delay, 2);
 791                DEBUG_RL_FULL_S("\n");
 792
 793                /*
 794                 * Broadcast to all PUPs current RL delays: DQS phase,leveling
 795                 * delay
 796                 */
 797                ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
 798
 799                /* Reset PHY read FIFO */
 800                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 801                        (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 802                /* 0x15B8 - Training SW 2 Register */
 803                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 804
 805                do {
 806                        reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 807                                (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 808                } while (reg);  /* Wait for '0' */
 809
 810                /* Read pattern from SDRAM */
 811                sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
 812                locked_pups = 0;
 813                if (MV_OK !=
 814                    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
 815                                       rl_pattern, LEN_STD_PATTERN,
 816                                       sdram_offset, 0, 0, NULL, 0))
 817                        return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN;
 818
 819                /* Octet evaluation */
 820                for (pup = 0; pup < (dram_info->num_of_std_pups *
 821                                     (1 - ecc) + ecc); pup++) {
 822                        /* pup_num = Q or 1 for ECC */
 823                        int idx;
 824
 825                        idx = pup + ecc * ECC_BIT;
 826
 827                        /* Check Overrun */
 828                        if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
 829                              (REG_DRAM_TRAINING_2_OVERRUN_OFFS +
 830                               pup)) & 0x1)) {
 831                                /* If no OverRun */
 832
 833                                /* Inside the window */
 834                                if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) {
 835                                        /*
 836                                         * Match expected value ? - Update
 837                                         * State Machine
 838                                         */
 839                                        if (((~locked_pups >> pup) & 0x1)
 840                                            && (final_delay == 0)) {
 841                                                /* Match - Still inside the Window */
 842                                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window  for pup: ",
 843                                                                (u32)pup, 1);
 844
 845                                        } else {
 846                                                /* We got fail -> this is the end of the window */
 847                                                dram_info->rl_val[cs][idx][DE] = delay;
 848                                                dram_info->rl_val[cs][idx][PE] = phase;
 849                                                /* Go to Final State */
 850                                                dram_info->rl_val[cs][idx][S]++;
 851                                                final_sum++;
 852                                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ",
 853                                                                (u32)pup, 1);
 854                                        }
 855
 856                                        /* Before the start of the window */
 857                                } else if (dram_info->rl_val[cs][idx][S] ==
 858                                           RL_UNLOCK_STATE) {
 859                                        /* Must be RL_UNLOCK_STATE */
 860                                        /*
 861                                         * Match expected value ? - Update
 862                                         * State Machine
 863                                         */
 864                                        if (dram_info->rl_val[cs][idx][C] <
 865                                            RL_RETRY_COUNT) {
 866                                                if (((~locked_pups >> pup) & 0x1)) {
 867                                                        /* Match */
 868                                                        DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
 869                                                                        (u32)pup, 1);
 870                                                        dram_info->rl_val[cs][idx][C]++;
 871
 872                                                        /* If pup got to last state - lock the delays */
 873                                                        if (dram_info->rl_val[cs][idx][C] ==
 874                                                            RL_RETRY_COUNT) {
 875                                                                dram_info->rl_val[cs][idx][C] = 0;
 876                                                                dram_info->rl_val[cs][idx][DS] =
 877                                                                        delay;
 878                                                                dram_info->rl_val[cs][idx][PS] =
 879                                                                        phase;
 880                                                                dram_info->rl_val[cs][idx][S]++;        /* Go to Window State */
 881                                                                locked_sum++;
 882                                                                /* Will count the pups that got locked */
 883
 884                                                                /* IF First lock - need to lock delays */
 885                                                                if (first_octet_locked == 0) {
 886                                                                        DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
 887                                                                                        (u32)pup, 1);
 888                                                                        first_octet_locked
 889                                                                            =
 890                                                                            1;
 891                                                                }
 892                                                        }
 893
 894                                                        /* if pup is in not in final state but there was match - dont increment counter */
 895                                                        else {
 896                                                                counter_in_progress
 897                                                                    = 1;
 898                                                        }
 899                                                }
 900                                        }
 901                                }
 902                        } else {
 903                                DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
 904                                                (u32)pup, 1);
 905                                counter_in_progress = 1;
 906                        }
 907                }
 908
 909                if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) {
 910                        all_locked = 1;
 911                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
 912                }
 913
 914                /*
 915                 * This is a fix for unstable condition where pups are
 916                 * toggling between match and no match
 917                 */
 918                /*
 919                 * If some of the pups is >1 <3, check if we did it too many
 920                 * times
 921                 */
 922                if (counter_in_progress == 1) {
 923                        if (repeat_max_cnt < RL_RETRY_COUNT) {
 924                                /* Notify at least one Counter is >=1 and < 3 */
 925                                repeat_max_cnt++;
 926                                counter_in_progress = 1;
 927                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
 928                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
 929                        } else {
 930                                DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
 931                                counter_in_progress = 0;
 932                        }
 933                }
 934
 935                /*
 936                 * Check some of the pups are in the middle of state machine
 937                 * and don't increment the delays
 938                 */
 939                if (!counter_in_progress && !all_locked) {
 940                        repeat_max_cnt = 0;
 941                        if (!ratio_2to1)
 942                                ui_max_delay = MAX_DELAY_INV;
 943                        else
 944                                ui_max_delay = MAX_DELAY;
 945
 946                        /* Increment Delay */
 947                        if (delay < ui_max_delay) {
 948                                /* Delay Incrementation */
 949                                delay++;
 950                                if (delay == ui_max_delay) {
 951                                        /*
 952                                         * Mark the last delay/pahse place
 953                                         * for window final place
 954                                         */
 955                                        if ((!ratio_2to1
 956                                             && phase == MAX_PHASE_RL_L_1TO1)
 957                                            || (ratio_2to1
 958                                                && phase ==
 959                                                MAX_PHASE_RL_L_2TO1))
 960                                                final_delay = 1;
 961                                }
 962                        } else {
 963                                /* Phase+CL Incrementation */
 964                                delay = 0;
 965                                if (!ratio_2to1) {
 966                                        /* 1:1 mode */
 967                                        if (first_octet_locked) {
 968                                                /* some pupet was Locked */
 969                                                if (phase < MAX_PHASE_RL_L_1TO1) {
 970#ifdef RL_WINDOW_WA
 971                                                        if (phase == 0)
 972#else
 973                                                        if (phase == 1)
 974#endif
 975                                                                phase = 4;
 976                                                        else
 977                                                                phase++;
 978                                                } else {
 979                                                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 980                                                        return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
 981                                                }
 982                                        } else {
 983                                                /* No Pup was Locked */
 984                                                if (phase < MAX_PHASE_RL_UL_1TO1) {
 985#ifdef RL_WINDOW_WA
 986                                                        if (phase == 0)
 987                                                                phase = 4;
 988#else
 989                                                        phase++;
 990#endif
 991                                                } else
 992                                                        phase = 0;
 993                                        }
 994                                } else {
 995                                        /* 2:1 mode */
 996                                        if (first_octet_locked) {
 997                                                /* Some Pup was Locked */
 998                                                if (phase < MAX_PHASE_RL_L_2TO1) {
 999                                                        phase++;
1000                                                } else {

1001                                                        DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
1002                                                        return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
1003                                                }
1004                                        } else {
1005                                                /* No Pup was Locked */
1006                                                if (phase < MAX_PHASE_RL_UL_2TO1)
1007                                                        phase++;
1008                                                else
1009                                                        phase = 0;
1010                                        }
1011                                }
1012
1013                                /*
1014                                 * If we finished a full Phases cycle (so
1015                                 * now phase = 0, need to increment
1016                                 * rd_sample_dly
1017                                 */
1018                                if (phase == 0 && first_octet_locked == 0) {
1019                                        rd_sample_delay++;
1020
1021                                        /* Set current rd_sample_delay  */
1022                                        reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
1023                                        reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
1024                                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS
1025                                                  * cs));
1026                                        reg |= (rd_sample_delay <<
1027                                                (REG_READ_DATA_SAMPLE_DELAYS_OFFS *
1028                                                 cs));
1029                                        reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
1030                                                  reg);
1031                                }
1032
1033                                /*
1034                                 * Set current rdReadyDelay according to the
1035                                 * hash table (Need to do this in every phase
1036                                 * change)
1037                                 */
1038                                if (!ratio_2to1) {
1039                                        /* 1:1 mode */
1040                                        add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1041                                        switch (phase) {
1042                                        case 0:
1043                                                add = add >>
1044                                                        REG_TRAINING_DEBUG_2_OFFS;
1045                                                break;
1046                                        case 1:
1047                                                add = add >>
1048                                                        (REG_TRAINING_DEBUG_2_OFFS
1049                                                         + 3);
1050                                                break;
1051                                        case 4:
1052                                                add = add >>
1053                                                        (REG_TRAINING_DEBUG_2_OFFS
1054                                                         + 6);
1055                                                break;
1056                                        case 5:
1057                                                add = add >>
1058                                                        (REG_TRAINING_DEBUG_2_OFFS
1059                                                         + 9);
1060                                                break;
1061                                        }
1062                                } else {
1063                                        /* 2:1 mode */
1064                                        add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1065                                        add = (add >> phase *
1066                                               REG_TRAINING_DEBUG_3_OFFS);
1067                                }
1068                                add &= REG_TRAINING_DEBUG_2_MASK;
1069                                reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1070                                reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
1071                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1072                                reg |= ((rd_sample_delay + add) <<
1073                                        (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1074                                reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1075                                dram_info->rd_smpl_dly = rd_sample_delay;
1076                                dram_info->rd_rdy_dly = rd_sample_delay + add;
1077                        }
1078
1079                        /* Reset counters for pups with states<RD_STATE_COUNT */
1080                        for (pup = 0;
1081                             pup <
1082                             (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1083                             pup++) {
1084                                if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
1085                                        dram_info->rl_val[cs][idx][C] = 0;
1086                        }
1087                }
1088        }
1089
1090        phase_min = 10;
1091
1092        for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
1093                DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: ");
1094                DEBUG_RL_D((u32) pup, 1);
1095                DEBUG_RL_S(", PS: ");
1096                DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1);
1097                DEBUG_RL_S(", DS: ");
1098                DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2);
1099                DEBUG_RL_S(", PE: ");
1100                DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1);
1101                DEBUG_RL_S(", DE: ");
1102                DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2);
1103                DEBUG_RL_S("\n");
1104        }
1105
1106        /* Find center of the window procedure */
1107        for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1108             pup++) {
1109#ifdef RL_WINDOW_WA
1110                if (!ratio_2to1) {      /* 1:1 mode */
1111                        if (dram_info->rl_val[cs][idx][PS] == 4)
1112                                dram_info->rl_val[cs][idx][PS] = 1;
1113                        if (dram_info->rl_val[cs][idx][PE] == 4)
1114                                dram_info->rl_val[cs][idx][PE] = 1;
1115
1116                        delay_s = dram_info->rl_val[cs][idx][PS] *
1117                                MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS];
1118                        delay_e = dram_info->rl_val[cs][idx][PE] *
1119                                MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE];
1120
1121                        tmp = (delay_e - delay_s) / 2 + delay_s;
1122                        phase = tmp / MAX_DELAY_INV;
1123                        if (phase == 1) /* 1:1 mode */
1124                                phase = 4;
1125
1126                        if (phase < phase_min)  /* for the read ready delay */
1127                                phase_min = phase;
1128
1129                        dram_info->rl_val[cs][idx][P] = phase;
1130                        dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV;
1131
1132                } else {
1133                        delay_s = dram_info->rl_val[cs][idx][PS] *
1134                                MAX_DELAY + dram_info->rl_val[cs][idx][DS];
1135                        delay_e = dram_info->rl_val[cs][idx][PE] *
1136                                MAX_DELAY + dram_info->rl_val[cs][idx][DE];
1137
1138                        tmp = (delay_e - delay_s) / 2 + delay_s;
1139                        phase = tmp / MAX_DELAY;
1140
1141                        if (phase < phase_min)  /* for the read ready delay */
1142                                phase_min = phase;
1143
1144                        dram_info->rl_val[cs][idx][P] = phase;
1145                        dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1146                }
1147#else
1148                if (!ratio_2to1) {      /* 1:1 mode */
1149                        if (dram_info->rl_val[cs][idx][PS] > 1)
1150                                dram_info->rl_val[cs][idx][PS] -= 2;
1151                        if (dram_info->rl_val[cs][idx][PE] > 1)
1152                                dram_info->rl_val[cs][idx][PE] -= 2;
1153                }
1154
1155                delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY +
1156                        dram_info->rl_val[cs][idx][DS];
1157                delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY +
1158                        dram_info->rl_val[cs][idx][DE];
1159
1160                tmp = (delay_e - delay_s) / 2 + delay_s;
1161                phase = tmp / MAX_DELAY;
1162                if (!ratio_2to1 && phase > 1)   /* 1:1 mode */
1163                        phase += 2;
1164
1165                if (phase < phase_min)  /* for the read ready delay */
1166                        phase_min = phase;
1167
1168                dram_info->rl_val[cs][idx][P] = phase;
1169                dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1170#endif
1171        }
1172
1173        /* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */
1174        if (!ratio_2to1) {      /* 1:1 mode */
1175                add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1176                switch (phase_min) {
1177                case 0:
1178                        add = (add >> REG_TRAINING_DEBUG_2_OFFS);
1179                        break;
1180                case 1:
1181                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
1182                        break;
1183                case 4:
1184                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
1185                        break;
1186                case 5:
1187                        add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
1188                        break;
1189                }
1190        } else {                /* 2:1 mode */
1191                add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1192                add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS);
1193        }
1194
1195        add &= REG_TRAINING_DEBUG_2_MASK;
1196        reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1197        reg &=
1198            ~(REG_READ_DATA_READY_DELAYS_MASK <<
1199              (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1200        reg |=
1201            ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1202        reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1203        dram_info->rd_rdy_dly = rd_sample_delay + add;
1204
1205        for (cs = 0; cs < dram_info->num_cs; cs++) {
1206                for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
1207                        reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
1208                        dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
1209                }
1210        }
1211
1212        return MV_OK;
1213}
1214#endif
1215