LXR uboot/drivers/ddr/marvell/axp/ddr3_write

   1/*
   2 * Copyright (C) Marvell International Ltd. and its affiliates
   3 *
   4 * SPDX-License-Identifier:     GPL-2.0
   5 */
   6
   7#include <common.h>
   8#include <i2c.h>
   9#include <spl.h>
  10#include <asm/io.h>
  11#include <asm/arch/cpu.h>
  12#include <asm/arch/soc.h>
  13
  14#include "ddr3_hw_training.h"
  15
  16/*
  17 * Debug
  18 */
  19#define DEBUG_WL_C(s, d, l) \
  20        DEBUG_WL_S(s); DEBUG_WL_D(d, l); DEBUG_WL_S("\n")
  21#define DEBUG_WL_FULL_C(s, d, l) \
  22        DEBUG_WL_FULL_S(s); DEBUG_WL_FULL_D(d, l); DEBUG_WL_FULL_S("\n")
  23
  24#ifdef MV_DEBUG_WL
  25#define DEBUG_WL_S(s)                   puts(s)
  26#define DEBUG_WL_D(d, l)                printf("%x", d)
  27#define DEBUG_RL_S(s) \
  28        debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  29#define DEBUG_RL_D(d, l) \
  30        debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  31#else
  32#define DEBUG_WL_S(s)
  33#define DEBUG_WL_D(d, l)
  34#endif
  35
  36#ifdef MV_DEBUG_WL_FULL
  37#define DEBUG_WL_FULL_S(s)              puts(s)
  38#define DEBUG_WL_FULL_D(d, l)           printf("%x", d)
  39#else
  40#define DEBUG_WL_FULL_S(s)
  41#define DEBUG_WL_FULL_D(d, l)
  42#endif
  43
  44#define WL_SUP_EXPECTED_DATA            0x21
  45#define WL_SUP_READ_DRAM_ENTRY          0x8
  46
  47static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
  48                                         u32 *result,
  49                                         MV_DRAM_INFO *dram_info);
  50static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr,
  51                                    u32 data);
  52
  53extern u16 odt_static[ODT_OPT][MAX_CS];
  54extern u16 odt_dynamic[ODT_OPT][MAX_CS];
  55extern u32 wl_sup_pattern[LEN_WL_SUP_PATTERN];
  56
  57/*
  58 * Name:     ddr3_write_leveling_hw
  59 * Desc:     Execute Write leveling phase by HW
  60 * Args:     freq      - current sequence frequency
  61 *           dram_info   - main struct
  62 * Notes:
  63 * Returns:  MV_OK if success, MV_FAIL if fail.
  64 */
  65int ddr3_write_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  66{
  67        u32 reg, phase, delay, cs, pup;
  68#ifdef MV88F67XX
  69        int dpde_flag = 0;
  70#endif
  71        /* Debug message - Start Read leveling procedure */
  72        DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
  73
  74#ifdef MV88F67XX
  75        /* Dynamic pad issue (BTS669) during WL */
  76        reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
  77        if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
  78                dpde_flag = 1;
  79                reg_write(REG_DUNIT_CTRL_LOW_ADDR,
  80                          reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
  81        }
  82#endif
  83
  84        reg = 1 << REG_DRAM_TRAINING_WL_OFFS;
  85        /* Config the retest number */
  86        reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
  87        reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
  88        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  89
  90        reg =  reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  91                (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  92        reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  93
  94        /* Wait */
  95        do {
  96                reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  97                        (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  98        } while (reg);          /* Wait for '0' */
  99
 100        reg = reg_read(REG_DRAM_TRAINING_ADDR);
 101        /* Check if Successful */
 102        if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
 103                /*
 104                 * Read results to arrays - Results are required for WL
 105                 * High freq Supplement and DQS Centralization
 106                 */
 107                for (cs = 0; cs < MAX_CS; cs++) {
 108                        if (dram_info->cs_ena & (1 << cs)) {
 109                                for (pup = 0;
 110                                     pup < dram_info->num_of_total_pups;
 111                                     pup++) {
 112                                        if (pup == dram_info->num_of_std_pups
 113                                            && dram_info->ecc_ena)
 114                                                pup = ECC_PUP;
 115                                        reg =
 116                                            ddr3_read_pup_reg(PUP_WL_MODE, cs,
 117                                                              pup);
 118                                        phase =
 119                                            (reg >> REG_PHY_PHASE_OFFS) &
 120                                            PUP_PHASE_MASK;
 121                                        delay = reg & PUP_DELAY_MASK;
 122                                        dram_info->wl_val[cs][pup][P] = phase;
 123                                        dram_info->wl_val[cs][pup][D] = delay;
 124                                        dram_info->wl_val[cs][pup][S] =
 125                                            WL_HI_FREQ_STATE - 1;
 126                                        reg =
 127                                            ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
 128                                                              cs, pup);
 129                                        dram_info->wl_val[cs][pup][DQS] =
 130                                            (reg & 0x3F);
 131                                }
 132
 133#ifdef MV_DEBUG_WL
 134                                /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
 135                                DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
 136                                DEBUG_WL_D((u32) cs, 1);
 137                                DEBUG_WL_S(" Results:\n");
 138                                for (pup = 0;
 139                                     pup < dram_info->num_of_total_pups;
 140                                     pup++) {
 141                                        if (pup == dram_info->num_of_std_pups
 142                                            && dram_info->ecc_ena)
 143                                                pup = ECC_PUP;
 144                                        DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
 145                                        DEBUG_WL_D((u32) pup, 1);
 146                                        DEBUG_WL_S(", Phase: ");
 147                                        DEBUG_WL_D((u32)
 148                                                   dram_info->wl_val[cs][pup]
 149                                                   [P], 1);
 150                                        DEBUG_WL_S(", Delay: ");
 151                                        DEBUG_WL_D((u32)
 152                                                   dram_info->wl_val[cs][pup]
 153                                                   [D], 2);
 154                                        DEBUG_WL_S("\n");
 155                                }
 156#endif
 157                        }
 158                }
 159
 160                /* Dynamic pad issue (BTS669) during WL */
 161#ifdef MV88F67XX
 162                if (dpde_flag) {
 163                        reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 164                                (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 165                        reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 166                }
 167#endif
 168
 169                DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 170
 171                return MV_OK;
 172        } else {
 173                DEBUG_WL_S("DDR3 - Write Leveling - HW WL Error\n");
 174                return MV_FAIL;
 175        }
 176}
 177
 178/*
 179 * Name:     ddr3_wl_supplement
 180 * Desc:     Write Leveling Supplement
 181 * Args:     dram_info   - main struct
 182 * Notes:
 183 * Returns:  MV_OK if success, MV_FAIL if fail.
 184 */
 185int ddr3_wl_supplement(MV_DRAM_INFO *dram_info)
 186{
 187        u32 cs, cnt, pup_num, sum, phase, delay, max_pup_num, pup, sdram_offset;
 188        u32 tmp_count, ecc, reg;
 189        u32 ddr_width, tmp_pup, idx;
 190        u32 sdram_pup_val, uj;
 191        u32 one_clk_err = 0, align_err = 0, no_err = 0, err = 0, err_n = 0;
 192        u32 sdram_data[LEN_WL_SUP_PATTERN] __aligned(32) = { 0 };
 193
 194        ddr_width = dram_info->ddr_width;
 195        no_err = 0;
 196
 197        DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Starting\n");
 198
 199        switch (ddr_width) {
 200                /* Data error from pos-adge to pos-adge */
 201        case 16:
 202                one_clk_err = 4;
 203                align_err = 4;
 204                break;
 205        case 32:
 206                one_clk_err = 8;
 207                align_err = 8;
 208                break;
 209        case 64:
 210                one_clk_err = 0x10;
 211                align_err = 0x10;
 212                break;
 213        default:
 214                DEBUG_WL_S("Error - bus width!!!\n");
 215                return MV_FAIL;
 216        }
 217
 218        /* Enable SW override */
 219        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 220                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 221
 222        /* [0] = 1 - Enable SW override  */
 223        /* 0x15B8 - Training SW 2 Register */
 224        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 225        DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - SW Override Enabled\n");
 226        reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 227        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 228        tmp_count = 0;
 229        for (cs = 0; cs < MAX_CS; cs++) {
 230                if (dram_info->cs_ena & (1 << cs)) {
 231                        sum = 0;
 232                        /*
 233                         * 2 iterations loop: 1)actual WL results 2) fix WL
 234                         * if needed
 235                         */
 236                        for (cnt = 0; cnt < COUNT_WL_HI_FREQ; cnt++) {
 237                                DEBUG_WL_C("COUNT = ", cnt, 1);
 238                                for (ecc = 0; ecc < (dram_info->ecc_ena + 1);
 239                                     ecc++) {
 240                                        if (ecc) {
 241                                                DEBUG_WL_S("ECC PUP:\n");
 242                                        } else {
 243                                                DEBUG_WL_S("DATA PUP:\n");
 244                                        }
 245
 246                                        max_pup_num =
 247                                            dram_info->num_of_std_pups * (1 -
 248                                                                          ecc) +
 249                                            ecc;
 250                                        /* ECC Support - Switch ECC Mux on ecc=1 */
 251                                        reg =
 252                                            (reg_read(REG_DRAM_TRAINING_2_ADDR)
 253                                             & ~(1 <<
 254                                                 REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
 255                                        reg |=
 256                                            (dram_info->ecc_ena *
 257                                             ecc <<
 258                                             REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 259                                        reg_write(REG_DRAM_TRAINING_2_ADDR,
 260                                                  reg);
 261                                        ddr3_reset_phy_read_fifo();
 262
 263                                        /* Write to memory */
 264                                        sdram_offset =
 265                                            tmp_count * (SDRAM_CS_SIZE + 1) +
 266                                            0x200;
 267                                        if (MV_OK != ddr3_dram_sram_burst((u32)
 268                                                                          wl_sup_pattern,
 269                                                                          sdram_offset,
 270                                                                          LEN_WL_SUP_PATTERN))
 271                                                return MV_FAIL;
 272
 273                                        /* Read from memory */
 274                                        if (MV_OK !=
 275                                            ddr3_dram_sram_burst(sdram_offset,
 276                                                                 (u32)
 277                                                                 sdram_data,
 278                                                                 LEN_WL_SUP_PATTERN))
 279                                                return MV_FAIL;
 280
 281                                        /* Print the buffer */
 282                                        for (uj = 0; uj < LEN_WL_SUP_PATTERN;
 283                                             uj++) {
 284                                                if ((uj % 4 == 0) && (uj != 0)) {
 285                                                        DEBUG_WL_S("\n");
 286                                                }
 287                                                DEBUG_WL_D(sdram_data[uj],
 288                                                           8);
 289                                                DEBUG_WL_S(" ");
 290                                        }
 291
 292                                        /* Check pup which DQS/DATA is error */
 293                                        for (pup = 0; pup < max_pup_num; pup++) {
 294                                                /* ECC support - bit 8 */
 295                                                pup_num = (ecc) ? ECC_PUP : pup;
 296                                                if (pup < 4) {  /* lower 32 bit */
 297                                                        tmp_pup = pup;
 298                                                        idx =
 299                                                            WL_SUP_READ_DRAM_ENTRY;
 300                                                } else {        /* higher 32 bit */
 301                                                        tmp_pup = pup - 4;
 302                                                        idx =
 303                                                            WL_SUP_READ_DRAM_ENTRY
 304                                                            + 1;
 305                                                }
 306                                                DEBUG_WL_S("\nCS: ");
 307                                                DEBUG_WL_D((u32) cs, 1);
 308                                                DEBUG_WL_S(" PUP: ");
 309                                                DEBUG_WL_D((u32) pup_num, 1);
 310                                                DEBUG_WL_S("\n");
 311                                                sdram_pup_val =
 312                                                    ((sdram_data[idx] >>
 313                                                      ((tmp_pup) * 8)) & 0xFF);
 314                                                DEBUG_WL_C("Actual Data = ",
 315                                                           sdram_pup_val, 2);
 316                                                DEBUG_WL_C("Expected Data = ",
 317                                                           (WL_SUP_EXPECTED_DATA
 318                                                            + pup), 2);
 319                                                /*
 320                                                 * ALINGHMENT: calculate
 321                                                 * expected data vs actual data
 322                                                 */
 323                                                err =
 324                                                    (WL_SUP_EXPECTED_DATA +
 325                                                     pup) - sdram_pup_val;
 326                                                /*
 327                                                 * CLOCK LONG: calculate
 328                                                 * expected data vs actual data
 329                                                 */
 330                                                err_n =
 331                                                    sdram_pup_val -
 332                                                    (WL_SUP_EXPECTED_DATA +
 333                                                     pup);
 334                                                DEBUG_WL_C("err = ", err, 2);
 335                                                DEBUG_WL_C("err_n = ", err_n,
 336                                                           2);
 337                                                if (err == no_err) {
 338                                                        /* PUP is correct - increment State */
 339                                                        dram_info->wl_val[cs]
 340                                                            [pup_num]
 341                                                            [S] = 1;
 342                                                } else if (err_n == one_clk_err) {
 343                                                        /* clock is longer than DQS */
 344                                                        phase =
 345                                                            ((dram_info->wl_val
 346                                                              [cs]
 347                                                              [pup_num][P] +
 348                                                              WL_HI_FREQ_SHIFT)
 349                                                             % MAX_PHASE_2TO1);
 350                                                        dram_info->wl_val[cs]
 351                                                            [pup_num]
 352                                                            [P] = phase;
 353                                                        delay =
 354                                                            dram_info->wl_val
 355                                                            [cs][pup_num]
 356                                                            [D];
 357                                                        DEBUG_WL_S("#### Clock is longer than DQS more than one clk cycle ####\n");
 358                                                        ddr3_write_pup_reg
 359                                                            (PUP_WL_MODE, cs,
 360                                                             pup * (1 - ecc) +
 361                                                             ECC_PUP * ecc,
 362                                                             phase, delay);
 363                                                } else if (err == align_err) {
 364                                                        /* clock is align to DQS */
 365                                                        phase =
 366                                                            dram_info->wl_val
 367                                                            [cs][pup_num]
 368                                                            [P];
 369                                                        delay =
 370                                                            dram_info->wl_val
 371                                                            [cs][pup_num]
 372                                                            [D];
 373                                                        DEBUG_WL_S("#### Alignment PUPS problem ####\n");
 374                                                        if ((phase == 0)
 375                                                            || ((phase == 1)
 376                                                                && (delay <=
 377                                                                    0x10))) {
 378                                                                DEBUG_WL_S("#### Warning - Possible Layout Violation (DQS is longer than CLK)####\n");
 379                                                        }
 380
 381                                                        phase = 0x0;
 382                                                        delay = 0x0;
 383                                                        dram_info->wl_val[cs]
 384                                                            [pup_num]
 385                                                            [P] = phase;
 386                                                        dram_info->wl_val[cs]
 387                                                            [pup_num]
 388                                                            [D] = delay;
 389                                                        ddr3_write_pup_reg
 390                                                            (PUP_WL_MODE, cs,
 391                                                             pup * (1 - ecc) +
 392                                                             ECC_PUP * ecc,
 393                                                             phase, delay);
 394                                                }
 395                                                /* Stop condition for ECC phase */
 396                                                pup = (ecc) ? max_pup_num : pup;
 397                                        }
 398
 399                                        /* ECC Support - Disable ECC MUX */
 400                                        reg =
 401                                            (reg_read(REG_DRAM_TRAINING_2_ADDR)
 402                                             & ~(1 <<
 403                                                 REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
 404                                        reg_write(REG_DRAM_TRAINING_2_ADDR,
 405                                                  reg);
 406                                }
 407                        }
 408
 409                        for (pup = 0; pup < dram_info->num_of_std_pups; pup++)
 410                                sum += dram_info->wl_val[cs][pup][S];
 411
 412                        if (dram_info->ecc_ena)
 413                                sum += dram_info->wl_val[cs][ECC_PUP][S];
 414
 415                        /* Checks if any pup is not locked after the change */
 416                        if (sum < (WL_HI_FREQ_STATE * (dram_info->num_of_total_pups))) {
 417                                DEBUG_WL_C("DDR3 - Write Leveling Hi-Freq Supplement - didn't work for Cs - ",
 418                                           (u32) cs, 1);
 419                                return MV_FAIL;
 420                        }
 421                        tmp_count++;
 422                }
 423        }
 424
 425        dram_info->wl_max_phase = 0;
 426        dram_info->wl_min_phase = 10;
 427
 428        /*
 429         * Read results to arrays - Results are required for DQS Centralization
 430         */
 431        for (cs = 0; cs < MAX_CS; cs++) {
 432                if (dram_info->cs_ena & (1 << cs)) {
 433                        for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
 434                                if (pup == dram_info->num_of_std_pups
 435                                    && dram_info->ecc_ena)
 436                                        pup = ECC_PUP;
 437                                reg = ddr3_read_pup_reg(PUP_WL_MODE, cs, pup);
 438                                phase =
 439                                    (reg >> REG_PHY_PHASE_OFFS) &
 440                                    PUP_PHASE_MASK;
 441                                if (phase > dram_info->wl_max_phase)
 442                                        dram_info->wl_max_phase = phase;
 443                                if (phase < dram_info->wl_min_phase)
 444                                        dram_info->wl_min_phase = phase;
 445                        }
 446                }
 447        }
 448
 449        /* Disable SW override - Must be in a different stage */
 450        /* [0]=0 - Enable SW override  */
 451        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 452        reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 453        /* 0x15B8 - Training SW 2 Register */
 454        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 455
 456        reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 457                (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 458        reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 459
 460        DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Ended Successfully\n");
 461
 462        return MV_OK;
 463}
 464
 465/*
 466 * Name:     ddr3_write_leveling_hw_reg_dimm
 467 * Desc:     Execute Write leveling phase by HW
 468 * Args:     freq      - current sequence frequency
 469 *           dram_info   - main struct
 470 * Notes:
 471 * Returns:  MV_OK if success, MV_FAIL if fail.
 472 */
 473int ddr3_write_leveling_hw_reg_dimm(u32 freq, MV_DRAM_INFO *dram_info)
 474{
 475        u32 reg, phase, delay, cs, pup, pup_num;
 476        __maybe_unused int dpde_flag = 0;
 477
 478        /* Debug message - Start Read leveling procedure */
 479        DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
 480
 481        if (dram_info->num_cs > 2) {
 482                DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 483                return MV_NO_CHANGE;
 484        }
 485
 486        /* If target freq = 400 move clock start point */
 487        /* Write to control PUP to Control Deskew Regs */
 488        if (freq <= DDR_400) {
 489                for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
 490                        /* PUP_DELAY_MASK 0x1F */
 491                        /* reg = 0x0C10001F + (uj << 16); */
 492                        ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
 493                                                0x1F);
 494                }
 495        }
 496
 497#ifdef MV88F67XX
 498        /* Dynamic pad issue (BTS669) during WL */
 499        reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 500        if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 501                dpde_flag = 1;
 502                reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 503                          reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 504        }
 505#endif
 506
 507        reg = (1 << REG_DRAM_TRAINING_WL_OFFS);
 508        /* Config the retest number */
 509        reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
 510        reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
 511        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 512
 513        reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
 514                (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 515        reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
 516
 517        /* Wait */
 518        do {
 519                reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
 520                        (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 521        } while (reg);          /* Wait for '0' */
 522
 523        reg = reg_read(REG_DRAM_TRAINING_ADDR);
 524        /* Check if Successful */
 525        if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
 526                /*
 527                 * Read results to arrays - Results are required for WL High
 528                 * freq Supplement and DQS Centralization
 529                 */
 530                for (cs = 0; cs < MAX_CS; cs++) {
 531                        if (dram_info->cs_ena & (1 << cs)) {
 532                                for (pup = 0;
 533                                     pup < dram_info->num_of_total_pups;
 534                                     pup++) {
 535                                        if (pup == dram_info->num_of_std_pups
 536                                            && dram_info->ecc_ena)
 537                                                pup = ECC_BIT;
 538                                        reg =
 539                                            ddr3_read_pup_reg(PUP_WL_MODE, cs,
 540                                                              pup);
 541                                        phase =
 542                                            (reg >> REG_PHY_PHASE_OFFS) &
 543                                            PUP_PHASE_MASK;
 544                                        delay = reg & PUP_DELAY_MASK;
 545                                        dram_info->wl_val[cs][pup][P] = phase;
 546                                        dram_info->wl_val[cs][pup][D] = delay;
 547                                        if ((phase == 1) && (delay >= 0x1D)) {
 548                                                /*
 549                                                 * Need to do it here for
 550                                                 * uncorrect WL values
 551                                                 */
 552                                                ddr3_write_pup_reg(PUP_WL_MODE,
 553                                                                   cs, pup, 0,
 554                                                                   0);
 555                                                dram_info->wl_val[cs][pup][P] =
 556                                                    0;
 557                                                dram_info->wl_val[cs][pup][D] =
 558                                                    0;
 559                                        }
 560                                        dram_info->wl_val[cs][pup][S] =
 561                                            WL_HI_FREQ_STATE - 1;
 562                                        reg =
 563                                            ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
 564                                                              cs, pup);
 565                                        dram_info->wl_val[cs][pup][DQS] =
 566                                            (reg & 0x3F);
 567                                }
 568#ifdef MV_DEBUG_WL
 569                                /*
 570                                 * Debug message - Print res for cs[i]:
 571                                 * cs,PUP,Phase,Delay
 572                                 */
 573                                DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
 574                                DEBUG_WL_D((u32) cs, 1);
 575                                DEBUG_WL_S(" Results:\n");
 576                                for (pup = 0;
 577                                     pup < dram_info->num_of_total_pups;
 578                                     pup++) {
 579                                        DEBUG_WL_S
 580                                            ("DDR3 - Write Leveling - PUP: ");
 581                                        DEBUG_WL_D((u32) pup, 1);
 582                                        DEBUG_WL_S(", Phase: ");
 583                                        DEBUG_WL_D((u32)
 584                                                   dram_info->wl_val[cs][pup]
 585                                                   [P], 1);
 586                                        DEBUG_WL_S(", Delay: ");
 587                                        DEBUG_WL_D((u32)
 588                                                   dram_info->wl_val[cs][pup]
 589                                                   [D], 2);
 590                                        DEBUG_WL_S("\n");
 591                                }
 592#endif
 593                        }
 594                }
 595
 596#ifdef MV88F67XX
 597                /* Dynamic pad issue (BTS669) during WL */
 598                if (dpde_flag) {
 599                        reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 600                                (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 601                        reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 602                }
 603#endif
 604                DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 605
 606                /* If target freq = 400 move clock back */
 607                /* Write to control PUP to Control Deskew Regs */
 608                if (freq <= DDR_400) {
 609                        for (pup = 0; pup <= dram_info->num_of_total_pups;
 610                             pup++) {
 611                                ddr3_write_ctrl_pup_reg(1, pup,
 612                                                        CNTRL_PUP_DESKEW + pup, 0);
 613                        }
 614                }
 615
 616                return MV_OK;
 617        } else {
 618                /* Configure Each PUP with locked leveling settings */
 619                for (cs = 0; cs < MAX_CS; cs++) {
 620                        if (dram_info->cs_ena & (1 << cs)) {
 621                                for (pup = 0;
 622                                     pup < dram_info->num_of_total_pups;
 623                                     pup++) {
 624                                        /* ECC support - bit 8 */
 625                                        pup_num = (pup == dram_info->num_of_std_pups) ?
 626                                                ECC_BIT : pup;
 627                                        ddr3_write_pup_reg(PUP_WL_MODE, cs,
 628                                                           pup_num, 0, 0);
 629                                }
 630                        }
 631                }
 632
 633                reg_write(REG_DRAM_TRAINING_ADDR, 0);
 634
 635                /* If target freq = 400 move clock back */
 636                /* Write to control PUP to Control Deskew Regs */
 637                if (freq <= DDR_400) {
 638                        for (pup = 0; pup <= dram_info->num_of_total_pups;
 639                             pup++) {
 640                                ddr3_write_ctrl_pup_reg(1, pup,
 641                                                        CNTRL_PUP_DESKEW + pup, 0);
 642                        }
 643                }
 644
 645                DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 646                return MV_NO_CHANGE;
 647        }
 648}
 649
 650/*
 651 * Name:     ddr3_write_leveling_sw
 652 * Desc:     Execute Write leveling phase by SW
 653 * Args:     freq      - current sequence frequency
 654 *           dram_info   - main struct
 655 * Notes:
 656 * Returns:  MV_OK if success, MV_FAIL if fail.
 657 */
 658int ddr3_write_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
 659{
 660        u32 reg, cs, cnt, pup, max_pup_num;
 661        u32 res[MAX_CS];
 662        max_pup_num = dram_info->num_of_total_pups;
 663        __maybe_unused int dpde_flag = 0;
 664
 665        /* Debug message - Start Write leveling procedure */
 666        DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
 667
 668#ifdef MV88F67XX
 669        /* Dynamic pad issue (BTS669) during WL */
 670        reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 671        if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 672                dpde_flag = 1;
 673                reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 674                          reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 675        }
 676#endif
 677
 678        /* Set Output buffer-off to all CS and correct ODT values */
 679        for (cs = 0; cs < MAX_CS; cs++) {
 680                if (dram_info->cs_ena & (1 << cs)) {
 681                        reg = reg_read(REG_DDR3_MR1_ADDR) &
 682                                REG_DDR3_MR1_ODT_MASK;
 683                        reg |= odt_static[dram_info->cs_ena][cs];
 684                        reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 685
 686                        /* 0x15D0 - DDR3 MR0 Register */
 687                        reg_write(REG_DDR3_MR1_ADDR, reg);
 688                        /* Issue MRS Command to current cs */
 689                        reg = REG_SDRAM_OPERATION_CMD_MR1 &
 690                                ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 691                        /*
 692                         * [3-0] = 0x4 - MR1 Command, [11-8] -
 693                         * enable current cs
 694                         */
 695                        /* 0x1418 - SDRAM Operation Register */
 696                        reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 697
 698                        udelay(MRS_DELAY);
 699                }
 700        }
 701
 702        DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
 703
 704        /* Enable SW override */
 705        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 706                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 707        /* [0] = 1 - Enable SW override  */
 708        /* 0x15B8 - Training SW 2 Register */
 709        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 710        DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
 711
 712        /* Enable PHY write leveling mode */
 713        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 714                ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 715        /* [2] = 0 - TrnWLMode - Enable */
 716        /* 0x15B8 - Training SW 2 Register */
 717        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 718        /* Reset WL results arry */
 719        memset(dram_info->wl_val, 0, sizeof(u32) * MAX_CS * MAX_PUP_NUM * 7);
 720
 721        /* Loop for each cs */
 722        for (cs = 0; cs < MAX_CS; cs++) {
 723                if (dram_info->cs_ena & (1 << cs)) {
 724                        DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
 725                                        (u32) cs, 1);
 726                        /* Refresh X9 current cs */
 727                        DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
 728                        for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
 729                                reg =
 730                                    REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
 731                                                                     (REG_SDRAM_OPERATION_CS_OFFS
 732                                                                      + cs));
 733                                /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
 734                                reg_write(REG_SDRAM_OPERATION_ADDR, reg);       /* 0x1418 - SDRAM Operation Register */
 735
 736                                do {
 737                                        reg =
 738                                            ((reg_read
 739                                              (REG_SDRAM_OPERATION_ADDR)) &
 740                                             REG_SDRAM_OPERATION_CMD_RFRS_DONE);
 741                                } while (reg);  /* Wait for '0' */
 742                        }
 743
 744                        /* Configure MR1 in Cs[CsNum] - write leveling on, output buffer on */
 745                        DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
 746                        reg = reg_read(REG_DDR3_MR1_ADDR) &
 747                                REG_DDR3_MR1_OUTBUF_WL_MASK;
 748                        /* Set ODT Values */
 749                        reg &= REG_DDR3_MR1_ODT_MASK;
 750                        reg |= odt_static[dram_info->cs_ena][cs];
 751                        /* Enable WL MODE */
 752                        reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
 753                        /* [7]=1, [12]=0 - Output Buffer and write leveling enabled */
 754                        reg_write(REG_DDR3_MR1_ADDR, reg);      /* 0x15D4 - DDR3 MR1 Register */
 755                        /* Issue MRS Command to current cs */
 756                        reg = REG_SDRAM_OPERATION_CMD_MR1 &
 757                                ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 758                        /*
 759                         * [3-0] = 0x4 - MR1 Command, [11-8] -
 760                         * enable current cs
 761                         */
 762                        /* 0x1418 - SDRAM Operation Register */
 763                        reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 764
 765                        udelay(MRS_DELAY);
 766
 767                        /* Write leveling  cs[cs] */
 768                        if (MV_OK !=
 769                            ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
 770                                                          (u32 *)(res + cs),
 771                                                          dram_info)) {
 772                                DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
 773                                                (u32) cs, 1);
 774                                for (pup = 0; pup < max_pup_num; pup++) {
 775                                        if (((res[cs] >> pup) & 0x1) == 0) {
 776                                                DEBUG_WL_C("Failed Byte : ",
 777                                                           pup, 1);
 778                                        }
 779                                }
 780                                return MV_FAIL;
 781                        }
 782
 783                        /* Set TrnWLDeUpd - After each CS is done */
 784                        reg = reg_read(REG_TRAINING_WL_ADDR) |
 785                                (1 << REG_TRAINING_WL_CS_DONE_OFFS);
 786                        /* 0x16AC - Training Write leveling register */
 787                        reg_write(REG_TRAINING_WL_ADDR, reg);
 788
 789                        /*
 790                         * Debug message - Finished Write leveling cs[cs] -
 791                         * each PUP Fail/Success
 792                         */
 793                        DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
 794                                        1);
 795                        DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
 796                                        (u32) res[cs], 3);
 797
 798                        /*
 799                         * Configure MR1 in cs[cs] - write leveling off (0),
 800                         * output buffer off (1)
 801                         */
 802                        reg = reg_read(REG_DDR3_MR1_ADDR) &
 803                                REG_DDR3_MR1_OUTBUF_WL_MASK;
 804                        reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 805                        /* No need to sort ODT since it is same CS */
 806                        /* 0x15D4 - DDR3 MR1 Register */
 807                        reg_write(REG_DDR3_MR1_ADDR, reg);
 808                        /* Issue MRS Command to current cs */
 809                        reg = REG_SDRAM_OPERATION_CMD_MR1 &
 810                                ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 811                        /*
 812                         * [3-0] = 0x4 - MR1 Command, [11-8] -
 813                         * enable current cs
 814                         */
 815                        /* 0x1418 - SDRAM Operation Register */
 816                        reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 817
 818                        udelay(MRS_DELAY);
 819                }
 820        }
 821
 822        /* Disable WL Mode */
 823        /* [2]=1 - TrnWLMode - Disable */
 824        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 825        reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 826        /* 0x15B8 - Training SW 2 Register */
 827        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 828
 829        /* Disable SW override - Must be in a different stage */
 830        /* [0]=0 - Enable SW override  */
 831        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 832        reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 833        /* 0x15B8 - Training SW 2 Register */
 834        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 835
 836        /* Set Output buffer-on to all CS and correct ODT values */
 837        for (cs = 0; cs < MAX_CS; cs++) {
 838                if (dram_info->cs_ena & (1 << cs)) {
 839                        reg = reg_read(REG_DDR3_MR1_ADDR) &
 840                                REG_DDR3_MR1_ODT_MASK;
 841                        reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
 842                        reg |= odt_static[dram_info->cs_ena][cs];
 843
 844                        /* 0x15D0 - DDR3 MR1 Register */
 845                        reg_write(REG_DDR3_MR1_ADDR, reg);
 846                        /* Issue MRS Command to current cs */
 847                        reg = REG_SDRAM_OPERATION_CMD_MR1 &
 848                                ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 849                        /*
 850                         * [3-0] = 0x4 - MR1 Command, [11-8] -
 851                         * enable current cs
 852                         */
 853                        /* 0x1418 - SDRAM Operation Register */
 854                        reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 855
 856                        udelay(MRS_DELAY);
 857                }
 858        }
 859
 860#ifdef MV88F67XX
 861        /* Dynamic pad issue (BTS669) during WL */
 862        if (dpde_flag) {
 863                reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 864                        (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 865                reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 866        }
 867#endif
 868        DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
 869
 870        return MV_OK;
 871}
 872
 873#if !defined(MV88F672X)
 874/*
 875 * Name:     ddr3_write_leveling_sw
 876 * Desc:     Execute Write leveling phase by SW
 877 * Args:     freq        - current sequence frequency
 878 *           dram_info   - main struct
 879 * Notes:
 880 * Returns:  MV_OK if success, MV_FAIL if fail.
 881 */
 882int ddr3_write_leveling_sw_reg_dimm(u32 freq, int ratio_2to1,
 883                                    MV_DRAM_INFO *dram_info)
 884{
 885        u32 reg, cs, cnt, pup;
 886        u32 res[MAX_CS];
 887        __maybe_unused int dpde_flag = 0;
 888
 889        /* Debug message - Start Write leveling procedure */
 890        DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
 891
 892#ifdef MV88F67XX
 893        /* Dynamic pad issue (BTS669) during WL */
 894        reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 895        if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 896                dpde_flag = 1;
 897                reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 898                          reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 899        }
 900#endif
 901
 902        /* If target freq = 400 move clock start point */
 903        /* Write to control PUP to Control Deskew Regs */
 904        if (freq <= DDR_400) {
 905                for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
 906                        /* PUP_DELAY_MASK 0x1F */
 907                        /* reg = 0x0C10001F + (uj << 16); */
 908                        ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
 909                                                0x1F);
 910                }
 911        }
 912
 913        /* Set Output buffer-off to all CS and correct ODT values */
 914        for (cs = 0; cs < MAX_CS; cs++) {
 915                if (dram_info->cs_ena & (1 << cs)) {
 916                        reg = reg_read(REG_DDR3_MR1_ADDR) &
 917                                REG_DDR3_MR1_ODT_MASK;
 918                        reg |= odt_static[dram_info->cs_ena][cs];
 919                        reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 920
 921                        /* 0x15D0 - DDR3 MR0 Register */
 922                        reg_write(REG_DDR3_MR1_ADDR, reg);
 923                        /* Issue MRS Command to current cs */
 924                        reg = REG_SDRAM_OPERATION_CMD_MR1 &
 925                                ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 926                        /*
 927                         * [3-0] = 0x4 - MR1 Command, [11-8] -
 928                         * enable current cs
 929                         */
 930                        /* 0x1418 - SDRAM Operation Register */
 931                        reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 932
 933                        udelay(MRS_DELAY);
 934                }
 935        }
 936
 937        DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
 938
 939        /* Enable SW override */
 940        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 941                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 942        /* [0] = 1 - Enable SW override  */
 943        /* 0x15B8 - Training SW 2 Register */
 944        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 945        DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
 946
 947        /* Enable PHY write leveling mode */
 948        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 949                ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 950        /* [2] = 0 - TrnWLMode - Enable */
 951        /* 0x15B8 - Training SW 2 Register */
 952        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 953
 954        /* Loop for each cs */
 955        for (cs = 0; cs < MAX_CS; cs++) {
 956                if (dram_info->cs_ena & (1 << cs)) {
 957                        DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
 958                                        (u32) cs, 1);
 959
 960                        /* Refresh X9 current cs */
 961                        DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
 962                        for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
 963                                reg =
 964                                    REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
 965                                                                     (REG_SDRAM_OPERATION_CS_OFFS
 966                                                                      + cs));
 967                                /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
 968                                reg_write(REG_SDRAM_OPERATION_ADDR, reg);       /* 0x1418 - SDRAM Operation Register */
 969
 970                                do {
 971                                        reg =
 972                                            ((reg_read
 973                                              (REG_SDRAM_OPERATION_ADDR)) &
 974                                             REG_SDRAM_OPERATION_CMD_RFRS_DONE);
 975                                } while (reg);  /* Wait for '0' */
 976                        }
 977
 978                        /*
 979                         * Configure MR1 in Cs[CsNum] - write leveling on,
 980                         * output buffer on
 981                         */
 982                        DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
 983                        reg = reg_read(REG_DDR3_MR1_ADDR) &
 984                                REG_DDR3_MR1_OUTBUF_WL_MASK;
 985                        /* Set ODT Values */
 986                        reg &= REG_DDR3_MR1_ODT_MASK;
 987                        reg |= odt_static[dram_info->cs_ena][cs];
 988                        /* Enable WL MODE */
 989                        reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
 990                        /*
 991                         * [7]=1, [12]=0 - Output Buffer and write leveling
 992                         * enabled
 993                         */
 994                        /* 0x15D4 - DDR3 MR1 Register */
 995                        reg_write(REG_DDR3_MR1_ADDR, reg);
 996                        /* Issue MRS Command to current cs */
 997                        reg = REG_SDRAM_OPERATION_CMD_MR1 &
 998                                ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 999                        /*
1000                         * [3-0] = 0x4 - MR1 Command, [11-8] -

1001                         * enable current cs
1002                         */
1003                        /* 0x1418 - SDRAM Operation Register */
1004                        reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1005
1006                        udelay(MRS_DELAY);
1007
1008                        /* Write leveling  cs[cs] */
1009                        if (MV_OK !=
1010                            ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
1011                                                          (u32 *)(res + cs),
1012                                                          dram_info)) {
1013                                DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
1014                                                (u32) cs, 1);
1015                                return MV_FAIL;
1016                        }
1017
1018                        /* Set TrnWLDeUpd - After each CS is done */
1019                        reg = reg_read(REG_TRAINING_WL_ADDR) |
1020                                (1 << REG_TRAINING_WL_CS_DONE_OFFS);
1021                        /* 0x16AC - Training Write leveling register */
1022                        reg_write(REG_TRAINING_WL_ADDR, reg);
1023
1024                        /*
1025                         * Debug message - Finished Write leveling cs[cs] -
1026                         * each PUP Fail/Success
1027                         */
1028                        DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
1029                                        1);
1030                        DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
1031                                        (u32) res[cs], 3);
1032
1033                        /* Configure MR1 in cs[cs] - write leveling off (0), output buffer off (1) */
1034                        reg = reg_read(REG_DDR3_MR1_ADDR) &
1035                                REG_DDR3_MR1_OUTBUF_WL_MASK;
1036                        reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
1037                        /* No need to sort ODT since it is same CS */
1038                        /* 0x15D4 - DDR3 MR1 Register */
1039                        reg_write(REG_DDR3_MR1_ADDR, reg);
1040                        /* Issue MRS Command to current cs */
1041                        reg = REG_SDRAM_OPERATION_CMD_MR1 &
1042                                ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1043                        /*
1044                         * [3-0] = 0x4 - MR1 Command, [11-8] -
1045                         * enable current cs
1046                         */
1047                        /* 0x1418 - SDRAM Operation Register */
1048                        reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1049
1050                        udelay(MRS_DELAY);
1051                }
1052        }
1053
1054        /* Disable WL Mode */
1055        /* [2]=1 - TrnWLMode - Disable */
1056        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1057        reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
1058        /* 0x15B8 - Training SW 2 Register */
1059        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1060
1061        /* Disable SW override - Must be in a different stage */
1062        /* [0]=0 - Enable SW override  */
1063        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1064        reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
1065        /* 0x15B8 - Training SW 2 Register */
1066        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1067
1068        /* Set Output buffer-on to all CS and correct ODT values */
1069        for (cs = 0; cs < MAX_CS; cs++) {
1070                if (dram_info->cs_ena & (1 << cs)) {
1071                        reg = reg_read(REG_DDR3_MR1_ADDR) &
1072                                REG_DDR3_MR1_ODT_MASK;
1073                        reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
1074                        reg |= odt_static[dram_info->cs_ena][cs];
1075
1076                        /* 0x15D0 - DDR3 MR1 Register */
1077                        reg_write(REG_DDR3_MR1_ADDR, reg);
1078                        /* Issue MRS Command to current cs */
1079                        reg = REG_SDRAM_OPERATION_CMD_MR1 &
1080                                ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1081                        /*
1082                         * [3-0] = 0x4 - MR1 Command, [11-8] -
1083                         * enable current cs
1084                         */
1085                        /* 0x1418 - SDRAM Operation Register */
1086                        reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1087
1088                        udelay(MRS_DELAY);
1089                }
1090        }
1091
1092#ifdef MV88F67XX
1093        /* Dynamic pad issue (BTS669) during WL */
1094        if (dpde_flag) {
1095                reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
1096                        (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
1097                reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
1098        }
1099#endif
1100
1101        /* If target freq = 400 move clock back */
1102        /* Write to control PUP to Control Deskew Regs */
1103        if (freq <= DDR_400) {
1104                for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
1105                        ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
1106                                                0);
1107                }
1108        }
1109
1110        DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
1111        return MV_OK;
1112}
1113#endif
1114
1115/*
1116 * Name:     ddr3_write_leveling_single_cs
1117 * Desc:     Execute Write leveling for single Chip select
1118 * Args:     cs          - current chip select
1119 *           freq        - current sequence frequency
1120 *           result      - res array
1121 *           dram_info   - main struct
1122 * Notes:
1123 * Returns:  MV_OK if success, MV_FAIL if fail.
1124 */
1125static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
1126                                         u32 *result, MV_DRAM_INFO *dram_info)
1127{
1128        u32 reg, pup_num, delay, phase, phaseMax, max_pup_num, pup,
1129                max_pup_mask;
1130
1131        max_pup_num = dram_info->num_of_total_pups;
1132        *result = 0;
1133        u32 flag[MAX_PUP_NUM] = { 0 };
1134
1135        DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - WL for Cs - ",
1136                        (u32) cs, 1);
1137
1138        switch (max_pup_num) {
1139        case 2:
1140                max_pup_mask = 0x3;
1141                break;
1142        case 4:
1143                max_pup_mask = 0xf;
1144                DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1145                break;
1146        case 5:
1147                max_pup_mask = 0x1f;
1148                DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1149                break;
1150        case 8:
1151                max_pup_mask = 0xff;
1152                DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1153                break;
1154        case 9:
1155                max_pup_mask = 0x1ff;
1156                DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1157                break;
1158        default:
1159                DEBUG_WL_C("ddr3_write_leveling_single_cs wrong max_pup_num =  ",
1160                           max_pup_num, 3);
1161                return MV_FAIL;
1162        }
1163
1164        /* CS ODT Override */
1165        reg = reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1166                REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1167        reg |= (REG_SDRAM_ODT_CTRL_HIGH_OVRD_ENA << (2 * cs));
1168        /* Set 0x3 - Enable ODT on the curent cs and disable on other cs */
1169        /* 0x1498 - SDRAM ODT Control high */
1170        reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1171
1172        DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - ODT Asserted for current Cs\n");
1173
1174        /* tWLMRD Delay */
1175        /* Delay of minimum 40 Dram clock cycles - 20 Tclk cycles */
1176        udelay(1);
1177
1178        /* [1:0] - current cs number */
1179        reg = (reg_read(REG_TRAINING_WL_ADDR) & REG_TRAINING_WL_CS_MASK) | cs;
1180        reg |= (1 << REG_TRAINING_WL_UPD_OFFS); /* [2] - trnWLCsUpd */
1181        /* 0x16AC - Training Write leveling register */
1182        reg_write(REG_TRAINING_WL_ADDR, reg);
1183
1184        /* Broadcast to all PUPs: Reset DQS phase, reset leveling delay */
1185        ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, 0, 0);
1186
1187        /* Seek Edge */
1188        DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Current Cs\n");
1189
1190        /* Drive DQS high for one cycle - All data PUPs */
1191        DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Driving DQS high for one cycle\n");
1192        if (!ratio_2to1) {
1193                reg = (reg_read(REG_TRAINING_WL_ADDR) &
1194                       REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_1TO1;
1195        } else {
1196                reg = (reg_read(REG_TRAINING_WL_ADDR) &
1197                       REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_2TO1;
1198        }
1199        /* 0x16AC - Training Write leveling register */
1200        reg_write(REG_TRAINING_WL_ADDR, reg);
1201
1202        /* Wait tWLdelay */
1203        do {
1204                /* [29] - trnWLDelayExp */
1205                reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1206                        REG_TRAINING_WL_DELAYEXP_MASK;
1207        } while (reg == 0x0);   /* Wait for '1' */
1208
1209        /* Read WL res */
1210        reg = (reg_read(REG_TRAINING_WL_ADDR) >> REG_TRAINING_WL_RESULTS_OFFS) &
1211                REG_TRAINING_WL_RESULTS_MASK;
1212        /* [28:20] - TrnWLResult */
1213
1214        if (!ratio_2to1) /* Different phase options for 2:1 or 1:1 modes */
1215                phaseMax = MAX_PHASE_1TO1;
1216        else
1217                phaseMax = MAX_PHASE_2TO1;
1218
1219        DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Shift DQS + Octet Leveling\n");
1220
1221        /* Shift DQS + Octet leveling */
1222        for (phase = 0; phase < phaseMax; phase++) {
1223                for (delay = 0; delay < MAX_DELAY; delay++) {
1224                        /*  Broadcast to all PUPs: DQS phase,leveling delay */
1225                        ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, phase,
1226                                           delay);
1227
1228                        udelay(1);      /* Delay of  3 Tclk cycles */
1229
1230                        DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge: Phase = ");
1231                        DEBUG_WL_FULL_D((u32) phase, 1);
1232                        DEBUG_WL_FULL_S(", Delay = ");
1233                        DEBUG_WL_FULL_D((u32) delay, 1);
1234                        DEBUG_WL_FULL_S("\n");
1235
1236                        /* Drive DQS high for one cycle - All data PUPs */
1237                        if (!ratio_2to1) {
1238                                reg = (reg_read(REG_TRAINING_WL_ADDR) &
1239                                       REG_TRAINING_WL_RATIO_MASK) |
1240                                        REG_TRAINING_WL_1TO1;
1241                        } else {
1242                                reg = (reg_read(REG_TRAINING_WL_ADDR) &
1243                                       REG_TRAINING_WL_RATIO_MASK) |
1244                                        REG_TRAINING_WL_2TO1;
1245                        }
1246                        reg_write(REG_TRAINING_WL_ADDR, reg);   /* 0x16AC  */
1247
1248                        /* Wait tWLdelay */
1249                        do {
1250                                reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1251                                        REG_TRAINING_WL_DELAYEXP_MASK;
1252                        } while (reg == 0x0);   /* [29] Wait for '1' */
1253
1254                        /* Read WL res */
1255                        reg = reg_read(REG_TRAINING_WL_ADDR);
1256                        reg = (reg >> REG_TRAINING_WL_RESULTS_OFFS) &
1257                                REG_TRAINING_WL_RESULTS_MASK;   /* [28:20] */
1258
1259                        DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - Seek Edge: Results =  ",
1260                                        (u32) reg, 3);
1261
1262                        /* Update State machine */
1263                        for (pup = 0; pup < (max_pup_num); pup++) {
1264                                /* ECC support - bit 8 */
1265                                pup_num = (pup == dram_info->num_of_std_pups) ?
1266                                        ECC_BIT : pup;
1267                                if (dram_info->wl_val[cs][pup][S] == 0) {
1268                                        /* Update phase to PUP */
1269                                        dram_info->wl_val[cs][pup][P] = phase;
1270                                        /* Update delay to PUP */
1271                                        dram_info->wl_val[cs][pup][D] = delay;
1272                                }
1273
1274                                if (((reg >> pup_num) & 0x1) == 0)
1275                                        flag[pup_num] = 1;
1276
1277                                if (((reg >> pup_num) & 0x1)
1278                                    && (flag[pup_num] == 1)
1279                                    && (dram_info->wl_val[cs][pup][S] == 0)) {
1280                                        /*
1281                                         * If the PUP is locked now and in last
1282                                         * counter states
1283                                         */
1284                                        /* Go to next state */
1285                                        dram_info->wl_val[cs][pup][S] = 1;
1286                                        /* Set res */
1287                                        *result = *result | (1 << pup_num);
1288                                }
1289                        }
1290
1291                        /* If all locked - Break the loops - Finished */
1292                        if (*result == max_pup_mask) {
1293                                phase = phaseMax;
1294                                delay = MAX_DELAY;
1295                                DEBUG_WL_S("DDR3 - Write Leveling Single Cs - Seek Edge: All Locked\n");
1296                        }
1297                }
1298        }
1299
1300        /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
1301        DEBUG_WL_C("DDR3 - Write Leveling - Results for CS - ", (u32) cs, 1);
1302        for (pup = 0; pup < (max_pup_num); pup++) {
1303                DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
1304                DEBUG_WL_D((u32) pup, 1);
1305                DEBUG_WL_S(", Phase: ");
1306                DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][P], 1);
1307                DEBUG_WL_S(", Delay: ");
1308                DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][D], 2);
1309                DEBUG_WL_S("\n");
1310        }
1311
1312        /* Check if some not locked and return error */
1313        if (*result != max_pup_mask) {
1314                DEBUG_WL_S("DDR3 - Write Leveling - ERROR - not all PUPS were locked\n");
1315                return MV_FAIL;
1316        }
1317
1318        /* Configure Each PUP with locked leveling settings */
1319        for (pup = 0; pup < (max_pup_num); pup++) {
1320                /* ECC support - bit 8 */
1321                pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
1322                phase = dram_info->wl_val[cs][pup][P];
1323                delay = dram_info->wl_val[cs][pup][D];
1324                ddr3_write_pup_reg(PUP_WL_MODE, cs, pup_num, phase, delay);
1325        }
1326
1327        /* CS ODT Override */
1328        reg =  reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1329                REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1330        /* 0x1498 - SDRAM ODT Control high */
1331        reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1332
1333        return MV_OK;
1334}
1335
1336/*
1337 * Perform DDR3 Control PUP Indirect Write
1338 */
1339static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr, u32 data)
1340{
1341        u32 reg = 0;
1342
1343        /* Store value for write */
1344        reg = (data & 0xFFFF);
1345
1346        /* Set bit 26 for control PHY access */
1347        reg |= (1 << REG_PHY_CNTRL_OFFS);
1348
1349        /* Configure BC or UC access to PHYs */
1350        if (bc_acc == 1)
1351                reg |= (1 << REG_PHY_BC_OFFS);
1352        else
1353                reg |= (pup << REG_PHY_PUP_OFFS);
1354
1355        /* Set PHY register address to write to */
1356        reg |= (reg_addr << REG_PHY_CS_OFFS);
1357
1358        reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1359        reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1360        reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1361
1362        do {
1363                reg = (reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR)) &
1364                        REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1365        } while (reg);          /* Wait for '0' to mark the end of the transaction */
1366}
1367