LXR uboot/drivers/ddr/marvell/axp/ddr3

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) Marvell International Ltd. and its affiliates
   4 */
   5
   6#include <common.h>
   7#include <i2c.h>
   8#include <spl.h>
   9#include <asm/io.h>
  10#include <asm/arch/cpu.h>
  11#include <asm/arch/soc.h>
  12#include <linux/delay.h>
  13
  14#include "ddr3_hw_training.h"
  15
  16/*
  17 * Debug
  18 */
  19#define DEBUG_PBS_FULL_C(s, d, l) \
  20        DEBUG_PBS_FULL_S(s); DEBUG_PBS_FULL_D(d, l); DEBUG_PBS_FULL_S("\n")
  21#define DEBUG_PBS_C(s, d, l) \
  22        DEBUG_PBS_S(s); DEBUG_PBS_D(d, l); DEBUG_PBS_S("\n")
  23
  24#ifdef MV_DEBUG_PBS
  25#define DEBUG_PBS_S(s)                  puts(s)
  26#define DEBUG_PBS_D(d, l)               printf("%x", d)
  27#else
  28#define DEBUG_PBS_S(s)
  29#define DEBUG_PBS_D(d, l)
  30#endif
  31
  32#ifdef MV_DEBUG_FULL_PBS
  33#define DEBUG_PBS_FULL_S(s)             puts(s)
  34#define DEBUG_PBS_FULL_D(d, l)          printf("%x", d)
  35#else
  36#define DEBUG_PBS_FULL_S(s)
  37#define DEBUG_PBS_FULL_D(d, l)
  38#endif
  39
  40#if defined(MV88F78X60) || defined(MV88F672X)
  41
  42/* Temp array for skew data storage */
  43static u32 skew_array[(MAX_PUP_NUM) * DQ_NUM] = { 0 };
  44
  45/* PBS locked dq (per pup) */
  46extern u32 pbs_locked_dq[MAX_PUP_NUM][DQ_NUM];
  47extern u32 pbs_locked_dm[MAX_PUP_NUM];
  48extern u32 pbs_locked_value[MAX_PUP_NUM][DQ_NUM];
  49
  50#if defined(MV88F672X)
  51extern u32 pbs_pattern[2][LEN_16BIT_PBS_PATTERN];
  52extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
  53#else
  54extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
  55extern u32 pbs_pattern_64b[2][LEN_PBS_PATTERN];
  56#endif
  57
  58extern u32 pbs_dq_mapping[PUP_NUM_64BIT + 1][DQ_NUM];
  59
  60static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
  61                u32 cur_pup, u32 pbs_pattern_idx, u32 ecc);
  62static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
  63                u32 pbs_pattern_idx, u32 ecc);
  64static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
  65                u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc);
  66static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx);
  67static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay);
  68
  69/*
  70 * Name:     ddr3_pbs_tx
  71 * Desc:     Execute the PBS TX phase.
  72 * Args:     dram_info   ddr3 training information struct
  73 * Notes:
  74 * Returns:  MV_OK if success, other error code if fail.
  75 */
  76int ddr3_pbs_tx(MV_DRAM_INFO *dram_info)
  77{
  78        /* Array of Deskew results */
  79
  80        /*
  81         * Array to hold the total sum of skew from all iterations
  82         * (for average purpose)
  83         */
  84        u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
  85
  86        /*
  87         * Array to hold the total average skew from both patterns
  88         * (for average purpose)
  89         */
  90        u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
  91
  92        u32 pbs_rep_time = 0;   /* counts number of loop in case of fail */
  93        /* bit array for unlock pups - used to repeat on the RX operation */
  94        u32 cur_pup;
  95        u32 max_pup;
  96        u32 pbs_retry;
  97        u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
  98        u32 pattern_idx;
  99        u32 ecc;
 100        /* indicates whether we need to start the loop again */
 101        int start_over;
 102
 103        DEBUG_PBS_S("DDR3 - PBS TX - Starting PBS TX procedure\n");
 104
 105        pups = dram_info->num_of_total_pups;
 106        max_pup = dram_info->num_of_total_pups;
 107
 108        /* Enable SW override */
 109        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 110                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 111        /* [0] = 1 - Enable SW override  */
 112        /* 0x15B8 - Training SW 2 Register */
 113        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 114        DEBUG_PBS_S("DDR3 - PBS RX - SW Override Enabled\n");
 115
 116        reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
 117        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 118
 119        /* Running twice for 2 different patterns. each patterns - 3 times */
 120        for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
 121                DEBUG_PBS_C("DDR3 - PBS TX - Working with pattern - ",
 122                            pattern_idx, 1);
 123
 124                /* Reset sum array */
 125                for (pup = 0; pup < pups; pup++) {
 126                        for (dq = 0; dq < DQ_NUM; dq++)
 127                                skew_sum_array[pup][dq] = 0;
 128                }
 129
 130                /*
 131                 * Perform PBS several of times (3 for each pattern).
 132                 * At the end, we'll use the average
 133                 */
 134                /* If there is ECC, do each PBS again with mux change */
 135                for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
 136                        for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
 137
 138                                /*
 139                                 * This parameter stores the current PUP
 140                                 * num - ecc mode dependent - 4-8 / 1 pups
 141                                 */
 142                                cur_max_pup = (1 - ecc) *
 143                                        dram_info->num_of_std_pups + ecc;
 144
 145                                if (ecc) {
 146                                        /* Only 1 pup in this case */
 147                                        valid_pup = 0x1;
 148                                } else if (cur_max_pup > 4) {
 149                                        /* 64 bit - 8 pups */
 150                                        valid_pup = 0xFF;
 151                                } else if (cur_max_pup == 4) {
 152                                        /* 32 bit - 4 pups */
 153                                        valid_pup = 0xF;
 154                                } else {
 155                                        /* 16 bit - 2 pups */
 156                                        valid_pup = 0x3;
 157                                }
 158
 159                                /* ECC Support - Switch ECC Mux on ecc=1 */
 160                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 161                                        ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 162                                reg |= (dram_info->ecc_ena * ecc <<
 163                                        REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 164                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 165
 166                                if (ecc)
 167                                        DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Enabled\n");
 168                                else
 169                                        DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Disabled\n");
 170
 171                                /* Init iteration values */
 172                                /* Clear the locked DQs */
 173                                for (pup = 0; pup < cur_max_pup; pup++) {
 174                                        for (dq = 0; dq < DQ_NUM; dq++) {
 175                                                pbs_locked_dq[
 176                                                        pup + ecc *
 177                                                        (max_pup - 1)][dq] =
 178                                                        0;
 179                                        }
 180                                }
 181
 182                                pbs_rep_time = 0;
 183                                cur_pup = valid_pup;
 184                                start_over = 0;
 185
 186                                /*
 187                                 * Run loop On current Pattern and current
 188                                 * pattern iteration (just to cover the false
 189                                 * fail problem)
 190                                 */
 191                                do {
 192                                        DEBUG_PBS_S("DDR3 - PBS Tx - Pbs Rep Loop is ");
 193                                        DEBUG_PBS_D(pbs_rep_time, 1);
 194                                        DEBUG_PBS_S(", for Retry No.");
 195                                        DEBUG_PBS_D(pbs_retry, 1);
 196                                        DEBUG_PBS_S("\n");
 197
 198                                        /* Set all PBS values to MIN (0) */
 199                                        DEBUG_PBS_S("DDR3 - PBS Tx - Set all PBS values to MIN\n");
 200
 201                                        for (dq = 0; dq < DQ_NUM; dq++) {
 202                                                ddr3_write_pup_reg(
 203                                                        PUP_PBS_TX +
 204                                                        pbs_dq_mapping[pup *
 205                                                                (1 - ecc) +
 206                                                                ecc * ECC_PUP]
 207                                                        [dq], CS0, (1 - ecc) *
 208                                                        PUP_BC + ecc * ECC_PUP, 0,
 209                                                        0);
 210                                        }
 211
 212                                        /*
 213                                         * Shift DQ ADLL right, One step before
 214                                         * fail
 215                                         */
 216                                        DEBUG_PBS_S("DDR3 - PBS Tx - ADLL shift right one phase before fail\n");
 217
 218                                        if (MV_OK != ddr3_tx_shift_dqs_adll_step_before_fail
 219                                            (dram_info, cur_pup, pattern_idx,
 220                                             ecc))
 221                                                return MV_DDR3_TRAINING_ERR_PBS_ADLL_SHR_1PHASE;
 222
 223                                        /* PBS For each bit */
 224                                        DEBUG_PBS_S("DDR3 - PBS Tx - perform PBS for each bit\n");
 225
 226                                        /*
 227                                         * In this stage - start_over = 0
 228                                         */
 229                                        if (MV_OK != ddr3_pbs_per_bit(
 230                                                    dram_info, &start_over, 1,
 231                                                    &cur_pup, pattern_idx, ecc))
 232                                                return MV_DDR3_TRAINING_ERR_PBS_TX_PER_BIT;
 233
 234                                } while ((start_over == 1) &&
 235                                         (++pbs_rep_time < COUNT_PBS_STARTOVER));
 236
 237                                if (pbs_rep_time == COUNT_PBS_STARTOVER &&
 238                                    start_over == 1) {
 239                                        DEBUG_PBS_S("DDR3 - PBS Tx - FAIL - Adll reach max value\n");
 240                                        return MV_DDR3_TRAINING_ERR_PBS_TX_MAX_VAL;
 241                                }
 242
 243                                DEBUG_PBS_FULL_C("DDR3 - PBS TX - values for iteration - ",
 244                                                 pbs_retry, 1);
 245                                for (pup = 0; pup < cur_max_pup; pup++) {
 246                                        /*
 247                                         * To minimize delay elements, inc
 248                                         * from pbs value the min pbs val
 249                                         */
 250                                        DEBUG_PBS_S("DDR3 - PBS - PUP");
 251                                        DEBUG_PBS_D((pup + (ecc * ECC_PUP)), 1);
 252                                        DEBUG_PBS_S(": ");
 253
 254                                        for (dq = 0; dq < DQ_NUM; dq++) {
 255                                                /* Set skew value for all dq */
 256                                                /*
 257                                                 * Bit# Deskew <- Bit# Deskew -
 258                                                 * last / first  failing bit
 259                                                 * Deskew For all bits (per PUP)
 260                                                 * (minimize delay elements)
 261                                                 */
 262                                                DEBUG_PBS_S("DQ");
 263                                                DEBUG_PBS_D(dq, 1);
 264                                                DEBUG_PBS_S("-");
 265                                                DEBUG_PBS_D(skew_array
 266                                                            [((pup) * DQ_NUM) +
 267                                                             dq], 2);
 268                                                DEBUG_PBS_S(", ");
 269                                        }
 270                                        DEBUG_PBS_S("\n");
 271                                }
 272
 273                                /*
 274                                 * Collect the results we got on this trial
 275                                 * of PBS
 276                                 */
 277                                for (pup = 0; pup < cur_max_pup; pup++) {
 278                                        for (dq = 0; dq < DQ_NUM; dq++) {
 279                                                skew_sum_array[pup + (ecc * (max_pup - 1))]
 280                                                        [dq] += skew_array
 281                                                        [((pup) * DQ_NUM) + dq];
 282                                        }
 283                                }
 284
 285                                /* ECC Support - Disable ECC MUX */
 286                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 287                                        ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 288                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 289                        }
 290                }
 291
 292                DEBUG_PBS_C("DDR3 - PBS TX - values for current pattern - ",
 293                            pattern_idx, 1);
 294                for (pup = 0; pup < max_pup; pup++) {
 295                        /*
 296                         * To minimize delay elements, inc from pbs value the
 297                         * min pbs val
 298                         */
 299                        DEBUG_PBS_S("DDR3 - PBS - PUP");
 300                        DEBUG_PBS_D(pup, 1);
 301                        DEBUG_PBS_S(": ");
 302
 303                        for (dq = 0; dq < DQ_NUM; dq++) {
 304                                /* set skew value for all dq */
 305                                /* Bit# Deskew <- Bit# Deskew - last / first  failing bit Deskew For all bits (per PUP) (minimize delay elements) */
 306                                DEBUG_PBS_S("DQ");
 307                                DEBUG_PBS_D(dq, 1);
 308                                DEBUG_PBS_S("-");
 309                                DEBUG_PBS_D(skew_sum_array[pup][dq] /
 310                                            COUNT_PBS_REPEAT, 2);
 311                                DEBUG_PBS_S(", ");
 312                        }
 313                        DEBUG_PBS_S("\n");
 314                }
 315
 316                /*
 317                 * Calculate the average skew for current pattern for each
 318                 * pup and each bit
 319                 */
 320                DEBUG_PBS_C("DDR3 - PBS TX - Average for pattern - ",
 321                            pattern_idx, 1);
 322
 323                for (pup = 0; pup < max_pup; pup++) {
 324                        /*
 325                         * FOR ECC only :: found min and max value for current
 326                         * pattern skew array
 327                         */
 328                        /* Loop for all dqs */
 329                        for (dq = 0; dq < DQ_NUM; dq++) {
 330                                pattern_skew_array[pup][dq] +=
 331                                        (skew_sum_array[pup][dq] /
 332                                         COUNT_PBS_REPEAT);
 333                        }
 334                }
 335        }
 336
 337        /* Calculate the average skew */
 338        for (pup = 0; pup < max_pup; pup++) {
 339                for (dq = 0; dq < DQ_NUM; dq++)
 340                        skew_array[((pup) * DQ_NUM) + dq] =
 341                                pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
 342        }
 343
 344        DEBUG_PBS_S("DDR3 - PBS TX - Average for all patterns:\n");
 345        for (pup = 0; pup < max_pup; pup++) {
 346                /*
 347                 * To minimize delay elements, inc from pbs value the min
 348                 * pbs val
 349                 */
 350                DEBUG_PBS_S("DDR3 - PBS - PUP");
 351                DEBUG_PBS_D(pup, 1);
 352                DEBUG_PBS_S(": ");
 353
 354                for (dq = 0; dq < DQ_NUM; dq++) {
 355                        /* Set skew value for all dq */
 356                        /*
 357                         * Bit# Deskew <- Bit# Deskew - last / first
 358                         * failing bit Deskew For all bits (per PUP)
 359                         * (minimize delay elements)
 360                         */
 361                        DEBUG_PBS_S("DQ");
 362                        DEBUG_PBS_D(dq, 1);
 363                        DEBUG_PBS_S("-");
 364                        DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
 365                        DEBUG_PBS_S(", ");
 366                }
 367                DEBUG_PBS_S("\n");
 368        }
 369
 370        /* Return ADLL to default value */
 371        for (pup = 0; pup < max_pup; pup++) {
 372                if (pup == (max_pup - 1) && dram_info->ecc_ena)
 373                        pup = ECC_PUP;
 374                ddr3_pbs_write_pup_dqs_reg(CS0, pup, INIT_WL_DELAY);
 375        }
 376
 377        /* Set averaged PBS results */
 378        ddr3_set_pbs_results(dram_info, 1);
 379
 380        /* Disable SW override - Must be in a different stage */
 381        /* [0]=0 - Enable SW override  */
 382        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 383        reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 384        /* 0x15B8 - Training SW 2 Register */
 385        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 386
 387        reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 388                (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 389        reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 390
 391        DEBUG_PBS_S("DDR3 - PBS Tx - PBS TX ended successfuly\n");
 392
 393        return MV_OK;
 394}
 395
 396/*
 397 * Name:     ddr3_tx_shift_dqs_adll_step_before_fail
 398 * Desc:     Execute the Tx shift DQ phase.
 399 * Args:     dram_info            ddr3 training information struct
 400 *           cur_pup              bit array of the function active pups.
 401 *           pbs_pattern_idx      Index of PBS pattern
 402 * Notes:
 403 * Returns:  MV_OK if success, other error code if fail.
 404 */
 405static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
 406                                                   u32 cur_pup,
 407                                                   u32 pbs_pattern_idx, u32 ecc)
 408{
 409        u32 unlock_pup;         /* bit array of unlock pups  */
 410        u32 new_lockup_pup;     /* bit array of compare failed pups */
 411        u32 adll_val = 4;       /* INIT_WL_DELAY */
 412        u32 cur_max_pup, pup;
 413        u32 dqs_dly_set[MAX_PUP_NUM] = { 0 };
 414        u32 *pattern_ptr;
 415
 416        /* Choose pattern */
 417        switch (dram_info->ddr_width) {
 418#if defined(MV88F672X)
 419        case 16:
 420                pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
 421                break;
 422#endif
 423        case 32:
 424                pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
 425                break;
 426#if defined(MV88F78X60)
 427        case 64:
 428                pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
 429                break;
 430#endif
 431        default:
 432                return MV_FAIL;
 433        }
 434
 435        /* Set current pup number */
 436        if (cur_pup == 0x1)     /* Ecc mode */
 437                cur_max_pup = 1;
 438        else
 439                cur_max_pup = dram_info->num_of_std_pups;
 440
 441        unlock_pup = cur_pup;   /* '1' for each unlocked pup */
 442
 443        /* Loop on all ADLL Vaules */
 444        do {
 445                /* Loop until found first fail */
 446                adll_val++;
 447
 448                /*
 449                 * Increment (Move to right - ADLL) DQ TX delay
 450                 * (broadcast to all Data PUPs)
 451                 */
 452                for (pup = 0; pup < cur_max_pup; pup++)
 453                        ddr3_pbs_write_pup_dqs_reg(CS0,
 454                                                   pup * (1 - ecc) +
 455                                                   ECC_PUP * ecc, adll_val);
 456
 457                /*
 458                 * Write and Read, compare results (read was already verified)
 459                 */
 460                /* 0 - all locked */
 461                new_lockup_pup = 0;
 462
 463                if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
 464                                                &new_lockup_pup,
 465                                                pattern_ptr, LEN_PBS_PATTERN,
 466                                                SDRAM_PBS_TX_OFFS, 1, 0,
 467                                                NULL,
 468                                                0))
 469                        return MV_FAIL;
 470
 471                unlock_pup &= ~new_lockup_pup;
 472
 473                DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
 474                DEBUG_PBS_FULL_D(unlock_pup, 2);
 475                DEBUG_PBS_FULL_C(", Set ADLL value = ", adll_val, 2);
 476
 477                /* If any PUP failed there is '1' to mark the PUP */
 478                if (new_lockup_pup != 0) {
 479                        /*
 480                         * Decrement (Move Back to Left two steps - ADLL)
 481                         * DQ TX delay for current failed pups and save
 482                         */
 483                        for (pup = 0; pup < cur_max_pup; pup++) {
 484                                if (((new_lockup_pup >> pup) & 0x1) &&
 485                                    dqs_dly_set[pup] == 0)
 486                                        dqs_dly_set[pup] = adll_val - 1;
 487                        }
 488                }
 489        } while ((unlock_pup != 0) && (adll_val != ADLL_MAX));
 490
 491        if (unlock_pup != 0) {
 492                DEBUG_PBS_FULL_S("DDR3 - PBS Tx - Shift DQ - Adll value reached maximum\n");
 493
 494                for (pup = 0; pup < cur_max_pup; pup++) {
 495                        if (((unlock_pup >> pup) & 0x1) &&
 496                            dqs_dly_set[pup] == 0)
 497                                dqs_dly_set[pup] = adll_val - 1;
 498                }
 499        }
 500
 501        DEBUG_PBS_FULL_C("PBS TX one step before fail last pups locked Adll ",
 502                         adll_val - 2, 2);
 503
 504        /* Set the PUP DQS DLY Values */
 505        for (pup = 0; pup < cur_max_pup; pup++)
 506                ddr3_pbs_write_pup_dqs_reg(CS0, pup * (1 - ecc) + ECC_PUP * ecc,
 507                                           dqs_dly_set[pup]);
 508
 509        /* Found one phase before fail */
 510        return MV_OK;
 511}
 512
 513/*
 514 * Name:     ddr3_pbs_rx
 515 * Desc:     Execute the PBS RX phase.
 516 * Args:     dram_info   ddr3 training information struct
 517 * Notes:
 518 * Returns:  MV_OK if success, other error code if fail.
 519 */
 520int ddr3_pbs_rx(MV_DRAM_INFO *dram_info)
 521{
 522        /*
 523         * Array to hold the total sum of skew from all iterations
 524         * (for average purpose)
 525         */
 526        u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
 527
 528        /*
 529         * Array to hold the total average skew from both patterns
 530         * (for average purpose)
 531         */
 532        u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
 533
 534        u32 pbs_rep_time = 0;   /* counts number of loop in case of fail */
 535        /* bit array for unlock pups - used to repeat on the RX operation */
 536        u32 cur_pup;
 537        u32 max_pup;
 538        u32 pbs_retry;
 539        u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
 540        u32 pattern_idx;
 541        u32 ecc;
 542        /* indicates whether we need to start the loop again */
 543        int start_over;
 544        int status;
 545
 546        DEBUG_PBS_S("DDR3 - PBS RX - Starting PBS RX procedure\n");
 547
 548        pups = dram_info->num_of_total_pups;
 549        max_pup = dram_info->num_of_total_pups;
 550
 551        /* Enable SW override */
 552        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 553                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 554        /* [0] = 1 - Enable SW override  */
 555        /* 0x15B8 - Training SW 2 Register */
 556        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 557        DEBUG_PBS_FULL_S("DDR3 - PBS RX - SW Override Enabled\n");
 558
 559        reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
 560        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 561
 562        /* Running twice for 2 different patterns. each patterns - 3 times */
 563        for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
 564                DEBUG_PBS_FULL_C("DDR3 - PBS RX - Working with pattern - ",
 565                                 pattern_idx, 1);
 566
 567                /* Reset sum array */
 568                for (pup = 0; pup < pups; pup++) {
 569                        for (dq = 0; dq < DQ_NUM; dq++)
 570                                skew_sum_array[pup][dq] = 0;
 571                }
 572
 573                /*
 574                 * Perform PBS several of times (3 for each pattern).
 575                 * At the end, we'll use the average
 576                 */
 577                /* If there is ECC, do each PBS again with mux change */
 578                for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
 579                        for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
 580                                /*
 581                                 * This parameter stores the current PUP
 582                                 * num - ecc mode dependent - 4-8 / 1 pups
 583                                 */
 584                                cur_max_pup = (1 - ecc) *
 585                                        dram_info->num_of_std_pups + ecc;
 586
 587                                if (ecc) {
 588                                        /* Only 1 pup in this case */
 589                                        valid_pup = 0x1;
 590                                } else if (cur_max_pup > 4) {
 591                                        /* 64 bit - 8 pups */
 592                                        valid_pup = 0xFF;
 593                                } else if (cur_max_pup == 4) {
 594                                        /* 32 bit - 4 pups */
 595                                        valid_pup = 0xF;
 596                                } else {
 597                                        /* 16 bit - 2 pups */
 598                                        valid_pup = 0x3;
 599                                }
 600
 601                                /* ECC Support - Switch ECC Mux on ecc=1 */
 602                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 603                                        ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 604                                reg |= (dram_info->ecc_ena * ecc <<
 605                                        REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 606                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 607
 608                                if (ecc)
 609                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Enabled\n");
 610                                else
 611                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Disabled\n");
 612
 613                                /* Init iteration values */
 614                                /* Clear the locked DQs */
 615                                for (pup = 0; pup < cur_max_pup; pup++) {
 616                                        for (dq = 0; dq < DQ_NUM; dq++) {
 617                                                pbs_locked_dq[
 618                                                        pup + ecc * (max_pup - 1)][dq] =
 619                                                        0;
 620                                        }
 621                                }
 622
 623                                pbs_rep_time = 0;
 624                                cur_pup = valid_pup;
 625                                start_over = 0;
 626
 627                                /*
 628                                 * Run loop On current Pattern and current
 629                                 * pattern iteration (just to cover the false
 630                                 * fail problem
 631                                 */
 632                                do {
 633                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Pbs Rep Loop is ");
 634                                        DEBUG_PBS_FULL_D(pbs_rep_time, 1);
 635                                        DEBUG_PBS_FULL_S(", for Retry No.");
 636                                        DEBUG_PBS_FULL_D(pbs_retry, 1);
 637                                        DEBUG_PBS_FULL_S("\n");
 638
 639                                        /* Set all PBS values to MAX (31) */
 640                                        for (pup = 0; pup < cur_max_pup; pup++) {
 641                                                for (dq = 0; dq < DQ_NUM; dq++)
 642                                                        ddr3_write_pup_reg(
 643                                                                PUP_PBS_RX +
 644                                                                pbs_dq_mapping[
 645                                                                pup * (1 - ecc)
 646                                                                + ecc * ECC_PUP]
 647                                                                [dq], CS0,
 648                                                                pup + ecc * ECC_PUP,
 649                                                                0, MAX_PBS);
 650                                        }
 651
 652                                        /* Set all DQS PBS values to MIN (0) */
 653                                        for (pup = 0; pup < cur_max_pup; pup++) {
 654                                                ddr3_write_pup_reg(PUP_PBS_RX +
 655                                                                   DQ_NUM, CS0,
 656                                                                   pup +
 657                                                                   ecc *
 658                                                                   ECC_PUP, 0,
 659                                                                   0);
 660                                        }
 661
 662                                        /* Shift DQS, To first Fail */
 663                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift RX DQS to first fail\n");
 664
 665                                        status = ddr3_rx_shift_dqs_to_first_fail
 666                                                (dram_info, cur_pup,
 667                                                 pattern_idx, ecc);
 668                                        if (MV_OK != status) {
 669                                                DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_rx_shift_dqs_to_first_fail failed.\n");
 670                                                DEBUG_PBS_D(status, 8);
 671                                                DEBUG_PBS_S("\nDDR3 - PBS Rx - SKIP.\n");
 672
 673                                                /* Reset read FIFO */
 674                                                reg = reg_read(REG_DRAM_TRAINING_ADDR);
 675                                                /* Start Auto Read Leveling procedure */
 676                                                reg |= (1 << REG_DRAM_TRAINING_RL_OFFS);
 677                                                /* 0x15B0 - Training Register */
 678                                                reg_write(REG_DRAM_TRAINING_ADDR, reg);
 679
 680                                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 681                                                reg |= ((1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS)
 682                                                        + (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS));
 683                                                /* [0] = 1 - Enable SW override, [4] = 1 - FIFO reset  */
 684                                                /* 0x15B8 - Training SW 2 Register */
 685                                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 686
 687                                                do {
 688                                                        reg = (reg_read(REG_DRAM_TRAINING_2_ADDR))
 689                                                                & (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 690                                                } while (reg);  /* Wait for '0' */
 691
 692                                                reg = reg_read(REG_DRAM_TRAINING_ADDR);
 693                                                /* Clear Auto Read Leveling procedure */
 694                                                reg &= ~(1 << REG_DRAM_TRAINING_RL_OFFS);
 695                                                /* 0x15B0 - Training Register */
 696                                                reg_write(REG_DRAM_TRAINING_ADDR, reg);
 697
 698                                                /* Set ADLL to 15 */
 699                                                for (pup = 0; pup < max_pup;
 700                                                     pup++) {
 701                                                        ddr3_write_pup_reg
 702                                                            (PUP_DQS_RD, CS0,
 703                                                             pup +
 704                                                             (ecc * ECC_PUP), 0,
 705                                                             15);
 706                                                }
 707
 708                                                /* Set all PBS values to MIN (0) */
 709                                                for (pup = 0; pup < cur_max_pup;
 710                                                     pup++) {
 711                                                        for (dq = 0;
 712                                                             dq < DQ_NUM; dq++)
 713                                                                ddr3_write_pup_reg
 714                                                                    (PUP_PBS_RX +
 715                                                                     pbs_dq_mapping
 716                                                                     [pup * (1 - ecc) +
 717                                                                      ecc * ECC_PUP]
 718                                                                     [dq], CS0,
 719                                                                     pup + ecc * ECC_PUP,
 720                                                                     0, MIN_PBS);
 721                                                }
 722
 723                                                return MV_OK;
 724                                        }
 725
 726                                        /* PBS For each bit */
 727                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - perform PBS for each bit\n");
 728                                        /* in this stage - start_over = 0; */
 729                                        if (MV_OK != ddr3_pbs_per_bit(
 730                                                    dram_info, &start_over,
 731                                                    0, &cur_pup,
 732                                                    pattern_idx, ecc)) {
 733                                                DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_pbs_per_bit failed.");
 734                                                return MV_DDR3_TRAINING_ERR_PBS_RX_PER_BIT;
 735                                        }
 736
 737                                } while ((start_over == 1) &&
 738                                         (++pbs_rep_time < COUNT_PBS_STARTOVER));
 739
 740                                if (pbs_rep_time == COUNT_PBS_STARTOVER &&
 741                                    start_over == 1) {
 742                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - FAIL - Algorithm failed doing RX PBS\n");
 743                                        return MV_DDR3_TRAINING_ERR_PBS_RX_MAX_VAL;
 744                                }
 745
 746                                /* Return DQS ADLL to default value - 15 */
 747                                /* Set all DQS PBS values to MIN (0) */
 748                                for (pup = 0; pup < cur_max_pup; pup++)
 749                                        ddr3_write_pup_reg(PUP_DQS_RD, CS0,
 750                                                           pup + ecc * ECC_PUP,
 751                                                           0, INIT_RL_DELAY);
 752
 753                                DEBUG_PBS_FULL_C("DDR3 - PBS RX - values for iteration - ",
 754                                                 pbs_retry, 1);
 755                                for (pup = 0; pup < cur_max_pup; pup++) {
 756                                        /*
 757                                         * To minimize delay elements, inc from
 758                                         * pbs value the min pbs val
 759                                         */
 760                                        DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
 761                                        DEBUG_PBS_FULL_D((pup +
 762                                                          (ecc * ECC_PUP)), 1);
 763                                        DEBUG_PBS_FULL_S(": ");
 764
 765                                        for (dq = 0; dq < DQ_NUM; dq++) {
 766                                                /* Set skew value for all dq */
 767                                                /*
 768                                                 * Bit# Deskew <- Bit# Deskew -
 769                                                 * last / first  failing bit
 770                                                 * Deskew For all bits (per PUP)
 771                                                 * (minimize delay elements)
 772                                                 */
 773                                                DEBUG_PBS_FULL_S("DQ");
 774                                                DEBUG_PBS_FULL_D(dq, 1);
 775                                                DEBUG_PBS_FULL_S("-");
 776                                                DEBUG_PBS_FULL_D(skew_array
 777                                                                 [((pup) *
 778                                                                   DQ_NUM) +
 779                                                                  dq], 2);
 780                                                DEBUG_PBS_FULL_S(", ");
 781                                        }
 782                                        DEBUG_PBS_FULL_S("\n");
 783                                }
 784
 785                                /*
 786                                 * Collect the results we got on this trial
 787                                 * of PBS
 788                                 */
 789                                for (pup = 0; pup < cur_max_pup; pup++) {
 790                                        for (dq = 0; dq < DQ_NUM; dq++) {
 791                                                skew_sum_array
 792                                                        [pup + (ecc * (max_pup - 1))]
 793                                                        [dq] +=
 794                                                        skew_array[((pup) * DQ_NUM) + dq];
 795                                        }
 796                                }
 797
 798                                /* ECC Support - Disable ECC MUX */
 799                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 800                                        ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 801                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 802                        }
 803                }
 804
 805                /*
 806                 * Calculate the average skew for current pattern for each
 807                 * pup and each bit
 808                 */
 809                DEBUG_PBS_FULL_C("DDR3 - PBS RX - Average for pattern - ",
 810                                 pattern_idx, 1);
 811                for (pup = 0; pup < max_pup; pup++) {
 812                        /*
 813                         * FOR ECC only :: found min and max value for
 814                         * current pattern skew array
 815                         */
 816                        /* Loop for all dqs */
 817                        for (dq = 0; dq < DQ_NUM; dq++) {
 818                                pattern_skew_array[pup][dq] +=
 819                                        (skew_sum_array[pup][dq] /
 820                                         COUNT_PBS_REPEAT);
 821                        }
 822                }
 823
 824                DEBUG_PBS_C("DDR3 - PBS RX - values for current pattern - ",
 825                            pattern_idx, 1);
 826                for (pup = 0; pup < max_pup; pup++) {
 827                        /*
 828                         * To minimize delay elements, inc from pbs value the
 829                         * min pbs val
 830                         */
 831                        DEBUG_PBS_S("DDR3 - PBS RX - PUP");
 832                        DEBUG_PBS_D(pup, 1);
 833                        DEBUG_PBS_S(": ");
 834
 835                        for (dq = 0; dq < DQ_NUM; dq++) {
 836                                /* Set skew value for all dq */
 837                                /*
 838                                 * Bit# Deskew <- Bit# Deskew - last / first
 839                                 * failing bit Deskew For all bits (per PUP)
 840                                 * (minimize delay elements)
 841                                 */
 842                                DEBUG_PBS_S("DQ");
 843                                DEBUG_PBS_D(dq, 1);
 844                                DEBUG_PBS_S("-");
 845                                DEBUG_PBS_D(skew_sum_array[pup][dq] /
 846                                            COUNT_PBS_REPEAT, 2);
 847                                DEBUG_PBS_S(", ");
 848                        }
 849                        DEBUG_PBS_S("\n");
 850                }
 851        }
 852
 853        /* Calculate the average skew */
 854        for (pup = 0; pup < max_pup; pup++) {
 855                for (dq = 0; dq < DQ_NUM; dq++)
 856                        skew_array[((pup) * DQ_NUM) + dq] =
 857                                pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
 858        }
 859
 860        DEBUG_PBS_S("DDR3 - PBS RX - Average for all patterns:\n");
 861        for (pup = 0; pup < max_pup; pup++) {
 862                /*
 863                 * To minimize delay elements, inc from pbs value the
 864                 * min pbs val
 865                 */
 866                DEBUG_PBS_S("DDR3 - PBS - PUP");
 867                DEBUG_PBS_D(pup, 1);
 868                DEBUG_PBS_S(": ");
 869
 870                for (dq = 0; dq < DQ_NUM; dq++) {
 871                        /* Set skew value for all dq */
 872                        /*
 873                         * Bit# Deskew <- Bit# Deskew - last / first
 874                         * failing bit Deskew For all bits (per PUP)
 875                         * (minimize delay elements)
 876                         */
 877                        DEBUG_PBS_S("DQ");
 878                        DEBUG_PBS_D(dq, 1);
 879                        DEBUG_PBS_S("-");
 880                        DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
 881                        DEBUG_PBS_S(", ");
 882                }
 883                DEBUG_PBS_S("\n");
 884        }
 885
 886        /* Return ADLL to default value */
 887        ddr3_write_pup_reg(PUP_DQS_RD, CS0, PUP_BC, 0, INIT_RL_DELAY);
 888
 889        /* Set averaged PBS results */
 890        ddr3_set_pbs_results(dram_info, 0);
 891
 892        /* Disable SW override - Must be in a different stage */
 893        /* [0]=0 - Enable SW override  */
 894        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 895        reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 896        /* 0x15B8 - Training SW 2 Register */
 897        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 898
 899        reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 900                (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 901        reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 902
 903        DEBUG_PBS_FULL_S("DDR3 - PBS RX - ended successfuly\n");
 904
 905        return MV_OK;
 906}
 907
 908/*
 909 * Name:     ddr3_rx_shift_dqs_to_first_fail
 910 * Desc:     Execute the Rx shift DQ phase.
 911 * Args:     dram_info           ddr3 training information struct
 912 *           cur_pup             bit array of the function active pups.
 913 *           pbs_pattern_idx     Index of PBS pattern
 914 * Notes:
 915 * Returns:  MV_OK if success, other error code if fail.
 916 */
 917static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
 918                                           u32 pbs_pattern_idx, u32 ecc)
 919{
 920        u32 unlock_pup;         /* bit array of unlock pups  */
 921        u32 new_lockup_pup;     /* bit array of compare failed pups */
 922        u32 adll_val = MAX_DELAY;
 923        u32 dqs_deskew_val = 0; /* current value of DQS PBS deskew */
 924        u32 cur_max_pup, pup, pass_pup;
 925        u32 *pattern_ptr;
 926
 927        /* Choose pattern */
 928        switch (dram_info->ddr_width) {
 929#if defined(MV88F672X)
 930        case 16:
 931                pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
 932                break;
 933#endif
 934        case 32:
 935                pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
 936                break;
 937#if defined(MV88F78X60)
 938        case 64:
 939                pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
 940                break;
 941#endif
 942        default:
 943                return MV_FAIL;
 944        }
 945
 946        /* Set current pup number */
 947        if (cur_pup == 0x1)     /* Ecc mode */
 948                cur_max_pup = 1;
 949        else
 950                cur_max_pup = dram_info->num_of_std_pups;
 951
 952        unlock_pup = cur_pup;   /* '1' for each unlocked pup */
 953
 954        DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Starting...\n");
 955
 956        /* Set DQS ADLL to MAX */
 957        DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Set DQS ADLL to Max for all PUPs\n");
 958        for (pup = 0; pup < cur_max_pup; pup++)
 959                ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP, 0,
 960                                   MAX_DELAY);
 961
 962        /* Loop on all ADLL Vaules */
 963        do {
 964                /* Loop until found fail for all pups */
 965                new_lockup_pup = 0;
 966                if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
 967                                                &new_lockup_pup,
 968                                                pattern_ptr, LEN_PBS_PATTERN,
 969                                                SDRAM_PBS_I_OFFS +
 970                                                pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
 971                                                0, 0, NULL, 0)) {
 972                        DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
 973                        return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
 974                }
 975
 976                if ((new_lockup_pup != 0) && (dqs_deskew_val <= 1)) {
 977                        /* Fail on start with first deskew value */
 978                        /* Decrement DQS ADLL */
 979                        --adll_val;
 980                        if (adll_val == ADLL_MIN) {
 981                                DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - fail on start with first deskew value\n");
 982                                return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
 983                        }
 984                        ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP,
 985                                           0, adll_val);
 986                        continue;
 987                }
 988
 989                /* Update all new locked pups */
 990                unlock_pup &= ~new_lockup_pup;
 991
 992                if ((unlock_pup == 0) || (dqs_deskew_val == MAX_PBS)) {
 993                        if (dqs_deskew_val == MAX_PBS) {
 994                                /*
 995                                 * Reach max value of dqs deskew or get fail
 996                                 * for all pups
 997                                 */
 998                                DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - DQS deskew reached maximum value\n");
 999                        }
1000                        break;

1001                }
1002
1003                DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Inc DQS deskew for PUPs: ");
1004                DEBUG_PBS_FULL_D(unlock_pup, 2);
1005                DEBUG_PBS_FULL_C(", deskew = ", dqs_deskew_val, 2);
1006
1007                /* Increment DQS deskew elements - Only for unlocked pups */
1008                dqs_deskew_val++;
1009                for (pup = 0; pup < cur_max_pup; pup++) {
1010                        if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1011                                ddr3_write_pup_reg(PUP_PBS_RX + DQS_DQ_NUM, CS0,
1012                                                   pup + ecc * ECC_PUP, 0,
1013                                                   dqs_deskew_val);
1014                        }
1015                }
1016        } while (1);
1017
1018        DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - ADLL shift one step before fail\n");
1019        /* Continue to ADLL shift one step before fail */
1020        unlock_pup = cur_pup;
1021        do {
1022                /* Loop until pass compare for all pups */
1023                new_lockup_pup = 0;
1024                /* Read and compare results  */
1025                if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup, &new_lockup_pup,
1026                                                pattern_ptr, LEN_PBS_PATTERN,
1027                                                SDRAM_PBS_I_OFFS +
1028                                                pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
1029                                                1, 0, NULL, 0)) {
1030                        DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
1031                        return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
1032                }
1033
1034                /*
1035                 * Get mask for pup which passed so their adll will be
1036                 * changed to 2 steps before fails
1037                 */
1038                pass_pup = unlock_pup & ~new_lockup_pup;
1039
1040                DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
1041                DEBUG_PBS_FULL_D(pass_pup, 2);
1042                DEBUG_PBS_FULL_C(", Set ADLL value = ", (adll_val - 2), 2);
1043
1044                /* Only for pass pups   */
1045                for (pup = 0; pup < cur_max_pup; pup++) {
1046                        if (IS_PUP_ACTIVE(pass_pup, pup) == 1) {
1047                                ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1048                                                   pup + ecc * ECC_PUP, 0,
1049                                                   (adll_val - 2));
1050                        }
1051                }
1052
1053                /* Locked pups that compare success  */
1054                unlock_pup &= new_lockup_pup;
1055
1056                if (unlock_pup == 0) {
1057                        /* All pups locked */
1058                        break;
1059                }
1060
1061                /* Found error */
1062                if (adll_val == 0) {
1063                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift DQS - Adll reach min value\n");
1064                        return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_MAX_VAL;
1065                }
1066
1067                /*
1068                 * Decrement (Move Back to Left one phase - ADLL) dqs RX delay
1069                 */
1070                adll_val--;
1071                for (pup = 0; pup < cur_max_pup; pup++) {
1072                        if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1073                                ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1074                                                   pup + ecc * ECC_PUP, 0,
1075                                                   adll_val);
1076                        }
1077                }
1078        } while (1);
1079
1080        return MV_OK;
1081}
1082
1083/*
1084 * lock_pups() extracted from ddr3_pbs_per_bit(). This just got too
1085 * much indented making it hard to read / edit.
1086 */
1087static void lock_pups(u32 pup, u32 *pup_locked, u8 *unlock_pup_dq_array,
1088                      u32 pbs_curr_val, u32 start_pbs, u32 ecc, int is_tx)
1089{
1090        u32 dq;
1091        int idx;
1092
1093        /* Lock PBS value for all remaining PUPs bits */
1094        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Lock PBS value for all remaining PUPs bits, pup ");
1095        DEBUG_PBS_FULL_D(pup, 1);
1096        DEBUG_PBS_FULL_C(" pbs value ", pbs_curr_val, 2);
1097
1098        idx = pup * (1 - ecc) + ecc * ECC_PUP;
1099        *pup_locked &= ~(1 << pup);
1100
1101        for (dq = 0; dq < DQ_NUM; dq++) {
1102                if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup) == 1) {
1103                        int offs;
1104
1105                        /* Lock current dq */
1106                        unlock_pup_dq_array[dq] &= ~(1 << pup);
1107                        skew_array[(pup * DQ_NUM) + dq] = pbs_curr_val;
1108
1109                        if (is_tx == 1)
1110                                offs = PUP_PBS_TX;
1111                        else
1112                                offs = PUP_PBS_RX;
1113
1114                        ddr3_write_pup_reg(offs +
1115                                           pbs_dq_mapping[idx][dq], CS0,
1116                                           idx, 0, start_pbs);
1117                }
1118        }
1119}
1120
1121/*
1122 * Name:     ddr3_pbs_per_bit
1123 * Desc:     Execute the Per Bit Skew phase.
1124 * Args:     start_over      Return whether need to start over the algorithm
1125 *           is_tx           Indicate whether Rx or Tx
1126 *           pcur_pup        bit array of the function active pups. return the
1127 *                           pups that need to repeat on the PBS
1128 *           pbs_pattern_idx Index of PBS pattern
1129 *
1130 * Notes:    Current implementation supports double activation of this function.
1131 *           i.e. in order to activate this function (using start_over) more than
1132 *           twice, the implementation should change.
1133 *           imlementation limitation are marked using
1134 *           ' CHIP-ONLY! - Implementation Limitation '
1135 * Returns:  MV_OK if success, other error code if fail.
1136 */
1137static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
1138                            u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc)
1139{
1140        /*
1141         * Bit array to indicate if we already get fail on bit per pup & dq bit
1142         */
1143        u8 unlock_pup_dq_array[DQ_NUM] = {
1144                *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup,
1145                *pcur_pup, *pcur_pup, *pcur_pup
1146        };
1147
1148        u8 cmp_unlock_pup_dq_array[COUNT_PBS_COMP_RETRY_NUM][DQ_NUM];
1149        u32 pup, dq;
1150        /* value of pbs is according to RX or TX */
1151        u32 start_pbs, last_pbs;
1152        u32 pbs_curr_val;
1153        /* bit array that indicates all dq of the pup locked */
1154        u32 pup_locked;
1155        u32 first_fail[MAX_PUP_NUM] = { 0 };    /* count first fail per pup */
1156        /* indicates whether we get first fail per pup */
1157        int first_failed[MAX_PUP_NUM] = { 0 };
1158        /* bit array that indicates pup already get fail */
1159        u32 sum_pup_fail;
1160        /* use to calculate diff between curr pbs to first fail pbs */
1161        u32 calc_pbs_diff;
1162        u32 pbs_cmp_retry;
1163        u32 max_pup;
1164
1165        /* Set init values for retry array - 8 retry */
1166        for (pbs_cmp_retry = 0; pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1167             pbs_cmp_retry++) {
1168                for (dq = 0; dq < DQ_NUM; dq++)
1169                        cmp_unlock_pup_dq_array[pbs_cmp_retry][dq] = *pcur_pup;
1170        }
1171
1172        memset(&skew_array, 0, MAX_PUP_NUM * DQ_NUM * sizeof(u32));
1173
1174        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Started\n");
1175
1176        /* The pbs value depends if rx or tx */
1177        if (is_tx == 1) {
1178                start_pbs = MIN_PBS;
1179                last_pbs = MAX_PBS;
1180        } else {
1181                start_pbs = MAX_PBS;
1182                last_pbs = MIN_PBS;
1183        }
1184
1185        pbs_curr_val = start_pbs;
1186        pup_locked = *pcur_pup;
1187
1188        /* Set current pup number */
1189        if (pup_locked == 0x1)  /* Ecc mode */
1190                max_pup = 1;
1191        else
1192                max_pup = dram_info->num_of_std_pups;
1193
1194        do {
1195                /* Increment/ decrement PBS for un-lock bits only */
1196                if (is_tx == 1)
1197                        pbs_curr_val++;
1198                else
1199                        pbs_curr_val--;
1200
1201                /* Set Current PBS delay  */
1202                for (dq = 0; dq < DQ_NUM; dq++) {
1203                        /* Check DQ bits to see if locked in all pups */
1204                        if (unlock_pup_dq_array[dq] == 0) {
1205                                DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - All pups are locked for DQ ");
1206                                DEBUG_PBS_FULL_D(dq, 1);
1207                                DEBUG_PBS_FULL_S("\n");
1208                                continue;
1209                        }
1210
1211                        for (pup = 0; pup < max_pup; pup++) {
1212                                int idx;
1213
1214                                idx = pup * (1 - ecc) + ecc * ECC_PUP;
1215
1216                                if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup)
1217                                    == 0)
1218                                        continue;
1219
1220                                if (is_tx == 1)
1221                                        ddr3_write_pup_reg(
1222                                                PUP_PBS_TX + pbs_dq_mapping[idx][dq],
1223                                                CS0, idx, 0, pbs_curr_val);
1224                                else
1225                                        ddr3_write_pup_reg(
1226                                                PUP_PBS_RX + pbs_dq_mapping[idx][dq],
1227                                                CS0, idx, 0, pbs_curr_val);
1228                        }
1229                }
1230
1231                /*
1232                 * Write Read and compare results - run the test
1233                 * DDR_PBS_COMP_RETRY_NUM times
1234                 */
1235                /* Run number of read and write to verify */
1236                for (pbs_cmp_retry = 0;
1237                     pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1238                     pbs_cmp_retry++) {
1239
1240                        if (MV_OK !=
1241                            ddr3_sdram_pbs_compare(dram_info, pup_locked, is_tx,
1242                                                   pbs_pattern_idx,
1243                                                   pbs_curr_val, start_pbs,
1244                                                   skew_array,
1245                                                   cmp_unlock_pup_dq_array
1246                                                   [pbs_cmp_retry], ecc))
1247                                return MV_FAIL;
1248
1249                        for (pup = 0; pup < max_pup; pup++) {
1250                                for (dq = 0; dq < DQ_NUM; dq++) {
1251                                        if ((IS_PUP_ACTIVE(unlock_pup_dq_array[dq],
1252                                                           pup) == 1)
1253                                            && (IS_PUP_ACTIVE(cmp_unlock_pup_dq_array
1254                                              [pbs_cmp_retry][dq],
1255                                              pup) == 0)) {
1256                                                DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PbsCurrVal: ");
1257                                                DEBUG_PBS_FULL_D(pbs_curr_val, 2);
1258                                                DEBUG_PBS_FULL_S(" PUP: ");
1259                                                DEBUG_PBS_FULL_D(pup, 1);
1260                                                DEBUG_PBS_FULL_S(" DQ: ");
1261                                                DEBUG_PBS_FULL_D(dq, 1);
1262                                                DEBUG_PBS_FULL_S(" - failed\n");
1263                                        }
1264                                }
1265                        }
1266
1267                        for (dq = 0; dq < DQ_NUM; dq++) {
1268                                unlock_pup_dq_array[dq] &=
1269                                    cmp_unlock_pup_dq_array[pbs_cmp_retry][dq];
1270                        }
1271                }
1272
1273                pup_locked = 0;
1274                sum_pup_fail = *pcur_pup;
1275
1276                /* Check which DQ is failed */
1277                for (dq = 0; dq < DQ_NUM; dq++) {
1278                        /* Summarize the locked pup */
1279                        pup_locked |= unlock_pup_dq_array[dq];
1280
1281                        /* Check if get fail */
1282                        sum_pup_fail &= unlock_pup_dq_array[dq];
1283                }
1284
1285                /* If all PUPS are locked in all DQ - Break */
1286                if (pup_locked == 0) {
1287                        /* All pups are locked */
1288                        *start_over = 0;
1289                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit -  All bit in all pups are successfully locked\n");
1290                        break;
1291                }
1292
1293                /* PBS deskew elements reach max ? */
1294                if (pbs_curr_val == last_pbs) {
1295                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PBS deskew elements reach max\n");
1296                        /* CHIP-ONLY! - Implementation Limitation */
1297                        *start_over = (sum_pup_fail != 0) && (!(*start_over));
1298                        *pcur_pup = pup_locked;
1299
1300                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - StartOver: ");
1301                        DEBUG_PBS_FULL_D(*start_over, 1);
1302                        DEBUG_PBS_FULL_S("  pup_locked: ");
1303                        DEBUG_PBS_FULL_D(pup_locked, 2);
1304                        DEBUG_PBS_FULL_S("  sum_pup_fail: ");
1305                        DEBUG_PBS_FULL_D(sum_pup_fail, 2);
1306                        DEBUG_PBS_FULL_S("\n");
1307
1308                        /* Lock PBS value for all remaining  bits */
1309                        for (pup = 0; pup < max_pup; pup++) {
1310                                /* Check if current pup already received error */
1311                                if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1312                                        /* Valid pup for current function */
1313                                        if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1314                                            1 && (*start_over == 1)) {
1315                                                DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - skipping lock of pup (first loop of pbs)",
1316                                                                 pup, 1);
1317                                                continue;
1318                                        } else
1319                                            if (IS_PUP_ACTIVE(sum_pup_fail, pup)
1320                                                == 1) {
1321                                                DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - Locking pup %d (even though it wasn't supposed to be locked)",
1322                                                                 pup, 1);
1323                                        }
1324
1325                                        /* Already got fail on the PUP */
1326                                        /* Lock PBS value for all remaining bits */
1327                                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Locking remaning DQs for pup - ");
1328                                        DEBUG_PBS_FULL_D(pup, 1);
1329                                        DEBUG_PBS_FULL_S(": ");
1330
1331                                        for (dq = 0; dq < DQ_NUM; dq++) {
1332                                                if (IS_PUP_ACTIVE
1333                                                    (unlock_pup_dq_array[dq],
1334                                                     pup) == 1) {
1335                                                        DEBUG_PBS_FULL_D(dq, 1);
1336                                                        DEBUG_PBS_FULL_S(",");
1337                                                        /* set current PBS */
1338                                                        skew_array[((pup) *
1339                                                                    DQ_NUM) +
1340                                                                   dq] =
1341                                                            pbs_curr_val;
1342                                                }
1343                                        }
1344
1345                                        if (*start_over == 1) {
1346                                                /*
1347                                                 * Reset this pup bit - when
1348                                                 * restart the PBS, ignore this
1349                                                 * pup
1350                                                 */
1351                                                *pcur_pup &= ~(1 << pup);
1352                                        }
1353                                        DEBUG_PBS_FULL_S("\n");
1354                                } else {
1355                                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Pup ");
1356                                        DEBUG_PBS_FULL_D(pup, 1);
1357                                        DEBUG_PBS_FULL_C(" is not set in puplocked - ",
1358                                                         pup_locked, 1);
1359                                }
1360                        }
1361
1362                        /* Need to start the PBS again */
1363                        if (*start_over == 1) {
1364                                DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - false fail - returning to start\n");
1365                                return MV_OK;
1366                        }
1367                        break;
1368                }
1369
1370                /* Diff Check */
1371                for (pup = 0; pup < max_pup; pup++) {
1372                        if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1373                                /* pup is not locked */
1374                                if (first_failed[pup] == 0) {
1375                                        /* No first fail until now */
1376                                        if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1377                                            0) {
1378                                                /* Get first fail */
1379                                                DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - First fail in pup ",
1380                                                                 pup, 1);
1381                                                first_failed[pup] = 1;
1382                                                first_fail[pup] = pbs_curr_val;
1383                                        }
1384                                } else {
1385                                        /* Already got first fail */
1386                                        if (is_tx == 1) {
1387                                                /* TX - inc pbs */
1388                                                calc_pbs_diff = pbs_curr_val -
1389                                                        first_fail[pup];
1390                                        } else {
1391                                                /* RX - dec pbs */
1392                                                calc_pbs_diff = first_fail[pup] -
1393                                                        pbs_curr_val;
1394                                        }
1395
1396                                        if (calc_pbs_diff >= PBS_DIFF_LIMIT) {
1397                                                lock_pups(pup, &pup_locked,
1398                                                          unlock_pup_dq_array,
1399                                                          pbs_curr_val,
1400                                                          start_pbs, ecc, is_tx);
1401                                        }
1402                                }
1403                        }
1404                }
1405        } while (1);
1406
1407        return MV_OK;
1408}
1409
1410/*
1411 * Name:         ddr3_set_pbs_results
1412 * Desc:         Set to HW the PBS phase results.
1413 * Args:         is_tx       Indicates whether to set Tx or RX results
1414 * Notes:
1415 * Returns:      MV_OK if success, other error code if fail.
1416 */
1417static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx)
1418{
1419        u32 pup, phys_pup, dq;
1420        u32 max_pup;            /* number of valid pups */
1421        u32 pbs_min;            /* minimal pbs val per pup */
1422        u32 pbs_max;            /* maximum pbs val per pup */
1423        u32 val[9];
1424
1425        max_pup = dram_info->num_of_total_pups;
1426        DEBUG_PBS_FULL_S("DDR3 - PBS - ddr3_set_pbs_results:\n");
1427
1428        /* Loop for all dqs & pups */
1429        for (pup = 0; pup < max_pup; pup++) {
1430                if (pup == (max_pup - 1) && dram_info->ecc_ena)
1431                        phys_pup = ECC_PUP;
1432                else
1433                        phys_pup = pup;
1434
1435                /*
1436                 * To minimize delay elements, inc from pbs value the min
1437                 * pbs val
1438                 */
1439                pbs_min = MAX_PBS;
1440                pbs_max = 0;
1441                for (dq = 0; dq < DQ_NUM; dq++) {
1442                        if (pbs_min > skew_array[(pup * DQ_NUM) + dq])
1443                                pbs_min = skew_array[(pup * DQ_NUM) + dq];
1444
1445                        if (pbs_max < skew_array[(pup * DQ_NUM) + dq])
1446                                pbs_max = skew_array[(pup * DQ_NUM) + dq];
1447                }
1448
1449                pbs_max -= pbs_min;
1450
1451                DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
1452                DEBUG_PBS_FULL_D(phys_pup, 1);
1453                DEBUG_PBS_FULL_S(": Min Val = ");
1454                DEBUG_PBS_FULL_D(pbs_min, 2);
1455                DEBUG_PBS_FULL_C(", Max Val = ", pbs_max, 2);
1456
1457                val[pup] = 0;
1458
1459                for (dq = 0; dq < DQ_NUM; dq++) {
1460                        int idx;
1461                        int offs;
1462
1463                        /* Set skew value for all dq */
1464                        /*
1465                         * Bit# Deskew <- Bit# Deskew - last / first
1466                         * failing bit Deskew For all bits (per PUP)
1467                         * (minimize delay elements)
1468                         */
1469
1470                        DEBUG_PBS_FULL_S("DQ");
1471                        DEBUG_PBS_FULL_D(dq, 1);
1472                        DEBUG_PBS_FULL_S("-");
1473                        DEBUG_PBS_FULL_D((skew_array[(pup * DQ_NUM) + dq] -
1474                                          pbs_min), 2);
1475                        DEBUG_PBS_FULL_S(", ");
1476
1477                        idx = (pup * DQ_NUM) + dq;
1478
1479                        if (is_tx == 1)
1480                                offs = PUP_PBS_TX;
1481                        else
1482                                offs = PUP_PBS_RX;
1483
1484                        ddr3_write_pup_reg(offs + pbs_dq_mapping[phys_pup][dq],
1485                                           CS0, phys_pup, 0,
1486                                           skew_array[idx] - pbs_min);
1487
1488                        if (is_tx == 1)
1489                                val[pup] += skew_array[idx] - pbs_min;
1490                }
1491
1492                DEBUG_PBS_FULL_S("\n");
1493
1494                /* Set the DQS the half of the Max PBS of the DQs  */
1495                if (is_tx == 1) {
1496                        ddr3_write_pup_reg(PUP_PBS_TX + 8, CS0, phys_pup, 0,
1497                                           pbs_max / 2);
1498                        ddr3_write_pup_reg(PUP_PBS_TX + 0xa, CS0, phys_pup, 0,
1499                                           val[pup] / 8);
1500                } else
1501                        ddr3_write_pup_reg(PUP_PBS_RX + 8, CS0, phys_pup, 0,
1502                                           pbs_max / 2);
1503        }
1504
1505        return MV_OK;
1506}
1507
1508static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay)
1509{
1510        u32 reg, delay;
1511
1512        reg = (ddr3_read_pup_reg(PUP_WL_MODE, cs, pup) & 0x3FF);
1513        delay = reg & PUP_DELAY_MASK;
1514        reg |= ((dqs_delay + delay) << REG_PHY_DQS_REF_DLY_OFFS);
1515        reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1516        reg |= (pup << REG_PHY_PUP_OFFS);
1517        reg |= ((0x4 * cs + PUP_WL_MODE) << REG_PHY_CS_OFFS);
1518
1519        reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1520        do {
1521                reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) &
1522                        REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1523        } while (reg);  /* Wait for '0' to mark the end of the transaction */
1524
1525        udelay(10);
1526}
1527
1528/*
1529 * Set training patterns
1530 */
1531int ddr3_load_pbs_patterns(MV_DRAM_INFO *dram_info)
1532{
1533        u32 cs, cs_count, cs_tmp;
1534        u32 sdram_addr;
1535        u32 *pattern_ptr0, *pattern_ptr1;
1536
1537        /* Choose pattern */
1538        switch (dram_info->ddr_width) {
1539#if defined(MV88F672X)
1540        case 16:
1541                pattern_ptr0 = (u32 *)&pbs_pattern[0];
1542                pattern_ptr1 = (u32 *)&pbs_pattern[1];
1543                break;
1544#endif
1545        case 32:
1546                pattern_ptr0 = (u32 *)&pbs_pattern_32b[0];
1547                pattern_ptr1 = (u32 *)&pbs_pattern_32b[1];
1548                break;
1549#if defined(MV88F78X60)
1550        case 64:
1551                pattern_ptr0 = (u32 *)&pbs_pattern_64b[0];
1552                pattern_ptr1 = (u32 *)&pbs_pattern_64b[1];
1553                break;
1554#endif
1555        default:
1556                return MV_FAIL;
1557        }
1558
1559        /* Loop for each CS */
1560        for (cs = 0; cs < MAX_CS; cs++) {
1561                if (dram_info->cs_ena & (1 << cs)) {
1562                        cs_count = 0;
1563                        for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
1564                                if (dram_info->cs_ena & (1 << cs_tmp))
1565                                        cs_count++;
1566                        }
1567
1568                        /* Init PBS I pattern */
1569                        sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1570                                      SDRAM_PBS_I_OFFS);
1571                        if (MV_OK !=
1572                            ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1573                                               pattern_ptr0, LEN_STD_PATTERN,
1574                                               sdram_addr, 1, 0, NULL,
1575                                               0))
1576                                return MV_FAIL;
1577
1578                        /* Init PBS II pattern */
1579                        sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1580                                      SDRAM_PBS_II_OFFS);
1581                        if (MV_OK !=
1582                            ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1583                                               pattern_ptr1, LEN_STD_PATTERN,
1584                                               sdram_addr, 1, 0, NULL,
1585                                               0))
1586                                return MV_FAIL;
1587                }
1588        }
1589
1590        return MV_OK;
1591}
1592#endif
1593