LXR uboot/drivers/ddr/marvell/axp/ddr3

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) Marvell International Ltd. and its affiliates
   4 */
   5
   6#include <common.h>
   7#include <i2c.h>
   8#include <spl.h>
   9#include <asm/io.h>
  10#include <asm/arch/cpu.h>
  11#include <asm/arch/soc.h>
  12
  13#include "ddr3_hw_training.h"
  14
  15/*
  16 * Debug
  17 */
  18#define DEBUG_PBS_FULL_C(s, d, l) \
  19        DEBUG_PBS_FULL_S(s); DEBUG_PBS_FULL_D(d, l); DEBUG_PBS_FULL_S("\n")
  20#define DEBUG_PBS_C(s, d, l) \
  21        DEBUG_PBS_S(s); DEBUG_PBS_D(d, l); DEBUG_PBS_S("\n")
  22
  23#ifdef MV_DEBUG_PBS
  24#define DEBUG_PBS_S(s)                  puts(s)
  25#define DEBUG_PBS_D(d, l)               printf("%x", d)
  26#else
  27#define DEBUG_PBS_S(s)
  28#define DEBUG_PBS_D(d, l)
  29#endif
  30
  31#ifdef MV_DEBUG_FULL_PBS
  32#define DEBUG_PBS_FULL_S(s)             puts(s)
  33#define DEBUG_PBS_FULL_D(d, l)          printf("%x", d)
  34#else
  35#define DEBUG_PBS_FULL_S(s)
  36#define DEBUG_PBS_FULL_D(d, l)
  37#endif
  38
  39#if defined(MV88F78X60) || defined(MV88F672X)
  40
  41/* Temp array for skew data storage */
  42static u32 skew_array[(MAX_PUP_NUM) * DQ_NUM] = { 0 };
  43
  44/* PBS locked dq (per pup) */
  45extern u32 pbs_locked_dq[MAX_PUP_NUM][DQ_NUM];
  46extern u32 pbs_locked_dm[MAX_PUP_NUM];
  47extern u32 pbs_locked_value[MAX_PUP_NUM][DQ_NUM];
  48
  49#if defined(MV88F672X)
  50extern u32 pbs_pattern[2][LEN_16BIT_PBS_PATTERN];
  51extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
  52#else
  53extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
  54extern u32 pbs_pattern_64b[2][LEN_PBS_PATTERN];
  55#endif
  56
  57extern u32 pbs_dq_mapping[PUP_NUM_64BIT + 1][DQ_NUM];
  58
  59static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
  60                u32 cur_pup, u32 pbs_pattern_idx, u32 ecc);
  61static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
  62                u32 pbs_pattern_idx, u32 ecc);
  63static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
  64                u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc);
  65static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx);
  66static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay);
  67
  68/*
  69 * Name:     ddr3_pbs_tx
  70 * Desc:     Execute the PBS TX phase.
  71 * Args:     dram_info   ddr3 training information struct
  72 * Notes:
  73 * Returns:  MV_OK if success, other error code if fail.
  74 */
  75int ddr3_pbs_tx(MV_DRAM_INFO *dram_info)
  76{
  77        /* Array of Deskew results */
  78
  79        /*
  80         * Array to hold the total sum of skew from all iterations
  81         * (for average purpose)
  82         */
  83        u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
  84
  85        /*
  86         * Array to hold the total average skew from both patterns
  87         * (for average purpose)
  88         */
  89        u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
  90
  91        u32 pbs_rep_time = 0;   /* counts number of loop in case of fail */
  92        /* bit array for unlock pups - used to repeat on the RX operation */
  93        u32 cur_pup;
  94        u32 max_pup;
  95        u32 pbs_retry;
  96        u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
  97        u32 pattern_idx;
  98        u32 ecc;
  99        /* indicates whether we need to start the loop again */
 100        int start_over;
 101
 102        DEBUG_PBS_S("DDR3 - PBS TX - Starting PBS TX procedure\n");
 103
 104        pups = dram_info->num_of_total_pups;
 105        max_pup = dram_info->num_of_total_pups;
 106
 107        /* Enable SW override */
 108        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 109                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 110        /* [0] = 1 - Enable SW override  */
 111        /* 0x15B8 - Training SW 2 Register */
 112        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 113        DEBUG_PBS_S("DDR3 - PBS RX - SW Override Enabled\n");
 114
 115        reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
 116        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 117
 118        /* Running twice for 2 different patterns. each patterns - 3 times */
 119        for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
 120                DEBUG_PBS_C("DDR3 - PBS TX - Working with pattern - ",
 121                            pattern_idx, 1);
 122
 123                /* Reset sum array */
 124                for (pup = 0; pup < pups; pup++) {
 125                        for (dq = 0; dq < DQ_NUM; dq++)
 126                                skew_sum_array[pup][dq] = 0;
 127                }
 128
 129                /*
 130                 * Perform PBS several of times (3 for each pattern).
 131                 * At the end, we'll use the average
 132                 */
 133                /* If there is ECC, do each PBS again with mux change */
 134                for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
 135                        for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
 136
 137                                /*
 138                                 * This parameter stores the current PUP
 139                                 * num - ecc mode dependent - 4-8 / 1 pups
 140                                 */
 141                                cur_max_pup = (1 - ecc) *
 142                                        dram_info->num_of_std_pups + ecc;
 143
 144                                if (ecc) {
 145                                        /* Only 1 pup in this case */
 146                                        valid_pup = 0x1;
 147                                } else if (cur_max_pup > 4) {
 148                                        /* 64 bit - 8 pups */
 149                                        valid_pup = 0xFF;
 150                                } else if (cur_max_pup == 4) {
 151                                        /* 32 bit - 4 pups */
 152                                        valid_pup = 0xF;
 153                                } else {
 154                                        /* 16 bit - 2 pups */
 155                                        valid_pup = 0x3;
 156                                }
 157
 158                                /* ECC Support - Switch ECC Mux on ecc=1 */
 159                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 160                                        ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 161                                reg |= (dram_info->ecc_ena * ecc <<
 162                                        REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 163                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 164
 165                                if (ecc)
 166                                        DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Enabled\n");
 167                                else
 168                                        DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Disabled\n");
 169
 170                                /* Init iteration values */
 171                                /* Clear the locked DQs */
 172                                for (pup = 0; pup < cur_max_pup; pup++) {
 173                                        for (dq = 0; dq < DQ_NUM; dq++) {
 174                                                pbs_locked_dq[
 175                                                        pup + ecc *
 176                                                        (max_pup - 1)][dq] =
 177                                                        0;
 178                                        }
 179                                }
 180
 181                                pbs_rep_time = 0;
 182                                cur_pup = valid_pup;
 183                                start_over = 0;
 184
 185                                /*
 186                                 * Run loop On current Pattern and current
 187                                 * pattern iteration (just to cover the false
 188                                 * fail problem)
 189                                 */
 190                                do {
 191                                        DEBUG_PBS_S("DDR3 - PBS Tx - Pbs Rep Loop is ");
 192                                        DEBUG_PBS_D(pbs_rep_time, 1);
 193                                        DEBUG_PBS_S(", for Retry No.");
 194                                        DEBUG_PBS_D(pbs_retry, 1);
 195                                        DEBUG_PBS_S("\n");
 196
 197                                        /* Set all PBS values to MIN (0) */
 198                                        DEBUG_PBS_S("DDR3 - PBS Tx - Set all PBS values to MIN\n");
 199
 200                                        for (dq = 0; dq < DQ_NUM; dq++) {
 201                                                ddr3_write_pup_reg(
 202                                                        PUP_PBS_TX +
 203                                                        pbs_dq_mapping[pup *
 204                                                                (1 - ecc) +
 205                                                                ecc * ECC_PUP]
 206                                                        [dq], CS0, (1 - ecc) *
 207                                                        PUP_BC + ecc * ECC_PUP, 0,
 208                                                        0);
 209                                        }
 210
 211                                        /*
 212                                         * Shift DQ ADLL right, One step before
 213                                         * fail
 214                                         */
 215                                        DEBUG_PBS_S("DDR3 - PBS Tx - ADLL shift right one phase before fail\n");
 216
 217                                        if (MV_OK != ddr3_tx_shift_dqs_adll_step_before_fail
 218                                            (dram_info, cur_pup, pattern_idx,
 219                                             ecc))
 220                                                return MV_DDR3_TRAINING_ERR_PBS_ADLL_SHR_1PHASE;
 221
 222                                        /* PBS For each bit */
 223                                        DEBUG_PBS_S("DDR3 - PBS Tx - perform PBS for each bit\n");
 224
 225                                        /*
 226                                         * In this stage - start_over = 0
 227                                         */
 228                                        if (MV_OK != ddr3_pbs_per_bit(
 229                                                    dram_info, &start_over, 1,
 230                                                    &cur_pup, pattern_idx, ecc))
 231                                                return MV_DDR3_TRAINING_ERR_PBS_TX_PER_BIT;
 232
 233                                } while ((start_over == 1) &&
 234                                         (++pbs_rep_time < COUNT_PBS_STARTOVER));
 235
 236                                if (pbs_rep_time == COUNT_PBS_STARTOVER &&
 237                                    start_over == 1) {
 238                                        DEBUG_PBS_S("DDR3 - PBS Tx - FAIL - Adll reach max value\n");
 239                                        return MV_DDR3_TRAINING_ERR_PBS_TX_MAX_VAL;
 240                                }
 241
 242                                DEBUG_PBS_FULL_C("DDR3 - PBS TX - values for iteration - ",
 243                                                 pbs_retry, 1);
 244                                for (pup = 0; pup < cur_max_pup; pup++) {
 245                                        /*
 246                                         * To minimize delay elements, inc
 247                                         * from pbs value the min pbs val
 248                                         */
 249                                        DEBUG_PBS_S("DDR3 - PBS - PUP");
 250                                        DEBUG_PBS_D((pup + (ecc * ECC_PUP)), 1);
 251                                        DEBUG_PBS_S(": ");
 252
 253                                        for (dq = 0; dq < DQ_NUM; dq++) {
 254                                                /* Set skew value for all dq */
 255                                                /*
 256                                                 * Bit# Deskew <- Bit# Deskew -
 257                                                 * last / first  failing bit
 258                                                 * Deskew For all bits (per PUP)
 259                                                 * (minimize delay elements)
 260                                                 */
 261                                                DEBUG_PBS_S("DQ");
 262                                                DEBUG_PBS_D(dq, 1);
 263                                                DEBUG_PBS_S("-");
 264                                                DEBUG_PBS_D(skew_array
 265                                                            [((pup) * DQ_NUM) +
 266                                                             dq], 2);
 267                                                DEBUG_PBS_S(", ");
 268                                        }
 269                                        DEBUG_PBS_S("\n");
 270                                }
 271
 272                                /*
 273                                 * Collect the results we got on this trial
 274                                 * of PBS
 275                                 */
 276                                for (pup = 0; pup < cur_max_pup; pup++) {
 277                                        for (dq = 0; dq < DQ_NUM; dq++) {
 278                                                skew_sum_array[pup + (ecc * (max_pup - 1))]
 279                                                        [dq] += skew_array
 280                                                        [((pup) * DQ_NUM) + dq];
 281                                        }
 282                                }
 283
 284                                /* ECC Support - Disable ECC MUX */
 285                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 286                                        ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 287                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 288                        }
 289                }
 290
 291                DEBUG_PBS_C("DDR3 - PBS TX - values for current pattern - ",
 292                            pattern_idx, 1);
 293                for (pup = 0; pup < max_pup; pup++) {
 294                        /*
 295                         * To minimize delay elements, inc from pbs value the
 296                         * min pbs val
 297                         */
 298                        DEBUG_PBS_S("DDR3 - PBS - PUP");
 299                        DEBUG_PBS_D(pup, 1);
 300                        DEBUG_PBS_S(": ");
 301
 302                        for (dq = 0; dq < DQ_NUM; dq++) {
 303                                /* set skew value for all dq */
 304                                /* Bit# Deskew <- Bit# Deskew - last / first  failing bit Deskew For all bits (per PUP) (minimize delay elements) */
 305                                DEBUG_PBS_S("DQ");
 306                                DEBUG_PBS_D(dq, 1);
 307                                DEBUG_PBS_S("-");
 308                                DEBUG_PBS_D(skew_sum_array[pup][dq] /
 309                                            COUNT_PBS_REPEAT, 2);
 310                                DEBUG_PBS_S(", ");
 311                        }
 312                        DEBUG_PBS_S("\n");
 313                }
 314
 315                /*
 316                 * Calculate the average skew for current pattern for each
 317                 * pup and each bit
 318                 */
 319                DEBUG_PBS_C("DDR3 - PBS TX - Average for pattern - ",
 320                            pattern_idx, 1);
 321
 322                for (pup = 0; pup < max_pup; pup++) {
 323                        /*
 324                         * FOR ECC only :: found min and max value for current
 325                         * pattern skew array
 326                         */
 327                        /* Loop for all dqs */
 328                        for (dq = 0; dq < DQ_NUM; dq++) {
 329                                pattern_skew_array[pup][dq] +=
 330                                        (skew_sum_array[pup][dq] /
 331                                         COUNT_PBS_REPEAT);
 332                        }
 333                }
 334        }
 335
 336        /* Calculate the average skew */
 337        for (pup = 0; pup < max_pup; pup++) {
 338                for (dq = 0; dq < DQ_NUM; dq++)
 339                        skew_array[((pup) * DQ_NUM) + dq] =
 340                                pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
 341        }
 342
 343        DEBUG_PBS_S("DDR3 - PBS TX - Average for all patterns:\n");
 344        for (pup = 0; pup < max_pup; pup++) {
 345                /*
 346                 * To minimize delay elements, inc from pbs value the min
 347                 * pbs val
 348                 */
 349                DEBUG_PBS_S("DDR3 - PBS - PUP");
 350                DEBUG_PBS_D(pup, 1);
 351                DEBUG_PBS_S(": ");
 352
 353                for (dq = 0; dq < DQ_NUM; dq++) {
 354                        /* Set skew value for all dq */
 355                        /*
 356                         * Bit# Deskew <- Bit# Deskew - last / first
 357                         * failing bit Deskew For all bits (per PUP)
 358                         * (minimize delay elements)
 359                         */
 360                        DEBUG_PBS_S("DQ");
 361                        DEBUG_PBS_D(dq, 1);
 362                        DEBUG_PBS_S("-");
 363                        DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
 364                        DEBUG_PBS_S(", ");
 365                }
 366                DEBUG_PBS_S("\n");
 367        }
 368
 369        /* Return ADLL to default value */
 370        for (pup = 0; pup < max_pup; pup++) {
 371                if (pup == (max_pup - 1) && dram_info->ecc_ena)
 372                        pup = ECC_PUP;
 373                ddr3_pbs_write_pup_dqs_reg(CS0, pup, INIT_WL_DELAY);
 374        }
 375
 376        /* Set averaged PBS results */
 377        ddr3_set_pbs_results(dram_info, 1);
 378
 379        /* Disable SW override - Must be in a different stage */
 380        /* [0]=0 - Enable SW override  */
 381        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 382        reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 383        /* 0x15B8 - Training SW 2 Register */
 384        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 385
 386        reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 387                (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 388        reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 389
 390        DEBUG_PBS_S("DDR3 - PBS Tx - PBS TX ended successfuly\n");
 391
 392        return MV_OK;
 393}
 394
 395/*
 396 * Name:     ddr3_tx_shift_dqs_adll_step_before_fail
 397 * Desc:     Execute the Tx shift DQ phase.
 398 * Args:     dram_info            ddr3 training information struct
 399 *           cur_pup              bit array of the function active pups.
 400 *           pbs_pattern_idx      Index of PBS pattern
 401 * Notes:
 402 * Returns:  MV_OK if success, other error code if fail.
 403 */
 404static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
 405                                                   u32 cur_pup,
 406                                                   u32 pbs_pattern_idx, u32 ecc)
 407{
 408        u32 unlock_pup;         /* bit array of unlock pups  */
 409        u32 new_lockup_pup;     /* bit array of compare failed pups */
 410        u32 adll_val = 4;       /* INIT_WL_DELAY */
 411        u32 cur_max_pup, pup;
 412        u32 dqs_dly_set[MAX_PUP_NUM] = { 0 };
 413        u32 *pattern_ptr;
 414
 415        /* Choose pattern */
 416        switch (dram_info->ddr_width) {
 417#if defined(MV88F672X)
 418        case 16:
 419                pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
 420                break;
 421#endif
 422        case 32:
 423                pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
 424                break;
 425#if defined(MV88F78X60)
 426        case 64:
 427                pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
 428                break;
 429#endif
 430        default:
 431                return MV_FAIL;
 432        }
 433
 434        /* Set current pup number */
 435        if (cur_pup == 0x1)     /* Ecc mode */
 436                cur_max_pup = 1;
 437        else
 438                cur_max_pup = dram_info->num_of_std_pups;
 439
 440        unlock_pup = cur_pup;   /* '1' for each unlocked pup */
 441
 442        /* Loop on all ADLL Vaules */
 443        do {
 444                /* Loop until found first fail */
 445                adll_val++;
 446
 447                /*
 448                 * Increment (Move to right - ADLL) DQ TX delay
 449                 * (broadcast to all Data PUPs)
 450                 */
 451                for (pup = 0; pup < cur_max_pup; pup++)
 452                        ddr3_pbs_write_pup_dqs_reg(CS0,
 453                                                   pup * (1 - ecc) +
 454                                                   ECC_PUP * ecc, adll_val);
 455
 456                /*
 457                 * Write and Read, compare results (read was already verified)
 458                 */
 459                /* 0 - all locked */
 460                new_lockup_pup = 0;
 461
 462                if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
 463                                                &new_lockup_pup,
 464                                                pattern_ptr, LEN_PBS_PATTERN,
 465                                                SDRAM_PBS_TX_OFFS, 1, 0,
 466                                                NULL,
 467                                                0))
 468                        return MV_FAIL;
 469
 470                unlock_pup &= ~new_lockup_pup;
 471
 472                DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
 473                DEBUG_PBS_FULL_D(unlock_pup, 2);
 474                DEBUG_PBS_FULL_C(", Set ADLL value = ", adll_val, 2);
 475
 476                /* If any PUP failed there is '1' to mark the PUP */
 477                if (new_lockup_pup != 0) {
 478                        /*
 479                         * Decrement (Move Back to Left two steps - ADLL)
 480                         * DQ TX delay for current failed pups and save
 481                         */
 482                        for (pup = 0; pup < cur_max_pup; pup++) {
 483                                if (((new_lockup_pup >> pup) & 0x1) &&
 484                                    dqs_dly_set[pup] == 0)
 485                                        dqs_dly_set[pup] = adll_val - 1;
 486                        }
 487                }
 488        } while ((unlock_pup != 0) && (adll_val != ADLL_MAX));
 489
 490        if (unlock_pup != 0) {
 491                DEBUG_PBS_FULL_S("DDR3 - PBS Tx - Shift DQ - Adll value reached maximum\n");
 492
 493                for (pup = 0; pup < cur_max_pup; pup++) {
 494                        if (((unlock_pup >> pup) & 0x1) &&
 495                            dqs_dly_set[pup] == 0)
 496                                dqs_dly_set[pup] = adll_val - 1;
 497                }
 498        }
 499
 500        DEBUG_PBS_FULL_C("PBS TX one step before fail last pups locked Adll ",
 501                         adll_val - 2, 2);
 502
 503        /* Set the PUP DQS DLY Values */
 504        for (pup = 0; pup < cur_max_pup; pup++)
 505                ddr3_pbs_write_pup_dqs_reg(CS0, pup * (1 - ecc) + ECC_PUP * ecc,
 506                                           dqs_dly_set[pup]);
 507
 508        /* Found one phase before fail */
 509        return MV_OK;
 510}
 511
 512/*
 513 * Name:     ddr3_pbs_rx
 514 * Desc:     Execute the PBS RX phase.
 515 * Args:     dram_info   ddr3 training information struct
 516 * Notes:
 517 * Returns:  MV_OK if success, other error code if fail.
 518 */
 519int ddr3_pbs_rx(MV_DRAM_INFO *dram_info)
 520{
 521        /*
 522         * Array to hold the total sum of skew from all iterations
 523         * (for average purpose)
 524         */
 525        u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
 526
 527        /*
 528         * Array to hold the total average skew from both patterns
 529         * (for average purpose)
 530         */
 531        u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
 532
 533        u32 pbs_rep_time = 0;   /* counts number of loop in case of fail */
 534        /* bit array for unlock pups - used to repeat on the RX operation */
 535        u32 cur_pup;
 536        u32 max_pup;
 537        u32 pbs_retry;
 538        u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
 539        u32 pattern_idx;
 540        u32 ecc;
 541        /* indicates whether we need to start the loop again */
 542        int start_over;
 543        int status;
 544
 545        DEBUG_PBS_S("DDR3 - PBS RX - Starting PBS RX procedure\n");
 546
 547        pups = dram_info->num_of_total_pups;
 548        max_pup = dram_info->num_of_total_pups;
 549
 550        /* Enable SW override */
 551        reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 552                (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 553        /* [0] = 1 - Enable SW override  */
 554        /* 0x15B8 - Training SW 2 Register */
 555        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 556        DEBUG_PBS_FULL_S("DDR3 - PBS RX - SW Override Enabled\n");
 557
 558        reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
 559        reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 560
 561        /* Running twice for 2 different patterns. each patterns - 3 times */
 562        for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
 563                DEBUG_PBS_FULL_C("DDR3 - PBS RX - Working with pattern - ",
 564                                 pattern_idx, 1);
 565
 566                /* Reset sum array */
 567                for (pup = 0; pup < pups; pup++) {
 568                        for (dq = 0; dq < DQ_NUM; dq++)
 569                                skew_sum_array[pup][dq] = 0;
 570                }
 571
 572                /*
 573                 * Perform PBS several of times (3 for each pattern).
 574                 * At the end, we'll use the average
 575                 */
 576                /* If there is ECC, do each PBS again with mux change */
 577                for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
 578                        for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
 579                                /*
 580                                 * This parameter stores the current PUP
 581                                 * num - ecc mode dependent - 4-8 / 1 pups
 582                                 */
 583                                cur_max_pup = (1 - ecc) *
 584                                        dram_info->num_of_std_pups + ecc;
 585
 586                                if (ecc) {
 587                                        /* Only 1 pup in this case */
 588                                        valid_pup = 0x1;
 589                                } else if (cur_max_pup > 4) {
 590                                        /* 64 bit - 8 pups */
 591                                        valid_pup = 0xFF;
 592                                } else if (cur_max_pup == 4) {
 593                                        /* 32 bit - 4 pups */
 594                                        valid_pup = 0xF;
 595                                } else {
 596                                        /* 16 bit - 2 pups */
 597                                        valid_pup = 0x3;
 598                                }
 599
 600                                /* ECC Support - Switch ECC Mux on ecc=1 */
 601                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 602                                        ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 603                                reg |= (dram_info->ecc_ena * ecc <<
 604                                        REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 605                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 606
 607                                if (ecc)
 608                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Enabled\n");
 609                                else
 610                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Disabled\n");
 611
 612                                /* Init iteration values */
 613                                /* Clear the locked DQs */
 614                                for (pup = 0; pup < cur_max_pup; pup++) {
 615                                        for (dq = 0; dq < DQ_NUM; dq++) {
 616                                                pbs_locked_dq[
 617                                                        pup + ecc * (max_pup - 1)][dq] =
 618                                                        0;
 619                                        }
 620                                }
 621
 622                                pbs_rep_time = 0;
 623                                cur_pup = valid_pup;
 624                                start_over = 0;
 625
 626                                /*
 627                                 * Run loop On current Pattern and current
 628                                 * pattern iteration (just to cover the false
 629                                 * fail problem
 630                                 */
 631                                do {
 632                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Pbs Rep Loop is ");
 633                                        DEBUG_PBS_FULL_D(pbs_rep_time, 1);
 634                                        DEBUG_PBS_FULL_S(", for Retry No.");
 635                                        DEBUG_PBS_FULL_D(pbs_retry, 1);
 636                                        DEBUG_PBS_FULL_S("\n");
 637
 638                                        /* Set all PBS values to MAX (31) */
 639                                        for (pup = 0; pup < cur_max_pup; pup++) {
 640                                                for (dq = 0; dq < DQ_NUM; dq++)
 641                                                        ddr3_write_pup_reg(
 642                                                                PUP_PBS_RX +
 643                                                                pbs_dq_mapping[
 644                                                                pup * (1 - ecc)
 645                                                                + ecc * ECC_PUP]
 646                                                                [dq], CS0,
 647                                                                pup + ecc * ECC_PUP,
 648                                                                0, MAX_PBS);
 649                                        }
 650
 651                                        /* Set all DQS PBS values to MIN (0) */
 652                                        for (pup = 0; pup < cur_max_pup; pup++) {
 653                                                ddr3_write_pup_reg(PUP_PBS_RX +
 654                                                                   DQ_NUM, CS0,
 655                                                                   pup +
 656                                                                   ecc *
 657                                                                   ECC_PUP, 0,
 658                                                                   0);
 659                                        }
 660
 661                                        /* Shift DQS, To first Fail */
 662                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift RX DQS to first fail\n");
 663
 664                                        status = ddr3_rx_shift_dqs_to_first_fail
 665                                                (dram_info, cur_pup,
 666                                                 pattern_idx, ecc);
 667                                        if (MV_OK != status) {
 668                                                DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_rx_shift_dqs_to_first_fail failed.\n");
 669                                                DEBUG_PBS_D(status, 8);
 670                                                DEBUG_PBS_S("\nDDR3 - PBS Rx - SKIP.\n");
 671
 672                                                /* Reset read FIFO */
 673                                                reg = reg_read(REG_DRAM_TRAINING_ADDR);
 674                                                /* Start Auto Read Leveling procedure */
 675                                                reg |= (1 << REG_DRAM_TRAINING_RL_OFFS);
 676                                                /* 0x15B0 - Training Register */
 677                                                reg_write(REG_DRAM_TRAINING_ADDR, reg);
 678
 679                                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 680                                                reg |= ((1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS)
 681                                                        + (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS));
 682                                                /* [0] = 1 - Enable SW override, [4] = 1 - FIFO reset  */
 683                                                /* 0x15B8 - Training SW 2 Register */
 684                                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 685
 686                                                do {
 687                                                        reg = (reg_read(REG_DRAM_TRAINING_2_ADDR))
 688                                                                & (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 689                                                } while (reg);  /* Wait for '0' */
 690
 691                                                reg = reg_read(REG_DRAM_TRAINING_ADDR);
 692                                                /* Clear Auto Read Leveling procedure */
 693                                                reg &= ~(1 << REG_DRAM_TRAINING_RL_OFFS);
 694                                                /* 0x15B0 - Training Register */
 695                                                reg_write(REG_DRAM_TRAINING_ADDR, reg);
 696
 697                                                /* Set ADLL to 15 */
 698                                                for (pup = 0; pup < max_pup;
 699                                                     pup++) {
 700                                                        ddr3_write_pup_reg
 701                                                            (PUP_DQS_RD, CS0,
 702                                                             pup +
 703                                                             (ecc * ECC_PUP), 0,
 704                                                             15);
 705                                                }
 706
 707                                                /* Set all PBS values to MIN (0) */
 708                                                for (pup = 0; pup < cur_max_pup;
 709                                                     pup++) {
 710                                                        for (dq = 0;
 711                                                             dq < DQ_NUM; dq++)
 712                                                                ddr3_write_pup_reg
 713                                                                    (PUP_PBS_RX +
 714                                                                     pbs_dq_mapping
 715                                                                     [pup * (1 - ecc) +
 716                                                                      ecc * ECC_PUP]
 717                                                                     [dq], CS0,
 718                                                                     pup + ecc * ECC_PUP,
 719                                                                     0, MIN_PBS);
 720                                                }
 721
 722                                                return MV_OK;
 723                                        }
 724
 725                                        /* PBS For each bit */
 726                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - perform PBS for each bit\n");
 727                                        /* in this stage - start_over = 0; */
 728                                        if (MV_OK != ddr3_pbs_per_bit(
 729                                                    dram_info, &start_over,
 730                                                    0, &cur_pup,
 731                                                    pattern_idx, ecc)) {
 732                                                DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_pbs_per_bit failed.");
 733                                                return MV_DDR3_TRAINING_ERR_PBS_RX_PER_BIT;
 734                                        }
 735
 736                                } while ((start_over == 1) &&
 737                                         (++pbs_rep_time < COUNT_PBS_STARTOVER));
 738
 739                                if (pbs_rep_time == COUNT_PBS_STARTOVER &&
 740                                    start_over == 1) {
 741                                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - FAIL - Algorithm failed doing RX PBS\n");
 742                                        return MV_DDR3_TRAINING_ERR_PBS_RX_MAX_VAL;
 743                                }
 744
 745                                /* Return DQS ADLL to default value - 15 */
 746                                /* Set all DQS PBS values to MIN (0) */
 747                                for (pup = 0; pup < cur_max_pup; pup++)
 748                                        ddr3_write_pup_reg(PUP_DQS_RD, CS0,
 749                                                           pup + ecc * ECC_PUP,
 750                                                           0, INIT_RL_DELAY);
 751
 752                                DEBUG_PBS_FULL_C("DDR3 - PBS RX - values for iteration - ",
 753                                                 pbs_retry, 1);
 754                                for (pup = 0; pup < cur_max_pup; pup++) {
 755                                        /*
 756                                         * To minimize delay elements, inc from
 757                                         * pbs value the min pbs val
 758                                         */
 759                                        DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
 760                                        DEBUG_PBS_FULL_D((pup +
 761                                                          (ecc * ECC_PUP)), 1);
 762                                        DEBUG_PBS_FULL_S(": ");
 763
 764                                        for (dq = 0; dq < DQ_NUM; dq++) {
 765                                                /* Set skew value for all dq */
 766                                                /*
 767                                                 * Bit# Deskew <- Bit# Deskew -
 768                                                 * last / first  failing bit
 769                                                 * Deskew For all bits (per PUP)
 770                                                 * (minimize delay elements)
 771                                                 */
 772                                                DEBUG_PBS_FULL_S("DQ");
 773                                                DEBUG_PBS_FULL_D(dq, 1);
 774                                                DEBUG_PBS_FULL_S("-");
 775                                                DEBUG_PBS_FULL_D(skew_array
 776                                                                 [((pup) *
 777                                                                   DQ_NUM) +
 778                                                                  dq], 2);
 779                                                DEBUG_PBS_FULL_S(", ");
 780                                        }
 781                                        DEBUG_PBS_FULL_S("\n");
 782                                }
 783
 784                                /*
 785                                 * Collect the results we got on this trial
 786                                 * of PBS
 787                                 */
 788                                for (pup = 0; pup < cur_max_pup; pup++) {
 789                                        for (dq = 0; dq < DQ_NUM; dq++) {
 790                                                skew_sum_array
 791                                                        [pup + (ecc * (max_pup - 1))]
 792                                                        [dq] +=
 793                                                        skew_array[((pup) * DQ_NUM) + dq];
 794                                        }
 795                                }
 796
 797                                /* ECC Support - Disable ECC MUX */
 798                                reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 799                                        ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 800                                reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 801                        }
 802                }
 803
 804                /*
 805                 * Calculate the average skew for current pattern for each
 806                 * pup and each bit
 807                 */
 808                DEBUG_PBS_FULL_C("DDR3 - PBS RX - Average for pattern - ",
 809                                 pattern_idx, 1);
 810                for (pup = 0; pup < max_pup; pup++) {
 811                        /*
 812                         * FOR ECC only :: found min and max value for
 813                         * current pattern skew array
 814                         */
 815                        /* Loop for all dqs */
 816                        for (dq = 0; dq < DQ_NUM; dq++) {
 817                                pattern_skew_array[pup][dq] +=
 818                                        (skew_sum_array[pup][dq] /
 819                                         COUNT_PBS_REPEAT);
 820                        }
 821                }
 822
 823                DEBUG_PBS_C("DDR3 - PBS RX - values for current pattern - ",
 824                            pattern_idx, 1);
 825                for (pup = 0; pup < max_pup; pup++) {
 826                        /*
 827                         * To minimize delay elements, inc from pbs value the
 828                         * min pbs val
 829                         */
 830                        DEBUG_PBS_S("DDR3 - PBS RX - PUP");
 831                        DEBUG_PBS_D(pup, 1);
 832                        DEBUG_PBS_S(": ");
 833
 834                        for (dq = 0; dq < DQ_NUM; dq++) {
 835                                /* Set skew value for all dq */
 836                                /*
 837                                 * Bit# Deskew <- Bit# Deskew - last / first
 838                                 * failing bit Deskew For all bits (per PUP)
 839                                 * (minimize delay elements)
 840                                 */
 841                                DEBUG_PBS_S("DQ");
 842                                DEBUG_PBS_D(dq, 1);
 843                                DEBUG_PBS_S("-");
 844                                DEBUG_PBS_D(skew_sum_array[pup][dq] /
 845                                            COUNT_PBS_REPEAT, 2);
 846                                DEBUG_PBS_S(", ");
 847                        }
 848                        DEBUG_PBS_S("\n");
 849                }
 850        }
 851
 852        /* Calculate the average skew */
 853        for (pup = 0; pup < max_pup; pup++) {
 854                for (dq = 0; dq < DQ_NUM; dq++)
 855                        skew_array[((pup) * DQ_NUM) + dq] =
 856                                pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
 857        }
 858
 859        DEBUG_PBS_S("DDR3 - PBS RX - Average for all patterns:\n");
 860        for (pup = 0; pup < max_pup; pup++) {
 861                /*
 862                 * To minimize delay elements, inc from pbs value the
 863                 * min pbs val
 864                 */
 865                DEBUG_PBS_S("DDR3 - PBS - PUP");
 866                DEBUG_PBS_D(pup, 1);
 867                DEBUG_PBS_S(": ");
 868
 869                for (dq = 0; dq < DQ_NUM; dq++) {
 870                        /* Set skew value for all dq */
 871                        /*
 872                         * Bit# Deskew <- Bit# Deskew - last / first
 873                         * failing bit Deskew For all bits (per PUP)
 874                         * (minimize delay elements)
 875                         */
 876                        DEBUG_PBS_S("DQ");
 877                        DEBUG_PBS_D(dq, 1);
 878                        DEBUG_PBS_S("-");
 879                        DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
 880                        DEBUG_PBS_S(", ");
 881                }
 882                DEBUG_PBS_S("\n");
 883        }
 884
 885        /* Return ADLL to default value */
 886        ddr3_write_pup_reg(PUP_DQS_RD, CS0, PUP_BC, 0, INIT_RL_DELAY);
 887
 888        /* Set averaged PBS results */
 889        ddr3_set_pbs_results(dram_info, 0);
 890
 891        /* Disable SW override - Must be in a different stage */
 892        /* [0]=0 - Enable SW override  */
 893        reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 894        reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 895        /* 0x15B8 - Training SW 2 Register */
 896        reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 897
 898        reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 899                (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 900        reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 901
 902        DEBUG_PBS_FULL_S("DDR3 - PBS RX - ended successfuly\n");
 903
 904        return MV_OK;
 905}
 906
 907/*
 908 * Name:     ddr3_rx_shift_dqs_to_first_fail
 909 * Desc:     Execute the Rx shift DQ phase.
 910 * Args:     dram_info           ddr3 training information struct
 911 *           cur_pup             bit array of the function active pups.
 912 *           pbs_pattern_idx     Index of PBS pattern
 913 * Notes:
 914 * Returns:  MV_OK if success, other error code if fail.
 915 */
 916static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
 917                                           u32 pbs_pattern_idx, u32 ecc)
 918{
 919        u32 unlock_pup;         /* bit array of unlock pups  */
 920        u32 new_lockup_pup;     /* bit array of compare failed pups */
 921        u32 adll_val = MAX_DELAY;
 922        u32 dqs_deskew_val = 0; /* current value of DQS PBS deskew */
 923        u32 cur_max_pup, pup, pass_pup;
 924        u32 *pattern_ptr;
 925
 926        /* Choose pattern */
 927        switch (dram_info->ddr_width) {
 928#if defined(MV88F672X)
 929        case 16:
 930                pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
 931                break;
 932#endif
 933        case 32:
 934                pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
 935                break;
 936#if defined(MV88F78X60)
 937        case 64:
 938                pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
 939                break;
 940#endif
 941        default:
 942                return MV_FAIL;
 943        }
 944
 945        /* Set current pup number */
 946        if (cur_pup == 0x1)     /* Ecc mode */
 947                cur_max_pup = 1;
 948        else
 949                cur_max_pup = dram_info->num_of_std_pups;
 950
 951        unlock_pup = cur_pup;   /* '1' for each unlocked pup */
 952
 953        DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Starting...\n");
 954
 955        /* Set DQS ADLL to MAX */
 956        DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Set DQS ADLL to Max for all PUPs\n");
 957        for (pup = 0; pup < cur_max_pup; pup++)
 958                ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP, 0,
 959                                   MAX_DELAY);
 960
 961        /* Loop on all ADLL Vaules */
 962        do {
 963                /* Loop until found fail for all pups */
 964                new_lockup_pup = 0;
 965                if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
 966                                                &new_lockup_pup,
 967                                                pattern_ptr, LEN_PBS_PATTERN,
 968                                                SDRAM_PBS_I_OFFS +
 969                                                pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
 970                                                0, 0, NULL, 0)) {
 971                        DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
 972                        return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
 973                }
 974
 975                if ((new_lockup_pup != 0) && (dqs_deskew_val <= 1)) {
 976                        /* Fail on start with first deskew value */
 977                        /* Decrement DQS ADLL */
 978                        --adll_val;
 979                        if (adll_val == ADLL_MIN) {
 980                                DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - fail on start with first deskew value\n");
 981                                return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
 982                        }
 983                        ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP,
 984                                           0, adll_val);
 985                        continue;
 986                }
 987
 988                /* Update all new locked pups */
 989                unlock_pup &= ~new_lockup_pup;
 990
 991                if ((unlock_pup == 0) || (dqs_deskew_val == MAX_PBS)) {
 992                        if (dqs_deskew_val == MAX_PBS) {
 993                                /*
 994                                 * Reach max value of dqs deskew or get fail
 995                                 * for all pups
 996                                 */
 997                                DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - DQS deskew reached maximum value\n");
 998                        }
 999                        break;
1000                }

1001
1002                DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Inc DQS deskew for PUPs: ");
1003                DEBUG_PBS_FULL_D(unlock_pup, 2);
1004                DEBUG_PBS_FULL_C(", deskew = ", dqs_deskew_val, 2);
1005
1006                /* Increment DQS deskew elements - Only for unlocked pups */
1007                dqs_deskew_val++;
1008                for (pup = 0; pup < cur_max_pup; pup++) {
1009                        if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1010                                ddr3_write_pup_reg(PUP_PBS_RX + DQS_DQ_NUM, CS0,
1011                                                   pup + ecc * ECC_PUP, 0,
1012                                                   dqs_deskew_val);
1013                        }
1014                }
1015        } while (1);
1016
1017        DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - ADLL shift one step before fail\n");
1018        /* Continue to ADLL shift one step before fail */
1019        unlock_pup = cur_pup;
1020        do {
1021                /* Loop until pass compare for all pups */
1022                new_lockup_pup = 0;
1023                /* Read and compare results  */
1024                if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup, &new_lockup_pup,
1025                                                pattern_ptr, LEN_PBS_PATTERN,
1026                                                SDRAM_PBS_I_OFFS +
1027                                                pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
1028                                                1, 0, NULL, 0)) {
1029                        DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
1030                        return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
1031                }
1032
1033                /*
1034                 * Get mask for pup which passed so their adll will be
1035                 * changed to 2 steps before fails
1036                 */
1037                pass_pup = unlock_pup & ~new_lockup_pup;
1038
1039                DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
1040                DEBUG_PBS_FULL_D(pass_pup, 2);
1041                DEBUG_PBS_FULL_C(", Set ADLL value = ", (adll_val - 2), 2);
1042
1043                /* Only for pass pups   */
1044                for (pup = 0; pup < cur_max_pup; pup++) {
1045                        if (IS_PUP_ACTIVE(pass_pup, pup) == 1) {
1046                                ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1047                                                   pup + ecc * ECC_PUP, 0,
1048                                                   (adll_val - 2));
1049                        }
1050                }
1051
1052                /* Locked pups that compare success  */
1053                unlock_pup &= new_lockup_pup;
1054
1055                if (unlock_pup == 0) {
1056                        /* All pups locked */
1057                        break;
1058                }
1059
1060                /* Found error */
1061                if (adll_val == 0) {
1062                        DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift DQS - Adll reach min value\n");
1063                        return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_MAX_VAL;
1064                }
1065
1066                /*
1067                 * Decrement (Move Back to Left one phase - ADLL) dqs RX delay
1068                 */
1069                adll_val--;
1070                for (pup = 0; pup < cur_max_pup; pup++) {
1071                        if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1072                                ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1073                                                   pup + ecc * ECC_PUP, 0,
1074                                                   adll_val);
1075                        }
1076                }
1077        } while (1);
1078
1079        return MV_OK;
1080}
1081
1082/*
1083 * lock_pups() extracted from ddr3_pbs_per_bit(). This just got too
1084 * much indented making it hard to read / edit.
1085 */
1086static void lock_pups(u32 pup, u32 *pup_locked, u8 *unlock_pup_dq_array,
1087                      u32 pbs_curr_val, u32 start_pbs, u32 ecc, int is_tx)
1088{
1089        u32 dq;
1090        int idx;
1091
1092        /* Lock PBS value for all remaining PUPs bits */
1093        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Lock PBS value for all remaining PUPs bits, pup ");
1094        DEBUG_PBS_FULL_D(pup, 1);
1095        DEBUG_PBS_FULL_C(" pbs value ", pbs_curr_val, 2);
1096
1097        idx = pup * (1 - ecc) + ecc * ECC_PUP;
1098        *pup_locked &= ~(1 << pup);
1099
1100        for (dq = 0; dq < DQ_NUM; dq++) {
1101                if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup) == 1) {
1102                        int offs;
1103
1104                        /* Lock current dq */
1105                        unlock_pup_dq_array[dq] &= ~(1 << pup);
1106                        skew_array[(pup * DQ_NUM) + dq] = pbs_curr_val;
1107
1108                        if (is_tx == 1)
1109                                offs = PUP_PBS_TX;
1110                        else
1111                                offs = PUP_PBS_RX;
1112
1113                        ddr3_write_pup_reg(offs +
1114                                           pbs_dq_mapping[idx][dq], CS0,
1115                                           idx, 0, start_pbs);
1116                }
1117        }
1118}
1119
1120/*
1121 * Name:     ddr3_pbs_per_bit
1122 * Desc:     Execute the Per Bit Skew phase.
1123 * Args:     start_over      Return whether need to start over the algorithm
1124 *           is_tx           Indicate whether Rx or Tx
1125 *           pcur_pup        bit array of the function active pups. return the
1126 *                           pups that need to repeat on the PBS
1127 *           pbs_pattern_idx Index of PBS pattern
1128 *
1129 * Notes:    Current implementation supports double activation of this function.
1130 *           i.e. in order to activate this function (using start_over) more than
1131 *           twice, the implementation should change.
1132 *           imlementation limitation are marked using
1133 *           ' CHIP-ONLY! - Implementation Limitation '
1134 * Returns:  MV_OK if success, other error code if fail.
1135 */
1136static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
1137                            u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc)
1138{
1139        /*
1140         * Bit array to indicate if we already get fail on bit per pup & dq bit
1141         */
1142        u8 unlock_pup_dq_array[DQ_NUM] = {
1143                *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup,
1144                *pcur_pup, *pcur_pup, *pcur_pup
1145        };
1146
1147        u8 cmp_unlock_pup_dq_array[COUNT_PBS_COMP_RETRY_NUM][DQ_NUM];
1148        u32 pup, dq;
1149        /* value of pbs is according to RX or TX */
1150        u32 start_pbs, last_pbs;
1151        u32 pbs_curr_val;
1152        /* bit array that indicates all dq of the pup locked */
1153        u32 pup_locked;
1154        u32 first_fail[MAX_PUP_NUM] = { 0 };    /* count first fail per pup */
1155        /* indicates whether we get first fail per pup */
1156        int first_failed[MAX_PUP_NUM] = { 0 };
1157        /* bit array that indicates pup already get fail */
1158        u32 sum_pup_fail;
1159        /* use to calculate diff between curr pbs to first fail pbs */
1160        u32 calc_pbs_diff;
1161        u32 pbs_cmp_retry;
1162        u32 max_pup;
1163
1164        /* Set init values for retry array - 8 retry */
1165        for (pbs_cmp_retry = 0; pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1166             pbs_cmp_retry++) {
1167                for (dq = 0; dq < DQ_NUM; dq++)
1168                        cmp_unlock_pup_dq_array[pbs_cmp_retry][dq] = *pcur_pup;
1169        }
1170
1171        memset(&skew_array, 0, MAX_PUP_NUM * DQ_NUM * sizeof(u32));
1172
1173        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Started\n");
1174
1175        /* The pbs value depends if rx or tx */
1176        if (is_tx == 1) {
1177                start_pbs = MIN_PBS;
1178                last_pbs = MAX_PBS;
1179        } else {
1180                start_pbs = MAX_PBS;
1181                last_pbs = MIN_PBS;
1182        }
1183
1184        pbs_curr_val = start_pbs;
1185        pup_locked = *pcur_pup;
1186
1187        /* Set current pup number */
1188        if (pup_locked == 0x1)  /* Ecc mode */
1189                max_pup = 1;
1190        else
1191                max_pup = dram_info->num_of_std_pups;
1192
1193        do {
1194                /* Increment/ decrement PBS for un-lock bits only */
1195                if (is_tx == 1)
1196                        pbs_curr_val++;
1197                else
1198                        pbs_curr_val--;
1199
1200                /* Set Current PBS delay  */
1201                for (dq = 0; dq < DQ_NUM; dq++) {
1202                        /* Check DQ bits to see if locked in all pups */
1203                        if (unlock_pup_dq_array[dq] == 0) {
1204                                DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - All pups are locked for DQ ");
1205                                DEBUG_PBS_FULL_D(dq, 1);
1206                                DEBUG_PBS_FULL_S("\n");
1207                                continue;
1208                        }
1209
1210                        for (pup = 0; pup < max_pup; pup++) {
1211                                int idx;
1212
1213                                idx = pup * (1 - ecc) + ecc * ECC_PUP;
1214
1215                                if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup)
1216                                    == 0)
1217                                        continue;
1218
1219                                if (is_tx == 1)
1220                                        ddr3_write_pup_reg(
1221                                                PUP_PBS_TX + pbs_dq_mapping[idx][dq],
1222                                                CS0, idx, 0, pbs_curr_val);
1223                                else
1224                                        ddr3_write_pup_reg(
1225                                                PUP_PBS_RX + pbs_dq_mapping[idx][dq],
1226                                                CS0, idx, 0, pbs_curr_val);
1227                        }
1228                }
1229
1230                /*
1231                 * Write Read and compare results - run the test
1232                 * DDR_PBS_COMP_RETRY_NUM times
1233                 */
1234                /* Run number of read and write to verify */
1235                for (pbs_cmp_retry = 0;
1236                     pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1237                     pbs_cmp_retry++) {
1238
1239                        if (MV_OK !=
1240                            ddr3_sdram_pbs_compare(dram_info, pup_locked, is_tx,
1241                                                   pbs_pattern_idx,
1242                                                   pbs_curr_val, start_pbs,
1243                                                   skew_array,
1244                                                   cmp_unlock_pup_dq_array
1245                                                   [pbs_cmp_retry], ecc))
1246                                return MV_FAIL;
1247
1248                        for (pup = 0; pup < max_pup; pup++) {
1249                                for (dq = 0; dq < DQ_NUM; dq++) {
1250                                        if ((IS_PUP_ACTIVE(unlock_pup_dq_array[dq],
1251                                                           pup) == 1)
1252                                            && (IS_PUP_ACTIVE(cmp_unlock_pup_dq_array
1253                                              [pbs_cmp_retry][dq],
1254                                              pup) == 0)) {
1255                                                DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PbsCurrVal: ");
1256                                                DEBUG_PBS_FULL_D(pbs_curr_val, 2);
1257                                                DEBUG_PBS_FULL_S(" PUP: ");
1258                                                DEBUG_PBS_FULL_D(pup, 1);
1259                                                DEBUG_PBS_FULL_S(" DQ: ");
1260                                                DEBUG_PBS_FULL_D(dq, 1);
1261                                                DEBUG_PBS_FULL_S(" - failed\n");
1262                                        }
1263                                }
1264                        }
1265
1266                        for (dq = 0; dq < DQ_NUM; dq++) {
1267                                unlock_pup_dq_array[dq] &=
1268                                    cmp_unlock_pup_dq_array[pbs_cmp_retry][dq];
1269                        }
1270                }
1271
1272                pup_locked = 0;
1273                sum_pup_fail = *pcur_pup;
1274
1275                /* Check which DQ is failed */
1276                for (dq = 0; dq < DQ_NUM; dq++) {
1277                        /* Summarize the locked pup */
1278                        pup_locked |= unlock_pup_dq_array[dq];
1279
1280                        /* Check if get fail */
1281                        sum_pup_fail &= unlock_pup_dq_array[dq];
1282                }
1283
1284                /* If all PUPS are locked in all DQ - Break */
1285                if (pup_locked == 0) {
1286                        /* All pups are locked */
1287                        *start_over = 0;
1288                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit -  All bit in all pups are successfully locked\n");
1289                        break;
1290                }
1291
1292                /* PBS deskew elements reach max ? */
1293                if (pbs_curr_val == last_pbs) {
1294                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PBS deskew elements reach max\n");
1295                        /* CHIP-ONLY! - Implementation Limitation */
1296                        *start_over = (sum_pup_fail != 0) && (!(*start_over));
1297                        *pcur_pup = pup_locked;
1298
1299                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - StartOver: ");
1300                        DEBUG_PBS_FULL_D(*start_over, 1);
1301                        DEBUG_PBS_FULL_S("  pup_locked: ");
1302                        DEBUG_PBS_FULL_D(pup_locked, 2);
1303                        DEBUG_PBS_FULL_S("  sum_pup_fail: ");
1304                        DEBUG_PBS_FULL_D(sum_pup_fail, 2);
1305                        DEBUG_PBS_FULL_S("\n");
1306
1307                        /* Lock PBS value for all remaining  bits */
1308                        for (pup = 0; pup < max_pup; pup++) {
1309                                /* Check if current pup already received error */
1310                                if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1311                                        /* Valid pup for current function */
1312                                        if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1313                                            1 && (*start_over == 1)) {
1314                                                DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - skipping lock of pup (first loop of pbs)",
1315                                                                 pup, 1);
1316                                                continue;
1317                                        } else
1318                                            if (IS_PUP_ACTIVE(sum_pup_fail, pup)
1319                                                == 1) {
1320                                                DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - Locking pup %d (even though it wasn't supposed to be locked)",
1321                                                                 pup, 1);
1322                                        }
1323
1324                                        /* Already got fail on the PUP */
1325                                        /* Lock PBS value for all remaining bits */
1326                                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Locking remaning DQs for pup - ");
1327                                        DEBUG_PBS_FULL_D(pup, 1);
1328                                        DEBUG_PBS_FULL_S(": ");
1329
1330                                        for (dq = 0; dq < DQ_NUM; dq++) {
1331                                                if (IS_PUP_ACTIVE
1332                                                    (unlock_pup_dq_array[dq],
1333                                                     pup) == 1) {
1334                                                        DEBUG_PBS_FULL_D(dq, 1);
1335                                                        DEBUG_PBS_FULL_S(",");
1336                                                        /* set current PBS */
1337                                                        skew_array[((pup) *
1338                                                                    DQ_NUM) +
1339                                                                   dq] =
1340                                                            pbs_curr_val;
1341                                                }
1342                                        }
1343
1344                                        if (*start_over == 1) {
1345                                                /*
1346                                                 * Reset this pup bit - when
1347                                                 * restart the PBS, ignore this
1348                                                 * pup
1349                                                 */
1350                                                *pcur_pup &= ~(1 << pup);
1351                                        }
1352                                        DEBUG_PBS_FULL_S("\n");
1353                                } else {
1354                                        DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Pup ");
1355                                        DEBUG_PBS_FULL_D(pup, 1);
1356                                        DEBUG_PBS_FULL_C(" is not set in puplocked - ",
1357                                                         pup_locked, 1);
1358                                }
1359                        }
1360
1361                        /* Need to start the PBS again */
1362                        if (*start_over == 1) {
1363                                DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - false fail - returning to start\n");
1364                                return MV_OK;
1365                        }
1366                        break;
1367                }
1368
1369                /* Diff Check */
1370                for (pup = 0; pup < max_pup; pup++) {
1371                        if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1372                                /* pup is not locked */
1373                                if (first_failed[pup] == 0) {
1374                                        /* No first fail until now */
1375                                        if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1376                                            0) {
1377                                                /* Get first fail */
1378                                                DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - First fail in pup ",
1379                                                                 pup, 1);
1380                                                first_failed[pup] = 1;
1381                                                first_fail[pup] = pbs_curr_val;
1382                                        }
1383                                } else {
1384                                        /* Already got first fail */
1385                                        if (is_tx == 1) {
1386                                                /* TX - inc pbs */
1387                                                calc_pbs_diff = pbs_curr_val -
1388                                                        first_fail[pup];
1389                                        } else {
1390                                                /* RX - dec pbs */
1391                                                calc_pbs_diff = first_fail[pup] -
1392                                                        pbs_curr_val;
1393                                        }
1394
1395                                        if (calc_pbs_diff >= PBS_DIFF_LIMIT) {
1396                                                lock_pups(pup, &pup_locked,
1397                                                          unlock_pup_dq_array,
1398                                                          pbs_curr_val,
1399                                                          start_pbs, ecc, is_tx);
1400                                        }
1401                                }
1402                        }
1403                }
1404        } while (1);
1405
1406        return MV_OK;
1407}
1408
1409/*
1410 * Name:         ddr3_set_pbs_results
1411 * Desc:         Set to HW the PBS phase results.
1412 * Args:         is_tx       Indicates whether to set Tx or RX results
1413 * Notes:
1414 * Returns:      MV_OK if success, other error code if fail.
1415 */
1416static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx)
1417{
1418        u32 pup, phys_pup, dq;
1419        u32 max_pup;            /* number of valid pups */
1420        u32 pbs_min;            /* minimal pbs val per pup */
1421        u32 pbs_max;            /* maximum pbs val per pup */
1422        u32 val[9];
1423
1424        max_pup = dram_info->num_of_total_pups;
1425        DEBUG_PBS_FULL_S("DDR3 - PBS - ddr3_set_pbs_results:\n");
1426
1427        /* Loop for all dqs & pups */
1428        for (pup = 0; pup < max_pup; pup++) {
1429                if (pup == (max_pup - 1) && dram_info->ecc_ena)
1430                        phys_pup = ECC_PUP;
1431                else
1432                        phys_pup = pup;
1433
1434                /*
1435                 * To minimize delay elements, inc from pbs value the min
1436                 * pbs val
1437                 */
1438                pbs_min = MAX_PBS;
1439                pbs_max = 0;
1440                for (dq = 0; dq < DQ_NUM; dq++) {
1441                        if (pbs_min > skew_array[(pup * DQ_NUM) + dq])
1442                                pbs_min = skew_array[(pup * DQ_NUM) + dq];
1443
1444                        if (pbs_max < skew_array[(pup * DQ_NUM) + dq])
1445                                pbs_max = skew_array[(pup * DQ_NUM) + dq];
1446                }
1447
1448                pbs_max -= pbs_min;
1449
1450                DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
1451                DEBUG_PBS_FULL_D(phys_pup, 1);
1452                DEBUG_PBS_FULL_S(": Min Val = ");
1453                DEBUG_PBS_FULL_D(pbs_min, 2);
1454                DEBUG_PBS_FULL_C(", Max Val = ", pbs_max, 2);
1455
1456                val[pup] = 0;
1457
1458                for (dq = 0; dq < DQ_NUM; dq++) {
1459                        int idx;
1460                        int offs;
1461
1462                        /* Set skew value for all dq */
1463                        /*
1464                         * Bit# Deskew <- Bit# Deskew - last / first
1465                         * failing bit Deskew For all bits (per PUP)
1466                         * (minimize delay elements)
1467                         */
1468
1469                        DEBUG_PBS_FULL_S("DQ");
1470                        DEBUG_PBS_FULL_D(dq, 1);
1471                        DEBUG_PBS_FULL_S("-");
1472                        DEBUG_PBS_FULL_D((skew_array[(pup * DQ_NUM) + dq] -
1473                                          pbs_min), 2);
1474                        DEBUG_PBS_FULL_S(", ");
1475
1476                        idx = (pup * DQ_NUM) + dq;
1477
1478                        if (is_tx == 1)
1479                                offs = PUP_PBS_TX;
1480                        else
1481                                offs = PUP_PBS_RX;
1482
1483                        ddr3_write_pup_reg(offs + pbs_dq_mapping[phys_pup][dq],
1484                                           CS0, phys_pup, 0,
1485                                           skew_array[idx] - pbs_min);
1486
1487                        if (is_tx == 1)
1488                                val[pup] += skew_array[idx] - pbs_min;
1489                }
1490
1491                DEBUG_PBS_FULL_S("\n");
1492
1493                /* Set the DQS the half of the Max PBS of the DQs  */
1494                if (is_tx == 1) {
1495                        ddr3_write_pup_reg(PUP_PBS_TX + 8, CS0, phys_pup, 0,
1496                                           pbs_max / 2);
1497                        ddr3_write_pup_reg(PUP_PBS_TX + 0xa, CS0, phys_pup, 0,
1498                                           val[pup] / 8);
1499                } else
1500                        ddr3_write_pup_reg(PUP_PBS_RX + 8, CS0, phys_pup, 0,
1501                                           pbs_max / 2);
1502        }
1503
1504        return MV_OK;
1505}
1506
1507static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay)
1508{
1509        u32 reg, delay;
1510
1511        reg = (ddr3_read_pup_reg(PUP_WL_MODE, cs, pup) & 0x3FF);
1512        delay = reg & PUP_DELAY_MASK;
1513        reg |= ((dqs_delay + delay) << REG_PHY_DQS_REF_DLY_OFFS);
1514        reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1515        reg |= (pup << REG_PHY_PUP_OFFS);
1516        reg |= ((0x4 * cs + PUP_WL_MODE) << REG_PHY_CS_OFFS);
1517
1518        reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1519        do {
1520                reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) &
1521                        REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1522        } while (reg);  /* Wait for '0' to mark the end of the transaction */
1523
1524        udelay(10);
1525}
1526
1527/*
1528 * Set training patterns
1529 */
1530int ddr3_load_pbs_patterns(MV_DRAM_INFO *dram_info)
1531{
1532        u32 cs, cs_count, cs_tmp;
1533        u32 sdram_addr;
1534        u32 *pattern_ptr0, *pattern_ptr1;
1535
1536        /* Choose pattern */
1537        switch (dram_info->ddr_width) {
1538#if defined(MV88F672X)
1539        case 16:
1540                pattern_ptr0 = (u32 *)&pbs_pattern[0];
1541                pattern_ptr1 = (u32 *)&pbs_pattern[1];
1542                break;
1543#endif
1544        case 32:
1545                pattern_ptr0 = (u32 *)&pbs_pattern_32b[0];
1546                pattern_ptr1 = (u32 *)&pbs_pattern_32b[1];
1547                break;
1548#if defined(MV88F78X60)
1549        case 64:
1550                pattern_ptr0 = (u32 *)&pbs_pattern_64b[0];
1551                pattern_ptr1 = (u32 *)&pbs_pattern_64b[1];
1552                break;
1553#endif
1554        default:
1555                return MV_FAIL;
1556        }
1557
1558        /* Loop for each CS */
1559        for (cs = 0; cs < MAX_CS; cs++) {
1560                if (dram_info->cs_ena & (1 << cs)) {
1561                        cs_count = 0;
1562                        for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
1563                                if (dram_info->cs_ena & (1 << cs_tmp))
1564                                        cs_count++;
1565                        }
1566
1567                        /* Init PBS I pattern */
1568                        sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1569                                      SDRAM_PBS_I_OFFS);
1570                        if (MV_OK !=
1571                            ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1572                                               pattern_ptr0, LEN_STD_PATTERN,
1573                                               sdram_addr, 1, 0, NULL,
1574                                               0))
1575                                return MV_FAIL;
1576
1577                        /* Init PBS II pattern */
1578                        sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1579                                      SDRAM_PBS_II_OFFS);
1580                        if (MV_OK !=
1581                            ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1582                                               pattern_ptr1, LEN_STD_PATTERN,
1583                                               sdram_addr, 1, 0, NULL,
1584                                               0))
1585                                return MV_FAIL;
1586                }
1587        }
1588
1589        return MV_OK;
1590}
1591#endif
1592