linux/drivers/spi/spi-tegra114.c
<<
>>
Prefs
   1/*
   2 * SPI driver for NVIDIA's Tegra114 SPI Controller.
   3 *
   4 * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms and conditions of the GNU General Public License,
   8 * version 2, as published by the Free Software Foundation.
   9 *
  10 * This program is distributed in the hope it will be useful, but WITHOUT
  11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 * more details.
  14 *
  15 * You should have received a copy of the GNU General Public License
  16 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19#include <linux/clk.h>
  20#include <linux/completion.h>
  21#include <linux/delay.h>
  22#include <linux/dmaengine.h>
  23#include <linux/dma-mapping.h>
  24#include <linux/dmapool.h>
  25#include <linux/err.h>
  26#include <linux/interrupt.h>
  27#include <linux/io.h>
  28#include <linux/kernel.h>
  29#include <linux/kthread.h>
  30#include <linux/module.h>
  31#include <linux/platform_device.h>
  32#include <linux/pm_runtime.h>
  33#include <linux/of.h>
  34#include <linux/of_device.h>
  35#include <linux/reset.h>
  36#include <linux/spi/spi.h>
  37
  38#define SPI_COMMAND1                            0x000
  39#define SPI_BIT_LENGTH(x)                       (((x) & 0x1f) << 0)
  40#define SPI_PACKED                              (1 << 5)
  41#define SPI_TX_EN                               (1 << 11)
  42#define SPI_RX_EN                               (1 << 12)
  43#define SPI_BOTH_EN_BYTE                        (1 << 13)
  44#define SPI_BOTH_EN_BIT                         (1 << 14)
  45#define SPI_LSBYTE_FE                           (1 << 15)
  46#define SPI_LSBIT_FE                            (1 << 16)
  47#define SPI_BIDIROE                             (1 << 17)
  48#define SPI_IDLE_SDA_DRIVE_LOW                  (0 << 18)
  49#define SPI_IDLE_SDA_DRIVE_HIGH                 (1 << 18)
  50#define SPI_IDLE_SDA_PULL_LOW                   (2 << 18)
  51#define SPI_IDLE_SDA_PULL_HIGH                  (3 << 18)
  52#define SPI_IDLE_SDA_MASK                       (3 << 18)
  53#define SPI_CS_SW_VAL                           (1 << 20)
  54#define SPI_CS_SW_HW                            (1 << 21)
  55/* SPI_CS_POL_INACTIVE bits are default high */
  56                                                /* n from 0 to 3 */
  57#define SPI_CS_POL_INACTIVE(n)                  (1 << (22 + (n)))
  58#define SPI_CS_POL_INACTIVE_MASK                (0xF << 22)
  59
  60#define SPI_CS_SEL_0                            (0 << 26)
  61#define SPI_CS_SEL_1                            (1 << 26)
  62#define SPI_CS_SEL_2                            (2 << 26)
  63#define SPI_CS_SEL_3                            (3 << 26)
  64#define SPI_CS_SEL_MASK                         (3 << 26)
  65#define SPI_CS_SEL(x)                           (((x) & 0x3) << 26)
  66#define SPI_CONTROL_MODE_0                      (0 << 28)
  67#define SPI_CONTROL_MODE_1                      (1 << 28)
  68#define SPI_CONTROL_MODE_2                      (2 << 28)
  69#define SPI_CONTROL_MODE_3                      (3 << 28)
  70#define SPI_CONTROL_MODE_MASK                   (3 << 28)
  71#define SPI_MODE_SEL(x)                         (((x) & 0x3) << 28)
  72#define SPI_M_S                                 (1 << 30)
  73#define SPI_PIO                                 (1 << 31)
  74
  75#define SPI_COMMAND2                            0x004
  76#define SPI_TX_TAP_DELAY(x)                     (((x) & 0x3F) << 6)
  77#define SPI_RX_TAP_DELAY(x)                     (((x) & 0x3F) << 0)
  78
  79#define SPI_CS_TIMING1                          0x008
  80#define SPI_SETUP_HOLD(setup, hold)             (((setup) << 4) | (hold))
  81#define SPI_CS_SETUP_HOLD(reg, cs, val)                 \
  82                ((((val) & 0xFFu) << ((cs) * 8)) |      \
  83                ((reg) & ~(0xFFu << ((cs) * 8))))
  84
  85#define SPI_CS_TIMING2                          0x00C
  86#define CYCLES_BETWEEN_PACKETS_0(x)             (((x) & 0x1F) << 0)
  87#define CS_ACTIVE_BETWEEN_PACKETS_0             (1 << 5)
  88#define CYCLES_BETWEEN_PACKETS_1(x)             (((x) & 0x1F) << 8)
  89#define CS_ACTIVE_BETWEEN_PACKETS_1             (1 << 13)
  90#define CYCLES_BETWEEN_PACKETS_2(x)             (((x) & 0x1F) << 16)
  91#define CS_ACTIVE_BETWEEN_PACKETS_2             (1 << 21)
  92#define CYCLES_BETWEEN_PACKETS_3(x)             (((x) & 0x1F) << 24)
  93#define CS_ACTIVE_BETWEEN_PACKETS_3             (1 << 29)
  94#define SPI_SET_CS_ACTIVE_BETWEEN_PACKETS(reg, cs, val)         \
  95                (reg = (((val) & 0x1) << ((cs) * 8 + 5)) |      \
  96                        ((reg) & ~(1 << ((cs) * 8 + 5))))
  97#define SPI_SET_CYCLES_BETWEEN_PACKETS(reg, cs, val)            \
  98                (reg = (((val) & 0xF) << ((cs) * 8)) |          \
  99                        ((reg) & ~(0xF << ((cs) * 8))))
 100
 101#define SPI_TRANS_STATUS                        0x010
 102#define SPI_BLK_CNT(val)                        (((val) >> 0) & 0xFFFF)
 103#define SPI_SLV_IDLE_COUNT(val)                 (((val) >> 16) & 0xFF)
 104#define SPI_RDY                                 (1 << 30)
 105
 106#define SPI_FIFO_STATUS                         0x014
 107#define SPI_RX_FIFO_EMPTY                       (1 << 0)
 108#define SPI_RX_FIFO_FULL                        (1 << 1)
 109#define SPI_TX_FIFO_EMPTY                       (1 << 2)
 110#define SPI_TX_FIFO_FULL                        (1 << 3)
 111#define SPI_RX_FIFO_UNF                         (1 << 4)
 112#define SPI_RX_FIFO_OVF                         (1 << 5)
 113#define SPI_TX_FIFO_UNF                         (1 << 6)
 114#define SPI_TX_FIFO_OVF                         (1 << 7)
 115#define SPI_ERR                                 (1 << 8)
 116#define SPI_TX_FIFO_FLUSH                       (1 << 14)
 117#define SPI_RX_FIFO_FLUSH                       (1 << 15)
 118#define SPI_TX_FIFO_EMPTY_COUNT(val)            (((val) >> 16) & 0x7F)
 119#define SPI_RX_FIFO_FULL_COUNT(val)             (((val) >> 23) & 0x7F)
 120#define SPI_FRAME_END                           (1 << 30)
 121#define SPI_CS_INACTIVE                         (1 << 31)
 122
 123#define SPI_FIFO_ERROR                          (SPI_RX_FIFO_UNF | \
 124                        SPI_RX_FIFO_OVF | SPI_TX_FIFO_UNF | SPI_TX_FIFO_OVF)
 125#define SPI_FIFO_EMPTY                  (SPI_RX_FIFO_EMPTY | SPI_TX_FIFO_EMPTY)
 126
 127#define SPI_TX_DATA                             0x018
 128#define SPI_RX_DATA                             0x01C
 129
 130#define SPI_DMA_CTL                             0x020
 131#define SPI_TX_TRIG_1                           (0 << 15)
 132#define SPI_TX_TRIG_4                           (1 << 15)
 133#define SPI_TX_TRIG_8                           (2 << 15)
 134#define SPI_TX_TRIG_16                          (3 << 15)
 135#define SPI_TX_TRIG_MASK                        (3 << 15)
 136#define SPI_RX_TRIG_1                           (0 << 19)
 137#define SPI_RX_TRIG_4                           (1 << 19)
 138#define SPI_RX_TRIG_8                           (2 << 19)
 139#define SPI_RX_TRIG_16                          (3 << 19)
 140#define SPI_RX_TRIG_MASK                        (3 << 19)
 141#define SPI_IE_TX                               (1 << 28)
 142#define SPI_IE_RX                               (1 << 29)
 143#define SPI_CONT                                (1 << 30)
 144#define SPI_DMA                                 (1 << 31)
 145#define SPI_DMA_EN                              SPI_DMA
 146
 147#define SPI_DMA_BLK                             0x024
 148#define SPI_DMA_BLK_SET(x)                      (((x) & 0xFFFF) << 0)
 149
 150#define SPI_TX_FIFO                             0x108
 151#define SPI_RX_FIFO                             0x188
 152#define MAX_CHIP_SELECT                         4
 153#define SPI_FIFO_DEPTH                          64
 154#define DATA_DIR_TX                             (1 << 0)
 155#define DATA_DIR_RX                             (1 << 1)
 156
 157#define SPI_DMA_TIMEOUT                         (msecs_to_jiffies(1000))
 158#define DEFAULT_SPI_DMA_BUF_LEN                 (16*1024)
 159#define TX_FIFO_EMPTY_COUNT_MAX                 SPI_TX_FIFO_EMPTY_COUNT(0x40)
 160#define RX_FIFO_FULL_COUNT_ZERO                 SPI_RX_FIFO_FULL_COUNT(0)
 161#define MAX_HOLD_CYCLES                         16
 162#define SPI_DEFAULT_SPEED                       25000000
 163
 164struct tegra_spi_data {
 165        struct device                           *dev;
 166        struct spi_master                       *master;
 167        spinlock_t                              lock;
 168
 169        struct clk                              *clk;
 170        struct reset_control                    *rst;
 171        void __iomem                            *base;
 172        phys_addr_t                             phys;
 173        unsigned                                irq;
 174        u32                                     cur_speed;
 175
 176        struct spi_device                       *cur_spi;
 177        struct spi_device                       *cs_control;
 178        unsigned                                cur_pos;
 179        unsigned                                words_per_32bit;
 180        unsigned                                bytes_per_word;
 181        unsigned                                curr_dma_words;
 182        unsigned                                cur_direction;
 183
 184        unsigned                                cur_rx_pos;
 185        unsigned                                cur_tx_pos;
 186
 187        unsigned                                dma_buf_size;
 188        unsigned                                max_buf_size;
 189        bool                                    is_curr_dma_xfer;
 190
 191        struct completion                       rx_dma_complete;
 192        struct completion                       tx_dma_complete;
 193
 194        u32                                     tx_status;
 195        u32                                     rx_status;
 196        u32                                     status_reg;
 197        bool                                    is_packed;
 198
 199        u32                                     command1_reg;
 200        u32                                     dma_control_reg;
 201        u32                                     def_command1_reg;
 202
 203        struct completion                       xfer_completion;
 204        struct spi_transfer                     *curr_xfer;
 205        struct dma_chan                         *rx_dma_chan;
 206        u32                                     *rx_dma_buf;
 207        dma_addr_t                              rx_dma_phys;
 208        struct dma_async_tx_descriptor          *rx_dma_desc;
 209
 210        struct dma_chan                         *tx_dma_chan;
 211        u32                                     *tx_dma_buf;
 212        dma_addr_t                              tx_dma_phys;
 213        struct dma_async_tx_descriptor          *tx_dma_desc;
 214};
 215
 216static int tegra_spi_runtime_suspend(struct device *dev);
 217static int tegra_spi_runtime_resume(struct device *dev);
 218
 219static inline u32 tegra_spi_readl(struct tegra_spi_data *tspi,
 220                unsigned long reg)
 221{
 222        return readl(tspi->base + reg);
 223}
 224
 225static inline void tegra_spi_writel(struct tegra_spi_data *tspi,
 226                u32 val, unsigned long reg)
 227{
 228        writel(val, tspi->base + reg);
 229
 230        /* Read back register to make sure that register writes completed */
 231        if (reg != SPI_TX_FIFO)
 232                readl(tspi->base + SPI_COMMAND1);
 233}
 234
 235static void tegra_spi_clear_status(struct tegra_spi_data *tspi)
 236{
 237        u32 val;
 238
 239        /* Write 1 to clear status register */
 240        val = tegra_spi_readl(tspi, SPI_TRANS_STATUS);
 241        tegra_spi_writel(tspi, val, SPI_TRANS_STATUS);
 242
 243        /* Clear fifo status error if any */
 244        val = tegra_spi_readl(tspi, SPI_FIFO_STATUS);
 245        if (val & SPI_ERR)
 246                tegra_spi_writel(tspi, SPI_ERR | SPI_FIFO_ERROR,
 247                                SPI_FIFO_STATUS);
 248}
 249
 250static unsigned tegra_spi_calculate_curr_xfer_param(
 251        struct spi_device *spi, struct tegra_spi_data *tspi,
 252        struct spi_transfer *t)
 253{
 254        unsigned remain_len = t->len - tspi->cur_pos;
 255        unsigned max_word;
 256        unsigned bits_per_word = t->bits_per_word;
 257        unsigned max_len;
 258        unsigned total_fifo_words;
 259
 260        tspi->bytes_per_word = DIV_ROUND_UP(bits_per_word, 8);
 261
 262        if (bits_per_word == 8 || bits_per_word == 16) {
 263                tspi->is_packed = 1;
 264                tspi->words_per_32bit = 32/bits_per_word;
 265        } else {
 266                tspi->is_packed = 0;
 267                tspi->words_per_32bit = 1;
 268        }
 269
 270        if (tspi->is_packed) {
 271                max_len = min(remain_len, tspi->max_buf_size);
 272                tspi->curr_dma_words = max_len/tspi->bytes_per_word;
 273                total_fifo_words = (max_len + 3) / 4;
 274        } else {
 275                max_word = (remain_len - 1) / tspi->bytes_per_word + 1;
 276                max_word = min(max_word, tspi->max_buf_size/4);
 277                tspi->curr_dma_words = max_word;
 278                total_fifo_words = max_word;
 279        }
 280        return total_fifo_words;
 281}
 282
 283static unsigned tegra_spi_fill_tx_fifo_from_client_txbuf(
 284        struct tegra_spi_data *tspi, struct spi_transfer *t)
 285{
 286        unsigned nbytes;
 287        unsigned tx_empty_count;
 288        u32 fifo_status;
 289        unsigned max_n_32bit;
 290        unsigned i, count;
 291        unsigned int written_words;
 292        unsigned fifo_words_left;
 293        u8 *tx_buf = (u8 *)t->tx_buf + tspi->cur_tx_pos;
 294
 295        fifo_status = tegra_spi_readl(tspi, SPI_FIFO_STATUS);
 296        tx_empty_count = SPI_TX_FIFO_EMPTY_COUNT(fifo_status);
 297
 298        if (tspi->is_packed) {
 299                fifo_words_left = tx_empty_count * tspi->words_per_32bit;
 300                written_words = min(fifo_words_left, tspi->curr_dma_words);
 301                nbytes = written_words * tspi->bytes_per_word;
 302                max_n_32bit = DIV_ROUND_UP(nbytes, 4);
 303                for (count = 0; count < max_n_32bit; count++) {
 304                        u32 x = 0;
 305
 306                        for (i = 0; (i < 4) && nbytes; i++, nbytes--)
 307                                x |= (u32)(*tx_buf++) << (i * 8);
 308                        tegra_spi_writel(tspi, x, SPI_TX_FIFO);
 309                }
 310        } else {
 311                max_n_32bit = min(tspi->curr_dma_words,  tx_empty_count);
 312                written_words = max_n_32bit;
 313                nbytes = written_words * tspi->bytes_per_word;
 314                for (count = 0; count < max_n_32bit; count++) {
 315                        u32 x = 0;
 316
 317                        for (i = 0; nbytes && (i < tspi->bytes_per_word);
 318                                                        i++, nbytes--)
 319                                x |= (u32)(*tx_buf++) << (i * 8);
 320                        tegra_spi_writel(tspi, x, SPI_TX_FIFO);
 321                }
 322        }
 323        tspi->cur_tx_pos += written_words * tspi->bytes_per_word;
 324        return written_words;
 325}
 326
 327static unsigned int tegra_spi_read_rx_fifo_to_client_rxbuf(
 328                struct tegra_spi_data *tspi, struct spi_transfer *t)
 329{
 330        unsigned rx_full_count;
 331        u32 fifo_status;
 332        unsigned i, count;
 333        unsigned int read_words = 0;
 334        unsigned len;
 335        u8 *rx_buf = (u8 *)t->rx_buf + tspi->cur_rx_pos;
 336
 337        fifo_status = tegra_spi_readl(tspi, SPI_FIFO_STATUS);
 338        rx_full_count = SPI_RX_FIFO_FULL_COUNT(fifo_status);
 339        if (tspi->is_packed) {
 340                len = tspi->curr_dma_words * tspi->bytes_per_word;
 341                for (count = 0; count < rx_full_count; count++) {
 342                        u32 x = tegra_spi_readl(tspi, SPI_RX_FIFO);
 343
 344                        for (i = 0; len && (i < 4); i++, len--)
 345                                *rx_buf++ = (x >> i*8) & 0xFF;
 346                }
 347                tspi->cur_rx_pos += tspi->curr_dma_words * tspi->bytes_per_word;
 348                read_words += tspi->curr_dma_words;
 349        } else {
 350                u32 rx_mask = ((u32)1 << t->bits_per_word) - 1;
 351
 352                for (count = 0; count < rx_full_count; count++) {
 353                        u32 x = tegra_spi_readl(tspi, SPI_RX_FIFO) & rx_mask;
 354
 355                        for (i = 0; (i < tspi->bytes_per_word); i++)
 356                                *rx_buf++ = (x >> (i*8)) & 0xFF;
 357                }
 358                tspi->cur_rx_pos += rx_full_count * tspi->bytes_per_word;
 359                read_words += rx_full_count;
 360        }
 361        return read_words;
 362}
 363
 364static void tegra_spi_copy_client_txbuf_to_spi_txbuf(
 365                struct tegra_spi_data *tspi, struct spi_transfer *t)
 366{
 367        /* Make the dma buffer to read by cpu */
 368        dma_sync_single_for_cpu(tspi->dev, tspi->tx_dma_phys,
 369                                tspi->dma_buf_size, DMA_TO_DEVICE);
 370
 371        if (tspi->is_packed) {
 372                unsigned len = tspi->curr_dma_words * tspi->bytes_per_word;
 373
 374                memcpy(tspi->tx_dma_buf, t->tx_buf + tspi->cur_pos, len);
 375        } else {
 376                unsigned int i;
 377                unsigned int count;
 378                u8 *tx_buf = (u8 *)t->tx_buf + tspi->cur_tx_pos;
 379                unsigned consume = tspi->curr_dma_words * tspi->bytes_per_word;
 380
 381                for (count = 0; count < tspi->curr_dma_words; count++) {
 382                        u32 x = 0;
 383
 384                        for (i = 0; consume && (i < tspi->bytes_per_word);
 385                                                        i++, consume--)
 386                                x |= (u32)(*tx_buf++) << (i * 8);
 387                        tspi->tx_dma_buf[count] = x;
 388                }
 389        }
 390        tspi->cur_tx_pos += tspi->curr_dma_words * tspi->bytes_per_word;
 391
 392        /* Make the dma buffer to read by dma */
 393        dma_sync_single_for_device(tspi->dev, tspi->tx_dma_phys,
 394                                tspi->dma_buf_size, DMA_TO_DEVICE);
 395}
 396
 397static void tegra_spi_copy_spi_rxbuf_to_client_rxbuf(
 398                struct tegra_spi_data *tspi, struct spi_transfer *t)
 399{
 400        /* Make the dma buffer to read by cpu */
 401        dma_sync_single_for_cpu(tspi->dev, tspi->rx_dma_phys,
 402                tspi->dma_buf_size, DMA_FROM_DEVICE);
 403
 404        if (tspi->is_packed) {
 405                unsigned len = tspi->curr_dma_words * tspi->bytes_per_word;
 406
 407                memcpy(t->rx_buf + tspi->cur_rx_pos, tspi->rx_dma_buf, len);
 408        } else {
 409                unsigned int i;
 410                unsigned int count;
 411                unsigned char *rx_buf = t->rx_buf + tspi->cur_rx_pos;
 412                u32 rx_mask = ((u32)1 << t->bits_per_word) - 1;
 413
 414                for (count = 0; count < tspi->curr_dma_words; count++) {
 415                        u32 x = tspi->rx_dma_buf[count] & rx_mask;
 416
 417                        for (i = 0; (i < tspi->bytes_per_word); i++)
 418                                *rx_buf++ = (x >> (i*8)) & 0xFF;
 419                }
 420        }
 421        tspi->cur_rx_pos += tspi->curr_dma_words * tspi->bytes_per_word;
 422
 423        /* Make the dma buffer to read by dma */
 424        dma_sync_single_for_device(tspi->dev, tspi->rx_dma_phys,
 425                tspi->dma_buf_size, DMA_FROM_DEVICE);
 426}
 427
 428static void tegra_spi_dma_complete(void *args)
 429{
 430        struct completion *dma_complete = args;
 431
 432        complete(dma_complete);
 433}
 434
 435static int tegra_spi_start_tx_dma(struct tegra_spi_data *tspi, int len)
 436{
 437        reinit_completion(&tspi->tx_dma_complete);
 438        tspi->tx_dma_desc = dmaengine_prep_slave_single(tspi->tx_dma_chan,
 439                                tspi->tx_dma_phys, len, DMA_MEM_TO_DEV,
 440                                DMA_PREP_INTERRUPT |  DMA_CTRL_ACK);
 441        if (!tspi->tx_dma_desc) {
 442                dev_err(tspi->dev, "Not able to get desc for Tx\n");
 443                return -EIO;
 444        }
 445
 446        tspi->tx_dma_desc->callback = tegra_spi_dma_complete;
 447        tspi->tx_dma_desc->callback_param = &tspi->tx_dma_complete;
 448
 449        dmaengine_submit(tspi->tx_dma_desc);
 450        dma_async_issue_pending(tspi->tx_dma_chan);
 451        return 0;
 452}
 453
 454static int tegra_spi_start_rx_dma(struct tegra_spi_data *tspi, int len)
 455{
 456        reinit_completion(&tspi->rx_dma_complete);
 457        tspi->rx_dma_desc = dmaengine_prep_slave_single(tspi->rx_dma_chan,
 458                                tspi->rx_dma_phys, len, DMA_DEV_TO_MEM,
 459                                DMA_PREP_INTERRUPT |  DMA_CTRL_ACK);
 460        if (!tspi->rx_dma_desc) {
 461                dev_err(tspi->dev, "Not able to get desc for Rx\n");
 462                return -EIO;
 463        }
 464
 465        tspi->rx_dma_desc->callback = tegra_spi_dma_complete;
 466        tspi->rx_dma_desc->callback_param = &tspi->rx_dma_complete;
 467
 468        dmaengine_submit(tspi->rx_dma_desc);
 469        dma_async_issue_pending(tspi->rx_dma_chan);
 470        return 0;
 471}
 472
 473static int tegra_spi_start_dma_based_transfer(
 474                struct tegra_spi_data *tspi, struct spi_transfer *t)
 475{
 476        u32 val;
 477        unsigned int len;
 478        int ret = 0;
 479        u32 status;
 480
 481        /* Make sure that Rx and Tx fifo are empty */
 482        status = tegra_spi_readl(tspi, SPI_FIFO_STATUS);
 483        if ((status & SPI_FIFO_EMPTY) != SPI_FIFO_EMPTY) {
 484                dev_err(tspi->dev, "Rx/Tx fifo are not empty status 0x%08x\n",
 485                        (unsigned)status);
 486                return -EIO;
 487        }
 488
 489        val = SPI_DMA_BLK_SET(tspi->curr_dma_words - 1);
 490        tegra_spi_writel(tspi, val, SPI_DMA_BLK);
 491
 492        if (tspi->is_packed)
 493                len = DIV_ROUND_UP(tspi->curr_dma_words * tspi->bytes_per_word,
 494                                        4) * 4;
 495        else
 496                len = tspi->curr_dma_words * 4;
 497
 498        /* Set attention level based on length of transfer */
 499        if (len & 0xF)
 500                val |= SPI_TX_TRIG_1 | SPI_RX_TRIG_1;
 501        else if (((len) >> 4) & 0x1)
 502                val |= SPI_TX_TRIG_4 | SPI_RX_TRIG_4;
 503        else
 504                val |= SPI_TX_TRIG_8 | SPI_RX_TRIG_8;
 505
 506        if (tspi->cur_direction & DATA_DIR_TX)
 507                val |= SPI_IE_TX;
 508
 509        if (tspi->cur_direction & DATA_DIR_RX)
 510                val |= SPI_IE_RX;
 511
 512        tegra_spi_writel(tspi, val, SPI_DMA_CTL);
 513        tspi->dma_control_reg = val;
 514
 515        if (tspi->cur_direction & DATA_DIR_TX) {
 516                tegra_spi_copy_client_txbuf_to_spi_txbuf(tspi, t);
 517                ret = tegra_spi_start_tx_dma(tspi, len);
 518                if (ret < 0) {
 519                        dev_err(tspi->dev,
 520                                "Starting tx dma failed, err %d\n", ret);
 521                        return ret;
 522                }
 523        }
 524
 525        if (tspi->cur_direction & DATA_DIR_RX) {
 526                /* Make the dma buffer to read by dma */
 527                dma_sync_single_for_device(tspi->dev, tspi->rx_dma_phys,
 528                                tspi->dma_buf_size, DMA_FROM_DEVICE);
 529
 530                ret = tegra_spi_start_rx_dma(tspi, len);
 531                if (ret < 0) {
 532                        dev_err(tspi->dev,
 533                                "Starting rx dma failed, err %d\n", ret);
 534                        if (tspi->cur_direction & DATA_DIR_TX)
 535                                dmaengine_terminate_all(tspi->tx_dma_chan);
 536                        return ret;
 537                }
 538        }
 539        tspi->is_curr_dma_xfer = true;
 540        tspi->dma_control_reg = val;
 541
 542        val |= SPI_DMA_EN;
 543        tegra_spi_writel(tspi, val, SPI_DMA_CTL);
 544        return ret;
 545}
 546
 547static int tegra_spi_start_cpu_based_transfer(
 548                struct tegra_spi_data *tspi, struct spi_transfer *t)
 549{
 550        u32 val;
 551        unsigned cur_words;
 552
 553        if (tspi->cur_direction & DATA_DIR_TX)
 554                cur_words = tegra_spi_fill_tx_fifo_from_client_txbuf(tspi, t);
 555        else
 556                cur_words = tspi->curr_dma_words;
 557
 558        val = SPI_DMA_BLK_SET(cur_words - 1);
 559        tegra_spi_writel(tspi, val, SPI_DMA_BLK);
 560
 561        val = 0;
 562        if (tspi->cur_direction & DATA_DIR_TX)
 563                val |= SPI_IE_TX;
 564
 565        if (tspi->cur_direction & DATA_DIR_RX)
 566                val |= SPI_IE_RX;
 567
 568        tegra_spi_writel(tspi, val, SPI_DMA_CTL);
 569        tspi->dma_control_reg = val;
 570
 571        tspi->is_curr_dma_xfer = false;
 572
 573        val |= SPI_DMA_EN;
 574        tegra_spi_writel(tspi, val, SPI_DMA_CTL);
 575        return 0;
 576}
 577
 578static int tegra_spi_init_dma_param(struct tegra_spi_data *tspi,
 579                        bool dma_to_memory)
 580{
 581        struct dma_chan *dma_chan;
 582        u32 *dma_buf;
 583        dma_addr_t dma_phys;
 584        int ret;
 585        struct dma_slave_config dma_sconfig;
 586
 587        dma_chan = dma_request_slave_channel_reason(tspi->dev,
 588                                        dma_to_memory ? "rx" : "tx");
 589        if (IS_ERR(dma_chan)) {
 590                ret = PTR_ERR(dma_chan);
 591                if (ret != -EPROBE_DEFER)
 592                        dev_err(tspi->dev,
 593                                "Dma channel is not available: %d\n", ret);
 594                return ret;
 595        }
 596
 597        dma_buf = dma_alloc_coherent(tspi->dev, tspi->dma_buf_size,
 598                                &dma_phys, GFP_KERNEL);
 599        if (!dma_buf) {
 600                dev_err(tspi->dev, " Not able to allocate the dma buffer\n");
 601                dma_release_channel(dma_chan);
 602                return -ENOMEM;
 603        }
 604
 605        if (dma_to_memory) {
 606                dma_sconfig.src_addr = tspi->phys + SPI_RX_FIFO;
 607                dma_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 608                dma_sconfig.src_maxburst = 0;
 609        } else {
 610                dma_sconfig.dst_addr = tspi->phys + SPI_TX_FIFO;
 611                dma_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 612                dma_sconfig.dst_maxburst = 0;
 613        }
 614
 615        ret = dmaengine_slave_config(dma_chan, &dma_sconfig);
 616        if (ret)
 617                goto scrub;
 618        if (dma_to_memory) {
 619                tspi->rx_dma_chan = dma_chan;
 620                tspi->rx_dma_buf = dma_buf;
 621                tspi->rx_dma_phys = dma_phys;
 622        } else {
 623                tspi->tx_dma_chan = dma_chan;
 624                tspi->tx_dma_buf = dma_buf;
 625                tspi->tx_dma_phys = dma_phys;
 626        }
 627        return 0;
 628
 629scrub:
 630        dma_free_coherent(tspi->dev, tspi->dma_buf_size, dma_buf, dma_phys);
 631        dma_release_channel(dma_chan);
 632        return ret;
 633}
 634
 635static void tegra_spi_deinit_dma_param(struct tegra_spi_data *tspi,
 636        bool dma_to_memory)
 637{
 638        u32 *dma_buf;
 639        dma_addr_t dma_phys;
 640        struct dma_chan *dma_chan;
 641
 642        if (dma_to_memory) {
 643                dma_buf = tspi->rx_dma_buf;
 644                dma_chan = tspi->rx_dma_chan;
 645                dma_phys = tspi->rx_dma_phys;
 646                tspi->rx_dma_chan = NULL;
 647                tspi->rx_dma_buf = NULL;
 648        } else {
 649                dma_buf = tspi->tx_dma_buf;
 650                dma_chan = tspi->tx_dma_chan;
 651                dma_phys = tspi->tx_dma_phys;
 652                tspi->tx_dma_buf = NULL;
 653                tspi->tx_dma_chan = NULL;
 654        }
 655        if (!dma_chan)
 656                return;
 657
 658        dma_free_coherent(tspi->dev, tspi->dma_buf_size, dma_buf, dma_phys);
 659        dma_release_channel(dma_chan);
 660}
 661
 662static u32 tegra_spi_setup_transfer_one(struct spi_device *spi,
 663                struct spi_transfer *t, bool is_first_of_msg)
 664{
 665        struct tegra_spi_data *tspi = spi_master_get_devdata(spi->master);
 666        u32 speed = t->speed_hz;
 667        u8 bits_per_word = t->bits_per_word;
 668        u32 command1;
 669        int req_mode;
 670
 671        if (speed != tspi->cur_speed) {
 672                clk_set_rate(tspi->clk, speed);
 673                tspi->cur_speed = speed;
 674        }
 675
 676        tspi->cur_spi = spi;
 677        tspi->cur_pos = 0;
 678        tspi->cur_rx_pos = 0;
 679        tspi->cur_tx_pos = 0;
 680        tspi->curr_xfer = t;
 681
 682        if (is_first_of_msg) {
 683                tegra_spi_clear_status(tspi);
 684
 685                command1 = tspi->def_command1_reg;
 686                command1 |= SPI_BIT_LENGTH(bits_per_word - 1);
 687
 688                command1 &= ~SPI_CONTROL_MODE_MASK;
 689                req_mode = spi->mode & 0x3;
 690                if (req_mode == SPI_MODE_0)
 691                        command1 |= SPI_CONTROL_MODE_0;
 692                else if (req_mode == SPI_MODE_1)
 693                        command1 |= SPI_CONTROL_MODE_1;
 694                else if (req_mode == SPI_MODE_2)
 695                        command1 |= SPI_CONTROL_MODE_2;
 696                else if (req_mode == SPI_MODE_3)
 697                        command1 |= SPI_CONTROL_MODE_3;
 698
 699                if (tspi->cs_control) {
 700                        if (tspi->cs_control != spi)
 701                                tegra_spi_writel(tspi, command1, SPI_COMMAND1);
 702                        tspi->cs_control = NULL;
 703                } else
 704                        tegra_spi_writel(tspi, command1, SPI_COMMAND1);
 705
 706                command1 |= SPI_CS_SW_HW;
 707                if (spi->mode & SPI_CS_HIGH)
 708                        command1 |= SPI_CS_SW_VAL;
 709                else
 710                        command1 &= ~SPI_CS_SW_VAL;
 711
 712                tegra_spi_writel(tspi, 0, SPI_COMMAND2);
 713        } else {
 714                command1 = tspi->command1_reg;
 715                command1 &= ~SPI_BIT_LENGTH(~0);
 716                command1 |= SPI_BIT_LENGTH(bits_per_word - 1);
 717        }
 718
 719        return command1;
 720}
 721
 722static int tegra_spi_start_transfer_one(struct spi_device *spi,
 723                struct spi_transfer *t, u32 command1)
 724{
 725        struct tegra_spi_data *tspi = spi_master_get_devdata(spi->master);
 726        unsigned total_fifo_words;
 727        int ret;
 728
 729        total_fifo_words = tegra_spi_calculate_curr_xfer_param(spi, tspi, t);
 730
 731        if (tspi->is_packed)
 732                command1 |= SPI_PACKED;
 733
 734        command1 &= ~(SPI_CS_SEL_MASK | SPI_TX_EN | SPI_RX_EN);
 735        tspi->cur_direction = 0;
 736        if (t->rx_buf) {
 737                command1 |= SPI_RX_EN;
 738                tspi->cur_direction |= DATA_DIR_RX;
 739        }
 740        if (t->tx_buf) {
 741                command1 |= SPI_TX_EN;
 742                tspi->cur_direction |= DATA_DIR_TX;
 743        }
 744        command1 |= SPI_CS_SEL(spi->chip_select);
 745        tegra_spi_writel(tspi, command1, SPI_COMMAND1);
 746        tspi->command1_reg = command1;
 747
 748        dev_dbg(tspi->dev, "The def 0x%x and written 0x%x\n",
 749                tspi->def_command1_reg, (unsigned)command1);
 750
 751        if (total_fifo_words > SPI_FIFO_DEPTH)
 752                ret = tegra_spi_start_dma_based_transfer(tspi, t);
 753        else
 754                ret = tegra_spi_start_cpu_based_transfer(tspi, t);
 755        return ret;
 756}
 757
 758static int tegra_spi_setup(struct spi_device *spi)
 759{
 760        struct tegra_spi_data *tspi = spi_master_get_devdata(spi->master);
 761        u32 val;
 762        unsigned long flags;
 763        int ret;
 764
 765        dev_dbg(&spi->dev, "setup %d bpw, %scpol, %scpha, %dHz\n",
 766                spi->bits_per_word,
 767                spi->mode & SPI_CPOL ? "" : "~",
 768                spi->mode & SPI_CPHA ? "" : "~",
 769                spi->max_speed_hz);
 770
 771        ret = pm_runtime_get_sync(tspi->dev);
 772        if (ret < 0) {
 773                dev_err(tspi->dev, "pm runtime failed, e = %d\n", ret);
 774                return ret;
 775        }
 776
 777        spin_lock_irqsave(&tspi->lock, flags);
 778        val = tspi->def_command1_reg;
 779        if (spi->mode & SPI_CS_HIGH)
 780                val &= ~SPI_CS_POL_INACTIVE(spi->chip_select);
 781        else
 782                val |= SPI_CS_POL_INACTIVE(spi->chip_select);
 783        tspi->def_command1_reg = val;
 784        tegra_spi_writel(tspi, tspi->def_command1_reg, SPI_COMMAND1);
 785        spin_unlock_irqrestore(&tspi->lock, flags);
 786
 787        pm_runtime_put(tspi->dev);
 788        return 0;
 789}
 790
 791static void tegra_spi_transfer_delay(int delay)
 792{
 793        if (!delay)
 794                return;
 795
 796        if (delay >= 1000)
 797                mdelay(delay / 1000);
 798
 799        udelay(delay % 1000);
 800}
 801
 802static int tegra_spi_transfer_one_message(struct spi_master *master,
 803                        struct spi_message *msg)
 804{
 805        bool is_first_msg = true;
 806        struct tegra_spi_data *tspi = spi_master_get_devdata(master);
 807        struct spi_transfer *xfer;
 808        struct spi_device *spi = msg->spi;
 809        int ret;
 810        bool skip = false;
 811
 812        msg->status = 0;
 813        msg->actual_length = 0;
 814
 815        list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 816                u32 cmd1;
 817
 818                reinit_completion(&tspi->xfer_completion);
 819
 820                cmd1 = tegra_spi_setup_transfer_one(spi, xfer, is_first_msg);
 821
 822                if (!xfer->len) {
 823                        ret = 0;
 824                        skip = true;
 825                        goto complete_xfer;
 826                }
 827
 828                ret = tegra_spi_start_transfer_one(spi, xfer, cmd1);
 829                if (ret < 0) {
 830                        dev_err(tspi->dev,
 831                                "spi can not start transfer, err %d\n", ret);
 832                        goto complete_xfer;
 833                }
 834
 835                is_first_msg = false;
 836                ret = wait_for_completion_timeout(&tspi->xfer_completion,
 837                                                SPI_DMA_TIMEOUT);
 838                if (WARN_ON(ret == 0)) {
 839                        dev_err(tspi->dev,
 840                                "spi transfer timeout, err %d\n", ret);
 841                        ret = -EIO;
 842                        goto complete_xfer;
 843                }
 844
 845                if (tspi->tx_status ||  tspi->rx_status) {
 846                        dev_err(tspi->dev, "Error in Transfer\n");
 847                        ret = -EIO;
 848                        goto complete_xfer;
 849                }
 850                msg->actual_length += xfer->len;
 851
 852complete_xfer:
 853                if (ret < 0 || skip) {
 854                        tegra_spi_writel(tspi, tspi->def_command1_reg,
 855                                        SPI_COMMAND1);
 856                        tegra_spi_transfer_delay(xfer->delay_usecs);
 857                        goto exit;
 858                } else if (list_is_last(&xfer->transfer_list,
 859                                        &msg->transfers)) {
 860                        if (xfer->cs_change)
 861                                tspi->cs_control = spi;
 862                        else {
 863                                tegra_spi_writel(tspi, tspi->def_command1_reg,
 864                                                SPI_COMMAND1);
 865                                tegra_spi_transfer_delay(xfer->delay_usecs);
 866                        }
 867                } else if (xfer->cs_change) {
 868                        tegra_spi_writel(tspi, tspi->def_command1_reg,
 869                                        SPI_COMMAND1);
 870                        tegra_spi_transfer_delay(xfer->delay_usecs);
 871                }
 872
 873        }
 874        ret = 0;
 875exit:
 876        msg->status = ret;
 877        spi_finalize_current_message(master);
 878        return ret;
 879}
 880
 881static irqreturn_t handle_cpu_based_xfer(struct tegra_spi_data *tspi)
 882{
 883        struct spi_transfer *t = tspi->curr_xfer;
 884        unsigned long flags;
 885
 886        spin_lock_irqsave(&tspi->lock, flags);
 887        if (tspi->tx_status ||  tspi->rx_status) {
 888                dev_err(tspi->dev, "CpuXfer ERROR bit set 0x%x\n",
 889                        tspi->status_reg);
 890                dev_err(tspi->dev, "CpuXfer 0x%08x:0x%08x\n",
 891                        tspi->command1_reg, tspi->dma_control_reg);
 892                reset_control_assert(tspi->rst);
 893                udelay(2);
 894                reset_control_deassert(tspi->rst);
 895                complete(&tspi->xfer_completion);
 896                goto exit;
 897        }
 898
 899        if (tspi->cur_direction & DATA_DIR_RX)
 900                tegra_spi_read_rx_fifo_to_client_rxbuf(tspi, t);
 901
 902        if (tspi->cur_direction & DATA_DIR_TX)
 903                tspi->cur_pos = tspi->cur_tx_pos;
 904        else
 905                tspi->cur_pos = tspi->cur_rx_pos;
 906
 907        if (tspi->cur_pos == t->len) {
 908                complete(&tspi->xfer_completion);
 909                goto exit;
 910        }
 911
 912        tegra_spi_calculate_curr_xfer_param(tspi->cur_spi, tspi, t);
 913        tegra_spi_start_cpu_based_transfer(tspi, t);
 914exit:
 915        spin_unlock_irqrestore(&tspi->lock, flags);
 916        return IRQ_HANDLED;
 917}
 918
 919static irqreturn_t handle_dma_based_xfer(struct tegra_spi_data *tspi)
 920{
 921        struct spi_transfer *t = tspi->curr_xfer;
 922        long wait_status;
 923        int err = 0;
 924        unsigned total_fifo_words;
 925        unsigned long flags;
 926
 927        /* Abort dmas if any error */
 928        if (tspi->cur_direction & DATA_DIR_TX) {
 929                if (tspi->tx_status) {
 930                        dmaengine_terminate_all(tspi->tx_dma_chan);
 931                        err += 1;
 932                } else {
 933                        wait_status = wait_for_completion_interruptible_timeout(
 934                                &tspi->tx_dma_complete, SPI_DMA_TIMEOUT);
 935                        if (wait_status <= 0) {
 936                                dmaengine_terminate_all(tspi->tx_dma_chan);
 937                                dev_err(tspi->dev, "TxDma Xfer failed\n");
 938                                err += 1;
 939                        }
 940                }
 941        }
 942
 943        if (tspi->cur_direction & DATA_DIR_RX) {
 944                if (tspi->rx_status) {
 945                        dmaengine_terminate_all(tspi->rx_dma_chan);
 946                        err += 2;
 947                } else {
 948                        wait_status = wait_for_completion_interruptible_timeout(
 949                                &tspi->rx_dma_complete, SPI_DMA_TIMEOUT);
 950                        if (wait_status <= 0) {
 951                                dmaengine_terminate_all(tspi->rx_dma_chan);
 952                                dev_err(tspi->dev, "RxDma Xfer failed\n");
 953                                err += 2;
 954                        }
 955                }
 956        }
 957
 958        spin_lock_irqsave(&tspi->lock, flags);
 959        if (err) {
 960                dev_err(tspi->dev, "DmaXfer: ERROR bit set 0x%x\n",
 961                        tspi->status_reg);
 962                dev_err(tspi->dev, "DmaXfer 0x%08x:0x%08x\n",
 963                        tspi->command1_reg, tspi->dma_control_reg);
 964                reset_control_assert(tspi->rst);
 965                udelay(2);
 966                reset_control_deassert(tspi->rst);
 967                complete(&tspi->xfer_completion);
 968                spin_unlock_irqrestore(&tspi->lock, flags);
 969                return IRQ_HANDLED;
 970        }
 971
 972        if (tspi->cur_direction & DATA_DIR_RX)
 973                tegra_spi_copy_spi_rxbuf_to_client_rxbuf(tspi, t);
 974
 975        if (tspi->cur_direction & DATA_DIR_TX)
 976                tspi->cur_pos = tspi->cur_tx_pos;
 977        else
 978                tspi->cur_pos = tspi->cur_rx_pos;
 979
 980        if (tspi->cur_pos == t->len) {
 981                complete(&tspi->xfer_completion);
 982                goto exit;
 983        }
 984
 985        /* Continue transfer in current message */
 986        total_fifo_words = tegra_spi_calculate_curr_xfer_param(tspi->cur_spi,
 987                                                        tspi, t);
 988        if (total_fifo_words > SPI_FIFO_DEPTH)
 989                err = tegra_spi_start_dma_based_transfer(tspi, t);
 990        else
 991                err = tegra_spi_start_cpu_based_transfer(tspi, t);
 992
 993exit:
 994        spin_unlock_irqrestore(&tspi->lock, flags);
 995        return IRQ_HANDLED;
 996}
 997
 998static irqreturn_t tegra_spi_isr_thread(int irq, void *context_data)
 999{
1000        struct tegra_spi_data *tspi = context_data;
1001
1002        if (!tspi->is_curr_dma_xfer)
1003                return handle_cpu_based_xfer(tspi);
1004        return handle_dma_based_xfer(tspi);
1005}
1006
1007static irqreturn_t tegra_spi_isr(int irq, void *context_data)
1008{
1009        struct tegra_spi_data *tspi = context_data;
1010
1011        tspi->status_reg = tegra_spi_readl(tspi, SPI_FIFO_STATUS);
1012        if (tspi->cur_direction & DATA_DIR_TX)
1013                tspi->tx_status = tspi->status_reg &
1014                                        (SPI_TX_FIFO_UNF | SPI_TX_FIFO_OVF);
1015
1016        if (tspi->cur_direction & DATA_DIR_RX)
1017                tspi->rx_status = tspi->status_reg &
1018                                        (SPI_RX_FIFO_OVF | SPI_RX_FIFO_UNF);
1019        tegra_spi_clear_status(tspi);
1020
1021        return IRQ_WAKE_THREAD;
1022}
1023
1024static const struct of_device_id tegra_spi_of_match[] = {
1025        { .compatible = "nvidia,tegra114-spi", },
1026        {}
1027};
1028MODULE_DEVICE_TABLE(of, tegra_spi_of_match);
1029
1030static int tegra_spi_probe(struct platform_device *pdev)
1031{
1032        struct spi_master       *master;
1033        struct tegra_spi_data   *tspi;
1034        struct resource         *r;
1035        int ret, spi_irq;
1036
1037        master = spi_alloc_master(&pdev->dev, sizeof(*tspi));
1038        if (!master) {
1039                dev_err(&pdev->dev, "master allocation failed\n");
1040                return -ENOMEM;
1041        }
1042        platform_set_drvdata(pdev, master);
1043        tspi = spi_master_get_devdata(master);
1044
1045        if (of_property_read_u32(pdev->dev.of_node, "spi-max-frequency",
1046                                 &master->max_speed_hz))
1047                master->max_speed_hz = 25000000; /* 25MHz */
1048
1049        /* the spi->mode bits understood by this driver: */
1050        master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1051        master->setup = tegra_spi_setup;
1052        master->transfer_one_message = tegra_spi_transfer_one_message;
1053        master->num_chipselect = MAX_CHIP_SELECT;
1054        master->auto_runtime_pm = true;
1055
1056        tspi->master = master;
1057        tspi->dev = &pdev->dev;
1058        spin_lock_init(&tspi->lock);
1059
1060        r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1061        tspi->base = devm_ioremap_resource(&pdev->dev, r);
1062        if (IS_ERR(tspi->base)) {
1063                ret = PTR_ERR(tspi->base);
1064                goto exit_free_master;
1065        }
1066        tspi->phys = r->start;
1067
1068        spi_irq = platform_get_irq(pdev, 0);
1069        tspi->irq = spi_irq;
1070        ret = request_threaded_irq(tspi->irq, tegra_spi_isr,
1071                        tegra_spi_isr_thread, IRQF_ONESHOT,
1072                        dev_name(&pdev->dev), tspi);
1073        if (ret < 0) {
1074                dev_err(&pdev->dev, "Failed to register ISR for IRQ %d\n",
1075                                        tspi->irq);
1076                goto exit_free_master;
1077        }
1078
1079        tspi->clk = devm_clk_get(&pdev->dev, "spi");
1080        if (IS_ERR(tspi->clk)) {
1081                dev_err(&pdev->dev, "can not get clock\n");
1082                ret = PTR_ERR(tspi->clk);
1083                goto exit_free_irq;
1084        }
1085
1086        tspi->rst = devm_reset_control_get_exclusive(&pdev->dev, "spi");
1087        if (IS_ERR(tspi->rst)) {
1088                dev_err(&pdev->dev, "can not get reset\n");
1089                ret = PTR_ERR(tspi->rst);
1090                goto exit_free_irq;
1091        }
1092
1093        tspi->max_buf_size = SPI_FIFO_DEPTH << 2;
1094        tspi->dma_buf_size = DEFAULT_SPI_DMA_BUF_LEN;
1095
1096        ret = tegra_spi_init_dma_param(tspi, true);
1097        if (ret < 0)
1098                goto exit_free_irq;
1099        ret = tegra_spi_init_dma_param(tspi, false);
1100        if (ret < 0)
1101                goto exit_rx_dma_free;
1102        tspi->max_buf_size = tspi->dma_buf_size;
1103        init_completion(&tspi->tx_dma_complete);
1104        init_completion(&tspi->rx_dma_complete);
1105
1106        init_completion(&tspi->xfer_completion);
1107
1108        pm_runtime_enable(&pdev->dev);
1109        if (!pm_runtime_enabled(&pdev->dev)) {
1110                ret = tegra_spi_runtime_resume(&pdev->dev);
1111                if (ret)
1112                        goto exit_pm_disable;
1113        }
1114
1115        ret = pm_runtime_get_sync(&pdev->dev);
1116        if (ret < 0) {
1117                dev_err(&pdev->dev, "pm runtime get failed, e = %d\n", ret);
1118                goto exit_pm_disable;
1119        }
1120        tspi->def_command1_reg  = SPI_M_S;
1121        tegra_spi_writel(tspi, tspi->def_command1_reg, SPI_COMMAND1);
1122        pm_runtime_put(&pdev->dev);
1123
1124        master->dev.of_node = pdev->dev.of_node;
1125        ret = devm_spi_register_master(&pdev->dev, master);
1126        if (ret < 0) {
1127                dev_err(&pdev->dev, "can not register to master err %d\n", ret);
1128                goto exit_pm_disable;
1129        }
1130        return ret;
1131
1132exit_pm_disable:
1133        pm_runtime_disable(&pdev->dev);
1134        if (!pm_runtime_status_suspended(&pdev->dev))
1135                tegra_spi_runtime_suspend(&pdev->dev);
1136        tegra_spi_deinit_dma_param(tspi, false);
1137exit_rx_dma_free:
1138        tegra_spi_deinit_dma_param(tspi, true);
1139exit_free_irq:
1140        free_irq(spi_irq, tspi);
1141exit_free_master:
1142        spi_master_put(master);
1143        return ret;
1144}
1145
1146static int tegra_spi_remove(struct platform_device *pdev)
1147{
1148        struct spi_master *master = platform_get_drvdata(pdev);
1149        struct tegra_spi_data   *tspi = spi_master_get_devdata(master);
1150
1151        free_irq(tspi->irq, tspi);
1152
1153        if (tspi->tx_dma_chan)
1154                tegra_spi_deinit_dma_param(tspi, false);
1155
1156        if (tspi->rx_dma_chan)
1157                tegra_spi_deinit_dma_param(tspi, true);
1158
1159        pm_runtime_disable(&pdev->dev);
1160        if (!pm_runtime_status_suspended(&pdev->dev))
1161                tegra_spi_runtime_suspend(&pdev->dev);
1162
1163        return 0;
1164}
1165
1166#ifdef CONFIG_PM_SLEEP
1167static int tegra_spi_suspend(struct device *dev)
1168{
1169        struct spi_master *master = dev_get_drvdata(dev);
1170
1171        return spi_master_suspend(master);
1172}
1173
1174static int tegra_spi_resume(struct device *dev)
1175{
1176        struct spi_master *master = dev_get_drvdata(dev);
1177        struct tegra_spi_data *tspi = spi_master_get_devdata(master);
1178        int ret;
1179
1180        ret = pm_runtime_get_sync(dev);
1181        if (ret < 0) {
1182                dev_err(dev, "pm runtime failed, e = %d\n", ret);
1183                return ret;
1184        }
1185        tegra_spi_writel(tspi, tspi->command1_reg, SPI_COMMAND1);
1186        pm_runtime_put(dev);
1187
1188        return spi_master_resume(master);
1189}
1190#endif
1191
1192static int tegra_spi_runtime_suspend(struct device *dev)
1193{
1194        struct spi_master *master = dev_get_drvdata(dev);
1195        struct tegra_spi_data *tspi = spi_master_get_devdata(master);
1196
1197        /* Flush all write which are in PPSB queue by reading back */
1198        tegra_spi_readl(tspi, SPI_COMMAND1);
1199
1200        clk_disable_unprepare(tspi->clk);
1201        return 0;
1202}
1203
1204static int tegra_spi_runtime_resume(struct device *dev)
1205{
1206        struct spi_master *master = dev_get_drvdata(dev);
1207        struct tegra_spi_data *tspi = spi_master_get_devdata(master);
1208        int ret;
1209
1210        ret = clk_prepare_enable(tspi->clk);
1211        if (ret < 0) {
1212                dev_err(tspi->dev, "clk_prepare failed: %d\n", ret);
1213                return ret;
1214        }
1215        return 0;
1216}
1217
1218static const struct dev_pm_ops tegra_spi_pm_ops = {
1219        SET_RUNTIME_PM_OPS(tegra_spi_runtime_suspend,
1220                tegra_spi_runtime_resume, NULL)
1221        SET_SYSTEM_SLEEP_PM_OPS(tegra_spi_suspend, tegra_spi_resume)
1222};
1223static struct platform_driver tegra_spi_driver = {
1224        .driver = {
1225                .name           = "spi-tegra114",
1226                .pm             = &tegra_spi_pm_ops,
1227                .of_match_table = tegra_spi_of_match,
1228        },
1229        .probe =        tegra_spi_probe,
1230        .remove =       tegra_spi_remove,
1231};
1232module_platform_driver(tegra_spi_driver);
1233
1234MODULE_ALIAS("platform:spi-tegra114");
1235MODULE_DESCRIPTION("NVIDIA Tegra114 SPI Controller Driver");
1236MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
1237MODULE_LICENSE("GPL v2");
1238