linux/drivers/scsi/ufs/ufshcd.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Universal Flash Storage Host controller driver Core
   4 * Copyright (C) 2011-2013 Samsung India Software Operations
   5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   6 *
   7 * Authors:
   8 *      Santosh Yaraganavi <santosh.sy@samsung.com>
   9 *      Vinayak Holikatti <h.vinayak@samsung.com>
  10 */
  11
  12#include <linux/async.h>
  13#include <linux/devfreq.h>
  14#include <linux/nls.h>
  15#include <linux/of.h>
  16#include <linux/bitfield.h>
  17#include <linux/blk-pm.h>
  18#include <linux/blkdev.h>
  19#include <scsi/scsi_driver.h>
  20#include "ufshcd.h"
  21#include "ufs_quirks.h"
  22#include "unipro.h"
  23#include "ufs-sysfs.h"
  24#include "ufs-debugfs.h"
  25#include "ufs-fault-injection.h"
  26#include "ufs_bsg.h"
  27#include "ufshcd-crypto.h"
  28#include "ufshpb.h"
  29#include <asm/unaligned.h>
  30
  31#define CREATE_TRACE_POINTS
  32#include <trace/events/ufs.h>
  33
  34#define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  35                                 UTP_TASK_REQ_COMPL |\
  36                                 UFSHCD_ERROR_MASK)
  37/* UIC command timeout, unit: ms */
  38#define UIC_CMD_TIMEOUT 500
  39
  40/* NOP OUT retries waiting for NOP IN response */
  41#define NOP_OUT_RETRIES    10
  42/* Timeout after 50 msecs if NOP OUT hangs without response */
  43#define NOP_OUT_TIMEOUT    50 /* msecs */
  44
  45/* Query request retries */
  46#define QUERY_REQ_RETRIES 3
  47/* Query request timeout */
  48#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  49
  50/* Task management command timeout */
  51#define TM_CMD_TIMEOUT  100 /* msecs */
  52
  53/* maximum number of retries for a general UIC command  */
  54#define UFS_UIC_COMMAND_RETRIES 3
  55
  56/* maximum number of link-startup retries */
  57#define DME_LINKSTARTUP_RETRIES 3
  58
  59/* Maximum retries for Hibern8 enter */
  60#define UIC_HIBERN8_ENTER_RETRIES 3
  61
  62/* maximum number of reset retries before giving up */
  63#define MAX_HOST_RESET_RETRIES 5
  64
  65/* Expose the flag value from utp_upiu_query.value */
  66#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  67
  68/* Interrupt aggregation default timeout, unit: 40us */
  69#define INT_AGGR_DEF_TO 0x02
  70
  71/* default delay of autosuspend: 2000 ms */
  72#define RPM_AUTOSUSPEND_DELAY_MS 2000
  73
  74/* Default delay of RPM device flush delayed work */
  75#define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
  76
  77/* Default value of wait time before gating device ref clock */
  78#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
  79
  80/* Polling time to wait for fDeviceInit */
  81#define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
  82
  83#define wlun_dev_to_hba(dv) shost_priv(to_scsi_device(dv)->host)
  84
  85#define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
  86        ({                                                              \
  87                int _ret;                                               \
  88                if (_on)                                                \
  89                        _ret = ufshcd_enable_vreg(_dev, _vreg);         \
  90                else                                                    \
  91                        _ret = ufshcd_disable_vreg(_dev, _vreg);        \
  92                _ret;                                                   \
  93        })
  94
  95#define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
  96        size_t __len = (len);                                            \
  97        print_hex_dump(KERN_ERR, prefix_str,                             \
  98                       __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
  99                       16, 4, buf, __len, false);                        \
 100} while (0)
 101
 102int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
 103                     const char *prefix)
 104{
 105        u32 *regs;
 106        size_t pos;
 107
 108        if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
 109                return -EINVAL;
 110
 111        regs = kzalloc(len, GFP_ATOMIC);
 112        if (!regs)
 113                return -ENOMEM;
 114
 115        for (pos = 0; pos < len; pos += 4)
 116                regs[pos / 4] = ufshcd_readl(hba, offset + pos);
 117
 118        ufshcd_hex_dump(prefix, regs, len);
 119        kfree(regs);
 120
 121        return 0;
 122}
 123EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
 124
 125enum {
 126        UFSHCD_MAX_CHANNEL      = 0,
 127        UFSHCD_MAX_ID           = 1,
 128        UFSHCD_CMD_PER_LUN      = 32,
 129        UFSHCD_CAN_QUEUE        = 32,
 130};
 131
 132/* UFSHCD error handling flags */
 133enum {
 134        UFSHCD_EH_IN_PROGRESS = (1 << 0),
 135};
 136
 137/* UFSHCD UIC layer error flags */
 138enum {
 139        UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
 140        UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
 141        UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
 142        UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
 143        UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
 144        UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
 145        UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
 146};
 147
 148#define ufshcd_set_eh_in_progress(h) \
 149        ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
 150#define ufshcd_eh_in_progress(h) \
 151        ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
 152#define ufshcd_clear_eh_in_progress(h) \
 153        ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
 154
 155struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
 156        [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 157        [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 158        [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 159        [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 160        [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 161        [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
 162        /*
 163         * For DeepSleep, the link is first put in hibern8 and then off.
 164         * Leaving the link in hibern8 is not supported.
 165         */
 166        [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
 167};
 168
 169static inline enum ufs_dev_pwr_mode
 170ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
 171{
 172        return ufs_pm_lvl_states[lvl].dev_state;
 173}
 174
 175static inline enum uic_link_state
 176ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
 177{
 178        return ufs_pm_lvl_states[lvl].link_state;
 179}
 180
 181static inline enum ufs_pm_level
 182ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
 183                                        enum uic_link_state link_state)
 184{
 185        enum ufs_pm_level lvl;
 186
 187        for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
 188                if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
 189                        (ufs_pm_lvl_states[lvl].link_state == link_state))
 190                        return lvl;
 191        }
 192
 193        /* if no match found, return the level 0 */
 194        return UFS_PM_LVL_0;
 195}
 196
 197static struct ufs_dev_fix ufs_fixups[] = {
 198        /* UFS cards deviations table */
 199        UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
 200                UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
 201                UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ),
 202        UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 203                UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
 204                UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
 205                UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
 206        UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
 207                UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
 208        UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
 209                UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
 210        UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
 211                UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 212        UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
 213                UFS_DEVICE_QUIRK_PA_TACTIVATE),
 214        UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
 215                UFS_DEVICE_QUIRK_PA_TACTIVATE),
 216        END_FIX
 217};
 218
 219static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
 220static void ufshcd_async_scan(void *data, async_cookie_t cookie);
 221static int ufshcd_reset_and_restore(struct ufs_hba *hba);
 222static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
 223static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
 224static void ufshcd_hba_exit(struct ufs_hba *hba);
 225static int ufshcd_clear_ua_wluns(struct ufs_hba *hba);
 226static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
 227static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
 228static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
 229static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
 230static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
 231static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
 232static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 233static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 234static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
 235static irqreturn_t ufshcd_intr(int irq, void *__hba);
 236static int ufshcd_change_power_mode(struct ufs_hba *hba,
 237                             struct ufs_pa_layer_attr *pwr_mode);
 238static void ufshcd_schedule_eh_work(struct ufs_hba *hba);
 239static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
 240static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
 241static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
 242                                         struct ufs_vreg *vreg);
 243static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
 244static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
 245static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
 246static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
 247static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
 248
 249static inline void ufshcd_enable_irq(struct ufs_hba *hba)
 250{
 251        if (!hba->is_irq_enabled) {
 252                enable_irq(hba->irq);
 253                hba->is_irq_enabled = true;
 254        }
 255}
 256
 257static inline void ufshcd_disable_irq(struct ufs_hba *hba)
 258{
 259        if (hba->is_irq_enabled) {
 260                disable_irq(hba->irq);
 261                hba->is_irq_enabled = false;
 262        }
 263}
 264
 265static inline void ufshcd_wb_config(struct ufs_hba *hba)
 266{
 267        if (!ufshcd_is_wb_allowed(hba))
 268                return;
 269
 270        ufshcd_wb_toggle(hba, true);
 271
 272        ufshcd_wb_toggle_flush_during_h8(hba, true);
 273        if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL))
 274                ufshcd_wb_toggle_flush(hba, true);
 275}
 276
 277static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
 278{
 279        if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
 280                scsi_unblock_requests(hba->host);
 281}
 282
 283static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
 284{
 285        if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
 286                scsi_block_requests(hba->host);
 287}
 288
 289static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 290                                      enum ufs_trace_str_t str_t)
 291{
 292        struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 293        struct utp_upiu_header *header;
 294
 295        if (!trace_ufshcd_upiu_enabled())
 296                return;
 297
 298        if (str_t == UFS_CMD_SEND)
 299                header = &rq->header;
 300        else
 301                header = &hba->lrb[tag].ucd_rsp_ptr->header;
 302
 303        trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
 304                          UFS_TSF_CDB);
 305}
 306
 307static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
 308                                        enum ufs_trace_str_t str_t,
 309                                        struct utp_upiu_req *rq_rsp)
 310{
 311        if (!trace_ufshcd_upiu_enabled())
 312                return;
 313
 314        trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
 315                          &rq_rsp->qr, UFS_TSF_OSF);
 316}
 317
 318static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 319                                     enum ufs_trace_str_t str_t)
 320{
 321        struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
 322
 323        if (!trace_ufshcd_upiu_enabled())
 324                return;
 325
 326        if (str_t == UFS_TM_SEND)
 327                trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 328                                  &descp->upiu_req.req_header,
 329                                  &descp->upiu_req.input_param1,
 330                                  UFS_TSF_TM_INPUT);
 331        else
 332                trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 333                                  &descp->upiu_rsp.rsp_header,
 334                                  &descp->upiu_rsp.output_param1,
 335                                  UFS_TSF_TM_OUTPUT);
 336}
 337
 338static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
 339                                         struct uic_command *ucmd,
 340                                         enum ufs_trace_str_t str_t)
 341{
 342        u32 cmd;
 343
 344        if (!trace_ufshcd_uic_command_enabled())
 345                return;
 346
 347        if (str_t == UFS_CMD_SEND)
 348                cmd = ucmd->command;
 349        else
 350                cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
 351
 352        trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
 353                                 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
 354                                 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
 355                                 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
 356}
 357
 358static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
 359                                     enum ufs_trace_str_t str_t)
 360{
 361        u64 lba;
 362        u8 opcode = 0, group_id = 0;
 363        u32 intr, doorbell;
 364        struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 365        struct scsi_cmnd *cmd = lrbp->cmd;
 366        struct request *rq = scsi_cmd_to_rq(cmd);
 367        int transfer_len = -1;
 368
 369        if (!cmd)
 370                return;
 371
 372        /* trace UPIU also */
 373        ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
 374        if (!trace_ufshcd_command_enabled())
 375                return;
 376
 377        opcode = cmd->cmnd[0];
 378        lba = scsi_get_lba(cmd);
 379
 380        if (opcode == READ_10 || opcode == WRITE_10) {
 381                /*
 382                 * Currently we only fully trace read(10) and write(10) commands
 383                 */
 384                transfer_len =
 385                       be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
 386                if (opcode == WRITE_10)
 387                        group_id = lrbp->cmd->cmnd[6];
 388        } else if (opcode == UNMAP) {
 389                /*
 390                 * The number of Bytes to be unmapped beginning with the lba.
 391                 */
 392                transfer_len = blk_rq_bytes(rq);
 393        }
 394
 395        intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 396        doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 397        trace_ufshcd_command(dev_name(hba->dev), str_t, tag,
 398                        doorbell, transfer_len, intr, lba, opcode, group_id);
 399}
 400
 401static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
 402{
 403        struct ufs_clk_info *clki;
 404        struct list_head *head = &hba->clk_list_head;
 405
 406        if (list_empty(head))
 407                return;
 408
 409        list_for_each_entry(clki, head, list) {
 410                if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
 411                                clki->max_freq)
 412                        dev_err(hba->dev, "clk: %s, rate: %u\n",
 413                                        clki->name, clki->curr_freq);
 414        }
 415}
 416
 417static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
 418                             char *err_name)
 419{
 420        int i;
 421        bool found = false;
 422        struct ufs_event_hist *e;
 423
 424        if (id >= UFS_EVT_CNT)
 425                return;
 426
 427        e = &hba->ufs_stats.event[id];
 428
 429        for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
 430                int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
 431
 432                if (e->tstamp[p] == 0)
 433                        continue;
 434                dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
 435                        e->val[p], ktime_to_us(e->tstamp[p]));
 436                found = true;
 437        }
 438
 439        if (!found)
 440                dev_err(hba->dev, "No record of %s\n", err_name);
 441        else
 442                dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
 443}
 444
 445static void ufshcd_print_evt_hist(struct ufs_hba *hba)
 446{
 447        ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
 448
 449        ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
 450        ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
 451        ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
 452        ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
 453        ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
 454        ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
 455                         "auto_hibern8_err");
 456        ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
 457        ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
 458                         "link_startup_fail");
 459        ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
 460        ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
 461                         "suspend_fail");
 462        ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
 463        ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
 464        ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
 465
 466        ufshcd_vops_dbg_register_dump(hba);
 467}
 468
 469static
 470void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
 471{
 472        struct ufshcd_lrb *lrbp;
 473        int prdt_length;
 474        int tag;
 475
 476        for_each_set_bit(tag, &bitmap, hba->nutrs) {
 477                lrbp = &hba->lrb[tag];
 478
 479                dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
 480                                tag, ktime_to_us(lrbp->issue_time_stamp));
 481                dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
 482                                tag, ktime_to_us(lrbp->compl_time_stamp));
 483                dev_err(hba->dev,
 484                        "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
 485                        tag, (u64)lrbp->utrd_dma_addr);
 486
 487                ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
 488                                sizeof(struct utp_transfer_req_desc));
 489                dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
 490                        (u64)lrbp->ucd_req_dma_addr);
 491                ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
 492                                sizeof(struct utp_upiu_req));
 493                dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
 494                        (u64)lrbp->ucd_rsp_dma_addr);
 495                ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
 496                                sizeof(struct utp_upiu_rsp));
 497
 498                prdt_length = le16_to_cpu(
 499                        lrbp->utr_descriptor_ptr->prd_table_length);
 500                if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
 501                        prdt_length /= sizeof(struct ufshcd_sg_entry);
 502
 503                dev_err(hba->dev,
 504                        "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
 505                        tag, prdt_length,
 506                        (u64)lrbp->ucd_prdt_dma_addr);
 507
 508                if (pr_prdt)
 509                        ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
 510                                sizeof(struct ufshcd_sg_entry) * prdt_length);
 511        }
 512}
 513
 514static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
 515{
 516        int tag;
 517
 518        for_each_set_bit(tag, &bitmap, hba->nutmrs) {
 519                struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
 520
 521                dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
 522                ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
 523        }
 524}
 525
 526static void ufshcd_print_host_state(struct ufs_hba *hba)
 527{
 528        struct scsi_device *sdev_ufs = hba->sdev_ufs_device;
 529
 530        dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
 531        dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
 532                hba->outstanding_reqs, hba->outstanding_tasks);
 533        dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
 534                hba->saved_err, hba->saved_uic_err);
 535        dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
 536                hba->curr_dev_pwr_mode, hba->uic_link_state);
 537        dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
 538                hba->pm_op_in_progress, hba->is_sys_suspended);
 539        dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
 540                hba->auto_bkops_enabled, hba->host->host_self_blocked);
 541        dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
 542        dev_err(hba->dev,
 543                "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
 544                ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
 545                hba->ufs_stats.hibern8_exit_cnt);
 546        dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
 547                ktime_to_us(hba->ufs_stats.last_intr_ts),
 548                hba->ufs_stats.last_intr_status);
 549        dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
 550                hba->eh_flags, hba->req_abort_count);
 551        dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
 552                hba->ufs_version, hba->capabilities, hba->caps);
 553        dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
 554                hba->dev_quirks);
 555        if (sdev_ufs)
 556                dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
 557                        sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
 558
 559        ufshcd_print_clk_freqs(hba);
 560}
 561
 562/**
 563 * ufshcd_print_pwr_info - print power params as saved in hba
 564 * power info
 565 * @hba: per-adapter instance
 566 */
 567static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 568{
 569        static const char * const names[] = {
 570                "INVALID MODE",
 571                "FAST MODE",
 572                "SLOW_MODE",
 573                "INVALID MODE",
 574                "FASTAUTO_MODE",
 575                "SLOWAUTO_MODE",
 576                "INVALID MODE",
 577        };
 578
 579        dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 580                 __func__,
 581                 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 582                 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 583                 names[hba->pwr_info.pwr_rx],
 584                 names[hba->pwr_info.pwr_tx],
 585                 hba->pwr_info.hs_rate);
 586}
 587
 588static void ufshcd_device_reset(struct ufs_hba *hba)
 589{
 590        int err;
 591
 592        err = ufshcd_vops_device_reset(hba);
 593
 594        if (!err) {
 595                ufshcd_set_ufs_dev_active(hba);
 596                if (ufshcd_is_wb_allowed(hba)) {
 597                        hba->dev_info.wb_enabled = false;
 598                        hba->dev_info.wb_buf_flush_enabled = false;
 599                }
 600        }
 601        if (err != -EOPNOTSUPP)
 602                ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
 603}
 604
 605void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
 606{
 607        if (!us)
 608                return;
 609
 610        if (us < 10)
 611                udelay(us);
 612        else
 613                usleep_range(us, us + tolerance);
 614}
 615EXPORT_SYMBOL_GPL(ufshcd_delay_us);
 616
 617/**
 618 * ufshcd_wait_for_register - wait for register value to change
 619 * @hba: per-adapter interface
 620 * @reg: mmio register offset
 621 * @mask: mask to apply to the read register value
 622 * @val: value to wait for
 623 * @interval_us: polling interval in microseconds
 624 * @timeout_ms: timeout in milliseconds
 625 *
 626 * Return:
 627 * -ETIMEDOUT on error, zero on success.
 628 */
 629int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 630                                u32 val, unsigned long interval_us,
 631                                unsigned long timeout_ms)
 632{
 633        int err = 0;
 634        unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 635
 636        /* ignore bits that we don't intend to wait on */
 637        val = val & mask;
 638
 639        while ((ufshcd_readl(hba, reg) & mask) != val) {
 640                usleep_range(interval_us, interval_us + 50);
 641                if (time_after(jiffies, timeout)) {
 642                        if ((ufshcd_readl(hba, reg) & mask) != val)
 643                                err = -ETIMEDOUT;
 644                        break;
 645                }
 646        }
 647
 648        return err;
 649}
 650
 651/**
 652 * ufshcd_get_intr_mask - Get the interrupt bit mask
 653 * @hba: Pointer to adapter instance
 654 *
 655 * Returns interrupt bit mask per version
 656 */
 657static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 658{
 659        if (hba->ufs_version == ufshci_version(1, 0))
 660                return INTERRUPT_MASK_ALL_VER_10;
 661        if (hba->ufs_version <= ufshci_version(2, 0))
 662                return INTERRUPT_MASK_ALL_VER_11;
 663
 664        return INTERRUPT_MASK_ALL_VER_21;
 665}
 666
 667/**
 668 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 669 * @hba: Pointer to adapter instance
 670 *
 671 * Returns UFSHCI version supported by the controller
 672 */
 673static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 674{
 675        u32 ufshci_ver;
 676
 677        if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
 678                ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
 679        else
 680                ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
 681
 682        /*
 683         * UFSHCI v1.x uses a different version scheme, in order
 684         * to allow the use of comparisons with the ufshci_version
 685         * function, we convert it to the same scheme as ufs 2.0+.
 686         */
 687        if (ufshci_ver & 0x00010000)
 688                return ufshci_version(1, ufshci_ver & 0x00000100);
 689
 690        return ufshci_ver;
 691}
 692
 693/**
 694 * ufshcd_is_device_present - Check if any device connected to
 695 *                            the host controller
 696 * @hba: pointer to adapter instance
 697 *
 698 * Returns true if device present, false if no device detected
 699 */
 700static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 701{
 702        return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
 703                                                DEVICE_PRESENT) ? true : false;
 704}
 705
 706/**
 707 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 708 * @lrbp: pointer to local command reference block
 709 *
 710 * This function is used to get the OCS field from UTRD
 711 * Returns the OCS field in the UTRD
 712 */
 713static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
 714{
 715        return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
 716}
 717
 718/**
 719 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
 720 * @hba: per adapter instance
 721 * @pos: position of the bit to be cleared
 722 */
 723static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 724{
 725        if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 726                ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 727        else
 728                ufshcd_writel(hba, ~(1 << pos),
 729                                REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 730}
 731
 732/**
 733 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
 734 * @hba: per adapter instance
 735 * @pos: position of the bit to be cleared
 736 */
 737static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
 738{
 739        if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 740                ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 741        else
 742                ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 743}
 744
 745/**
 746 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 747 * @reg: Register value of host controller status
 748 *
 749 * Returns integer, 0 on Success and positive value if failed
 750 */
 751static inline int ufshcd_get_lists_status(u32 reg)
 752{
 753        return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 754}
 755
 756/**
 757 * ufshcd_get_uic_cmd_result - Get the UIC command result
 758 * @hba: Pointer to adapter instance
 759 *
 760 * This function gets the result of UIC command completion
 761 * Returns 0 on success, non zero value on error
 762 */
 763static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 764{
 765        return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 766               MASK_UIC_COMMAND_RESULT;
 767}
 768
 769/**
 770 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 771 * @hba: Pointer to adapter instance
 772 *
 773 * This function gets UIC command argument3
 774 * Returns 0 on success, non zero value on error
 775 */
 776static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 777{
 778        return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 779}
 780
 781/**
 782 * ufshcd_get_req_rsp - returns the TR response transaction type
 783 * @ucd_rsp_ptr: pointer to response UPIU
 784 */
 785static inline int
 786ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 787{
 788        return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 789}
 790
 791/**
 792 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
 793 * @ucd_rsp_ptr: pointer to response UPIU
 794 *
 795 * This function gets the response status and scsi_status from response UPIU
 796 * Returns the response result code.
 797 */
 798static inline int
 799ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 800{
 801        return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 802}
 803
 804/*
 805 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
 806 *                              from response UPIU
 807 * @ucd_rsp_ptr: pointer to response UPIU
 808 *
 809 * Return the data segment length.
 810 */
 811static inline unsigned int
 812ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
 813{
 814        return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 815                MASK_RSP_UPIU_DATA_SEG_LEN;
 816}
 817
 818/**
 819 * ufshcd_is_exception_event - Check if the device raised an exception event
 820 * @ucd_rsp_ptr: pointer to response UPIU
 821 *
 822 * The function checks if the device raised an exception event indicated in
 823 * the Device Information field of response UPIU.
 824 *
 825 * Returns true if exception is raised, false otherwise.
 826 */
 827static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 828{
 829        return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 830                        MASK_RSP_EXCEPTION_EVENT ? true : false;
 831}
 832
 833/**
 834 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
 835 * @hba: per adapter instance
 836 */
 837static inline void
 838ufshcd_reset_intr_aggr(struct ufs_hba *hba)
 839{
 840        ufshcd_writel(hba, INT_AGGR_ENABLE |
 841                      INT_AGGR_COUNTER_AND_TIMER_RESET,
 842                      REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 843}
 844
 845/**
 846 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
 847 * @hba: per adapter instance
 848 * @cnt: Interrupt aggregation counter threshold
 849 * @tmout: Interrupt aggregation timeout value
 850 */
 851static inline void
 852ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
 853{
 854        ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 855                      INT_AGGR_COUNTER_THLD_VAL(cnt) |
 856                      INT_AGGR_TIMEOUT_VAL(tmout),
 857                      REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 858}
 859
 860/**
 861 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 862 * @hba: per adapter instance
 863 */
 864static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 865{
 866        ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 867}
 868
 869/**
 870 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 871 *                      When run-stop registers are set to 1, it indicates the
 872 *                      host controller that it can process the requests
 873 * @hba: per adapter instance
 874 */
 875static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 876{
 877        ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 878                      REG_UTP_TASK_REQ_LIST_RUN_STOP);
 879        ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 880                      REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 881}
 882
 883/**
 884 * ufshcd_hba_start - Start controller initialization sequence
 885 * @hba: per adapter instance
 886 */
 887static inline void ufshcd_hba_start(struct ufs_hba *hba)
 888{
 889        u32 val = CONTROLLER_ENABLE;
 890
 891        if (ufshcd_crypto_enable(hba))
 892                val |= CRYPTO_GENERAL_ENABLE;
 893
 894        ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
 895}
 896
 897/**
 898 * ufshcd_is_hba_active - Get controller state
 899 * @hba: per adapter instance
 900 *
 901 * Returns false if controller is active, true otherwise
 902 */
 903static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
 904{
 905        return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
 906                ? false : true;
 907}
 908
 909u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
 910{
 911        /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
 912        if (hba->ufs_version <= ufshci_version(1, 1))
 913                return UFS_UNIPRO_VER_1_41;
 914        else
 915                return UFS_UNIPRO_VER_1_6;
 916}
 917EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
 918
 919static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
 920{
 921        /*
 922         * If both host and device support UniPro ver1.6 or later, PA layer
 923         * parameters tuning happens during link startup itself.
 924         *
 925         * We can manually tune PA layer parameters if either host or device
 926         * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
 927         * logic simple, we will only do manual tuning if local unipro version
 928         * doesn't support ver1.6 or later.
 929         */
 930        if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
 931                return true;
 932        else
 933                return false;
 934}
 935
 936/**
 937 * ufshcd_set_clk_freq - set UFS controller clock frequencies
 938 * @hba: per adapter instance
 939 * @scale_up: If True, set max possible frequency othewise set low frequency
 940 *
 941 * Returns 0 if successful
 942 * Returns < 0 for any other errors
 943 */
 944static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
 945{
 946        int ret = 0;
 947        struct ufs_clk_info *clki;
 948        struct list_head *head = &hba->clk_list_head;
 949
 950        if (list_empty(head))
 951                goto out;
 952
 953        list_for_each_entry(clki, head, list) {
 954                if (!IS_ERR_OR_NULL(clki->clk)) {
 955                        if (scale_up && clki->max_freq) {
 956                                if (clki->curr_freq == clki->max_freq)
 957                                        continue;
 958
 959                                ret = clk_set_rate(clki->clk, clki->max_freq);
 960                                if (ret) {
 961                                        dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
 962                                                __func__, clki->name,
 963                                                clki->max_freq, ret);
 964                                        break;
 965                                }
 966                                trace_ufshcd_clk_scaling(dev_name(hba->dev),
 967                                                "scaled up", clki->name,
 968                                                clki->curr_freq,
 969                                                clki->max_freq);
 970
 971                                clki->curr_freq = clki->max_freq;
 972
 973                        } else if (!scale_up && clki->min_freq) {
 974                                if (clki->curr_freq == clki->min_freq)
 975                                        continue;
 976
 977                                ret = clk_set_rate(clki->clk, clki->min_freq);
 978                                if (ret) {
 979                                        dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
 980                                                __func__, clki->name,
 981                                                clki->min_freq, ret);
 982                                        break;
 983                                }
 984                                trace_ufshcd_clk_scaling(dev_name(hba->dev),
 985                                                "scaled down", clki->name,
 986                                                clki->curr_freq,
 987                                                clki->min_freq);
 988                                clki->curr_freq = clki->min_freq;
 989                        }
 990                }
 991                dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
 992                                clki->name, clk_get_rate(clki->clk));
 993        }
 994
 995out:
 996        return ret;
 997}
 998
 999/**
1000 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1001 * @hba: per adapter instance
1002 * @scale_up: True if scaling up and false if scaling down
1003 *
1004 * Returns 0 if successful
1005 * Returns < 0 for any other errors
1006 */
1007static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
1008{
1009        int ret = 0;
1010        ktime_t start = ktime_get();
1011
1012        ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1013        if (ret)
1014                goto out;
1015
1016        ret = ufshcd_set_clk_freq(hba, scale_up);
1017        if (ret)
1018                goto out;
1019
1020        ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1021        if (ret)
1022                ufshcd_set_clk_freq(hba, !scale_up);
1023
1024out:
1025        trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1026                        (scale_up ? "up" : "down"),
1027                        ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1028        return ret;
1029}
1030
1031/**
1032 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1033 * @hba: per adapter instance
1034 * @scale_up: True if scaling up and false if scaling down
1035 *
1036 * Returns true if scaling is required, false otherwise.
1037 */
1038static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1039                                               bool scale_up)
1040{
1041        struct ufs_clk_info *clki;
1042        struct list_head *head = &hba->clk_list_head;
1043
1044        if (list_empty(head))
1045                return false;
1046
1047        list_for_each_entry(clki, head, list) {
1048                if (!IS_ERR_OR_NULL(clki->clk)) {
1049                        if (scale_up && clki->max_freq) {
1050                                if (clki->curr_freq == clki->max_freq)
1051                                        continue;
1052                                return true;
1053                        } else if (!scale_up && clki->min_freq) {
1054                                if (clki->curr_freq == clki->min_freq)
1055                                        continue;
1056                                return true;
1057                        }
1058                }
1059        }
1060
1061        return false;
1062}
1063
1064static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1065                                        u64 wait_timeout_us)
1066{
1067        unsigned long flags;
1068        int ret = 0;
1069        u32 tm_doorbell;
1070        u32 tr_doorbell;
1071        bool timeout = false, do_last_check = false;
1072        ktime_t start;
1073
1074        ufshcd_hold(hba, false);
1075        spin_lock_irqsave(hba->host->host_lock, flags);
1076        /*
1077         * Wait for all the outstanding tasks/transfer requests.
1078         * Verify by checking the doorbell registers are clear.
1079         */
1080        start = ktime_get();
1081        do {
1082                if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1083                        ret = -EBUSY;
1084                        goto out;
1085                }
1086
1087                tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1088                tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1089                if (!tm_doorbell && !tr_doorbell) {
1090                        timeout = false;
1091                        break;
1092                } else if (do_last_check) {
1093                        break;
1094                }
1095
1096                spin_unlock_irqrestore(hba->host->host_lock, flags);
1097                schedule();
1098                if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1099                    wait_timeout_us) {
1100                        timeout = true;
1101                        /*
1102                         * We might have scheduled out for long time so make
1103                         * sure to check if doorbells are cleared by this time
1104                         * or not.
1105                         */
1106                        do_last_check = true;
1107                }
1108                spin_lock_irqsave(hba->host->host_lock, flags);
1109        } while (tm_doorbell || tr_doorbell);
1110
1111        if (timeout) {
1112                dev_err(hba->dev,
1113                        "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1114                        __func__, tm_doorbell, tr_doorbell);
1115                ret = -EBUSY;
1116        }
1117out:
1118        spin_unlock_irqrestore(hba->host->host_lock, flags);
1119        ufshcd_release(hba);
1120        return ret;
1121}
1122
1123/**
1124 * ufshcd_scale_gear - scale up/down UFS gear
1125 * @hba: per adapter instance
1126 * @scale_up: True for scaling up gear and false for scaling down
1127 *
1128 * Returns 0 for success,
1129 * Returns -EBUSY if scaling can't happen at this time
1130 * Returns non-zero for any other errors
1131 */
1132static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1133{
1134        int ret = 0;
1135        struct ufs_pa_layer_attr new_pwr_info;
1136
1137        if (scale_up) {
1138                memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1139                       sizeof(struct ufs_pa_layer_attr));
1140        } else {
1141                memcpy(&new_pwr_info, &hba->pwr_info,
1142                       sizeof(struct ufs_pa_layer_attr));
1143
1144                if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1145                    hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1146                        /* save the current power mode */
1147                        memcpy(&hba->clk_scaling.saved_pwr_info.info,
1148                                &hba->pwr_info,
1149                                sizeof(struct ufs_pa_layer_attr));
1150
1151                        /* scale down gear */
1152                        new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1153                        new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1154                }
1155        }
1156
1157        /* check if the power mode needs to be changed or not? */
1158        ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1159        if (ret)
1160                dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1161                        __func__, ret,
1162                        hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1163                        new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1164
1165        return ret;
1166}
1167
1168static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1169{
1170        #define DOORBELL_CLR_TOUT_US            (1000 * 1000) /* 1 sec */
1171        int ret = 0;
1172        /*
1173         * make sure that there are no outstanding requests when
1174         * clock scaling is in progress
1175         */
1176        ufshcd_scsi_block_requests(hba);
1177        down_write(&hba->clk_scaling_lock);
1178
1179        if (!hba->clk_scaling.is_allowed ||
1180            ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1181                ret = -EBUSY;
1182                up_write(&hba->clk_scaling_lock);
1183                ufshcd_scsi_unblock_requests(hba);
1184                goto out;
1185        }
1186
1187        /* let's not get into low power until clock scaling is completed */
1188        ufshcd_hold(hba, false);
1189
1190out:
1191        return ret;
1192}
1193
1194static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, bool writelock)
1195{
1196        if (writelock)
1197                up_write(&hba->clk_scaling_lock);
1198        else
1199                up_read(&hba->clk_scaling_lock);
1200        ufshcd_scsi_unblock_requests(hba);
1201        ufshcd_release(hba);
1202}
1203
1204/**
1205 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1206 * @hba: per adapter instance
1207 * @scale_up: True for scaling up and false for scalin down
1208 *
1209 * Returns 0 for success,
1210 * Returns -EBUSY if scaling can't happen at this time
1211 * Returns non-zero for any other errors
1212 */
1213static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1214{
1215        int ret = 0;
1216        bool is_writelock = true;
1217
1218        ret = ufshcd_clock_scaling_prepare(hba);
1219        if (ret)
1220                return ret;
1221
1222        /* scale down the gear before scaling down clocks */
1223        if (!scale_up) {
1224                ret = ufshcd_scale_gear(hba, false);
1225                if (ret)
1226                        goto out_unprepare;
1227        }
1228
1229        ret = ufshcd_scale_clks(hba, scale_up);
1230        if (ret) {
1231                if (!scale_up)
1232                        ufshcd_scale_gear(hba, true);
1233                goto out_unprepare;
1234        }
1235
1236        /* scale up the gear after scaling up clocks */
1237        if (scale_up) {
1238                ret = ufshcd_scale_gear(hba, true);
1239                if (ret) {
1240                        ufshcd_scale_clks(hba, false);
1241                        goto out_unprepare;
1242                }
1243        }
1244
1245        /* Enable Write Booster if we have scaled up else disable it */
1246        downgrade_write(&hba->clk_scaling_lock);
1247        is_writelock = false;
1248        ufshcd_wb_toggle(hba, scale_up);
1249
1250out_unprepare:
1251        ufshcd_clock_scaling_unprepare(hba, is_writelock);
1252        return ret;
1253}
1254
1255static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1256{
1257        struct ufs_hba *hba = container_of(work, struct ufs_hba,
1258                                           clk_scaling.suspend_work);
1259        unsigned long irq_flags;
1260
1261        spin_lock_irqsave(hba->host->host_lock, irq_flags);
1262        if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1263                spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1264                return;
1265        }
1266        hba->clk_scaling.is_suspended = true;
1267        spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1268
1269        __ufshcd_suspend_clkscaling(hba);
1270}
1271
1272static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1273{
1274        struct ufs_hba *hba = container_of(work, struct ufs_hba,
1275                                           clk_scaling.resume_work);
1276        unsigned long irq_flags;
1277
1278        spin_lock_irqsave(hba->host->host_lock, irq_flags);
1279        if (!hba->clk_scaling.is_suspended) {
1280                spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1281                return;
1282        }
1283        hba->clk_scaling.is_suspended = false;
1284        spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1285
1286        devfreq_resume_device(hba->devfreq);
1287}
1288
1289static int ufshcd_devfreq_target(struct device *dev,
1290                                unsigned long *freq, u32 flags)
1291{
1292        int ret = 0;
1293        struct ufs_hba *hba = dev_get_drvdata(dev);
1294        ktime_t start;
1295        bool scale_up, sched_clk_scaling_suspend_work = false;
1296        struct list_head *clk_list = &hba->clk_list_head;
1297        struct ufs_clk_info *clki;
1298        unsigned long irq_flags;
1299
1300        if (!ufshcd_is_clkscaling_supported(hba))
1301                return -EINVAL;
1302
1303        clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1304        /* Override with the closest supported frequency */
1305        *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1306        spin_lock_irqsave(hba->host->host_lock, irq_flags);
1307        if (ufshcd_eh_in_progress(hba)) {
1308                spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1309                return 0;
1310        }
1311
1312        if (!hba->clk_scaling.active_reqs)
1313                sched_clk_scaling_suspend_work = true;
1314
1315        if (list_empty(clk_list)) {
1316                spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1317                goto out;
1318        }
1319
1320        /* Decide based on the rounded-off frequency and update */
1321        scale_up = (*freq == clki->max_freq) ? true : false;
1322        if (!scale_up)
1323                *freq = clki->min_freq;
1324        /* Update the frequency */
1325        if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1326                spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1327                ret = 0;
1328                goto out; /* no state change required */
1329        }
1330        spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1331
1332        start = ktime_get();
1333        ret = ufshcd_devfreq_scale(hba, scale_up);
1334
1335        trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1336                (scale_up ? "up" : "down"),
1337                ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1338
1339out:
1340        if (sched_clk_scaling_suspend_work)
1341                queue_work(hba->clk_scaling.workq,
1342                           &hba->clk_scaling.suspend_work);
1343
1344        return ret;
1345}
1346
1347static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1348{
1349        int *busy = priv;
1350
1351        WARN_ON_ONCE(reserved);
1352        (*busy)++;
1353        return false;
1354}
1355
1356/* Whether or not any tag is in use by a request that is in progress. */
1357static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1358{
1359        struct request_queue *q = hba->cmd_queue;
1360        int busy = 0;
1361
1362        blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1363        return busy;
1364}
1365
1366static int ufshcd_devfreq_get_dev_status(struct device *dev,
1367                struct devfreq_dev_status *stat)
1368{
1369        struct ufs_hba *hba = dev_get_drvdata(dev);
1370        struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1371        unsigned long flags;
1372        struct list_head *clk_list = &hba->clk_list_head;
1373        struct ufs_clk_info *clki;
1374        ktime_t curr_t;
1375
1376        if (!ufshcd_is_clkscaling_supported(hba))
1377                return -EINVAL;
1378
1379        memset(stat, 0, sizeof(*stat));
1380
1381        spin_lock_irqsave(hba->host->host_lock, flags);
1382        curr_t = ktime_get();
1383        if (!scaling->window_start_t)
1384                goto start_window;
1385
1386        clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1387        /*
1388         * If current frequency is 0, then the ondemand governor considers
1389         * there's no initial frequency set. And it always requests to set
1390         * to max. frequency.
1391         */
1392        stat->current_frequency = clki->curr_freq;
1393        if (scaling->is_busy_started)
1394                scaling->tot_busy_t += ktime_us_delta(curr_t,
1395                                scaling->busy_start_t);
1396
1397        stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1398        stat->busy_time = scaling->tot_busy_t;
1399start_window:
1400        scaling->window_start_t = curr_t;
1401        scaling->tot_busy_t = 0;
1402
1403        if (hba->outstanding_reqs) {
1404                scaling->busy_start_t = curr_t;
1405                scaling->is_busy_started = true;
1406        } else {
1407                scaling->busy_start_t = 0;
1408                scaling->is_busy_started = false;
1409        }
1410        spin_unlock_irqrestore(hba->host->host_lock, flags);
1411        return 0;
1412}
1413
1414static int ufshcd_devfreq_init(struct ufs_hba *hba)
1415{
1416        struct list_head *clk_list = &hba->clk_list_head;
1417        struct ufs_clk_info *clki;
1418        struct devfreq *devfreq;
1419        int ret;
1420
1421        /* Skip devfreq if we don't have any clocks in the list */
1422        if (list_empty(clk_list))
1423                return 0;
1424
1425        clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1426        dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1427        dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1428
1429        ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1430                                         &hba->vps->ondemand_data);
1431        devfreq = devfreq_add_device(hba->dev,
1432                        &hba->vps->devfreq_profile,
1433                        DEVFREQ_GOV_SIMPLE_ONDEMAND,
1434                        &hba->vps->ondemand_data);
1435        if (IS_ERR(devfreq)) {
1436                ret = PTR_ERR(devfreq);
1437                dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1438
1439                dev_pm_opp_remove(hba->dev, clki->min_freq);
1440                dev_pm_opp_remove(hba->dev, clki->max_freq);
1441                return ret;
1442        }
1443
1444        hba->devfreq = devfreq;
1445
1446        return 0;
1447}
1448
1449static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1450{
1451        struct list_head *clk_list = &hba->clk_list_head;
1452        struct ufs_clk_info *clki;
1453
1454        if (!hba->devfreq)
1455                return;
1456
1457        devfreq_remove_device(hba->devfreq);
1458        hba->devfreq = NULL;
1459
1460        clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1461        dev_pm_opp_remove(hba->dev, clki->min_freq);
1462        dev_pm_opp_remove(hba->dev, clki->max_freq);
1463}
1464
1465static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1466{
1467        unsigned long flags;
1468
1469        devfreq_suspend_device(hba->devfreq);
1470        spin_lock_irqsave(hba->host->host_lock, flags);
1471        hba->clk_scaling.window_start_t = 0;
1472        spin_unlock_irqrestore(hba->host->host_lock, flags);
1473}
1474
1475static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1476{
1477        unsigned long flags;
1478        bool suspend = false;
1479
1480        cancel_work_sync(&hba->clk_scaling.suspend_work);
1481        cancel_work_sync(&hba->clk_scaling.resume_work);
1482
1483        spin_lock_irqsave(hba->host->host_lock, flags);
1484        if (!hba->clk_scaling.is_suspended) {
1485                suspend = true;
1486                hba->clk_scaling.is_suspended = true;
1487        }
1488        spin_unlock_irqrestore(hba->host->host_lock, flags);
1489
1490        if (suspend)
1491                __ufshcd_suspend_clkscaling(hba);
1492}
1493
1494static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1495{
1496        unsigned long flags;
1497        bool resume = false;
1498
1499        spin_lock_irqsave(hba->host->host_lock, flags);
1500        if (hba->clk_scaling.is_suspended) {
1501                resume = true;
1502                hba->clk_scaling.is_suspended = false;
1503        }
1504        spin_unlock_irqrestore(hba->host->host_lock, flags);
1505
1506        if (resume)
1507                devfreq_resume_device(hba->devfreq);
1508}
1509
1510static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1511                struct device_attribute *attr, char *buf)
1512{
1513        struct ufs_hba *hba = dev_get_drvdata(dev);
1514
1515        return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1516}
1517
1518static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1519                struct device_attribute *attr, const char *buf, size_t count)
1520{
1521        struct ufs_hba *hba = dev_get_drvdata(dev);
1522        u32 value;
1523        int err = 0;
1524
1525        if (kstrtou32(buf, 0, &value))
1526                return -EINVAL;
1527
1528        down(&hba->host_sem);
1529        if (!ufshcd_is_user_access_allowed(hba)) {
1530                err = -EBUSY;
1531                goto out;
1532        }
1533
1534        value = !!value;
1535        if (value == hba->clk_scaling.is_enabled)
1536                goto out;
1537
1538        ufshcd_rpm_get_sync(hba);
1539        ufshcd_hold(hba, false);
1540
1541        hba->clk_scaling.is_enabled = value;
1542
1543        if (value) {
1544                ufshcd_resume_clkscaling(hba);
1545        } else {
1546                ufshcd_suspend_clkscaling(hba);
1547                err = ufshcd_devfreq_scale(hba, true);
1548                if (err)
1549                        dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1550                                        __func__, err);
1551        }
1552
1553        ufshcd_release(hba);
1554        ufshcd_rpm_put_sync(hba);
1555out:
1556        up(&hba->host_sem);
1557        return err ? err : count;
1558}
1559
1560static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1561{
1562        hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1563        hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1564        sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1565        hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1566        hba->clk_scaling.enable_attr.attr.mode = 0644;
1567        if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1568                dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1569}
1570
1571static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1572{
1573        if (hba->clk_scaling.enable_attr.attr.name)
1574                device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1575}
1576
1577static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1578{
1579        char wq_name[sizeof("ufs_clkscaling_00")];
1580
1581        if (!ufshcd_is_clkscaling_supported(hba))
1582                return;
1583
1584        if (!hba->clk_scaling.min_gear)
1585                hba->clk_scaling.min_gear = UFS_HS_G1;
1586
1587        INIT_WORK(&hba->clk_scaling.suspend_work,
1588                  ufshcd_clk_scaling_suspend_work);
1589        INIT_WORK(&hba->clk_scaling.resume_work,
1590                  ufshcd_clk_scaling_resume_work);
1591
1592        snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1593                 hba->host->host_no);
1594        hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1595
1596        hba->clk_scaling.is_initialized = true;
1597}
1598
1599static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1600{
1601        if (!hba->clk_scaling.is_initialized)
1602                return;
1603
1604        ufshcd_remove_clk_scaling_sysfs(hba);
1605        destroy_workqueue(hba->clk_scaling.workq);
1606        ufshcd_devfreq_remove(hba);
1607        hba->clk_scaling.is_initialized = false;
1608}
1609
1610static void ufshcd_ungate_work(struct work_struct *work)
1611{
1612        int ret;
1613        unsigned long flags;
1614        struct ufs_hba *hba = container_of(work, struct ufs_hba,
1615                        clk_gating.ungate_work);
1616
1617        cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1618
1619        spin_lock_irqsave(hba->host->host_lock, flags);
1620        if (hba->clk_gating.state == CLKS_ON) {
1621                spin_unlock_irqrestore(hba->host->host_lock, flags);
1622                goto unblock_reqs;
1623        }
1624
1625        spin_unlock_irqrestore(hba->host->host_lock, flags);
1626        ufshcd_hba_vreg_set_hpm(hba);
1627        ufshcd_setup_clocks(hba, true);
1628
1629        ufshcd_enable_irq(hba);
1630
1631        /* Exit from hibern8 */
1632        if (ufshcd_can_hibern8_during_gating(hba)) {
1633                /* Prevent gating in this path */
1634                hba->clk_gating.is_suspended = true;
1635                if (ufshcd_is_link_hibern8(hba)) {
1636                        ret = ufshcd_uic_hibern8_exit(hba);
1637                        if (ret)
1638                                dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1639                                        __func__, ret);
1640                        else
1641                                ufshcd_set_link_active(hba);
1642                }
1643                hba->clk_gating.is_suspended = false;
1644        }
1645unblock_reqs:
1646        ufshcd_scsi_unblock_requests(hba);
1647}
1648
1649/**
1650 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1651 * Also, exit from hibern8 mode and set the link as active.
1652 * @hba: per adapter instance
1653 * @async: This indicates whether caller should ungate clocks asynchronously.
1654 */
1655int ufshcd_hold(struct ufs_hba *hba, bool async)
1656{
1657        int rc = 0;
1658        bool flush_result;
1659        unsigned long flags;
1660
1661        if (!ufshcd_is_clkgating_allowed(hba))
1662                goto out;
1663        spin_lock_irqsave(hba->host->host_lock, flags);
1664        hba->clk_gating.active_reqs++;
1665
1666start:
1667        switch (hba->clk_gating.state) {
1668        case CLKS_ON:
1669                /*
1670                 * Wait for the ungate work to complete if in progress.
1671                 * Though the clocks may be in ON state, the link could
1672                 * still be in hibner8 state if hibern8 is allowed
1673                 * during clock gating.
1674                 * Make sure we exit hibern8 state also in addition to
1675                 * clocks being ON.
1676                 */
1677                if (ufshcd_can_hibern8_during_gating(hba) &&
1678                    ufshcd_is_link_hibern8(hba)) {
1679                        if (async) {
1680                                rc = -EAGAIN;
1681                                hba->clk_gating.active_reqs--;
1682                                break;
1683                        }
1684                        spin_unlock_irqrestore(hba->host->host_lock, flags);
1685                        flush_result = flush_work(&hba->clk_gating.ungate_work);
1686                        if (hba->clk_gating.is_suspended && !flush_result)
1687                                goto out;
1688                        spin_lock_irqsave(hba->host->host_lock, flags);
1689                        goto start;
1690                }
1691                break;
1692        case REQ_CLKS_OFF:
1693                if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1694                        hba->clk_gating.state = CLKS_ON;
1695                        trace_ufshcd_clk_gating(dev_name(hba->dev),
1696                                                hba->clk_gating.state);
1697                        break;
1698                }
1699                /*
1700                 * If we are here, it means gating work is either done or
1701                 * currently running. Hence, fall through to cancel gating
1702                 * work and to enable clocks.
1703                 */
1704                fallthrough;
1705        case CLKS_OFF:
1706                hba->clk_gating.state = REQ_CLKS_ON;
1707                trace_ufshcd_clk_gating(dev_name(hba->dev),
1708                                        hba->clk_gating.state);
1709                if (queue_work(hba->clk_gating.clk_gating_workq,
1710                               &hba->clk_gating.ungate_work))
1711                        ufshcd_scsi_block_requests(hba);
1712                /*
1713                 * fall through to check if we should wait for this
1714                 * work to be done or not.
1715                 */
1716                fallthrough;
1717        case REQ_CLKS_ON:
1718                if (async) {
1719                        rc = -EAGAIN;
1720                        hba->clk_gating.active_reqs--;
1721                        break;
1722                }
1723
1724                spin_unlock_irqrestore(hba->host->host_lock, flags);
1725                flush_work(&hba->clk_gating.ungate_work);
1726                /* Make sure state is CLKS_ON before returning */
1727                spin_lock_irqsave(hba->host->host_lock, flags);
1728                goto start;
1729        default:
1730                dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1731                                __func__, hba->clk_gating.state);
1732                break;
1733        }
1734        spin_unlock_irqrestore(hba->host->host_lock, flags);
1735out:
1736        return rc;
1737}
1738EXPORT_SYMBOL_GPL(ufshcd_hold);
1739
1740static void ufshcd_gate_work(struct work_struct *work)
1741{
1742        struct ufs_hba *hba = container_of(work, struct ufs_hba,
1743                        clk_gating.gate_work.work);
1744        unsigned long flags;
1745        int ret;
1746
1747        spin_lock_irqsave(hba->host->host_lock, flags);
1748        /*
1749         * In case you are here to cancel this work the gating state
1750         * would be marked as REQ_CLKS_ON. In this case save time by
1751         * skipping the gating work and exit after changing the clock
1752         * state to CLKS_ON.
1753         */
1754        if (hba->clk_gating.is_suspended ||
1755                (hba->clk_gating.state != REQ_CLKS_OFF)) {
1756                hba->clk_gating.state = CLKS_ON;
1757                trace_ufshcd_clk_gating(dev_name(hba->dev),
1758                                        hba->clk_gating.state);
1759                goto rel_lock;
1760        }
1761
1762        if (hba->clk_gating.active_reqs
1763                || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1764                || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1765                || hba->active_uic_cmd || hba->uic_async_done)
1766                goto rel_lock;
1767
1768        spin_unlock_irqrestore(hba->host->host_lock, flags);
1769
1770        /* put the link into hibern8 mode before turning off clocks */
1771        if (ufshcd_can_hibern8_during_gating(hba)) {
1772                ret = ufshcd_uic_hibern8_enter(hba);
1773                if (ret) {
1774                        hba->clk_gating.state = CLKS_ON;
1775                        dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1776                                        __func__, ret);
1777                        trace_ufshcd_clk_gating(dev_name(hba->dev),
1778                                                hba->clk_gating.state);
1779                        goto out;
1780                }
1781                ufshcd_set_link_hibern8(hba);
1782        }
1783
1784        ufshcd_disable_irq(hba);
1785
1786        ufshcd_setup_clocks(hba, false);
1787
1788        /* Put the host controller in low power mode if possible */
1789        ufshcd_hba_vreg_set_lpm(hba);
1790        /*
1791         * In case you are here to cancel this work the gating state
1792         * would be marked as REQ_CLKS_ON. In this case keep the state
1793         * as REQ_CLKS_ON which would anyway imply that clocks are off
1794         * and a request to turn them on is pending. By doing this way,
1795         * we keep the state machine in tact and this would ultimately
1796         * prevent from doing cancel work multiple times when there are
1797         * new requests arriving before the current cancel work is done.
1798         */
1799        spin_lock_irqsave(hba->host->host_lock, flags);
1800        if (hba->clk_gating.state == REQ_CLKS_OFF) {
1801                hba->clk_gating.state = CLKS_OFF;
1802                trace_ufshcd_clk_gating(dev_name(hba->dev),
1803                                        hba->clk_gating.state);
1804        }
1805rel_lock:
1806        spin_unlock_irqrestore(hba->host->host_lock, flags);
1807out:
1808        return;
1809}
1810
1811/* host lock must be held before calling this variant */
1812static void __ufshcd_release(struct ufs_hba *hba)
1813{
1814        if (!ufshcd_is_clkgating_allowed(hba))
1815                return;
1816
1817        hba->clk_gating.active_reqs--;
1818
1819        if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1820            hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1821            hba->outstanding_tasks ||
1822            hba->active_uic_cmd || hba->uic_async_done ||
1823            hba->clk_gating.state == CLKS_OFF)
1824                return;
1825
1826        hba->clk_gating.state = REQ_CLKS_OFF;
1827        trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1828        queue_delayed_work(hba->clk_gating.clk_gating_workq,
1829                           &hba->clk_gating.gate_work,
1830                           msecs_to_jiffies(hba->clk_gating.delay_ms));
1831}
1832
1833void ufshcd_release(struct ufs_hba *hba)
1834{
1835        unsigned long flags;
1836
1837        spin_lock_irqsave(hba->host->host_lock, flags);
1838        __ufshcd_release(hba);
1839        spin_unlock_irqrestore(hba->host->host_lock, flags);
1840}
1841EXPORT_SYMBOL_GPL(ufshcd_release);
1842
1843static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1844                struct device_attribute *attr, char *buf)
1845{
1846        struct ufs_hba *hba = dev_get_drvdata(dev);
1847
1848        return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
1849}
1850
1851static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1852                struct device_attribute *attr, const char *buf, size_t count)
1853{
1854        struct ufs_hba *hba = dev_get_drvdata(dev);
1855        unsigned long flags, value;
1856
1857        if (kstrtoul(buf, 0, &value))
1858                return -EINVAL;
1859
1860        spin_lock_irqsave(hba->host->host_lock, flags);
1861        hba->clk_gating.delay_ms = value;
1862        spin_unlock_irqrestore(hba->host->host_lock, flags);
1863        return count;
1864}
1865
1866static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1867                struct device_attribute *attr, char *buf)
1868{
1869        struct ufs_hba *hba = dev_get_drvdata(dev);
1870
1871        return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
1872}
1873
1874static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1875                struct device_attribute *attr, const char *buf, size_t count)
1876{
1877        struct ufs_hba *hba = dev_get_drvdata(dev);
1878        unsigned long flags;
1879        u32 value;
1880
1881        if (kstrtou32(buf, 0, &value))
1882                return -EINVAL;
1883
1884        value = !!value;
1885
1886        spin_lock_irqsave(hba->host->host_lock, flags);
1887        if (value == hba->clk_gating.is_enabled)
1888                goto out;
1889
1890        if (value)
1891                __ufshcd_release(hba);
1892        else
1893                hba->clk_gating.active_reqs++;
1894
1895        hba->clk_gating.is_enabled = value;
1896out:
1897        spin_unlock_irqrestore(hba->host->host_lock, flags);
1898        return count;
1899}
1900
1901static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
1902{
1903        hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1904        hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1905        sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1906        hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1907        hba->clk_gating.delay_attr.attr.mode = 0644;
1908        if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1909                dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1910
1911        hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1912        hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1913        sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1914        hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1915        hba->clk_gating.enable_attr.attr.mode = 0644;
1916        if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1917                dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1918}
1919
1920static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
1921{
1922        if (hba->clk_gating.delay_attr.attr.name)
1923                device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1924        if (hba->clk_gating.enable_attr.attr.name)
1925                device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1926}
1927
1928static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1929{
1930        char wq_name[sizeof("ufs_clk_gating_00")];
1931
1932        if (!ufshcd_is_clkgating_allowed(hba))
1933                return;
1934
1935        hba->clk_gating.state = CLKS_ON;
1936
1937        hba->clk_gating.delay_ms = 150;
1938        INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1939        INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1940
1941        snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1942                 hba->host->host_no);
1943        hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1944                                        WQ_MEM_RECLAIM | WQ_HIGHPRI);
1945
1946        ufshcd_init_clk_gating_sysfs(hba);
1947
1948        hba->clk_gating.is_enabled = true;
1949        hba->clk_gating.is_initialized = true;
1950}
1951
1952static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1953{
1954        if (!hba->clk_gating.is_initialized)
1955                return;
1956        ufshcd_remove_clk_gating_sysfs(hba);
1957        cancel_work_sync(&hba->clk_gating.ungate_work);
1958        cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1959        destroy_workqueue(hba->clk_gating.clk_gating_workq);
1960        hba->clk_gating.is_initialized = false;
1961}
1962
1963/* Must be called with host lock acquired */
1964static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1965{
1966        bool queue_resume_work = false;
1967        ktime_t curr_t = ktime_get();
1968        unsigned long flags;
1969
1970        if (!ufshcd_is_clkscaling_supported(hba))
1971                return;
1972
1973        spin_lock_irqsave(hba->host->host_lock, flags);
1974        if (!hba->clk_scaling.active_reqs++)
1975                queue_resume_work = true;
1976
1977        if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
1978                spin_unlock_irqrestore(hba->host->host_lock, flags);
1979                return;
1980        }
1981
1982        if (queue_resume_work)
1983                queue_work(hba->clk_scaling.workq,
1984                           &hba->clk_scaling.resume_work);
1985
1986        if (!hba->clk_scaling.window_start_t) {
1987                hba->clk_scaling.window_start_t = curr_t;
1988                hba->clk_scaling.tot_busy_t = 0;
1989                hba->clk_scaling.is_busy_started = false;
1990        }
1991
1992        if (!hba->clk_scaling.is_busy_started) {
1993                hba->clk_scaling.busy_start_t = curr_t;
1994                hba->clk_scaling.is_busy_started = true;
1995        }
1996        spin_unlock_irqrestore(hba->host->host_lock, flags);
1997}
1998
1999static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2000{
2001        struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2002        unsigned long flags;
2003
2004        if (!ufshcd_is_clkscaling_supported(hba))
2005                return;
2006
2007        spin_lock_irqsave(hba->host->host_lock, flags);
2008        hba->clk_scaling.active_reqs--;
2009        if (!hba->outstanding_reqs && scaling->is_busy_started) {
2010                scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2011                                        scaling->busy_start_t));
2012                scaling->busy_start_t = 0;
2013                scaling->is_busy_started = false;
2014        }
2015        spin_unlock_irqrestore(hba->host->host_lock, flags);
2016}
2017
2018static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2019{
2020        if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2021                return READ;
2022        else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2023                return WRITE;
2024        else
2025                return -EINVAL;
2026}
2027
2028static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2029                                                struct ufshcd_lrb *lrbp)
2030{
2031        struct ufs_hba_monitor *m = &hba->monitor;
2032
2033        return (m->enabled && lrbp && lrbp->cmd &&
2034                (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2035                ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2036}
2037
2038static void ufshcd_start_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2039{
2040        int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2041        unsigned long flags;
2042
2043        spin_lock_irqsave(hba->host->host_lock, flags);
2044        if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2045                hba->monitor.busy_start_ts[dir] = ktime_get();
2046        spin_unlock_irqrestore(hba->host->host_lock, flags);
2047}
2048
2049static void ufshcd_update_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2050{
2051        int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2052        unsigned long flags;
2053
2054        spin_lock_irqsave(hba->host->host_lock, flags);
2055        if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2056                struct request *req = scsi_cmd_to_rq(lrbp->cmd);
2057                struct ufs_hba_monitor *m = &hba->monitor;
2058                ktime_t now, inc, lat;
2059
2060                now = lrbp->compl_time_stamp;
2061                inc = ktime_sub(now, m->busy_start_ts[dir]);
2062                m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2063                m->nr_sec_rw[dir] += blk_rq_sectors(req);
2064
2065                /* Update latencies */
2066                m->nr_req[dir]++;
2067                lat = ktime_sub(now, lrbp->issue_time_stamp);
2068                m->lat_sum[dir] += lat;
2069                if (m->lat_max[dir] < lat || !m->lat_max[dir])
2070                        m->lat_max[dir] = lat;
2071                if (m->lat_min[dir] > lat || !m->lat_min[dir])
2072                        m->lat_min[dir] = lat;
2073
2074                m->nr_queued[dir]--;
2075                /* Push forward the busy start of monitor */
2076                m->busy_start_ts[dir] = now;
2077        }
2078        spin_unlock_irqrestore(hba->host->host_lock, flags);
2079}
2080
2081/**
2082 * ufshcd_send_command - Send SCSI or device management commands
2083 * @hba: per adapter instance
2084 * @task_tag: Task tag of the command
2085 */
2086static inline
2087void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
2088{
2089        struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2090        unsigned long flags;
2091
2092        lrbp->issue_time_stamp = ktime_get();
2093        lrbp->compl_time_stamp = ktime_set(0, 0);
2094        ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
2095        ufshcd_clk_scaling_start_busy(hba);
2096        if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2097                ufshcd_start_monitor(hba, lrbp);
2098
2099        spin_lock_irqsave(&hba->outstanding_lock, flags);
2100        if (hba->vops && hba->vops->setup_xfer_req)
2101                hba->vops->setup_xfer_req(hba, task_tag, !!lrbp->cmd);
2102        __set_bit(task_tag, &hba->outstanding_reqs);
2103        ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
2104        spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2105
2106        /* Make sure that doorbell is committed immediately */
2107        wmb();
2108}
2109
2110/**
2111 * ufshcd_copy_sense_data - Copy sense data in case of check condition
2112 * @lrbp: pointer to local reference block
2113 */
2114static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2115{
2116        int len;
2117        if (lrbp->sense_buffer &&
2118            ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
2119                int len_to_copy;
2120
2121                len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2122                len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2123
2124                memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2125                       len_to_copy);
2126        }
2127}
2128
2129/**
2130 * ufshcd_copy_query_response() - Copy the Query Response and the data
2131 * descriptor
2132 * @hba: per adapter instance
2133 * @lrbp: pointer to local reference block
2134 */
2135static
2136int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2137{
2138        struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2139
2140        memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2141
2142        /* Get the descriptor */
2143        if (hba->dev_cmd.query.descriptor &&
2144            lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2145                u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2146                                GENERAL_UPIU_REQUEST_SIZE;
2147                u16 resp_len;
2148                u16 buf_len;
2149
2150                /* data segment length */
2151                resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
2152                                                MASK_QUERY_DATA_SEG_LEN;
2153                buf_len = be16_to_cpu(
2154                                hba->dev_cmd.query.request.upiu_req.length);
2155                if (likely(buf_len >= resp_len)) {
2156                        memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2157                } else {
2158                        dev_warn(hba->dev,
2159                                 "%s: rsp size %d is bigger than buffer size %d",
2160                                 __func__, resp_len, buf_len);
2161                        return -EINVAL;
2162                }
2163        }
2164
2165        return 0;
2166}
2167
2168/**
2169 * ufshcd_hba_capabilities - Read controller capabilities
2170 * @hba: per adapter instance
2171 *
2172 * Return: 0 on success, negative on error.
2173 */
2174static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2175{
2176        int err;
2177
2178        hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2179
2180        /* nutrs and nutmrs are 0 based values */
2181        hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2182        hba->nutmrs =
2183        ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2184
2185        /* Read crypto capabilities */
2186        err = ufshcd_hba_init_crypto_capabilities(hba);
2187        if (err)
2188                dev_err(hba->dev, "crypto setup failed\n");
2189
2190        return err;
2191}
2192
2193/**
2194 * ufshcd_ready_for_uic_cmd - Check if controller is ready
2195 *                            to accept UIC commands
2196 * @hba: per adapter instance
2197 * Return true on success, else false
2198 */
2199static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2200{
2201        if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
2202                return true;
2203        else
2204                return false;
2205}
2206
2207/**
2208 * ufshcd_get_upmcrs - Get the power mode change request status
2209 * @hba: Pointer to adapter instance
2210 *
2211 * This function gets the UPMCRS field of HCS register
2212 * Returns value of UPMCRS field
2213 */
2214static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2215{
2216        return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2217}
2218
2219/**
2220 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
2221 * @hba: per adapter instance
2222 * @uic_cmd: UIC command
2223 */
2224static inline void
2225ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2226{
2227        lockdep_assert_held(&hba->uic_cmd_mutex);
2228
2229        WARN_ON(hba->active_uic_cmd);
2230
2231        hba->active_uic_cmd = uic_cmd;
2232
2233        /* Write Args */
2234        ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2235        ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2236        ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2237
2238        ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
2239
2240        /* Write UIC Cmd */
2241        ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2242                      REG_UIC_COMMAND);
2243}
2244
2245/**
2246 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
2247 * @hba: per adapter instance
2248 * @uic_cmd: UIC command
2249 *
2250 * Returns 0 only if success.
2251 */
2252static int
2253ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2254{
2255        int ret;
2256        unsigned long flags;
2257
2258        lockdep_assert_held(&hba->uic_cmd_mutex);
2259
2260        if (wait_for_completion_timeout(&uic_cmd->done,
2261                                        msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2262                ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2263        } else {
2264                ret = -ETIMEDOUT;
2265                dev_err(hba->dev,
2266                        "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2267                        uic_cmd->command, uic_cmd->argument3);
2268
2269                if (!uic_cmd->cmd_active) {
2270                        dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2271                                __func__);
2272                        ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2273                }
2274        }
2275
2276        spin_lock_irqsave(hba->host->host_lock, flags);
2277        hba->active_uic_cmd = NULL;
2278        spin_unlock_irqrestore(hba->host->host_lock, flags);
2279
2280        return ret;
2281}
2282
2283/**
2284 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2285 * @hba: per adapter instance
2286 * @uic_cmd: UIC command
2287 * @completion: initialize the completion only if this is set to true
2288 *
2289 * Returns 0 only if success.
2290 */
2291static int
2292__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2293                      bool completion)
2294{
2295        lockdep_assert_held(&hba->uic_cmd_mutex);
2296        lockdep_assert_held(hba->host->host_lock);
2297
2298        if (!ufshcd_ready_for_uic_cmd(hba)) {
2299                dev_err(hba->dev,
2300                        "Controller not ready to accept UIC commands\n");
2301                return -EIO;
2302        }
2303
2304        if (completion)
2305                init_completion(&uic_cmd->done);
2306
2307        uic_cmd->cmd_active = 1;
2308        ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2309
2310        return 0;
2311}
2312
2313/**
2314 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2315 * @hba: per adapter instance
2316 * @uic_cmd: UIC command
2317 *
2318 * Returns 0 only if success.
2319 */
2320int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2321{
2322        int ret;
2323        unsigned long flags;
2324
2325        ufshcd_hold(hba, false);
2326        mutex_lock(&hba->uic_cmd_mutex);
2327        ufshcd_add_delay_before_dme_cmd(hba);
2328
2329        spin_lock_irqsave(hba->host->host_lock, flags);
2330        ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2331        spin_unlock_irqrestore(hba->host->host_lock, flags);
2332        if (!ret)
2333                ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2334
2335        mutex_unlock(&hba->uic_cmd_mutex);
2336
2337        ufshcd_release(hba);
2338        return ret;
2339}
2340
2341/**
2342 * ufshcd_map_sg - Map scatter-gather list to prdt
2343 * @hba: per adapter instance
2344 * @lrbp: pointer to local reference block
2345 *
2346 * Returns 0 in case of success, non-zero value in case of failure
2347 */
2348static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2349{
2350        struct ufshcd_sg_entry *prd_table;
2351        struct scatterlist *sg;
2352        struct scsi_cmnd *cmd;
2353        int sg_segments;
2354        int i;
2355
2356        cmd = lrbp->cmd;
2357        sg_segments = scsi_dma_map(cmd);
2358        if (sg_segments < 0)
2359                return sg_segments;
2360
2361        if (sg_segments) {
2362
2363                if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2364                        lrbp->utr_descriptor_ptr->prd_table_length =
2365                                cpu_to_le16((sg_segments *
2366                                        sizeof(struct ufshcd_sg_entry)));
2367                else
2368                        lrbp->utr_descriptor_ptr->prd_table_length =
2369                                cpu_to_le16((u16) (sg_segments));
2370
2371                prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2372
2373                scsi_for_each_sg(cmd, sg, sg_segments, i) {
2374                        prd_table[i].size  =
2375                                cpu_to_le32(((u32) sg_dma_len(sg))-1);
2376                        prd_table[i].base_addr =
2377                                cpu_to_le32(lower_32_bits(sg->dma_address));
2378                        prd_table[i].upper_addr =
2379                                cpu_to_le32(upper_32_bits(sg->dma_address));
2380                        prd_table[i].reserved = 0;
2381                }
2382        } else {
2383                lrbp->utr_descriptor_ptr->prd_table_length = 0;
2384        }
2385
2386        return 0;
2387}
2388
2389/**
2390 * ufshcd_enable_intr - enable interrupts
2391 * @hba: per adapter instance
2392 * @intrs: interrupt bits
2393 */
2394static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2395{
2396        u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2397
2398        if (hba->ufs_version == ufshci_version(1, 0)) {
2399                u32 rw;
2400                rw = set & INTERRUPT_MASK_RW_VER_10;
2401                set = rw | ((set ^ intrs) & intrs);
2402        } else {
2403                set |= intrs;
2404        }
2405
2406        ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2407}
2408
2409/**
2410 * ufshcd_disable_intr - disable interrupts
2411 * @hba: per adapter instance
2412 * @intrs: interrupt bits
2413 */
2414static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2415{
2416        u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2417
2418        if (hba->ufs_version == ufshci_version(1, 0)) {
2419                u32 rw;
2420                rw = (set & INTERRUPT_MASK_RW_VER_10) &
2421                        ~(intrs & INTERRUPT_MASK_RW_VER_10);
2422                set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2423
2424        } else {
2425                set &= ~intrs;
2426        }
2427
2428        ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2429}
2430
2431/**
2432 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2433 * descriptor according to request
2434 * @lrbp: pointer to local reference block
2435 * @upiu_flags: flags required in the header
2436 * @cmd_dir: requests data direction
2437 */
2438static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2439                        u8 *upiu_flags, enum dma_data_direction cmd_dir)
2440{
2441        struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2442        u32 data_direction;
2443        u32 dword_0;
2444        u32 dword_1 = 0;
2445        u32 dword_3 = 0;
2446
2447        if (cmd_dir == DMA_FROM_DEVICE) {
2448                data_direction = UTP_DEVICE_TO_HOST;
2449                *upiu_flags = UPIU_CMD_FLAGS_READ;
2450        } else if (cmd_dir == DMA_TO_DEVICE) {
2451                data_direction = UTP_HOST_TO_DEVICE;
2452                *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2453        } else {
2454                data_direction = UTP_NO_DATA_TRANSFER;
2455                *upiu_flags = UPIU_CMD_FLAGS_NONE;
2456        }
2457
2458        dword_0 = data_direction | (lrbp->command_type
2459                                << UPIU_COMMAND_TYPE_OFFSET);
2460        if (lrbp->intr_cmd)
2461                dword_0 |= UTP_REQ_DESC_INT_CMD;
2462
2463        /* Prepare crypto related dwords */
2464        ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2465
2466        /* Transfer request descriptor header fields */
2467        req_desc->header.dword_0 = cpu_to_le32(dword_0);
2468        req_desc->header.dword_1 = cpu_to_le32(dword_1);
2469        /*
2470         * assigning invalid value for command status. Controller
2471         * updates OCS on command completion, with the command
2472         * status
2473         */
2474        req_desc->header.dword_2 =
2475                cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2476        req_desc->header.dword_3 = cpu_to_le32(dword_3);
2477
2478        req_desc->prd_table_length = 0;
2479}
2480
2481/**
2482 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2483 * for scsi commands
2484 * @lrbp: local reference block pointer
2485 * @upiu_flags: flags
2486 */
2487static
2488void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2489{
2490        struct scsi_cmnd *cmd = lrbp->cmd;
2491        struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2492        unsigned short cdb_len;
2493
2494        /* command descriptor fields */
2495        ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2496                                UPIU_TRANSACTION_COMMAND, upiu_flags,
2497                                lrbp->lun, lrbp->task_tag);
2498        ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2499                                UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2500
2501        /* Total EHS length and Data segment length will be zero */
2502        ucd_req_ptr->header.dword_2 = 0;
2503
2504        ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2505
2506        cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2507        memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2508        memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2509
2510        memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2511}
2512
2513/**
2514 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2515 * for query requsts
2516 * @hba: UFS hba
2517 * @lrbp: local reference block pointer
2518 * @upiu_flags: flags
2519 */
2520static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2521                                struct ufshcd_lrb *lrbp, u8 upiu_flags)
2522{
2523        struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2524        struct ufs_query *query = &hba->dev_cmd.query;
2525        u16 len = be16_to_cpu(query->request.upiu_req.length);
2526
2527        /* Query request header */
2528        ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2529                        UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2530                        lrbp->lun, lrbp->task_tag);
2531        ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2532                        0, query->request.query_func, 0, 0);
2533
2534        /* Data segment length only need for WRITE_DESC */
2535        if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2536                ucd_req_ptr->header.dword_2 =
2537                        UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2538        else
2539                ucd_req_ptr->header.dword_2 = 0;
2540
2541        /* Copy the Query Request buffer as is */
2542        memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2543                        QUERY_OSF_SIZE);
2544
2545        /* Copy the Descriptor */
2546        if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2547                memcpy(ucd_req_ptr + 1, query->descriptor, len);
2548
2549        memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2550}
2551
2552static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2553{
2554        struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2555
2556        memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2557
2558        /* command descriptor fields */
2559        ucd_req_ptr->header.dword_0 =
2560                UPIU_HEADER_DWORD(
2561                        UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2562        /* clear rest of the fields of basic header */
2563        ucd_req_ptr->header.dword_1 = 0;
2564        ucd_req_ptr->header.dword_2 = 0;
2565
2566        memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2567}
2568
2569/**
2570 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2571 *                           for Device Management Purposes
2572 * @hba: per adapter instance
2573 * @lrbp: pointer to local reference block
2574 */
2575static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2576                                      struct ufshcd_lrb *lrbp)
2577{
2578        u8 upiu_flags;
2579        int ret = 0;
2580
2581        if (hba->ufs_version <= ufshci_version(1, 1))
2582                lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2583        else
2584                lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2585
2586        ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2587        if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2588                ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2589        else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2590                ufshcd_prepare_utp_nop_upiu(lrbp);
2591        else
2592                ret = -EINVAL;
2593
2594        return ret;
2595}
2596
2597/**
2598 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2599 *                         for SCSI Purposes
2600 * @hba: per adapter instance
2601 * @lrbp: pointer to local reference block
2602 */
2603static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2604{
2605        u8 upiu_flags;
2606        int ret = 0;
2607
2608        if (hba->ufs_version <= ufshci_version(1, 1))
2609                lrbp->command_type = UTP_CMD_TYPE_SCSI;
2610        else
2611                lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2612
2613        if (likely(lrbp->cmd)) {
2614                ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2615                                                lrbp->cmd->sc_data_direction);
2616                ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2617        } else {
2618                ret = -EINVAL;
2619        }
2620
2621        return ret;
2622}
2623
2624/**
2625 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2626 * @upiu_wlun_id: UPIU W-LUN id
2627 *
2628 * Returns SCSI W-LUN id
2629 */
2630static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2631{
2632        return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2633}
2634
2635static inline bool is_rpmb_wlun(struct scsi_device *sdev)
2636{
2637        return sdev->lun == ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN);
2638}
2639
2640static inline bool is_device_wlun(struct scsi_device *sdev)
2641{
2642        return sdev->lun ==
2643                ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2644}
2645
2646static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2647{
2648        struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2649        struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2650        dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2651                i * sizeof(struct utp_transfer_cmd_desc);
2652        u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2653                                       response_upiu);
2654        u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2655
2656        lrb->utr_descriptor_ptr = utrdlp + i;
2657        lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2658                i * sizeof(struct utp_transfer_req_desc);
2659        lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2660        lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2661        lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2662        lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2663        lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2664        lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2665}
2666
2667/**
2668 * ufshcd_queuecommand - main entry point for SCSI requests
2669 * @host: SCSI host pointer
2670 * @cmd: command from SCSI Midlayer
2671 *
2672 * Returns 0 for success, non-zero in case of failure
2673 */
2674static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2675{
2676        struct ufs_hba *hba = shost_priv(host);
2677        int tag = scsi_cmd_to_rq(cmd)->tag;
2678        struct ufshcd_lrb *lrbp;
2679        int err = 0;
2680
2681        WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
2682
2683        if (!down_read_trylock(&hba->clk_scaling_lock))
2684                return SCSI_MLQUEUE_HOST_BUSY;
2685
2686        switch (hba->ufshcd_state) {
2687        case UFSHCD_STATE_OPERATIONAL:
2688        case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2689                break;
2690        case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2691                /*
2692                 * pm_runtime_get_sync() is used at error handling preparation
2693                 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2694                 * PM ops, it can never be finished if we let SCSI layer keep
2695                 * retrying it, which gets err handler stuck forever. Neither
2696                 * can we let the scsi cmd pass through, because UFS is in bad
2697                 * state, the scsi cmd may eventually time out, which will get
2698                 * err handler blocked for too long. So, just fail the scsi cmd
2699                 * sent from PM ops, err handler can recover PM error anyways.
2700                 */
2701                if (hba->pm_op_in_progress) {
2702                        hba->force_reset = true;
2703                        set_host_byte(cmd, DID_BAD_TARGET);
2704                        cmd->scsi_done(cmd);
2705                        goto out;
2706                }
2707                fallthrough;
2708        case UFSHCD_STATE_RESET:
2709                err = SCSI_MLQUEUE_HOST_BUSY;
2710                goto out;
2711        case UFSHCD_STATE_ERROR:
2712                set_host_byte(cmd, DID_ERROR);
2713                cmd->scsi_done(cmd);
2714                goto out;
2715        }
2716
2717        hba->req_abort_count = 0;
2718
2719        err = ufshcd_hold(hba, true);
2720        if (err) {
2721                err = SCSI_MLQUEUE_HOST_BUSY;
2722                goto out;
2723        }
2724        WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2725                (hba->clk_gating.state != CLKS_ON));
2726
2727        lrbp = &hba->lrb[tag];
2728        WARN_ON(lrbp->cmd);
2729        lrbp->cmd = cmd;
2730        lrbp->sense_bufflen = UFS_SENSE_SIZE;
2731        lrbp->sense_buffer = cmd->sense_buffer;
2732        lrbp->task_tag = tag;
2733        lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2734        lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2735
2736        ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp);
2737
2738        lrbp->req_abort_skip = false;
2739
2740        ufshpb_prep(hba, lrbp);
2741
2742        ufshcd_comp_scsi_upiu(hba, lrbp);
2743
2744        err = ufshcd_map_sg(hba, lrbp);
2745        if (err) {
2746                lrbp->cmd = NULL;
2747                ufshcd_release(hba);
2748                goto out;
2749        }
2750
2751        ufshcd_send_command(hba, tag);
2752out:
2753        up_read(&hba->clk_scaling_lock);
2754
2755        if (ufs_trigger_eh()) {
2756                unsigned long flags;
2757
2758                spin_lock_irqsave(hba->host->host_lock, flags);
2759                ufshcd_schedule_eh_work(hba);
2760                spin_unlock_irqrestore(hba->host->host_lock, flags);
2761        }
2762
2763        return err;
2764}
2765
2766static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2767                struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2768{
2769        lrbp->cmd = NULL;
2770        lrbp->sense_bufflen = 0;
2771        lrbp->sense_buffer = NULL;
2772        lrbp->task_tag = tag;
2773        lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2774        lrbp->intr_cmd = true; /* No interrupt aggregation */
2775        ufshcd_prepare_lrbp_crypto(NULL, lrbp);
2776        hba->dev_cmd.type = cmd_type;
2777
2778        return ufshcd_compose_devman_upiu(hba, lrbp);
2779}
2780
2781static int
2782ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2783{
2784        int err = 0;
2785        unsigned long flags;
2786        u32 mask = 1 << tag;
2787
2788        /* clear outstanding transaction before retry */
2789        spin_lock_irqsave(hba->host->host_lock, flags);
2790        ufshcd_utrl_clear(hba, tag);
2791        spin_unlock_irqrestore(hba->host->host_lock, flags);
2792
2793        /*
2794         * wait for h/w to clear corresponding bit in door-bell.
2795         * max. wait is 1 sec.
2796         */
2797        err = ufshcd_wait_for_register(hba,
2798                        REG_UTP_TRANSFER_REQ_DOOR_BELL,
2799                        mask, ~mask, 1000, 1000);
2800
2801        return err;
2802}
2803
2804static int
2805ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2806{
2807        struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2808
2809        /* Get the UPIU response */
2810        query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2811                                UPIU_RSP_CODE_OFFSET;
2812        return query_res->response;
2813}
2814
2815/**
2816 * ufshcd_dev_cmd_completion() - handles device management command responses
2817 * @hba: per adapter instance
2818 * @lrbp: pointer to local reference block
2819 */
2820static int
2821ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2822{
2823        int resp;
2824        int err = 0;
2825
2826        hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2827        resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2828
2829        switch (resp) {
2830        case UPIU_TRANSACTION_NOP_IN:
2831                if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2832                        err = -EINVAL;
2833                        dev_err(hba->dev, "%s: unexpected response %x\n",
2834                                        __func__, resp);
2835                }
2836                break;
2837        case UPIU_TRANSACTION_QUERY_RSP:
2838                err = ufshcd_check_query_response(hba, lrbp);
2839                if (!err)
2840                        err = ufshcd_copy_query_response(hba, lrbp);
2841                break;
2842        case UPIU_TRANSACTION_REJECT_UPIU:
2843                /* TODO: handle Reject UPIU Response */
2844                err = -EPERM;
2845                dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2846                                __func__);
2847                break;
2848        default:
2849                err = -EINVAL;
2850                dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2851                                __func__, resp);
2852                break;
2853        }
2854
2855        return err;
2856}
2857
2858static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2859                struct ufshcd_lrb *lrbp, int max_timeout)
2860{
2861        int err = 0;
2862        unsigned long time_left;
2863        unsigned long flags;
2864
2865        time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2866                        msecs_to_jiffies(max_timeout));
2867
2868        spin_lock_irqsave(hba->host->host_lock, flags);
2869        hba->dev_cmd.complete = NULL;
2870        if (likely(time_left)) {
2871                err = ufshcd_get_tr_ocs(lrbp);
2872                if (!err)
2873                        err = ufshcd_dev_cmd_completion(hba, lrbp);
2874        }
2875        spin_unlock_irqrestore(hba->host->host_lock, flags);
2876
2877        if (!time_left) {
2878                err = -ETIMEDOUT;
2879                dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2880                        __func__, lrbp->task_tag);
2881                if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2882                        /* successfully cleared the command, retry if needed */
2883                        err = -EAGAIN;
2884                /*
2885                 * in case of an error, after clearing the doorbell,
2886                 * we also need to clear the outstanding_request
2887                 * field in hba
2888                 */
2889                spin_lock_irqsave(&hba->outstanding_lock, flags);
2890                __clear_bit(lrbp->task_tag, &hba->outstanding_reqs);
2891                spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2892        }
2893
2894        return err;
2895}
2896
2897/**
2898 * ufshcd_exec_dev_cmd - API for sending device management requests
2899 * @hba: UFS hba
2900 * @cmd_type: specifies the type (NOP, Query...)
2901 * @timeout: timeout in milliseconds
2902 *
2903 * NOTE: Since there is only one available tag for device management commands,
2904 * it is expected you hold the hba->dev_cmd.lock mutex.
2905 */
2906static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2907                enum dev_cmd_type cmd_type, int timeout)
2908{
2909        struct request_queue *q = hba->cmd_queue;
2910        DECLARE_COMPLETION_ONSTACK(wait);
2911        struct request *req;
2912        struct ufshcd_lrb *lrbp;
2913        int err;
2914        int tag;
2915
2916        down_read(&hba->clk_scaling_lock);
2917
2918        /*
2919         * Get free slot, sleep if slots are unavailable.
2920         * Even though we use wait_event() which sleeps indefinitely,
2921         * the maximum wait time is bounded by SCSI request timeout.
2922         */
2923        req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
2924        if (IS_ERR(req)) {
2925                err = PTR_ERR(req);
2926                goto out_unlock;
2927        }
2928        tag = req->tag;
2929        WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
2930        /* Set the timeout such that the SCSI error handler is not activated. */
2931        req->timeout = msecs_to_jiffies(2 * timeout);
2932        blk_mq_start_request(req);
2933
2934        lrbp = &hba->lrb[tag];
2935        WARN_ON(lrbp->cmd);
2936        err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2937        if (unlikely(err))
2938                goto out;
2939
2940        hba->dev_cmd.complete = &wait;
2941
2942        ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
2943
2944        ufshcd_send_command(hba, tag);
2945        err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2946        ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
2947                                    (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
2948
2949out:
2950        blk_put_request(req);
2951out_unlock:
2952        up_read(&hba->clk_scaling_lock);
2953        return err;
2954}
2955
2956/**
2957 * ufshcd_init_query() - init the query response and request parameters
2958 * @hba: per-adapter instance
2959 * @request: address of the request pointer to be initialized
2960 * @response: address of the response pointer to be initialized
2961 * @opcode: operation to perform
2962 * @idn: flag idn to access
2963 * @index: LU number to access
2964 * @selector: query/flag/descriptor further identification
2965 */
2966static inline void ufshcd_init_query(struct ufs_hba *hba,
2967                struct ufs_query_req **request, struct ufs_query_res **response,
2968                enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2969{
2970        *request = &hba->dev_cmd.query.request;
2971        *response = &hba->dev_cmd.query.response;
2972        memset(*request, 0, sizeof(struct ufs_query_req));
2973        memset(*response, 0, sizeof(struct ufs_query_res));
2974        (*request)->upiu_req.opcode = opcode;
2975        (*request)->upiu_req.idn = idn;
2976        (*request)->upiu_req.index = index;
2977        (*request)->upiu_req.selector = selector;
2978}
2979
2980static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2981        enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
2982{
2983        int ret;
2984        int retries;
2985
2986        for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2987                ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
2988                if (ret)
2989                        dev_dbg(hba->dev,
2990                                "%s: failed with error %d, retries %d\n",
2991                                __func__, ret, retries);
2992                else
2993                        break;
2994        }
2995
2996        if (ret)
2997                dev_err(hba->dev,
2998                        "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2999                        __func__, opcode, idn, ret, retries);
3000        return ret;
3001}
3002
3003/**
3004 * ufshcd_query_flag() - API function for sending flag query requests
3005 * @hba: per-adapter instance
3006 * @opcode: flag query to perform
3007 * @idn: flag idn to access
3008 * @index: flag index to access
3009 * @flag_res: the flag value after the query request completes
3010 *
3011 * Returns 0 for success, non-zero in case of failure
3012 */
3013int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3014                        enum flag_idn idn, u8 index, bool *flag_res)
3015{
3016        struct ufs_query_req *request = NULL;
3017        struct ufs_query_res *response = NULL;
3018        int err, selector = 0;
3019        int timeout = QUERY_REQ_TIMEOUT;
3020
3021        BUG_ON(!hba);
3022
3023        ufshcd_hold(hba, false);
3024        mutex_lock(&hba->dev_cmd.lock);
3025        ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3026                        selector);
3027
3028        switch (opcode) {
3029        case UPIU_QUERY_OPCODE_SET_FLAG:
3030        case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3031        case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3032                request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3033                break;
3034        case UPIU_QUERY_OPCODE_READ_FLAG:
3035                request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3036                if (!flag_res) {
3037                        /* No dummy reads */
3038                        dev_err(hba->dev, "%s: Invalid argument for read request\n",
3039                                        __func__);
3040                        err = -EINVAL;
3041                        goto out_unlock;
3042                }
3043                break;
3044        default:
3045                dev_err(hba->dev,
3046                        "%s: Expected query flag opcode but got = %d\n",
3047                        __func__, opcode);
3048                err = -EINVAL;
3049                goto out_unlock;
3050        }
3051
3052        err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3053
3054        if (err) {
3055                dev_err(hba->dev,
3056                        "%s: Sending flag query for idn %d failed, err = %d\n",
3057                        __func__, idn, err);
3058                goto out_unlock;
3059        }
3060
3061        if (flag_res)
3062                *flag_res = (be32_to_cpu(response->upiu_res.value) &
3063                                MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3064
3065out_unlock:
3066        mutex_unlock(&hba->dev_cmd.lock);
3067        ufshcd_release(hba);
3068        return err;
3069}
3070
3071/**
3072 * ufshcd_query_attr - API function for sending attribute requests
3073 * @hba: per-adapter instance
3074 * @opcode: attribute opcode
3075 * @idn: attribute idn to access
3076 * @index: index field
3077 * @selector: selector field
3078 * @attr_val: the attribute value after the query request completes
3079 *
3080 * Returns 0 for success, non-zero in case of failure
3081*/
3082int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3083                      enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3084{
3085        struct ufs_query_req *request = NULL;
3086        struct ufs_query_res *response = NULL;
3087        int err;
3088
3089        BUG_ON(!hba);
3090
3091        if (!attr_val) {
3092                dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3093                                __func__, opcode);
3094                return -EINVAL;
3095        }
3096
3097        ufshcd_hold(hba, false);
3098
3099        mutex_lock(&hba->dev_cmd.lock);
3100        ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3101                        selector);
3102
3103        switch (opcode) {
3104        case UPIU_QUERY_OPCODE_WRITE_ATTR:
3105                request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3106                request->upiu_req.value = cpu_to_be32(*attr_val);
3107                break;
3108        case UPIU_QUERY_OPCODE_READ_ATTR:
3109                request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3110                break;
3111        default:
3112                dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3113                                __func__, opcode);
3114                err = -EINVAL;
3115                goto out_unlock;
3116        }
3117
3118        err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3119
3120        if (err) {
3121                dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3122                                __func__, opcode, idn, index, err);
3123                goto out_unlock;
3124        }
3125
3126        *attr_val = be32_to_cpu(response->upiu_res.value);
3127
3128out_unlock:
3129        mutex_unlock(&hba->dev_cmd.lock);
3130        ufshcd_release(hba);
3131        return err;
3132}
3133
3134/**
3135 * ufshcd_query_attr_retry() - API function for sending query
3136 * attribute with retries
3137 * @hba: per-adapter instance
3138 * @opcode: attribute opcode
3139 * @idn: attribute idn to access
3140 * @index: index field
3141 * @selector: selector field
3142 * @attr_val: the attribute value after the query request
3143 * completes
3144 *
3145 * Returns 0 for success, non-zero in case of failure
3146*/
3147int ufshcd_query_attr_retry(struct ufs_hba *hba,
3148        enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3149        u32 *attr_val)
3150{
3151        int ret = 0;
3152        u32 retries;
3153
3154        for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3155                ret = ufshcd_query_attr(hba, opcode, idn, index,
3156                                                selector, attr_val);
3157                if (ret)
3158                        dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3159                                __func__, ret, retries);
3160                else
3161                        break;
3162        }
3163
3164        if (ret)
3165                dev_err(hba->dev,
3166                        "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3167                        __func__, idn, ret, QUERY_REQ_RETRIES);
3168        return ret;
3169}
3170
3171static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3172                        enum query_opcode opcode, enum desc_idn idn, u8 index,
3173                        u8 selector, u8 *desc_buf, int *buf_len)
3174{
3175        struct ufs_query_req *request = NULL;
3176        struct ufs_query_res *response = NULL;
3177        int err;
3178
3179        BUG_ON(!hba);
3180
3181        if (!desc_buf) {
3182                dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3183                                __func__, opcode);
3184                return -EINVAL;
3185        }
3186
3187        if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3188                dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3189                                __func__, *buf_len);
3190                return -EINVAL;
3191        }
3192
3193        ufshcd_hold(hba, false);
3194
3195        mutex_lock(&hba->dev_cmd.lock);
3196        ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3197                        selector);
3198        hba->dev_cmd.query.descriptor = desc_buf;
3199        request->upiu_req.length = cpu_to_be16(*buf_len);
3200
3201        switch (opcode) {
3202        case UPIU_QUERY_OPCODE_WRITE_DESC:
3203                request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3204                break;
3205        case UPIU_QUERY_OPCODE_READ_DESC:
3206                request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3207                break;
3208        default:
3209                dev_err(hba->dev,
3210                                "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3211                                __func__, opcode);
3212                err = -EINVAL;
3213                goto out_unlock;
3214        }
3215
3216        err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3217
3218        if (err) {
3219                dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3220                                __func__, opcode, idn, index, err);
3221                goto out_unlock;
3222        }
3223
3224        *buf_len = be16_to_cpu(response->upiu_res.length);
3225
3226out_unlock:
3227        hba->dev_cmd.query.descriptor = NULL;
3228        mutex_unlock(&hba->dev_cmd.lock);
3229        ufshcd_release(hba);
3230        return err;
3231}
3232
3233/**
3234 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3235 * @hba: per-adapter instance
3236 * @opcode: attribute opcode
3237 * @idn: attribute idn to access
3238 * @index: index field
3239 * @selector: selector field
3240 * @desc_buf: the buffer that contains the descriptor
3241 * @buf_len: length parameter passed to the device
3242 *
3243 * Returns 0 for success, non-zero in case of failure.
3244 * The buf_len parameter will contain, on return, the length parameter
3245 * received on the response.
3246 */
3247int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3248                                  enum query_opcode opcode,
3249                                  enum desc_idn idn, u8 index,
3250                                  u8 selector,
3251                                  u8 *desc_buf, int *buf_len)
3252{
3253        int err;
3254        int retries;
3255
3256        for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3257                err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3258                                                selector, desc_buf, buf_len);
3259                if (!err || err == -EINVAL)
3260                        break;
3261        }
3262
3263        return err;
3264}
3265
3266/**
3267 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3268 * @hba: Pointer to adapter instance
3269 * @desc_id: descriptor idn value
3270 * @desc_len: mapped desc length (out)
3271 */
3272void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3273                                  int *desc_len)
3274{
3275        if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3276            desc_id == QUERY_DESC_IDN_RFU_1)
3277                *desc_len = 0;
3278        else
3279                *desc_len = hba->desc_size[desc_id];
3280}
3281EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3282
3283static void ufshcd_update_desc_length(struct ufs_hba *hba,
3284                                      enum desc_idn desc_id, int desc_index,
3285                                      unsigned char desc_len)
3286{
3287        if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
3288            desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3289                /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3290                 * than the RPMB unit, however, both descriptors share the same
3291                 * desc_idn, to cover both unit descriptors with one length, we
3292                 * choose the normal unit descriptor length by desc_index.
3293                 */
3294                hba->desc_size[desc_id] = desc_len;
3295}
3296
3297/**
3298 * ufshcd_read_desc_param - read the specified descriptor parameter
3299 * @hba: Pointer to adapter instance
3300 * @desc_id: descriptor idn value
3301 * @desc_index: descriptor index
3302 * @param_offset: offset of the parameter to read
3303 * @param_read_buf: pointer to buffer where parameter would be read
3304 * @param_size: sizeof(param_read_buf)
3305 *
3306 * Return 0 in case of success, non-zero otherwise
3307 */
3308int ufshcd_read_desc_param(struct ufs_hba *hba,
3309                           enum desc_idn desc_id,
3310                           int desc_index,
3311                           u8 param_offset,
3312                           u8 *param_read_buf,
3313                           u8 param_size)
3314{
3315        int ret;
3316        u8 *desc_buf;
3317        int buff_len;
3318        bool is_kmalloc = true;
3319
3320        /* Safety check */
3321        if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3322                return -EINVAL;
3323
3324        /* Get the length of descriptor */
3325        ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3326        if (!buff_len) {
3327                dev_err(hba->dev, "%s: Failed to get desc length\n", __func__);
3328                return -EINVAL;
3329        }
3330
3331        if (param_offset >= buff_len) {
3332                dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3333                        __func__, param_offset, desc_id, buff_len);
3334                return -EINVAL;
3335        }
3336
3337        /* Check whether we need temp memory */
3338        if (param_offset != 0 || param_size < buff_len) {
3339                desc_buf = kzalloc(buff_len, GFP_KERNEL);
3340                if (!desc_buf)
3341                        return -ENOMEM;
3342        } else {
3343                desc_buf = param_read_buf;
3344                is_kmalloc = false;
3345        }
3346
3347        /* Request for full descriptor */
3348        ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3349                                        desc_id, desc_index, 0,
3350                                        desc_buf, &buff_len);
3351
3352        if (ret) {
3353                dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3354                        __func__, desc_id, desc_index, param_offset, ret);
3355                goto out;
3356        }
3357
3358        /* Sanity check */
3359        if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3360                dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3361                        __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3362                ret = -EINVAL;
3363                goto out;
3364        }
3365
3366        /* Update descriptor length */
3367        buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3368        ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
3369
3370        if (is_kmalloc) {
3371                /* Make sure we don't copy more data than available */
3372                if (param_offset >= buff_len)
3373                        ret = -EINVAL;
3374                else
3375                        memcpy(param_read_buf, &desc_buf[param_offset],
3376                               min_t(u32, param_size, buff_len - param_offset));
3377        }
3378out:
3379        if (is_kmalloc)
3380                kfree(desc_buf);
3381        return ret;
3382}
3383
3384/**
3385 * struct uc_string_id - unicode string
3386 *
3387 * @len: size of this descriptor inclusive
3388 * @type: descriptor type
3389 * @uc: unicode string character
3390 */
3391struct uc_string_id {
3392        u8 len;
3393        u8 type;
3394        wchar_t uc[];
3395} __packed;
3396
3397/* replace non-printable or non-ASCII characters with spaces */
3398static inline char ufshcd_remove_non_printable(u8 ch)
3399{
3400        return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3401}
3402
3403/**
3404 * ufshcd_read_string_desc - read string descriptor
3405 * @hba: pointer to adapter instance
3406 * @desc_index: descriptor index
3407 * @buf: pointer to buffer where descriptor would be read,
3408 *       the caller should free the memory.
3409 * @ascii: if true convert from unicode to ascii characters
3410 *         null terminated string.
3411 *
3412 * Return:
3413 * *      string size on success.
3414 * *      -ENOMEM: on allocation failure
3415 * *      -EINVAL: on a wrong parameter
3416 */
3417int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3418                            u8 **buf, bool ascii)
3419{
3420        struct uc_string_id *uc_str;
3421        u8 *str;
3422        int ret;
3423
3424        if (!buf)
3425                return -EINVAL;
3426
3427        uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3428        if (!uc_str)
3429                return -ENOMEM;
3430
3431        ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3432                                     (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3433        if (ret < 0) {
3434                dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3435                        QUERY_REQ_RETRIES, ret);
3436                str = NULL;
3437                goto out;
3438        }
3439
3440        if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3441                dev_dbg(hba->dev, "String Desc is of zero length\n");
3442                str = NULL;
3443                ret = 0;
3444                goto out;
3445        }
3446
3447        if (ascii) {
3448                ssize_t ascii_len;
3449                int i;
3450                /* remove header and divide by 2 to move from UTF16 to UTF8 */
3451                ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3452                str = kzalloc(ascii_len, GFP_KERNEL);
3453                if (!str) {
3454                        ret = -ENOMEM;
3455                        goto out;
3456                }
3457
3458                /*
3459                 * the descriptor contains string in UTF16 format
3460                 * we need to convert to utf-8 so it can be displayed
3461                 */
3462                ret = utf16s_to_utf8s(uc_str->uc,
3463                                      uc_str->len - QUERY_DESC_HDR_SIZE,
3464                                      UTF16_BIG_ENDIAN, str, ascii_len);
3465
3466                /* replace non-printable or non-ASCII characters with spaces */
3467                for (i = 0; i < ret; i++)
3468                        str[i] = ufshcd_remove_non_printable(str[i]);
3469
3470                str[ret++] = '\0';
3471
3472        } else {
3473                str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3474                if (!str) {
3475                        ret = -ENOMEM;
3476                        goto out;
3477                }
3478                ret = uc_str->len;
3479        }
3480out:
3481        *buf = str;
3482        kfree(uc_str);
3483        return ret;
3484}
3485
3486/**
3487 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3488 * @hba: Pointer to adapter instance
3489 * @lun: lun id
3490 * @param_offset: offset of the parameter to read
3491 * @param_read_buf: pointer to buffer where parameter would be read
3492 * @param_size: sizeof(param_read_buf)
3493 *
3494 * Return 0 in case of success, non-zero otherwise
3495 */
3496static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3497                                              int lun,
3498                                              enum unit_desc_param param_offset,
3499                                              u8 *param_read_buf,
3500                                              u32 param_size)
3501{
3502        /*
3503         * Unit descriptors are only available for general purpose LUs (LUN id
3504         * from 0 to 7) and RPMB Well known LU.
3505         */
3506        if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun, param_offset))
3507                return -EOPNOTSUPP;
3508
3509        return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3510                                      param_offset, param_read_buf, param_size);
3511}
3512
3513static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3514{
3515        int err = 0;
3516        u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3517
3518        if (hba->dev_info.wspecversion >= 0x300) {
3519                err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3520                                QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3521                                &gating_wait);
3522                if (err)
3523                        dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3524                                         err, gating_wait);
3525
3526                if (gating_wait == 0) {
3527                        gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3528                        dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3529                                         gating_wait);
3530                }
3531
3532                hba->dev_info.clk_gating_wait_us = gating_wait;
3533        }
3534
3535        return err;
3536}
3537
3538/**
3539 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3540 * @hba: per adapter instance
3541 *
3542 * 1. Allocate DMA memory for Command Descriptor array
3543 *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3544 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3545 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3546 *      (UTMRDL)
3547 * 4. Allocate memory for local reference block(lrb).
3548 *
3549 * Returns 0 for success, non-zero in case of failure
3550 */
3551static int ufshcd_memory_alloc(struct ufs_hba *hba)
3552{
3553        size_t utmrdl_size, utrdl_size, ucdl_size;
3554
3555        /* Allocate memory for UTP command descriptors */
3556        ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
3557        hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3558                                                  ucdl_size,
3559                                                  &hba->ucdl_dma_addr,
3560                                                  GFP_KERNEL);
3561
3562        /*
3563         * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3564         * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3565         * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3566         * be aligned to 128 bytes as well
3567         */
3568        if (!hba->ucdl_base_addr ||
3569            WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3570                dev_err(hba->dev,
3571                        "Command Descriptor Memory allocation failed\n");
3572                goto out;
3573        }
3574
3575        /*
3576         * Allocate memory for UTP Transfer descriptors
3577         * UFSHCI requires 1024 byte alignment of UTRD
3578         */
3579        utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3580        hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3581                                                   utrdl_size,
3582                                                   &hba->utrdl_dma_addr,
3583                                                   GFP_KERNEL);
3584        if (!hba->utrdl_base_addr ||
3585            WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3586                dev_err(hba->dev,
3587                        "Transfer Descriptor Memory allocation failed\n");
3588                goto out;
3589        }
3590
3591        /*
3592         * Allocate memory for UTP Task Management descriptors
3593         * UFSHCI requires 1024 byte alignment of UTMRD
3594         */
3595        utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3596        hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3597                                                    utmrdl_size,
3598                                                    &hba->utmrdl_dma_addr,
3599                                                    GFP_KERNEL);
3600        if (!hba->utmrdl_base_addr ||
3601            WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3602                dev_err(hba->dev,
3603                "Task Management Descriptor Memory allocation failed\n");
3604                goto out;
3605        }
3606
3607        /* Allocate memory for local reference block */
3608        hba->lrb = devm_kcalloc(hba->dev,
3609                                hba->nutrs, sizeof(struct ufshcd_lrb),
3610                                GFP_KERNEL);
3611        if (!hba->lrb) {
3612                dev_err(hba->dev, "LRB Memory allocation failed\n");
3613                goto out;
3614        }
3615        return 0;
3616out:
3617        return -ENOMEM;
3618}
3619
3620/**
3621 * ufshcd_host_memory_configure - configure local reference block with
3622 *                              memory offsets
3623 * @hba: per adapter instance
3624 *
3625 * Configure Host memory space
3626 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3627 * address.
3628 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3629 * and PRDT offset.
3630 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3631 * into local reference block.
3632 */
3633static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3634{
3635        struct utp_transfer_req_desc *utrdlp;
3636        dma_addr_t cmd_desc_dma_addr;
3637        dma_addr_t cmd_desc_element_addr;
3638        u16 response_offset;
3639        u16 prdt_offset;
3640        int cmd_desc_size;
3641        int i;
3642
3643        utrdlp = hba->utrdl_base_addr;
3644
3645        response_offset =
3646                offsetof(struct utp_transfer_cmd_desc, response_upiu);
3647        prdt_offset =
3648                offsetof(struct utp_transfer_cmd_desc, prd_table);
3649
3650        cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3651        cmd_desc_dma_addr = hba->ucdl_dma_addr;
3652
3653        for (i = 0; i < hba->nutrs; i++) {
3654                /* Configure UTRD with command descriptor base address */
3655                cmd_desc_element_addr =
3656                                (cmd_desc_dma_addr + (cmd_desc_size * i));
3657                utrdlp[i].command_desc_base_addr_lo =
3658                                cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3659                utrdlp[i].command_desc_base_addr_hi =
3660                                cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3661
3662                /* Response upiu and prdt offset should be in double words */
3663                if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3664                        utrdlp[i].response_upiu_offset =
3665                                cpu_to_le16(response_offset);
3666                        utrdlp[i].prd_table_offset =
3667                                cpu_to_le16(prdt_offset);
3668                        utrdlp[i].response_upiu_length =
3669                                cpu_to_le16(ALIGNED_UPIU_SIZE);
3670                } else {
3671                        utrdlp[i].response_upiu_offset =
3672                                cpu_to_le16(response_offset >> 2);
3673                        utrdlp[i].prd_table_offset =
3674                                cpu_to_le16(prdt_offset >> 2);
3675                        utrdlp[i].response_upiu_length =
3676                                cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3677                }
3678
3679                ufshcd_init_lrb(hba, &hba->lrb[i], i);
3680        }
3681}
3682
3683/**
3684 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3685 * @hba: per adapter instance
3686 *
3687 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3688 * in order to initialize the Unipro link startup procedure.
3689 * Once the Unipro links are up, the device connected to the controller
3690 * is detected.
3691 *
3692 * Returns 0 on success, non-zero value on failure
3693 */
3694static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3695{
3696        struct uic_command uic_cmd = {0};
3697        int ret;
3698
3699        uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3700
3701        ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3702        if (ret)
3703                dev_dbg(hba->dev,
3704                        "dme-link-startup: error code %d\n", ret);
3705        return ret;
3706}
3707/**
3708 * ufshcd_dme_reset - UIC command for DME_RESET
3709 * @hba: per adapter instance
3710 *
3711 * DME_RESET command is issued in order to reset UniPro stack.
3712 * This function now deals with cold reset.
3713 *
3714 * Returns 0 on success, non-zero value on failure
3715 */
3716static int ufshcd_dme_reset(struct ufs_hba *hba)
3717{
3718        struct uic_command uic_cmd = {0};
3719        int ret;
3720
3721        uic_cmd.command = UIC_CMD_DME_RESET;
3722
3723        ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3724        if (ret)
3725                dev_err(hba->dev,
3726                        "dme-reset: error code %d\n", ret);
3727
3728        return ret;
3729}
3730
3731int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
3732                               int agreed_gear,
3733                               int adapt_val)
3734{
3735        int ret;
3736
3737        if (agreed_gear != UFS_HS_G4)
3738                adapt_val = PA_NO_ADAPT;
3739
3740        ret = ufshcd_dme_set(hba,
3741                             UIC_ARG_MIB(PA_TXHSADAPTTYPE),
3742                             adapt_val);
3743        return ret;
3744}
3745EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
3746
3747/**
3748 * ufshcd_dme_enable - UIC command for DME_ENABLE
3749 * @hba: per adapter instance
3750 *
3751 * DME_ENABLE command is issued in order to enable UniPro stack.
3752 *
3753 * Returns 0 on success, non-zero value on failure
3754 */
3755static int ufshcd_dme_enable(struct ufs_hba *hba)
3756{
3757        struct uic_command uic_cmd = {0};
3758        int ret;
3759
3760        uic_cmd.command = UIC_CMD_DME_ENABLE;
3761
3762        ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3763        if (ret)
3764                dev_err(hba->dev,
3765                        "dme-enable: error code %d\n", ret);
3766
3767        return ret;
3768}
3769
3770static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3771{
3772        #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
3773        unsigned long min_sleep_time_us;
3774
3775        if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3776                return;
3777
3778        /*
3779         * last_dme_cmd_tstamp will be 0 only for 1st call to
3780         * this function
3781         */
3782        if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3783                min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3784        } else {
3785                unsigned long delta =
3786                        (unsigned long) ktime_to_us(
3787                                ktime_sub(ktime_get(),
3788                                hba->last_dme_cmd_tstamp));
3789
3790                if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3791                        min_sleep_time_us =
3792                                MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3793                else
3794                        return; /* no more delay required */
3795        }
3796
3797        /* allow sleep for extra 50us if needed */
3798        usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3799}
3800
3801/**
3802 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3803 * @hba: per adapter instance
3804 * @attr_sel: uic command argument1
3805 * @attr_set: attribute set type as uic command argument2
3806 * @mib_val: setting value as uic command argument3
3807 * @peer: indicate whether peer or local
3808 *
3809 * Returns 0 on success, non-zero value on failure
3810 */
3811int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3812                        u8 attr_set, u32 mib_val, u8 peer)
3813{
3814        struct uic_command uic_cmd = {0};
3815        static const char *const action[] = {
3816                "dme-set",
3817                "dme-peer-set"
3818        };
3819        const char *set = action[!!peer];
3820        int ret;
3821        int retries = UFS_UIC_COMMAND_RETRIES;
3822
3823        uic_cmd.command = peer ?
3824                UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3825        uic_cmd.argument1 = attr_sel;
3826        uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3827        uic_cmd.argument3 = mib_val;
3828
3829        do {
3830                /* for peer attributes we retry upon failure */
3831                ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3832                if (ret)
3833                        dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3834                                set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3835        } while (ret && peer && --retries);
3836
3837        if (ret)
3838                dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
3839                        set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3840                        UFS_UIC_COMMAND_RETRIES - retries);
3841
3842        return ret;
3843}
3844EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3845
3846/**
3847 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3848 * @hba: per adapter instance
3849 * @attr_sel: uic command argument1
3850 * @mib_val: the value of the attribute as returned by the UIC command
3851 * @peer: indicate whether peer or local
3852 *
3853 * Returns 0 on success, non-zero value on failure
3854 */
3855int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3856                        u32 *mib_val, u8 peer)
3857{
3858        struct uic_command uic_cmd = {0};
3859        static const char *const action[] = {
3860                "dme-get",
3861                "dme-peer-get"
3862        };
3863        const char *get = action[!!peer];
3864        int ret;
3865        int retries = UFS_UIC_COMMAND_RETRIES;
3866        struct ufs_pa_layer_attr orig_pwr_info;
3867        struct ufs_pa_layer_attr temp_pwr_info;
3868        bool pwr_mode_change = false;
3869
3870        if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3871                orig_pwr_info = hba->pwr_info;
3872                temp_pwr_info = orig_pwr_info;
3873
3874                if (orig_pwr_info.pwr_tx == FAST_MODE ||
3875                    orig_pwr_info.pwr_rx == FAST_MODE) {
3876                        temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3877                        temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3878                        pwr_mode_change = true;
3879                } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3880                    orig_pwr_info.pwr_rx == SLOW_MODE) {
3881                        temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3882                        temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3883                        pwr_mode_change = true;
3884                }
3885                if (pwr_mode_change) {
3886                        ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3887                        if (ret)
3888                                goto out;
3889                }
3890        }
3891
3892        uic_cmd.command = peer ?
3893                UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3894        uic_cmd.argument1 = attr_sel;
3895
3896        do {
3897                /* for peer attributes we retry upon failure */
3898                ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3899                if (ret)
3900                        dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3901                                get, UIC_GET_ATTR_ID(attr_sel), ret);
3902        } while (ret && peer && --retries);
3903
3904        if (ret)
3905                dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
3906                        get, UIC_GET_ATTR_ID(attr_sel),
3907                        UFS_UIC_COMMAND_RETRIES - retries);
3908
3909        if (mib_val && !ret)
3910                *mib_val = uic_cmd.argument3;
3911
3912        if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3913            && pwr_mode_change)
3914                ufshcd_change_power_mode(hba, &orig_pwr_info);
3915out:
3916        return ret;
3917}
3918EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3919
3920/**
3921 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3922 * state) and waits for it to take effect.
3923 *
3924 * @hba: per adapter instance
3925 * @cmd: UIC command to execute
3926 *
3927 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3928 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3929 * and device UniPro link and hence it's final completion would be indicated by
3930 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3931 * addition to normal UIC command completion Status (UCCS). This function only
3932 * returns after the relevant status bits indicate the completion.
3933 *
3934 * Returns 0 on success, non-zero value on failure
3935 */
3936static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
3937{
3938        DECLARE_COMPLETION_ONSTACK(uic_async_done);
3939        unsigned long flags;
3940        u8 status;
3941        int ret;
3942        bool reenable_intr = false;
3943
3944        mutex_lock(&hba->uic_cmd_mutex);
3945        ufshcd_add_delay_before_dme_cmd(hba);
3946
3947        spin_lock_irqsave(hba->host->host_lock, flags);
3948        if (ufshcd_is_link_broken(hba)) {
3949                ret = -ENOLINK;
3950                goto out_unlock;
3951        }
3952        hba->uic_async_done = &uic_async_done;
3953        if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3954                ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3955                /*
3956                 * Make sure UIC command completion interrupt is disabled before
3957                 * issuing UIC command.
3958                 */
3959                wmb();
3960                reenable_intr = true;
3961        }
3962        ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3963        spin_unlock_irqrestore(hba->host->host_lock, flags);
3964        if (ret) {
3965                dev_err(hba->dev,
3966                        "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3967                        cmd->command, cmd->argument3, ret);
3968                goto out;
3969        }
3970
3971        if (!wait_for_completion_timeout(hba->uic_async_done,
3972                                         msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3973                dev_err(hba->dev,
3974                        "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3975                        cmd->command, cmd->argument3);
3976
3977                if (!cmd->cmd_active) {
3978                        dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
3979                                __func__);
3980                        goto check_upmcrs;
3981                }
3982
3983                ret = -ETIMEDOUT;
3984                goto out;
3985        }
3986
3987check_upmcrs:
3988        status = ufshcd_get_upmcrs(hba);
3989        if (status != PWR_LOCAL) {
3990                dev_err(hba->dev,
3991                        "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
3992                        cmd->command, status);
3993                ret = (status != PWR_OK) ? status : -1;
3994        }
3995out:
3996        if (ret) {
3997                ufshcd_print_host_state(hba);
3998                ufshcd_print_pwr_info(hba);
3999                ufshcd_print_evt_hist(hba);
4000        }
4001
4002        spin_lock_irqsave(hba->host->host_lock, flags);
4003        hba->active_uic_cmd = NULL;
4004        hba->uic_async_done = NULL;
4005        if (reenable_intr)
4006                ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4007        if (ret) {
4008                ufshcd_set_link_broken(hba);
4009                ufshcd_schedule_eh_work(hba);
4010        }
4011out_unlock:
4012        spin_unlock_irqrestore(hba->host->host_lock, flags);
4013        mutex_unlock(&hba->uic_cmd_mutex);
4014
4015        return ret;
4016}
4017
4018/**
4019 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4020 *                              using DME_SET primitives.
4021 * @hba: per adapter instance
4022 * @mode: powr mode value
4023 *
4024 * Returns 0 on success, non-zero value on failure
4025 */
4026static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4027{
4028        struct uic_command uic_cmd = {0};
4029        int ret;
4030
4031        if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4032                ret = ufshcd_dme_set(hba,
4033                                UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4034                if (ret) {
4035                        dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4036                                                __func__, ret);
4037                        goto out;
4038                }
4039        }
4040
4041        uic_cmd.command = UIC_CMD_DME_SET;
4042        uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
4043        uic_cmd.argument3 = mode;
4044        ufshcd_hold(hba, false);
4045        ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4046        ufshcd_release(hba);
4047
4048out:
4049        return ret;
4050}
4051
4052int ufshcd_link_recovery(struct ufs_hba *hba)
4053{
4054        int ret;
4055        unsigned long flags;
4056
4057        spin_lock_irqsave(hba->host->host_lock, flags);
4058        hba->ufshcd_state = UFSHCD_STATE_RESET;
4059        ufshcd_set_eh_in_progress(hba);
4060        spin_unlock_irqrestore(hba->host->host_lock, flags);
4061
4062        /* Reset the attached device */
4063        ufshcd_device_reset(hba);
4064
4065        ret = ufshcd_host_reset_and_restore(hba);
4066
4067        spin_lock_irqsave(hba->host->host_lock, flags);
4068        if (ret)
4069                hba->ufshcd_state = UFSHCD_STATE_ERROR;
4070        ufshcd_clear_eh_in_progress(hba);
4071        spin_unlock_irqrestore(hba->host->host_lock, flags);
4072
4073        if (ret)
4074                dev_err(hba->dev, "%s: link recovery failed, err %d",
4075                        __func__, ret);
4076        else
4077                ufshcd_clear_ua_wluns(hba);
4078
4079        return ret;
4080}
4081EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4082
4083static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4084{
4085        int ret;
4086        struct uic_command uic_cmd = {0};
4087        ktime_t start = ktime_get();
4088
4089        ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4090
4091        uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
4092        ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4093        trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4094                             ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4095
4096        if (ret)
4097                dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4098                        __func__, ret);
4099        else
4100                ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4101                                                                POST_CHANGE);
4102
4103        return ret;
4104}
4105
4106int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4107{
4108        struct uic_command uic_cmd = {0};
4109        int ret;
4110        ktime_t start = ktime_get();
4111
4112        ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4113
4114        uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
4115        ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4116        trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4117                             ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4118
4119        if (ret) {
4120                dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4121                        __func__, ret);
4122        } else {
4123                ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4124                                                                POST_CHANGE);
4125                hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
4126                hba->ufs_stats.hibern8_exit_cnt++;
4127        }
4128
4129        return ret;
4130}
4131EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4132
4133void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4134{
4135        unsigned long flags;
4136        bool update = false;
4137
4138        if (!ufshcd_is_auto_hibern8_supported(hba))
4139                return;
4140
4141        spin_lock_irqsave(hba->host->host_lock, flags);
4142        if (hba->ahit != ahit) {
4143                hba->ahit = ahit;
4144                update = true;
4145        }
4146        spin_unlock_irqrestore(hba->host->host_lock, flags);
4147
4148        if (update &&
4149            !pm_runtime_suspended(&hba->sdev_ufs_device->sdev_gendev)) {
4150                ufshcd_rpm_get_sync(hba);
4151                ufshcd_hold(hba, false);
4152                ufshcd_auto_hibern8_enable(hba);
4153                ufshcd_release(hba);
4154                ufshcd_rpm_put_sync(hba);
4155        }
4156}
4157EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4158
4159void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
4160{
4161        unsigned long flags;
4162
4163        if (!ufshcd_is_auto_hibern8_supported(hba))
4164                return;
4165
4166        spin_lock_irqsave(hba->host->host_lock, flags);
4167        ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4168        spin_unlock_irqrestore(hba->host->host_lock, flags);
4169}
4170
4171 /**
4172 * ufshcd_init_pwr_info - setting the POR (power on reset)
4173 * values in hba power info
4174 * @hba: per-adapter instance
4175 */
4176static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4177{
4178        hba->pwr_info.gear_rx = UFS_PWM_G1;
4179        hba->pwr_info.gear_tx = UFS_PWM_G1;
4180        hba->pwr_info.lane_rx = 1;
4181        hba->pwr_info.lane_tx = 1;
4182        hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4183        hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4184        hba->pwr_info.hs_rate = 0;
4185}
4186
4187/**
4188 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4189 * @hba: per-adapter instance
4190 */
4191static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4192{
4193        struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4194
4195        if (hba->max_pwr_info.is_valid)
4196                return 0;
4197
4198        pwr_info->pwr_tx = FAST_MODE;
4199        pwr_info->pwr_rx = FAST_MODE;
4200        pwr_info->hs_rate = PA_HS_MODE_B;
4201
4202        /* Get the connected lane count */
4203        ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4204                        &pwr_info->lane_rx);
4205        ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4206                        &pwr_info->lane_tx);
4207
4208        if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4209                dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4210                                __func__,
4211                                pwr_info->lane_rx,
4212                                pwr_info->lane_tx);
4213                return -EINVAL;
4214        }
4215
4216        /*
4217         * First, get the maximum gears of HS speed.
4218         * If a zero value, it means there is no HSGEAR capability.
4219         * Then, get the maximum gears of PWM speed.
4220         */
4221        ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4222        if (!pwr_info->gear_rx) {
4223                ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4224                                &pwr_info->gear_rx);
4225                if (!pwr_info->gear_rx) {
4226                        dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4227                                __func__, pwr_info->gear_rx);
4228                        return -EINVAL;
4229                }
4230                pwr_info->pwr_rx = SLOW_MODE;
4231        }
4232
4233        ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4234                        &pwr_info->gear_tx);
4235        if (!pwr_info->gear_tx) {
4236                ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4237                                &pwr_info->gear_tx);
4238                if (!pwr_info->gear_tx) {
4239                        dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4240                                __func__, pwr_info->gear_tx);
4241                        return -EINVAL;
4242                }
4243                pwr_info->pwr_tx = SLOW_MODE;
4244        }
4245
4246        hba->max_pwr_info.is_valid = true;
4247        return 0;
4248}
4249
4250static int ufshcd_change_power_mode(struct ufs_hba *hba,
4251                             struct ufs_pa_layer_attr *pwr_mode)
4252{
4253        int ret;
4254
4255        /* if already configured to the requested pwr_mode */
4256        if (!hba->force_pmc &&
4257            pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4258            pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4259            pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4260            pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4261            pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4262            pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4263            pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4264                dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4265                return 0;
4266        }
4267
4268        /*
4269         * Configure attributes for power mode change with below.
4270         * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4271         * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4272         * - PA_HSSERIES
4273         */
4274        ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4275        ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4276                        pwr_mode->lane_rx);
4277        if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4278                        pwr_mode->pwr_rx == FAST_MODE)
4279                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
4280        else
4281                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
4282
4283        ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4284        ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4285                        pwr_mode->lane_tx);
4286        if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4287                        pwr_mode->pwr_tx == FAST_MODE)
4288                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
4289        else
4290                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
4291
4292        if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4293            pwr_mode->pwr_tx == FASTAUTO_MODE ||
4294            pwr_mode->pwr_rx == FAST_MODE ||
4295            pwr_mode->pwr_tx == FAST_MODE)
4296                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4297                                                pwr_mode->hs_rate);
4298
4299        if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4300                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4301                                DL_FC0ProtectionTimeOutVal_Default);
4302                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4303                                DL_TC0ReplayTimeOutVal_Default);
4304                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4305                                DL_AFC0ReqTimeOutVal_Default);
4306                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4307                                DL_FC1ProtectionTimeOutVal_Default);
4308                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4309                                DL_TC1ReplayTimeOutVal_Default);
4310                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4311                                DL_AFC1ReqTimeOutVal_Default);
4312
4313                ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4314                                DL_FC0ProtectionTimeOutVal_Default);
4315                ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4316                                DL_TC0ReplayTimeOutVal_Default);
4317                ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4318                                DL_AFC0ReqTimeOutVal_Default);
4319        }
4320
4321        ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4322                        | pwr_mode->pwr_tx);
4323
4324        if (ret) {
4325                dev_err(hba->dev,
4326                        "%s: power mode change failed %d\n", __func__, ret);
4327        } else {
4328                ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4329                                                                pwr_mode);
4330
4331                memcpy(&hba->pwr_info, pwr_mode,
4332                        sizeof(struct ufs_pa_layer_attr));
4333        }
4334
4335        return ret;
4336}
4337
4338/**
4339 * ufshcd_config_pwr_mode - configure a new power mode
4340 * @hba: per-adapter instance
4341 * @desired_pwr_mode: desired power configuration
4342 */
4343int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4344                struct ufs_pa_layer_attr *desired_pwr_mode)
4345{
4346        struct ufs_pa_layer_attr final_params = { 0 };
4347        int ret;
4348
4349        ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4350                                        desired_pwr_mode, &final_params);
4351
4352        if (ret)
4353                memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4354
4355        ret = ufshcd_change_power_mode(hba, &final_params);
4356
4357        return ret;
4358}
4359EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4360
4361/**
4362 * ufshcd_complete_dev_init() - checks device readiness
4363 * @hba: per-adapter instance
4364 *
4365 * Set fDeviceInit flag and poll until device toggles it.
4366 */
4367static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4368{
4369        int err;
4370        bool flag_res = true;
4371        ktime_t timeout;
4372
4373        err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4374                QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4375        if (err) {
4376                dev_err(hba->dev,
4377                        "%s setting fDeviceInit flag failed with error %d\n",
4378                        __func__, err);
4379                goto out;
4380        }
4381
4382        /* Poll fDeviceInit flag to be cleared */
4383        timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4384        do {
4385                err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4386                                        QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4387                if (!flag_res)
4388                        break;
4389                usleep_range(5000, 10000);
4390        } while (ktime_before(ktime_get(), timeout));
4391
4392        if (err) {
4393                dev_err(hba->dev,
4394                                "%s reading fDeviceInit flag failed with error %d\n",
4395                                __func__, err);
4396        } else if (flag_res) {
4397                dev_err(hba->dev,
4398                                "%s fDeviceInit was not cleared by the device\n",
4399                                __func__);
4400                err = -EBUSY;
4401        }
4402out:
4403        return err;
4404}
4405
4406/**
4407 * ufshcd_make_hba_operational - Make UFS controller operational
4408 * @hba: per adapter instance
4409 *
4410 * To bring UFS host controller to operational state,
4411 * 1. Enable required interrupts
4412 * 2. Configure interrupt aggregation
4413 * 3. Program UTRL and UTMRL base address
4414 * 4. Configure run-stop-registers
4415 *
4416 * Returns 0 on success, non-zero value on failure
4417 */
4418int ufshcd_make_hba_operational(struct ufs_hba *hba)
4419{
4420        int err = 0;
4421        u32 reg;
4422
4423        /* Enable required interrupts */
4424        ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4425
4426        /* Configure interrupt aggregation */
4427        if (ufshcd_is_intr_aggr_allowed(hba))
4428                ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4429        else
4430                ufshcd_disable_intr_aggr(hba);
4431
4432        /* Configure UTRL and UTMRL base address registers */
4433        ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4434                        REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4435        ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4436                        REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4437        ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4438                        REG_UTP_TASK_REQ_LIST_BASE_L);
4439        ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4440                        REG_UTP_TASK_REQ_LIST_BASE_H);
4441
4442        /*
4443         * Make sure base address and interrupt setup are updated before
4444         * enabling the run/stop registers below.
4445         */
4446        wmb();
4447
4448        /*
4449         * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4450         */
4451        reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4452        if (!(ufshcd_get_lists_status(reg))) {
4453                ufshcd_enable_run_stop_reg(hba);
4454        } else {
4455                dev_err(hba->dev,
4456                        "Host controller not ready to process requests");
4457                err = -EIO;
4458        }
4459
4460        return err;
4461}
4462EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4463
4464/**
4465 * ufshcd_hba_stop - Send controller to reset state
4466 * @hba: per adapter instance
4467 */
4468void ufshcd_hba_stop(struct ufs_hba *hba)
4469{
4470        unsigned long flags;
4471        int err;
4472
4473        /*
4474         * Obtain the host lock to prevent that the controller is disabled
4475         * while the UFS interrupt handler is active on another CPU.
4476         */
4477        spin_lock_irqsave(hba->host->host_lock, flags);
4478        ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
4479        spin_unlock_irqrestore(hba->host->host_lock, flags);
4480
4481        err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4482                                        CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4483                                        10, 1);
4484        if (err)
4485                dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4486}
4487EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4488
4489/**
4490 * ufshcd_hba_execute_hce - initialize the controller
4491 * @hba: per adapter instance
4492 *
4493 * The controller resets itself and controller firmware initialization
4494 * sequence kicks off. When controller is ready it will set
4495 * the Host Controller Enable bit to 1.
4496 *
4497 * Returns 0 on success, non-zero value on failure
4498 */
4499static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4500{
4501        int retry_outer = 3;
4502        int retry_inner;
4503
4504start:
4505        if (!ufshcd_is_hba_active(hba))
4506                /* change controller state to "reset state" */
4507                ufshcd_hba_stop(hba);
4508
4509        /* UniPro link is disabled at this point */
4510        ufshcd_set_link_off(hba);
4511
4512        ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4513
4514        /* start controller initialization sequence */
4515        ufshcd_hba_start(hba);
4516
4517        /*
4518         * To initialize a UFS host controller HCE bit must be set to 1.
4519         * During initialization the HCE bit value changes from 1->0->1.
4520         * When the host controller completes initialization sequence
4521         * it sets the value of HCE bit to 1. The same HCE bit is read back
4522         * to check if the controller has completed initialization sequence.
4523         * So without this delay the value HCE = 1, set in the previous
4524         * instruction might be read back.
4525         * This delay can be changed based on the controller.
4526         */
4527        ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4528
4529        /* wait for the host controller to complete initialization */
4530        retry_inner = 50;
4531        while (ufshcd_is_hba_active(hba)) {
4532                if (retry_inner) {
4533                        retry_inner--;
4534                } else {
4535                        dev_err(hba->dev,
4536                                "Controller enable failed\n");
4537                        if (retry_outer) {
4538                                retry_outer--;
4539                                goto start;
4540                        }
4541                        return -EIO;
4542                }
4543                usleep_range(1000, 1100);
4544        }
4545
4546        /* enable UIC related interrupts */
4547        ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4548
4549        ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4550
4551        return 0;
4552}
4553
4554int ufshcd_hba_enable(struct ufs_hba *hba)
4555{
4556        int ret;
4557
4558        if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4559                ufshcd_set_link_off(hba);
4560                ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4561
4562                /* enable UIC related interrupts */
4563                ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4564                ret = ufshcd_dme_reset(hba);
4565                if (!ret) {
4566                        ret = ufshcd_dme_enable(hba);
4567                        if (!ret)
4568                                ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4569                        if (ret)
4570                                dev_err(hba->dev,
4571                                        "Host controller enable failed with non-hce\n");
4572                }
4573        } else {
4574                ret = ufshcd_hba_execute_hce(hba);
4575        }
4576
4577        return ret;
4578}
4579EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4580
4581static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4582{
4583        int tx_lanes = 0, i, err = 0;
4584
4585        if (!peer)
4586                ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4587                               &tx_lanes);
4588        else
4589                ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4590                                    &tx_lanes);
4591        for (i = 0; i < tx_lanes; i++) {
4592                if (!peer)
4593                        err = ufshcd_dme_set(hba,
4594                                UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4595                                        UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4596                                        0);
4597                else
4598                        err = ufshcd_dme_peer_set(hba,
4599                                UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4600                                        UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4601                                        0);
4602                if (err) {
4603                        dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4604                                __func__, peer, i, err);
4605                        break;
4606                }
4607        }
4608
4609        return err;
4610}
4611
4612static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4613{
4614        return ufshcd_disable_tx_lcc(hba, true);
4615}
4616
4617void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4618{
4619        struct ufs_event_hist *e;
4620
4621        if (id >= UFS_EVT_CNT)
4622                return;
4623
4624        e = &hba->ufs_stats.event[id];
4625        e->val[e->pos] = val;
4626        e->tstamp[e->pos] = ktime_get();
4627        e->cnt += 1;
4628        e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4629
4630        ufshcd_vops_event_notify(hba, id, &val);
4631}
4632EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4633
4634/**
4635 * ufshcd_link_startup - Initialize unipro link startup
4636 * @hba: per adapter instance
4637 *
4638 * Returns 0 for success, non-zero in case of failure
4639 */
4640static int ufshcd_link_startup(struct ufs_hba *hba)
4641{
4642        int ret;
4643        int retries = DME_LINKSTARTUP_RETRIES;
4644        bool link_startup_again = false;
4645
4646        /*
4647         * If UFS device isn't active then we will have to issue link startup
4648         * 2 times to make sure the device state move to active.
4649         */
4650        if (!ufshcd_is_ufs_dev_active(hba))
4651                link_startup_again = true;
4652
4653link_startup:
4654        do {
4655                ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4656
4657                ret = ufshcd_dme_link_startup(hba);
4658
4659                /* check if device is detected by inter-connect layer */
4660                if (!ret && !ufshcd_is_device_present(hba)) {
4661                        ufshcd_update_evt_hist(hba,
4662                                               UFS_EVT_LINK_STARTUP_FAIL,
4663                                               0);
4664                        dev_err(hba->dev, "%s: Device not present\n", __func__);
4665                        ret = -ENXIO;
4666                        goto out;
4667                }
4668
4669                /*
4670                 * DME link lost indication is only received when link is up,
4671                 * but we can't be sure if the link is up until link startup
4672                 * succeeds. So reset the local Uni-Pro and try again.
4673                 */
4674                if (ret && ufshcd_hba_enable(hba)) {
4675                        ufshcd_update_evt_hist(hba,
4676                                               UFS_EVT_LINK_STARTUP_FAIL,
4677                                               (u32)ret);
4678                        goto out;
4679                }
4680        } while (ret && retries--);
4681
4682        if (ret) {
4683                /* failed to get the link up... retire */
4684                ufshcd_update_evt_hist(hba,
4685                                       UFS_EVT_LINK_STARTUP_FAIL,
4686                                       (u32)ret);
4687                goto out;
4688        }
4689
4690        if (link_startup_again) {
4691                link_startup_again = false;
4692                retries = DME_LINKSTARTUP_RETRIES;
4693                goto link_startup;
4694        }
4695
4696        /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4697        ufshcd_init_pwr_info(hba);
4698        ufshcd_print_pwr_info(hba);
4699
4700        if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4701                ret = ufshcd_disable_device_tx_lcc(hba);
4702                if (ret)
4703                        goto out;
4704        }
4705
4706        /* Include any host controller configuration via UIC commands */
4707        ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4708        if (ret)
4709                goto out;
4710
4711        /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4712        ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
4713        ret = ufshcd_make_hba_operational(hba);
4714out:
4715        if (ret) {
4716                dev_err(hba->dev, "link startup failed %d\n", ret);
4717                ufshcd_print_host_state(hba);
4718                ufshcd_print_pwr_info(hba);
4719                ufshcd_print_evt_hist(hba);
4720        }
4721        return ret;
4722}
4723
4724/**
4725 * ufshcd_verify_dev_init() - Verify device initialization
4726 * @hba: per-adapter instance
4727 *
4728 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4729 * device Transport Protocol (UTP) layer is ready after a reset.
4730 * If the UTP layer at the device side is not initialized, it may
4731 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4732 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4733 */
4734static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4735{
4736        int err = 0;
4737        int retries;
4738
4739        ufshcd_hold(hba, false);
4740        mutex_lock(&hba->dev_cmd.lock);
4741        for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4742                err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4743                                          hba->nop_out_timeout);
4744
4745                if (!err || err == -ETIMEDOUT)
4746                        break;
4747
4748                dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4749        }
4750        mutex_unlock(&hba->dev_cmd.lock);
4751        ufshcd_release(hba);
4752
4753        if (err)
4754                dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4755        return err;
4756}
4757
4758/**
4759 * ufshcd_set_queue_depth - set lun queue depth
4760 * @sdev: pointer to SCSI device
4761 *
4762 * Read bLUQueueDepth value and activate scsi tagged command
4763 * queueing. For WLUN, queue depth is set to 1. For best-effort
4764 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4765 * value that host can queue.
4766 */
4767static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4768{
4769        int ret = 0;
4770        u8 lun_qdepth;
4771        struct ufs_hba *hba;
4772
4773        hba = shost_priv(sdev->host);
4774
4775        lun_qdepth = hba->nutrs;
4776        ret = ufshcd_read_unit_desc_param(hba,
4777                                          ufshcd_scsi_to_upiu_lun(sdev->lun),
4778                                          UNIT_DESC_PARAM_LU_Q_DEPTH,
4779                                          &lun_qdepth,
4780                                          sizeof(lun_qdepth));
4781
4782        /* Some WLUN doesn't support unit descriptor */
4783        if (ret == -EOPNOTSUPP)
4784                lun_qdepth = 1;
4785        else if (!lun_qdepth)
4786                /* eventually, we can figure out the real queue depth */
4787                lun_qdepth = hba->nutrs;
4788        else
4789                lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4790
4791        dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4792                        __func__, lun_qdepth);
4793        scsi_change_queue_depth(sdev, lun_qdepth);
4794}
4795
4796/*
4797 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4798 * @hba: per-adapter instance
4799 * @lun: UFS device lun id
4800 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4801 *
4802 * Returns 0 in case of success and b_lu_write_protect status would be returned
4803 * @b_lu_write_protect parameter.
4804 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4805 * Returns -EINVAL in case of invalid parameters passed to this function.
4806 */
4807static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4808                            u8 lun,
4809                            u8 *b_lu_write_protect)
4810{
4811        int ret;
4812
4813        if (!b_lu_write_protect)
4814                ret = -EINVAL;
4815        /*
4816         * According to UFS device spec, RPMB LU can't be write
4817         * protected so skip reading bLUWriteProtect parameter for
4818         * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4819         */
4820        else if (lun >= hba->dev_info.max_lu_supported)
4821                ret = -ENOTSUPP;
4822        else
4823                ret = ufshcd_read_unit_desc_param(hba,
4824                                          lun,
4825                                          UNIT_DESC_PARAM_LU_WR_PROTECT,
4826                                          b_lu_write_protect,
4827                                          sizeof(*b_lu_write_protect));
4828        return ret;
4829}
4830
4831/**
4832 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4833 * status
4834 * @hba: per-adapter instance
4835 * @sdev: pointer to SCSI device
4836 *
4837 */
4838static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4839                                                    struct scsi_device *sdev)
4840{
4841        if (hba->dev_info.f_power_on_wp_en &&
4842            !hba->dev_info.is_lu_power_on_wp) {
4843                u8 b_lu_write_protect;
4844
4845                if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4846                                      &b_lu_write_protect) &&
4847                    (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4848                        hba->dev_info.is_lu_power_on_wp = true;
4849        }
4850}
4851
4852/**
4853 * ufshcd_setup_links - associate link b/w device wlun and other luns
4854 * @sdev: pointer to SCSI device
4855 * @hba: pointer to ufs hba
4856 */
4857static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
4858{
4859        struct device_link *link;
4860
4861        /*
4862         * Device wlun is the supplier & rest of the luns are consumers.
4863         * This ensures that device wlun suspends after all other luns.
4864         */
4865        if (hba->sdev_ufs_device) {
4866                link = device_link_add(&sdev->sdev_gendev,
4867                                       &hba->sdev_ufs_device->sdev_gendev,
4868                                       DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
4869                if (!link) {
4870                        dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
4871                                dev_name(&hba->sdev_ufs_device->sdev_gendev));
4872                        return;
4873                }
4874                hba->luns_avail--;
4875                /* Ignore REPORT_LUN wlun probing */
4876                if (hba->luns_avail == 1) {
4877                        ufshcd_rpm_put(hba);
4878                        return;
4879                }
4880        } else {
4881                /*
4882                 * Device wlun is probed. The assumption is that WLUNs are
4883                 * scanned before other LUNs.
4884                 */
4885                hba->luns_avail--;
4886        }
4887}
4888
4889/**
4890 * ufshcd_slave_alloc - handle initial SCSI device configurations
4891 * @sdev: pointer to SCSI device
4892 *
4893 * Returns success
4894 */
4895static int ufshcd_slave_alloc(struct scsi_device *sdev)
4896{
4897        struct ufs_hba *hba;
4898
4899        hba = shost_priv(sdev->host);
4900
4901        /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4902        sdev->use_10_for_ms = 1;
4903
4904        /* DBD field should be set to 1 in mode sense(10) */
4905        sdev->set_dbd_for_ms = 1;
4906
4907        /* allow SCSI layer to restart the device in case of errors */
4908        sdev->allow_restart = 1;
4909
4910        /* REPORT SUPPORTED OPERATION CODES is not supported */
4911        sdev->no_report_opcodes = 1;
4912
4913        /* WRITE_SAME command is not supported */
4914        sdev->no_write_same = 1;
4915
4916        ufshcd_set_queue_depth(sdev);
4917
4918        ufshcd_get_lu_power_on_wp_status(hba, sdev);
4919
4920        ufshcd_setup_links(hba, sdev);
4921
4922        return 0;
4923}
4924
4925/**
4926 * ufshcd_change_queue_depth - change queue depth
4927 * @sdev: pointer to SCSI device
4928 * @depth: required depth to set
4929 *
4930 * Change queue depth and make sure the max. limits are not crossed.
4931 */
4932static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4933{
4934        struct ufs_hba *hba = shost_priv(sdev->host);
4935
4936        if (depth > hba->nutrs)
4937                depth = hba->nutrs;
4938        return scsi_change_queue_depth(sdev, depth);
4939}
4940
4941static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev)
4942{
4943        /* skip well-known LU */
4944        if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
4945            !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
4946                return;
4947
4948        ufshpb_destroy_lu(hba, sdev);
4949}
4950
4951static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev)
4952{
4953        /* skip well-known LU */
4954        if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
4955            !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
4956                return;
4957
4958        ufshpb_init_hpb_lu(hba, sdev);
4959}
4960
4961/**
4962 * ufshcd_slave_configure - adjust SCSI device configurations
4963 * @sdev: pointer to SCSI device
4964 */
4965static int ufshcd_slave_configure(struct scsi_device *sdev)
4966{
4967        struct ufs_hba *hba = shost_priv(sdev->host);
4968        struct request_queue *q = sdev->request_queue;
4969
4970        ufshcd_hpb_configure(hba, sdev);
4971
4972        blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
4973        if (hba->quirks & UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE)
4974                blk_queue_update_dma_alignment(q, PAGE_SIZE - 1);
4975        /*
4976         * Block runtime-pm until all consumers are added.
4977         * Refer ufshcd_setup_links().
4978         */
4979        if (is_device_wlun(sdev))
4980                pm_runtime_get_noresume(&sdev->sdev_gendev);
4981        else if (ufshcd_is_rpm_autosuspend_allowed(hba))
4982                sdev->rpm_autosuspend = 1;
4983
4984        ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
4985
4986        return 0;
4987}
4988
4989/**
4990 * ufshcd_slave_destroy - remove SCSI device configurations
4991 * @sdev: pointer to SCSI device
4992 */
4993static void ufshcd_slave_destroy(struct scsi_device *sdev)
4994{
4995        struct ufs_hba *hba;
4996        unsigned long flags;
4997
4998        hba = shost_priv(sdev->host);
4999
5000        ufshcd_hpb_destroy(hba, sdev);
5001
5002        /* Drop the reference as it won't be needed anymore */
5003        if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5004                spin_lock_irqsave(hba->host->host_lock, flags);
5005                hba->sdev_ufs_device = NULL;
5006                spin_unlock_irqrestore(hba->host->host_lock, flags);
5007        } else if (hba->sdev_ufs_device) {
5008                struct device *supplier = NULL;
5009
5010                /* Ensure UFS Device WLUN exists and does not disappear */
5011                spin_lock_irqsave(hba->host->host_lock, flags);
5012                if (hba->sdev_ufs_device) {
5013                        supplier = &hba->sdev_ufs_device->sdev_gendev;
5014                        get_device(supplier);
5015                }
5016                spin_unlock_irqrestore(hba->host->host_lock, flags);
5017
5018                if (supplier) {
5019                        /*
5020                         * If a LUN fails to probe (e.g. absent BOOT WLUN), the
5021                         * device will not have been registered but can still
5022                         * have a device link holding a reference to the device.
5023                         */
5024                        device_link_remove(&sdev->sdev_gendev, supplier);
5025                        put_device(supplier);
5026                }
5027        }
5028}
5029
5030/**
5031 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5032 * @lrbp: pointer to local reference block of completed command
5033 * @scsi_status: SCSI command status
5034 *
5035 * Returns value base on SCSI command status
5036 */
5037static inline int
5038ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5039{
5040        int result = 0;
5041
5042        switch (scsi_status) {
5043        case SAM_STAT_CHECK_CONDITION:
5044                ufshcd_copy_sense_data(lrbp);
5045                fallthrough;
5046        case SAM_STAT_GOOD:
5047                result |= DID_OK << 16 | scsi_status;
5048                break;
5049        case SAM_STAT_TASK_SET_FULL:
5050        case SAM_STAT_BUSY:
5051        case SAM_STAT_TASK_ABORTED:
5052                ufshcd_copy_sense_data(lrbp);
5053                result |= scsi_status;
5054                break;
5055        default:
5056                result |= DID_ERROR << 16;
5057                break;
5058        } /* end of switch */
5059
5060        return result;
5061}
5062
5063/**
5064 * ufshcd_transfer_rsp_status - Get overall status of the response
5065 * @hba: per adapter instance
5066 * @lrbp: pointer to local reference block of completed command
5067 *
5068 * Returns result of the command to notify SCSI midlayer
5069 */
5070static inline int
5071ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
5072{
5073        int result = 0;
5074        int scsi_status;
5075        int ocs;
5076
5077        /* overall command status of utrd */
5078        ocs = ufshcd_get_tr_ocs(lrbp);
5079
5080        if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5081                if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
5082                                        MASK_RSP_UPIU_RESULT)
5083                        ocs = OCS_SUCCESS;
5084        }
5085
5086        switch (ocs) {
5087        case OCS_SUCCESS:
5088                result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
5089                hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5090                switch (result) {
5091                case UPIU_TRANSACTION_RESPONSE:
5092                        /*
5093                         * get the response UPIU result to extract
5094                         * the SCSI command status
5095                         */
5096                        result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
5097
5098                        /*
5099                         * get the result based on SCSI status response
5100                         * to notify the SCSI midlayer of the command status
5101                         */
5102                        scsi_status = result & MASK_SCSI_STATUS;
5103                        result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5104
5105                        /*
5106                         * Currently we are only supporting BKOPs exception
5107                         * events hence we can ignore BKOPs exception event
5108                         * during power management callbacks. BKOPs exception
5109                         * event is not expected to be raised in runtime suspend
5110                         * callback as it allows the urgent bkops.
5111                         * During system suspend, we are anyway forcefully
5112                         * disabling the bkops and if urgent bkops is needed
5113                         * it will be enabled on system resume. Long term
5114                         * solution could be to abort the system suspend if
5115                         * UFS device needs urgent BKOPs.
5116                         */
5117                        if (!hba->pm_op_in_progress &&
5118                            !ufshcd_eh_in_progress(hba) &&
5119                            ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5120                                /* Flushed in suspend */
5121                                schedule_work(&hba->eeh_work);
5122
5123                        if (scsi_status == SAM_STAT_GOOD)
5124                                ufshpb_rsp_upiu(hba, lrbp);
5125                        break;
5126                case UPIU_TRANSACTION_REJECT_UPIU:
5127                        /* TODO: handle Reject UPIU Response */
5128                        result = DID_ERROR << 16;
5129                        dev_err(hba->dev,
5130                                "Reject UPIU not fully implemented\n");
5131                        break;
5132                default:
5133                        dev_err(hba->dev,
5134                                "Unexpected request response code = %x\n",
5135                                result);
5136                        result = DID_ERROR << 16;
5137                        break;
5138                }
5139                break;
5140        case OCS_ABORTED:
5141                result |= DID_ABORT << 16;
5142                break;
5143        case OCS_INVALID_COMMAND_STATUS:
5144                result |= DID_REQUEUE << 16;
5145                break;
5146        case OCS_INVALID_CMD_TABLE_ATTR:
5147        case OCS_INVALID_PRDT_ATTR:
5148        case OCS_MISMATCH_DATA_BUF_SIZE:
5149        case OCS_MISMATCH_RESP_UPIU_SIZE:
5150        case OCS_PEER_COMM_FAILURE:
5151        case OCS_FATAL_ERROR:
5152        case OCS_DEVICE_FATAL_ERROR:
5153        case OCS_INVALID_CRYPTO_CONFIG:
5154        case OCS_GENERAL_CRYPTO_ERROR:
5155        default:
5156                result |= DID_ERROR << 16;
5157                dev_err(hba->dev,
5158                                "OCS error from controller = %x for tag %d\n",
5159                                ocs, lrbp->task_tag);
5160                ufshcd_print_evt_hist(hba);
5161                ufshcd_print_host_state(hba);
5162                break;
5163        } /* end of switch */
5164
5165        if ((host_byte(result) != DID_OK) &&
5166            (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5167                ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
5168        return result;
5169}
5170
5171static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5172                                         u32 intr_mask)
5173{
5174        if (!ufshcd_is_auto_hibern8_supported(hba) ||
5175            !ufshcd_is_auto_hibern8_enabled(hba))
5176                return false;
5177
5178        if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5179                return false;
5180
5181        if (hba->active_uic_cmd &&
5182            (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5183            hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5184                return false;
5185
5186        return true;
5187}
5188
5189/**
5190 * ufshcd_uic_cmd_compl - handle completion of uic command
5191 * @hba: per adapter instance
5192 * @intr_status: interrupt status generated by the controller
5193 *
5194 * Returns
5195 *  IRQ_HANDLED - If interrupt is valid
5196 *  IRQ_NONE    - If invalid interrupt
5197 */
5198static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5199{
5200        irqreturn_t retval = IRQ_NONE;
5201
5202        spin_lock(hba->host->host_lock);
5203        if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5204                hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5205
5206        if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5207                hba->active_uic_cmd->argument2 |=
5208                        ufshcd_get_uic_cmd_result(hba);
5209                hba->active_uic_cmd->argument3 =
5210                        ufshcd_get_dme_attr_val(hba);
5211                if (!hba->uic_async_done)
5212                        hba->active_uic_cmd->cmd_active = 0;
5213                complete(&hba->active_uic_cmd->done);
5214                retval = IRQ_HANDLED;
5215        }
5216
5217        if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5218                hba->active_uic_cmd->cmd_active = 0;
5219                complete(hba->uic_async_done);
5220                retval = IRQ_HANDLED;
5221        }
5222
5223        if (retval == IRQ_HANDLED)
5224                ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5225                                             UFS_CMD_COMP);
5226        spin_unlock(hba->host->host_lock);
5227        return retval;
5228}
5229
5230/**
5231 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5232 * @hba: per adapter instance
5233 * @completed_reqs: bitmask that indicates which requests to complete
5234 * @retry_requests: whether to ask the SCSI core to retry completed requests
5235 */
5236static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5237                                        unsigned long completed_reqs,
5238                                        bool retry_requests)
5239{
5240        struct ufshcd_lrb *lrbp;
5241        struct scsi_cmnd *cmd;
5242        int result;
5243        int index;
5244        bool update_scaling = false;
5245
5246        for_each_set_bit(index, &completed_reqs, hba->nutrs) {
5247                lrbp = &hba->lrb[index];
5248                lrbp->compl_time_stamp = ktime_get();
5249                cmd = lrbp->cmd;
5250                if (cmd) {
5251                        if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5252                                ufshcd_update_monitor(hba, lrbp);
5253                        ufshcd_add_command_trace(hba, index, UFS_CMD_COMP);
5254                        result = retry_requests ? DID_BUS_BUSY << 16 :
5255                                ufshcd_transfer_rsp_status(hba, lrbp);
5256                        scsi_dma_unmap(cmd);
5257                        cmd->result = result;
5258                        /* Mark completed command as NULL in LRB */
5259                        lrbp->cmd = NULL;
5260                        /* Do not touch lrbp after scsi done */
5261                        cmd->scsi_done(cmd);
5262                        ufshcd_release(hba);
5263                        update_scaling = true;
5264                } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5265                        lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
5266                        if (hba->dev_cmd.complete) {
5267                                ufshcd_add_command_trace(hba, index,
5268                                                         UFS_DEV_COMP);
5269                                complete(hba->dev_cmd.complete);
5270                                update_scaling = true;
5271                        }
5272                }
5273                if (update_scaling)
5274                        ufshcd_clk_scaling_update_busy(hba);
5275        }
5276}
5277
5278/**
5279 * ufshcd_transfer_req_compl - handle SCSI and query command completion
5280 * @hba: per adapter instance
5281 * @retry_requests: whether or not to ask to retry requests
5282 *
5283 * Returns
5284 *  IRQ_HANDLED - If interrupt is valid
5285 *  IRQ_NONE    - If invalid interrupt
5286 */
5287static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba,
5288                                             bool retry_requests)
5289{
5290        unsigned long completed_reqs, flags;
5291        u32 tr_doorbell;
5292
5293        /* Resetting interrupt aggregation counters first and reading the
5294         * DOOR_BELL afterward allows us to handle all the completed requests.
5295         * In order to prevent other interrupts starvation the DB is read once
5296         * after reset. The down side of this solution is the possibility of
5297         * false interrupt if device completes another request after resetting
5298         * aggregation and before reading the DB.
5299         */
5300        if (ufshcd_is_intr_aggr_allowed(hba) &&
5301            !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5302                ufshcd_reset_intr_aggr(hba);
5303
5304        if (ufs_fail_completion())
5305                return IRQ_HANDLED;
5306
5307        spin_lock_irqsave(&hba->outstanding_lock, flags);
5308        tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5309        completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5310        WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5311                  "completed: %#lx; outstanding: %#lx\n", completed_reqs,
5312                  hba->outstanding_reqs);
5313        hba->outstanding_reqs &= ~completed_reqs;
5314        spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5315
5316        if (completed_reqs) {
5317                __ufshcd_transfer_req_compl(hba, completed_reqs,
5318                                            retry_requests);
5319                return IRQ_HANDLED;
5320        } else {
5321                return IRQ_NONE;
5322        }
5323}
5324
5325int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
5326{
5327        return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5328                                       QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5329                                       &ee_ctrl_mask);
5330}
5331
5332int ufshcd_write_ee_control(struct ufs_hba *hba)
5333{
5334        int err;
5335
5336        mutex_lock(&hba->ee_ctrl_mutex);
5337        err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5338        mutex_unlock(&hba->ee_ctrl_mutex);
5339        if (err)
5340                dev_err(hba->dev, "%s: failed to write ee control %d\n",
5341                        __func__, err);
5342        return err;
5343}
5344
5345int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, u16 *other_mask,
5346                             u16 set, u16 clr)
5347{
5348