linux/drivers/target/iscsi/iscsi_target_util.c
<<
>>
Prefs
   1/*******************************************************************************
   2 * This file contains the iSCSI Target specific utility functions.
   3 *
   4 * (c) Copyright 2007-2013 Datera, Inc.
   5 *
   6 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   7 *
   8 * This program is free software; you can redistribute it and/or modify
   9 * it under the terms of the GNU General Public License as published by
  10 * the Free Software Foundation; either version 2 of the License, or
  11 * (at your option) any later version.
  12 *
  13 * This program is distributed in the hope that it will be useful,
  14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 * GNU General Public License for more details.
  17 ******************************************************************************/
  18
  19#include <linux/list.h>
  20#include <linux/percpu_ida.h>
  21#include <scsi/scsi_tcq.h>
  22#include <scsi/iscsi_proto.h>
  23#include <target/target_core_base.h>
  24#include <target/target_core_fabric.h>
  25#include <target/target_core_configfs.h>
  26#include <target/iscsi/iscsi_transport.h>
  27
  28#include "iscsi_target_core.h"
  29#include "iscsi_target_parameters.h"
  30#include "iscsi_target_seq_pdu_list.h"
  31#include "iscsi_target_datain_values.h"
  32#include "iscsi_target_erl0.h"
  33#include "iscsi_target_erl1.h"
  34#include "iscsi_target_erl2.h"
  35#include "iscsi_target_tpg.h"
  36#include "iscsi_target_tq.h"
  37#include "iscsi_target_util.h"
  38#include "iscsi_target.h"
  39
  40#define PRINT_BUFF(buff, len)                                   \
  41{                                                               \
  42        int zzz;                                                \
  43                                                                \
  44        pr_debug("%d:\n", __LINE__);                            \
  45        for (zzz = 0; zzz < len; zzz++) {                       \
  46                if (zzz % 16 == 0) {                            \
  47                        if (zzz)                                \
  48                                pr_debug("\n");                 \
  49                        pr_debug("%4i: ", zzz);                 \
  50                }                                               \
  51                pr_debug("%02x ", (unsigned char) (buff)[zzz]); \
  52        }                                                       \
  53        if ((len + 1) % 16)                                     \
  54                pr_debug("\n");                                 \
  55}
  56
  57extern struct list_head g_tiqn_list;
  58extern spinlock_t tiqn_lock;
  59
  60/*
  61 *      Called with cmd->r2t_lock held.
  62 */
  63int iscsit_add_r2t_to_list(
  64        struct iscsi_cmd *cmd,
  65        u32 offset,
  66        u32 xfer_len,
  67        int recovery,
  68        u32 r2t_sn)
  69{
  70        struct iscsi_r2t *r2t;
  71
  72        r2t = kmem_cache_zalloc(lio_r2t_cache, GFP_ATOMIC);
  73        if (!r2t) {
  74                pr_err("Unable to allocate memory for struct iscsi_r2t.\n");
  75                return -1;
  76        }
  77        INIT_LIST_HEAD(&r2t->r2t_list);
  78
  79        r2t->recovery_r2t = recovery;
  80        r2t->r2t_sn = (!r2t_sn) ? cmd->r2t_sn++ : r2t_sn;
  81        r2t->offset = offset;
  82        r2t->xfer_len = xfer_len;
  83        list_add_tail(&r2t->r2t_list, &cmd->cmd_r2t_list);
  84        spin_unlock_bh(&cmd->r2t_lock);
  85
  86        iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T);
  87
  88        spin_lock_bh(&cmd->r2t_lock);
  89        return 0;
  90}
  91
  92struct iscsi_r2t *iscsit_get_r2t_for_eos(
  93        struct iscsi_cmd *cmd,
  94        u32 offset,
  95        u32 length)
  96{
  97        struct iscsi_r2t *r2t;
  98
  99        spin_lock_bh(&cmd->r2t_lock);
 100        list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
 101                if ((r2t->offset <= offset) &&
 102                    (r2t->offset + r2t->xfer_len) >= (offset + length)) {
 103                        spin_unlock_bh(&cmd->r2t_lock);
 104                        return r2t;
 105                }
 106        }
 107        spin_unlock_bh(&cmd->r2t_lock);
 108
 109        pr_err("Unable to locate R2T for Offset: %u, Length:"
 110                        " %u\n", offset, length);
 111        return NULL;
 112}
 113
 114struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *cmd)
 115{
 116        struct iscsi_r2t *r2t;
 117
 118        spin_lock_bh(&cmd->r2t_lock);
 119        list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
 120                if (!r2t->sent_r2t) {
 121                        spin_unlock_bh(&cmd->r2t_lock);
 122                        return r2t;
 123                }
 124        }
 125        spin_unlock_bh(&cmd->r2t_lock);
 126
 127        pr_err("Unable to locate next R2T to send for ITT:"
 128                        " 0x%08x.\n", cmd->init_task_tag);
 129        return NULL;
 130}
 131
 132/*
 133 *      Called with cmd->r2t_lock held.
 134 */
 135void iscsit_free_r2t(struct iscsi_r2t *r2t, struct iscsi_cmd *cmd)
 136{
 137        list_del(&r2t->r2t_list);
 138        kmem_cache_free(lio_r2t_cache, r2t);
 139}
 140
 141void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd)
 142{
 143        struct iscsi_r2t *r2t, *r2t_tmp;
 144
 145        spin_lock_bh(&cmd->r2t_lock);
 146        list_for_each_entry_safe(r2t, r2t_tmp, &cmd->cmd_r2t_list, r2t_list)
 147                iscsit_free_r2t(r2t, cmd);
 148        spin_unlock_bh(&cmd->r2t_lock);
 149}
 150
 151/*
 152 * May be called from software interrupt (timer) context for allocating
 153 * iSCSI NopINs.
 154 */
 155struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, int state)
 156{
 157        struct iscsi_cmd *cmd;
 158        struct se_session *se_sess = conn->sess->se_sess;
 159        int size, tag;
 160
 161        tag = percpu_ida_alloc(&se_sess->sess_tag_pool, state);
 162        if (tag < 0)
 163                return NULL;
 164
 165        size = sizeof(struct iscsi_cmd) + conn->conn_transport->priv_size;
 166        cmd = (struct iscsi_cmd *)(se_sess->sess_cmd_map + (tag * size));
 167        memset(cmd, 0, size);
 168
 169        cmd->se_cmd.map_tag = tag;
 170        cmd->conn = conn;
 171        INIT_LIST_HEAD(&cmd->i_conn_node);
 172        INIT_LIST_HEAD(&cmd->datain_list);
 173        INIT_LIST_HEAD(&cmd->cmd_r2t_list);
 174        spin_lock_init(&cmd->datain_lock);
 175        spin_lock_init(&cmd->dataout_timeout_lock);
 176        spin_lock_init(&cmd->istate_lock);
 177        spin_lock_init(&cmd->error_lock);
 178        spin_lock_init(&cmd->r2t_lock);
 179
 180        return cmd;
 181}
 182EXPORT_SYMBOL(iscsit_allocate_cmd);
 183
 184struct iscsi_seq *iscsit_get_seq_holder_for_datain(
 185        struct iscsi_cmd *cmd,
 186        u32 seq_send_order)
 187{
 188        u32 i;
 189
 190        for (i = 0; i < cmd->seq_count; i++)
 191                if (cmd->seq_list[i].seq_send_order == seq_send_order)
 192                        return &cmd->seq_list[i];
 193
 194        return NULL;
 195}
 196
 197struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *cmd)
 198{
 199        u32 i;
 200
 201        if (!cmd->seq_list) {
 202                pr_err("struct iscsi_cmd->seq_list is NULL!\n");
 203                return NULL;
 204        }
 205
 206        for (i = 0; i < cmd->seq_count; i++) {
 207                if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
 208                        continue;
 209                if (cmd->seq_list[i].seq_send_order == cmd->seq_send_order) {
 210                        cmd->seq_send_order++;
 211                        return &cmd->seq_list[i];
 212                }
 213        }
 214
 215        return NULL;
 216}
 217
 218struct iscsi_r2t *iscsit_get_holder_for_r2tsn(
 219        struct iscsi_cmd *cmd,
 220        u32 r2t_sn)
 221{
 222        struct iscsi_r2t *r2t;
 223
 224        spin_lock_bh(&cmd->r2t_lock);
 225        list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
 226                if (r2t->r2t_sn == r2t_sn) {
 227                        spin_unlock_bh(&cmd->r2t_lock);
 228                        return r2t;
 229                }
 230        }
 231        spin_unlock_bh(&cmd->r2t_lock);
 232
 233        return NULL;
 234}
 235
 236static inline int iscsit_check_received_cmdsn(struct iscsi_session *sess, u32 cmdsn)
 237{
 238        int ret;
 239
 240        /*
 241         * This is the proper method of checking received CmdSN against
 242         * ExpCmdSN and MaxCmdSN values, as well as accounting for out
 243         * or order CmdSNs due to multiple connection sessions and/or
 244         * CRC failures.
 245         */
 246        if (iscsi_sna_gt(cmdsn, sess->max_cmd_sn)) {
 247                pr_err("Received CmdSN: 0x%08x is greater than"
 248                       " MaxCmdSN: 0x%08x, ignoring.\n", cmdsn,
 249                       sess->max_cmd_sn);
 250                ret = CMDSN_MAXCMDSN_OVERRUN;
 251
 252        } else if (cmdsn == sess->exp_cmd_sn) {
 253                sess->exp_cmd_sn++;
 254                pr_debug("Received CmdSN matches ExpCmdSN,"
 255                      " incremented ExpCmdSN to: 0x%08x\n",
 256                      sess->exp_cmd_sn);
 257                ret = CMDSN_NORMAL_OPERATION;
 258
 259        } else if (iscsi_sna_gt(cmdsn, sess->exp_cmd_sn)) {
 260                pr_debug("Received CmdSN: 0x%08x is greater"
 261                      " than ExpCmdSN: 0x%08x, not acknowledging.\n",
 262                      cmdsn, sess->exp_cmd_sn);
 263                ret = CMDSN_HIGHER_THAN_EXP;
 264
 265        } else {
 266                pr_err("Received CmdSN: 0x%08x is less than"
 267                       " ExpCmdSN: 0x%08x, ignoring.\n", cmdsn,
 268                       sess->exp_cmd_sn);
 269                ret = CMDSN_LOWER_THAN_EXP;
 270        }
 271
 272        return ret;
 273}
 274
 275/*
 276 * Commands may be received out of order if MC/S is in use.
 277 * Ensure they are executed in CmdSN order.
 278 */
 279int iscsit_sequence_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 280                        unsigned char *buf, __be32 cmdsn)
 281{
 282        int ret, cmdsn_ret;
 283        bool reject = false;
 284        u8 reason = ISCSI_REASON_BOOKMARK_NO_RESOURCES;
 285
 286        mutex_lock(&conn->sess->cmdsn_mutex);
 287
 288        cmdsn_ret = iscsit_check_received_cmdsn(conn->sess, be32_to_cpu(cmdsn));
 289        switch (cmdsn_ret) {
 290        case CMDSN_NORMAL_OPERATION:
 291                ret = iscsit_execute_cmd(cmd, 0);
 292                if ((ret >= 0) && !list_empty(&conn->sess->sess_ooo_cmdsn_list))
 293                        iscsit_execute_ooo_cmdsns(conn->sess);
 294                else if (ret < 0) {
 295                        reject = true;
 296                        ret = CMDSN_ERROR_CANNOT_RECOVER;
 297                }
 298                break;
 299        case CMDSN_HIGHER_THAN_EXP:
 300                ret = iscsit_handle_ooo_cmdsn(conn->sess, cmd, be32_to_cpu(cmdsn));
 301                if (ret < 0) {
 302                        reject = true;
 303                        ret = CMDSN_ERROR_CANNOT_RECOVER;
 304                        break;
 305                }
 306                ret = CMDSN_HIGHER_THAN_EXP;
 307                break;
 308        case CMDSN_LOWER_THAN_EXP:
 309        case CMDSN_MAXCMDSN_OVERRUN:
 310        default:
 311                cmd->i_state = ISTATE_REMOVE;
 312                iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
 313                /*
 314                 * Existing callers for iscsit_sequence_cmd() will silently
 315                 * ignore commands with CMDSN_LOWER_THAN_EXP, so force this
 316                 * return for CMDSN_MAXCMDSN_OVERRUN as well..
 317                 */
 318                ret = CMDSN_LOWER_THAN_EXP;
 319                break;
 320        }
 321        mutex_unlock(&conn->sess->cmdsn_mutex);
 322
 323        if (reject)
 324                iscsit_reject_cmd(cmd, reason, buf);
 325
 326        return ret;
 327}
 328EXPORT_SYMBOL(iscsit_sequence_cmd);
 329
 330int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf)
 331{
 332        struct iscsi_conn *conn = cmd->conn;
 333        struct se_cmd *se_cmd = &cmd->se_cmd;
 334        struct iscsi_data *hdr = (struct iscsi_data *) buf;
 335        u32 payload_length = ntoh24(hdr->dlength);
 336
 337        if (conn->sess->sess_ops->InitialR2T) {
 338                pr_err("Received unexpected unsolicited data"
 339                        " while InitialR2T=Yes, protocol error.\n");
 340                transport_send_check_condition_and_sense(se_cmd,
 341                                TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
 342                return -1;
 343        }
 344
 345        if ((cmd->first_burst_len + payload_length) >
 346             conn->sess->sess_ops->FirstBurstLength) {
 347                pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 348                        " for this Unsolicited DataOut Burst.\n",
 349                        (cmd->first_burst_len + payload_length),
 350                                conn->sess->sess_ops->FirstBurstLength);
 351                transport_send_check_condition_and_sense(se_cmd,
 352                                TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 353                return -1;
 354        }
 355
 356        if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))
 357                return 0;
 358
 359        if (((cmd->first_burst_len + payload_length) != cmd->se_cmd.data_length) &&
 360            ((cmd->first_burst_len + payload_length) !=
 361              conn->sess->sess_ops->FirstBurstLength)) {
 362                pr_err("Unsolicited non-immediate data received %u"
 363                        " does not equal FirstBurstLength: %u, and does"
 364                        " not equal ExpXferLen %u.\n",
 365                        (cmd->first_burst_len + payload_length),
 366                        conn->sess->sess_ops->FirstBurstLength, cmd->se_cmd.data_length);
 367                transport_send_check_condition_and_sense(se_cmd,
 368                                TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 369                return -1;
 370        }
 371        return 0;
 372}
 373
 374struct iscsi_cmd *iscsit_find_cmd_from_itt(
 375        struct iscsi_conn *conn,
 376        itt_t init_task_tag)
 377{
 378        struct iscsi_cmd *cmd;
 379
 380        spin_lock_bh(&conn->cmd_lock);
 381        list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 382                if (cmd->init_task_tag == init_task_tag) {
 383                        spin_unlock_bh(&conn->cmd_lock);
 384                        return cmd;
 385                }
 386        }
 387        spin_unlock_bh(&conn->cmd_lock);
 388
 389        pr_err("Unable to locate ITT: 0x%08x on CID: %hu",
 390                        init_task_tag, conn->cid);
 391        return NULL;
 392}
 393
 394struct iscsi_cmd *iscsit_find_cmd_from_itt_or_dump(
 395        struct iscsi_conn *conn,
 396        itt_t init_task_tag,
 397        u32 length)
 398{
 399        struct iscsi_cmd *cmd;
 400
 401        spin_lock_bh(&conn->cmd_lock);
 402        list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 403                if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT)
 404                        continue;
 405                if (cmd->init_task_tag == init_task_tag) {
 406                        spin_unlock_bh(&conn->cmd_lock);
 407                        return cmd;
 408                }
 409        }
 410        spin_unlock_bh(&conn->cmd_lock);
 411
 412        pr_err("Unable to locate ITT: 0x%08x on CID: %hu,"
 413                        " dumping payload\n", init_task_tag, conn->cid);
 414        if (length)
 415                iscsit_dump_data_payload(conn, length, 1);
 416
 417        return NULL;
 418}
 419
 420struct iscsi_cmd *iscsit_find_cmd_from_ttt(
 421        struct iscsi_conn *conn,
 422        u32 targ_xfer_tag)
 423{
 424        struct iscsi_cmd *cmd = NULL;
 425
 426        spin_lock_bh(&conn->cmd_lock);
 427        list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 428                if (cmd->targ_xfer_tag == targ_xfer_tag) {
 429                        spin_unlock_bh(&conn->cmd_lock);
 430                        return cmd;
 431                }
 432        }
 433        spin_unlock_bh(&conn->cmd_lock);
 434
 435        pr_err("Unable to locate TTT: 0x%08x on CID: %hu\n",
 436                        targ_xfer_tag, conn->cid);
 437        return NULL;
 438}
 439
 440int iscsit_find_cmd_for_recovery(
 441        struct iscsi_session *sess,
 442        struct iscsi_cmd **cmd_ptr,
 443        struct iscsi_conn_recovery **cr_ptr,
 444        itt_t init_task_tag)
 445{
 446        struct iscsi_cmd *cmd = NULL;
 447        struct iscsi_conn_recovery *cr;
 448        /*
 449         * Scan through the inactive connection recovery list's command list.
 450         * If init_task_tag matches the command is still alligent.
 451         */
 452        spin_lock(&sess->cr_i_lock);
 453        list_for_each_entry(cr, &sess->cr_inactive_list, cr_list) {
 454                spin_lock(&cr->conn_recovery_cmd_lock);
 455                list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
 456                        if (cmd->init_task_tag == init_task_tag) {
 457                                spin_unlock(&cr->conn_recovery_cmd_lock);
 458                                spin_unlock(&sess->cr_i_lock);
 459
 460                                *cr_ptr = cr;
 461                                *cmd_ptr = cmd;
 462                                return -2;
 463                        }
 464                }
 465                spin_unlock(&cr->conn_recovery_cmd_lock);
 466        }
 467        spin_unlock(&sess->cr_i_lock);
 468        /*
 469         * Scan through the active connection recovery list's command list.
 470         * If init_task_tag matches the command is ready to be reassigned.
 471         */
 472        spin_lock(&sess->cr_a_lock);
 473        list_for_each_entry(cr, &sess->cr_active_list, cr_list) {
 474                spin_lock(&cr->conn_recovery_cmd_lock);
 475                list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
 476                        if (cmd->init_task_tag == init_task_tag) {
 477                                spin_unlock(&cr->conn_recovery_cmd_lock);
 478                                spin_unlock(&sess->cr_a_lock);
 479
 480                                *cr_ptr = cr;
 481                                *cmd_ptr = cmd;
 482                                return 0;
 483                        }
 484                }
 485                spin_unlock(&cr->conn_recovery_cmd_lock);
 486        }
 487        spin_unlock(&sess->cr_a_lock);
 488
 489        return -1;
 490}
 491
 492void iscsit_add_cmd_to_immediate_queue(
 493        struct iscsi_cmd *cmd,
 494        struct iscsi_conn *conn,
 495        u8 state)
 496{
 497        struct iscsi_queue_req *qr;
 498
 499        qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
 500        if (!qr) {
 501                pr_err("Unable to allocate memory for"
 502                                " struct iscsi_queue_req\n");
 503                return;
 504        }
 505        INIT_LIST_HEAD(&qr->qr_list);
 506        qr->cmd = cmd;
 507        qr->state = state;
 508
 509        spin_lock_bh(&conn->immed_queue_lock);
 510        list_add_tail(&qr->qr_list, &conn->immed_queue_list);
 511        atomic_inc(&cmd->immed_queue_count);
 512        atomic_set(&conn->check_immediate_queue, 1);
 513        spin_unlock_bh(&conn->immed_queue_lock);
 514
 515        wake_up(&conn->queues_wq);
 516}
 517
 518struct iscsi_queue_req *iscsit_get_cmd_from_immediate_queue(struct iscsi_conn *conn)
 519{
 520        struct iscsi_queue_req *qr;
 521
 522        spin_lock_bh(&conn->immed_queue_lock);
 523        if (list_empty(&conn->immed_queue_list)) {
 524                spin_unlock_bh(&conn->immed_queue_lock);
 525                return NULL;
 526        }
 527        qr = list_first_entry(&conn->immed_queue_list,
 528                              struct iscsi_queue_req, qr_list);
 529
 530        list_del(&qr->qr_list);
 531        if (qr->cmd)
 532                atomic_dec(&qr->cmd->immed_queue_count);
 533        spin_unlock_bh(&conn->immed_queue_lock);
 534
 535        return qr;
 536}
 537
 538static void iscsit_remove_cmd_from_immediate_queue(
 539        struct iscsi_cmd *cmd,
 540        struct iscsi_conn *conn)
 541{
 542        struct iscsi_queue_req *qr, *qr_tmp;
 543
 544        spin_lock_bh(&conn->immed_queue_lock);
 545        if (!atomic_read(&cmd->immed_queue_count)) {
 546                spin_unlock_bh(&conn->immed_queue_lock);
 547                return;
 548        }
 549
 550        list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
 551                if (qr->cmd != cmd)
 552                        continue;
 553
 554                atomic_dec(&qr->cmd->immed_queue_count);
 555                list_del(&qr->qr_list);
 556                kmem_cache_free(lio_qr_cache, qr);
 557        }
 558        spin_unlock_bh(&conn->immed_queue_lock);
 559
 560        if (atomic_read(&cmd->immed_queue_count)) {
 561                pr_err("ITT: 0x%08x immed_queue_count: %d\n",
 562                        cmd->init_task_tag,
 563                        atomic_read(&cmd->immed_queue_count));
 564        }
 565}
 566
 567void iscsit_add_cmd_to_response_queue(
 568        struct iscsi_cmd *cmd,
 569        struct iscsi_conn *conn,
 570        u8 state)
 571{
 572        struct iscsi_queue_req *qr;
 573
 574        qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
 575        if (!qr) {
 576                pr_err("Unable to allocate memory for"
 577                        " struct iscsi_queue_req\n");
 578                return;
 579        }
 580        INIT_LIST_HEAD(&qr->qr_list);
 581        qr->cmd = cmd;
 582        qr->state = state;
 583
 584        spin_lock_bh(&conn->response_queue_lock);
 585        list_add_tail(&qr->qr_list, &conn->response_queue_list);
 586        atomic_inc(&cmd->response_queue_count);
 587        spin_unlock_bh(&conn->response_queue_lock);
 588
 589        wake_up(&conn->queues_wq);
 590}
 591
 592struct iscsi_queue_req *iscsit_get_cmd_from_response_queue(struct iscsi_conn *conn)
 593{
 594        struct iscsi_queue_req *qr;
 595
 596        spin_lock_bh(&conn->response_queue_lock);
 597        if (list_empty(&conn->response_queue_list)) {
 598                spin_unlock_bh(&conn->response_queue_lock);
 599                return NULL;
 600        }
 601
 602        qr = list_first_entry(&conn->response_queue_list,
 603                              struct iscsi_queue_req, qr_list);
 604
 605        list_del(&qr->qr_list);
 606        if (qr->cmd)
 607                atomic_dec(&qr->cmd->response_queue_count);
 608        spin_unlock_bh(&conn->response_queue_lock);
 609
 610        return qr;
 611}
 612
 613static void iscsit_remove_cmd_from_response_queue(
 614        struct iscsi_cmd *cmd,
 615        struct iscsi_conn *conn)
 616{
 617        struct iscsi_queue_req *qr, *qr_tmp;
 618
 619        spin_lock_bh(&conn->response_queue_lock);
 620        if (!atomic_read(&cmd->response_queue_count)) {
 621                spin_unlock_bh(&conn->response_queue_lock);
 622                return;
 623        }
 624
 625        list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
 626                                qr_list) {
 627                if (qr->cmd != cmd)
 628                        continue;
 629
 630                atomic_dec(&qr->cmd->response_queue_count);
 631                list_del(&qr->qr_list);
 632                kmem_cache_free(lio_qr_cache, qr);
 633        }
 634        spin_unlock_bh(&conn->response_queue_lock);
 635
 636        if (atomic_read(&cmd->response_queue_count)) {
 637                pr_err("ITT: 0x%08x response_queue_count: %d\n",
 638                        cmd->init_task_tag,
 639                        atomic_read(&cmd->response_queue_count));
 640        }
 641}
 642
 643bool iscsit_conn_all_queues_empty(struct iscsi_conn *conn)
 644{
 645        bool empty;
 646
 647        spin_lock_bh(&conn->immed_queue_lock);
 648        empty = list_empty(&conn->immed_queue_list);
 649        spin_unlock_bh(&conn->immed_queue_lock);
 650
 651        if (!empty)
 652                return empty;
 653
 654        spin_lock_bh(&conn->response_queue_lock);
 655        empty = list_empty(&conn->response_queue_list);
 656        spin_unlock_bh(&conn->response_queue_lock);
 657
 658        return empty;
 659}
 660
 661void iscsit_free_queue_reqs_for_conn(struct iscsi_conn *conn)
 662{
 663        struct iscsi_queue_req *qr, *qr_tmp;
 664
 665        spin_lock_bh(&conn->immed_queue_lock);
 666        list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
 667                list_del(&qr->qr_list);
 668                if (qr->cmd)
 669                        atomic_dec(&qr->cmd->immed_queue_count);
 670
 671                kmem_cache_free(lio_qr_cache, qr);
 672        }
 673        spin_unlock_bh(&conn->immed_queue_lock);
 674
 675        spin_lock_bh(&conn->response_queue_lock);
 676        list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
 677                        qr_list) {
 678                list_del(&qr->qr_list);
 679                if (qr->cmd)
 680                        atomic_dec(&qr->cmd->response_queue_count);
 681
 682                kmem_cache_free(lio_qr_cache, qr);
 683        }
 684        spin_unlock_bh(&conn->response_queue_lock);
 685}
 686
 687void iscsit_release_cmd(struct iscsi_cmd *cmd)
 688{
 689        struct iscsi_session *sess;
 690        struct se_cmd *se_cmd = &cmd->se_cmd;
 691
 692        if (cmd->conn)
 693                sess = cmd->conn->sess;
 694        else
 695                sess = cmd->sess;
 696
 697        BUG_ON(!sess || !sess->se_sess);
 698
 699        kfree(cmd->buf_ptr);
 700        kfree(cmd->pdu_list);
 701        kfree(cmd->seq_list);
 702        kfree(cmd->tmr_req);
 703        kfree(cmd->iov_data);
 704        kfree(cmd->text_in_ptr);
 705
 706        percpu_ida_free(&sess->se_sess->sess_tag_pool, se_cmd->map_tag);
 707}
 708EXPORT_SYMBOL(iscsit_release_cmd);
 709
 710void __iscsit_free_cmd(struct iscsi_cmd *cmd, bool scsi_cmd,
 711                       bool check_queues)
 712{
 713        struct iscsi_conn *conn = cmd->conn;
 714
 715        if (scsi_cmd) {
 716                if (cmd->data_direction == DMA_TO_DEVICE) {
 717                        iscsit_stop_dataout_timer(cmd);
 718                        iscsit_free_r2ts_from_list(cmd);
 719                }
 720                if (cmd->data_direction == DMA_FROM_DEVICE)
 721                        iscsit_free_all_datain_reqs(cmd);
 722        }
 723
 724        if (conn && check_queues) {
 725                iscsit_remove_cmd_from_immediate_queue(cmd, conn);
 726                iscsit_remove_cmd_from_response_queue(cmd, conn);
 727        }
 728}
 729
 730void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown)
 731{
 732        struct se_cmd *se_cmd = NULL;
 733        int rc;
 734        /*
 735         * Determine if a struct se_cmd is associated with
 736         * this struct iscsi_cmd.
 737         */
 738        switch (cmd->iscsi_opcode) {
 739        case ISCSI_OP_SCSI_CMD:
 740                se_cmd = &cmd->se_cmd;
 741                __iscsit_free_cmd(cmd, true, shutdown);
 742                /*
 743                 * Fallthrough
 744                 */
 745        case ISCSI_OP_SCSI_TMFUNC:
 746                rc = transport_generic_free_cmd(&cmd->se_cmd, shutdown);
 747                if (!rc && shutdown && se_cmd && se_cmd->se_sess) {
 748                        __iscsit_free_cmd(cmd, true, shutdown);
 749                        target_put_sess_cmd(se_cmd->se_sess, se_cmd);
 750                }
 751                break;
 752        case ISCSI_OP_REJECT:
 753                /*
 754                 * Handle special case for REJECT when iscsi_add_reject*() has
 755                 * overwritten the original iscsi_opcode assignment, and the
 756                 * associated cmd->se_cmd needs to be released.
 757                 */
 758                if (cmd->se_cmd.se_tfo != NULL) {
 759                        se_cmd = &cmd->se_cmd;
 760                        __iscsit_free_cmd(cmd, true, shutdown);
 761
 762                        rc = transport_generic_free_cmd(&cmd->se_cmd, shutdown);
 763                        if (!rc && shutdown && se_cmd->se_sess) {
 764                                __iscsit_free_cmd(cmd, true, shutdown);
 765                                target_put_sess_cmd(se_cmd->se_sess, se_cmd);
 766                        }
 767                        break;
 768                }
 769                /* Fall-through */
 770        default:
 771                __iscsit_free_cmd(cmd, false, shutdown);
 772                iscsit_release_cmd(cmd);
 773                break;
 774        }
 775}
 776
 777int iscsit_check_session_usage_count(struct iscsi_session *sess)
 778{
 779        spin_lock_bh(&sess->session_usage_lock);
 780        if (sess->session_usage_count != 0) {
 781                sess->session_waiting_on_uc = 1;
 782                spin_unlock_bh(&sess->session_usage_lock);
 783                if (in_interrupt())
 784                        return 2;
 785
 786                wait_for_completion(&sess->session_waiting_on_uc_comp);
 787                return 1;
 788        }
 789        spin_unlock_bh(&sess->session_usage_lock);
 790
 791        return 0;
 792}
 793
 794void iscsit_dec_session_usage_count(struct iscsi_session *sess)
 795{
 796        spin_lock_bh(&sess->session_usage_lock);
 797        sess->session_usage_count--;
 798
 799        if (!sess->session_usage_count && sess->session_waiting_on_uc)
 800                complete(&sess->session_waiting_on_uc_comp);
 801
 802        spin_unlock_bh(&sess->session_usage_lock);
 803}
 804
 805void iscsit_inc_session_usage_count(struct iscsi_session *sess)
 806{
 807        spin_lock_bh(&sess->session_usage_lock);
 808        sess->session_usage_count++;
 809        spin_unlock_bh(&sess->session_usage_lock);
 810}
 811
 812/*
 813 *      Setup conn->if_marker and conn->of_marker values based upon
 814 *      the initial marker-less interval. (see iSCSI v19 A.2)
 815 */
 816int iscsit_set_sync_and_steering_values(struct iscsi_conn *conn)
 817{
 818        int login_ifmarker_count = 0, login_ofmarker_count = 0, next_marker = 0;
 819        /*
 820         * IFMarkInt and OFMarkInt are negotiated as 32-bit words.
 821         */
 822        u32 IFMarkInt = (conn->conn_ops->IFMarkInt * 4);
 823        u32 OFMarkInt = (conn->conn_ops->OFMarkInt * 4);
 824
 825        if (conn->conn_ops->OFMarker) {
 826                /*
 827                 * Account for the first Login Command received not
 828                 * via iscsi_recv_msg().
 829                 */
 830                conn->of_marker += ISCSI_HDR_LEN;
 831                if (conn->of_marker <= OFMarkInt) {
 832                        conn->of_marker = (OFMarkInt - conn->of_marker);
 833                } else {
 834                        login_ofmarker_count = (conn->of_marker / OFMarkInt);
 835                        next_marker = (OFMarkInt * (login_ofmarker_count + 1)) +
 836                                        (login_ofmarker_count * MARKER_SIZE);
 837                        conn->of_marker = (next_marker - conn->of_marker);
 838                }
 839                conn->of_marker_offset = 0;
 840                pr_debug("Setting OFMarker value to %u based on Initial"
 841                        " Markerless Interval.\n", conn->of_marker);
 842        }
 843
 844        if (conn->conn_ops->IFMarker) {
 845                if (conn->if_marker <= IFMarkInt) {
 846                        conn->if_marker = (IFMarkInt - conn->if_marker);
 847                } else {
 848                        login_ifmarker_count = (conn->if_marker / IFMarkInt);
 849                        next_marker = (IFMarkInt * (login_ifmarker_count + 1)) +
 850                                        (login_ifmarker_count * MARKER_SIZE);
 851                        conn->if_marker = (next_marker - conn->if_marker);
 852                }
 853                pr_debug("Setting IFMarker value to %u based on Initial"
 854                        " Markerless Interval.\n", conn->if_marker);
 855        }
 856
 857        return 0;
 858}
 859
 860struct iscsi_conn *iscsit_get_conn_from_cid(struct iscsi_session *sess, u16 cid)
 861{
 862        struct iscsi_conn *conn;
 863
 864        spin_lock_bh(&sess->conn_lock);
 865        list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
 866                if ((conn->cid == cid) &&
 867                    (conn->conn_state == TARG_CONN_STATE_LOGGED_IN)) {
 868                        iscsit_inc_conn_usage_count(conn);
 869                        spin_unlock_bh(&sess->conn_lock);
 870                        return conn;
 871                }
 872        }
 873        spin_unlock_bh(&sess->conn_lock);
 874
 875        return NULL;
 876}
 877
 878struct iscsi_conn *iscsit_get_conn_from_cid_rcfr(struct iscsi_session *sess, u16 cid)
 879{
 880        struct iscsi_conn *conn;
 881
 882        spin_lock_bh(&sess->conn_lock);
 883        list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
 884                if (conn->cid == cid) {
 885                        iscsit_inc_conn_usage_count(conn);
 886                        spin_lock(&conn->state_lock);
 887                        atomic_set(&conn->connection_wait_rcfr, 1);
 888                        spin_unlock(&conn->state_lock);
 889                        spin_unlock_bh(&sess->conn_lock);
 890                        return conn;
 891                }
 892        }
 893        spin_unlock_bh(&sess->conn_lock);
 894
 895        return NULL;
 896}
 897
 898void iscsit_check_conn_usage_count(struct iscsi_conn *conn)
 899{
 900        spin_lock_bh(&conn->conn_usage_lock);
 901        if (conn->conn_usage_count != 0) {
 902                conn->conn_waiting_on_uc = 1;
 903                spin_unlock_bh(&conn->conn_usage_lock);
 904
 905                wait_for_completion(&conn->conn_waiting_on_uc_comp);
 906                return;
 907        }
 908        spin_unlock_bh(&conn->conn_usage_lock);
 909}
 910
 911void iscsit_dec_conn_usage_count(struct iscsi_conn *conn)
 912{
 913        spin_lock_bh(&conn->conn_usage_lock);
 914        conn->conn_usage_count--;
 915
 916        if (!conn->conn_usage_count && conn->conn_waiting_on_uc)
 917                complete(&conn->conn_waiting_on_uc_comp);
 918
 919        spin_unlock_bh(&conn->conn_usage_lock);
 920}
 921
 922void iscsit_inc_conn_usage_count(struct iscsi_conn *conn)
 923{
 924        spin_lock_bh(&conn->conn_usage_lock);
 925        conn->conn_usage_count++;
 926        spin_unlock_bh(&conn->conn_usage_lock);
 927}
 928
 929static int iscsit_add_nopin(struct iscsi_conn *conn, int want_response)
 930{
 931        u8 state;
 932        struct iscsi_cmd *cmd;
 933
 934        cmd = iscsit_allocate_cmd(conn, TASK_RUNNING);
 935        if (!cmd)
 936                return -1;
 937
 938        cmd->iscsi_opcode = ISCSI_OP_NOOP_IN;
 939        state = (want_response) ? ISTATE_SEND_NOPIN_WANT_RESPONSE :
 940                                ISTATE_SEND_NOPIN_NO_RESPONSE;
 941        cmd->init_task_tag = RESERVED_ITT;
 942        spin_lock_bh(&conn->sess->ttt_lock);
 943        cmd->targ_xfer_tag = (want_response) ? conn->sess->targ_xfer_tag++ :
 944                        0xFFFFFFFF;
 945        if (want_response && (cmd->targ_xfer_tag == 0xFFFFFFFF))
 946                cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 947        spin_unlock_bh(&conn->sess->ttt_lock);
 948
 949        spin_lock_bh(&conn->cmd_lock);
 950        list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 951        spin_unlock_bh(&conn->cmd_lock);
 952
 953        if (want_response)
 954                iscsit_start_nopin_response_timer(conn);
 955        iscsit_add_cmd_to_immediate_queue(cmd, conn, state);
 956
 957        return 0;
 958}
 959
 960static void iscsit_handle_nopin_response_timeout(unsigned long data)
 961{
 962        struct iscsi_conn *conn = (struct iscsi_conn *) data;
 963
 964        iscsit_inc_conn_usage_count(conn);
 965
 966        spin_lock_bh(&conn->nopin_timer_lock);
 967        if (conn->nopin_response_timer_flags & ISCSI_TF_STOP) {
 968                spin_unlock_bh(&conn->nopin_timer_lock);
 969                iscsit_dec_conn_usage_count(conn);
 970                return;
 971        }
 972
 973        pr_debug("Did not receive response to NOPIN on CID: %hu on"
 974                " SID: %u, failing connection.\n", conn->cid,
 975                        conn->sess->sid);
 976        conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
 977        spin_unlock_bh(&conn->nopin_timer_lock);
 978
 979        {
 980        struct iscsi_portal_group *tpg = conn->sess->tpg;
 981        struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 982
 983        if (tiqn) {
 984                spin_lock_bh(&tiqn->sess_err_stats.lock);
 985                strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
 986                                conn->sess->sess_ops->InitiatorName);
 987                tiqn->sess_err_stats.last_sess_failure_type =
 988                                ISCSI_SESS_ERR_CXN_TIMEOUT;
 989                tiqn->sess_err_stats.cxn_timeout_errors++;
 990                atomic_long_inc(&conn->sess->conn_timeout_errors);
 991                spin_unlock_bh(&tiqn->sess_err_stats.lock);
 992        }
 993        }
 994
 995        iscsit_cause_connection_reinstatement(conn, 0);
 996        iscsit_dec_conn_usage_count(conn);
 997}
 998
 999void iscsit_mod_nopin_response_timer(struct iscsi_conn *conn)
1000{
1001        struct iscsi_session *sess = conn->sess;
1002        struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1003
1004        spin_lock_bh(&conn->nopin_timer_lock);
1005        if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
1006                spin_unlock_bh(&conn->nopin_timer_lock);
1007                return;
1008        }
1009
1010        mod_timer(&conn->nopin_response_timer,
1011                (get_jiffies_64() + na->nopin_response_timeout * HZ));
1012        spin_unlock_bh(&conn->nopin_timer_lock);
1013}
1014
1015/*
1016 *      Called with conn->nopin_timer_lock held.
1017 */
1018void iscsit_start_nopin_response_timer(struct iscsi_conn *conn)
1019{
1020        struct iscsi_session *sess = conn->sess;
1021        struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1022
1023        spin_lock_bh(&conn->nopin_timer_lock);
1024        if (conn->nopin_response_timer_flags & ISCSI_TF_RUNNING) {
1025                spin_unlock_bh(&conn->nopin_timer_lock);
1026                return;
1027        }
1028
1029        init_timer(&conn->nopin_response_timer);
1030        conn->nopin_response_timer.expires =
1031                (get_jiffies_64() + na->nopin_response_timeout * HZ);
1032        conn->nopin_response_timer.data = (unsigned long)conn;
1033        conn->nopin_response_timer.function = iscsit_handle_nopin_response_timeout;
1034        conn->nopin_response_timer_flags &= ~ISCSI_TF_STOP;
1035        conn->nopin_response_timer_flags |= ISCSI_TF_RUNNING;
1036        add_timer(&conn->nopin_response_timer);
1037
1038        pr_debug("Started NOPIN Response Timer on CID: %d to %u"
1039                " seconds\n", conn->cid, na->nopin_response_timeout);
1040        spin_unlock_bh(&conn->nopin_timer_lock);
1041}
1042
1043void iscsit_stop_nopin_response_timer(struct iscsi_conn *conn)
1044{
1045        spin_lock_bh(&conn->nopin_timer_lock);
1046        if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
1047                spin_unlock_bh(&conn->nopin_timer_lock);
1048                return;
1049        }
1050        conn->nopin_response_timer_flags |= ISCSI_TF_STOP;
1051        spin_unlock_bh(&conn->nopin_timer_lock);
1052
1053        del_timer_sync(&conn->nopin_response_timer);
1054
1055        spin_lock_bh(&conn->nopin_timer_lock);
1056        conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
1057        spin_unlock_bh(&conn->nopin_timer_lock);
1058}
1059
1060static void iscsit_handle_nopin_timeout(unsigned long data)
1061{
1062        struct iscsi_conn *conn = (struct iscsi_conn *) data;
1063
1064        iscsit_inc_conn_usage_count(conn);
1065
1066        spin_lock_bh(&conn->nopin_timer_lock);
1067        if (conn->nopin_timer_flags & ISCSI_TF_STOP) {
1068                spin_unlock_bh(&conn->nopin_timer_lock);
1069                iscsit_dec_conn_usage_count(conn);
1070                return;
1071        }
1072        conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1073        spin_unlock_bh(&conn->nopin_timer_lock);
1074
1075        iscsit_add_nopin(conn, 1);
1076        iscsit_dec_conn_usage_count(conn);
1077}
1078
1079/*
1080 * Called with conn->nopin_timer_lock held.
1081 */
1082void __iscsit_start_nopin_timer(struct iscsi_conn *conn)
1083{
1084        struct iscsi_session *sess = conn->sess;
1085        struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1086        /*
1087        * NOPIN timeout is disabled.
1088         */
1089        if (!na->nopin_timeout)
1090                return;
1091
1092        if (conn->nopin_timer_flags & ISCSI_TF_RUNNING)
1093                return;
1094
1095        init_timer(&conn->nopin_timer);
1096        conn->nopin_timer.expires = (get_jiffies_64() + na->nopin_timeout * HZ);
1097        conn->nopin_timer.data = (unsigned long)conn;
1098        conn->nopin_timer.function = iscsit_handle_nopin_timeout;
1099        conn->nopin_timer_flags &= ~ISCSI_TF_STOP;
1100        conn->nopin_timer_flags |= ISCSI_TF_RUNNING;
1101        add_timer(&conn->nopin_timer);
1102
1103        pr_debug("Started NOPIN Timer on CID: %d at %u second"
1104                " interval\n", conn->cid, na->nopin_timeout);
1105}
1106
1107void iscsit_start_nopin_timer(struct iscsi_conn *conn)
1108{
1109        struct iscsi_session *sess = conn->sess;
1110        struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1111        /*
1112         * NOPIN timeout is disabled..
1113         */
1114        if (!na->nopin_timeout)
1115                return;
1116
1117        spin_lock_bh(&conn->nopin_timer_lock);
1118        if (conn->nopin_timer_flags & ISCSI_TF_RUNNING) {
1119                spin_unlock_bh(&conn->nopin_timer_lock);
1120                return;
1121        }
1122
1123        init_timer(&conn->nopin_timer);
1124        conn->nopin_timer.expires = (get_jiffies_64() + na->nopin_timeout * HZ);
1125        conn->nopin_timer.data = (unsigned long)conn;
1126        conn->nopin_timer.function = iscsit_handle_nopin_timeout;
1127        conn->nopin_timer_flags &= ~ISCSI_TF_STOP;
1128        conn->nopin_timer_flags |= ISCSI_TF_RUNNING;
1129        add_timer(&conn->nopin_timer);
1130
1131        pr_debug("Started NOPIN Timer on CID: %d at %u second"
1132                        " interval\n", conn->cid, na->nopin_timeout);
1133        spin_unlock_bh(&conn->nopin_timer_lock);
1134}
1135
1136void iscsit_stop_nopin_timer(struct iscsi_conn *conn)
1137{
1138        spin_lock_bh(&conn->nopin_timer_lock);
1139        if (!(conn->nopin_timer_flags & ISCSI_TF_RUNNING)) {
1140                spin_unlock_bh(&conn->nopin_timer_lock);
1141                return;
1142        }
1143        conn->nopin_timer_flags |= ISCSI_TF_STOP;
1144        spin_unlock_bh(&conn->nopin_timer_lock);
1145
1146        del_timer_sync(&conn->nopin_timer);
1147
1148        spin_lock_bh(&conn->nopin_timer_lock);
1149        conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1150        spin_unlock_bh(&conn->nopin_timer_lock);
1151}
1152
1153int iscsit_send_tx_data(
1154        struct iscsi_cmd *cmd,
1155        struct iscsi_conn *conn,
1156        int use_misc)
1157{
1158        int tx_sent, tx_size;
1159        u32 iov_count;
1160        struct kvec *iov;
1161
1162send_data:
1163        tx_size = cmd->tx_size;
1164
1165        if (!use_misc) {
1166                iov = &cmd->iov_data[0];
1167                iov_count = cmd->iov_data_count;
1168        } else {
1169                iov = &cmd->iov_misc[0];
1170                iov_count = cmd->iov_misc_count;
1171        }
1172
1173        tx_sent = tx_data(conn, &iov[0], iov_count, tx_size);
1174        if (tx_size != tx_sent) {
1175                if (tx_sent == -EAGAIN) {
1176                        pr_err("tx_data() returned -EAGAIN\n");
1177                        goto send_data;
1178                } else
1179                        return -1;
1180        }
1181        cmd->tx_size = 0;
1182
1183        return 0;
1184}
1185
1186int iscsit_fe_sendpage_sg(
1187        struct iscsi_cmd *cmd,
1188        struct iscsi_conn *conn)
1189{
1190        struct scatterlist *sg = cmd->first_data_sg;
1191        struct kvec iov;
1192        u32 tx_hdr_size, data_len;
1193        u32 offset = cmd->first_data_sg_off;
1194        int tx_sent, iov_off;
1195
1196send_hdr:
1197        tx_hdr_size = ISCSI_HDR_LEN;
1198        if (conn->conn_ops->HeaderDigest)
1199                tx_hdr_size += ISCSI_CRC_LEN;
1200
1201        iov.iov_base = cmd->pdu;
1202        iov.iov_len = tx_hdr_size;
1203
1204        tx_sent = tx_data(conn, &iov, 1, tx_hdr_size);
1205        if (tx_hdr_size != tx_sent) {
1206                if (tx_sent == -EAGAIN) {
1207                        pr_err("tx_data() returned -EAGAIN\n");
1208                        goto send_hdr;
1209                }
1210                return -1;
1211        }
1212
1213        data_len = cmd->tx_size - tx_hdr_size - cmd->padding;
1214        /*
1215         * Set iov_off used by padding and data digest tx_data() calls below
1216         * in order to determine proper offset into cmd->iov_data[]
1217         */
1218        if (conn->conn_ops->DataDigest) {
1219                data_len -= ISCSI_CRC_LEN;
1220                if (cmd->padding)
1221                        iov_off = (cmd->iov_data_count - 2);
1222                else
1223                        iov_off = (cmd->iov_data_count - 1);
1224        } else {
1225                iov_off = (cmd->iov_data_count - 1);
1226        }
1227        /*
1228         * Perform sendpage() for each page in the scatterlist
1229         */
1230        while (data_len) {
1231                u32 space = (sg->length - offset);
1232                u32 sub_len = min_t(u32, data_len, space);
1233send_pg:
1234                tx_sent = conn->sock->ops->sendpage(conn->sock,
1235                                        sg_page(sg), sg->offset + offset, sub_len, 0);
1236                if (tx_sent != sub_len) {
1237                        if (tx_sent == -EAGAIN) {
1238                                pr_err("tcp_sendpage() returned"
1239                                                " -EAGAIN\n");
1240                                goto send_pg;
1241                        }
1242
1243                        pr_err("tcp_sendpage() failure: %d\n",
1244                                        tx_sent);
1245                        return -1;
1246                }
1247
1248                data_len -= sub_len;
1249                offset = 0;
1250                sg = sg_next(sg);
1251        }
1252
1253send_padding:
1254        if (cmd->padding) {
1255                struct kvec *iov_p = &cmd->iov_data[iov_off++];
1256
1257                tx_sent = tx_data(conn, iov_p, 1, cmd->padding);
1258                if (cmd->padding != tx_sent) {
1259                        if (tx_sent == -EAGAIN) {
1260                                pr_err("tx_data() returned -EAGAIN\n");
1261                                goto send_padding;
1262                        }
1263                        return -1;
1264                }
1265        }
1266
1267send_datacrc:
1268        if (conn->conn_ops->DataDigest) {
1269                struct kvec *iov_d = &cmd->iov_data[iov_off];
1270
1271                tx_sent = tx_data(conn, iov_d, 1, ISCSI_CRC_LEN);
1272                if (ISCSI_CRC_LEN != tx_sent) {
1273                        if (tx_sent == -EAGAIN) {
1274                                pr_err("tx_data() returned -EAGAIN\n");
1275                                goto send_datacrc;
1276                        }
1277                        return -1;
1278                }
1279        }
1280
1281        return 0;
1282}
1283
1284/*
1285 *      This function is used for mainly sending a ISCSI_TARG_LOGIN_RSP PDU
1286 *      back to the Initiator when an expection condition occurs with the
1287 *      errors set in status_class and status_detail.
1288 *
1289 *      Parameters:     iSCSI Connection, Status Class, Status Detail.
1290 *      Returns:        0 on success, -1 on error.
1291 */
1292int iscsit_tx_login_rsp(struct iscsi_conn *conn, u8 status_class, u8 status_detail)
1293{
1294        struct iscsi_login_rsp *hdr;
1295        struct iscsi_login *login = conn->conn_login;
1296
1297        login->login_failed = 1;
1298        iscsit_collect_login_stats(conn, status_class, status_detail);
1299
1300        memset(&login->rsp[0], 0, ISCSI_HDR_LEN);
1301
1302        hdr     = (struct iscsi_login_rsp *)&login->rsp[0];
1303        hdr->opcode             = ISCSI_OP_LOGIN_RSP;
1304        hdr->status_class       = status_class;
1305        hdr->status_detail      = status_detail;
1306        hdr->itt                = conn->login_itt;
1307
1308        return conn->conn_transport->iscsit_put_login_tx(conn, login, 0);
1309}
1310
1311void iscsit_print_session_params(struct iscsi_session *sess)
1312{
1313        struct iscsi_conn *conn;
1314
1315        pr_debug("-----------------------------[Session Params for"
1316                " SID: %u]-----------------------------\n", sess->sid);
1317        spin_lock_bh(&sess->conn_lock);
1318        list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
1319                iscsi_dump_conn_ops(conn->conn_ops);
1320        spin_unlock_bh(&sess->conn_lock);
1321
1322        iscsi_dump_sess_ops(sess->sess_ops);
1323}
1324
1325static int iscsit_do_rx_data(
1326        struct iscsi_conn *conn,
1327        struct iscsi_data_count *count)
1328{
1329        int data = count->data_length, rx_loop = 0, total_rx = 0, iov_len;
1330        struct kvec *iov_p;
1331        struct msghdr msg;
1332
1333        if (!conn || !conn->sock || !conn->conn_ops)
1334                return -1;
1335
1336        memset(&msg, 0, sizeof(struct msghdr));
1337
1338        iov_p = count->iov;
1339        iov_len = count->iov_count;
1340
1341        while (total_rx < data) {
1342                rx_loop = kernel_recvmsg(conn->sock, &msg, iov_p, iov_len,
1343                                        (data - total_rx), MSG_WAITALL);
1344                if (rx_loop <= 0) {
1345                        pr_debug("rx_loop: %d total_rx: %d\n",
1346                                rx_loop, total_rx);
1347                        return rx_loop;
1348                }
1349                total_rx += rx_loop;
1350                pr_debug("rx_loop: %d, total_rx: %d, data: %d\n",
1351                                rx_loop, total_rx, data);
1352        }
1353
1354        return total_rx;
1355}
1356
1357static int iscsit_do_tx_data(
1358        struct iscsi_conn *conn,
1359        struct iscsi_data_count *count)
1360{
1361        int data = count->data_length, total_tx = 0, tx_loop = 0, iov_len;
1362        struct kvec *iov_p;
1363        struct msghdr msg;
1364
1365        if (!conn || !conn->sock || !conn->conn_ops)
1366                return -1;
1367
1368        if (data <= 0) {
1369                pr_err("Data length is: %d\n", data);
1370                return -1;
1371        }
1372
1373        memset(&msg, 0, sizeof(struct msghdr));
1374
1375        iov_p = count->iov;
1376        iov_len = count->iov_count;
1377
1378        while (total_tx < data) {
1379                tx_loop = kernel_sendmsg(conn->sock, &msg, iov_p, iov_len,
1380                                        (data - total_tx));
1381                if (tx_loop <= 0) {
1382                        pr_debug("tx_loop: %d total_tx %d\n",
1383                                tx_loop, total_tx);
1384                        return tx_loop;
1385                }
1386                total_tx += tx_loop;
1387                pr_debug("tx_loop: %d, total_tx: %d, data: %d\n",
1388                                        tx_loop, total_tx, data);
1389        }
1390
1391        return total_tx;
1392}
1393
1394int rx_data(
1395        struct iscsi_conn *conn,
1396        struct kvec *iov,
1397        int iov_count,
1398        int data)
1399{
1400        struct iscsi_data_count c;
1401
1402        if (!conn || !conn->sock || !conn->conn_ops)
1403                return -1;
1404
1405        memset(&c, 0, sizeof(struct iscsi_data_count));
1406        c.iov = iov;
1407        c.iov_count = iov_count;
1408        c.data_length = data;
1409        c.type = ISCSI_RX_DATA;
1410
1411        return iscsit_do_rx_data(conn, &c);
1412}
1413
1414int tx_data(
1415        struct iscsi_conn *conn,
1416        struct kvec *iov,
1417        int iov_count,
1418        int data)
1419{
1420        struct iscsi_data_count c;
1421
1422        if (!conn || !conn->sock || !conn->conn_ops)
1423                return -1;
1424
1425        memset(&c, 0, sizeof(struct iscsi_data_count));
1426        c.iov = iov;
1427        c.iov_count = iov_count;
1428        c.data_length = data;
1429        c.type = ISCSI_TX_DATA;
1430
1431        return iscsit_do_tx_data(conn, &c);
1432}
1433
1434void iscsit_collect_login_stats(
1435        struct iscsi_conn *conn,
1436        u8 status_class,
1437        u8 status_detail)
1438{
1439        struct iscsi_param *intrname = NULL;
1440        struct iscsi_tiqn *tiqn;
1441        struct iscsi_login_stats *ls;
1442
1443        tiqn = iscsit_snmp_get_tiqn(conn);
1444        if (!tiqn)
1445                return;
1446
1447        ls = &tiqn->login_stats;
1448
1449        spin_lock(&ls->lock);
1450        if (!strcmp(conn->login_ip, ls->last_intr_fail_ip_addr) &&
1451            ((get_jiffies_64() - ls->last_fail_time) < 10)) {
1452                /* We already have the failure info for this login */
1453                spin_unlock(&ls->lock);
1454                return;
1455        }
1456
1457        if (status_class == ISCSI_STATUS_CLS_SUCCESS)
1458                ls->accepts++;
1459        else if (status_class == ISCSI_STATUS_CLS_REDIRECT) {
1460                ls->redirects++;
1461                ls->last_fail_type = ISCSI_LOGIN_FAIL_REDIRECT;
1462        } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1463                 (status_detail == ISCSI_LOGIN_STATUS_AUTH_FAILED)) {
1464                ls->authenticate_fails++;
1465                ls->last_fail_type =  ISCSI_LOGIN_FAIL_AUTHENTICATE;
1466        } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1467                 (status_detail == ISCSI_LOGIN_STATUS_TGT_FORBIDDEN)) {
1468                ls->authorize_fails++;
1469                ls->last_fail_type = ISCSI_LOGIN_FAIL_AUTHORIZE;
1470        } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR) &&
1471                 (status_detail == ISCSI_LOGIN_STATUS_INIT_ERR)) {
1472                ls->negotiate_fails++;
1473                ls->last_fail_type = ISCSI_LOGIN_FAIL_NEGOTIATE;
1474        } else {
1475                ls->other_fails++;
1476                ls->last_fail_type = ISCSI_LOGIN_FAIL_OTHER;
1477        }
1478
1479        /* Save initiator name, ip address and time, if it is a failed login */
1480        if (status_class != ISCSI_STATUS_CLS_SUCCESS) {
1481                if (conn->param_list)
1482                        intrname = iscsi_find_param_from_key(INITIATORNAME,
1483                                                             conn->param_list);
1484                strcpy(ls->last_intr_fail_name,
1485                       (intrname ? intrname->value : "Unknown"));
1486
1487                ls->last_intr_fail_ip_family = conn->login_family;
1488
1489                snprintf(ls->last_intr_fail_ip_addr, IPV6_ADDRESS_SPACE,
1490                                "%s", conn->login_ip);
1491                ls->last_fail_time = get_jiffies_64();
1492        }
1493
1494        spin_unlock(&ls->lock);
1495}
1496
1497struct iscsi_tiqn *iscsit_snmp_get_tiqn(struct iscsi_conn *conn)
1498{
1499        struct iscsi_portal_group *tpg;
1500
1501        if (!conn || !conn->sess)
1502                return NULL;
1503
1504        tpg = conn->sess->tpg;
1505        if (!tpg)
1506                return NULL;
1507
1508        if (!tpg->tpg_tiqn)
1509                return NULL;
1510
1511        return tpg->tpg_tiqn;
1512}
1513