1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/net.h>
27#include <linux/delay.h>
28#include <linux/string.h>
29#include <linux/timer.h>
30#include <linux/slab.h>
31#include <linux/spinlock.h>
32#include <linux/kthread.h>
33#include <linux/in.h>
34#include <linux/cdrom.h>
35#include <linux/module.h>
36#include <linux/ratelimit.h>
37#include <linux/vmalloc.h>
38#include <asm/unaligned.h>
39#include <net/sock.h>
40#include <net/tcp.h>
41#include <scsi/scsi.h>
42#include <scsi/scsi_eh.h>
43#include <scsi/scsi_cmnd.h>
44#include <scsi/scsi_tcq.h>
45
46#include <target/target_core_base.h>
47#include <target/target_core_backend.h>
48#include <target/target_core_fabric.h>
49
50#include "target_core_internal.h"
51#include "target_core_alua.h"
52#include "target_core_pr.h"
53#include "target_core_ua.h"
54
55#define CREATE_TRACE_POINTS
56#include <trace/events/target.h>
57
58static struct workqueue_struct *target_completion_wq;
59static struct kmem_cache *se_sess_cache;
60struct kmem_cache *se_ua_cache;
61struct kmem_cache *t10_pr_reg_cache;
62struct kmem_cache *t10_alua_lu_gp_cache;
63struct kmem_cache *t10_alua_lu_gp_mem_cache;
64struct kmem_cache *t10_alua_tg_pt_gp_cache;
65struct kmem_cache *t10_alua_lba_map_cache;
66struct kmem_cache *t10_alua_lba_map_mem_cache;
67
68static void transport_complete_task_attr(struct se_cmd *cmd);
69static void transport_handle_queue_full(struct se_cmd *cmd,
70 struct se_device *dev, int err, bool write_pending);
71static void target_complete_ok_work(struct work_struct *work);
72
73int init_se_kmem_caches(void)
74{
75 se_sess_cache = kmem_cache_create("se_sess_cache",
76 sizeof(struct se_session), __alignof__(struct se_session),
77 0, NULL);
78 if (!se_sess_cache) {
79 pr_err("kmem_cache_create() for struct se_session"
80 " failed\n");
81 goto out;
82 }
83 se_ua_cache = kmem_cache_create("se_ua_cache",
84 sizeof(struct se_ua), __alignof__(struct se_ua),
85 0, NULL);
86 if (!se_ua_cache) {
87 pr_err("kmem_cache_create() for struct se_ua failed\n");
88 goto out_free_sess_cache;
89 }
90 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
91 sizeof(struct t10_pr_registration),
92 __alignof__(struct t10_pr_registration), 0, NULL);
93 if (!t10_pr_reg_cache) {
94 pr_err("kmem_cache_create() for struct t10_pr_registration"
95 " failed\n");
96 goto out_free_ua_cache;
97 }
98 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
99 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
100 0, NULL);
101 if (!t10_alua_lu_gp_cache) {
102 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
103 " failed\n");
104 goto out_free_pr_reg_cache;
105 }
106 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
107 sizeof(struct t10_alua_lu_gp_member),
108 __alignof__(struct t10_alua_lu_gp_member), 0, NULL);
109 if (!t10_alua_lu_gp_mem_cache) {
110 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
111 "cache failed\n");
112 goto out_free_lu_gp_cache;
113 }
114 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
115 sizeof(struct t10_alua_tg_pt_gp),
116 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
117 if (!t10_alua_tg_pt_gp_cache) {
118 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
119 "cache failed\n");
120 goto out_free_lu_gp_mem_cache;
121 }
122 t10_alua_lba_map_cache = kmem_cache_create(
123 "t10_alua_lba_map_cache",
124 sizeof(struct t10_alua_lba_map),
125 __alignof__(struct t10_alua_lba_map), 0, NULL);
126 if (!t10_alua_lba_map_cache) {
127 pr_err("kmem_cache_create() for t10_alua_lba_map_"
128 "cache failed\n");
129 goto out_free_tg_pt_gp_cache;
130 }
131 t10_alua_lba_map_mem_cache = kmem_cache_create(
132 "t10_alua_lba_map_mem_cache",
133 sizeof(struct t10_alua_lba_map_member),
134 __alignof__(struct t10_alua_lba_map_member), 0, NULL);
135 if (!t10_alua_lba_map_mem_cache) {
136 pr_err("kmem_cache_create() for t10_alua_lba_map_mem_"
137 "cache failed\n");
138 goto out_free_lba_map_cache;
139 }
140
141 target_completion_wq = alloc_workqueue("target_completion",
142 WQ_MEM_RECLAIM, 0);
143 if (!target_completion_wq)
144 goto out_free_lba_map_mem_cache;
145
146 return 0;
147
148out_free_lba_map_mem_cache:
149 kmem_cache_destroy(t10_alua_lba_map_mem_cache);
150out_free_lba_map_cache:
151 kmem_cache_destroy(t10_alua_lba_map_cache);
152out_free_tg_pt_gp_cache:
153 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
154out_free_lu_gp_mem_cache:
155 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
156out_free_lu_gp_cache:
157 kmem_cache_destroy(t10_alua_lu_gp_cache);
158out_free_pr_reg_cache:
159 kmem_cache_destroy(t10_pr_reg_cache);
160out_free_ua_cache:
161 kmem_cache_destroy(se_ua_cache);
162out_free_sess_cache:
163 kmem_cache_destroy(se_sess_cache);
164out:
165 return -ENOMEM;
166}
167
168void release_se_kmem_caches(void)
169{
170 destroy_workqueue(target_completion_wq);
171 kmem_cache_destroy(se_sess_cache);
172 kmem_cache_destroy(se_ua_cache);
173 kmem_cache_destroy(t10_pr_reg_cache);
174 kmem_cache_destroy(t10_alua_lu_gp_cache);
175 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
176 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
177 kmem_cache_destroy(t10_alua_lba_map_cache);
178 kmem_cache_destroy(t10_alua_lba_map_mem_cache);
179}
180
181
182static DEFINE_SPINLOCK(scsi_mib_index_lock);
183static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
184
185
186
187
188u32 scsi_get_new_index(scsi_index_t type)
189{
190 u32 new_index;
191
192 BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
193
194 spin_lock(&scsi_mib_index_lock);
195 new_index = ++scsi_mib_index[type];
196 spin_unlock(&scsi_mib_index_lock);
197
198 return new_index;
199}
200
201void transport_subsystem_check_init(void)
202{
203 int ret;
204 static int sub_api_initialized;
205
206 if (sub_api_initialized)
207 return;
208
209 ret = request_module("target_core_iblock");
210 if (ret != 0)
211 pr_err("Unable to load target_core_iblock\n");
212
213 ret = request_module("target_core_file");
214 if (ret != 0)
215 pr_err("Unable to load target_core_file\n");
216
217 ret = request_module("target_core_pscsi");
218 if (ret != 0)
219 pr_err("Unable to load target_core_pscsi\n");
220
221 ret = request_module("target_core_user");
222 if (ret != 0)
223 pr_err("Unable to load target_core_user\n");
224
225 sub_api_initialized = 1;
226}
227
228struct se_session *transport_init_session(enum target_prot_op sup_prot_ops)
229{
230 struct se_session *se_sess;
231
232 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
233 if (!se_sess) {
234 pr_err("Unable to allocate struct se_session from"
235 " se_sess_cache\n");
236 return ERR_PTR(-ENOMEM);
237 }
238 INIT_LIST_HEAD(&se_sess->sess_list);
239 INIT_LIST_HEAD(&se_sess->sess_acl_list);
240 INIT_LIST_HEAD(&se_sess->sess_cmd_list);
241 INIT_LIST_HEAD(&se_sess->sess_wait_list);
242 spin_lock_init(&se_sess->sess_cmd_lock);
243 kref_init(&se_sess->sess_kref);
244 se_sess->sup_prot_ops = sup_prot_ops;
245
246 return se_sess;
247}
248EXPORT_SYMBOL(transport_init_session);
249
250int transport_alloc_session_tags(struct se_session *se_sess,
251 unsigned int tag_num, unsigned int tag_size)
252{
253 int rc;
254
255 se_sess->sess_cmd_map = kzalloc(tag_num * tag_size,
256 GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
257 if (!se_sess->sess_cmd_map) {
258 se_sess->sess_cmd_map = vzalloc(tag_num * tag_size);
259 if (!se_sess->sess_cmd_map) {
260 pr_err("Unable to allocate se_sess->sess_cmd_map\n");
261 return -ENOMEM;
262 }
263 }
264
265 rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num);
266 if (rc < 0) {
267 pr_err("Unable to init se_sess->sess_tag_pool,"
268 " tag_num: %u\n", tag_num);
269 kvfree(se_sess->sess_cmd_map);
270 se_sess->sess_cmd_map = NULL;
271 return -ENOMEM;
272 }
273
274 return 0;
275}
276EXPORT_SYMBOL(transport_alloc_session_tags);
277
278struct se_session *transport_init_session_tags(unsigned int tag_num,
279 unsigned int tag_size,
280 enum target_prot_op sup_prot_ops)
281{
282 struct se_session *se_sess;
283 int rc;
284
285 if (tag_num != 0 && !tag_size) {
286 pr_err("init_session_tags called with percpu-ida tag_num:"
287 " %u, but zero tag_size\n", tag_num);
288 return ERR_PTR(-EINVAL);
289 }
290 if (!tag_num && tag_size) {
291 pr_err("init_session_tags called with percpu-ida tag_size:"
292 " %u, but zero tag_num\n", tag_size);
293 return ERR_PTR(-EINVAL);
294 }
295
296 se_sess = transport_init_session(sup_prot_ops);
297 if (IS_ERR(se_sess))
298 return se_sess;
299
300 rc = transport_alloc_session_tags(se_sess, tag_num, tag_size);
301 if (rc < 0) {
302 transport_free_session(se_sess);
303 return ERR_PTR(-ENOMEM);
304 }
305
306 return se_sess;
307}
308EXPORT_SYMBOL(transport_init_session_tags);
309
310
311
312
313void __transport_register_session(
314 struct se_portal_group *se_tpg,
315 struct se_node_acl *se_nacl,
316 struct se_session *se_sess,
317 void *fabric_sess_ptr)
318{
319 const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo;
320 unsigned char buf[PR_REG_ISID_LEN];
321
322 se_sess->se_tpg = se_tpg;
323 se_sess->fabric_sess_ptr = fabric_sess_ptr;
324
325
326
327
328
329
330 if (se_nacl) {
331
332
333
334
335
336
337
338
339
340
341 if (se_nacl->saved_prot_type)
342 se_sess->sess_prot_type = se_nacl->saved_prot_type;
343 else if (tfo->tpg_check_prot_fabric_only)
344 se_sess->sess_prot_type = se_nacl->saved_prot_type =
345 tfo->tpg_check_prot_fabric_only(se_tpg);
346
347
348
349
350 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
351 memset(&buf[0], 0, PR_REG_ISID_LEN);
352 se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
353 &buf[0], PR_REG_ISID_LEN);
354 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
355 }
356
357 spin_lock_irq(&se_nacl->nacl_sess_lock);
358
359
360
361
362 se_nacl->nacl_sess = se_sess;
363
364 list_add_tail(&se_sess->sess_acl_list,
365 &se_nacl->acl_sess_list);
366 spin_unlock_irq(&se_nacl->nacl_sess_lock);
367 }
368 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
369
370 pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
371 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
372}
373EXPORT_SYMBOL(__transport_register_session);
374
375void transport_register_session(
376 struct se_portal_group *se_tpg,
377 struct se_node_acl *se_nacl,
378 struct se_session *se_sess,
379 void *fabric_sess_ptr)
380{
381 unsigned long flags;
382
383 spin_lock_irqsave(&se_tpg->session_lock, flags);
384 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
385 spin_unlock_irqrestore(&se_tpg->session_lock, flags);
386}
387EXPORT_SYMBOL(transport_register_session);
388
389struct se_session *
390target_alloc_session(struct se_portal_group *tpg,
391 unsigned int tag_num, unsigned int tag_size,
392 enum target_prot_op prot_op,
393 const char *initiatorname, void *private,
394 int (*callback)(struct se_portal_group *,
395 struct se_session *, void *))
396{
397 struct se_session *sess;
398
399
400
401
402
403 if (tag_num != 0)
404 sess = transport_init_session_tags(tag_num, tag_size, prot_op);
405 else
406 sess = transport_init_session(prot_op);
407
408 if (IS_ERR(sess))
409 return sess;
410
411 sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg,
412 (unsigned char *)initiatorname);
413 if (!sess->se_node_acl) {
414 transport_free_session(sess);
415 return ERR_PTR(-EACCES);
416 }
417
418
419
420
421 if (callback != NULL) {
422 int rc = callback(tpg, sess, private);
423 if (rc) {
424 transport_free_session(sess);
425 return ERR_PTR(rc);
426 }
427 }
428
429 transport_register_session(tpg, sess->se_node_acl, sess, private);
430 return sess;
431}
432EXPORT_SYMBOL(target_alloc_session);
433
434static void target_release_session(struct kref *kref)
435{
436 struct se_session *se_sess = container_of(kref,
437 struct se_session, sess_kref);
438 struct se_portal_group *se_tpg = se_sess->se_tpg;
439
440 if (se_tpg->se_tpg_tfo->close_session)
441 se_tpg->se_tpg_tfo->close_session(se_sess);
442}
443
444int target_get_session(struct se_session *se_sess)
445{
446 return kref_get_unless_zero(&se_sess->sess_kref);
447}
448EXPORT_SYMBOL(target_get_session);
449
450void target_put_session(struct se_session *se_sess)
451{
452 kref_put(&se_sess->sess_kref, target_release_session);
453}
454EXPORT_SYMBOL(target_put_session);
455
456ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page)
457{
458 struct se_session *se_sess;
459 ssize_t len = 0;
460
461 spin_lock_bh(&se_tpg->session_lock);
462 list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) {
463 if (!se_sess->se_node_acl)
464 continue;
465 if (!se_sess->se_node_acl->dynamic_node_acl)
466 continue;
467 if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE)
468 break;
469
470 len += snprintf(page + len, PAGE_SIZE - len, "%s\n",
471 se_sess->se_node_acl->initiatorname);
472 len += 1;
473 }
474 spin_unlock_bh(&se_tpg->session_lock);
475
476 return len;
477}
478EXPORT_SYMBOL(target_show_dynamic_sessions);
479
480static void target_complete_nacl(struct kref *kref)
481{
482 struct se_node_acl *nacl = container_of(kref,
483 struct se_node_acl, acl_kref);
484 struct se_portal_group *se_tpg = nacl->se_tpg;
485
486 if (!nacl->dynamic_stop) {
487 complete(&nacl->acl_free_comp);
488 return;
489 }
490
491 mutex_lock(&se_tpg->acl_node_mutex);
492 list_del_init(&nacl->acl_list);
493 mutex_unlock(&se_tpg->acl_node_mutex);
494
495 core_tpg_wait_for_nacl_pr_ref(nacl);
496 core_free_device_list_for_node(nacl, se_tpg);
497 kfree(nacl);
498}
499
500void target_put_nacl(struct se_node_acl *nacl)
501{
502 kref_put(&nacl->acl_kref, target_complete_nacl);
503}
504EXPORT_SYMBOL(target_put_nacl);
505
506void transport_deregister_session_configfs(struct se_session *se_sess)
507{
508 struct se_node_acl *se_nacl;
509 unsigned long flags;
510
511
512
513 se_nacl = se_sess->se_node_acl;
514 if (se_nacl) {
515 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
516 if (!list_empty(&se_sess->sess_acl_list))
517 list_del_init(&se_sess->sess_acl_list);
518
519
520
521
522
523 if (list_empty(&se_nacl->acl_sess_list))
524 se_nacl->nacl_sess = NULL;
525 else {
526 se_nacl->nacl_sess = container_of(
527 se_nacl->acl_sess_list.prev,
528 struct se_session, sess_acl_list);
529 }
530 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
531 }
532}
533EXPORT_SYMBOL(transport_deregister_session_configfs);
534
535void transport_free_session(struct se_session *se_sess)
536{
537 struct se_node_acl *se_nacl = se_sess->se_node_acl;
538
539
540
541
542
543 if (se_nacl) {
544 struct se_portal_group *se_tpg = se_nacl->se_tpg;
545 const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo;
546 unsigned long flags;
547
548 se_sess->se_node_acl = NULL;
549
550
551
552
553
554
555 mutex_lock(&se_tpg->acl_node_mutex);
556 if (se_nacl->dynamic_node_acl &&
557 !se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
558 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
559 if (list_empty(&se_nacl->acl_sess_list))
560 se_nacl->dynamic_stop = true;
561 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
562
563 if (se_nacl->dynamic_stop)
564 list_del_init(&se_nacl->acl_list);
565 }
566 mutex_unlock(&se_tpg->acl_node_mutex);
567
568 if (se_nacl->dynamic_stop)
569 target_put_nacl(se_nacl);
570
571 target_put_nacl(se_nacl);
572 }
573 if (se_sess->sess_cmd_map) {
574 percpu_ida_destroy(&se_sess->sess_tag_pool);
575 kvfree(se_sess->sess_cmd_map);
576 }
577 kmem_cache_free(se_sess_cache, se_sess);
578}
579EXPORT_SYMBOL(transport_free_session);
580
581void transport_deregister_session(struct se_session *se_sess)
582{
583 struct se_portal_group *se_tpg = se_sess->se_tpg;
584 unsigned long flags;
585
586 if (!se_tpg) {
587 transport_free_session(se_sess);
588 return;
589 }
590
591 spin_lock_irqsave(&se_tpg->session_lock, flags);
592 list_del(&se_sess->sess_list);
593 se_sess->se_tpg = NULL;
594 se_sess->fabric_sess_ptr = NULL;
595 spin_unlock_irqrestore(&se_tpg->session_lock, flags);
596
597 pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
598 se_tpg->se_tpg_tfo->get_fabric_name());
599
600
601
602
603
604
605
606
607
608 transport_free_session(se_sess);
609}
610EXPORT_SYMBOL(transport_deregister_session);
611
612static void target_remove_from_state_list(struct se_cmd *cmd)
613{
614 struct se_device *dev = cmd->se_dev;
615 unsigned long flags;
616
617 if (!dev)
618 return;
619
620 spin_lock_irqsave(&dev->execute_task_lock, flags);
621 if (cmd->state_active) {
622 list_del(&cmd->state_list);
623 cmd->state_active = false;
624 }
625 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
626}
627
628static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
629{
630 unsigned long flags;
631
632 target_remove_from_state_list(cmd);
633
634
635
636
637 cmd->se_lun = NULL;
638
639 spin_lock_irqsave(&cmd->t_state_lock, flags);
640
641
642
643
644 if (cmd->transport_state & CMD_T_STOP) {
645 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
646 __func__, __LINE__, cmd->tag);
647
648 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
649
650 complete_all(&cmd->t_transport_stop_comp);
651 return 1;
652 }
653 cmd->transport_state &= ~CMD_T_ACTIVE;
654 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
655
656
657
658
659
660
661
662
663 return cmd->se_tfo->check_stop_free(cmd);
664}
665
666static void transport_lun_remove_cmd(struct se_cmd *cmd)
667{
668 struct se_lun *lun = cmd->se_lun;
669
670 if (!lun)
671 return;
672
673 if (cmpxchg(&cmd->lun_ref_active, true, false))
674 percpu_ref_put(&lun->lun_ref);
675}
676
677int transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
678{
679 bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF);
680 int ret = 0;
681
682 if (cmd->se_cmd_flags & SCF_SE_LUN_CMD)
683 transport_lun_remove_cmd(cmd);
684
685
686
687
688 if (remove)
689 cmd->se_tfo->aborted_task(cmd);
690
691 if (transport_cmd_check_stop_to_fabric(cmd))
692 return 1;
693 if (remove && ack_kref)
694 ret = target_put_sess_cmd(cmd);
695
696 return ret;
697}
698
699static void target_complete_failure_work(struct work_struct *work)
700{
701 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
702
703 transport_generic_request_failure(cmd,
704 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
705}
706
707
708
709
710
711static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd)
712{
713 struct se_device *dev = cmd->se_dev;
714
715 WARN_ON(!cmd->se_lun);
716
717 if (!dev)
718 return NULL;
719
720 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION)
721 return NULL;
722
723 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
724
725 pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n",
726 dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status);
727 return cmd->sense_buffer;
728}
729
730void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense)
731{
732 unsigned char *cmd_sense_buf;
733 unsigned long flags;
734
735 spin_lock_irqsave(&cmd->t_state_lock, flags);
736 cmd_sense_buf = transport_get_sense_buffer(cmd);
737 if (!cmd_sense_buf) {
738 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
739 return;
740 }
741
742 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
743 memcpy(cmd_sense_buf, sense, cmd->scsi_sense_length);
744 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
745}
746EXPORT_SYMBOL(transport_copy_sense_to_cmd);
747
748void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
749{
750 struct se_device *dev = cmd->se_dev;
751 int success;
752 unsigned long flags;
753
754 cmd->scsi_status = scsi_status;
755
756 spin_lock_irqsave(&cmd->t_state_lock, flags);
757
758 if (dev && dev->transport->transport_complete) {
759 dev->transport->transport_complete(cmd,
760 cmd->t_data_sg,
761 transport_get_sense_buffer(cmd));
762 }
763 switch (cmd->scsi_status) {
764 case SAM_STAT_CHECK_CONDITION:
765 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
766 success = 1;
767 else
768 success = 0;
769 break;
770 default:
771 success = 1;
772 break;
773 }
774
775
776
777
778
779 if (cmd->transport_state & CMD_T_ABORTED ||
780 cmd->transport_state & CMD_T_STOP) {
781 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
782
783
784
785
786
787
788 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) {
789 up(&dev->caw_sem);
790 }
791 complete_all(&cmd->t_transport_stop_comp);
792 return;
793 } else if (!success) {
794 INIT_WORK(&cmd->work, target_complete_failure_work);
795 } else {
796 INIT_WORK(&cmd->work, target_complete_ok_work);
797 }
798
799 cmd->t_state = TRANSPORT_COMPLETE;
800 cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
801 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
802
803 if (cmd->se_cmd_flags & SCF_USE_CPUID)
804 queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work);
805 else
806 queue_work(target_completion_wq, &cmd->work);
807}
808EXPORT_SYMBOL(target_complete_cmd);
809
810void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length)
811{
812 if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) {
813 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
814 cmd->residual_count += cmd->data_length - length;
815 } else {
816 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
817 cmd->residual_count = cmd->data_length - length;
818 }
819
820 cmd->data_length = length;
821 }
822
823 target_complete_cmd(cmd, scsi_status);
824}
825EXPORT_SYMBOL(target_complete_cmd_with_length);
826
827static void target_add_to_state_list(struct se_cmd *cmd)
828{
829 struct se_device *dev = cmd->se_dev;
830 unsigned long flags;
831
832 spin_lock_irqsave(&dev->execute_task_lock, flags);
833 if (!cmd->state_active) {
834 list_add_tail(&cmd->state_list, &dev->state_list);
835 cmd->state_active = true;
836 }
837 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
838}
839
840
841
842
843static void transport_write_pending_qf(struct se_cmd *cmd);
844static void transport_complete_qf(struct se_cmd *cmd);
845
846void target_qf_do_work(struct work_struct *work)
847{
848 struct se_device *dev = container_of(work, struct se_device,
849 qf_work_queue);
850 LIST_HEAD(qf_cmd_list);
851 struct se_cmd *cmd, *cmd_tmp;
852
853 spin_lock_irq(&dev->qf_cmd_lock);
854 list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
855 spin_unlock_irq(&dev->qf_cmd_lock);
856
857 list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
858 list_del(&cmd->se_qf_node);
859 atomic_dec_mb(&dev->dev_qf_count);
860
861 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
862 " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
863 (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
864 (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
865 : "UNKNOWN");
866
867 if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP)
868 transport_write_pending_qf(cmd);
869 else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK ||
870 cmd->t_state == TRANSPORT_COMPLETE_QF_ERR)
871 transport_complete_qf(cmd);
872 }
873}
874
875unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
876{
877 switch (cmd->data_direction) {
878 case DMA_NONE:
879 return "NONE";
880 case DMA_FROM_DEVICE:
881 return "READ";
882 case DMA_TO_DEVICE:
883 return "WRITE";
884 case DMA_BIDIRECTIONAL:
885 return "BIDI";
886 default:
887 break;
888 }
889
890 return "UNKNOWN";
891}
892
893void transport_dump_dev_state(
894 struct se_device *dev,
895 char *b,
896 int *bl)
897{
898 *bl += sprintf(b + *bl, "Status: ");
899 if (dev->export_count)
900 *bl += sprintf(b + *bl, "ACTIVATED");
901 else
902 *bl += sprintf(b + *bl, "DEACTIVATED");
903
904 *bl += sprintf(b + *bl, " Max Queue Depth: %d", dev->queue_depth);
905 *bl += sprintf(b + *bl, " SectorSize: %u HwMaxSectors: %u\n",
906 dev->dev_attrib.block_size,
907 dev->dev_attrib.hw_max_sectors);
908 *bl += sprintf(b + *bl, " ");
909}
910
911void transport_dump_vpd_proto_id(
912 struct t10_vpd *vpd,
913 unsigned char *p_buf,
914 int p_buf_len)
915{
916 unsigned char buf[VPD_TMP_BUF_SIZE];
917 int len;
918
919 memset(buf, 0, VPD_TMP_BUF_SIZE);
920 len = sprintf(buf, "T10 VPD Protocol Identifier: ");
921
922 switch (vpd->protocol_identifier) {
923 case 0x00:
924 sprintf(buf+len, "Fibre Channel\n");
925 break;
926 case 0x10:
927 sprintf(buf+len, "Parallel SCSI\n");
928 break;
929 case 0x20:
930 sprintf(buf+len, "SSA\n");
931 break;
932 case 0x30:
933 sprintf(buf+len, "IEEE 1394\n");
934 break;
935 case 0x40:
936 sprintf(buf+len, "SCSI Remote Direct Memory Access"
937 " Protocol\n");
938 break;
939 case 0x50:
940 sprintf(buf+len, "Internet SCSI (iSCSI)\n");
941 break;
942 case 0x60:
943 sprintf(buf+len, "SAS Serial SCSI Protocol\n");
944 break;
945 case 0x70:
946 sprintf(buf+len, "Automation/Drive Interface Transport"
947 " Protocol\n");
948 break;
949 case 0x80:
950 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
951 break;
952 default:
953 sprintf(buf+len, "Unknown 0x%02x\n",
954 vpd->protocol_identifier);
955 break;
956 }
957
958 if (p_buf)
959 strncpy(p_buf, buf, p_buf_len);
960 else
961 pr_debug("%s", buf);
962}
963
964void
965transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
966{
967
968
969
970
971
972 if (page_83[1] & 0x80) {
973 vpd->protocol_identifier = (page_83[0] & 0xf0);
974 vpd->protocol_identifier_set = 1;
975 transport_dump_vpd_proto_id(vpd, NULL, 0);
976 }
977}
978EXPORT_SYMBOL(transport_set_vpd_proto_id);
979
980int transport_dump_vpd_assoc(
981 struct t10_vpd *vpd,
982 unsigned char *p_buf,
983 int p_buf_len)
984{
985 unsigned char buf[VPD_TMP_BUF_SIZE];
986 int ret = 0;
987 int len;
988
989 memset(buf, 0, VPD_TMP_BUF_SIZE);
990 len = sprintf(buf, "T10 VPD Identifier Association: ");
991
992 switch (vpd->association) {
993 case 0x00:
994 sprintf(buf+len, "addressed logical unit\n");
995 break;
996 case 0x10:
997 sprintf(buf+len, "target port\n");
998 break;
999 case 0x20:
1000 sprintf(buf+len, "SCSI target device\n");
1001 break;
1002 default:
1003 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
1004 ret = -EINVAL;
1005 break;
1006 }
1007
1008 if (p_buf)
1009 strncpy(p_buf, buf, p_buf_len);
1010 else
1011 pr_debug("%s", buf);
1012
1013 return ret;
1014}
1015
1016int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
1017{
1018
1019
1020
1021
1022
1023 vpd->association = (page_83[1] & 0x30);
1024 return transport_dump_vpd_assoc(vpd, NULL, 0);
1025}
1026EXPORT_SYMBOL(transport_set_vpd_assoc);
1027
1028int transport_dump_vpd_ident_type(
1029 struct t10_vpd *vpd,
1030 unsigned char *p_buf,
1031 int p_buf_len)
1032{
1033 unsigned char buf[VPD_TMP_BUF_SIZE];
1034 int ret = 0;
1035 int len;
1036
1037 memset(buf, 0, VPD_TMP_BUF_SIZE);
1038 len = sprintf(buf, "T10 VPD Identifier Type: ");
1039
1040 switch (vpd->device_identifier_type) {
1041 case 0x00:
1042 sprintf(buf+len, "Vendor specific\n");
1043 break;
1044 case 0x01:
1045 sprintf(buf+len, "T10 Vendor ID based\n");
1046 break;
1047 case 0x02:
1048 sprintf(buf+len, "EUI-64 based\n");
1049 break;
1050 case 0x03:
1051 sprintf(buf+len, "NAA\n");
1052 break;
1053 case 0x04:
1054 sprintf(buf+len, "Relative target port identifier\n");
1055 break;
1056 case 0x08:
1057 sprintf(buf+len, "SCSI name string\n");
1058 break;
1059 default:
1060 sprintf(buf+len, "Unsupported: 0x%02x\n",
1061 vpd->device_identifier_type);
1062 ret = -EINVAL;
1063 break;
1064 }
1065
1066 if (p_buf) {
1067 if (p_buf_len < strlen(buf)+1)
1068 return -EINVAL;
1069 strncpy(p_buf, buf, p_buf_len);
1070 } else {
1071 pr_debug("%s", buf);
1072 }
1073
1074 return ret;
1075}
1076
1077int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
1078{
1079
1080
1081
1082
1083
1084 vpd->device_identifier_type = (page_83[1] & 0x0f);
1085 return transport_dump_vpd_ident_type(vpd, NULL, 0);
1086}
1087EXPORT_SYMBOL(transport_set_vpd_ident_type);
1088
1089int transport_dump_vpd_ident(
1090 struct t10_vpd *vpd,
1091 unsigned char *p_buf,
1092 int p_buf_len)
1093{
1094 unsigned char buf[VPD_TMP_BUF_SIZE];
1095 int ret = 0;
1096
1097 memset(buf, 0, VPD_TMP_BUF_SIZE);
1098
1099 switch (vpd->device_identifier_code_set) {
1100 case 0x01:
1101 snprintf(buf, sizeof(buf),
1102 "T10 VPD Binary Device Identifier: %s\n",
1103 &vpd->device_identifier[0]);
1104 break;
1105 case 0x02:
1106 snprintf(buf, sizeof(buf),
1107 "T10 VPD ASCII Device Identifier: %s\n",
1108 &vpd->device_identifier[0]);
1109 break;
1110 case 0x03:
1111 snprintf(buf, sizeof(buf),
1112 "T10 VPD UTF-8 Device Identifier: %s\n",
1113 &vpd->device_identifier[0]);
1114 break;
1115 default:
1116 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
1117 " 0x%02x", vpd->device_identifier_code_set);
1118 ret = -EINVAL;
1119 break;
1120 }
1121
1122 if (p_buf)
1123 strncpy(p_buf, buf, p_buf_len);
1124 else
1125 pr_debug("%s", buf);
1126
1127 return ret;
1128}
1129
1130int
1131transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
1132{
1133 static const char hex_str[] = "0123456789abcdef";
1134 int j = 0, i = 4;
1135
1136
1137
1138
1139
1140
1141 vpd->device_identifier_code_set = (page_83[0] & 0x0f);
1142 switch (vpd->device_identifier_code_set) {
1143 case 0x01:
1144 vpd->device_identifier[j++] =
1145 hex_str[vpd->device_identifier_type];
1146 while (i < (4 + page_83[3])) {
1147 vpd->device_identifier[j++] =
1148 hex_str[(page_83[i] & 0xf0) >> 4];
1149 vpd->device_identifier[j++] =
1150 hex_str[page_83[i] & 0x0f];
1151 i++;
1152 }
1153 break;
1154 case 0x02:
1155 case 0x03:
1156 while (i < (4 + page_83[3]))
1157 vpd->device_identifier[j++] = page_83[i++];
1158 break;
1159 default:
1160 break;
1161 }
1162
1163 return transport_dump_vpd_ident(vpd, NULL, 0);
1164}
1165EXPORT_SYMBOL(transport_set_vpd_ident);
1166
1167static sense_reason_t
1168target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev,
1169 unsigned int size)
1170{
1171 u32 mtl;
1172
1173 if (!cmd->se_tfo->max_data_sg_nents)
1174 return TCM_NO_SENSE;
1175
1176
1177
1178
1179
1180
1181 mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE);
1182 if (cmd->data_length > mtl) {
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
1195 cmd->residual_count = (size - mtl);
1196 } else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
1197 u32 orig_dl = size + cmd->residual_count;
1198 cmd->residual_count = (orig_dl - mtl);
1199 } else {
1200 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
1201 cmd->residual_count = (cmd->data_length - mtl);
1202 }
1203 cmd->data_length = mtl;
1204
1205
1206
1207
1208 if (cmd->prot_length) {
1209 u32 sectors = (mtl / dev->dev_attrib.block_size);
1210 cmd->prot_length = dev->prot_length * sectors;
1211 }
1212 }
1213 return TCM_NO_SENSE;
1214}
1215
1216sense_reason_t
1217target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
1218{
1219 struct se_device *dev = cmd->se_dev;
1220
1221 if (cmd->unknown_data_length) {
1222 cmd->data_length = size;
1223 } else if (size != cmd->data_length) {
1224 pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:"
1225 " %u does not match SCSI CDB Length: %u for SAM Opcode:"
1226 " 0x%02x\n", cmd->se_tfo->get_fabric_name(),
1227 cmd->data_length, size, cmd->t_task_cdb[0]);
1228
1229 if (cmd->data_direction == DMA_TO_DEVICE) {
1230 if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) {
1231 pr_err_ratelimited("Rejecting underflow/overflow"
1232 " for WRITE data CDB\n");
1233 return TCM_INVALID_CDB_FIELD;
1234 }
1235
1236
1237
1238
1239
1240
1241 if (size > cmd->data_length) {
1242 pr_err_ratelimited("Rejecting overflow for"
1243 " WRITE control CDB\n");
1244 return TCM_INVALID_CDB_FIELD;
1245 }
1246 }
1247
1248
1249
1250
1251 if (dev->dev_attrib.block_size != 512) {
1252 pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
1253 " CDB on non 512-byte sector setup subsystem"
1254 " plugin: %s\n", dev->transport->name);
1255
1256 return TCM_INVALID_CDB_FIELD;
1257 }
1258
1259
1260
1261
1262
1263
1264 if (size > cmd->data_length) {
1265 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
1266 cmd->residual_count = (size - cmd->data_length);
1267 } else {
1268 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
1269 cmd->residual_count = (cmd->data_length - size);
1270 cmd->data_length = size;
1271 }
1272 }
1273
1274 return target_check_max_data_sg_nents(cmd, dev, size);
1275
1276}
1277
1278
1279
1280
1281
1282
1283
1284void transport_init_se_cmd(
1285 struct se_cmd *cmd,
1286 const struct target_core_fabric_ops *tfo,
1287 struct se_session *se_sess,
1288 u32 data_length,
1289 int data_direction,
1290 int task_attr,
1291 unsigned char *sense_buffer)
1292{
1293 INIT_LIST_HEAD(&cmd->se_delayed_node);
1294 INIT_LIST_HEAD(&cmd->se_qf_node);
1295 INIT_LIST_HEAD(&cmd->se_cmd_list);
1296 INIT_LIST_HEAD(&cmd->state_list);
1297 init_completion(&cmd->t_transport_stop_comp);
1298 init_completion(&cmd->cmd_wait_comp);
1299 spin_lock_init(&cmd->t_state_lock);
1300 INIT_WORK(&cmd->work, NULL);
1301 kref_init(&cmd->cmd_kref);
1302
1303 cmd->se_tfo = tfo;
1304 cmd->se_sess = se_sess;
1305 cmd->data_length = data_length;
1306 cmd->data_direction = data_direction;
1307 cmd->sam_task_attr = task_attr;
1308 cmd->sense_buffer = sense_buffer;
1309
1310 cmd->state_active = false;
1311}
1312EXPORT_SYMBOL(transport_init_se_cmd);
1313
1314static sense_reason_t
1315transport_check_alloc_task_attr(struct se_cmd *cmd)
1316{
1317 struct se_device *dev = cmd->se_dev;
1318
1319
1320
1321
1322
1323 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1324 return 0;
1325
1326 if (cmd->sam_task_attr == TCM_ACA_TAG) {
1327 pr_debug("SAM Task Attribute ACA"
1328 " emulation is not supported\n");
1329 return TCM_INVALID_CDB_FIELD;
1330 }
1331
1332 return 0;
1333}
1334
1335sense_reason_t
1336target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
1337{
1338 struct se_device *dev = cmd->se_dev;
1339 sense_reason_t ret;
1340
1341
1342
1343
1344
1345 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1346 pr_err("Received SCSI CDB with command_size: %d that"
1347 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1348 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1349 return TCM_INVALID_CDB_FIELD;
1350 }
1351
1352
1353
1354
1355
1356 if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
1357 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
1358 GFP_KERNEL);
1359 if (!cmd->t_task_cdb) {
1360 pr_err("Unable to allocate cmd->t_task_cdb"
1361 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
1362 scsi_command_size(cdb),
1363 (unsigned long)sizeof(cmd->__t_task_cdb));
1364 return TCM_OUT_OF_RESOURCES;
1365 }
1366 } else
1367 cmd->t_task_cdb = &cmd->__t_task_cdb[0];
1368
1369
1370
1371 memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
1372
1373 trace_target_sequencer_start(cmd);
1374
1375 ret = dev->transport->parse_cdb(cmd);
1376 if (ret == TCM_UNSUPPORTED_SCSI_OPCODE)
1377 pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n",
1378 cmd->se_tfo->get_fabric_name(),
1379 cmd->se_sess->se_node_acl->initiatorname,
1380 cmd->t_task_cdb[0]);
1381 if (ret)
1382 return ret;
1383
1384 ret = transport_check_alloc_task_attr(cmd);
1385 if (ret)
1386 return ret;
1387
1388 cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
1389 atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus);
1390 return 0;
1391}
1392EXPORT_SYMBOL(target_setup_cmd_from_cdb);
1393
1394
1395
1396
1397
1398int transport_handle_cdb_direct(
1399 struct se_cmd *cmd)
1400{
1401 sense_reason_t ret;
1402
1403 if (!cmd->se_lun) {
1404 dump_stack();
1405 pr_err("cmd->se_lun is NULL\n");
1406 return -EINVAL;
1407 }
1408 if (in_interrupt()) {
1409 dump_stack();
1410 pr_err("transport_generic_handle_cdb cannot be called"
1411 " from interrupt context\n");
1412 return -EINVAL;
1413 }
1414
1415
1416
1417
1418
1419
1420
1421
1422 cmd->t_state = TRANSPORT_NEW_CMD;
1423 cmd->transport_state |= CMD_T_ACTIVE;
1424
1425
1426
1427
1428
1429
1430 ret = transport_generic_new_cmd(cmd);
1431 if (ret)
1432 transport_generic_request_failure(cmd, ret);
1433 return 0;
1434}
1435EXPORT_SYMBOL(transport_handle_cdb_direct);
1436
1437sense_reason_t
1438transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
1439 u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count)
1440{
1441 if (!sgl || !sgl_count)
1442 return 0;
1443
1444
1445
1446
1447
1448
1449 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
1450 pr_warn("Rejecting SCSI DATA overflow for fabric using"
1451 " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
1452 return TCM_INVALID_CDB_FIELD;
1453 }
1454
1455 cmd->t_data_sg = sgl;
1456 cmd->t_data_nents = sgl_count;
1457 cmd->t_bidi_data_sg = sgl_bidi;
1458 cmd->t_bidi_data_nents = sgl_bidi_count;
1459
1460 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1461 return 0;
1462}
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess,
1494 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
1495 u32 data_length, int task_attr, int data_dir, int flags,
1496 struct scatterlist *sgl, u32 sgl_count,
1497 struct scatterlist *sgl_bidi, u32 sgl_bidi_count,
1498 struct scatterlist *sgl_prot, u32 sgl_prot_count)
1499{
1500 struct se_portal_group *se_tpg;
1501 sense_reason_t rc;
1502 int ret;
1503
1504 se_tpg = se_sess->se_tpg;
1505 BUG_ON(!se_tpg);
1506 BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
1507 BUG_ON(in_interrupt());
1508
1509
1510
1511
1512
1513 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1514 data_length, data_dir, task_attr, sense);
1515
1516 if (flags & TARGET_SCF_USE_CPUID)
1517 se_cmd->se_cmd_flags |= SCF_USE_CPUID;
1518 else
1519 se_cmd->cpuid = WORK_CPU_UNBOUND;
1520
1521 if (flags & TARGET_SCF_UNKNOWN_SIZE)
1522 se_cmd->unknown_data_length = 1;
1523
1524
1525
1526
1527
1528
1529 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
1530 if (ret)
1531 return ret;
1532
1533
1534
1535 if (flags & TARGET_SCF_BIDI_OP)
1536 se_cmd->se_cmd_flags |= SCF_BIDI;
1537
1538
1539
1540 rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun);
1541 if (rc) {
1542 transport_send_check_condition_and_sense(se_cmd, rc, 0);
1543 target_put_sess_cmd(se_cmd);
1544 return 0;
1545 }
1546
1547 rc = target_setup_cmd_from_cdb(se_cmd, cdb);
1548 if (rc != 0) {
1549 transport_generic_request_failure(se_cmd, rc);
1550 return 0;
1551 }
1552
1553
1554
1555
1556
1557 if (sgl_prot_count) {
1558 se_cmd->t_prot_sg = sgl_prot;
1559 se_cmd->t_prot_nents = sgl_prot_count;
1560 se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC;
1561 }
1562
1563
1564
1565
1566
1567
1568 if (sgl_count != 0) {
1569 BUG_ON(!sgl);
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579 if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) &&
1580 se_cmd->data_direction == DMA_FROM_DEVICE) {
1581 unsigned char *buf = NULL;
1582
1583 if (sgl)
1584 buf = kmap(sg_page(sgl)) + sgl->offset;
1585
1586 if (buf) {
1587 memset(buf, 0, sgl->length);
1588 kunmap(sg_page(sgl));
1589 }
1590 }
1591
1592 rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count,
1593 sgl_bidi, sgl_bidi_count);
1594 if (rc != 0) {
1595 transport_generic_request_failure(se_cmd, rc);
1596 return 0;
1597 }
1598 }
1599
1600
1601
1602
1603
1604 core_alua_check_nonop_delay(se_cmd);
1605
1606 transport_handle_cdb_direct(se_cmd);
1607 return 0;
1608}
1609EXPORT_SYMBOL(target_submit_cmd_map_sgls);
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
1636 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
1637 u32 data_length, int task_attr, int data_dir, int flags)
1638{
1639 return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense,
1640 unpacked_lun, data_length, task_attr, data_dir,
1641 flags, NULL, 0, NULL, 0, NULL, 0);
1642}
1643EXPORT_SYMBOL(target_submit_cmd);
1644
1645static void target_complete_tmr_failure(struct work_struct *work)
1646{
1647 struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);
1648
1649 se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
1650 se_cmd->se_tfo->queue_tm_rsp(se_cmd);
1651
1652 transport_lun_remove_cmd(se_cmd);
1653 transport_cmd_check_stop_to_fabric(se_cmd);
1654}
1655
1656static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag,
1657 u64 *unpacked_lun)
1658{
1659 struct se_cmd *se_cmd;
1660 unsigned long flags;
1661 bool ret = false;
1662
1663 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
1664 list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) {
1665 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
1666 continue;
1667
1668 if (se_cmd->tag == tag) {
1669 *unpacked_lun = se_cmd->orig_fe_lun;
1670 ret = true;
1671 break;
1672 }
1673 }
1674 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
1675
1676 return ret;
1677}
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
1697 unsigned char *sense, u64 unpacked_lun,
1698 void *fabric_tmr_ptr, unsigned char tm_type,
1699 gfp_t gfp, u64 tag, int flags)
1700{
1701 struct se_portal_group *se_tpg;
1702 int ret;
1703
1704 se_tpg = se_sess->se_tpg;
1705 BUG_ON(!se_tpg);
1706
1707 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1708 0, DMA_NONE, TCM_SIMPLE_TAG, sense);
1709
1710
1711
1712
1713 ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp);
1714 if (ret < 0)
1715 return -ENOMEM;
1716
1717 if (tm_type == TMR_ABORT_TASK)
1718 se_cmd->se_tmr_req->ref_task_tag = tag;
1719
1720
1721 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
1722 if (ret) {
1723 core_tmr_release_req(se_cmd->se_tmr_req);
1724 return ret;
1725 }
1726
1727
1728
1729
1730
1731 if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) {
1732 if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun))
1733 goto failure;
1734 }
1735
1736 ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
1737 if (ret)
1738 goto failure;
1739
1740 transport_generic_handle_tmr(se_cmd);
1741 return 0;
1742
1743
1744
1745
1746
1747failure:
1748 INIT_WORK(&se_cmd->work, target_complete_tmr_failure);
1749 schedule_work(&se_cmd->work);
1750 return 0;
1751}
1752EXPORT_SYMBOL(target_submit_tmr);
1753
1754
1755
1756
1757void transport_generic_request_failure(struct se_cmd *cmd,
1758 sense_reason_t sense_reason)
1759{
1760 int ret = 0, post_ret = 0;
1761
1762 pr_debug("-----[ Storage Engine Exception; sense_reason %d\n",
1763 sense_reason);
1764 target_show_cmd("-----[ ", cmd);
1765
1766
1767
1768
1769 transport_complete_task_attr(cmd);
1770
1771
1772
1773
1774
1775 if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
1776 cmd->transport_complete_callback)
1777 cmd->transport_complete_callback(cmd, false, &post_ret);
1778
1779 if (transport_check_aborted_status(cmd, 1))
1780 return;
1781
1782 switch (sense_reason) {
1783 case TCM_NON_EXISTENT_LUN:
1784 case TCM_UNSUPPORTED_SCSI_OPCODE:
1785 case TCM_INVALID_CDB_FIELD:
1786 case TCM_INVALID_PARAMETER_LIST:
1787 case TCM_PARAMETER_LIST_LENGTH_ERROR:
1788 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
1789 case TCM_UNKNOWN_MODE_PAGE:
1790 case TCM_WRITE_PROTECTED:
1791 case TCM_ADDRESS_OUT_OF_RANGE:
1792 case TCM_CHECK_CONDITION_ABORT_CMD:
1793 case TCM_CHECK_CONDITION_UNIT_ATTENTION:
1794 case TCM_CHECK_CONDITION_NOT_READY:
1795 case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED:
1796 case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED:
1797 case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED:
1798 case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE:
1799 case TCM_TOO_MANY_TARGET_DESCS:
1800 case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE:
1801 case TCM_TOO_MANY_SEGMENT_DESCS:
1802 case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE:
1803 break;
1804 case TCM_OUT_OF_RESOURCES:
1805 cmd->scsi_status = SAM_STAT_TASK_SET_FULL;
1806 goto queue_status;
1807 case TCM_LUN_BUSY:
1808 cmd->scsi_status = SAM_STAT_BUSY;
1809 goto queue_status;
1810 case TCM_RESERVATION_CONFLICT:
1811
1812
1813
1814
1815
1816
1817 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1818
1819
1820
1821
1822
1823
1824
1825 if (cmd->se_sess &&
1826 cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) {
1827 target_ua_allocate_lun(cmd->se_sess->se_node_acl,
1828 cmd->orig_fe_lun, 0x2C,
1829 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
1830 }
1831
1832 goto queue_status;
1833 default:
1834 pr_err("Unknown transport error for CDB 0x%02x: %d\n",
1835 cmd->t_task_cdb[0], sense_reason);
1836 sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
1837 break;
1838 }
1839
1840 ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0);
1841 if (ret)
1842 goto queue_full;
1843
1844check_stop:
1845 transport_lun_remove_cmd(cmd);
1846 transport_cmd_check_stop_to_fabric(cmd);
1847 return;
1848
1849queue_status:
1850 trace_target_cmd_complete(cmd);
1851 ret = cmd->se_tfo->queue_status(cmd);
1852 if (!ret)
1853 goto check_stop;
1854queue_full:
1855 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
1856}
1857EXPORT_SYMBOL(transport_generic_request_failure);
1858
1859void __target_execute_cmd(struct se_cmd *cmd, bool do_checks)
1860{
1861 sense_reason_t ret;
1862
1863 if (!cmd->execute_cmd) {
1864 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1865 goto err;
1866 }
1867 if (do_checks) {
1868
1869
1870
1871
1872
1873
1874 ret = target_scsi3_ua_check(cmd);
1875 if (ret)
1876 goto err;
1877
1878 ret = target_alua_state_check(cmd);
1879 if (ret)
1880 goto err;
1881
1882 ret = target_check_reservation(cmd);
1883 if (ret) {
1884 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1885 goto err;
1886 }
1887 }
1888
1889 ret = cmd->execute_cmd(cmd);
1890 if (!ret)
1891 return;
1892err:
1893 spin_lock_irq(&cmd->t_state_lock);
1894 cmd->transport_state &= ~CMD_T_SENT;
1895 spin_unlock_irq(&cmd->t_state_lock);
1896
1897 transport_generic_request_failure(cmd, ret);
1898}
1899
1900static int target_write_prot_action(struct se_cmd *cmd)
1901{
1902 u32 sectors;
1903
1904
1905
1906
1907
1908 switch (cmd->prot_op) {
1909 case TARGET_PROT_DOUT_INSERT:
1910 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT))
1911 sbc_dif_generate(cmd);
1912 break;
1913 case TARGET_PROT_DOUT_STRIP:
1914 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP)
1915 break;
1916
1917 sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size);
1918 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
1919 sectors, 0, cmd->t_prot_sg, 0);
1920 if (unlikely(cmd->pi_err)) {
1921 spin_lock_irq(&cmd->t_state_lock);
1922 cmd->transport_state &= ~CMD_T_SENT;
1923 spin_unlock_irq(&cmd->t_state_lock);
1924 transport_generic_request_failure(cmd, cmd->pi_err);
1925 return -1;
1926 }
1927 break;
1928 default:
1929 break;
1930 }
1931
1932 return 0;
1933}
1934
1935static bool target_handle_task_attr(struct se_cmd *cmd)
1936{
1937 struct se_device *dev = cmd->se_dev;
1938
1939 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1940 return false;
1941
1942 cmd->se_cmd_flags |= SCF_TASK_ATTR_SET;
1943
1944
1945
1946
1947
1948 switch (cmd->sam_task_attr) {
1949 case TCM_HEAD_TAG:
1950 pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n",
1951 cmd->t_task_cdb[0]);
1952 return false;
1953 case TCM_ORDERED_TAG:
1954 atomic_inc_mb(&dev->dev_ordered_sync);
1955
1956 pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n",
1957 cmd->t_task_cdb[0]);
1958
1959
1960
1961
1962
1963 if (!atomic_read(&dev->simple_cmds))
1964 return false;
1965 break;
1966 default:
1967
1968
1969
1970 atomic_inc_mb(&dev->simple_cmds);
1971 break;
1972 }
1973
1974 if (atomic_read(&dev->dev_ordered_sync) == 0)
1975 return false;
1976
1977 spin_lock(&dev->delayed_cmd_lock);
1978 list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list);
1979 spin_unlock(&dev->delayed_cmd_lock);
1980
1981 pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn",
1982 cmd->t_task_cdb[0], cmd->sam_task_attr);
1983 return true;
1984}
1985
1986static int __transport_check_aborted_status(struct se_cmd *, int);
1987
1988void target_execute_cmd(struct se_cmd *cmd)
1989{
1990
1991
1992
1993
1994
1995
1996 spin_lock_irq(&cmd->t_state_lock);
1997 if (__transport_check_aborted_status(cmd, 1)) {
1998 spin_unlock_irq(&cmd->t_state_lock);
1999 return;
2000 }
2001 if (cmd->transport_state & CMD_T_STOP) {
2002 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
2003 __func__, __LINE__, cmd->tag);
2004
2005 spin_unlock_irq(&cmd->t_state_lock);
2006 complete_all(&cmd->t_transport_stop_comp);
2007 return;
2008 }
2009
2010 cmd->t_state = TRANSPORT_PROCESSING;
2011 cmd->transport_state &= ~CMD_T_PRE_EXECUTE;
2012 cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT;
2013 spin_unlock_irq(&cmd->t_state_lock);
2014
2015 if (target_write_prot_action(cmd))
2016 return;
2017
2018 if (target_handle_task_attr(cmd)) {
2019 spin_lock_irq(&cmd->t_state_lock);
2020 cmd->transport_state &= ~CMD_T_SENT;
2021 spin_unlock_irq(&cmd->t_state_lock);
2022 return;
2023 }
2024
2025 __target_execute_cmd(cmd, true);
2026}
2027EXPORT_SYMBOL(target_execute_cmd);
2028
2029
2030
2031
2032
2033static void target_restart_delayed_cmds(struct se_device *dev)
2034{
2035 for (;;) {
2036 struct se_cmd *cmd;
2037
2038 spin_lock(&dev->delayed_cmd_lock);
2039 if (list_empty(&dev->delayed_cmd_list)) {
2040 spin_unlock(&dev->delayed_cmd_lock);
2041 break;
2042 }
2043
2044 cmd = list_entry(dev->delayed_cmd_list.next,
2045 struct se_cmd, se_delayed_node);
2046 list_del(&cmd->se_delayed_node);
2047 spin_unlock(&dev->delayed_cmd_lock);
2048
2049 cmd->transport_state |= CMD_T_SENT;
2050
2051 __target_execute_cmd(cmd, true);
2052
2053 if (cmd->sam_task_attr == TCM_ORDERED_TAG)
2054 break;
2055 }
2056}
2057
2058
2059
2060
2061
2062static void transport_complete_task_attr(struct se_cmd *cmd)
2063{
2064 struct se_device *dev = cmd->se_dev;
2065
2066 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
2067 return;
2068
2069 if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET))
2070 goto restart;
2071
2072 if (cmd->sam_task_attr == TCM_SIMPLE_TAG) {
2073 atomic_dec_mb(&dev->simple_cmds);
2074 dev->dev_cur_ordered_id++;
2075 } else if (cmd->sam_task_attr == TCM_HEAD_TAG) {
2076 dev->dev_cur_ordered_id++;
2077 pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n",
2078 dev->dev_cur_ordered_id);
2079 } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) {
2080 atomic_dec_mb(&dev->dev_ordered_sync);
2081
2082 dev->dev_cur_ordered_id++;
2083 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n",
2084 dev->dev_cur_ordered_id);
2085 }
2086 cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET;
2087
2088restart:
2089 target_restart_delayed_cmds(dev);
2090}
2091
2092static void transport_complete_qf(struct se_cmd *cmd)
2093{
2094 int ret = 0;
2095
2096 transport_complete_task_attr(cmd);
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106 if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) {
2107 if (cmd->scsi_status)
2108 goto queue_status;
2109
2110 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
2111 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
2112 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
2113 goto queue_status;
2114 }
2115
2116 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
2117 goto queue_status;
2118
2119 switch (cmd->data_direction) {
2120 case DMA_FROM_DEVICE:
2121 if (cmd->scsi_status)
2122 goto queue_status;
2123
2124 trace_target_cmd_complete(cmd);
2125 ret = cmd->se_tfo->queue_data_in(cmd);
2126 break;
2127 case DMA_TO_DEVICE:
2128 if (cmd->se_cmd_flags & SCF_BIDI) {
2129 ret = cmd->se_tfo->queue_data_in(cmd);
2130 break;
2131 }
2132
2133 case DMA_NONE:
2134queue_status:
2135 trace_target_cmd_complete(cmd);
2136 ret = cmd->se_tfo->queue_status(cmd);
2137 break;
2138 default:
2139 break;
2140 }
2141
2142 if (ret < 0) {
2143 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
2144 return;
2145 }
2146 transport_lun_remove_cmd(cmd);
2147 transport_cmd_check_stop_to_fabric(cmd);
2148}
2149
2150static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev,
2151 int err, bool write_pending)
2152{
2153
2154
2155
2156
2157
2158
2159
2160
2161 if (err == -EAGAIN || err == -ENOMEM) {
2162 cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP :
2163 TRANSPORT_COMPLETE_QF_OK;
2164 } else {
2165 pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err);
2166 cmd->t_state = TRANSPORT_COMPLETE_QF_ERR;
2167 }
2168
2169 spin_lock_irq(&dev->qf_cmd_lock);
2170 list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
2171 atomic_inc_mb(&dev->dev_qf_count);
2172 spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);
2173
2174 schedule_work(&cmd->se_dev->qf_work_queue);
2175}
2176
2177static bool target_read_prot_action(struct se_cmd *cmd)
2178{
2179 switch (cmd->prot_op) {
2180 case TARGET_PROT_DIN_STRIP:
2181 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) {
2182 u32 sectors = cmd->data_length >>
2183 ilog2(cmd->se_dev->dev_attrib.block_size);
2184
2185 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
2186 sectors, 0, cmd->t_prot_sg,
2187 0);
2188 if (cmd->pi_err)
2189 return true;
2190 }
2191 break;
2192 case TARGET_PROT_DIN_INSERT:
2193 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT)
2194 break;
2195
2196 sbc_dif_generate(cmd);
2197 break;
2198 default:
2199 break;
2200 }
2201
2202 return false;
2203}
2204
2205static void target_complete_ok_work(struct work_struct *work)
2206{
2207 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
2208 int ret;
2209
2210
2211
2212
2213
2214
2215 transport_complete_task_attr(cmd);
2216
2217
2218
2219
2220
2221 if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
2222 schedule_work(&cmd->se_dev->qf_work_queue);
2223
2224
2225
2226
2227
2228 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
2229 WARN_ON(!cmd->scsi_status);
2230 ret = transport_send_check_condition_and_sense(
2231 cmd, 0, 1);
2232 if (ret)
2233 goto queue_full;
2234
2235 transport_lun_remove_cmd(cmd);
2236 transport_cmd_check_stop_to_fabric(cmd);
2237 return;
2238 }
2239
2240
2241
2242
2243 if (cmd->transport_complete_callback) {
2244 sense_reason_t rc;
2245 bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE);
2246 bool zero_dl = !(cmd->data_length);
2247 int post_ret = 0;
2248
2249 rc = cmd->transport_complete_callback(cmd, true, &post_ret);
2250 if (!rc && !post_ret) {
2251 if (caw && zero_dl)
2252 goto queue_rsp;
2253
2254 return;
2255 } else if (rc) {
2256 ret = transport_send_check_condition_and_sense(cmd,
2257 rc, 0);
2258 if (ret)
2259 goto queue_full;
2260
2261 transport_lun_remove_cmd(cmd);
2262 transport_cmd_check_stop_to_fabric(cmd);
2263 return;
2264 }
2265 }
2266
2267queue_rsp:
2268 switch (cmd->data_direction) {
2269 case DMA_FROM_DEVICE:
2270 if (cmd->scsi_status)
2271 goto queue_status;
2272
2273 atomic_long_add(cmd->data_length,
2274 &cmd->se_lun->lun_stats.tx_data_octets);
2275
2276
2277
2278
2279
2280 if (target_read_prot_action(cmd)) {
2281 ret = transport_send_check_condition_and_sense(cmd,
2282 cmd->pi_err, 0);
2283 if (ret)
2284 goto queue_full;
2285
2286 transport_lun_remove_cmd(cmd);
2287 transport_cmd_check_stop_to_fabric(cmd);
2288 return;
2289 }
2290
2291 trace_target_cmd_complete(cmd);
2292 ret = cmd->se_tfo->queue_data_in(cmd);
2293 if (ret)
2294 goto queue_full;
2295 break;
2296 case DMA_TO_DEVICE:
2297 atomic_long_add(cmd->data_length,
2298 &cmd->se_lun->lun_stats.rx_data_octets);
2299
2300
2301
2302 if (cmd->se_cmd_flags & SCF_BIDI) {
2303 atomic_long_add(cmd->data_length,
2304 &cmd->se_lun->lun_stats.tx_data_octets);
2305 ret = cmd->se_tfo->queue_data_in(cmd);
2306 if (ret)
2307 goto queue_full;
2308 break;
2309 }
2310
2311 case DMA_NONE:
2312queue_status:
2313 trace_target_cmd_complete(cmd);
2314 ret = cmd->se_tfo->queue_status(cmd);
2315 if (ret)
2316 goto queue_full;
2317 break;
2318 default:
2319 break;
2320 }
2321
2322 transport_lun_remove_cmd(cmd);
2323 transport_cmd_check_stop_to_fabric(cmd);
2324 return;
2325
2326queue_full:
2327 pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
2328 " data_direction: %d\n", cmd, cmd->data_direction);
2329
2330 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
2331}
2332
2333void target_free_sgl(struct scatterlist *sgl, int nents)
2334{
2335 struct scatterlist *sg;
2336 int count;
2337
2338 for_each_sg(sgl, sg, nents, count)
2339 __free_page(sg_page(sg));
2340
2341 kfree(sgl);
2342}
2343EXPORT_SYMBOL(target_free_sgl);
2344
2345static inline void transport_reset_sgl_orig(struct se_cmd *cmd)
2346{
2347
2348
2349
2350
2351 if (!cmd->t_data_sg_orig)
2352 return;
2353
2354 kfree(cmd->t_data_sg);
2355 cmd->t_data_sg = cmd->t_data_sg_orig;
2356 cmd->t_data_sg_orig = NULL;
2357 cmd->t_data_nents = cmd->t_data_nents_orig;
2358 cmd->t_data_nents_orig = 0;
2359}
2360
2361static inline void transport_free_pages(struct se_cmd *cmd)
2362{
2363 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
2364 target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents);
2365 cmd->t_prot_sg = NULL;
2366 cmd->t_prot_nents = 0;
2367 }
2368
2369 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
2370
2371
2372
2373
2374 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) {
2375 target_free_sgl(cmd->t_bidi_data_sg,
2376 cmd->t_bidi_data_nents);
2377 cmd->t_bidi_data_sg = NULL;
2378 cmd->t_bidi_data_nents = 0;
2379 }
2380 transport_reset_sgl_orig(cmd);
2381 return;
2382 }
2383 transport_reset_sgl_orig(cmd);
2384
2385 target_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
2386 cmd->t_data_sg = NULL;
2387 cmd->t_data_nents = 0;
2388
2389 target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
2390 cmd->t_bidi_data_sg = NULL;
2391 cmd->t_bidi_data_nents = 0;
2392}
2393
2394void *transport_kmap_data_sg(struct se_cmd *cmd)
2395{
2396 struct scatterlist *sg = cmd->t_data_sg;
2397 struct page **pages;
2398 int i;
2399
2400
2401
2402
2403
2404
2405 if (!cmd->t_data_nents)
2406 return NULL;
2407
2408 BUG_ON(!sg);
2409 if (cmd->t_data_nents == 1)
2410 return kmap(sg_page(sg)) + sg->offset;
2411
2412
2413 pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL);
2414 if (!pages)
2415 return NULL;
2416
2417
2418 for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
2419 pages[i] = sg_page(sg);
2420 }
2421
2422 cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL);
2423 kfree(pages);
2424 if (!cmd->t_data_vmap)
2425 return NULL;
2426
2427 return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
2428}
2429EXPORT_SYMBOL(transport_kmap_data_sg);
2430
2431void transport_kunmap_data_sg(struct se_cmd *cmd)
2432{
2433 if (!cmd->t_data_nents) {
2434 return;
2435 } else if (cmd->t_data_nents == 1) {
2436 kunmap(sg_page(cmd->t_data_sg));
2437 return;
2438 }
2439
2440 vunmap(cmd->t_data_vmap);
2441 cmd->t_data_vmap = NULL;
2442}
2443EXPORT_SYMBOL(transport_kunmap_data_sg);
2444
2445int
2446target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length,
2447 bool zero_page, bool chainable)
2448{
2449 struct scatterlist *sg;
2450 struct page *page;
2451 gfp_t zero_flag = (zero_page) ? __GFP_ZERO : 0;
2452 unsigned int nalloc, nent;
2453 int i = 0;
2454
2455 nalloc = nent = DIV_ROUND_UP(length, PAGE_SIZE);
2456 if (chainable)
2457 nalloc++;
2458 sg = kmalloc_array(nalloc, sizeof(struct scatterlist), GFP_KERNEL);
2459 if (!sg)
2460 return -ENOMEM;
2461
2462 sg_init_table(sg, nalloc);
2463
2464 while (length) {
2465 u32 page_len = min_t(u32, length, PAGE_SIZE);
2466 page = alloc_page(GFP_KERNEL | zero_flag);
2467 if (!page)
2468 goto out;
2469
2470 sg_set_page(&sg[i], page, page_len, 0);
2471 length -= page_len;
2472 i++;
2473 }
2474 *sgl = sg;
2475 *nents = nent;
2476 return 0;
2477
2478out:
2479 while (i > 0) {
2480 i--;
2481 __free_page(sg_page(&sg[i]));
2482 }
2483 kfree(sg);
2484 return -ENOMEM;
2485}
2486EXPORT_SYMBOL(target_alloc_sgl);
2487
2488
2489
2490
2491
2492
2493sense_reason_t
2494transport_generic_new_cmd(struct se_cmd *cmd)
2495{
2496 unsigned long flags;
2497 int ret = 0;
2498 bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB);
2499
2500 if (cmd->prot_op != TARGET_PROT_NORMAL &&
2501 !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
2502 ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents,
2503 cmd->prot_length, true, false);
2504 if (ret < 0)
2505 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2506 }
2507
2508
2509
2510
2511
2512
2513 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
2514 cmd->data_length) {
2515
2516 if ((cmd->se_cmd_flags & SCF_BIDI) ||
2517 (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) {
2518 u32 bidi_length;
2519
2520 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)
2521 bidi_length = cmd->t_task_nolb *
2522 cmd->se_dev->dev_attrib.block_size;
2523 else
2524 bidi_length = cmd->data_length;
2525
2526 ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
2527 &cmd->t_bidi_data_nents,
2528 bidi_length, zero_flag, false);
2529 if (ret < 0)
2530 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2531 }
2532
2533 ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents,
2534 cmd->data_length, zero_flag, false);
2535 if (ret < 0)
2536 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2537 } else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
2538 cmd->data_length) {
2539
2540
2541
2542
2543 u32 caw_length = cmd->t_task_nolb *
2544 cmd->se_dev->dev_attrib.block_size;
2545
2546 ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
2547 &cmd->t_bidi_data_nents,
2548 caw_length, zero_flag, false);
2549 if (ret < 0)
2550 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2551 }
2552
2553
2554
2555
2556
2557 target_add_to_state_list(cmd);
2558 if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) {
2559 target_execute_cmd(cmd);
2560 return 0;
2561 }
2562
2563 spin_lock_irqsave(&cmd->t_state_lock, flags);
2564 cmd->t_state = TRANSPORT_WRITE_PENDING;
2565
2566
2567
2568
2569 if (cmd->transport_state & CMD_T_STOP) {
2570 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
2571 __func__, __LINE__, cmd->tag);
2572
2573 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2574
2575 complete_all(&cmd->t_transport_stop_comp);
2576 return 0;
2577 }
2578 cmd->transport_state &= ~CMD_T_ACTIVE;
2579 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2580
2581 ret = cmd->se_tfo->write_pending(cmd);
2582 if (ret)
2583 goto queue_full;
2584
2585 return 0;
2586
2587queue_full:
2588 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
2589 transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
2590 return 0;
2591}
2592EXPORT_SYMBOL(transport_generic_new_cmd);
2593
2594static void transport_write_pending_qf(struct se_cmd *cmd)
2595{
2596 unsigned long flags;
2597 int ret;
2598 bool stop;
2599
2600 spin_lock_irqsave(&cmd->t_state_lock, flags);
2601 stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED));
2602 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2603
2604 if (stop) {
2605 pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n",
2606 __func__, __LINE__, cmd->tag);
2607 complete_all(&cmd->t_transport_stop_comp);
2608 return;
2609 }
2610
2611 ret = cmd->se_tfo->write_pending(cmd);
2612 if (ret) {
2613 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
2614 cmd);
2615 transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
2616 }
2617}
2618
2619static bool
2620__transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *,
2621 unsigned long *flags);
2622
2623static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas)
2624{
2625 unsigned long flags;
2626
2627 spin_lock_irqsave(&cmd->t_state_lock, flags);
2628 __transport_wait_for_tasks(cmd, true, aborted, tas, &flags);
2629 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2630}
2631
2632int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
2633{
2634 int ret = 0;
2635 bool aborted = false, tas = false;
2636
2637 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) {
2638 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2639 target_wait_free_cmd(cmd, &aborted, &tas);
2640
2641 if (!aborted || tas)
2642 ret = target_put_sess_cmd(cmd);
2643 } else {
2644 if (wait_for_tasks)
2645 target_wait_free_cmd(cmd, &aborted, &tas);
2646
2647
2648
2649
2650
2651 if (cmd->state_active)
2652 target_remove_from_state_list(cmd);
2653
2654 if (cmd->se_lun)
2655 transport_lun_remove_cmd(cmd);
2656
2657 if (!aborted || tas)
2658 ret = target_put_sess_cmd(cmd);
2659 }
2660
2661
2662
2663
2664
2665
2666 if (aborted) {
2667 pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag);
2668 wait_for_completion(&cmd->cmd_wait_comp);
2669 cmd->se_tfo->release_cmd(cmd);
2670 ret = 1;
2671 }
2672 return ret;
2673}
2674EXPORT_SYMBOL(transport_generic_free_cmd);
2675
2676
2677
2678
2679
2680int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref)
2681{
2682 struct se_session *se_sess = se_cmd->se_sess;
2683 unsigned long flags;
2684 int ret = 0;
2685
2686
2687
2688
2689
2690
2691 if (ack_kref) {
2692 if (!kref_get_unless_zero(&se_cmd->cmd_kref))
2693 return -EINVAL;
2694
2695 se_cmd->se_cmd_flags |= SCF_ACK_KREF;
2696 }
2697
2698 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2699 if (se_sess->sess_tearing_down) {
2700 ret = -ESHUTDOWN;
2701 goto out;
2702 }
2703 se_cmd->transport_state |= CMD_T_PRE_EXECUTE;
2704 list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
2705out:
2706 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2707
2708 if (ret && ack_kref)
2709 target_put_sess_cmd(se_cmd);
2710
2711 return ret;
2712}
2713EXPORT_SYMBOL(target_get_sess_cmd);
2714
2715static void target_free_cmd_mem(struct se_cmd *cmd)
2716{
2717 transport_free_pages(cmd);
2718
2719 if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
2720 core_tmr_release_req(cmd->se_tmr_req);
2721 if (cmd->t_task_cdb != cmd->__t_task_cdb)
2722 kfree(cmd->t_task_cdb);
2723}
2724
2725static void target_release_cmd_kref(struct kref *kref)
2726{
2727 struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
2728 struct se_session *se_sess = se_cmd->se_sess;
2729 unsigned long flags;
2730 bool fabric_stop;
2731
2732 if (se_sess) {
2733 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2734
2735 spin_lock(&se_cmd->t_state_lock);
2736 fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) &&
2737 (se_cmd->transport_state & CMD_T_ABORTED);
2738 spin_unlock(&se_cmd->t_state_lock);
2739
2740 if (se_cmd->cmd_wait_set || fabric_stop) {
2741 list_del_init(&se_cmd->se_cmd_list);
2742 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2743 target_free_cmd_mem(se_cmd);
2744 complete(&se_cmd->cmd_wait_comp);
2745 return;
2746 }
2747 list_del_init(&se_cmd->se_cmd_list);
2748 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2749 }
2750
2751 target_free_cmd_mem(se_cmd);
2752 se_cmd->se_tfo->release_cmd(se_cmd);
2753}
2754
2755
2756
2757
2758
2759
2760
2761
2762int target_put_sess_cmd(struct se_cmd *se_cmd)
2763{
2764 return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
2765}
2766EXPORT_SYMBOL(target_put_sess_cmd);
2767
2768static const char *data_dir_name(enum dma_data_direction d)
2769{
2770 switch (d) {
2771 case DMA_BIDIRECTIONAL: return "BIDI";
2772 case DMA_TO_DEVICE: return "WRITE";
2773 case DMA_FROM_DEVICE: return "READ";
2774 case DMA_NONE: return "NONE";
2775 }
2776
2777 return "(?)";
2778}
2779
2780static const char *cmd_state_name(enum transport_state_table t)
2781{
2782 switch (t) {
2783 case TRANSPORT_NO_STATE: return "NO_STATE";
2784 case TRANSPORT_NEW_CMD: return "NEW_CMD";
2785 case TRANSPORT_WRITE_PENDING: return "WRITE_PENDING";
2786 case TRANSPORT_PROCESSING: return "PROCESSING";
2787 case TRANSPORT_COMPLETE: return "COMPLETE";
2788 case TRANSPORT_ISTATE_PROCESSING:
2789 return "ISTATE_PROCESSING";
2790 case TRANSPORT_COMPLETE_QF_WP: return "COMPLETE_QF_WP";
2791 case TRANSPORT_COMPLETE_QF_OK: return "COMPLETE_QF_OK";
2792 case TRANSPORT_COMPLETE_QF_ERR: return "COMPLETE_QF_ERR";
2793 }
2794
2795 return "(?)";
2796}
2797
2798static void target_append_str(char **str, const char *txt)
2799{
2800 char *prev = *str;
2801
2802 *str = *str ? kasprintf(GFP_ATOMIC, "%s,%s", *str, txt) :
2803 kstrdup(txt, GFP_ATOMIC);
2804 kfree(prev);
2805}
2806
2807
2808
2809
2810
2811static char *target_ts_to_str(u32 ts)
2812{
2813 char *str = NULL;
2814
2815 if (ts & CMD_T_ABORTED)
2816 target_append_str(&str, "aborted");
2817 if (ts & CMD_T_ACTIVE)
2818 target_append_str(&str, "active");
2819 if (ts & CMD_T_COMPLETE)
2820 target_append_str(&str, "complete");
2821 if (ts & CMD_T_SENT)
2822 target_append_str(&str, "sent");
2823 if (ts & CMD_T_STOP)
2824 target_append_str(&str, "stop");
2825 if (ts & CMD_T_FABRIC_STOP)
2826 target_append_str(&str, "fabric_stop");
2827
2828 return str;
2829}
2830
2831static const char *target_tmf_name(enum tcm_tmreq_table tmf)
2832{
2833 switch (tmf) {
2834 case TMR_ABORT_TASK: return "ABORT_TASK";
2835 case TMR_ABORT_TASK_SET: return "ABORT_TASK_SET";
2836 case TMR_CLEAR_ACA: return "CLEAR_ACA";
2837 case TMR_CLEAR_TASK_SET: return "CLEAR_TASK_SET";
2838 case TMR_LUN_RESET: return "LUN_RESET";
2839 case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET";
2840 case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET";
2841 case TMR_UNKNOWN: break;
2842 }
2843 return "(?)";
2844}
2845
2846void target_show_cmd(const char *pfx, struct se_cmd *cmd)
2847{
2848 char *ts_str = target_ts_to_str(cmd->transport_state);
2849 const u8 *cdb = cmd->t_task_cdb;
2850 struct se_tmr_req *tmf = cmd->se_tmr_req;
2851
2852 if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
2853 pr_debug("%scmd %#02x:%#02x with tag %#llx dir %s i_state %d t_state %s len %d refcnt %d transport_state %s\n",
2854 pfx, cdb[0], cdb[1], cmd->tag,
2855 data_dir_name(cmd->data_direction),
2856 cmd->se_tfo->get_cmd_state(cmd),
2857 cmd_state_name(cmd->t_state), cmd->data_length,
2858 kref_read(&cmd->cmd_kref), ts_str);
2859 } else {
2860 pr_debug("%stmf %s with tag %#llx ref_task_tag %#llx i_state %d t_state %s refcnt %d transport_state %s\n",
2861 pfx, target_tmf_name(tmf->function), cmd->tag,
2862 tmf->ref_task_tag, cmd->se_tfo->get_cmd_state(cmd),
2863 cmd_state_name(cmd->t_state),
2864 kref_read(&cmd->cmd_kref), ts_str);
2865 }
2866 kfree(ts_str);
2867}
2868EXPORT_SYMBOL(target_show_cmd);
2869
2870
2871
2872
2873
2874
2875void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
2876{
2877 struct se_cmd *se_cmd, *tmp_cmd;
2878 unsigned long flags;
2879 int rc;
2880
2881 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2882 if (se_sess->sess_tearing_down) {
2883 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2884 return;
2885 }
2886 se_sess->sess_tearing_down = 1;
2887 list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list);
2888
2889 list_for_each_entry_safe(se_cmd, tmp_cmd,
2890 &se_sess->sess_wait_list, se_cmd_list) {
2891 rc = kref_get_unless_zero(&se_cmd->cmd_kref);
2892 if (rc) {
2893 se_cmd->cmd_wait_set = 1;
2894 spin_lock(&se_cmd->t_state_lock);
2895 se_cmd->transport_state |= CMD_T_FABRIC_STOP;
2896 spin_unlock(&se_cmd->t_state_lock);
2897 } else
2898 list_del_init(&se_cmd->se_cmd_list);
2899 }
2900
2901 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2902}
2903EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
2904
2905
2906
2907
2908void target_wait_for_sess_cmds(struct se_session *se_sess)
2909{
2910 struct se_cmd *se_cmd, *tmp_cmd;
2911 unsigned long flags;
2912 bool tas;
2913
2914 list_for_each_entry_safe(se_cmd, tmp_cmd,
2915 &se_sess->sess_wait_list, se_cmd_list) {
2916 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:"
2917 " %d\n", se_cmd, se_cmd->t_state,
2918 se_cmd->se_tfo->get_cmd_state(se_cmd));
2919
2920 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
2921 tas = (se_cmd->transport_state & CMD_T_TAS);
2922 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
2923
2924 if (!target_put_sess_cmd(se_cmd)) {
2925 if (tas)
2926 target_put_sess_cmd(se_cmd);
2927 }
2928
2929 wait_for_completion(&se_cmd->cmd_wait_comp);
2930 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d"
2931 " fabric state: %d\n", se_cmd, se_cmd->t_state,
2932 se_cmd->se_tfo->get_cmd_state(se_cmd));
2933
2934 se_cmd->se_tfo->release_cmd(se_cmd);
2935 }
2936
2937 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2938 WARN_ON(!list_empty(&se_sess->sess_cmd_list));
2939 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2940
2941}
2942EXPORT_SYMBOL(target_wait_for_sess_cmds);
2943
2944static void target_lun_confirm(struct percpu_ref *ref)
2945{
2946 struct se_lun *lun = container_of(ref, struct se_lun, lun_ref);
2947
2948 complete(&lun->lun_ref_comp);
2949}
2950
2951void transport_clear_lun_ref(struct se_lun *lun)
2952{
2953
2954
2955
2956
2957
2958 percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm);
2959
2960
2961
2962
2963
2964
2965
2966 wait_for_completion(&lun->lun_ref_comp);
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976 wait_for_completion(&lun->lun_shutdown_comp);
2977}
2978
2979static bool
2980__transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop,
2981 bool *aborted, bool *tas, unsigned long *flags)
2982 __releases(&cmd->t_state_lock)
2983 __acquires(&cmd->t_state_lock)
2984{
2985
2986 assert_spin_locked(&cmd->t_state_lock);
2987 WARN_ON_ONCE(!irqs_disabled());
2988
2989 if (fabric_stop)
2990 cmd->transport_state |= CMD_T_FABRIC_STOP;
2991
2992 if (cmd->transport_state & CMD_T_ABORTED)
2993 *aborted = true;
2994
2995 if (cmd->transport_state & CMD_T_TAS)
2996 *tas = true;
2997
2998 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
2999 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
3000 return false;
3001
3002 if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
3003 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
3004 return false;
3005
3006 if (!(cmd->transport_state & CMD_T_ACTIVE))
3007 return false;
3008
3009 if (fabric_stop && *aborted)
3010 return false;
3011
3012 cmd->transport_state |= CMD_T_STOP;
3013
3014 target_show_cmd("wait_for_tasks: Stopping ", cmd);
3015
3016 spin_unlock_irqrestore(&cmd->t_state_lock, *flags);
3017
3018 while (!wait_for_completion_timeout(&cmd->t_transport_stop_comp,
3019 180 * HZ))
3020 target_show_cmd("wait for tasks: ", cmd);
3021
3022 spin_lock_irqsave(&cmd->t_state_lock, *flags);
3023 cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
3024
3025 pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->"
3026 "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag);
3027
3028 return true;
3029}
3030
3031
3032
3033
3034
3035bool transport_wait_for_tasks(struct se_cmd *cmd)
3036{
3037 unsigned long flags;
3038 bool ret, aborted = false, tas = false;
3039
3040 spin_lock_irqsave(&cmd->t_state_lock, flags);
3041 ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags);
3042 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3043
3044 return ret;
3045}
3046EXPORT_SYMBOL(transport_wait_for_tasks);
3047
3048static
3049void transport_err_sector_info(unsigned char *buffer, sector_t bad_sector)
3050{
3051
3052 buffer[SPC_ADD_SENSE_LEN_OFFSET] = 0xc;
3053 buffer[SPC_DESC_TYPE_OFFSET] = 0;
3054 buffer[SPC_ADDITIONAL_DESC_LEN_OFFSET] = 0xa;
3055 buffer[SPC_VALIDITY_OFFSET] = 0x80;
3056
3057
3058 put_unaligned_be64(bad_sector, &buffer[12]);
3059}
3060
3061struct sense_info {
3062 u8 key;
3063 u8 asc;
3064 u8 ascq;
3065 bool add_sector_info;
3066};
3067
3068static const struct sense_info sense_info_table[] = {
3069 [TCM_NO_SENSE] = {
3070 .key = NOT_READY
3071 },
3072 [TCM_NON_EXISTENT_LUN] = {
3073 .key = ILLEGAL_REQUEST,
3074 .asc = 0x25
3075 },
3076 [TCM_UNSUPPORTED_SCSI_OPCODE] = {
3077 .key = ILLEGAL_REQUEST,
3078 .asc = 0x20,
3079 },
3080 [TCM_SECTOR_COUNT_TOO_MANY] = {
3081 .key = ILLEGAL_REQUEST,
3082 .asc = 0x20,
3083 },
3084 [TCM_UNKNOWN_MODE_PAGE] = {
3085 .key = ILLEGAL_REQUEST,
3086 .asc = 0x24,
3087 },
3088 [TCM_CHECK_CONDITION_ABORT_CMD] = {
3089 .key = ABORTED_COMMAND,
3090 .asc = 0x29,
3091 .ascq = 0x03,
3092 },
3093 [TCM_INCORRECT_AMOUNT_OF_DATA] = {
3094 .key = ABORTED_COMMAND,
3095 .asc = 0x0c,
3096 .ascq = 0x0d,
3097 },
3098 [TCM_INVALID_CDB_FIELD] = {
3099 .key = ILLEGAL_REQUEST,
3100 .asc = 0x24,
3101 },
3102 [TCM_INVALID_PARAMETER_LIST] = {
3103 .key = ILLEGAL_REQUEST,
3104 .asc = 0x26,
3105 },
3106 [TCM_TOO_MANY_TARGET_DESCS] = {
3107 .key = ILLEGAL_REQUEST,
3108 .asc = 0x26,
3109 .ascq = 0x06,
3110 },
3111 [TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = {
3112 .key = ILLEGAL_REQUEST,
3113 .asc = 0x26,
3114 .ascq = 0x07,
3115 },
3116 [TCM_TOO_MANY_SEGMENT_DESCS] = {
3117 .key = ILLEGAL_REQUEST,
3118 .asc = 0x26,
3119 .ascq = 0x08,
3120 },
3121 [TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = {
3122 .key = ILLEGAL_REQUEST,
3123 .asc = 0x26,
3124 .ascq = 0x09,
3125 },
3126 [TCM_PARAMETER_LIST_LENGTH_ERROR] = {
3127 .key = ILLEGAL_REQUEST,
3128 .asc = 0x1a,
3129 },
3130 [TCM_UNEXPECTED_UNSOLICITED_DATA] = {
3131 .key = ILLEGAL_REQUEST,
3132 .asc = 0x0c,
3133 .ascq = 0x0c,
3134 },
3135 [TCM_SERVICE_CRC_ERROR] = {
3136 .key = ABORTED_COMMAND,
3137 .asc = 0x47,
3138 .ascq = 0x05,
3139 },
3140 [TCM_SNACK_REJECTED] = {
3141 .key = ABORTED_COMMAND,
3142 .asc = 0x11,
3143 .ascq = 0x13,
3144 },
3145 [TCM_WRITE_PROTECTED] = {
3146 .key = DATA_PROTECT,
3147 .asc = 0x27,
3148 },
3149 [TCM_ADDRESS_OUT_OF_RANGE] = {
3150 .key = ILLEGAL_REQUEST,
3151 .asc = 0x21,
3152 },
3153 [TCM_CHECK_CONDITION_UNIT_ATTENTION] = {
3154 .key = UNIT_ATTENTION,
3155 },
3156 [TCM_CHECK_CONDITION_NOT_READY] = {
3157 .key = NOT_READY,
3158 },
3159 [TCM_MISCOMPARE_VERIFY] = {
3160 .key = MISCOMPARE,
3161 .asc = 0x1d,
3162 .ascq = 0x00,
3163 },
3164 [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = {
3165 .key = ABORTED_COMMAND,
3166 .asc = 0x10,
3167 .ascq = 0x01,
3168 .add_sector_info = true,
3169 },
3170 [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = {
3171 .key = ABORTED_COMMAND,
3172 .asc = 0x10,
3173 .ascq = 0x02,
3174 .add_sector_info = true,
3175 },
3176 [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = {
3177 .key = ABORTED_COMMAND,
3178 .asc = 0x10,
3179 .ascq = 0x03,
3180 .add_sector_info = true,
3181 },
3182 [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = {
3183 .key = COPY_ABORTED,
3184 .asc = 0x0d,
3185 .ascq = 0x02,
3186
3187 },
3188 [TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = {
3189
3190
3191
3192
3193
3194
3195 .key = NOT_READY,
3196 .asc = 0x08,
3197 },
3198 [TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = {
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209 .key = ILLEGAL_REQUEST,
3210 .asc = 0x55,
3211 .ascq = 0x04,
3212 },
3213};
3214
3215static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason)
3216{
3217 const struct sense_info *si;
3218 u8 *buffer = cmd->sense_buffer;
3219 int r = (__force int)reason;
3220 u8 asc, ascq;
3221 bool desc_format = target_sense_desc_format(cmd->se_dev);
3222
3223 if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key)
3224 si = &sense_info_table[r];
3225 else
3226 si = &sense_info_table[(__force int)
3227 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE];
3228
3229 if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) {
3230 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
3231 WARN_ON_ONCE(asc == 0);
3232 } else if (si->asc == 0) {
3233 WARN_ON_ONCE(cmd->scsi_asc == 0);
3234 asc = cmd->scsi_asc;
3235 ascq = cmd->scsi_ascq;
3236 } else {
3237 asc = si->asc;
3238 ascq = si->ascq;
3239 }
3240
3241 scsi_build_sense_buffer(desc_format, buffer, si->key, asc, ascq);
3242 if (si->add_sector_info)
3243 transport_err_sector_info(cmd->sense_buffer, cmd->bad_sector);
3244}
3245
3246int
3247transport_send_check_condition_and_sense(struct se_cmd *cmd,
3248 sense_reason_t reason, int from_transport)
3249{
3250 unsigned long flags;
3251
3252 spin_lock_irqsave(&cmd->t_state_lock, flags);
3253 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
3254 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3255 return 0;
3256 }
3257 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
3258 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3259
3260 if (!from_transport) {
3261 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
3262 translate_sense_reason(cmd, reason);
3263 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
3264 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
3265 }
3266
3267 trace_target_cmd_complete(cmd);
3268 return cmd->se_tfo->queue_status(cmd);
3269}
3270EXPORT_SYMBOL(transport_send_check_condition_and_sense);
3271
3272static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status)
3273 __releases(&cmd->t_state_lock)
3274 __acquires(&cmd->t_state_lock)
3275{
3276 assert_spin_locked(&cmd->t_state_lock);
3277 WARN_ON_ONCE(!irqs_disabled());
3278
3279 if (!(cmd->transport_state & CMD_T_ABORTED))
3280 return 0;
3281
3282
3283
3284
3285 if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) {
3286 if (send_status)
3287 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS;
3288 return 1;
3289 }
3290
3291 pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:"
3292 " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag);
3293
3294 cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS;
3295 cmd->scsi_status = SAM_STAT_TASK_ABORTED;
3296 trace_target_cmd_complete(cmd);
3297
3298 spin_unlock_irq(&cmd->t_state_lock);
3299 cmd->se_tfo->queue_status(cmd);
3300 spin_lock_irq(&cmd->t_state_lock);
3301
3302 return 1;
3303}
3304
3305int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
3306{
3307 int ret;
3308
3309 spin_lock_irq(&cmd->t_state_lock);
3310 ret = __transport_check_aborted_status(cmd, send_status);
3311 spin_unlock_irq(&cmd->t_state_lock);
3312
3313 return ret;
3314}
3315EXPORT_SYMBOL(transport_check_aborted_status);
3316
3317void transport_send_task_abort(struct se_cmd *cmd)
3318{
3319 unsigned long flags;
3320 int ret;
3321
3322 spin_lock_irqsave(&cmd->t_state_lock, flags);
3323 if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) {
3324 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3325 return;
3326 }
3327 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3328
3329
3330
3331
3332
3333
3334
3335 if (cmd->data_direction == DMA_TO_DEVICE) {
3336 if (cmd->se_tfo->write_pending_status(cmd) != 0) {
3337 spin_lock_irqsave(&cmd->t_state_lock, flags);
3338 if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) {
3339 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3340 goto send_abort;
3341 }
3342 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS;
3343 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3344 return;
3345 }
3346 }
3347send_abort:
3348 cmd->scsi_status = SAM_STAT_TASK_ABORTED;
3349
3350 transport_lun_remove_cmd(cmd);
3351
3352 pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n",
3353 cmd->t_task_cdb[0], cmd->tag);
3354
3355 trace_target_cmd_complete(cmd);
3356 ret = cmd->se_tfo->queue_status(cmd);
3357 if (ret)
3358 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
3359}
3360
3361static void target_tmr_work(struct work_struct *work)
3362{
3363 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
3364 struct se_device *dev = cmd->se_dev;
3365 struct se_tmr_req *tmr = cmd->se_tmr_req;
3366 unsigned long flags;
3367 int ret;
3368
3369 spin_lock_irqsave(&cmd->t_state_lock, flags);
3370 if (cmd->transport_state & CMD_T_ABORTED) {
3371 tmr->response = TMR_FUNCTION_REJECTED;
3372 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3373 goto check_stop;
3374 }
3375 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3376
3377 switch (tmr->function) {
3378 case TMR_ABORT_TASK:
3379 core_tmr_abort_task(dev, tmr, cmd->se_sess);
3380 break;
3381 case TMR_ABORT_TASK_SET:
3382 case TMR_CLEAR_ACA:
3383 case TMR_CLEAR_TASK_SET:
3384 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
3385 break;
3386 case TMR_LUN_RESET:
3387 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
3388 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
3389 TMR_FUNCTION_REJECTED;
3390 if (tmr->response == TMR_FUNCTION_COMPLETE) {
3391 target_ua_allocate_lun(cmd->se_sess->se_node_acl,
3392 cmd->orig_fe_lun, 0x29,
3393 ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED);
3394 }
3395 break;
3396 case TMR_TARGET_WARM_RESET:
3397 tmr->response = TMR_FUNCTION_REJECTED;
3398 break;
3399 case TMR_TARGET_COLD_RESET:
3400 tmr->response = TMR_FUNCTION_REJECTED;
3401 break;
3402 default:
3403 pr_err("Uknown TMR function: 0x%02x.\n",
3404 tmr->function);
3405 tmr->response = TMR_FUNCTION_REJECTED;
3406 break;
3407 }
3408
3409 spin_lock_irqsave(&cmd->t_state_lock, flags);
3410 if (cmd->transport_state & CMD_T_ABORTED) {
3411 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3412 goto check_stop;
3413 }
3414 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3415
3416 cmd->se_tfo->queue_tm_rsp(cmd);
3417
3418check_stop:
3419 transport_lun_remove_cmd(cmd);
3420 transport_cmd_check_stop_to_fabric(cmd);
3421}
3422
3423int transport_generic_handle_tmr(
3424 struct se_cmd *cmd)
3425{
3426 unsigned long flags;
3427 bool aborted = false;
3428
3429 spin_lock_irqsave(&cmd->t_state_lock, flags);
3430 if (cmd->transport_state & CMD_T_ABORTED) {
3431 aborted = true;
3432 } else {
3433 cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
3434 cmd->transport_state |= CMD_T_ACTIVE;
3435 }
3436 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3437
3438 if (aborted) {
3439 pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d"
3440 "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function,
3441 cmd->se_tmr_req->ref_task_tag, cmd->tag);
3442 transport_lun_remove_cmd(cmd);
3443 transport_cmd_check_stop_to_fabric(cmd);
3444 return 0;
3445 }
3446
3447 INIT_WORK(&cmd->work, target_tmr_work);
3448 queue_work(cmd->se_dev->tmr_wq, &cmd->work);
3449 return 0;
3450}
3451EXPORT_SYMBOL(transport_generic_handle_tmr);
3452
3453bool
3454target_check_wce(struct se_device *dev)
3455{
3456 bool wce = false;
3457
3458 if (dev->transport->get_write_cache)
3459 wce = dev->transport->get_write_cache(dev);
3460 else if (dev->dev_attrib.emulate_write_cache > 0)
3461 wce = true;
3462
3463 return wce;
3464}
3465
3466bool
3467target_check_fua(struct se_device *dev)
3468{
3469 return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0;
3470}
3471