1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/net.h>
27#include <linux/delay.h>
28#include <linux/string.h>
29#include <linux/timer.h>
30#include <linux/slab.h>
31#include <linux/spinlock.h>
32#include <linux/kthread.h>
33#include <linux/in.h>
34#include <linux/cdrom.h>
35#include <linux/module.h>
36#include <linux/ratelimit.h>
37#include <linux/vmalloc.h>
38#include <asm/unaligned.h>
39#include <net/sock.h>
40#include <net/tcp.h>
41#include <scsi/scsi.h>
42#include <scsi/scsi_eh.h>
43#include <scsi/scsi_cmnd.h>
44#include <scsi/scsi_tcq.h>
45
46#include <target/target_core_base.h>
47#include <target/target_core_backend.h>
48#include <target/target_core_fabric.h>
49
50#include "target_core_internal.h"
51#include "target_core_alua.h"
52#include "target_core_pr.h"
53#include "target_core_ua.h"
54
55#define CREATE_TRACE_POINTS
56#include <trace/events/target.h>
57
58static struct workqueue_struct *target_completion_wq;
59static struct kmem_cache *se_sess_cache;
60struct kmem_cache *se_ua_cache;
61struct kmem_cache *t10_pr_reg_cache;
62struct kmem_cache *t10_alua_lu_gp_cache;
63struct kmem_cache *t10_alua_lu_gp_mem_cache;
64struct kmem_cache *t10_alua_tg_pt_gp_cache;
65struct kmem_cache *t10_alua_lba_map_cache;
66struct kmem_cache *t10_alua_lba_map_mem_cache;
67
68static void transport_complete_task_attr(struct se_cmd *cmd);
69static void transport_handle_queue_full(struct se_cmd *cmd,
70 struct se_device *dev, int err, bool write_pending);
71static void target_complete_ok_work(struct work_struct *work);
72
73int init_se_kmem_caches(void)
74{
75 se_sess_cache = kmem_cache_create("se_sess_cache",
76 sizeof(struct se_session), __alignof__(struct se_session),
77 0, NULL);
78 if (!se_sess_cache) {
79 pr_err("kmem_cache_create() for struct se_session"
80 " failed\n");
81 goto out;
82 }
83 se_ua_cache = kmem_cache_create("se_ua_cache",
84 sizeof(struct se_ua), __alignof__(struct se_ua),
85 0, NULL);
86 if (!se_ua_cache) {
87 pr_err("kmem_cache_create() for struct se_ua failed\n");
88 goto out_free_sess_cache;
89 }
90 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
91 sizeof(struct t10_pr_registration),
92 __alignof__(struct t10_pr_registration), 0, NULL);
93 if (!t10_pr_reg_cache) {
94 pr_err("kmem_cache_create() for struct t10_pr_registration"
95 " failed\n");
96 goto out_free_ua_cache;
97 }
98 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
99 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
100 0, NULL);
101 if (!t10_alua_lu_gp_cache) {
102 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
103 " failed\n");
104 goto out_free_pr_reg_cache;
105 }
106 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
107 sizeof(struct t10_alua_lu_gp_member),
108 __alignof__(struct t10_alua_lu_gp_member), 0, NULL);
109 if (!t10_alua_lu_gp_mem_cache) {
110 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
111 "cache failed\n");
112 goto out_free_lu_gp_cache;
113 }
114 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
115 sizeof(struct t10_alua_tg_pt_gp),
116 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
117 if (!t10_alua_tg_pt_gp_cache) {
118 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
119 "cache failed\n");
120 goto out_free_lu_gp_mem_cache;
121 }
122 t10_alua_lba_map_cache = kmem_cache_create(
123 "t10_alua_lba_map_cache",
124 sizeof(struct t10_alua_lba_map),
125 __alignof__(struct t10_alua_lba_map), 0, NULL);
126 if (!t10_alua_lba_map_cache) {
127 pr_err("kmem_cache_create() for t10_alua_lba_map_"
128 "cache failed\n");
129 goto out_free_tg_pt_gp_cache;
130 }
131 t10_alua_lba_map_mem_cache = kmem_cache_create(
132 "t10_alua_lba_map_mem_cache",
133 sizeof(struct t10_alua_lba_map_member),
134 __alignof__(struct t10_alua_lba_map_member), 0, NULL);
135 if (!t10_alua_lba_map_mem_cache) {
136 pr_err("kmem_cache_create() for t10_alua_lba_map_mem_"
137 "cache failed\n");
138 goto out_free_lba_map_cache;
139 }
140
141 target_completion_wq = alloc_workqueue("target_completion",
142 WQ_MEM_RECLAIM, 0);
143 if (!target_completion_wq)
144 goto out_free_lba_map_mem_cache;
145
146 return 0;
147
148out_free_lba_map_mem_cache:
149 kmem_cache_destroy(t10_alua_lba_map_mem_cache);
150out_free_lba_map_cache:
151 kmem_cache_destroy(t10_alua_lba_map_cache);
152out_free_tg_pt_gp_cache:
153 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
154out_free_lu_gp_mem_cache:
155 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
156out_free_lu_gp_cache:
157 kmem_cache_destroy(t10_alua_lu_gp_cache);
158out_free_pr_reg_cache:
159 kmem_cache_destroy(t10_pr_reg_cache);
160out_free_ua_cache:
161 kmem_cache_destroy(se_ua_cache);
162out_free_sess_cache:
163 kmem_cache_destroy(se_sess_cache);
164out:
165 return -ENOMEM;
166}
167
168void release_se_kmem_caches(void)
169{
170 destroy_workqueue(target_completion_wq);
171 kmem_cache_destroy(se_sess_cache);
172 kmem_cache_destroy(se_ua_cache);
173 kmem_cache_destroy(t10_pr_reg_cache);
174 kmem_cache_destroy(t10_alua_lu_gp_cache);
175 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
176 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
177 kmem_cache_destroy(t10_alua_lba_map_cache);
178 kmem_cache_destroy(t10_alua_lba_map_mem_cache);
179}
180
181
182static DEFINE_SPINLOCK(scsi_mib_index_lock);
183static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
184
185
186
187
188u32 scsi_get_new_index(scsi_index_t type)
189{
190 u32 new_index;
191
192 BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
193
194 spin_lock(&scsi_mib_index_lock);
195 new_index = ++scsi_mib_index[type];
196 spin_unlock(&scsi_mib_index_lock);
197
198 return new_index;
199}
200
201void transport_subsystem_check_init(void)
202{
203 int ret;
204 static int sub_api_initialized;
205
206 if (sub_api_initialized)
207 return;
208
209 ret = IS_ENABLED(CONFIG_TCM_IBLOCK) && request_module("target_core_iblock");
210 if (ret != 0)
211 pr_err("Unable to load target_core_iblock\n");
212
213 ret = IS_ENABLED(CONFIG_TCM_FILEIO) && request_module("target_core_file");
214 if (ret != 0)
215 pr_err("Unable to load target_core_file\n");
216
217 ret = IS_ENABLED(CONFIG_TCM_PSCSI) && request_module("target_core_pscsi");
218 if (ret != 0)
219 pr_err("Unable to load target_core_pscsi\n");
220
221 ret = IS_ENABLED(CONFIG_TCM_USER2) && request_module("target_core_user");
222 if (ret != 0)
223 pr_err("Unable to load target_core_user\n");
224
225 sub_api_initialized = 1;
226}
227
228struct se_session *transport_init_session(enum target_prot_op sup_prot_ops)
229{
230 struct se_session *se_sess;
231
232 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
233 if (!se_sess) {
234 pr_err("Unable to allocate struct se_session from"
235 " se_sess_cache\n");
236 return ERR_PTR(-ENOMEM);
237 }
238 INIT_LIST_HEAD(&se_sess->sess_list);
239 INIT_LIST_HEAD(&se_sess->sess_acl_list);
240 INIT_LIST_HEAD(&se_sess->sess_cmd_list);
241 INIT_LIST_HEAD(&se_sess->sess_wait_list);
242 spin_lock_init(&se_sess->sess_cmd_lock);
243 kref_init(&se_sess->sess_kref);
244 se_sess->sup_prot_ops = sup_prot_ops;
245
246 return se_sess;
247}
248EXPORT_SYMBOL(transport_init_session);
249
250int transport_alloc_session_tags(struct se_session *se_sess,
251 unsigned int tag_num, unsigned int tag_size)
252{
253 int rc;
254
255 se_sess->sess_cmd_map = kzalloc(tag_num * tag_size,
256 GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
257 if (!se_sess->sess_cmd_map) {
258 se_sess->sess_cmd_map = vzalloc(tag_num * tag_size);
259 if (!se_sess->sess_cmd_map) {
260 pr_err("Unable to allocate se_sess->sess_cmd_map\n");
261 return -ENOMEM;
262 }
263 }
264
265 rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num);
266 if (rc < 0) {
267 pr_err("Unable to init se_sess->sess_tag_pool,"
268 " tag_num: %u\n", tag_num);
269 kvfree(se_sess->sess_cmd_map);
270 se_sess->sess_cmd_map = NULL;
271 return -ENOMEM;
272 }
273
274 return 0;
275}
276EXPORT_SYMBOL(transport_alloc_session_tags);
277
278struct se_session *transport_init_session_tags(unsigned int tag_num,
279 unsigned int tag_size,
280 enum target_prot_op sup_prot_ops)
281{
282 struct se_session *se_sess;
283 int rc;
284
285 if (tag_num != 0 && !tag_size) {
286 pr_err("init_session_tags called with percpu-ida tag_num:"
287 " %u, but zero tag_size\n", tag_num);
288 return ERR_PTR(-EINVAL);
289 }
290 if (!tag_num && tag_size) {
291 pr_err("init_session_tags called with percpu-ida tag_size:"
292 " %u, but zero tag_num\n", tag_size);
293 return ERR_PTR(-EINVAL);
294 }
295
296 se_sess = transport_init_session(sup_prot_ops);
297 if (IS_ERR(se_sess))
298 return se_sess;
299
300 rc = transport_alloc_session_tags(se_sess, tag_num, tag_size);
301 if (rc < 0) {
302 transport_free_session(se_sess);
303 return ERR_PTR(-ENOMEM);
304 }
305
306 return se_sess;
307}
308EXPORT_SYMBOL(transport_init_session_tags);
309
310
311
312
313void __transport_register_session(
314 struct se_portal_group *se_tpg,
315 struct se_node_acl *se_nacl,
316 struct se_session *se_sess,
317 void *fabric_sess_ptr)
318{
319 const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo;
320 unsigned char buf[PR_REG_ISID_LEN];
321 unsigned long flags;
322
323 se_sess->se_tpg = se_tpg;
324 se_sess->fabric_sess_ptr = fabric_sess_ptr;
325
326
327
328
329
330
331 if (se_nacl) {
332
333
334
335
336
337
338
339
340
341
342 if (se_nacl->saved_prot_type)
343 se_sess->sess_prot_type = se_nacl->saved_prot_type;
344 else if (tfo->tpg_check_prot_fabric_only)
345 se_sess->sess_prot_type = se_nacl->saved_prot_type =
346 tfo->tpg_check_prot_fabric_only(se_tpg);
347
348
349
350
351 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
352 memset(&buf[0], 0, PR_REG_ISID_LEN);
353 se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
354 &buf[0], PR_REG_ISID_LEN);
355 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
356 }
357
358 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
359
360
361
362
363 se_nacl->nacl_sess = se_sess;
364
365 list_add_tail(&se_sess->sess_acl_list,
366 &se_nacl->acl_sess_list);
367 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
368 }
369 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
370
371 pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
372 se_tpg->se_tpg_tfo->fabric_name, se_sess->fabric_sess_ptr);
373}
374EXPORT_SYMBOL(__transport_register_session);
375
376void transport_register_session(
377 struct se_portal_group *se_tpg,
378 struct se_node_acl *se_nacl,
379 struct se_session *se_sess,
380 void *fabric_sess_ptr)
381{
382 unsigned long flags;
383
384 spin_lock_irqsave(&se_tpg->session_lock, flags);
385 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
386 spin_unlock_irqrestore(&se_tpg->session_lock, flags);
387}
388EXPORT_SYMBOL(transport_register_session);
389
390struct se_session *
391target_setup_session(struct se_portal_group *tpg,
392 unsigned int tag_num, unsigned int tag_size,
393 enum target_prot_op prot_op,
394 const char *initiatorname, void *private,
395 int (*callback)(struct se_portal_group *,
396 struct se_session *, void *))
397{
398 struct se_session *sess;
399
400
401
402
403
404 if (tag_num != 0)
405 sess = transport_init_session_tags(tag_num, tag_size, prot_op);
406 else
407 sess = transport_init_session(prot_op);
408
409 if (IS_ERR(sess))
410 return sess;
411
412 sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg,
413 (unsigned char *)initiatorname);
414 if (!sess->se_node_acl) {
415 transport_free_session(sess);
416 return ERR_PTR(-EACCES);
417 }
418
419
420
421
422 if (callback != NULL) {
423 int rc = callback(tpg, sess, private);
424 if (rc) {
425 transport_free_session(sess);
426 return ERR_PTR(rc);
427 }
428 }
429
430 transport_register_session(tpg, sess->se_node_acl, sess, private);
431 return sess;
432}
433EXPORT_SYMBOL(target_setup_session);
434
435static void target_release_session(struct kref *kref)
436{
437 struct se_session *se_sess = container_of(kref,
438 struct se_session, sess_kref);
439 struct se_portal_group *se_tpg = se_sess->se_tpg;
440
441 if (se_tpg->se_tpg_tfo->close_session)
442 se_tpg->se_tpg_tfo->close_session(se_sess);
443}
444
445int target_get_session(struct se_session *se_sess)
446{
447 return kref_get_unless_zero(&se_sess->sess_kref);
448}
449EXPORT_SYMBOL(target_get_session);
450
451void target_put_session(struct se_session *se_sess)
452{
453 kref_put(&se_sess->sess_kref, target_release_session);
454}
455EXPORT_SYMBOL(target_put_session);
456
457ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page)
458{
459 struct se_session *se_sess;
460 ssize_t len = 0;
461
462 spin_lock_bh(&se_tpg->session_lock);
463 list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) {
464 if (!se_sess->se_node_acl)
465 continue;
466 if (!se_sess->se_node_acl->dynamic_node_acl)
467 continue;
468 if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE)
469 break;
470
471 len += snprintf(page + len, PAGE_SIZE - len, "%s\n",
472 se_sess->se_node_acl->initiatorname);
473 len += 1;
474 }
475 spin_unlock_bh(&se_tpg->session_lock);
476
477 return len;
478}
479EXPORT_SYMBOL(target_show_dynamic_sessions);
480
481static void target_complete_nacl(struct kref *kref)
482{
483 struct se_node_acl *nacl = container_of(kref,
484 struct se_node_acl, acl_kref);
485 struct se_portal_group *se_tpg = nacl->se_tpg;
486
487 if (!nacl->dynamic_stop) {
488 complete(&nacl->acl_free_comp);
489 return;
490 }
491
492 mutex_lock(&se_tpg->acl_node_mutex);
493 list_del_init(&nacl->acl_list);
494 mutex_unlock(&se_tpg->acl_node_mutex);
495
496 core_tpg_wait_for_nacl_pr_ref(nacl);
497 core_free_device_list_for_node(nacl, se_tpg);
498 kfree(nacl);
499}
500
501void target_put_nacl(struct se_node_acl *nacl)
502{
503 kref_put(&nacl->acl_kref, target_complete_nacl);
504}
505EXPORT_SYMBOL(target_put_nacl);
506
507void transport_deregister_session_configfs(struct se_session *se_sess)
508{
509 struct se_node_acl *se_nacl;
510 unsigned long flags;
511
512
513
514 se_nacl = se_sess->se_node_acl;
515 if (se_nacl) {
516 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
517 if (!list_empty(&se_sess->sess_acl_list))
518 list_del_init(&se_sess->sess_acl_list);
519
520
521
522
523
524 if (list_empty(&se_nacl->acl_sess_list))
525 se_nacl->nacl_sess = NULL;
526 else {
527 se_nacl->nacl_sess = container_of(
528 se_nacl->acl_sess_list.prev,
529 struct se_session, sess_acl_list);
530 }
531 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
532 }
533}
534EXPORT_SYMBOL(transport_deregister_session_configfs);
535
536void transport_free_session(struct se_session *se_sess)
537{
538 struct se_node_acl *se_nacl = se_sess->se_node_acl;
539
540
541
542
543
544 if (se_nacl) {
545 struct se_portal_group *se_tpg = se_nacl->se_tpg;
546 const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo;
547 unsigned long flags;
548
549 se_sess->se_node_acl = NULL;
550
551
552
553
554
555
556 mutex_lock(&se_tpg->acl_node_mutex);
557 if (se_nacl->dynamic_node_acl &&
558 !se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
559 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
560 if (list_empty(&se_nacl->acl_sess_list))
561 se_nacl->dynamic_stop = true;
562 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
563
564 if (se_nacl->dynamic_stop)
565 list_del_init(&se_nacl->acl_list);
566 }
567 mutex_unlock(&se_tpg->acl_node_mutex);
568
569 if (se_nacl->dynamic_stop)
570 target_put_nacl(se_nacl);
571
572 target_put_nacl(se_nacl);
573 }
574 if (se_sess->sess_cmd_map) {
575 percpu_ida_destroy(&se_sess->sess_tag_pool);
576 kvfree(se_sess->sess_cmd_map);
577 }
578 kmem_cache_free(se_sess_cache, se_sess);
579}
580EXPORT_SYMBOL(transport_free_session);
581
582void transport_deregister_session(struct se_session *se_sess)
583{
584 struct se_portal_group *se_tpg = se_sess->se_tpg;
585 unsigned long flags;
586
587 if (!se_tpg) {
588 transport_free_session(se_sess);
589 return;
590 }
591
592 spin_lock_irqsave(&se_tpg->session_lock, flags);
593 list_del(&se_sess->sess_list);
594 se_sess->se_tpg = NULL;
595 se_sess->fabric_sess_ptr = NULL;
596 spin_unlock_irqrestore(&se_tpg->session_lock, flags);
597
598 pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
599 se_tpg->se_tpg_tfo->fabric_name);
600
601
602
603
604
605
606
607
608
609 transport_free_session(se_sess);
610}
611EXPORT_SYMBOL(transport_deregister_session);
612
613void target_remove_session(struct se_session *se_sess)
614{
615 transport_deregister_session_configfs(se_sess);
616 transport_deregister_session(se_sess);
617}
618EXPORT_SYMBOL(target_remove_session);
619
620static void target_remove_from_state_list(struct se_cmd *cmd)
621{
622 struct se_device *dev = cmd->se_dev;
623 unsigned long flags;
624
625 if (!dev)
626 return;
627
628 spin_lock_irqsave(&dev->execute_task_lock, flags);
629 if (cmd->state_active) {
630 list_del(&cmd->state_list);
631 cmd->state_active = false;
632 }
633 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
634}
635
636static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
637{
638 unsigned long flags;
639
640 target_remove_from_state_list(cmd);
641
642
643
644
645 cmd->se_lun = NULL;
646
647 spin_lock_irqsave(&cmd->t_state_lock, flags);
648
649
650
651
652 if (cmd->transport_state & CMD_T_STOP) {
653 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
654 __func__, __LINE__, cmd->tag);
655
656 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
657
658 complete_all(&cmd->t_transport_stop_comp);
659 return 1;
660 }
661 cmd->transport_state &= ~CMD_T_ACTIVE;
662 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
663
664
665
666
667
668
669
670
671 return cmd->se_tfo->check_stop_free(cmd);
672}
673
674static void transport_lun_remove_cmd(struct se_cmd *cmd)
675{
676 struct se_lun *lun = cmd->se_lun;
677
678 if (!lun)
679 return;
680
681 if (cmpxchg(&cmd->lun_ref_active, true, false))
682 percpu_ref_put(&lun->lun_ref);
683}
684
685int transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
686{
687 bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF);
688 int ret = 0;
689
690 if (cmd->se_cmd_flags & SCF_SE_LUN_CMD)
691 transport_lun_remove_cmd(cmd);
692
693
694
695
696 if (remove)
697 cmd->se_tfo->aborted_task(cmd);
698
699 if (transport_cmd_check_stop_to_fabric(cmd))
700 return 1;
701 if (remove && ack_kref)
702 ret = target_put_sess_cmd(cmd);
703
704 return ret;
705}
706
707static void target_complete_failure_work(struct work_struct *work)
708{
709 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
710
711 transport_generic_request_failure(cmd,
712 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
713}
714
715
716
717
718
719static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd)
720{
721 struct se_device *dev = cmd->se_dev;
722
723 WARN_ON(!cmd->se_lun);
724
725 if (!dev)
726 return NULL;
727
728 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION)
729 return NULL;
730
731 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
732
733 pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n",
734 dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status);
735 return cmd->sense_buffer;
736}
737
738void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense)
739{
740 unsigned char *cmd_sense_buf;
741 unsigned long flags;
742
743 spin_lock_irqsave(&cmd->t_state_lock, flags);
744 cmd_sense_buf = transport_get_sense_buffer(cmd);
745 if (!cmd_sense_buf) {
746 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
747 return;
748 }
749
750 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
751 memcpy(cmd_sense_buf, sense, cmd->scsi_sense_length);
752 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
753}
754EXPORT_SYMBOL(transport_copy_sense_to_cmd);
755
756void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
757{
758 struct se_device *dev = cmd->se_dev;
759 int success;
760 unsigned long flags;
761
762 cmd->scsi_status = scsi_status;
763
764 spin_lock_irqsave(&cmd->t_state_lock, flags);
765
766 if (dev && dev->transport->transport_complete) {
767 dev->transport->transport_complete(cmd,
768 cmd->t_data_sg,
769 transport_get_sense_buffer(cmd));
770 }
771 switch (cmd->scsi_status) {
772 case SAM_STAT_CHECK_CONDITION:
773 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
774 success = 1;
775 else
776 success = 0;
777 break;
778 default:
779 success = 1;
780 break;
781 }
782
783
784
785
786
787 if (cmd->transport_state & CMD_T_ABORTED ||
788 cmd->transport_state & CMD_T_STOP) {
789 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
790
791
792
793
794
795
796 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) {
797 up(&dev->caw_sem);
798 }
799 complete_all(&cmd->t_transport_stop_comp);
800 return;
801 } else if (!success) {
802 INIT_WORK(&cmd->work, target_complete_failure_work);
803 } else {
804 INIT_WORK(&cmd->work, target_complete_ok_work);
805 }
806
807 cmd->t_state = TRANSPORT_COMPLETE;
808 cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
809 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
810
811 if (cmd->se_cmd_flags & SCF_USE_CPUID)
812 queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work);
813 else
814 queue_work(target_completion_wq, &cmd->work);
815}
816EXPORT_SYMBOL(target_complete_cmd);
817
818void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length)
819{
820 if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) {
821 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
822 cmd->residual_count += cmd->data_length - length;
823 } else {
824 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
825 cmd->residual_count = cmd->data_length - length;
826 }
827
828 cmd->data_length = length;
829 }
830
831 target_complete_cmd(cmd, scsi_status);
832}
833EXPORT_SYMBOL(target_complete_cmd_with_length);
834
835static void target_add_to_state_list(struct se_cmd *cmd)
836{
837 struct se_device *dev = cmd->se_dev;
838 unsigned long flags;
839
840 spin_lock_irqsave(&dev->execute_task_lock, flags);
841 if (!cmd->state_active) {
842 list_add_tail(&cmd->state_list, &dev->state_list);
843 cmd->state_active = true;
844 }
845 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
846}
847
848
849
850
851static void transport_write_pending_qf(struct se_cmd *cmd);
852static void transport_complete_qf(struct se_cmd *cmd);
853
854void target_qf_do_work(struct work_struct *work)
855{
856 struct se_device *dev = container_of(work, struct se_device,
857 qf_work_queue);
858 LIST_HEAD(qf_cmd_list);
859 struct se_cmd *cmd, *cmd_tmp;
860
861 spin_lock_irq(&dev->qf_cmd_lock);
862 list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
863 spin_unlock_irq(&dev->qf_cmd_lock);
864
865 list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
866 list_del(&cmd->se_qf_node);
867 atomic_dec_mb(&dev->dev_qf_count);
868
869 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
870 " context: %s\n", cmd->se_tfo->fabric_name, cmd,
871 (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
872 (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
873 : "UNKNOWN");
874
875 if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP)
876 transport_write_pending_qf(cmd);
877 else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK ||
878 cmd->t_state == TRANSPORT_COMPLETE_QF_ERR)
879 transport_complete_qf(cmd);
880 }
881}
882
883unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
884{
885 switch (cmd->data_direction) {
886 case DMA_NONE:
887 return "NONE";
888 case DMA_FROM_DEVICE:
889 return "READ";
890 case DMA_TO_DEVICE:
891 return "WRITE";
892 case DMA_BIDIRECTIONAL:
893 return "BIDI";
894 default:
895 break;
896 }
897
898 return "UNKNOWN";
899}
900
901void transport_dump_dev_state(
902 struct se_device *dev,
903 char *b,
904 int *bl)
905{
906 *bl += sprintf(b + *bl, "Status: ");
907 if (dev->export_count)
908 *bl += sprintf(b + *bl, "ACTIVATED");
909 else
910 *bl += sprintf(b + *bl, "DEACTIVATED");
911
912 *bl += sprintf(b + *bl, " Max Queue Depth: %d", dev->queue_depth);
913 *bl += sprintf(b + *bl, " SectorSize: %u HwMaxSectors: %u\n",
914 dev->dev_attrib.block_size,
915 dev->dev_attrib.hw_max_sectors);
916 *bl += sprintf(b + *bl, " ");
917}
918
919void transport_dump_vpd_proto_id(
920 struct t10_vpd *vpd,
921 unsigned char *p_buf,
922 int p_buf_len)
923{
924 unsigned char buf[VPD_TMP_BUF_SIZE];
925 int len;
926
927 memset(buf, 0, VPD_TMP_BUF_SIZE);
928 len = sprintf(buf, "T10 VPD Protocol Identifier: ");
929
930 switch (vpd->protocol_identifier) {
931 case 0x00:
932 sprintf(buf+len, "Fibre Channel\n");
933 break;
934 case 0x10:
935 sprintf(buf+len, "Parallel SCSI\n");
936 break;
937 case 0x20:
938 sprintf(buf+len, "SSA\n");
939 break;
940 case 0x30:
941 sprintf(buf+len, "IEEE 1394\n");
942 break;
943 case 0x40:
944 sprintf(buf+len, "SCSI Remote Direct Memory Access"
945 " Protocol\n");
946 break;
947 case 0x50:
948 sprintf(buf+len, "Internet SCSI (iSCSI)\n");
949 break;
950 case 0x60:
951 sprintf(buf+len, "SAS Serial SCSI Protocol\n");
952 break;
953 case 0x70:
954 sprintf(buf+len, "Automation/Drive Interface Transport"
955 " Protocol\n");
956 break;
957 case 0x80:
958 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
959 break;
960 default:
961 sprintf(buf+len, "Unknown 0x%02x\n",
962 vpd->protocol_identifier);
963 break;
964 }
965
966 if (p_buf)
967 strncpy(p_buf, buf, p_buf_len);
968 else
969 pr_debug("%s", buf);
970}
971
972void
973transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
974{
975
976
977
978
979
980 if (page_83[1] & 0x80) {
981 vpd->protocol_identifier = (page_83[0] & 0xf0);
982 vpd->protocol_identifier_set = 1;
983 transport_dump_vpd_proto_id(vpd, NULL, 0);
984 }
985}
986EXPORT_SYMBOL(transport_set_vpd_proto_id);
987
988int transport_dump_vpd_assoc(
989 struct t10_vpd *vpd,
990 unsigned char *p_buf,
991 int p_buf_len)
992{
993 unsigned char buf[VPD_TMP_BUF_SIZE];
994 int ret = 0;
995 int len;
996
997 memset(buf, 0, VPD_TMP_BUF_SIZE);
998 len = sprintf(buf, "T10 VPD Identifier Association: ");
999
1000 switch (vpd->association) {
1001 case 0x00:
1002 sprintf(buf+len, "addressed logical unit\n");
1003 break;
1004 case 0x10:
1005 sprintf(buf+len, "target port\n");
1006 break;
1007 case 0x20:
1008 sprintf(buf+len, "SCSI target device\n");
1009 break;
1010 default:
1011 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
1012 ret = -EINVAL;
1013 break;
1014 }
1015
1016 if (p_buf)
1017 strncpy(p_buf, buf, p_buf_len);
1018 else
1019 pr_debug("%s", buf);
1020
1021 return ret;
1022}
1023
1024int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
1025{
1026
1027
1028
1029
1030
1031 vpd->association = (page_83[1] & 0x30);
1032 return transport_dump_vpd_assoc(vpd, NULL, 0);
1033}
1034EXPORT_SYMBOL(transport_set_vpd_assoc);
1035
1036int transport_dump_vpd_ident_type(
1037 struct t10_vpd *vpd,
1038 unsigned char *p_buf,
1039 int p_buf_len)
1040{
1041 unsigned char buf[VPD_TMP_BUF_SIZE];
1042 int ret = 0;
1043 int len;
1044
1045 memset(buf, 0, VPD_TMP_BUF_SIZE);
1046 len = sprintf(buf, "T10 VPD Identifier Type: ");
1047
1048 switch (vpd->device_identifier_type) {
1049 case 0x00:
1050 sprintf(buf+len, "Vendor specific\n");
1051 break;
1052 case 0x01:
1053 sprintf(buf+len, "T10 Vendor ID based\n");
1054 break;
1055 case 0x02:
1056 sprintf(buf+len, "EUI-64 based\n");
1057 break;
1058 case 0x03:
1059 sprintf(buf+len, "NAA\n");
1060 break;
1061 case 0x04:
1062 sprintf(buf+len, "Relative target port identifier\n");
1063 break;
1064 case 0x08:
1065 sprintf(buf+len, "SCSI name string\n");
1066 break;
1067 default:
1068 sprintf(buf+len, "Unsupported: 0x%02x\n",
1069 vpd->device_identifier_type);
1070 ret = -EINVAL;
1071 break;
1072 }
1073
1074 if (p_buf) {
1075 if (p_buf_len < strlen(buf)+1)
1076 return -EINVAL;
1077 strncpy(p_buf, buf, p_buf_len);
1078 } else {
1079 pr_debug("%s", buf);
1080 }
1081
1082 return ret;
1083}
1084
1085int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
1086{
1087
1088
1089
1090
1091
1092 vpd->device_identifier_type = (page_83[1] & 0x0f);
1093 return transport_dump_vpd_ident_type(vpd, NULL, 0);
1094}
1095EXPORT_SYMBOL(transport_set_vpd_ident_type);
1096
1097int transport_dump_vpd_ident(
1098 struct t10_vpd *vpd,
1099 unsigned char *p_buf,
1100 int p_buf_len)
1101{
1102 unsigned char buf[VPD_TMP_BUF_SIZE];
1103 int ret = 0;
1104
1105 memset(buf, 0, VPD_TMP_BUF_SIZE);
1106
1107 switch (vpd->device_identifier_code_set) {
1108 case 0x01:
1109 snprintf(buf, sizeof(buf),
1110 "T10 VPD Binary Device Identifier: %s\n",
1111 &vpd->device_identifier[0]);
1112 break;
1113 case 0x02:
1114 snprintf(buf, sizeof(buf),
1115 "T10 VPD ASCII Device Identifier: %s\n",
1116 &vpd->device_identifier[0]);
1117 break;
1118 case 0x03:
1119 snprintf(buf, sizeof(buf),
1120 "T10 VPD UTF-8 Device Identifier: %s\n",
1121 &vpd->device_identifier[0]);
1122 break;
1123 default:
1124 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
1125 " 0x%02x", vpd->device_identifier_code_set);
1126 ret = -EINVAL;
1127 break;
1128 }
1129
1130 if (p_buf)
1131 strncpy(p_buf, buf, p_buf_len);
1132 else
1133 pr_debug("%s", buf);
1134
1135 return ret;
1136}
1137
1138int
1139transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
1140{
1141 static const char hex_str[] = "0123456789abcdef";
1142 int j = 0, i = 4;
1143
1144
1145
1146
1147
1148
1149 vpd->device_identifier_code_set = (page_83[0] & 0x0f);
1150 switch (vpd->device_identifier_code_set) {
1151 case 0x01:
1152 vpd->device_identifier[j++] =
1153 hex_str[vpd->device_identifier_type];
1154 while (i < (4 + page_83[3])) {
1155 vpd->device_identifier[j++] =
1156 hex_str[(page_83[i] & 0xf0) >> 4];
1157 vpd->device_identifier[j++] =
1158 hex_str[page_83[i] & 0x0f];
1159 i++;
1160 }
1161 break;
1162 case 0x02:
1163 case 0x03:
1164 while (i < (4 + page_83[3]))
1165 vpd->device_identifier[j++] = page_83[i++];
1166 break;
1167 default:
1168 break;
1169 }
1170
1171 return transport_dump_vpd_ident(vpd, NULL, 0);
1172}
1173EXPORT_SYMBOL(transport_set_vpd_ident);
1174
1175static sense_reason_t
1176target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev,
1177 unsigned int size)
1178{
1179 u32 mtl;
1180
1181 if (!cmd->se_tfo->max_data_sg_nents)
1182 return TCM_NO_SENSE;
1183
1184
1185
1186
1187
1188
1189 mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE);
1190 if (cmd->data_length > mtl) {
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
1203 cmd->residual_count = (size - mtl);
1204 } else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
1205 u32 orig_dl = size + cmd->residual_count;
1206 cmd->residual_count = (orig_dl - mtl);
1207 } else {
1208 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
1209 cmd->residual_count = (cmd->data_length - mtl);
1210 }
1211 cmd->data_length = mtl;
1212
1213
1214
1215
1216 if (cmd->prot_length) {
1217 u32 sectors = (mtl / dev->dev_attrib.block_size);
1218 cmd->prot_length = dev->prot_length * sectors;
1219 }
1220 }
1221 return TCM_NO_SENSE;
1222}
1223
1224sense_reason_t
1225target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
1226{
1227 struct se_device *dev = cmd->se_dev;
1228
1229 if (cmd->unknown_data_length) {
1230 cmd->data_length = size;
1231 } else if (size != cmd->data_length) {
1232 pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:"
1233 " %u does not match SCSI CDB Length: %u for SAM Opcode:"
1234 " 0x%02x\n", cmd->se_tfo->fabric_name,
1235 cmd->data_length, size, cmd->t_task_cdb[0]);
1236
1237 if (cmd->data_direction == DMA_TO_DEVICE) {
1238 if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) {
1239 pr_err_ratelimited("Rejecting underflow/overflow"
1240 " for WRITE data CDB\n");
1241 return TCM_INVALID_CDB_FIELD;
1242 }
1243
1244
1245
1246
1247
1248
1249 if (size > cmd->data_length) {
1250 pr_err_ratelimited("Rejecting overflow for"
1251 " WRITE control CDB\n");
1252 return TCM_INVALID_CDB_FIELD;
1253 }
1254 }
1255
1256
1257
1258
1259 if (dev->dev_attrib.block_size != 512) {
1260 pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
1261 " CDB on non 512-byte sector setup subsystem"
1262 " plugin: %s\n", dev->transport->name);
1263
1264 return TCM_INVALID_CDB_FIELD;
1265 }
1266
1267
1268
1269
1270
1271
1272 if (size > cmd->data_length) {
1273 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
1274 cmd->residual_count = (size - cmd->data_length);
1275 } else {
1276 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
1277 cmd->residual_count = (cmd->data_length - size);
1278 cmd->data_length = size;
1279 }
1280 }
1281
1282 return target_check_max_data_sg_nents(cmd, dev, size);
1283
1284}
1285
1286
1287
1288
1289
1290
1291
1292void transport_init_se_cmd(
1293 struct se_cmd *cmd,
1294 const struct target_core_fabric_ops *tfo,
1295 struct se_session *se_sess,
1296 u32 data_length,
1297 int data_direction,
1298 int task_attr,
1299 unsigned char *sense_buffer)
1300{
1301 INIT_LIST_HEAD(&cmd->se_delayed_node);
1302 INIT_LIST_HEAD(&cmd->se_qf_node);
1303 INIT_LIST_HEAD(&cmd->se_cmd_list);
1304 INIT_LIST_HEAD(&cmd->state_list);
1305 init_completion(&cmd->t_transport_stop_comp);
1306 init_completion(&cmd->cmd_wait_comp);
1307 spin_lock_init(&cmd->t_state_lock);
1308 INIT_WORK(&cmd->work, NULL);
1309 kref_init(&cmd->cmd_kref);
1310
1311 cmd->se_tfo = tfo;
1312 cmd->se_sess = se_sess;
1313 cmd->data_length = data_length;
1314 cmd->data_direction = data_direction;
1315 cmd->sam_task_attr = task_attr;
1316 cmd->sense_buffer = sense_buffer;
1317
1318 cmd->state_active = false;
1319}
1320EXPORT_SYMBOL(transport_init_se_cmd);
1321
1322static sense_reason_t
1323transport_check_alloc_task_attr(struct se_cmd *cmd)
1324{
1325 struct se_device *dev = cmd->se_dev;
1326
1327
1328
1329
1330
1331 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1332 return 0;
1333
1334 if (cmd->sam_task_attr == TCM_ACA_TAG) {
1335 pr_debug("SAM Task Attribute ACA"
1336 " emulation is not supported\n");
1337 return TCM_INVALID_CDB_FIELD;
1338 }
1339
1340 return 0;
1341}
1342
1343sense_reason_t
1344target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
1345{
1346 struct se_device *dev = cmd->se_dev;
1347 sense_reason_t ret;
1348
1349
1350
1351
1352
1353 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1354 pr_err("Received SCSI CDB with command_size: %d that"
1355 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1356 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1357 return TCM_INVALID_CDB_FIELD;
1358 }
1359
1360
1361
1362
1363
1364 if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
1365 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
1366 GFP_KERNEL);
1367 if (!cmd->t_task_cdb) {
1368 pr_err("Unable to allocate cmd->t_task_cdb"
1369 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
1370 scsi_command_size(cdb),
1371 (unsigned long)sizeof(cmd->__t_task_cdb));
1372 return TCM_OUT_OF_RESOURCES;
1373 }
1374 } else
1375 cmd->t_task_cdb = &cmd->__t_task_cdb[0];
1376
1377
1378
1379 memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
1380
1381 trace_target_sequencer_start(cmd);
1382
1383 ret = dev->transport->parse_cdb(cmd);
1384 if (ret == TCM_UNSUPPORTED_SCSI_OPCODE)
1385 pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n",
1386 cmd->se_tfo->fabric_name,
1387 cmd->se_sess->se_node_acl->initiatorname,
1388 cmd->t_task_cdb[0]);
1389 if (ret)
1390 return ret;
1391
1392 ret = transport_check_alloc_task_attr(cmd);
1393 if (ret)
1394 return ret;
1395
1396 cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
1397 atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus);
1398 return 0;
1399}
1400EXPORT_SYMBOL(target_setup_cmd_from_cdb);
1401
1402
1403
1404
1405
1406int transport_handle_cdb_direct(
1407 struct se_cmd *cmd)
1408{
1409 sense_reason_t ret;
1410
1411 if (!cmd->se_lun) {
1412 dump_stack();
1413 pr_err("cmd->se_lun is NULL\n");
1414 return -EINVAL;
1415 }
1416 if (in_interrupt()) {
1417 dump_stack();
1418 pr_err("transport_generic_handle_cdb cannot be called"
1419 " from interrupt context\n");
1420 return -EINVAL;
1421 }
1422
1423
1424
1425
1426
1427
1428
1429
1430 cmd->t_state = TRANSPORT_NEW_CMD;
1431 cmd->transport_state |= CMD_T_ACTIVE;
1432
1433
1434
1435
1436
1437
1438 ret = transport_generic_new_cmd(cmd);
1439 if (ret)
1440 transport_generic_request_failure(cmd, ret);
1441 return 0;
1442}
1443EXPORT_SYMBOL(transport_handle_cdb_direct);
1444
1445sense_reason_t
1446transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
1447 u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count)
1448{
1449 if (!sgl || !sgl_count)
1450 return 0;
1451
1452
1453
1454
1455
1456
1457 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
1458 pr_warn("Rejecting SCSI DATA overflow for fabric using"
1459 " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
1460 return TCM_INVALID_CDB_FIELD;
1461 }
1462
1463 cmd->t_data_sg = sgl;
1464 cmd->t_data_nents = sgl_count;
1465 cmd->t_bidi_data_sg = sgl_bidi;
1466 cmd->t_bidi_data_nents = sgl_bidi_count;
1467
1468 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1469 return 0;
1470}
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess,
1502 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
1503 u32 data_length, int task_attr, int data_dir, int flags,
1504 struct scatterlist *sgl, u32 sgl_count,
1505 struct scatterlist *sgl_bidi, u32 sgl_bidi_count,
1506 struct scatterlist *sgl_prot, u32 sgl_prot_count)
1507{
1508 struct se_portal_group *se_tpg;
1509 sense_reason_t rc;
1510 int ret;
1511
1512 se_tpg = se_sess->se_tpg;
1513 BUG_ON(!se_tpg);
1514 BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
1515 BUG_ON(in_interrupt());
1516
1517
1518
1519
1520
1521 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1522 data_length, data_dir, task_attr, sense);
1523
1524 if (flags & TARGET_SCF_USE_CPUID)
1525 se_cmd->se_cmd_flags |= SCF_USE_CPUID;
1526 else
1527 se_cmd->cpuid = WORK_CPU_UNBOUND;
1528
1529 if (flags & TARGET_SCF_UNKNOWN_SIZE)
1530 se_cmd->unknown_data_length = 1;
1531
1532
1533
1534
1535
1536
1537 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
1538 if (ret)
1539 return ret;
1540
1541
1542
1543 if (flags & TARGET_SCF_BIDI_OP)
1544 se_cmd->se_cmd_flags |= SCF_BIDI;
1545
1546
1547
1548 rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun);
1549 if (rc) {
1550 transport_send_check_condition_and_sense(se_cmd, rc, 0);
1551 target_put_sess_cmd(se_cmd);
1552 return 0;
1553 }
1554
1555 rc = target_setup_cmd_from_cdb(se_cmd, cdb);
1556 if (rc != 0) {
1557 transport_generic_request_failure(se_cmd, rc);
1558 return 0;
1559 }
1560
1561
1562
1563
1564
1565 if (sgl_prot_count) {
1566 se_cmd->t_prot_sg = sgl_prot;
1567 se_cmd->t_prot_nents = sgl_prot_count;
1568 se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC;
1569 }
1570
1571
1572
1573
1574
1575
1576 if (sgl_count != 0) {
1577 BUG_ON(!sgl);
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587 if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) &&
1588 se_cmd->data_direction == DMA_FROM_DEVICE) {
1589 unsigned char *buf = NULL;
1590
1591 if (sgl)
1592 buf = kmap(sg_page(sgl)) + sgl->offset;
1593
1594 if (buf) {
1595 memset(buf, 0, sgl->length);
1596 kunmap(sg_page(sgl));
1597 }
1598 }
1599
1600 rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count,
1601 sgl_bidi, sgl_bidi_count);
1602 if (rc != 0) {
1603 transport_generic_request_failure(se_cmd, rc);
1604 return 0;
1605 }
1606 }
1607
1608
1609
1610
1611
1612 core_alua_check_nonop_delay(se_cmd);
1613
1614 transport_handle_cdb_direct(se_cmd);
1615 return 0;
1616}
1617EXPORT_SYMBOL(target_submit_cmd_map_sgls);
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
1644 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
1645 u32 data_length, int task_attr, int data_dir, int flags)
1646{
1647 return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense,
1648 unpacked_lun, data_length, task_attr, data_dir,
1649 flags, NULL, 0, NULL, 0, NULL, 0);
1650}
1651EXPORT_SYMBOL(target_submit_cmd);
1652
1653static void target_complete_tmr_failure(struct work_struct *work)
1654{
1655 struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);
1656
1657 se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
1658 se_cmd->se_tfo->queue_tm_rsp(se_cmd);
1659
1660 transport_lun_remove_cmd(se_cmd);
1661 transport_cmd_check_stop_to_fabric(se_cmd);
1662}
1663
1664static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag,
1665 u64 *unpacked_lun)
1666{
1667 struct se_cmd *se_cmd;
1668 unsigned long flags;
1669 bool ret = false;
1670
1671 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
1672 list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) {
1673 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
1674 continue;
1675
1676 if (se_cmd->tag == tag) {
1677 *unpacked_lun = se_cmd->orig_fe_lun;
1678 ret = true;
1679 break;
1680 }
1681 }
1682 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
1683
1684 return ret;
1685}
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
1705 unsigned char *sense, u64 unpacked_lun,
1706 void *fabric_tmr_ptr, unsigned char tm_type,
1707 gfp_t gfp, u64 tag, int flags)
1708{
1709 struct se_portal_group *se_tpg;
1710 int ret;
1711
1712 se_tpg = se_sess->se_tpg;
1713 BUG_ON(!se_tpg);
1714
1715 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1716 0, DMA_NONE, TCM_SIMPLE_TAG, sense);
1717
1718
1719
1720
1721 ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp);
1722 if (ret < 0)
1723 return -ENOMEM;
1724
1725 if (tm_type == TMR_ABORT_TASK)
1726 se_cmd->se_tmr_req->ref_task_tag = tag;
1727
1728
1729 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
1730 if (ret) {
1731 core_tmr_release_req(se_cmd->se_tmr_req);
1732 return ret;
1733 }
1734
1735
1736
1737
1738
1739 if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) {
1740 if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun))
1741 goto failure;
1742 }
1743
1744 ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
1745 if (ret)
1746 goto failure;
1747
1748 transport_generic_handle_tmr(se_cmd);
1749 return 0;
1750
1751
1752
1753
1754
1755failure:
1756 INIT_WORK(&se_cmd->work, target_complete_tmr_failure);
1757 schedule_work(&se_cmd->work);
1758 return 0;
1759}
1760EXPORT_SYMBOL(target_submit_tmr);
1761
1762
1763
1764
1765void transport_generic_request_failure(struct se_cmd *cmd,
1766 sense_reason_t sense_reason)
1767{
1768 int ret = 0, post_ret;
1769
1770 pr_debug("-----[ Storage Engine Exception; sense_reason %d\n",
1771 sense_reason);
1772 target_show_cmd("-----[ ", cmd);
1773
1774
1775
1776
1777 transport_complete_task_attr(cmd);
1778
1779 if (cmd->transport_complete_callback)
1780 cmd->transport_complete_callback(cmd, false, &post_ret);
1781
1782 if (transport_check_aborted_status(cmd, 1))
1783 return;
1784
1785 switch (sense_reason) {
1786 case TCM_NON_EXISTENT_LUN:
1787 case TCM_UNSUPPORTED_SCSI_OPCODE:
1788 case TCM_INVALID_CDB_FIELD:
1789 case TCM_INVALID_PARAMETER_LIST:
1790 case TCM_PARAMETER_LIST_LENGTH_ERROR:
1791 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
1792 case TCM_UNKNOWN_MODE_PAGE:
1793 case TCM_WRITE_PROTECTED:
1794 case TCM_ADDRESS_OUT_OF_RANGE:
1795 case TCM_CHECK_CONDITION_ABORT_CMD:
1796 case TCM_CHECK_CONDITION_UNIT_ATTENTION:
1797 case TCM_CHECK_CONDITION_NOT_READY:
1798 case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED:
1799 case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED:
1800 case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED:
1801 case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE:
1802 case TCM_TOO_MANY_TARGET_DESCS:
1803 case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE:
1804 case TCM_TOO_MANY_SEGMENT_DESCS:
1805 case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE:
1806 break;
1807 case TCM_OUT_OF_RESOURCES:
1808 cmd->scsi_status = SAM_STAT_TASK_SET_FULL;
1809 goto queue_status;
1810 case TCM_LUN_BUSY:
1811 cmd->scsi_status = SAM_STAT_BUSY;
1812 goto queue_status;
1813 case TCM_RESERVATION_CONFLICT:
1814
1815
1816
1817
1818
1819
1820 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1821
1822
1823
1824
1825
1826
1827
1828 if (cmd->se_sess &&
1829 cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) {
1830 target_ua_allocate_lun(cmd->se_sess->se_node_acl,
1831 cmd->orig_fe_lun, 0x2C,
1832 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
1833 }
1834
1835 goto queue_status;
1836 default:
1837 pr_err("Unknown transport error for CDB 0x%02x: %d\n",
1838 cmd->t_task_cdb[0], sense_reason);
1839 sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
1840 break;
1841 }
1842
1843 ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0);
1844 if (ret)
1845 goto queue_full;
1846
1847check_stop:
1848 transport_lun_remove_cmd(cmd);
1849 transport_cmd_check_stop_to_fabric(cmd);
1850 return;
1851
1852queue_status:
1853 trace_target_cmd_complete(cmd);
1854 ret = cmd->se_tfo->queue_status(cmd);
1855 if (!ret)
1856 goto check_stop;
1857queue_full:
1858 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
1859}
1860EXPORT_SYMBOL(transport_generic_request_failure);
1861
1862void __target_execute_cmd(struct se_cmd *cmd, bool do_checks)
1863{
1864 sense_reason_t ret;
1865
1866 if (!cmd->execute_cmd) {
1867 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1868 goto err;
1869 }
1870 if (do_checks) {
1871
1872
1873
1874
1875
1876
1877 ret = target_scsi3_ua_check(cmd);
1878 if (ret)
1879 goto err;
1880
1881 ret = target_alua_state_check(cmd);
1882 if (ret)
1883 goto err;
1884
1885 ret = target_check_reservation(cmd);
1886 if (ret) {
1887 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1888 goto err;
1889 }
1890 }
1891
1892 ret = cmd->execute_cmd(cmd);
1893 if (!ret)
1894 return;
1895err:
1896 spin_lock_irq(&cmd->t_state_lock);
1897 cmd->transport_state &= ~CMD_T_SENT;
1898 spin_unlock_irq(&cmd->t_state_lock);
1899
1900 transport_generic_request_failure(cmd, ret);
1901}
1902
1903static int target_write_prot_action(struct se_cmd *cmd)
1904{
1905 u32 sectors;
1906
1907
1908
1909
1910
1911 switch (cmd->prot_op) {
1912 case TARGET_PROT_DOUT_INSERT:
1913 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT))
1914 sbc_dif_generate(cmd);
1915 break;
1916 case TARGET_PROT_DOUT_STRIP:
1917 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP)
1918 break;
1919
1920 sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size);
1921 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
1922 sectors, 0, cmd->t_prot_sg, 0);
1923 if (unlikely(cmd->pi_err)) {
1924 spin_lock_irq(&cmd->t_state_lock);
1925 cmd->transport_state &= ~CMD_T_SENT;
1926 spin_unlock_irq(&cmd->t_state_lock);
1927 transport_generic_request_failure(cmd, cmd->pi_err);
1928 return -1;
1929 }
1930 break;
1931 default:
1932 break;
1933 }
1934
1935 return 0;
1936}
1937
1938static bool target_handle_task_attr(struct se_cmd *cmd)
1939{
1940 struct se_device *dev = cmd->se_dev;
1941
1942 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1943 return false;
1944
1945 cmd->se_cmd_flags |= SCF_TASK_ATTR_SET;
1946
1947
1948
1949
1950
1951 switch (cmd->sam_task_attr) {
1952 case TCM_HEAD_TAG:
1953 pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n",
1954 cmd->t_task_cdb[0]);
1955 return false;
1956 case TCM_ORDERED_TAG:
1957 atomic_inc_mb(&dev->dev_ordered_sync);
1958
1959 pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n",
1960 cmd->t_task_cdb[0]);
1961
1962
1963
1964
1965
1966 if (!atomic_read(&dev->simple_cmds))
1967 return false;
1968 break;
1969 default:
1970
1971
1972
1973 atomic_inc_mb(&dev->simple_cmds);
1974 break;
1975 }
1976
1977 if (atomic_read(&dev->dev_ordered_sync) == 0)
1978 return false;
1979
1980 spin_lock(&dev->delayed_cmd_lock);
1981 list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list);
1982 spin_unlock(&dev->delayed_cmd_lock);
1983
1984 pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn",
1985 cmd->t_task_cdb[0], cmd->sam_task_attr);
1986 return true;
1987}
1988
1989static int __transport_check_aborted_status(struct se_cmd *, int);
1990
1991void target_execute_cmd(struct se_cmd *cmd)
1992{
1993
1994
1995
1996
1997
1998
1999 spin_lock_irq(&cmd->t_state_lock);
2000 if (__transport_check_aborted_status(cmd, 1)) {
2001 spin_unlock_irq(&cmd->t_state_lock);
2002 return;
2003 }
2004 if (cmd->transport_state & CMD_T_STOP) {
2005 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
2006 __func__, __LINE__, cmd->tag);
2007
2008 spin_unlock_irq(&cmd->t_state_lock);
2009 complete_all(&cmd->t_transport_stop_comp);
2010 return;
2011 }
2012
2013 cmd->t_state = TRANSPORT_PROCESSING;
2014 cmd->transport_state &= ~CMD_T_PRE_EXECUTE;
2015 cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT;
2016 spin_unlock_irq(&cmd->t_state_lock);
2017
2018 if (target_write_prot_action(cmd))
2019 return;
2020
2021 if (target_handle_task_attr(cmd)) {
2022 spin_lock_irq(&cmd->t_state_lock);
2023 cmd->transport_state &= ~CMD_T_SENT;
2024 spin_unlock_irq(&cmd->t_state_lock);
2025 return;
2026 }
2027
2028 __target_execute_cmd(cmd, true);
2029}
2030EXPORT_SYMBOL(target_execute_cmd);
2031
2032
2033
2034
2035
2036static void target_restart_delayed_cmds(struct se_device *dev)
2037{
2038 for (;;) {
2039 struct se_cmd *cmd;
2040
2041 spin_lock(&dev->delayed_cmd_lock);
2042 if (list_empty(&dev->delayed_cmd_list)) {
2043 spin_unlock(&dev->delayed_cmd_lock);
2044 break;
2045 }
2046
2047 cmd = list_entry(dev->delayed_cmd_list.next,
2048 struct se_cmd, se_delayed_node);
2049 list_del(&cmd->se_delayed_node);
2050 spin_unlock(&dev->delayed_cmd_lock);
2051
2052 cmd->transport_state |= CMD_T_SENT;
2053
2054 __target_execute_cmd(cmd, true);
2055
2056 if (cmd->sam_task_attr == TCM_ORDERED_TAG)
2057 break;
2058 }
2059}
2060
2061
2062
2063
2064
2065static void transport_complete_task_attr(struct se_cmd *cmd)
2066{
2067 struct se_device *dev = cmd->se_dev;
2068
2069 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
2070 return;
2071
2072 if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET))
2073 goto restart;
2074
2075 if (cmd->sam_task_attr == TCM_SIMPLE_TAG) {
2076 atomic_dec_mb(&dev->simple_cmds);
2077 dev->dev_cur_ordered_id++;
2078 } else if (cmd->sam_task_attr == TCM_HEAD_TAG) {
2079 dev->dev_cur_ordered_id++;
2080 pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n",
2081 dev->dev_cur_ordered_id);
2082 } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) {
2083 atomic_dec_mb(&dev->dev_ordered_sync);
2084
2085 dev->dev_cur_ordered_id++;
2086 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n",
2087 dev->dev_cur_ordered_id);
2088 }
2089 cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET;
2090
2091restart:
2092 target_restart_delayed_cmds(dev);
2093}
2094
2095static void transport_complete_qf(struct se_cmd *cmd)
2096{
2097 int ret = 0;
2098
2099 transport_complete_task_attr(cmd);
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109 if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) {
2110 if (cmd->scsi_status)
2111 goto queue_status;
2112
2113 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
2114 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
2115 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
2116 goto queue_status;
2117 }
2118
2119 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
2120 goto queue_status;
2121
2122 switch (cmd->data_direction) {
2123 case DMA_FROM_DEVICE:
2124 if (cmd->scsi_status)
2125 goto queue_status;
2126
2127 trace_target_cmd_complete(cmd);
2128 ret = cmd->se_tfo->queue_data_in(cmd);
2129 break;
2130 case DMA_TO_DEVICE:
2131 if (cmd->se_cmd_flags & SCF_BIDI) {
2132 ret = cmd->se_tfo->queue_data_in(cmd);
2133 break;
2134 }
2135
2136 case DMA_NONE:
2137queue_status:
2138 trace_target_cmd_complete(cmd);
2139 ret = cmd->se_tfo->queue_status(cmd);
2140 break;
2141 default:
2142 break;
2143 }
2144
2145 if (ret < 0) {
2146 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
2147 return;
2148 }
2149 transport_lun_remove_cmd(cmd);
2150 transport_cmd_check_stop_to_fabric(cmd);
2151}
2152
2153static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev,
2154 int err, bool write_pending)
2155{
2156
2157
2158
2159
2160
2161
2162
2163
2164 if (err == -EAGAIN || err == -ENOMEM) {
2165 cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP :
2166 TRANSPORT_COMPLETE_QF_OK;
2167 } else {
2168 pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err);
2169 cmd->t_state = TRANSPORT_COMPLETE_QF_ERR;
2170 }
2171
2172 spin_lock_irq(&dev->qf_cmd_lock);
2173 list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
2174 atomic_inc_mb(&dev->dev_qf_count);
2175 spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);
2176
2177 schedule_work(&cmd->se_dev->qf_work_queue);
2178}
2179
2180static bool target_read_prot_action(struct se_cmd *cmd)
2181{
2182 switch (cmd->prot_op) {
2183 case TARGET_PROT_DIN_STRIP:
2184 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) {
2185 u32 sectors = cmd->data_length >>
2186 ilog2(cmd->se_dev->dev_attrib.block_size);
2187
2188 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
2189 sectors, 0, cmd->t_prot_sg,
2190 0);
2191 if (cmd->pi_err)
2192 return true;
2193 }
2194 break;
2195 case TARGET_PROT_DIN_INSERT:
2196 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT)
2197 break;
2198
2199 sbc_dif_generate(cmd);
2200 break;
2201 default:
2202 break;
2203 }
2204
2205 return false;
2206}
2207
2208static void target_complete_ok_work(struct work_struct *work)
2209{
2210 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
2211 int ret;
2212
2213
2214
2215
2216
2217
2218 transport_complete_task_attr(cmd);
2219
2220
2221
2222
2223
2224 if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
2225 schedule_work(&cmd->se_dev->qf_work_queue);
2226
2227
2228
2229
2230
2231 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
2232 WARN_ON(!cmd->scsi_status);
2233 ret = transport_send_check_condition_and_sense(
2234 cmd, 0, 1);
2235 if (ret)
2236 goto queue_full;
2237
2238 transport_lun_remove_cmd(cmd);
2239 transport_cmd_check_stop_to_fabric(cmd);
2240 return;
2241 }
2242
2243
2244
2245
2246 if (cmd->transport_complete_callback) {
2247 sense_reason_t rc;
2248 bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE);
2249 bool zero_dl = !(cmd->data_length);
2250 int post_ret = 0;
2251
2252 rc = cmd->transport_complete_callback(cmd, true, &post_ret);
2253 if (!rc && !post_ret) {
2254 if (caw && zero_dl)
2255 goto queue_rsp;
2256
2257 return;
2258 } else if (rc) {
2259 ret = transport_send_check_condition_and_sense(cmd,
2260 rc, 0);
2261 if (ret)
2262 goto queue_full;
2263
2264 transport_lun_remove_cmd(cmd);
2265 transport_cmd_check_stop_to_fabric(cmd);
2266 return;
2267 }
2268 }
2269
2270queue_rsp:
2271 switch (cmd->data_direction) {
2272 case DMA_FROM_DEVICE:
2273 if (cmd->scsi_status)
2274 goto queue_status;
2275
2276 atomic_long_add(cmd->data_length,
2277 &cmd->se_lun->lun_stats.tx_data_octets);
2278
2279
2280
2281
2282
2283 if (target_read_prot_action(cmd)) {
2284 ret = transport_send_check_condition_and_sense(cmd,
2285 cmd->pi_err, 0);
2286 if (ret)
2287 goto queue_full;
2288
2289 transport_lun_remove_cmd(cmd);
2290 transport_cmd_check_stop_to_fabric(cmd);
2291 return;
2292 }
2293
2294 trace_target_cmd_complete(cmd);
2295 ret = cmd->se_tfo->queue_data_in(cmd);
2296 if (ret)
2297 goto queue_full;
2298 break;
2299 case DMA_TO_DEVICE:
2300 atomic_long_add(cmd->data_length,
2301 &cmd->se_lun->lun_stats.rx_data_octets);
2302
2303
2304
2305 if (cmd->se_cmd_flags & SCF_BIDI) {
2306 atomic_long_add(cmd->data_length,
2307 &cmd->se_lun->lun_stats.tx_data_octets);
2308 ret = cmd->se_tfo->queue_data_in(cmd);
2309 if (ret)
2310 goto queue_full;
2311 break;
2312 }
2313
2314 case DMA_NONE:
2315queue_status:
2316 trace_target_cmd_complete(cmd);
2317 ret = cmd->se_tfo->queue_status(cmd);
2318 if (ret)
2319 goto queue_full;
2320 break;
2321 default:
2322 break;
2323 }
2324
2325 transport_lun_remove_cmd(cmd);
2326 transport_cmd_check_stop_to_fabric(cmd);
2327 return;
2328
2329queue_full:
2330 pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
2331 " data_direction: %d\n", cmd, cmd->data_direction);
2332
2333 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
2334}
2335
2336void target_free_sgl(struct scatterlist *sgl, int nents)
2337{
2338 struct scatterlist *sg;
2339 int count;
2340
2341 for_each_sg(sgl, sg, nents, count)
2342 __free_page(sg_page(sg));
2343
2344 kfree(sgl);
2345}
2346EXPORT_SYMBOL(target_free_sgl);
2347
2348static inline void transport_reset_sgl_orig(struct se_cmd *cmd)
2349{
2350
2351
2352
2353
2354 if (!cmd->t_data_sg_orig)
2355 return;
2356
2357 kfree(cmd->t_data_sg);
2358 cmd->t_data_sg = cmd->t_data_sg_orig;
2359 cmd->t_data_sg_orig = NULL;
2360 cmd->t_data_nents = cmd->t_data_nents_orig;
2361 cmd->t_data_nents_orig = 0;
2362}
2363
2364static inline void transport_free_pages(struct se_cmd *cmd)
2365{
2366 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
2367 target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents);
2368 cmd->t_prot_sg = NULL;
2369 cmd->t_prot_nents = 0;
2370 }
2371
2372 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
2373
2374
2375
2376
2377 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) {
2378 target_free_sgl(cmd->t_bidi_data_sg,
2379 cmd->t_bidi_data_nents);
2380 cmd->t_bidi_data_sg = NULL;
2381 cmd->t_bidi_data_nents = 0;
2382 }
2383 transport_reset_sgl_orig(cmd);
2384 return;
2385 }
2386 transport_reset_sgl_orig(cmd);
2387
2388 target_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
2389 cmd->t_data_sg = NULL;
2390 cmd->t_data_nents = 0;
2391
2392 target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
2393 cmd->t_bidi_data_sg = NULL;
2394 cmd->t_bidi_data_nents = 0;
2395}
2396
2397void *transport_kmap_data_sg(struct se_cmd *cmd)
2398{
2399 struct scatterlist *sg = cmd->t_data_sg;
2400 struct page **pages;
2401 int i;
2402
2403
2404
2405
2406
2407
2408 if (!cmd->t_data_nents)
2409 return NULL;
2410
2411 BUG_ON(!sg);
2412 if (cmd->t_data_nents == 1)
2413 return kmap(sg_page(sg)) + sg->offset;
2414
2415
2416 pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL);
2417 if (!pages)
2418 return NULL;
2419
2420
2421 for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
2422 pages[i] = sg_page(sg);
2423 }
2424
2425 cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL);
2426 kfree(pages);
2427 if (!cmd->t_data_vmap)
2428 return NULL;
2429
2430 return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
2431}
2432EXPORT_SYMBOL(transport_kmap_data_sg);
2433
2434void transport_kunmap_data_sg(struct se_cmd *cmd)
2435{
2436 if (!cmd->t_data_nents) {
2437 return;
2438 } else if (cmd->t_data_nents == 1) {
2439 kunmap(sg_page(cmd->t_data_sg));
2440 return;
2441 }
2442
2443 vunmap(cmd->t_data_vmap);
2444 cmd->t_data_vmap = NULL;
2445}
2446EXPORT_SYMBOL(transport_kunmap_data_sg);
2447
2448int
2449target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length,
2450 bool zero_page, bool chainable)
2451{
2452 struct scatterlist *sg;
2453 struct page *page;
2454 gfp_t zero_flag = (zero_page) ? __GFP_ZERO : 0;
2455 unsigned int nalloc, nent;
2456 int i = 0;
2457
2458 nalloc = nent = DIV_ROUND_UP(length, PAGE_SIZE);
2459 if (chainable)
2460 nalloc++;
2461 sg = kmalloc_array(nalloc, sizeof(struct scatterlist), GFP_KERNEL);
2462 if (!sg)
2463 return -ENOMEM;
2464
2465 sg_init_table(sg, nalloc);
2466
2467 while (length) {
2468 u32 page_len = min_t(u32, length, PAGE_SIZE);
2469 page = alloc_page(GFP_KERNEL | zero_flag);
2470 if (!page)
2471 goto out;
2472
2473 sg_set_page(&sg[i], page, page_len, 0);
2474 length -= page_len;
2475 i++;
2476 }
2477 *sgl = sg;
2478 *nents = nent;
2479 return 0;
2480
2481out:
2482 while (i > 0) {
2483 i--;
2484 __free_page(sg_page(&sg[i]));
2485 }
2486 kfree(sg);
2487 return -ENOMEM;
2488}
2489EXPORT_SYMBOL(target_alloc_sgl);
2490
2491
2492
2493
2494
2495
2496sense_reason_t
2497transport_generic_new_cmd(struct se_cmd *cmd)
2498{
2499 unsigned long flags;
2500 int ret = 0;
2501 bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB);
2502
2503 if (cmd->prot_op != TARGET_PROT_NORMAL &&
2504 !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
2505 ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents,
2506 cmd->prot_length, true, false);
2507 if (ret < 0)
2508 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2509 }
2510
2511
2512
2513
2514
2515
2516 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
2517 cmd->data_length) {
2518
2519 if ((cmd->se_cmd_flags & SCF_BIDI) ||
2520 (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) {
2521 u32 bidi_length;
2522
2523 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)
2524 bidi_length = cmd->t_task_nolb *
2525 cmd->se_dev->dev_attrib.block_size;
2526 else
2527 bidi_length = cmd->data_length;
2528
2529 ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
2530 &cmd->t_bidi_data_nents,
2531 bidi_length, zero_flag, false);
2532 if (ret < 0)
2533 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2534 }
2535
2536 ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents,
2537 cmd->data_length, zero_flag, false);
2538 if (ret < 0)
2539 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2540 } else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
2541 cmd->data_length) {
2542
2543
2544
2545
2546 u32 caw_length = cmd->t_task_nolb *
2547 cmd->se_dev->dev_attrib.block_size;
2548
2549 ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
2550 &cmd->t_bidi_data_nents,
2551 caw_length, zero_flag, false);
2552 if (ret < 0)
2553 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2554 }
2555
2556
2557
2558
2559
2560 target_add_to_state_list(cmd);
2561 if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) {
2562 target_execute_cmd(cmd);
2563 return 0;
2564 }
2565
2566 spin_lock_irqsave(&cmd->t_state_lock, flags);
2567 cmd->t_state = TRANSPORT_WRITE_PENDING;
2568
2569
2570
2571
2572 if (cmd->transport_state & CMD_T_STOP) {
2573 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
2574 __func__, __LINE__, cmd->tag);
2575
2576 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2577
2578 complete_all(&cmd->t_transport_stop_comp);
2579 return 0;
2580 }
2581 cmd->transport_state &= ~CMD_T_ACTIVE;
2582 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2583
2584 ret = cmd->se_tfo->write_pending(cmd);
2585 if (ret)
2586 goto queue_full;
2587
2588 return 0;
2589
2590queue_full:
2591 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
2592 transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
2593 return 0;
2594}
2595EXPORT_SYMBOL(transport_generic_new_cmd);
2596
2597static void transport_write_pending_qf(struct se_cmd *cmd)
2598{
2599 unsigned long flags;
2600 int ret;
2601 bool stop;
2602
2603 spin_lock_irqsave(&cmd->t_state_lock, flags);
2604 stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED));
2605 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2606
2607 if (stop) {
2608 pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n",
2609 __func__, __LINE__, cmd->tag);
2610 complete_all(&cmd->t_transport_stop_comp);
2611 return;
2612 }
2613
2614 ret = cmd->se_tfo->write_pending(cmd);
2615 if (ret) {
2616 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
2617 cmd);
2618 transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
2619 }
2620}
2621
2622static bool
2623__transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *,
2624 unsigned long *flags);
2625
2626static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas)
2627{
2628 unsigned long flags;
2629
2630 spin_lock_irqsave(&cmd->t_state_lock, flags);
2631 __transport_wait_for_tasks(cmd, true, aborted, tas, &flags);
2632 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2633}
2634
2635int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
2636{
2637 int ret = 0;
2638 bool aborted = false, tas = false;
2639
2640 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) {
2641 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2642 target_wait_free_cmd(cmd, &aborted, &tas);
2643
2644 if (!aborted || tas)
2645 ret = target_put_sess_cmd(cmd);
2646 } else {
2647 if (wait_for_tasks)
2648 target_wait_free_cmd(cmd, &aborted, &tas);
2649
2650
2651
2652
2653
2654 if (cmd->state_active)
2655 target_remove_from_state_list(cmd);
2656
2657 if (cmd->se_lun)
2658 transport_lun_remove_cmd(cmd);
2659
2660 if (!aborted || tas)
2661 ret = target_put_sess_cmd(cmd);
2662 }
2663
2664
2665
2666
2667
2668
2669 if (aborted) {
2670 pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag);
2671 wait_for_completion(&cmd->cmd_wait_comp);
2672 cmd->se_tfo->release_cmd(cmd);
2673 ret = 1;
2674 }
2675 return ret;
2676}
2677EXPORT_SYMBOL(transport_generic_free_cmd);
2678
2679
2680
2681
2682
2683int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref)
2684{
2685 struct se_session *se_sess = se_cmd->se_sess;
2686 unsigned long flags;
2687 int ret = 0;
2688
2689
2690
2691
2692
2693
2694 if (ack_kref) {
2695 if (!kref_get_unless_zero(&se_cmd->cmd_kref))
2696 return -EINVAL;
2697
2698 se_cmd->se_cmd_flags |= SCF_ACK_KREF;
2699 }
2700
2701 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2702 if (se_sess->sess_tearing_down) {
2703 ret = -ESHUTDOWN;
2704 goto out;
2705 }
2706 se_cmd->transport_state |= CMD_T_PRE_EXECUTE;
2707 list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
2708out:
2709 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2710
2711 if (ret && ack_kref)
2712 target_put_sess_cmd(se_cmd);
2713
2714 return ret;
2715}
2716EXPORT_SYMBOL(target_get_sess_cmd);
2717
2718static void target_free_cmd_mem(struct se_cmd *cmd)
2719{
2720 transport_free_pages(cmd);
2721
2722 if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
2723 core_tmr_release_req(cmd->se_tmr_req);
2724 if (cmd->t_task_cdb != cmd->__t_task_cdb)
2725 kfree(cmd->t_task_cdb);
2726}
2727
2728static void target_release_cmd_kref(struct kref *kref)
2729{
2730 struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
2731 struct se_session *se_sess = se_cmd->se_sess;
2732 unsigned long flags;
2733 bool fabric_stop;
2734
2735 if (se_sess) {
2736 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2737
2738 spin_lock(&se_cmd->t_state_lock);
2739 fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) &&
2740 (se_cmd->transport_state & CMD_T_ABORTED);
2741 spin_unlock(&se_cmd->t_state_lock);
2742
2743 if (se_cmd->cmd_wait_set || fabric_stop) {
2744 list_del_init(&se_cmd->se_cmd_list);
2745 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2746 target_free_cmd_mem(se_cmd);
2747 complete(&se_cmd->cmd_wait_comp);
2748 return;
2749 }
2750 list_del_init(&se_cmd->se_cmd_list);
2751 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2752 }
2753
2754 target_free_cmd_mem(se_cmd);
2755 se_cmd->se_tfo->release_cmd(se_cmd);
2756}
2757
2758
2759
2760
2761
2762
2763
2764
2765int target_put_sess_cmd(struct se_cmd *se_cmd)
2766{
2767 return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
2768}
2769EXPORT_SYMBOL(target_put_sess_cmd);
2770
2771static const char *data_dir_name(enum dma_data_direction d)
2772{
2773 switch (d) {
2774 case DMA_BIDIRECTIONAL: return "BIDI";
2775 case DMA_TO_DEVICE: return "WRITE";
2776 case DMA_FROM_DEVICE: return "READ";
2777 case DMA_NONE: return "NONE";
2778 }
2779
2780 return "(?)";
2781}
2782
2783static const char *cmd_state_name(enum transport_state_table t)
2784{
2785 switch (t) {
2786 case TRANSPORT_NO_STATE: return "NO_STATE";
2787 case TRANSPORT_NEW_CMD: return "NEW_CMD";
2788 case TRANSPORT_WRITE_PENDING: return "WRITE_PENDING";
2789 case TRANSPORT_PROCESSING: return "PROCESSING";
2790 case TRANSPORT_COMPLETE: return "COMPLETE";
2791 case TRANSPORT_ISTATE_PROCESSING:
2792 return "ISTATE_PROCESSING";
2793 case TRANSPORT_COMPLETE_QF_WP: return "COMPLETE_QF_WP";
2794 case TRANSPORT_COMPLETE_QF_OK: return "COMPLETE_QF_OK";
2795 case TRANSPORT_COMPLETE_QF_ERR: return "COMPLETE_QF_ERR";
2796 }
2797
2798 return "(?)";
2799}
2800
2801static void target_append_str(char **str, const char *txt)
2802{
2803 char *prev = *str;
2804
2805 *str = *str ? kasprintf(GFP_ATOMIC, "%s,%s", *str, txt) :
2806 kstrdup(txt, GFP_ATOMIC);
2807 kfree(prev);
2808}
2809
2810
2811
2812
2813
2814static char *target_ts_to_str(u32 ts)
2815{
2816 char *str = NULL;
2817
2818 if (ts & CMD_T_ABORTED)
2819 target_append_str(&str, "aborted");
2820 if (ts & CMD_T_ACTIVE)
2821 target_append_str(&str, "active");
2822 if (ts & CMD_T_COMPLETE)
2823 target_append_str(&str, "complete");
2824 if (ts & CMD_T_SENT)
2825 target_append_str(&str, "sent");
2826 if (ts & CMD_T_STOP)
2827 target_append_str(&str, "stop");
2828 if (ts & CMD_T_FABRIC_STOP)
2829 target_append_str(&str, "fabric_stop");
2830
2831 return str;
2832}
2833
2834static const char *target_tmf_name(enum tcm_tmreq_table tmf)
2835{
2836 switch (tmf) {
2837 case TMR_ABORT_TASK: return "ABORT_TASK";
2838 case TMR_ABORT_TASK_SET: return "ABORT_TASK_SET";
2839 case TMR_CLEAR_ACA: return "CLEAR_ACA";
2840 case TMR_CLEAR_TASK_SET: return "CLEAR_TASK_SET";
2841 case TMR_LUN_RESET: return "LUN_RESET";
2842 case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET";
2843 case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET";
2844 case TMR_UNKNOWN: break;
2845 }
2846 return "(?)";
2847}
2848
2849void target_show_cmd(const char *pfx, struct se_cmd *cmd)
2850{
2851 char *ts_str = target_ts_to_str(cmd->transport_state);
2852 const u8 *cdb = cmd->t_task_cdb;
2853 struct se_tmr_req *tmf = cmd->se_tmr_req;
2854
2855 if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
2856 pr_debug("%scmd %#02x:%#02x with tag %#llx dir %s i_state %d t_state %s len %d refcnt %d transport_state %s\n",
2857 pfx, cdb[0], cdb[1], cmd->tag,
2858 data_dir_name(cmd->data_direction),
2859 cmd->se_tfo->get_cmd_state(cmd),
2860 cmd_state_name(cmd->t_state), cmd->data_length,
2861 kref_read(&cmd->cmd_kref), ts_str);
2862 } else {
2863 pr_debug("%stmf %s with tag %#llx ref_task_tag %#llx i_state %d t_state %s refcnt %d transport_state %s\n",
2864 pfx, target_tmf_name(tmf->function), cmd->tag,
2865 tmf->ref_task_tag, cmd->se_tfo->get_cmd_state(cmd),
2866 cmd_state_name(cmd->t_state),
2867 kref_read(&cmd->cmd_kref), ts_str);
2868 }
2869 kfree(ts_str);
2870}
2871EXPORT_SYMBOL(target_show_cmd);
2872
2873
2874
2875
2876
2877
2878void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
2879{
2880 struct se_cmd *se_cmd, *tmp_cmd;
2881 unsigned long flags;
2882 int rc;
2883
2884 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2885 if (se_sess->sess_tearing_down) {
2886 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2887 return;
2888 }
2889 se_sess->sess_tearing_down = 1;
2890 list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list);
2891
2892 list_for_each_entry_safe(se_cmd, tmp_cmd,
2893 &se_sess->sess_wait_list, se_cmd_list) {
2894 rc = kref_get_unless_zero(&se_cmd->cmd_kref);
2895 if (rc) {
2896 se_cmd->cmd_wait_set = 1;
2897 spin_lock(&se_cmd->t_state_lock);
2898 se_cmd->transport_state |= CMD_T_FABRIC_STOP;
2899 spin_unlock(&se_cmd->t_state_lock);
2900 } else
2901 list_del_init(&se_cmd->se_cmd_list);
2902 }
2903
2904 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2905}
2906EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
2907
2908
2909
2910
2911void target_wait_for_sess_cmds(struct se_session *se_sess)
2912{
2913 struct se_cmd *se_cmd, *tmp_cmd;
2914 unsigned long flags;
2915 bool tas;
2916
2917 list_for_each_entry_safe(se_cmd, tmp_cmd,
2918 &se_sess->sess_wait_list, se_cmd_list) {
2919 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:"
2920 " %d\n", se_cmd, se_cmd->t_state,
2921 se_cmd->se_tfo->get_cmd_state(se_cmd));
2922
2923 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
2924 tas = (se_cmd->transport_state & CMD_T_TAS);
2925 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
2926
2927 if (!target_put_sess_cmd(se_cmd)) {
2928 if (tas)
2929 target_put_sess_cmd(se_cmd);
2930 }
2931
2932 wait_for_completion(&se_cmd->cmd_wait_comp);
2933 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d"
2934 " fabric state: %d\n", se_cmd, se_cmd->t_state,
2935 se_cmd->se_tfo->get_cmd_state(se_cmd));
2936
2937 se_cmd->se_tfo->release_cmd(se_cmd);
2938 }
2939
2940 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2941 WARN_ON(!list_empty(&se_sess->sess_cmd_list));
2942 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2943
2944}
2945EXPORT_SYMBOL(target_wait_for_sess_cmds);
2946
2947static void target_lun_confirm(struct percpu_ref *ref)
2948{
2949 struct se_lun *lun = container_of(ref, struct se_lun, lun_ref);
2950
2951 complete(&lun->lun_ref_comp);
2952}
2953
2954void transport_clear_lun_ref(struct se_lun *lun)
2955{
2956
2957
2958
2959
2960
2961 percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm);
2962
2963
2964
2965
2966
2967
2968
2969 wait_for_completion(&lun->lun_ref_comp);
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979 wait_for_completion(&lun->lun_shutdown_comp);
2980}
2981
2982static bool
2983__transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop,
2984 bool *aborted, bool *tas, unsigned long *flags)
2985 __releases(&cmd->t_state_lock)
2986 __acquires(&cmd->t_state_lock)
2987{
2988
2989 assert_spin_locked(&cmd->t_state_lock);
2990 WARN_ON_ONCE(!irqs_disabled());
2991
2992 if (fabric_stop)
2993 cmd->transport_state |= CMD_T_FABRIC_STOP;
2994
2995 if (cmd->transport_state & CMD_T_ABORTED)
2996 *aborted = true;
2997
2998 if (cmd->transport_state & CMD_T_TAS)
2999 *tas = true;
3000
3001 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
3002 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
3003 return false;
3004
3005 if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
3006 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
3007 return false;
3008
3009 if (!(cmd->transport_state & CMD_T_ACTIVE))
3010 return false;
3011
3012 if (fabric_stop && *aborted)
3013 return false;
3014
3015 cmd->transport_state |= CMD_T_STOP;
3016
3017 target_show_cmd("wait_for_tasks: Stopping ", cmd);
3018
3019 spin_unlock_irqrestore(&cmd->t_state_lock, *flags);
3020
3021 while (!wait_for_completion_timeout(&cmd->t_transport_stop_comp,
3022 180 * HZ))
3023 target_show_cmd("wait for tasks: ", cmd);
3024
3025 spin_lock_irqsave(&cmd->t_state_lock, *flags);
3026 cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
3027
3028 pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->"
3029 "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag);
3030
3031 return true;
3032}
3033
3034
3035
3036
3037
3038bool transport_wait_for_tasks(struct se_cmd *cmd)
3039{
3040 unsigned long flags;
3041 bool ret, aborted = false, tas = false;
3042
3043 spin_lock_irqsave(&cmd->t_state_lock, flags);
3044 ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags);
3045 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3046
3047 return ret;
3048}
3049EXPORT_SYMBOL(transport_wait_for_tasks);
3050
3051static
3052void transport_err_sector_info(unsigned char *buffer, sector_t bad_sector)
3053{
3054
3055 buffer[SPC_ADD_SENSE_LEN_OFFSET] = 0xc;
3056 buffer[SPC_DESC_TYPE_OFFSET] = 0;
3057 buffer[SPC_ADDITIONAL_DESC_LEN_OFFSET] = 0xa;
3058 buffer[SPC_VALIDITY_OFFSET] = 0x80;
3059
3060
3061 put_unaligned_be64(bad_sector, &buffer[12]);
3062}
3063
3064struct sense_info {
3065 u8 key;
3066 u8 asc;
3067 u8 ascq;
3068 bool add_sector_info;
3069};
3070
3071static const struct sense_info sense_info_table[] = {
3072 [TCM_NO_SENSE] = {
3073 .key = NOT_READY
3074 },
3075 [TCM_NON_EXISTENT_LUN] = {
3076 .key = ILLEGAL_REQUEST,
3077 .asc = 0x25
3078 },
3079 [TCM_UNSUPPORTED_SCSI_OPCODE] = {
3080 .key = ILLEGAL_REQUEST,
3081 .asc = 0x20,
3082 },
3083 [TCM_SECTOR_COUNT_TOO_MANY] = {
3084 .key = ILLEGAL_REQUEST,
3085 .asc = 0x20,
3086 },
3087 [TCM_UNKNOWN_MODE_PAGE] = {
3088 .key = ILLEGAL_REQUEST,
3089 .asc = 0x24,
3090 },
3091 [TCM_CHECK_CONDITION_ABORT_CMD] = {
3092 .key = ABORTED_COMMAND,
3093 .asc = 0x29,
3094 .ascq = 0x03,
3095 },
3096 [TCM_INCORRECT_AMOUNT_OF_DATA] = {
3097 .key = ABORTED_COMMAND,
3098 .asc = 0x0c,
3099 .ascq = 0x0d,
3100 },
3101 [TCM_INVALID_CDB_FIELD] = {
3102 .key = ILLEGAL_REQUEST,
3103 .asc = 0x24,
3104 },
3105 [TCM_INVALID_PARAMETER_LIST] = {
3106 .key = ILLEGAL_REQUEST,
3107 .asc = 0x26,
3108 },
3109 [TCM_TOO_MANY_TARGET_DESCS] = {
3110 .key = ILLEGAL_REQUEST,
3111 .asc = 0x26,
3112 .ascq = 0x06,
3113 },
3114 [TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = {
3115 .key = ILLEGAL_REQUEST,
3116 .asc = 0x26,
3117 .ascq = 0x07,
3118 },
3119 [TCM_TOO_MANY_SEGMENT_DESCS] = {
3120 .key = ILLEGAL_REQUEST,
3121 .asc = 0x26,
3122 .ascq = 0x08,
3123 },
3124 [TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = {
3125 .key = ILLEGAL_REQUEST,
3126 .asc = 0x26,
3127 .ascq = 0x09,
3128 },
3129 [TCM_PARAMETER_LIST_LENGTH_ERROR] = {
3130 .key = ILLEGAL_REQUEST,
3131 .asc = 0x1a,
3132 },
3133 [TCM_UNEXPECTED_UNSOLICITED_DATA] = {
3134 .key = ILLEGAL_REQUEST,
3135 .asc = 0x0c,
3136 .ascq = 0x0c,
3137 },
3138 [TCM_SERVICE_CRC_ERROR] = {
3139 .key = ABORTED_COMMAND,
3140 .asc = 0x47,
3141 .ascq = 0x05,
3142 },
3143 [TCM_SNACK_REJECTED] = {
3144 .key = ABORTED_COMMAND,
3145 .asc = 0x11,
3146 .ascq = 0x13,
3147 },
3148 [TCM_WRITE_PROTECTED] = {
3149 .key = DATA_PROTECT,
3150 .asc = 0x27,
3151 },
3152 [TCM_ADDRESS_OUT_OF_RANGE] = {
3153 .key = ILLEGAL_REQUEST,
3154 .asc = 0x21,
3155 },
3156 [TCM_CHECK_CONDITION_UNIT_ATTENTION] = {
3157 .key = UNIT_ATTENTION,
3158 },
3159 [TCM_CHECK_CONDITION_NOT_READY] = {
3160 .key = NOT_READY,
3161 },
3162 [TCM_MISCOMPARE_VERIFY] = {
3163 .key = MISCOMPARE,
3164 .asc = 0x1d,
3165 .ascq = 0x00,
3166 },
3167 [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = {
3168 .key = ABORTED_COMMAND,
3169 .asc = 0x10,
3170 .ascq = 0x01,
3171 .add_sector_info = true,
3172 },
3173 [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = {
3174 .key = ABORTED_COMMAND,
3175 .asc = 0x10,
3176 .ascq = 0x02,
3177 .add_sector_info = true,
3178 },
3179 [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = {
3180 .key = ABORTED_COMMAND,
3181 .asc = 0x10,
3182 .ascq = 0x03,
3183 .add_sector_info = true,
3184 },
3185 [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = {
3186 .key = COPY_ABORTED,
3187 .asc = 0x0d,
3188 .ascq = 0x02,
3189
3190 },
3191 [TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = {
3192
3193
3194
3195
3196
3197
3198 .key = NOT_READY,
3199 .asc = 0x08,
3200 },
3201 [TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = {
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212 .key = ILLEGAL_REQUEST,
3213 .asc = 0x55,
3214 .ascq = 0x04,
3215 },
3216};
3217
3218static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason)
3219{
3220 const struct sense_info *si;
3221 u8 *buffer = cmd->sense_buffer;
3222 int r = (__force int)reason;
3223 u8 asc, ascq;
3224 bool desc_format = target_sense_desc_format(cmd->se_dev);
3225
3226 if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key)
3227 si = &sense_info_table[r];
3228 else
3229 si = &sense_info_table[(__force int)
3230 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE];
3231
3232 if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) {
3233 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
3234 WARN_ON_ONCE(asc == 0);
3235 } else if (si->asc == 0) {
3236 WARN_ON_ONCE(cmd->scsi_asc == 0);
3237 asc = cmd->scsi_asc;
3238 ascq = cmd->scsi_ascq;
3239 } else {
3240 asc = si->asc;
3241 ascq = si->ascq;
3242 }
3243
3244 scsi_build_sense_buffer(desc_format, buffer, si->key, asc, ascq);
3245 if (si->add_sector_info)
3246 transport_err_sector_info(cmd->sense_buffer, cmd->bad_sector);
3247}
3248
3249int
3250transport_send_check_condition_and_sense(struct se_cmd *cmd,
3251 sense_reason_t reason, int from_transport)
3252{
3253 unsigned long flags;
3254
3255 spin_lock_irqsave(&cmd->t_state_lock, flags);
3256 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
3257 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3258 return 0;
3259 }
3260 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
3261 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3262
3263 if (!from_transport) {
3264 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
3265 translate_sense_reason(cmd, reason);
3266 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
3267 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
3268 }
3269
3270 trace_target_cmd_complete(cmd);
3271 return cmd->se_tfo->queue_status(cmd);
3272}
3273EXPORT_SYMBOL(transport_send_check_condition_and_sense);
3274
3275static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status)
3276 __releases(&cmd->t_state_lock)
3277 __acquires(&cmd->t_state_lock)
3278{
3279 assert_spin_locked(&cmd->t_state_lock);
3280 WARN_ON_ONCE(!irqs_disabled());
3281
3282 if (!(cmd->transport_state & CMD_T_ABORTED))
3283 return 0;
3284
3285
3286
3287
3288 if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) {
3289 if (send_status)
3290 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS;
3291 return 1;
3292 }
3293
3294 pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:"
3295 " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag);
3296
3297 cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS;
3298 cmd->scsi_status = SAM_STAT_TASK_ABORTED;
3299 trace_target_cmd_complete(cmd);
3300
3301 spin_unlock_irq(&cmd->t_state_lock);
3302 cmd->se_tfo->queue_status(cmd);
3303 spin_lock_irq(&cmd->t_state_lock);
3304
3305 return 1;
3306}
3307
3308int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
3309{
3310 int ret;
3311
3312 spin_lock_irq(&cmd->t_state_lock);
3313 ret = __transport_check_aborted_status(cmd, send_status);
3314 spin_unlock_irq(&cmd->t_state_lock);
3315
3316 return ret;
3317}
3318EXPORT_SYMBOL(transport_check_aborted_status);
3319
3320void transport_send_task_abort(struct se_cmd *cmd)
3321{
3322 unsigned long flags;
3323 int ret;
3324
3325 spin_lock_irqsave(&cmd->t_state_lock, flags);
3326 if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) {
3327 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3328 return;
3329 }
3330 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3331
3332
3333
3334
3335
3336
3337
3338 if (cmd->data_direction == DMA_TO_DEVICE) {
3339 if (cmd->se_tfo->write_pending_status(cmd) != 0) {
3340 spin_lock_irqsave(&cmd->t_state_lock, flags);
3341 if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) {
3342 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3343 goto send_abort;
3344 }
3345 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS;
3346 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3347 return;
3348 }
3349 }
3350send_abort:
3351 cmd->scsi_status = SAM_STAT_TASK_ABORTED;
3352
3353 transport_lun_remove_cmd(cmd);
3354
3355 pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n",
3356 cmd->t_task_cdb[0], cmd->tag);
3357
3358 trace_target_cmd_complete(cmd);
3359 ret = cmd->se_tfo->queue_status(cmd);
3360 if (ret)
3361 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
3362}
3363
3364static void target_tmr_work(struct work_struct *work)
3365{
3366 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
3367 struct se_device *dev = cmd->se_dev;
3368 struct se_tmr_req *tmr = cmd->se_tmr_req;
3369 unsigned long flags;
3370 int ret;
3371
3372 spin_lock_irqsave(&cmd->t_state_lock, flags);
3373 if (cmd->transport_state & CMD_T_ABORTED) {
3374 tmr->response = TMR_FUNCTION_REJECTED;
3375 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3376 goto check_stop;
3377 }
3378 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3379
3380 switch (tmr->function) {
3381 case TMR_ABORT_TASK:
3382 core_tmr_abort_task(dev, tmr, cmd->se_sess);
3383 break;
3384 case TMR_ABORT_TASK_SET:
3385 case TMR_CLEAR_ACA:
3386 case TMR_CLEAR_TASK_SET:
3387 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
3388 break;
3389 case TMR_LUN_RESET:
3390 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
3391 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
3392 TMR_FUNCTION_REJECTED;
3393 if (tmr->response == TMR_FUNCTION_COMPLETE) {
3394 target_ua_allocate_lun(cmd->se_sess->se_node_acl,
3395 cmd->orig_fe_lun, 0x29,
3396 ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED);
3397 }
3398 break;
3399 case TMR_TARGET_WARM_RESET:
3400 tmr->response = TMR_FUNCTION_REJECTED;
3401 break;
3402 case TMR_TARGET_COLD_RESET:
3403 tmr->response = TMR_FUNCTION_REJECTED;
3404 break;
3405 default:
3406 pr_err("Uknown TMR function: 0x%02x.\n",
3407 tmr->function);
3408 tmr->response = TMR_FUNCTION_REJECTED;
3409 break;
3410 }
3411
3412 spin_lock_irqsave(&cmd->t_state_lock, flags);
3413 if (cmd->transport_state & CMD_T_ABORTED) {
3414 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3415 goto check_stop;
3416 }
3417 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3418
3419 cmd->se_tfo->queue_tm_rsp(cmd);
3420
3421check_stop:
3422 transport_lun_remove_cmd(cmd);
3423 transport_cmd_check_stop_to_fabric(cmd);
3424}
3425
3426int transport_generic_handle_tmr(
3427 struct se_cmd *cmd)
3428{
3429 unsigned long flags;
3430 bool aborted = false;
3431
3432 spin_lock_irqsave(&cmd->t_state_lock, flags);
3433 if (cmd->transport_state & CMD_T_ABORTED) {
3434 aborted = true;
3435 } else {
3436 cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
3437 cmd->transport_state |= CMD_T_ACTIVE;
3438 }
3439 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3440
3441 if (aborted) {
3442 pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d"
3443 "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function,
3444 cmd->se_tmr_req->ref_task_tag, cmd->tag);
3445 transport_lun_remove_cmd(cmd);
3446 transport_cmd_check_stop_to_fabric(cmd);
3447 return 0;
3448 }
3449
3450 INIT_WORK(&cmd->work, target_tmr_work);
3451 queue_work(cmd->se_dev->tmr_wq, &cmd->work);
3452 return 0;
3453}
3454EXPORT_SYMBOL(transport_generic_handle_tmr);
3455
3456bool
3457target_check_wce(struct se_device *dev)
3458{
3459 bool wce = false;
3460
3461 if (dev->transport->get_write_cache)
3462 wce = dev->transport->get_write_cache(dev);
3463 else if (dev->dev_attrib.emulate_write_cache > 0)
3464 wce = true;
3465
3466 return wce;
3467}
3468
3469bool
3470target_check_fua(struct se_device *dev)
3471{
3472 return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0;
3473}
3474