1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/reboot.h>
21#include <linux/delay.h>
22#include <linux/interrupt.h>
23#include <linux/blkdev.h>
24#include <linux/pci.h>
25#include <linux/string.h>
26#include <linux/kernel.h>
27#include <linux/semaphore.h>
28
29#include <scsi/libiscsi.h>
30#include <scsi/scsi_transport_iscsi.h>
31#include <scsi/scsi_transport.h>
32#include <scsi/scsi_cmnd.h>
33#include <scsi/scsi_device.h>
34#include <scsi/scsi_host.h>
35#include <scsi/scsi.h>
36#include "be_main.h"
37#include "be_iscsi.h"
38#include "be_mgmt.h"
39
40static unsigned int be_iopoll_budget = 10;
41static unsigned int be_max_phys_size = 64;
42static unsigned int enable_msix;
43
44MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
45MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
46MODULE_AUTHOR("ServerEngines Corporation");
47MODULE_LICENSE("GPL");
48module_param(be_iopoll_budget, int, 0);
49module_param(enable_msix, int, 0);
50module_param(be_max_phys_size, uint, S_IRUGO);
51MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
52 "contiguous memory that can be allocated."
53 "Range is 16 - 128");
54
55static int beiscsi_slave_configure(struct scsi_device *sdev)
56{
57 blk_queue_max_segment_size(sdev->request_queue, 65536);
58 return 0;
59}
60
61static struct scsi_host_template beiscsi_sht = {
62 .module = THIS_MODULE,
63 .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
64 .proc_name = DRV_NAME,
65 .queuecommand = iscsi_queuecommand,
66 .eh_abort_handler = iscsi_eh_abort,
67 .change_queue_depth = iscsi_change_queue_depth,
68 .slave_configure = beiscsi_slave_configure,
69 .target_alloc = iscsi_target_alloc,
70 .eh_device_reset_handler = iscsi_eh_device_reset,
71 .eh_target_reset_handler = iscsi_eh_target_reset,
72 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
73 .can_queue = BE2_IO_DEPTH,
74 .this_id = -1,
75 .max_sectors = BEISCSI_MAX_SECTORS,
76 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
77 .use_clustering = ENABLE_CLUSTERING,
78};
79static struct scsi_transport_template *beiscsi_scsi_transport;
80
81
82static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
83 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
84 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
85 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
86 { 0 }
87};
88MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
89
90static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
91{
92 struct beiscsi_hba *phba;
93 struct Scsi_Host *shost;
94
95 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
96 if (!shost) {
97 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
98 "iscsi_host_alloc failed \n");
99 return NULL;
100 }
101 shost->dma_boundary = pcidev->dma_mask;
102 shost->max_id = BE2_MAX_SESSIONS;
103 shost->max_channel = 0;
104 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
105 shost->max_lun = BEISCSI_NUM_MAX_LUN;
106 shost->transportt = beiscsi_scsi_transport;
107
108 phba = iscsi_host_priv(shost);
109 memset(phba, 0, sizeof(*phba));
110 phba->shost = shost;
111 phba->pcidev = pci_dev_get(pcidev);
112
113 if (iscsi_host_add(shost, &phba->pcidev->dev))
114 goto free_devices;
115 return phba;
116
117free_devices:
118 pci_dev_put(phba->pcidev);
119 iscsi_host_free(phba->shost);
120 return NULL;
121}
122
123static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
124{
125 if (phba->csr_va) {
126 iounmap(phba->csr_va);
127 phba->csr_va = NULL;
128 }
129 if (phba->db_va) {
130 iounmap(phba->db_va);
131 phba->db_va = NULL;
132 }
133 if (phba->pci_va) {
134 iounmap(phba->pci_va);
135 phba->pci_va = NULL;
136 }
137}
138
139static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
140 struct pci_dev *pcidev)
141{
142 u8 __iomem *addr;
143
144 addr = ioremap_nocache(pci_resource_start(pcidev, 2),
145 pci_resource_len(pcidev, 2));
146 if (addr == NULL)
147 return -ENOMEM;
148 phba->ctrl.csr = addr;
149 phba->csr_va = addr;
150 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
151
152 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
153 if (addr == NULL)
154 goto pci_map_err;
155 phba->ctrl.db = addr;
156 phba->db_va = addr;
157 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
158
159 addr = ioremap_nocache(pci_resource_start(pcidev, 1),
160 pci_resource_len(pcidev, 1));
161 if (addr == NULL)
162 goto pci_map_err;
163 phba->ctrl.pcicfg = addr;
164 phba->pci_va = addr;
165 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, 1);
166 return 0;
167
168pci_map_err:
169 beiscsi_unmap_pci_function(phba);
170 return -ENOMEM;
171}
172
173static int beiscsi_enable_pci(struct pci_dev *pcidev)
174{
175 int ret;
176
177 ret = pci_enable_device(pcidev);
178 if (ret) {
179 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
180 "failed. Returning -ENODEV\n");
181 return ret;
182 }
183
184 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
185 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
186 if (ret) {
187 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
188 pci_disable_device(pcidev);
189 return ret;
190 }
191 }
192 return 0;
193}
194
195static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
196{
197 struct be_ctrl_info *ctrl = &phba->ctrl;
198 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
199 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
200 int status = 0;
201
202 ctrl->pdev = pdev;
203 status = beiscsi_map_pci_bars(phba, pdev);
204 if (status)
205 return status;
206
207 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
208 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
209 mbox_mem_alloc->size,
210 &mbox_mem_alloc->dma);
211 if (!mbox_mem_alloc->va) {
212 beiscsi_unmap_pci_function(phba);
213 status = -ENOMEM;
214 return status;
215 }
216
217 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
218 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
219 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
220 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
221 spin_lock_init(&ctrl->mbox_lock);
222 return status;
223}
224
225static void beiscsi_get_params(struct beiscsi_hba *phba)
226{
227 phba->params.ios_per_ctrl = BE2_IO_DEPTH;
228 phba->params.cxns_per_ctrl = BE2_MAX_SESSIONS;
229 phba->params.asyncpdus_per_ctrl = BE2_ASYNCPDUS;
230 phba->params.icds_per_ctrl = BE2_MAX_ICDS / 2;
231 phba->params.num_sge_per_io = BE2_SGE;
232 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
233 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
234 phba->params.eq_timer = 64;
235 phba->params.num_eq_entries =
236 (((BE2_CMDS_PER_CXN * 2 + BE2_LOGOUTS + BE2_TMFS + BE2_ASYNCPDUS) /
237 512) + 1) * 512;
238 phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
239 ? 1024 : phba->params.num_eq_entries;
240 SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
241 phba->params.num_eq_entries);
242 phba->params.num_cq_entries =
243 (((BE2_CMDS_PER_CXN * 2 + BE2_LOGOUTS + BE2_TMFS + BE2_ASYNCPDUS) /
244 512) + 1) * 512;
245 SE_DEBUG(DBG_LVL_8,
246 "phba->params.num_cq_entries=%d BE2_CMDS_PER_CXN=%d"
247 "BE2_LOGOUTS=%d BE2_TMFS=%d BE2_ASYNCPDUS=%d \n",
248 phba->params.num_cq_entries, BE2_CMDS_PER_CXN,
249 BE2_LOGOUTS, BE2_TMFS, BE2_ASYNCPDUS);
250 phba->params.wrbs_per_cxn = 256;
251}
252
253static void hwi_ring_eq_db(struct beiscsi_hba *phba,
254 unsigned int id, unsigned int clr_interrupt,
255 unsigned int num_processed,
256 unsigned char rearm, unsigned char event)
257{
258 u32 val = 0;
259 val |= id & DB_EQ_RING_ID_MASK;
260 if (rearm)
261 val |= 1 << DB_EQ_REARM_SHIFT;
262 if (clr_interrupt)
263 val |= 1 << DB_EQ_CLR_SHIFT;
264 if (event)
265 val |= 1 << DB_EQ_EVNT_SHIFT;
266 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
267 iowrite32(val, phba->db_va + DB_EQ_OFFSET);
268}
269
270
271
272
273
274
275static irqreturn_t be_isr(int irq, void *dev_id)
276{
277 struct beiscsi_hba *phba;
278 struct hwi_controller *phwi_ctrlr;
279 struct hwi_context_memory *phwi_context;
280 struct be_eq_entry *eqe = NULL;
281 struct be_queue_info *eq;
282 struct be_queue_info *cq;
283 unsigned long flags, index;
284 unsigned int num_eq_processed;
285 struct be_ctrl_info *ctrl;
286 int isr;
287
288 phba = dev_id;
289 if (!enable_msix) {
290 ctrl = &phba->ctrl;;
291 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
292 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
293 if (!isr)
294 return IRQ_NONE;
295 }
296
297 phwi_ctrlr = phba->phwi_ctrlr;
298 phwi_context = phwi_ctrlr->phwi_ctxt;
299 eq = &phwi_context->be_eq.q;
300 cq = &phwi_context->be_cq;
301 index = 0;
302 eqe = queue_tail_node(eq);
303 if (!eqe)
304 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
305
306 num_eq_processed = 0;
307 if (blk_iopoll_enabled) {
308 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
309 & EQE_VALID_MASK) {
310 if (!blk_iopoll_sched_prep(&phba->iopoll))
311 blk_iopoll_sched(&phba->iopoll);
312
313 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
314 queue_tail_inc(eq);
315 eqe = queue_tail_node(eq);
316 num_eq_processed++;
317 SE_DEBUG(DBG_LVL_8, "Valid EQE\n");
318 }
319 if (num_eq_processed) {
320 hwi_ring_eq_db(phba, eq->id, 0, num_eq_processed, 0, 1);
321 return IRQ_HANDLED;
322 } else
323 return IRQ_NONE;
324 } else {
325 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
326 & EQE_VALID_MASK) {
327
328 if (((eqe->dw[offsetof(struct amap_eq_entry,
329 resource_id) / 32] &
330 EQE_RESID_MASK) >> 16) != cq->id) {
331 spin_lock_irqsave(&phba->isr_lock, flags);
332 phba->todo_mcc_cq = 1;
333 spin_unlock_irqrestore(&phba->isr_lock, flags);
334 } else {
335 spin_lock_irqsave(&phba->isr_lock, flags);
336 phba->todo_cq = 1;
337 spin_unlock_irqrestore(&phba->isr_lock, flags);
338 }
339 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
340 queue_tail_inc(eq);
341 eqe = queue_tail_node(eq);
342 num_eq_processed++;
343 }
344 if (phba->todo_cq || phba->todo_mcc_cq)
345 queue_work(phba->wq, &phba->work_cqs);
346
347 if (num_eq_processed) {
348 hwi_ring_eq_db(phba, eq->id, 0, num_eq_processed, 1, 1);
349 return IRQ_HANDLED;
350 } else
351 return IRQ_NONE;
352 }
353}
354
355static int beiscsi_init_irqs(struct beiscsi_hba *phba)
356{
357 struct pci_dev *pcidev = phba->pcidev;
358 int ret;
359
360 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, "beiscsi", phba);
361 if (ret) {
362 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
363 "Failed to register irq\\n");
364 return ret;
365 }
366 return 0;
367}
368
369static void hwi_ring_cq_db(struct beiscsi_hba *phba,
370 unsigned int id, unsigned int num_processed,
371 unsigned char rearm, unsigned char event)
372{
373 u32 val = 0;
374 val |= id & DB_CQ_RING_ID_MASK;
375 if (rearm)
376 val |= 1 << DB_CQ_REARM_SHIFT;
377 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
378 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
379}
380
381
382
383
384
385
386
387
388
389
390static unsigned int
391beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
392 struct beiscsi_hba *phba,
393 unsigned short cid,
394 struct pdu_base *ppdu,
395 unsigned long pdu_len,
396 void *pbuffer, unsigned long buf_len)
397{
398 struct iscsi_conn *conn = beiscsi_conn->conn;
399 struct iscsi_session *session = conn->session;
400
401 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
402 PDUBASE_OPCODE_MASK) {
403 case ISCSI_OP_NOOP_IN:
404 pbuffer = NULL;
405 buf_len = 0;
406 break;
407 case ISCSI_OP_ASYNC_EVENT:
408 break;
409 case ISCSI_OP_REJECT:
410 WARN_ON(!pbuffer);
411 WARN_ON(!(buf_len == 48));
412 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
413 break;
414 case ISCSI_OP_LOGIN_RSP:
415 break;
416 default:
417 shost_printk(KERN_WARNING, phba->shost,
418 "Unrecognized opcode 0x%x in async msg \n",
419 (ppdu->
420 dw[offsetof(struct amap_pdu_base, opcode) / 32]
421 & PDUBASE_OPCODE_MASK));
422 return 1;
423 }
424
425 spin_lock_bh(&session->lock);
426 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
427 spin_unlock_bh(&session->lock);
428 return 0;
429}
430
431static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
432{
433 struct sgl_handle *psgl_handle;
434
435 if (phba->io_sgl_hndl_avbl) {
436 SE_DEBUG(DBG_LVL_8,
437 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
438 phba->io_sgl_alloc_index);
439 psgl_handle = phba->io_sgl_hndl_base[phba->
440 io_sgl_alloc_index];
441 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
442 phba->io_sgl_hndl_avbl--;
443 if (phba->io_sgl_alloc_index == (phba->params.ios_per_ctrl - 1))
444 phba->io_sgl_alloc_index = 0;
445 else
446 phba->io_sgl_alloc_index++;
447 } else
448 psgl_handle = NULL;
449 return psgl_handle;
450}
451
452static void
453free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
454{
455 SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
456 phba->io_sgl_free_index);
457 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
458
459
460
461
462 SE_DEBUG(DBG_LVL_8,
463 "Double Free in IO SGL io_sgl_free_index=%d,"
464 "value there=%p \n", phba->io_sgl_free_index,
465 phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
466 return;
467 }
468 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
469 phba->io_sgl_hndl_avbl++;
470 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
471 phba->io_sgl_free_index = 0;
472 else
473 phba->io_sgl_free_index++;
474}
475
476
477
478
479
480
481
482
483
484struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid,
485 int index)
486{
487 struct hwi_wrb_context *pwrb_context;
488 struct hwi_controller *phwi_ctrlr;
489 struct wrb_handle *pwrb_handle;
490
491 phwi_ctrlr = phba->phwi_ctrlr;
492 pwrb_context = &phwi_ctrlr->wrb_context[cid];
493 pwrb_handle = pwrb_context->pwrb_handle_base[index];
494 pwrb_handle->wrb_index = index;
495 pwrb_handle->nxt_wrb_index = index;
496 return pwrb_handle;
497}
498
499
500
501
502
503
504
505
506
507static void
508free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
509 struct wrb_handle *pwrb_handle)
510{
511 SE_DEBUG(DBG_LVL_8,
512 "FREE WRB: pwrb_handle=%p free_index=%d=0x%x"
513 "wrb_handles_available=%d \n",
514 pwrb_handle, pwrb_context->free_index,
515 pwrb_context->free_index, pwrb_context->wrb_handles_available);
516}
517
518static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
519{
520 struct sgl_handle *psgl_handle;
521
522 if (phba->eh_sgl_hndl_avbl) {
523 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
524 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
525 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
526 phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
527 phba->eh_sgl_hndl_avbl--;
528 if (phba->eh_sgl_alloc_index ==
529 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
530 1))
531 phba->eh_sgl_alloc_index = 0;
532 else
533 phba->eh_sgl_alloc_index++;
534 } else
535 psgl_handle = NULL;
536 return psgl_handle;
537}
538
539void
540free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
541{
542
543 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
544
545
546
547
548 SE_DEBUG(DBG_LVL_8,
549 "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
550 phba->eh_sgl_free_index);
551 return;
552 }
553 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
554 phba->eh_sgl_hndl_avbl++;
555 if (phba->eh_sgl_free_index ==
556 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
557 phba->eh_sgl_free_index = 0;
558 else
559 phba->eh_sgl_free_index++;
560}
561
562static void
563be_complete_io(struct beiscsi_conn *beiscsi_conn,
564 struct iscsi_task *task, struct sol_cqe *psol)
565{
566 struct beiscsi_io_task *io_task = task->dd_data;
567 struct be_status_bhs *sts_bhs =
568 (struct be_status_bhs *)io_task->cmd_bhs;
569 struct iscsi_conn *conn = beiscsi_conn->conn;
570 unsigned int sense_len;
571 unsigned char *sense;
572 u32 resid = 0, exp_cmdsn, max_cmdsn;
573 u8 rsp, status, flags;
574
575 exp_cmdsn = be32_to_cpu(psol->
576 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
577 & SOL_EXP_CMD_SN_MASK);
578 max_cmdsn = be32_to_cpu((psol->
579 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
580 & SOL_EXP_CMD_SN_MASK) +
581 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
582 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
583 rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
584 & SOL_RESP_MASK) >> 16);
585 status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
586 & SOL_STS_MASK) >> 8);
587 flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
588 & SOL_FLAGS_MASK) >> 24) | 0x80;
589
590 task->sc->result = (DID_OK << 16) | status;
591 if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
592 task->sc->result = DID_ERROR << 16;
593 goto unmap;
594 }
595
596
597 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
598 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
599 32] & SOL_RES_CNT_MASK);
600
601 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
602 task->sc->result = DID_ERROR << 16;
603
604 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
605 scsi_set_resid(task->sc, resid);
606 if (!status && (scsi_bufflen(task->sc) - resid <
607 task->sc->underflow))
608 task->sc->result = DID_ERROR << 16;
609 }
610 }
611
612 if (status == SAM_STAT_CHECK_CONDITION) {
613 sense = sts_bhs->sense_info + sizeof(unsigned short);
614 sense_len =
615 cpu_to_be16((unsigned short)(sts_bhs->sense_info[0]));
616 memcpy(task->sc->sense_buffer, sense,
617 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
618 }
619 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
620 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
621 & SOL_RES_CNT_MASK)
622 conn->rxdata_octets += (psol->
623 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
624 & SOL_RES_CNT_MASK);
625 }
626unmap:
627 scsi_dma_unmap(io_task->scsi_cmnd);
628 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
629}
630
631static void
632be_complete_logout(struct beiscsi_conn *beiscsi_conn,
633 struct iscsi_task *task, struct sol_cqe *psol)
634{
635 struct iscsi_logout_rsp *hdr;
636 struct iscsi_conn *conn = beiscsi_conn->conn;
637
638 hdr = (struct iscsi_logout_rsp *)task->hdr;
639 hdr->t2wait = 5;
640 hdr->t2retain = 0;
641 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
642 & SOL_FLAGS_MASK) >> 24) | 0x80;
643 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
644 32] & SOL_RESP_MASK);
645 hdr->exp_cmdsn = cpu_to_be32(psol->
646 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
647 & SOL_EXP_CMD_SN_MASK);
648 hdr->max_cmdsn = be32_to_cpu((psol->
649 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
650 & SOL_EXP_CMD_SN_MASK) +
651 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
652 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
653 hdr->hlength = 0;
654
655 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
656}
657
658static void
659be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
660 struct iscsi_task *task, struct sol_cqe *psol)
661{
662 struct iscsi_tm_rsp *hdr;
663 struct iscsi_conn *conn = beiscsi_conn->conn;
664
665 hdr = (struct iscsi_tm_rsp *)task->hdr;
666 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
667 & SOL_FLAGS_MASK) >> 24) | 0x80;
668 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
669 32] & SOL_RESP_MASK);
670 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
671 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
672 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
673 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
674 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
675 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
676 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
677}
678
679static void
680hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
681 struct beiscsi_hba *phba, struct sol_cqe *psol)
682{
683 struct hwi_wrb_context *pwrb_context;
684 struct wrb_handle *pwrb_handle;
685 struct hwi_controller *phwi_ctrlr;
686 struct iscsi_conn *conn = beiscsi_conn->conn;
687 struct iscsi_session *session = conn->session;
688
689 phwi_ctrlr = phba->phwi_ctrlr;
690 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
691 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
692 SOL_CID_MASK) >> 6)];
693 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
694 dw[offsetof(struct amap_sol_cqe, wrb_index) /
695 32] & SOL_WRB_INDEX_MASK) >> 16)];
696 spin_lock_bh(&session->lock);
697 free_wrb_handle(phba, pwrb_context, pwrb_handle);
698 spin_unlock_bh(&session->lock);
699}
700
701static void
702be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
703 struct iscsi_task *task, struct sol_cqe *psol)
704{
705 struct iscsi_nopin *hdr;
706 struct iscsi_conn *conn = beiscsi_conn->conn;
707
708 hdr = (struct iscsi_nopin *)task->hdr;
709 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
710 & SOL_FLAGS_MASK) >> 24) | 0x80;
711 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
712 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
713 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
714 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
715 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
716 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
717 hdr->opcode = ISCSI_OP_NOOP_IN;
718 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
719}
720
721static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
722 struct beiscsi_hba *phba, struct sol_cqe *psol)
723{
724 struct hwi_wrb_context *pwrb_context;
725 struct wrb_handle *pwrb_handle;
726 struct iscsi_wrb *pwrb = NULL;
727 struct hwi_controller *phwi_ctrlr;
728 struct iscsi_task *task;
729 struct beiscsi_io_task *io_task;
730 struct iscsi_conn *conn = beiscsi_conn->conn;
731 struct iscsi_session *session = conn->session;
732
733 phwi_ctrlr = phba->phwi_ctrlr;
734
735 pwrb_context = &phwi_ctrlr->
736 wrb_context[((psol->dw[offsetof(struct amap_sol_cqe, cid) / 32]
737 & SOL_CID_MASK) >> 6)];
738 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
739 dw[offsetof(struct amap_sol_cqe, wrb_index) /
740 32] & SOL_WRB_INDEX_MASK) >> 16)];
741
742 task = pwrb_handle->pio_handle;
743 io_task = task->dd_data;
744 spin_lock_bh(&session->lock);
745 pwrb = pwrb_handle->pwrb;
746 switch ((pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
747 WRB_TYPE_MASK) >> 28) {
748 case HWH_TYPE_IO:
749 case HWH_TYPE_IO_RD:
750 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
751 ISCSI_OP_NOOP_OUT) {
752 be_complete_nopin_resp(beiscsi_conn, task, psol);
753 } else
754 be_complete_io(beiscsi_conn, task, psol);
755 break;
756
757 case HWH_TYPE_LOGOUT:
758 be_complete_logout(beiscsi_conn, task, psol);
759 break;
760
761 case HWH_TYPE_LOGIN:
762 SE_DEBUG(DBG_LVL_1,
763 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
764 "- Solicited path \n");
765 break;
766
767 case HWH_TYPE_TMF:
768 be_complete_tmf(beiscsi_conn, task, psol);
769 break;
770
771 case HWH_TYPE_NOP:
772 be_complete_nopin_resp(beiscsi_conn, task, psol);
773 break;
774
775 default:
776 shost_printk(KERN_WARNING, phba->shost,
777 "wrb_index 0x%x CID 0x%x\n",
778 ((psol->dw[offsetof(struct amap_iscsi_wrb, type) /
779 32] & SOL_WRB_INDEX_MASK) >> 16),
780 ((psol->dw[offsetof(struct amap_sol_cqe, cid) / 32]
781 & SOL_CID_MASK) >> 6));
782 break;
783 }
784
785 spin_unlock_bh(&session->lock);
786}
787
788static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
789 *pasync_ctx, unsigned int is_header,
790 unsigned int host_write_ptr)
791{
792 if (is_header)
793 return &pasync_ctx->async_entry[host_write_ptr].
794 header_busy_list;
795 else
796 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
797}
798
799static struct async_pdu_handle *
800hwi_get_async_handle(struct beiscsi_hba *phba,
801 struct beiscsi_conn *beiscsi_conn,
802 struct hwi_async_pdu_context *pasync_ctx,
803 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
804{
805 struct be_bus_address phys_addr;
806 struct list_head *pbusy_list;
807 struct async_pdu_handle *pasync_handle = NULL;
808 int buffer_len = 0;
809 unsigned char buffer_index = -1;
810 unsigned char is_header = 0;
811
812 phys_addr.u.a32.address_lo =
813 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
814 ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
815 & PDUCQE_DPL_MASK) >> 16);
816 phys_addr.u.a32.address_hi =
817 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
818
819 phys_addr.u.a64.address =
820 *((unsigned long long *)(&phys_addr.u.a64.address));
821
822 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
823 & PDUCQE_CODE_MASK) {
824 case UNSOL_HDR_NOTIFY:
825 is_header = 1;
826
827 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
828 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
829 index) / 32] & PDUCQE_INDEX_MASK));
830
831 buffer_len = (unsigned int)(phys_addr.u.a64.address -
832 pasync_ctx->async_header.pa_base.u.a64.address);
833
834 buffer_index = buffer_len /
835 pasync_ctx->async_header.buffer_size;
836
837 break;
838 case UNSOL_DATA_NOTIFY:
839 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
840 dw[offsetof(struct amap_i_t_dpdu_cqe,
841 index) / 32] & PDUCQE_INDEX_MASK));
842 buffer_len = (unsigned long)(phys_addr.u.a64.address -
843 pasync_ctx->async_data.pa_base.u.
844 a64.address);
845 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
846 break;
847 default:
848 pbusy_list = NULL;
849 shost_printk(KERN_WARNING, phba->shost,
850 "Unexpected code=%d \n",
851 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
852 code) / 32] & PDUCQE_CODE_MASK);
853 return NULL;
854 }
855
856 WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
857 WARN_ON(list_empty(pbusy_list));
858 list_for_each_entry(pasync_handle, pbusy_list, link) {
859 WARN_ON(pasync_handle->consumed);
860 if (pasync_handle->index == buffer_index)
861 break;
862 }
863
864 WARN_ON(!pasync_handle);
865
866 pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid;
867 pasync_handle->is_header = is_header;
868 pasync_handle->buffer_len = ((pdpdu_cqe->
869 dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
870 & PDUCQE_DPL_MASK) >> 16);
871
872 *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
873 index) / 32] & PDUCQE_INDEX_MASK);
874 return pasync_handle;
875}
876
877static unsigned int
878hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
879 unsigned int is_header, unsigned int cq_index)
880{
881 struct list_head *pbusy_list;
882 struct async_pdu_handle *pasync_handle;
883 unsigned int num_entries, writables = 0;
884 unsigned int *pep_read_ptr, *pwritables;
885
886
887 if (is_header) {
888 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
889 pwritables = &pasync_ctx->async_header.writables;
890 num_entries = pasync_ctx->async_header.num_entries;
891 } else {
892 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
893 pwritables = &pasync_ctx->async_data.writables;
894 num_entries = pasync_ctx->async_data.num_entries;
895 }
896
897 while ((*pep_read_ptr) != cq_index) {
898 (*pep_read_ptr)++;
899 *pep_read_ptr = (*pep_read_ptr) % num_entries;
900
901 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
902 *pep_read_ptr);
903 if (writables == 0)
904 WARN_ON(list_empty(pbusy_list));
905
906 if (!list_empty(pbusy_list)) {
907 pasync_handle = list_entry(pbusy_list->next,
908 struct async_pdu_handle,
909 link);
910 WARN_ON(!pasync_handle);
911 pasync_handle->consumed = 1;
912 }
913
914 writables++;
915 }
916
917 if (!writables) {
918 SE_DEBUG(DBG_LVL_1,
919 "Duplicate notification received - index 0x%x!!\n",
920 cq_index);
921 WARN_ON(1);
922 }
923
924 *pwritables = *pwritables + writables;
925 return 0;
926}
927
928static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
929 unsigned int cri)
930{
931 struct hwi_controller *phwi_ctrlr;
932 struct hwi_async_pdu_context *pasync_ctx;
933 struct async_pdu_handle *pasync_handle, *tmp_handle;
934 struct list_head *plist;
935 unsigned int i = 0;
936
937 phwi_ctrlr = phba->phwi_ctrlr;
938 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
939
940 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
941
942 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
943 list_del(&pasync_handle->link);
944
945 if (i == 0) {
946 list_add_tail(&pasync_handle->link,
947 &pasync_ctx->async_header.free_list);
948 pasync_ctx->async_header.free_entries++;
949 i++;
950 } else {
951 list_add_tail(&pasync_handle->link,
952 &pasync_ctx->async_data.free_list);
953 pasync_ctx->async_data.free_entries++;
954 i++;
955 }
956 }
957
958 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
959 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
960 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
961 return 0;
962}
963
964static struct phys_addr *
965hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
966 unsigned int is_header, unsigned int host_write_ptr)
967{
968 struct phys_addr *pasync_sge = NULL;
969
970 if (is_header)
971 pasync_sge = pasync_ctx->async_header.ring_base;
972 else
973 pasync_sge = pasync_ctx->async_data.ring_base;
974
975 return pasync_sge + host_write_ptr;
976}
977
978static void hwi_post_async_buffers(struct beiscsi_hba *phba,
979 unsigned int is_header)
980{
981 struct hwi_controller *phwi_ctrlr;
982 struct hwi_async_pdu_context *pasync_ctx;
983 struct async_pdu_handle *pasync_handle;
984 struct list_head *pfree_link, *pbusy_list;
985 struct phys_addr *pasync_sge;
986 unsigned int ring_id, num_entries;
987 unsigned int host_write_num;
988 unsigned int writables;
989 unsigned int i = 0;
990 u32 doorbell = 0;
991
992 phwi_ctrlr = phba->phwi_ctrlr;
993 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
994
995 if (is_header) {
996 num_entries = pasync_ctx->async_header.num_entries;
997 writables = min(pasync_ctx->async_header.writables,
998 pasync_ctx->async_header.free_entries);
999 pfree_link = pasync_ctx->async_header.free_list.next;
1000 host_write_num = pasync_ctx->async_header.host_write_ptr;
1001 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1002 } else {
1003 num_entries = pasync_ctx->async_data.num_entries;
1004 writables = min(pasync_ctx->async_data.writables,
1005 pasync_ctx->async_data.free_entries);
1006 pfree_link = pasync_ctx->async_data.free_list.next;
1007 host_write_num = pasync_ctx->async_data.host_write_ptr;
1008 ring_id = phwi_ctrlr->default_pdu_data.id;
1009 }
1010
1011 writables = (writables / 8) * 8;
1012 if (writables) {
1013 for (i = 0; i < writables; i++) {
1014 pbusy_list =
1015 hwi_get_async_busy_list(pasync_ctx, is_header,
1016 host_write_num);
1017 pasync_handle =
1018 list_entry(pfree_link, struct async_pdu_handle,
1019 link);
1020 WARN_ON(!pasync_handle);
1021 pasync_handle->consumed = 0;
1022
1023 pfree_link = pfree_link->next;
1024
1025 pasync_sge = hwi_get_ring_address(pasync_ctx,
1026 is_header, host_write_num);
1027
1028 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1029 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1030
1031 list_move(&pasync_handle->link, pbusy_list);
1032
1033 host_write_num++;
1034 host_write_num = host_write_num % num_entries;
1035 }
1036
1037 if (is_header) {
1038 pasync_ctx->async_header.host_write_ptr =
1039 host_write_num;
1040 pasync_ctx->async_header.free_entries -= writables;
1041 pasync_ctx->async_header.writables -= writables;
1042 pasync_ctx->async_header.busy_entries += writables;
1043 } else {
1044 pasync_ctx->async_data.host_write_ptr = host_write_num;
1045 pasync_ctx->async_data.free_entries -= writables;
1046 pasync_ctx->async_data.writables -= writables;
1047 pasync_ctx->async_data.busy_entries += writables;
1048 }
1049
1050 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1051 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1052 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1053 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1054 << DB_DEF_PDU_CQPROC_SHIFT;
1055
1056 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1057 }
1058}
1059
1060static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1061 struct beiscsi_conn *beiscsi_conn,
1062 struct i_t_dpdu_cqe *pdpdu_cqe)
1063{
1064 struct hwi_controller *phwi_ctrlr;
1065 struct hwi_async_pdu_context *pasync_ctx;
1066 struct async_pdu_handle *pasync_handle = NULL;
1067 unsigned int cq_index = -1;
1068
1069 phwi_ctrlr = phba->phwi_ctrlr;
1070 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1071
1072 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1073 pdpdu_cqe, &cq_index);
1074 BUG_ON(pasync_handle->is_header != 0);
1075 if (pasync_handle->consumed == 0)
1076 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1077 cq_index);
1078
1079 hwi_free_async_msg(phba, pasync_handle->cri);
1080 hwi_post_async_buffers(phba, pasync_handle->is_header);
1081}
1082
1083static unsigned int
1084hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1085 struct beiscsi_hba *phba,
1086 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1087{
1088 struct list_head *plist;
1089 struct async_pdu_handle *pasync_handle;
1090 void *phdr = NULL;
1091 unsigned int hdr_len = 0, buf_len = 0;
1092 unsigned int status, index = 0, offset = 0;
1093 void *pfirst_buffer = NULL;
1094 unsigned int num_buf = 0;
1095
1096 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1097
1098 list_for_each_entry(pasync_handle, plist, link) {
1099 if (index == 0) {
1100 phdr = pasync_handle->pbuffer;
1101 hdr_len = pasync_handle->buffer_len;
1102 } else {
1103 buf_len = pasync_handle->buffer_len;
1104 if (!num_buf) {
1105 pfirst_buffer = pasync_handle->pbuffer;
1106 num_buf++;
1107 }
1108 memcpy(pfirst_buffer + offset,
1109 pasync_handle->pbuffer, buf_len);
1110 offset = buf_len;
1111 }
1112 index++;
1113 }
1114
1115 status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1116 beiscsi_conn->beiscsi_conn_cid,
1117 phdr, hdr_len, pfirst_buffer,
1118 buf_len);
1119
1120 if (status == 0)
1121 hwi_free_async_msg(phba, cri);
1122 return 0;
1123}
1124
1125static unsigned int
1126hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1127 struct beiscsi_hba *phba,
1128 struct async_pdu_handle *pasync_handle)
1129{
1130 struct hwi_async_pdu_context *pasync_ctx;
1131 struct hwi_controller *phwi_ctrlr;
1132 unsigned int bytes_needed = 0, status = 0;
1133 unsigned short cri = pasync_handle->cri;
1134 struct pdu_base *ppdu;
1135
1136 phwi_ctrlr = phba->phwi_ctrlr;
1137 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1138
1139 list_del(&pasync_handle->link);
1140 if (pasync_handle->is_header) {
1141 pasync_ctx->async_header.busy_entries--;
1142 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1143 hwi_free_async_msg(phba, cri);
1144 BUG();
1145 }
1146
1147 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1148 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1149 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1150 (unsigned short)pasync_handle->buffer_len;
1151 list_add_tail(&pasync_handle->link,
1152 &pasync_ctx->async_entry[cri].wait_queue.list);
1153
1154 ppdu = pasync_handle->pbuffer;
1155 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1156 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1157 0xFFFF0000) | ((be16_to_cpu((ppdu->
1158 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1159 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1160
1161 if (status == 0) {
1162 pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1163 bytes_needed;
1164
1165 if (bytes_needed == 0)
1166 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1167 pasync_ctx, cri);
1168 }
1169 } else {
1170 pasync_ctx->async_data.busy_entries--;
1171 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1172 list_add_tail(&pasync_handle->link,
1173 &pasync_ctx->async_entry[cri].wait_queue.
1174 list);
1175 pasync_ctx->async_entry[cri].wait_queue.
1176 bytes_received +=
1177 (unsigned short)pasync_handle->buffer_len;
1178
1179 if (pasync_ctx->async_entry[cri].wait_queue.
1180 bytes_received >=
1181 pasync_ctx->async_entry[cri].wait_queue.
1182 bytes_needed)
1183 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1184 pasync_ctx, cri);
1185 }
1186 }
1187 return status;
1188}
1189
1190static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1191 struct beiscsi_hba *phba,
1192 struct i_t_dpdu_cqe *pdpdu_cqe)
1193{
1194 struct hwi_controller *phwi_ctrlr;
1195 struct hwi_async_pdu_context *pasync_ctx;
1196 struct async_pdu_handle *pasync_handle = NULL;
1197 unsigned int cq_index = -1;
1198
1199 phwi_ctrlr = phba->phwi_ctrlr;
1200 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1201 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1202 pdpdu_cqe, &cq_index);
1203
1204 if (pasync_handle->consumed == 0)
1205 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1206 cq_index);
1207 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1208 hwi_post_async_buffers(phba, pasync_handle->is_header);
1209}
1210
1211static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1212{
1213 struct hwi_controller *phwi_ctrlr;
1214 struct hwi_context_memory *phwi_context;
1215 struct be_queue_info *cq;
1216 struct sol_cqe *sol;
1217 struct dmsg_cqe *dmsg;
1218 unsigned int num_processed = 0;
1219 unsigned int tot_nump = 0;
1220 struct beiscsi_conn *beiscsi_conn;
1221
1222 phwi_ctrlr = phba->phwi_ctrlr;
1223 phwi_context = phwi_ctrlr->phwi_ctxt;
1224 cq = &phwi_context->be_cq;
1225 sol = queue_tail_node(cq);
1226
1227 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1228 CQE_VALID_MASK) {
1229 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1230
1231 beiscsi_conn = phba->conn_table[(u32) (sol->
1232 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1233 SOL_CID_MASK) >> 6];
1234
1235 if (!beiscsi_conn || !beiscsi_conn->ep) {
1236 shost_printk(KERN_WARNING, phba->shost,
1237 "Connection table empty for cid = %d\n",
1238 (u32)(sol->dw[offsetof(struct amap_sol_cqe,
1239 cid) / 32] & SOL_CID_MASK) >> 6);
1240 return 0;
1241 }
1242
1243 if (num_processed >= 32) {
1244 hwi_ring_cq_db(phba, phwi_context->be_cq.id,
1245 num_processed, 0, 0);
1246 tot_nump += num_processed;
1247 num_processed = 0;
1248 }
1249
1250 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1251 32] & CQE_CODE_MASK) {
1252 case SOL_CMD_COMPLETE:
1253 hwi_complete_cmd(beiscsi_conn, phba, sol);
1254 break;
1255 case DRIVERMSG_NOTIFY:
1256 SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1257 dmsg = (struct dmsg_cqe *)sol;
1258 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1259 break;
1260 case UNSOL_HDR_NOTIFY:
1261 case UNSOL_DATA_NOTIFY:
1262 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR/DATA_NOTIFY\n");
1263 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1264 (struct i_t_dpdu_cqe *)sol);
1265 break;
1266 case CXN_INVALIDATE_INDEX_NOTIFY:
1267 case CMD_INVALIDATED_NOTIFY:
1268 case CXN_INVALIDATE_NOTIFY:
1269 SE_DEBUG(DBG_LVL_1,
1270 "Ignoring CQ Error notification for cmd/cxn"
1271 "invalidate\n");
1272 break;
1273 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1274 case CMD_KILLED_INVALID_STATSN_RCVD:
1275 case CMD_KILLED_INVALID_R2T_RCVD:
1276 case CMD_CXN_KILLED_LUN_INVALID:
1277 case CMD_CXN_KILLED_ICD_INVALID:
1278 case CMD_CXN_KILLED_ITT_INVALID:
1279 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1280 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1281 SE_DEBUG(DBG_LVL_1,
1282 "CQ Error notification for cmd.. "
1283 "code %d cid 0x%x\n",
1284 sol->dw[offsetof(struct amap_sol_cqe, code) /
1285 32] & CQE_CODE_MASK,
1286 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1287 32] & SOL_CID_MASK));
1288 break;
1289 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1290 SE_DEBUG(DBG_LVL_1,
1291 "Digest error on def pdu ring, dropping..\n");
1292 hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1293 (struct i_t_dpdu_cqe *) sol);
1294 break;
1295 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1296 case CXN_KILLED_BURST_LEN_MISMATCH:
1297 case CXN_KILLED_AHS_RCVD:
1298 case CXN_KILLED_HDR_DIGEST_ERR:
1299 case CXN_KILLED_UNKNOWN_HDR:
1300 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1301 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1302 case CXN_KILLED_TIMED_OUT:
1303 case CXN_KILLED_FIN_RCVD:
1304 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1305 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1306 case CXN_KILLED_OVER_RUN_RESIDUAL:
1307 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1308 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1309 SE_DEBUG(DBG_LVL_1, "CQ Error %d, resetting CID "
1310 "0x%x...\n",
1311 sol->dw[offsetof(struct amap_sol_cqe, code) /
1312 32] & CQE_CODE_MASK,
1313 sol->dw[offsetof(struct amap_sol_cqe, cid) /
1314 32] & CQE_CID_MASK);
1315 iscsi_conn_failure(beiscsi_conn->conn,
1316 ISCSI_ERR_CONN_FAILED);
1317 break;
1318 case CXN_KILLED_RST_SENT:
1319 case CXN_KILLED_RST_RCVD:
1320 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset received/sent "
1321 "on CID 0x%x...\n",
1322 sol->dw[offsetof(struct amap_sol_cqe, code) /
1323 32] & CQE_CODE_MASK,
1324 sol->dw[offsetof(struct amap_sol_cqe, cid) /
1325 32] & CQE_CID_MASK);
1326 iscsi_conn_failure(beiscsi_conn->conn,
1327 ISCSI_ERR_CONN_FAILED);
1328 break;
1329 default:
1330 SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1331 "received on CID 0x%x...\n",
1332 sol->dw[offsetof(struct amap_sol_cqe, code) /
1333 32] & CQE_CODE_MASK,
1334 sol->dw[offsetof(struct amap_sol_cqe, cid) /
1335 32] & CQE_CID_MASK);
1336 break;
1337 }
1338
1339 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1340 queue_tail_inc(cq);
1341 sol = queue_tail_node(cq);
1342 num_processed++;
1343 }
1344
1345 if (num_processed > 0) {
1346 tot_nump += num_processed;
1347 hwi_ring_cq_db(phba, phwi_context->be_cq.id, num_processed,
1348 1, 0);
1349 }
1350 return tot_nump;
1351}
1352
1353static void beiscsi_process_all_cqs(struct work_struct *work)
1354{
1355 unsigned long flags;
1356 struct beiscsi_hba *phba =
1357 container_of(work, struct beiscsi_hba, work_cqs);
1358
1359 if (phba->todo_mcc_cq) {
1360 spin_lock_irqsave(&phba->isr_lock, flags);
1361 phba->todo_mcc_cq = 0;
1362 spin_unlock_irqrestore(&phba->isr_lock, flags);
1363 SE_DEBUG(DBG_LVL_1, "MCC Interrupt Not expected \n");
1364 }
1365
1366 if (phba->todo_cq) {
1367 spin_lock_irqsave(&phba->isr_lock, flags);
1368 phba->todo_cq = 0;
1369 spin_unlock_irqrestore(&phba->isr_lock, flags);
1370 beiscsi_process_cq(phba);
1371 }
1372}
1373
1374static int be_iopoll(struct blk_iopoll *iop, int budget)
1375{
1376 static unsigned int ret;
1377 struct beiscsi_hba *phba;
1378
1379 phba = container_of(iop, struct beiscsi_hba, iopoll);
1380
1381 ret = beiscsi_process_cq(phba);
1382 if (ret < budget) {
1383 struct hwi_controller *phwi_ctrlr;
1384 struct hwi_context_memory *phwi_context;
1385
1386 phwi_ctrlr = phba->phwi_ctrlr;
1387 phwi_context = phwi_ctrlr->phwi_ctxt;
1388 blk_iopoll_complete(iop);
1389 hwi_ring_eq_db(phba, phwi_context->be_eq.q.id, 0,
1390 0, 1, 1);
1391 }
1392 return ret;
1393}
1394
1395static void
1396hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1397 unsigned int num_sg, struct beiscsi_io_task *io_task)
1398{
1399 struct iscsi_sge *psgl;
1400 unsigned short sg_len, index;
1401 unsigned int sge_len = 0;
1402 unsigned long long addr;
1403 struct scatterlist *l_sg;
1404 unsigned int offset;
1405
1406 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1407 io_task->bhs_pa.u.a32.address_lo);
1408 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1409 io_task->bhs_pa.u.a32.address_hi);
1410
1411 l_sg = sg;
1412 for (index = 0; (index < num_sg) && (index < 2); index++, sg_next(sg)) {
1413 if (index == 0) {
1414 sg_len = sg_dma_len(sg);
1415 addr = (u64) sg_dma_address(sg);
1416 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1417 (addr & 0xFFFFFFFF));
1418 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1419 (addr >> 32));
1420 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1421 sg_len);
1422 sge_len = sg_len;
1423 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1424 1);
1425 } else {
1426 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1427 0);
1428 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1429 pwrb, sge_len);
1430 sg_len = sg_dma_len(sg);
1431 addr = (u64) sg_dma_address(sg);
1432 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1433 (addr & 0xFFFFFFFF));
1434 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1435 (addr >> 32));
1436 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1437 sg_len);
1438 }
1439 }
1440 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1441 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1442
1443 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1444
1445 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1446 io_task->bhs_pa.u.a32.address_hi);
1447 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1448 io_task->bhs_pa.u.a32.address_lo);
1449
1450 if (num_sg == 2)
1451 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1);
1452 sg = l_sg;
1453 psgl++;
1454 psgl++;
1455 offset = 0;
1456 for (index = 0; index < num_sg; index++, sg_next(sg), psgl++) {
1457 sg_len = sg_dma_len(sg);
1458 addr = (u64) sg_dma_address(sg);
1459 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1460 (addr & 0xFFFFFFFF));
1461 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1462 (addr >> 32));
1463 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1464 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1465 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1466 offset += sg_len;
1467 }
1468 psgl--;
1469 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1470}
1471
1472static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1473{
1474 struct iscsi_sge *psgl;
1475 unsigned long long addr;
1476 struct beiscsi_io_task *io_task = task->dd_data;
1477 struct beiscsi_conn *beiscsi_conn = io_task->conn;
1478 struct beiscsi_hba *phba = beiscsi_conn->phba;
1479
1480 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1481 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1482 io_task->bhs_pa.u.a32.address_lo);
1483 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1484 io_task->bhs_pa.u.a32.address_hi);
1485
1486 if (task->data) {
1487 if (task->data_count) {
1488 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1489 addr = (u64) pci_map_single(phba->pcidev,
1490 task->data,
1491 task->data_count, 1);
1492 } else {
1493 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1494 addr = 0;
1495 }
1496 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1497 (addr & 0xFFFFFFFF));
1498 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1499 (addr >> 32));
1500 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1501 task->data_count);
1502
1503 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1504 } else {
1505 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1506 addr = 0;
1507 }
1508
1509 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1510
1511 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1512
1513 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1514 io_task->bhs_pa.u.a32.address_hi);
1515 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1516 io_task->bhs_pa.u.a32.address_lo);
1517 if (task->data) {
1518 psgl++;
1519 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1520 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1521 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1522 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1523 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1524 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1525
1526 psgl++;
1527 if (task->data) {
1528 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1529 (addr & 0xFFFFFFFF));
1530 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1531 (addr >> 32));
1532 }
1533 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1534 }
1535 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1536}
1537
1538static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1539{
1540 unsigned int num_cq_pages, num_eq_pages, num_async_pdu_buf_pages;
1541 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1542 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1543
1544 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1545 sizeof(struct sol_cqe));
1546 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
1547 sizeof(struct be_eq_entry));
1548 num_async_pdu_buf_pages =
1549 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1550 phba->params.defpdu_hdr_sz);
1551 num_async_pdu_buf_sgl_pages =
1552 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1553 sizeof(struct phys_addr));
1554 num_async_pdu_data_pages =
1555 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1556 phba->params.defpdu_data_sz);
1557 num_async_pdu_data_sgl_pages =
1558 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1559 sizeof(struct phys_addr));
1560
1561 phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1562
1563 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1564 BE_ISCSI_PDU_HEADER_SIZE;
1565 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1566 sizeof(struct hwi_context_memory);
1567
1568 phba->mem_req[HWI_MEM_CQ] = num_cq_pages * PAGE_SIZE;
1569 phba->mem_req[HWI_MEM_EQ] = num_eq_pages * PAGE_SIZE;
1570
1571 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1572 * (phba->params.wrbs_per_cxn)
1573 * phba->params.cxns_per_ctrl;
1574 wrb_sz_per_cxn = sizeof(struct wrb_handle) *
1575 (phba->params.wrbs_per_cxn);
1576 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1577 phba->params.cxns_per_ctrl);
1578
1579 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1580 phba->params.icds_per_ctrl;
1581 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1582 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1583
1584 phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1585 num_async_pdu_buf_pages * PAGE_SIZE;
1586 phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1587 num_async_pdu_data_pages * PAGE_SIZE;
1588 phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1589 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1590 phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1591 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1592 phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1593 phba->params.asyncpdus_per_ctrl *
1594 sizeof(struct async_pdu_handle);
1595 phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1596 phba->params.asyncpdus_per_ctrl *
1597 sizeof(struct async_pdu_handle);
1598 phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1599 sizeof(struct hwi_async_pdu_context) +
1600 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1601}
1602
1603static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1604{
1605 struct be_mem_descriptor *mem_descr;
1606 dma_addr_t bus_add;
1607 struct mem_array *mem_arr, *mem_arr_orig;
1608 unsigned int i, j, alloc_size, curr_alloc_size;
1609
1610 phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1611 if (!phba->phwi_ctrlr)
1612 return -ENOMEM;
1613
1614 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1615 GFP_KERNEL);
1616 if (!phba->init_mem) {
1617 kfree(phba->phwi_ctrlr);
1618 return -ENOMEM;
1619 }
1620
1621 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1622 GFP_KERNEL);
1623 if (!mem_arr_orig) {
1624 kfree(phba->init_mem);
1625 kfree(phba->phwi_ctrlr);
1626 return -ENOMEM;
1627 }
1628
1629 mem_descr = phba->init_mem;
1630 for (i = 0; i < SE_MEM_MAX; i++) {
1631 j = 0;
1632 mem_arr = mem_arr_orig;
1633 alloc_size = phba->mem_req[i];
1634 memset(mem_arr, 0, sizeof(struct mem_array) *
1635 BEISCSI_MAX_FRAGS_INIT);
1636 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
1637 do {
1638 mem_arr->virtual_address = pci_alloc_consistent(
1639 phba->pcidev,
1640 curr_alloc_size,
1641 &bus_add);
1642 if (!mem_arr->virtual_address) {
1643 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
1644 goto free_mem;
1645 if (curr_alloc_size -
1646 rounddown_pow_of_two(curr_alloc_size))
1647 curr_alloc_size = rounddown_pow_of_two
1648 (curr_alloc_size);
1649 else
1650 curr_alloc_size = curr_alloc_size / 2;
1651 } else {
1652 mem_arr->bus_address.u.
1653 a64.address = (__u64) bus_add;
1654 mem_arr->size = curr_alloc_size;
1655 alloc_size -= curr_alloc_size;
1656 curr_alloc_size = min(be_max_phys_size *
1657 1024, alloc_size);
1658 j++;
1659 mem_arr++;
1660 }
1661 } while (alloc_size);
1662 mem_descr->num_elements = j;
1663 mem_descr->size_in_bytes = phba->mem_req[i];
1664 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
1665 GFP_KERNEL);
1666 if (!mem_descr->mem_array)
1667 goto free_mem;
1668
1669 memcpy(mem_descr->mem_array, mem_arr_orig,
1670 sizeof(struct mem_array) * j);
1671 mem_descr++;
1672 }
1673 kfree(mem_arr_orig);
1674 return 0;
1675free_mem:
1676 mem_descr->num_elements = j;
1677 while ((i) || (j)) {
1678 for (j = mem_descr->num_elements; j > 0; j--) {
1679 pci_free_consistent(phba->pcidev,
1680 mem_descr->mem_array[j - 1].size,
1681 mem_descr->mem_array[j - 1].
1682 virtual_address,
1683 mem_descr->mem_array[j - 1].
1684 bus_address.u.a64.address);
1685 }
1686 if (i) {
1687 i--;
1688 kfree(mem_descr->mem_array);
1689 mem_descr--;
1690 }
1691 }
1692 kfree(mem_arr_orig);
1693 kfree(phba->init_mem);
1694 kfree(phba->phwi_ctrlr);
1695 return -ENOMEM;
1696}
1697
1698static int beiscsi_get_memory(struct beiscsi_hba *phba)
1699{
1700 beiscsi_find_mem_req(phba);
1701 return beiscsi_alloc_mem(phba);
1702}
1703
1704static void iscsi_init_global_templates(struct beiscsi_hba *phba)
1705{
1706 struct pdu_data_out *pdata_out;
1707 struct pdu_nop_out *pnop_out;
1708 struct be_mem_descriptor *mem_descr;
1709
1710 mem_descr = phba->init_mem;
1711 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
1712 pdata_out =
1713 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
1714 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1715
1716 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
1717 IIOC_SCSI_DATA);
1718
1719 pnop_out =
1720 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
1721 virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
1722
1723 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1724 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
1725 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
1726 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
1727}
1728
1729static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1730{
1731 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
1732 struct wrb_handle *pwrb_handle;
1733 struct hwi_controller *phwi_ctrlr;
1734 struct hwi_wrb_context *pwrb_context;
1735 struct iscsi_wrb *pwrb;
1736 unsigned int num_cxn_wrbh;
1737 unsigned int num_cxn_wrb, j, idx, index;
1738
1739 mem_descr_wrbh = phba->init_mem;
1740 mem_descr_wrbh += HWI_MEM_WRBH;
1741
1742 mem_descr_wrb = phba->init_mem;
1743 mem_descr_wrb += HWI_MEM_WRB;
1744
1745 idx = 0;
1746 pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
1747 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
1748 ((sizeof(struct wrb_handle)) *
1749 phba->params.wrbs_per_cxn));
1750 phwi_ctrlr = phba->phwi_ctrlr;
1751
1752 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
1753 pwrb_context = &phwi_ctrlr->wrb_context[index];
1754 SE_DEBUG(DBG_LVL_8, "cid=%d pwrb_context=%p \n", index,
1755 pwrb_context);
1756 pwrb_context->pwrb_handle_base =
1757 kzalloc(sizeof(struct wrb_handle *) *
1758 phba->params.wrbs_per_cxn, GFP_KERNEL);
1759 pwrb_context->pwrb_handle_basestd =
1760 kzalloc(sizeof(struct wrb_handle *) *
1761 phba->params.wrbs_per_cxn, GFP_KERNEL);
1762 if (num_cxn_wrbh) {
1763 pwrb_context->alloc_index = 0;
1764 pwrb_context->wrb_handles_available = 0;
1765 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1766 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
1767 pwrb_context->pwrb_handle_basestd[j] =
1768 pwrb_handle;
1769 pwrb_context->wrb_handles_available++;
1770 pwrb_handle++;
1771 }
1772 pwrb_context->free_index = 0;
1773 num_cxn_wrbh--;
1774 } else {
1775 idx++;
1776 pwrb_handle =
1777 mem_descr_wrbh->mem_array[idx].virtual_address;
1778 num_cxn_wrbh =
1779 ((mem_descr_wrbh->mem_array[idx].size) /
1780 ((sizeof(struct wrb_handle)) *
1781 phba->params.wrbs_per_cxn));
1782 pwrb_context->alloc_index = 0;
1783 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1784 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
1785 pwrb_context->pwrb_handle_basestd[j] =
1786 pwrb_handle;
1787 pwrb_context->wrb_handles_available++;
1788 pwrb_handle++;
1789 }
1790 pwrb_context->free_index = 0;
1791 num_cxn_wrbh--;
1792 }
1793 }
1794 idx = 0;
1795 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
1796 num_cxn_wrb =
1797 ((mem_descr_wrb->mem_array[idx].size) / (sizeof(struct iscsi_wrb)) *
1798 phba->params.wrbs_per_cxn);
1799
1800 for (index = 0; index < phba->params.cxns_per_ctrl; index += 2) {
1801 pwrb_context = &phwi_ctrlr->wrb_context[index];
1802 if (num_cxn_wrb) {
1803 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1804 pwrb_handle = pwrb_context->pwrb_handle_base[j];
1805 pwrb_handle->pwrb = pwrb;
1806 pwrb++;
1807 }
1808 num_cxn_wrb--;
1809 } else {
1810 idx++;
1811 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
1812 num_cxn_wrb = ((mem_descr_wrb->mem_array[idx].size) /
1813 (sizeof(struct iscsi_wrb)) *
1814 phba->params.wrbs_per_cxn);
1815 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1816 pwrb_handle = pwrb_context->pwrb_handle_base[j];
1817 pwrb_handle->pwrb = pwrb;
1818 pwrb++;
1819 }
1820 num_cxn_wrb--;
1821 }
1822 }
1823}
1824
1825static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
1826{
1827 struct hwi_controller *phwi_ctrlr;
1828 struct hba_parameters *p = &phba->params;
1829 struct hwi_async_pdu_context *pasync_ctx;
1830 struct async_pdu_handle *pasync_header_h, *pasync_data_h;
1831 unsigned int index;
1832 struct be_mem_descriptor *mem_descr;
1833
1834 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1835 mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
1836
1837 phwi_ctrlr = phba->phwi_ctrlr;
1838 phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
1839 mem_descr->mem_array[0].virtual_address;
1840 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
1841 memset(pasync_ctx, 0, sizeof(*pasync_ctx));
1842
1843 pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
1844 pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
1845 pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
1846 pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
1847
1848 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1849 mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
1850 if (mem_descr->mem_array[0].virtual_address) {
1851 SE_DEBUG(DBG_LVL_8,
1852 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
1853 "va=%p \n", mem_descr->mem_array[0].virtual_address);
1854 } else
1855 shost_printk(KERN_WARNING, phba->shost,
1856 "No Virtual address \n");
1857
1858 pasync_ctx->async_header.va_base =
1859 mem_descr->mem_array[0].virtual_address;
1860
1861 pasync_ctx->async_header.pa_base.u.a64.address =
1862 mem_descr->mem_array[0].bus_address.u.a64.address;
1863
1864 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1865 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
1866 if (mem_descr->mem_array[0].virtual_address) {
1867 SE_DEBUG(DBG_LVL_8,
1868 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
1869 "va=%p \n", mem_descr->mem_array[0].virtual_address);
1870 } else
1871 shost_printk(KERN_WARNING, phba->shost,
1872 "No Virtual address \n");
1873 pasync_ctx->async_header.ring_base =
1874 mem_descr->mem_array[0].virtual_address;
1875
1876 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1877 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
1878 if (mem_descr->mem_array[0].virtual_address) {
1879 SE_DEBUG(DBG_LVL_8,
1880 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
1881 "va=%p \n", mem_descr->mem_array[0].virtual_address);
1882 } else
1883 shost_printk(KERN_WARNING, phba->shost,
1884 "No Virtual address \n");
1885
1886 pasync_ctx->async_header.handle_base =
1887 mem_descr->mem_array[0].virtual_address;
1888 pasync_ctx->async_header.writables = 0;
1889 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
1890
1891 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1892 mem_descr += HWI_MEM_ASYNC_DATA_BUF;
1893 if (mem_descr->mem_array[0].virtual_address) {
1894 SE_DEBUG(DBG_LVL_8,
1895 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
1896 "va=%p \n", mem_descr->mem_array[0].virtual_address);
1897 } else
1898 shost_printk(KERN_WARNING, phba->shost,
1899 "No Virtual address \n");
1900 pasync_ctx->async_data.va_base =
1901 mem_descr->mem_array[0].virtual_address;
1902 pasync_ctx->async_data.pa_base.u.a64.address =
1903 mem_descr->mem_array[0].bus_address.u.a64.address;
1904
1905 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1906 mem_descr += HWI_MEM_ASYNC_DATA_RING;
1907 if (mem_descr->mem_array[0].virtual_address) {
1908 SE_DEBUG(DBG_LVL_8,
1909 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
1910 "va=%p \n", mem_descr->mem_array[0].virtual_address);
1911 } else
1912 shost_printk(KERN_WARNING, phba->shost,
1913 "No Virtual address \n");
1914
1915 pasync_ctx->async_data.ring_base =
1916 mem_descr->mem_array[0].virtual_address;
1917
1918 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1919 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
1920 if (!mem_descr->mem_array[0].virtual_address)
1921 shost_printk(KERN_WARNING, phba->shost,
1922 "No Virtual address \n");
1923
1924 pasync_ctx->async_data.handle_base =
1925 mem_descr->mem_array[0].virtual_address;
1926 pasync_ctx->async_data.writables = 0;
1927 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
1928
1929 pasync_header_h =
1930 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
1931 pasync_data_h =
1932 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
1933
1934 for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
1935 pasync_header_h->cri = -1;
1936 pasync_header_h->index = (char)index;
1937 INIT_LIST_HEAD(&pasync_header_h->link);
1938 pasync_header_h->pbuffer =
1939 (void *)((unsigned long)
1940 (pasync_ctx->async_header.va_base) +
1941 (p->defpdu_hdr_sz * index));
1942
1943 pasync_header_h->pa.u.a64.address =
1944 pasync_ctx->async_header.pa_base.u.a64.address +
1945 (p->defpdu_hdr_sz * index);
1946
1947 list_add_tail(&pasync_header_h->link,
1948 &pasync_ctx->async_header.free_list);
1949 pasync_header_h++;
1950 pasync_ctx->async_header.free_entries++;
1951 pasync_ctx->async_header.writables++;
1952
1953 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
1954 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
1955 header_busy_list);
1956 pasync_data_h->cri = -1;
1957 pasync_data_h->index = (char)index;
1958 INIT_LIST_HEAD(&pasync_data_h->link);
1959 pasync_data_h->pbuffer =
1960 (void *)((unsigned long)
1961 (pasync_ctx->async_data.va_base) +
1962 (p->defpdu_data_sz * index));
1963
1964 pasync_data_h->pa.u.a64.address =
1965 pasync_ctx->async_data.pa_base.u.a64.address +
1966 (p->defpdu_data_sz * index);
1967
1968 list_add_tail(&pasync_data_h->link,
1969 &pasync_ctx->async_data.free_list);
1970 pasync_data_h++;
1971 pasync_ctx->async_data.free_entries++;
1972 pasync_ctx->async_data.writables++;
1973
1974 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
1975 }
1976
1977 pasync_ctx->async_header.host_write_ptr = 0;
1978 pasync_ctx->async_header.ep_read_ptr = -1;
1979 pasync_ctx->async_data.host_write_ptr = 0;
1980 pasync_ctx->async_data.ep_read_ptr = -1;
1981}
1982
1983static int
1984be_sgl_create_contiguous(void *virtual_address,
1985 u64 physical_address, u32 length,
1986 struct be_dma_mem *sgl)
1987{
1988 WARN_ON(!virtual_address);
1989 WARN_ON(!physical_address);
1990 WARN_ON(!length > 0);
1991 WARN_ON(!sgl);
1992
1993 sgl->va = virtual_address;
1994 sgl->dma = physical_address;
1995 sgl->size = length;
1996
1997 return 0;
1998}
1999
2000static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2001{
2002 memset(sgl, 0, sizeof(*sgl));
2003}
2004
2005static void
2006hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2007 struct mem_array *pmem, struct be_dma_mem *sgl)
2008{
2009 if (sgl->va)
2010 be_sgl_destroy_contiguous(sgl);
2011
2012 be_sgl_create_contiguous(pmem->virtual_address,
2013 pmem->bus_address.u.a64.address,
2014 pmem->size, sgl);
2015}
2016
2017static void
2018hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2019 struct mem_array *pmem, struct be_dma_mem *sgl)
2020{
2021 if (sgl->va)
2022 be_sgl_destroy_contiguous(sgl);
2023
2024 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2025 pmem->bus_address.u.a64.address,
2026 pmem->size, sgl);
2027}
2028
2029static int be_fill_queue(struct be_queue_info *q,
2030 u16 len, u16 entry_size, void *vaddress)
2031{
2032 struct be_dma_mem *mem = &q->dma_mem;
2033
2034 memset(q, 0, sizeof(*q));
2035 q->len = len;
2036 q->entry_size = entry_size;
2037 mem->size = len * entry_size;
2038 mem->va = vaddress;
2039 if (!mem->va)
2040 return -ENOMEM;
2041 memset(mem->va, 0, mem->size);
2042 return 0;
2043}
2044
2045static int beiscsi_create_eq(struct beiscsi_hba *phba,
2046 struct hwi_context_memory *phwi_context)
2047{
2048 unsigned int idx;
2049 int ret;
2050 struct be_queue_info *eq;
2051 struct be_dma_mem *mem;
2052 struct be_mem_descriptor *mem_descr;
2053 void *eq_vaddress;
2054
2055 idx = 0;
2056 eq = &phwi_context->be_eq.q;
2057 mem = &eq->dma_mem;
2058 mem_descr = phba->init_mem;
2059 mem_descr += HWI_MEM_EQ;
2060 eq_vaddress = mem_descr->mem_array[idx].virtual_address;
2061
2062 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2063 sizeof(struct be_eq_entry), eq_vaddress);
2064 if (ret) {
2065 shost_printk(KERN_ERR, phba->shost,
2066 "be_fill_queue Failed for EQ \n");
2067 return ret;
2068 }
2069
2070 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2071
2072 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2073 phwi_context->be_eq.cur_eqd);
2074 if (ret) {
2075 shost_printk(KERN_ERR, phba->shost, "beiscsi_cmd_eq_create"
2076 "Failedfor EQ \n");
2077 return ret;
2078 }
2079 SE_DEBUG(DBG_LVL_8, "eq id is %d\n", phwi_context->be_eq.q.id);
2080 return 0;
2081}
2082
2083static int beiscsi_create_cq(struct beiscsi_hba *phba,
2084 struct hwi_context_memory *phwi_context)
2085{
2086 unsigned int idx;
2087 int ret;
2088 struct be_queue_info *cq, *eq;
2089 struct be_dma_mem *mem;
2090 struct be_mem_descriptor *mem_descr;
2091 void *cq_vaddress;
2092
2093 idx = 0;
2094 cq = &phwi_context->be_cq;
2095 eq = &phwi_context->be_eq.q;
2096 mem = &cq->dma_mem;
2097 mem_descr = phba->init_mem;
2098 mem_descr += HWI_MEM_CQ;
2099 cq_vaddress = mem_descr->mem_array[idx].virtual_address;
2100 ret = be_fill_queue(cq, phba->params.icds_per_ctrl / 2,
2101 sizeof(struct sol_cqe), cq_vaddress);
2102 if (ret) {
2103 shost_printk(KERN_ERR, phba->shost,
2104 "be_fill_queue Failed for ISCSI CQ \n");
2105 return ret;
2106 }
2107
2108 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2109 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, false, 0);
2110 if (ret) {
2111 shost_printk(KERN_ERR, phba->shost,
2112 "beiscsi_cmd_eq_create Failed for ISCSI CQ \n");
2113 return ret;
2114 }
2115 SE_DEBUG(DBG_LVL_8, "iscsi cq id is %d\n", phwi_context->be_cq.id);
2116 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2117 return 0;
2118}
2119
2120static int
2121beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2122 struct hwi_context_memory *phwi_context,
2123 struct hwi_controller *phwi_ctrlr,
2124 unsigned int def_pdu_ring_sz)
2125{
2126 unsigned int idx;
2127 int ret;
2128 struct be_queue_info *dq, *cq;
2129 struct be_dma_mem *mem;
2130 struct be_mem_descriptor *mem_descr;
2131 void *dq_vaddress;
2132
2133 idx = 0;
2134 dq = &phwi_context->be_def_hdrq;
2135 cq = &phwi_context->be_cq;
2136 mem = &dq->dma_mem;
2137 mem_descr = phba->init_mem;
2138 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2139 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2140 ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2141 sizeof(struct phys_addr),
2142 sizeof(struct phys_addr), dq_vaddress);
2143 if (ret) {
2144 shost_printk(KERN_ERR, phba->shost,
2145 "be_fill_queue Failed for DEF PDU HDR\n");
2146 return ret;
2147 }
2148 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2149 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2150 def_pdu_ring_sz,
2151 phba->params.defpdu_hdr_sz);
2152 if (ret) {
2153 shost_printk(KERN_ERR, phba->shost,
2154 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2155 return ret;
2156 }
2157 phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2158 SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2159 phwi_context->be_def_hdrq.id);
2160 hwi_post_async_buffers(phba, 1);
2161 return 0;
2162}
2163
2164static int
2165beiscsi_create_def_data(struct beiscsi_hba *phba,
2166 struct hwi_context_memory *phwi_context,
2167 struct hwi_controller *phwi_ctrlr,
2168 unsigned int def_pdu_ring_sz)
2169{
2170 unsigned int idx;
2171 int ret;
2172 struct be_queue_info *dataq, *cq;
2173 struct be_dma_mem *mem;
2174 struct be_mem_descriptor *mem_descr;
2175 void *dq_vaddress;
2176
2177 idx = 0;
2178 dataq = &phwi_context->be_def_dataq;
2179 cq = &phwi_context->be_cq;
2180 mem = &dataq->dma_mem;
2181 mem_descr = phba->init_mem;
2182 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2183 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2184 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2185 sizeof(struct phys_addr),
2186 sizeof(struct phys_addr), dq_vaddress);
2187 if (ret) {
2188 shost_printk(KERN_ERR, phba->shost,
2189 "be_fill_queue Failed for DEF PDU DATA\n");
2190 return ret;
2191 }
2192 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2193 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2194 def_pdu_ring_sz,
2195 phba->params.defpdu_data_sz);
2196 if (ret) {
2197 shost_printk(KERN_ERR, phba->shost,
2198 "be_cmd_create_default_pdu_queue Failed"
2199 " for DEF PDU DATA\n");
2200 return ret;
2201 }
2202 phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2203 SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2204 phwi_context->be_def_dataq.id);
2205 hwi_post_async_buffers(phba, 0);
2206 SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2207 return 0;
2208}
2209
2210static int
2211beiscsi_post_pages(struct beiscsi_hba *phba)
2212{
2213 struct be_mem_descriptor *mem_descr;
2214 struct mem_array *pm_arr;
2215 unsigned int page_offset, i;
2216 struct be_dma_mem sgl;
2217 int status;
2218
2219 mem_descr = phba->init_mem;
2220 mem_descr += HWI_MEM_SGE;
2221 pm_arr = mem_descr->mem_array;
2222
2223 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2224 phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2225 for (i = 0; i < mem_descr->num_elements; i++) {
2226 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2227 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2228 page_offset,
2229 (pm_arr->size / PAGE_SIZE));
2230 page_offset += pm_arr->size / PAGE_SIZE;
2231 if (status != 0) {
2232 shost_printk(KERN_ERR, phba->shost,
2233 "post sgl failed.\n");
2234 return status;
2235 }
2236 pm_arr++;
2237 }
2238 SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2239 return 0;
2240}
2241
2242static int
2243beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2244 struct hwi_context_memory *phwi_context,
2245 struct hwi_controller *phwi_ctrlr)
2246{
2247 unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2248 u64 pa_addr_lo;
2249 unsigned int idx, num, i;
2250 struct mem_array *pwrb_arr;
2251 void *wrb_vaddr;
2252 struct be_dma_mem sgl;
2253 struct be_mem_descriptor *mem_descr;
2254 int status;
2255
2256 idx = 0;
2257 mem_descr = phba->init_mem;
2258 mem_descr += HWI_MEM_WRB;
2259 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2260 GFP_KERNEL);
2261 if (!pwrb_arr) {
2262 shost_printk(KERN_ERR, phba->shost,
2263 "Memory alloc failed in create wrb ring.\n");
2264 return -ENOMEM;
2265 }
2266 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2267 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2268 num_wrb_rings = mem_descr->mem_array[idx].size /
2269 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2270
2271 for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2272 if (num_wrb_rings) {
2273 pwrb_arr[num].virtual_address = wrb_vaddr;
2274 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2275 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2276 sizeof(struct iscsi_wrb);
2277 wrb_vaddr += pwrb_arr[num].size;
2278 pa_addr_lo += pwrb_arr[num].size;
2279 num_wrb_rings--;
2280 } else {
2281 idx++;
2282 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2283 pa_addr_lo = mem_descr->mem_array[idx].\
2284 bus_address.u.a64.address;
2285 num_wrb_rings = mem_descr->mem_array[idx].size /
2286 (phba->params.wrbs_per_cxn *
2287 sizeof(struct iscsi_wrb));
2288 pwrb_arr[num].virtual_address = wrb_vaddr;
2289 pwrb_arr[num].bus_address.u.a64.address\
2290 = pa_addr_lo;
2291 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2292 sizeof(struct iscsi_wrb);
2293 wrb_vaddr += pwrb_arr[num].size;
2294 pa_addr_lo += pwrb_arr[num].size;
2295 num_wrb_rings--;
2296 }
2297 }
2298 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2299 wrb_mem_index = 0;
2300 offset = 0;
2301 size = 0;
2302
2303 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2304 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2305 &phwi_context->be_wrbq[i]);
2306 if (status != 0) {
2307 shost_printk(KERN_ERR, phba->shost,
2308 "wrbq create failed.");
2309 return status;
2310 }
2311 phwi_ctrlr->wrb_context[i].cid = phwi_context->be_wrbq[i].id;
2312 }
2313 kfree(pwrb_arr);
2314 return 0;
2315}
2316
2317static void free_wrb_handles(struct beiscsi_hba *phba)
2318{
2319 unsigned int index;
2320 struct hwi_controller *phwi_ctrlr;
2321 struct hwi_wrb_context *pwrb_context;
2322
2323 phwi_ctrlr = phba->phwi_ctrlr;
2324 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2325 pwrb_context = &phwi_ctrlr->wrb_context[index];
2326 kfree(pwrb_context->pwrb_handle_base);
2327 kfree(pwrb_context->pwrb_handle_basestd);
2328 }
2329}
2330
2331static void hwi_cleanup(struct beiscsi_hba *phba)
2332{
2333 struct be_queue_info *q;
2334 struct be_ctrl_info *ctrl = &phba->ctrl;
2335 struct hwi_controller *phwi_ctrlr;
2336 struct hwi_context_memory *phwi_context;
2337 int i;
2338
2339 phwi_ctrlr = phba->phwi_ctrlr;
2340 phwi_context = phwi_ctrlr->phwi_ctxt;
2341 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2342 q = &phwi_context->be_wrbq[i];
2343 if (q->created)
2344 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2345 }
2346
2347 free_wrb_handles(phba);
2348
2349 q = &phwi_context->be_def_hdrq;
2350 if (q->created)
2351 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2352
2353 q = &phwi_context->be_def_dataq;
2354 if (q->created)
2355 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2356
2357 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2358
2359 q = &phwi_context->be_cq;
2360 if (q->created)
2361 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2362
2363 q = &phwi_context->be_eq.q;
2364 if (q->created)
2365 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2366}
2367
2368static int hwi_init_port(struct beiscsi_hba *phba)
2369{
2370 struct hwi_controller *phwi_ctrlr;
2371 struct hwi_context_memory *phwi_context;
2372 unsigned int def_pdu_ring_sz;
2373 struct be_ctrl_info *ctrl = &phba->ctrl;
2374 int status;
2375
2376 def_pdu_ring_sz =
2377 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2378 phwi_ctrlr = phba->phwi_ctrlr;
2379
2380 phwi_context = phwi_ctrlr->phwi_ctxt;
2381 phwi_context->be_eq.max_eqd = 0;
2382 phwi_context->be_eq.min_eqd = 0;
2383 phwi_context->be_eq.cur_eqd = 64;
2384 phwi_context->be_eq.enable_aic = false;
2385 be_cmd_fw_initialize(&phba->ctrl);
2386 status = beiscsi_create_eq(phba, phwi_context);
2387 if (status != 0) {
2388 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2389 goto error;
2390 }
2391
2392 status = mgmt_check_supported_fw(ctrl);
2393 if (status != 0) {
2394 shost_printk(KERN_ERR, phba->shost,
2395 "Unsupported fw version \n");
2396 goto error;
2397 }
2398
2399 status = mgmt_get_fw_config(ctrl, phba);
2400 if (status != 0) {
2401 shost_printk(KERN_ERR, phba->shost,
2402 "Error getting fw config\n");
2403 goto error;
2404 }
2405
2406 status = beiscsi_create_cq(phba, phwi_context);
2407 if (status != 0) {
2408 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2409 goto error;
2410 }
2411
2412 status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2413 def_pdu_ring_sz);
2414 if (status != 0) {
2415 shost_printk(KERN_ERR, phba->shost,
2416 "Default Header not created\n");
2417 goto error;
2418 }
2419
2420 status = beiscsi_create_def_data(phba, phwi_context,
2421 phwi_ctrlr, def_pdu_ring_sz);
2422 if (status != 0) {
2423 shost_printk(KERN_ERR, phba->shost,
2424 "Default Data not created\n");
2425 goto error;
2426 }
2427
2428 status = beiscsi_post_pages(phba);
2429 if (status != 0) {
2430 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2431 goto error;
2432 }
2433
2434 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2435 if (status != 0) {
2436 shost_printk(KERN_ERR, phba->shost,
2437 "WRB Rings not created\n");
2438 goto error;
2439 }
2440
2441 SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2442 return 0;
2443
2444error:
2445 shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2446 hwi_cleanup(phba);
2447 return -ENOMEM;
2448}
2449
2450
2451static int hwi_init_controller(struct beiscsi_hba *phba)
2452{
2453 struct hwi_controller *phwi_ctrlr;
2454
2455 phwi_ctrlr = phba->phwi_ctrlr;
2456 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2457 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2458 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2459 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2460 phwi_ctrlr->phwi_ctxt);
2461 } else {
2462 shost_printk(KERN_ERR, phba->shost,
2463 "HWI_MEM_ADDN_CONTEXT is more than one element."
2464 "Failing to load\n");
2465 return -ENOMEM;
2466 }
2467
2468 iscsi_init_global_templates(phba);
2469 beiscsi_init_wrb_handle(phba);
2470 hwi_init_async_pdu_ctx(phba);
2471 if (hwi_init_port(phba) != 0) {
2472 shost_printk(KERN_ERR, phba->shost,
2473 "hwi_init_controller failed\n");
2474 return -ENOMEM;
2475 }
2476 return 0;
2477}
2478
2479static void beiscsi_free_mem(struct beiscsi_hba *phba)
2480{
2481 struct be_mem_descriptor *mem_descr;
2482 int i, j;
2483
2484 mem_descr = phba->init_mem;
2485 i = 0;
2486 j = 0;
2487 for (i = 0; i < SE_MEM_MAX; i++) {
2488 for (j = mem_descr->num_elements; j > 0; j--) {
2489 pci_free_consistent(phba->pcidev,
2490 mem_descr->mem_array[j - 1].size,
2491 mem_descr->mem_array[j - 1].virtual_address,
2492 mem_descr->mem_array[j - 1].bus_address.
2493 u.a64.address);
2494 }
2495 kfree(mem_descr->mem_array);
2496 mem_descr++;
2497 }
2498 kfree(phba->init_mem);
2499 kfree(phba->phwi_ctrlr);
2500}
2501
2502static int beiscsi_init_controller(struct beiscsi_hba *phba)
2503{
2504 int ret = -ENOMEM;
2505
2506 ret = beiscsi_get_memory(phba);
2507 if (ret < 0) {
2508 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
2509 "Failed in beiscsi_alloc_memory \n");
2510 return ret;
2511 }
2512
2513 ret = hwi_init_controller(phba);
2514 if (ret)
2515 goto free_init;
2516 SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
2517 return 0;
2518
2519free_init:
2520 beiscsi_free_mem(phba);
2521 return -ENOMEM;
2522}
2523
2524static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
2525{
2526 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
2527 struct sgl_handle *psgl_handle;
2528 struct iscsi_sge *pfrag;
2529 unsigned int arr_index, i, idx;
2530
2531 phba->io_sgl_hndl_avbl = 0;
2532 phba->eh_sgl_hndl_avbl = 0;
2533 mem_descr_sglh = phba->init_mem;
2534 mem_descr_sglh += HWI_MEM_SGLH;
2535 if (1 == mem_descr_sglh->num_elements) {
2536 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2537 phba->params.ios_per_ctrl,
2538 GFP_KERNEL);
2539 if (!phba->io_sgl_hndl_base) {
2540 shost_printk(KERN_ERR, phba->shost,
2541 "Mem Alloc Failed. Failing to load\n");
2542 return -ENOMEM;
2543 }
2544 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2545 (phba->params.icds_per_ctrl -
2546 phba->params.ios_per_ctrl),
2547 GFP_KERNEL);
2548 if (!phba->eh_sgl_hndl_base) {
2549 kfree(phba->io_sgl_hndl_base);
2550 shost_printk(KERN_ERR, phba->shost,
2551 "Mem Alloc Failed. Failing to load\n");
2552 return -ENOMEM;
2553 }
2554 } else {
2555 shost_printk(KERN_ERR, phba->shost,
2556 "HWI_MEM_SGLH is more than one element."
2557 "Failing to load\n");
2558 return -ENOMEM;
2559 }
2560
2561 arr_index = 0;
2562 idx = 0;
2563 while (idx < mem_descr_sglh->num_elements) {
2564 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
2565
2566 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
2567 sizeof(struct sgl_handle)); i++) {
2568 if (arr_index < phba->params.ios_per_ctrl) {
2569 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
2570 phba->io_sgl_hndl_avbl++;
2571 arr_index++;
2572 } else {
2573 phba->eh_sgl_hndl_base[arr_index -
2574 phba->params.ios_per_ctrl] =
2575 psgl_handle;
2576 arr_index++;
2577 phba->eh_sgl_hndl_avbl++;
2578 }
2579 psgl_handle++;
2580 }
2581 idx++;
2582 }
2583 SE_DEBUG(DBG_LVL_8,
2584 "phba->io_sgl_hndl_avbl=%d"
2585 "phba->eh_sgl_hndl_avbl=%d \n",
2586 phba->io_sgl_hndl_avbl,
2587 phba->eh_sgl_hndl_avbl);
2588 mem_descr_sg = phba->init_mem;
2589 mem_descr_sg += HWI_MEM_SGE;
2590 SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
2591 mem_descr_sg->num_elements);
2592 arr_index = 0;
2593 idx = 0;
2594 while (idx < mem_descr_sg->num_elements) {
2595 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
2596
2597 for (i = 0;
2598 i < (mem_descr_sg->mem_array[idx].size) /
2599 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
2600 i++) {
2601 if (arr_index < phba->params.ios_per_ctrl)
2602 psgl_handle = phba->io_sgl_hndl_base[arr_index];
2603 else
2604 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
2605 phba->params.ios_per_ctrl];
2606 psgl_handle->pfrag = pfrag;
2607 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
2608 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
2609 pfrag += phba->params.num_sge_per_io;
2610 psgl_handle->sgl_index =
2611 phba->fw_config.iscsi_cid_start + arr_index++;
2612 }
2613 idx++;
2614 }
2615 phba->io_sgl_free_index = 0;
2616 phba->io_sgl_alloc_index = 0;
2617 phba->eh_sgl_free_index = 0;
2618 phba->eh_sgl_alloc_index = 0;
2619 return 0;
2620}
2621
2622static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
2623{
2624 int i, new_cid;
2625
2626 phba->cid_array = kmalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
2627 GFP_KERNEL);
2628 if (!phba->cid_array) {
2629 shost_printk(KERN_ERR, phba->shost,
2630 "Failed to allocate memory in "
2631 "hba_setup_cid_tbls\n");
2632 return -ENOMEM;
2633 }
2634 phba->ep_array = kmalloc(sizeof(struct iscsi_endpoint *) *
2635 phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
2636 if (!phba->ep_array) {
2637 shost_printk(KERN_ERR, phba->shost,
2638 "Failed to allocate memory in "
2639 "hba_setup_cid_tbls \n");
2640 kfree(phba->cid_array);
2641 return -ENOMEM;
2642 }
2643 new_cid = phba->fw_config.iscsi_icd_start;
2644 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2645 phba->cid_array[i] = new_cid;
2646 new_cid += 2;
2647 }
2648 phba->avlbl_cids = phba->params.cxns_per_ctrl;
2649 return 0;
2650}
2651
2652static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
2653{
2654 struct be_ctrl_info *ctrl = &phba->ctrl;
2655 struct hwi_controller *phwi_ctrlr;
2656 struct hwi_context_memory *phwi_context;
2657 struct be_queue_info *eq;
2658 u8 __iomem *addr;
2659 u32 reg;
2660 u32 enabled;
2661
2662 phwi_ctrlr = phba->phwi_ctrlr;
2663 phwi_context = phwi_ctrlr->phwi_ctxt;
2664
2665 eq = &phwi_context->be_eq.q;
2666 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
2667 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
2668 reg = ioread32(addr);
2669 SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
2670
2671 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2672 if (!enabled) {
2673 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2674 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
2675 iowrite32(reg, addr);
2676 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
2677
2678 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
2679 } else
2680 shost_printk(KERN_WARNING, phba->shost,
2681 "In hwi_enable_intr, Not Enabled \n");
2682 return true;
2683}
2684
2685static void hwi_disable_intr(struct beiscsi_hba *phba)
2686{
2687 struct be_ctrl_info *ctrl = &phba->ctrl;
2688
2689 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
2690 u32 reg = ioread32(addr);
2691
2692 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2693 if (enabled) {
2694 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2695 iowrite32(reg, addr);
2696 } else
2697 shost_printk(KERN_WARNING, phba->shost,
2698 "In hwi_disable_intr, Already Disabled \n");
2699}
2700
2701static int beiscsi_init_port(struct beiscsi_hba *phba)
2702{
2703 int ret;
2704
2705 ret = beiscsi_init_controller(phba);
2706 if (ret < 0) {
2707 shost_printk(KERN_ERR, phba->shost,
2708 "beiscsi_dev_probe - Failed in"
2709 "beiscsi_init_controller \n");
2710 return ret;
2711 }
2712 ret = beiscsi_init_sgl_handle(phba);
2713 if (ret < 0) {
2714 shost_printk(KERN_ERR, phba->shost,
2715 "beiscsi_dev_probe - Failed in"
2716 "beiscsi_init_sgl_handle \n");
2717 goto do_cleanup_ctrlr;
2718 }
2719
2720 if (hba_setup_cid_tbls(phba)) {
2721 shost_printk(KERN_ERR, phba->shost,
2722 "Failed in hba_setup_cid_tbls\n");
2723 kfree(phba->io_sgl_hndl_base);
2724 kfree(phba->eh_sgl_hndl_base);
2725 goto do_cleanup_ctrlr;
2726 }
2727
2728 return ret;
2729
2730do_cleanup_ctrlr:
2731 hwi_cleanup(phba);
2732 return ret;
2733}
2734
2735static void hwi_purge_eq(struct beiscsi_hba *phba)
2736{
2737 struct hwi_controller *phwi_ctrlr;
2738 struct hwi_context_memory *phwi_context;
2739 struct be_queue_info *eq;
2740 struct be_eq_entry *eqe = NULL;
2741
2742 phwi_ctrlr = phba->phwi_ctrlr;
2743 phwi_context = phwi_ctrlr->phwi_ctxt;
2744 eq = &phwi_context->be_eq.q;
2745 eqe = queue_tail_node(eq);
2746
2747 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
2748 & EQE_VALID_MASK) {
2749 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
2750 queue_tail_inc(eq);
2751 eqe = queue_tail_node(eq);
2752 }
2753}
2754
2755static void beiscsi_clean_port(struct beiscsi_hba *phba)
2756{
2757 unsigned char mgmt_status;
2758
2759 mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
2760 if (mgmt_status)
2761 shost_printk(KERN_WARNING, phba->shost,
2762 "mgmt_epfw_cleanup FAILED \n");
2763 hwi_cleanup(phba);
2764 hwi_purge_eq(phba);
2765 kfree(phba->io_sgl_hndl_base);
2766 kfree(phba->eh_sgl_hndl_base);
2767 kfree(phba->cid_array);
2768 kfree(phba->ep_array);
2769}
2770
2771void
2772beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
2773 struct beiscsi_offload_params *params)
2774{
2775 struct wrb_handle *pwrb_handle;
2776 struct iscsi_target_context_update_wrb *pwrb = NULL;
2777 struct be_mem_descriptor *mem_descr;
2778 struct beiscsi_hba *phba = beiscsi_conn->phba;
2779 u32 doorbell = 0;
2780
2781
2782
2783
2784
2785 pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid, 0);
2786 pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
2787 memset(pwrb, 0, sizeof(*pwrb));
2788 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2789 max_burst_length, pwrb, params->dw[offsetof
2790 (struct amap_beiscsi_offload_params,
2791 max_burst_length) / 32]);
2792 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2793 max_send_data_segment_length, pwrb,
2794 params->dw[offsetof(struct amap_beiscsi_offload_params,
2795 max_send_data_segment_length) / 32]);
2796 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2797 first_burst_length,
2798 pwrb,
2799 params->dw[offsetof(struct amap_beiscsi_offload_params,
2800 first_burst_length) / 32]);
2801
2802 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
2803 (params->dw[offsetof(struct amap_beiscsi_offload_params,
2804 erl) / 32] & OFFLD_PARAMS_ERL));
2805 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
2806 (params->dw[offsetof(struct amap_beiscsi_offload_params,
2807 dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
2808 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
2809 (params->dw[offsetof(struct amap_beiscsi_offload_params,
2810 hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
2811 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
2812 (params->dw[offsetof(struct amap_beiscsi_offload_params,
2813 ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
2814 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
2815 (params->dw[offsetof(struct amap_beiscsi_offload_params,
2816 imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
2817 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
2818 pwrb,
2819 (params->dw[offsetof(struct amap_beiscsi_offload_params,
2820 exp_statsn) / 32] + 1));
2821 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
2822 0x7);
2823 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
2824 pwrb, pwrb_handle->wrb_index);
2825 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
2826 pwrb, pwrb_handle->nxt_wrb_index);
2827 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2828 session_state, pwrb, 0);
2829 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
2830 pwrb, 1);
2831 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
2832 pwrb, 0);
2833 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
2834 0);
2835
2836 mem_descr = phba->init_mem;
2837 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2838
2839 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2840 pad_buffer_addr_hi, pwrb,
2841 mem_descr->mem_array[0].bus_address.u.a32.address_hi);
2842 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2843 pad_buffer_addr_lo, pwrb,
2844 mem_descr->mem_array[0].bus_address.u.a32.address_lo);
2845
2846 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
2847
2848 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
2849 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) <<
2850 DB_DEF_PDU_WRB_INDEX_SHIFT;
2851 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
2852
2853 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
2854}
2855
2856static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
2857 int *index, int *age)
2858{
2859 *index = be32_to_cpu(itt) >> 16;
2860 if (age)
2861 *age = conn->session->age;
2862}
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
2875{
2876 struct beiscsi_io_task *io_task = task->dd_data;
2877 struct iscsi_conn *conn = task->conn;
2878 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
2879 struct beiscsi_hba *phba = beiscsi_conn->phba;
2880 struct hwi_wrb_context *pwrb_context;
2881 struct hwi_controller *phwi_ctrlr;
2882 itt_t itt;
2883 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
2884 dma_addr_t paddr;
2885
2886 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
2887 GFP_KERNEL, &paddr);
2888
2889 if (!io_task->cmd_bhs)
2890 return -ENOMEM;
2891
2892 io_task->bhs_pa.u.a64.address = paddr;
2893 io_task->pwrb_handle = alloc_wrb_handle(phba,
2894 beiscsi_conn->beiscsi_conn_cid,
2895 task->itt);
2896 io_task->pwrb_handle->pio_handle = task;
2897 io_task->conn = beiscsi_conn;
2898
2899 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
2900 task->hdr_max = sizeof(struct be_cmd_bhs);
2901
2902 if (task->sc) {
2903 spin_lock(&phba->io_sgl_lock);
2904 io_task->psgl_handle = alloc_io_sgl_handle(phba);
2905 spin_unlock(&phba->io_sgl_lock);
2906 if (!io_task->psgl_handle)
2907 goto free_hndls;
2908
2909 } else {
2910 io_task->scsi_cmnd = NULL;
2911 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
2912 if (!beiscsi_conn->login_in_progress) {
2913 spin_lock(&phba->mgmt_sgl_lock);
2914 io_task->psgl_handle = (struct sgl_handle *)
2915 alloc_mgmt_sgl_handle(phba);
2916 spin_unlock(&phba->mgmt_sgl_lock);
2917 if (!io_task->psgl_handle)
2918 goto free_hndls;
2919
2920 beiscsi_conn->login_in_progress = 1;
2921 beiscsi_conn->plogin_sgl_handle =
2922 io_task->psgl_handle;
2923 } else {
2924 io_task->psgl_handle =
2925 beiscsi_conn->plogin_sgl_handle;
2926 }
2927 } else {
2928 spin_lock(&phba->mgmt_sgl_lock);
2929 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
2930 spin_unlock(&phba->mgmt_sgl_lock);
2931 if (!io_task->psgl_handle)
2932 goto free_hndls;
2933 }
2934 }
2935 itt = (itt_t) cpu_to_be32(((unsigned int)task->itt << 16) |
2936 (unsigned int)(io_task->psgl_handle->sgl_index));
2937 io_task->cmd_bhs->iscsi_hdr.itt = itt;
2938 return 0;
2939
2940free_hndls:
2941 phwi_ctrlr = phba->phwi_ctrlr;
2942 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid];
2943 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2944 io_task->pwrb_handle = NULL;
2945 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
2946 io_task->bhs_pa.u.a64.address);
2947 SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed \n");
2948 return -ENOMEM;
2949}
2950
2951static void beiscsi_cleanup_task(struct iscsi_task *task)
2952{
2953 struct beiscsi_io_task *io_task = task->dd_data;
2954 struct iscsi_conn *conn = task->conn;
2955 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
2956 struct beiscsi_hba *phba = beiscsi_conn->phba;
2957 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
2958 struct hwi_wrb_context *pwrb_context;
2959 struct hwi_controller *phwi_ctrlr;
2960
2961 phwi_ctrlr = phba->phwi_ctrlr;
2962 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid];
2963 if (io_task->pwrb_handle) {
2964 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2965 io_task->pwrb_handle = NULL;
2966 }
2967
2968 if (io_task->cmd_bhs) {
2969 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
2970 io_task->bhs_pa.u.a64.address);
2971 }
2972
2973 if (task->sc) {
2974 if (io_task->psgl_handle) {
2975 spin_lock(&phba->io_sgl_lock);
2976 free_io_sgl_handle(phba, io_task->psgl_handle);
2977 spin_unlock(&phba->io_sgl_lock);
2978 io_task->psgl_handle = NULL;
2979 }
2980 } else {
2981 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
2982 return;
2983 if (io_task->psgl_handle) {
2984 spin_lock(&phba->mgmt_sgl_lock);
2985 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
2986 spin_unlock(&phba->mgmt_sgl_lock);
2987 io_task->psgl_handle = NULL;
2988 }
2989 }
2990}
2991
2992static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
2993 unsigned int num_sg, unsigned int xferlen,
2994 unsigned int writedir)
2995{
2996
2997 struct beiscsi_io_task *io_task = task->dd_data;
2998 struct iscsi_conn *conn = task->conn;
2999 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3000 struct beiscsi_hba *phba = beiscsi_conn->phba;
3001 struct iscsi_wrb *pwrb = NULL;
3002 unsigned int doorbell = 0;
3003
3004 pwrb = io_task->pwrb_handle->pwrb;
3005 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3006 io_task->bhs_len = sizeof(struct be_cmd_bhs);
3007
3008 if (writedir) {
3009 SE_DEBUG(DBG_LVL_4, " WRITE Command \t");
3010 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3011 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3012 &io_task->cmd_bhs->iscsi_data_pdu,
3013 (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3014 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3015 &io_task->cmd_bhs->iscsi_data_pdu,
3016 ISCSI_OPCODE_SCSI_DATA_OUT);
3017 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3018 &io_task->cmd_bhs->iscsi_data_pdu, 1);
3019 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_WR_CMD);
3020 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3021 } else {
3022 SE_DEBUG(DBG_LVL_4, "READ Command \t");
3023 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_RD_CMD);
3024 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3025 }
3026 memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3027 dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3028 io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3029
3030 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3031 cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3032 lun[0]));
3033 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3034 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3035 io_task->pwrb_handle->wrb_index);
3036 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3037 be32_to_cpu(task->cmdsn));
3038 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3039 io_task->psgl_handle->sgl_index);
3040
3041 hwi_write_sgl(pwrb, sg, num_sg, io_task);
3042
3043 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3044 io_task->pwrb_handle->nxt_wrb_index);
3045 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3046
3047 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3048 doorbell |= (io_task->pwrb_handle->wrb_index &
3049 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3050 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3051
3052 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3053 return 0;
3054}
3055
3056static int beiscsi_mtask(struct iscsi_task *task)
3057{
3058 struct beiscsi_io_task *aborted_io_task, *io_task = task->dd_data;
3059 struct iscsi_conn *conn = task->conn;
3060 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3061 struct beiscsi_hba *phba = beiscsi_conn->phba;
3062 struct iscsi_wrb *pwrb = NULL;
3063 unsigned int doorbell = 0;
3064 struct iscsi_task *aborted_task;
3065
3066 pwrb = io_task->pwrb_handle->pwrb;
3067 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3068 be32_to_cpu(task->cmdsn));
3069 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3070 io_task->pwrb_handle->wrb_index);
3071 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3072 io_task->psgl_handle->sgl_index);
3073
3074 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3075 case ISCSI_OP_LOGIN:
3076 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, TGT_DM_CMD);
3077 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3078 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3079 hwi_write_buffer(pwrb, task);
3080 break;
3081 case ISCSI_OP_NOOP_OUT:
3082 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_RD_CMD);
3083 hwi_write_buffer(pwrb, task);
3084 break;
3085 case ISCSI_OP_TEXT:
3086 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_WR_CMD);
3087 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3088 hwi_write_buffer(pwrb, task);
3089 break;
3090 case ISCSI_OP_SCSI_TMFUNC:
3091 aborted_task = iscsi_itt_to_task(conn,
3092 ((struct iscsi_tm *)task->hdr)->rtt);
3093 if (!aborted_task)
3094 return 0;
3095 aborted_io_task = aborted_task->dd_data;
3096 if (!aborted_io_task->scsi_cmnd)
3097 return 0;
3098
3099 mgmt_invalidate_icds(phba,
3100 aborted_io_task->psgl_handle->sgl_index,
3101 beiscsi_conn->beiscsi_conn_cid);
3102 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_TMF_CMD);
3103 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3104 hwi_write_buffer(pwrb, task);
3105 break;
3106 case ISCSI_OP_LOGOUT:
3107 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3108 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3109 HWH_TYPE_LOGOUT);
3110 hwi_write_buffer(pwrb, task);
3111 break;
3112
3113 default:
3114 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3115 task->hdr->opcode & ISCSI_OPCODE_MASK);
3116 return -EINVAL;
3117 }
3118
3119 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3120 be32_to_cpu(task->data_count));
3121 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3122 io_task->pwrb_handle->nxt_wrb_index);
3123 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3124
3125 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3126 doorbell |= (io_task->pwrb_handle->wrb_index &
3127 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3128 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3129 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3130 return 0;
3131}
3132
3133static int beiscsi_task_xmit(struct iscsi_task *task)
3134{
3135 struct iscsi_conn *conn = task->conn;
3136 struct beiscsi_io_task *io_task = task->dd_data;
3137 struct scsi_cmnd *sc = task->sc;
3138 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3139 struct scatterlist *sg;
3140 int num_sg;
3141 unsigned int writedir = 0, xferlen = 0;
3142
3143 SE_DEBUG(DBG_LVL_4, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t"
3144 "beiscsi_conn=%p \n", beiscsi_conn->beiscsi_conn_cid,
3145 task, conn, beiscsi_conn);
3146 if (!sc)
3147 return beiscsi_mtask(task);
3148
3149 io_task->scsi_cmnd = sc;
3150 num_sg = scsi_dma_map(sc);
3151 if (num_sg < 0) {
3152 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3153 return num_sg;
3154 }
3155 SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3156 (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3157 xferlen = scsi_bufflen(sc);
3158 sg = scsi_sglist(sc);
3159 if (sc->sc_data_direction == DMA_TO_DEVICE) {
3160 writedir = 1;
3161 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3162 task->imm_count);
3163 } else
3164 writedir = 0;
3165 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3166}
3167
3168static void beiscsi_remove(struct pci_dev *pcidev)
3169{
3170 struct beiscsi_hba *phba = NULL;
3171
3172 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3173 if (!phba) {
3174 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3175 return;
3176 }
3177
3178 hwi_disable_intr(phba);
3179 if (phba->pcidev->irq)
3180 free_irq(phba->pcidev->irq, phba);
3181 destroy_workqueue(phba->wq);
3182 if (blk_iopoll_enabled)
3183 blk_iopoll_disable(&phba->iopoll);
3184
3185 beiscsi_clean_port(phba);
3186 beiscsi_free_mem(phba);
3187 beiscsi_unmap_pci_function(phba);
3188 pci_free_consistent(phba->pcidev,
3189 phba->ctrl.mbox_mem_alloced.size,
3190 phba->ctrl.mbox_mem_alloced.va,
3191 phba->ctrl.mbox_mem_alloced.dma);
3192 iscsi_host_remove(phba->shost);
3193 pci_dev_put(phba->pcidev);
3194 iscsi_host_free(phba->shost);
3195}
3196
3197static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3198 const struct pci_device_id *id)
3199{
3200 struct beiscsi_hba *phba = NULL;
3201 int ret;
3202
3203 ret = beiscsi_enable_pci(pcidev);
3204 if (ret < 0) {
3205 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3206 "Failed to enable pci device \n");
3207 return ret;
3208 }
3209
3210 phba = beiscsi_hba_alloc(pcidev);
3211 if (!phba) {
3212 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3213 " Failed in beiscsi_hba_alloc \n");
3214 goto disable_pci;
3215 }
3216
3217 pci_set_drvdata(pcidev, phba);
3218 ret = be_ctrl_init(phba, pcidev);
3219 if (ret) {
3220 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3221 "Failed in be_ctrl_init\n");
3222 goto hba_free;
3223 }
3224
3225 spin_lock_init(&phba->io_sgl_lock);
3226 spin_lock_init(&phba->mgmt_sgl_lock);
3227 spin_lock_init(&phba->isr_lock);
3228 beiscsi_get_params(phba);
3229 ret = beiscsi_init_port(phba);
3230 if (ret < 0) {
3231 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3232 "Failed in beiscsi_init_port\n");
3233 goto free_port;
3234 }
3235
3236 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3237 phba->shost->host_no);
3238 phba->wq = create_singlethread_workqueue(phba->wq_name);
3239 if (!phba->wq) {
3240 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3241 "Failed to allocate work queue\n");
3242 goto free_twq;
3243 }
3244
3245 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3246
3247 if (blk_iopoll_enabled) {
3248 blk_iopoll_init(&phba->iopoll, be_iopoll_budget, be_iopoll);
3249 blk_iopoll_enable(&phba->iopoll);
3250 }
3251
3252 ret = beiscsi_init_irqs(phba);
3253 if (ret < 0) {
3254 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3255 "Failed to beiscsi_init_irqs\n");
3256 goto free_blkenbld;
3257 }
3258 ret = hwi_enable_intr(phba);
3259 if (ret < 0) {
3260 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3261 "Failed to hwi_enable_intr\n");
3262 goto free_ctrlr;
3263 }
3264
3265 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3266 return 0;
3267
3268free_ctrlr:
3269 if (phba->pcidev->irq)
3270 free_irq(phba->pcidev->irq, phba);
3271free_blkenbld:
3272 destroy_workqueue(phba->wq);
3273 if (blk_iopoll_enabled)
3274 blk_iopoll_disable(&phba->iopoll);
3275free_twq:
3276 beiscsi_clean_port(phba);
3277 beiscsi_free_mem(phba);
3278free_port:
3279 pci_free_consistent(phba->pcidev,
3280 phba->ctrl.mbox_mem_alloced.size,
3281 phba->ctrl.mbox_mem_alloced.va,
3282 phba->ctrl.mbox_mem_alloced.dma);
3283 beiscsi_unmap_pci_function(phba);
3284hba_free:
3285 iscsi_host_remove(phba->shost);
3286 pci_dev_put(phba->pcidev);
3287 iscsi_host_free(phba->shost);
3288disable_pci:
3289 pci_disable_device(pcidev);
3290 return ret;
3291}
3292
3293struct iscsi_transport beiscsi_iscsi_transport = {
3294 .owner = THIS_MODULE,
3295 .name = DRV_NAME,
3296 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST |
3297 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3298 .param_mask = ISCSI_MAX_RECV_DLENGTH |
3299 ISCSI_MAX_XMIT_DLENGTH |
3300 ISCSI_HDRDGST_EN |
3301 ISCSI_DATADGST_EN |
3302 ISCSI_INITIAL_R2T_EN |
3303 ISCSI_MAX_R2T |
3304 ISCSI_IMM_DATA_EN |
3305 ISCSI_FIRST_BURST |
3306 ISCSI_MAX_BURST |
3307 ISCSI_PDU_INORDER_EN |
3308 ISCSI_DATASEQ_INORDER_EN |
3309 ISCSI_ERL |
3310 ISCSI_CONN_PORT |
3311 ISCSI_CONN_ADDRESS |
3312 ISCSI_EXP_STATSN |
3313 ISCSI_PERSISTENT_PORT |
3314 ISCSI_PERSISTENT_ADDRESS |
3315 ISCSI_TARGET_NAME | ISCSI_TPGT |
3316 ISCSI_USERNAME | ISCSI_PASSWORD |
3317 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3318 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3319 ISCSI_LU_RESET_TMO |
3320 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3321 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3322 .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3323 ISCSI_HOST_INITIATOR_NAME,
3324 .create_session = beiscsi_session_create,
3325 .destroy_session = beiscsi_session_destroy,
3326 .create_conn = beiscsi_conn_create,
3327 .bind_conn = beiscsi_conn_bind,
3328 .destroy_conn = iscsi_conn_teardown,
3329 .set_param = beiscsi_set_param,
3330 .get_conn_param = beiscsi_conn_get_param,
3331 .get_session_param = iscsi_session_get_param,
3332 .get_host_param = beiscsi_get_host_param,
3333 .start_conn = beiscsi_conn_start,
3334 .stop_conn = beiscsi_conn_stop,
3335 .send_pdu = iscsi_conn_send_pdu,
3336 .xmit_task = beiscsi_task_xmit,
3337 .cleanup_task = beiscsi_cleanup_task,
3338 .alloc_pdu = beiscsi_alloc_pdu,
3339 .parse_pdu_itt = beiscsi_parse_pdu,
3340 .get_stats = beiscsi_conn_get_stats,
3341 .ep_connect = beiscsi_ep_connect,
3342 .ep_poll = beiscsi_ep_poll,
3343 .ep_disconnect = beiscsi_ep_disconnect,
3344 .session_recovery_timedout = iscsi_session_recovery_timedout,
3345};
3346
3347static struct pci_driver beiscsi_pci_driver = {
3348 .name = DRV_NAME,
3349 .probe = beiscsi_dev_probe,
3350 .remove = beiscsi_remove,
3351 .id_table = beiscsi_pci_id_table
3352};
3353
3354static int __init beiscsi_module_init(void)
3355{
3356 int ret;
3357
3358 beiscsi_scsi_transport =
3359 iscsi_register_transport(&beiscsi_iscsi_transport);
3360 if (!beiscsi_scsi_transport) {
3361 SE_DEBUG(DBG_LVL_1,
3362 "beiscsi_module_init - Unable to register beiscsi"
3363 "transport.\n");
3364 ret = -ENOMEM;
3365 }
3366 SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3367 &beiscsi_iscsi_transport);
3368
3369 ret = pci_register_driver(&beiscsi_pci_driver);
3370 if (ret) {
3371 SE_DEBUG(DBG_LVL_1,
3372 "beiscsi_module_init - Unable to register"
3373 "beiscsi pci driver.\n");
3374 goto unregister_iscsi_transport;
3375 }
3376 return 0;
3377
3378unregister_iscsi_transport:
3379 iscsi_unregister_transport(&beiscsi_iscsi_transport);
3380 return ret;
3381}
3382
3383static void __exit beiscsi_module_exit(void)
3384{
3385 pci_unregister_driver(&beiscsi_pci_driver);
3386 iscsi_unregister_transport(&beiscsi_iscsi_transport);
3387}
3388
3389module_init(beiscsi_module_init);
3390module_exit(beiscsi_module_exit);
3391
