1
2
3
4
5
6
7#include <linux/kernel.h>
8#include <linux/types.h>
9#include <linux/pci.h>
10#include <linux/device.h>
11#include <linux/errno.h>
12#include <linux/slab.h>
13#include <linux/semaphore.h>
14#include <linux/dma-mapping.h>
15#include <linux/io.h>
16#include <linux/err.h>
17
18#include "hinic_hw_if.h"
19#include "hinic_hw_eqs.h"
20#include "hinic_hw_wqe.h"
21#include "hinic_hw_wq.h"
22#include "hinic_hw_cmdq.h"
23#include "hinic_hw_qp_ctxt.h"
24#include "hinic_hw_qp.h"
25#include "hinic_hw_io.h"
26
27#define CI_Q_ADDR_SIZE sizeof(u32)
28
29#define CI_ADDR(base_addr, q_id) ((base_addr) + \
30 (q_id) * CI_Q_ADDR_SIZE)
31
32#define CI_TABLE_SIZE(num_qps) ((num_qps) * CI_Q_ADDR_SIZE)
33
34#define DB_IDX(db, db_base) \
35 (((unsigned long)(db) - (unsigned long)(db_base)) / HINIC_DB_PAGE_SIZE)
36
37enum io_cmd {
38 IO_CMD_MODIFY_QUEUE_CTXT = 0,
39 IO_CMD_CLEAN_QUEUE_CTXT,
40};
41
42static void init_db_area_idx(struct hinic_free_db_area *free_db_area)
43{
44 int i;
45
46 for (i = 0; i < HINIC_DB_MAX_AREAS; i++)
47 free_db_area->db_idx[i] = i;
48
49 free_db_area->alloc_pos = 0;
50 free_db_area->return_pos = HINIC_DB_MAX_AREAS;
51
52 free_db_area->num_free = HINIC_DB_MAX_AREAS;
53
54 sema_init(&free_db_area->idx_lock, 1);
55}
56
57static void __iomem *get_db_area(struct hinic_func_to_io *func_to_io)
58{
59 struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
60 int pos, idx;
61
62 down(&free_db_area->idx_lock);
63
64 free_db_area->num_free--;
65
66 if (free_db_area->num_free < 0) {
67 free_db_area->num_free++;
68 up(&free_db_area->idx_lock);
69 return ERR_PTR(-ENOMEM);
70 }
71
72 pos = free_db_area->alloc_pos++;
73 pos &= HINIC_DB_MAX_AREAS - 1;
74
75 idx = free_db_area->db_idx[pos];
76
77 free_db_area->db_idx[pos] = -1;
78
79 up(&free_db_area->idx_lock);
80
81 return func_to_io->db_base + idx * HINIC_DB_PAGE_SIZE;
82}
83
84static void return_db_area(struct hinic_func_to_io *func_to_io,
85 void __iomem *db_base)
86{
87 struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
88 int pos, idx = DB_IDX(db_base, func_to_io->db_base);
89
90 down(&free_db_area->idx_lock);
91
92 pos = free_db_area->return_pos++;
93 pos &= HINIC_DB_MAX_AREAS - 1;
94
95 free_db_area->db_idx[pos] = idx;
96
97 free_db_area->num_free++;
98
99 up(&free_db_area->idx_lock);
100}
101
102static int write_sq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
103 u16 num_sqs)
104{
105 struct hinic_hwif *hwif = func_to_io->hwif;
106 struct hinic_sq_ctxt_block *sq_ctxt_block;
107 struct pci_dev *pdev = hwif->pdev;
108 struct hinic_cmdq_buf cmdq_buf;
109 struct hinic_sq_ctxt *sq_ctxt;
110 struct hinic_qp *qp;
111 u64 out_param;
112 int err, i;
113
114 err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
115 if (err) {
116 dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
117 return err;
118 }
119
120 sq_ctxt_block = cmdq_buf.buf;
121 sq_ctxt = sq_ctxt_block->sq_ctxt;
122
123 hinic_qp_prepare_header(&sq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_SQ,
124 num_sqs, func_to_io->max_qps);
125 for (i = 0; i < num_sqs; i++) {
126 qp = &func_to_io->qps[i];
127
128 hinic_sq_prepare_ctxt(&sq_ctxt[i], &qp->sq,
129 base_qpn + qp->q_id);
130 }
131
132 cmdq_buf.size = HINIC_SQ_CTXT_SIZE(num_sqs);
133
134 err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
135 IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
136 &out_param);
137 if ((err) || (out_param != 0)) {
138 dev_err(&pdev->dev, "Failed to set SQ ctxts\n");
139 err = -EFAULT;
140 }
141
142 hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
143 return err;
144}
145
146static int write_rq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
147 u16 num_rqs)
148{
149 struct hinic_hwif *hwif = func_to_io->hwif;
150 struct hinic_rq_ctxt_block *rq_ctxt_block;
151 struct pci_dev *pdev = hwif->pdev;
152 struct hinic_cmdq_buf cmdq_buf;
153 struct hinic_rq_ctxt *rq_ctxt;
154 struct hinic_qp *qp;
155 u64 out_param;
156 int err, i;
157
158 err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
159 if (err) {
160 dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
161 return err;
162 }
163
164 rq_ctxt_block = cmdq_buf.buf;
165 rq_ctxt = rq_ctxt_block->rq_ctxt;
166
167 hinic_qp_prepare_header(&rq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_RQ,
168 num_rqs, func_to_io->max_qps);
169 for (i = 0; i < num_rqs; i++) {
170 qp = &func_to_io->qps[i];
171
172 hinic_rq_prepare_ctxt(&rq_ctxt[i], &qp->rq,
173 base_qpn + qp->q_id);
174 }
175
176 cmdq_buf.size = HINIC_RQ_CTXT_SIZE(num_rqs);
177
178 err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
179 IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
180 &out_param);
181 if ((err) || (out_param != 0)) {
182 dev_err(&pdev->dev, "Failed to set RQ ctxts\n");
183 err = -EFAULT;
184 }
185
186 hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
187 return err;
188}
189
190
191
192
193
194
195
196
197
198static int write_qp_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
199 u16 num_qps)
200{
201 return (write_sq_ctxts(func_to_io, base_qpn, num_qps) ||
202 write_rq_ctxts(func_to_io, base_qpn, num_qps));
203}
204
205static int hinic_clean_queue_offload_ctxt(struct hinic_func_to_io *func_to_io,
206 enum hinic_qp_ctxt_type ctxt_type)
207{
208 struct hinic_hwif *hwif = func_to_io->hwif;
209 struct hinic_clean_queue_ctxt *ctxt_block;
210 struct pci_dev *pdev = hwif->pdev;
211 struct hinic_cmdq_buf cmdq_buf;
212 u64 out_param = 0;
213 int err;
214
215 err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
216 if (err) {
217 dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
218 return err;
219 }
220
221 ctxt_block = cmdq_buf.buf;
222 ctxt_block->cmdq_hdr.num_queues = func_to_io->max_qps;
223 ctxt_block->cmdq_hdr.queue_type = ctxt_type;
224 ctxt_block->cmdq_hdr.addr_offset = 0;
225
226
227 ctxt_block->ctxt_size = 0x3;
228
229 hinic_cpu_to_be32(ctxt_block, sizeof(*ctxt_block));
230
231 cmdq_buf.size = sizeof(*ctxt_block);
232
233 err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
234 IO_CMD_CLEAN_QUEUE_CTXT,
235 &cmdq_buf, &out_param);
236
237 if (err || out_param) {
238 dev_err(&pdev->dev, "Failed to clean offload ctxts, err: %d, out_param: 0x%llx\n",
239 err, out_param);
240
241 err = -EFAULT;
242 }
243
244 hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
245
246 return err;
247}
248
249static int hinic_clean_qp_offload_ctxt(struct hinic_func_to_io *func_to_io)
250{
251
252 return (hinic_clean_queue_offload_ctxt(func_to_io,
253 HINIC_QP_CTXT_TYPE_SQ) ||
254 hinic_clean_queue_offload_ctxt(func_to_io,
255 HINIC_QP_CTXT_TYPE_RQ));
256}
257
258
259
260
261
262
263
264
265
266
267
268static int init_qp(struct hinic_func_to_io *func_to_io,
269 struct hinic_qp *qp, int q_id,
270 struct msix_entry *sq_msix_entry,
271 struct msix_entry *rq_msix_entry)
272{
273 struct hinic_hwif *hwif = func_to_io->hwif;
274 struct pci_dev *pdev = hwif->pdev;
275 void __iomem *db_base;
276 int err;
277
278 qp->q_id = q_id;
279
280 err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->sq_wq[q_id],
281 HINIC_SQ_WQEBB_SIZE, HINIC_SQ_PAGE_SIZE,
282 HINIC_SQ_DEPTH, HINIC_SQ_WQE_MAX_SIZE);
283 if (err) {
284 dev_err(&pdev->dev, "Failed to allocate WQ for SQ\n");
285 return err;
286 }
287
288 err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->rq_wq[q_id],
289 HINIC_RQ_WQEBB_SIZE, HINIC_RQ_PAGE_SIZE,
290 HINIC_RQ_DEPTH, HINIC_RQ_WQE_SIZE);
291 if (err) {
292 dev_err(&pdev->dev, "Failed to allocate WQ for RQ\n");
293 goto err_rq_alloc;
294 }
295
296 db_base = get_db_area(func_to_io);
297 if (IS_ERR(db_base)) {
298 dev_err(&pdev->dev, "Failed to get DB area for SQ\n");
299 err = PTR_ERR(db_base);
300 goto err_get_db;
301 }
302
303 func_to_io->sq_db[q_id] = db_base;
304
305 err = hinic_init_sq(&qp->sq, hwif, &func_to_io->sq_wq[q_id],
306 sq_msix_entry,
307 CI_ADDR(func_to_io->ci_addr_base, q_id),
308 CI_ADDR(func_to_io->ci_dma_base, q_id), db_base);
309 if (err) {
310 dev_err(&pdev->dev, "Failed to init SQ\n");
311 goto err_sq_init;
312 }
313
314 err = hinic_init_rq(&qp->rq, hwif, &func_to_io->rq_wq[q_id],
315 rq_msix_entry);
316 if (err) {
317 dev_err(&pdev->dev, "Failed to init RQ\n");
318 goto err_rq_init;
319 }
320
321 return 0;
322
323err_rq_init:
324 hinic_clean_sq(&qp->sq);
325
326err_sq_init:
327 return_db_area(func_to_io, db_base);
328
329err_get_db:
330 hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
331
332err_rq_alloc:
333 hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
334 return err;
335}
336
337
338
339
340
341
342static void destroy_qp(struct hinic_func_to_io *func_to_io,
343 struct hinic_qp *qp)
344{
345 int q_id = qp->q_id;
346
347 hinic_clean_rq(&qp->rq);
348 hinic_clean_sq(&qp->sq);
349
350 return_db_area(func_to_io, func_to_io->sq_db[q_id]);
351
352 hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
353 hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
354}
355
356
357
358
359
360
361
362
363
364
365
366int hinic_io_create_qps(struct hinic_func_to_io *func_to_io,
367 u16 base_qpn, int num_qps,
368 struct msix_entry *sq_msix_entries,
369 struct msix_entry *rq_msix_entries)
370{
371 struct hinic_hwif *hwif = func_to_io->hwif;
372 struct pci_dev *pdev = hwif->pdev;
373 size_t qps_size, wq_size, db_size;
374 void *ci_addr_base;
375 int i, j, err;
376
377 qps_size = num_qps * sizeof(*func_to_io->qps);
378 func_to_io->qps = devm_kzalloc(&pdev->dev, qps_size, GFP_KERNEL);
379 if (!func_to_io->qps)
380 return -ENOMEM;
381
382 wq_size = num_qps * sizeof(*func_to_io->sq_wq);
383 func_to_io->sq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
384 if (!func_to_io->sq_wq) {
385 err = -ENOMEM;
386 goto err_sq_wq;
387 }
388
389 wq_size = num_qps * sizeof(*func_to_io->rq_wq);
390 func_to_io->rq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
391 if (!func_to_io->rq_wq) {
392 err = -ENOMEM;
393 goto err_rq_wq;
394 }
395
396 db_size = num_qps * sizeof(*func_to_io->sq_db);
397 func_to_io->sq_db = devm_kzalloc(&pdev->dev, db_size, GFP_KERNEL);
398 if (!func_to_io->sq_db) {
399 err = -ENOMEM;
400 goto err_sq_db;
401 }
402
403 ci_addr_base = dma_alloc_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
404 &func_to_io->ci_dma_base,
405 GFP_KERNEL);
406 if (!ci_addr_base) {
407 dev_err(&pdev->dev, "Failed to allocate CI area\n");
408 err = -ENOMEM;
409 goto err_ci_base;
410 }
411
412 func_to_io->ci_addr_base = ci_addr_base;
413
414 for (i = 0; i < num_qps; i++) {
415 err = init_qp(func_to_io, &func_to_io->qps[i], i,
416 &sq_msix_entries[i], &rq_msix_entries[i]);
417 if (err) {
418 dev_err(&pdev->dev, "Failed to create QP %d\n", i);
419 goto err_init_qp;
420 }
421 }
422
423 err = write_qp_ctxts(func_to_io, base_qpn, num_qps);
424 if (err) {
425 dev_err(&pdev->dev, "Failed to init QP ctxts\n");
426 goto err_write_qp_ctxts;
427 }
428
429 err = hinic_clean_qp_offload_ctxt(func_to_io);
430 if (err) {
431 dev_err(&pdev->dev, "Failed to clean QP contexts space\n");
432 goto err_write_qp_ctxts;
433 }
434
435 return 0;
436
437err_write_qp_ctxts:
438err_init_qp:
439 for (j = 0; j < i; j++)
440 destroy_qp(func_to_io, &func_to_io->qps[j]);
441
442 dma_free_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
443 func_to_io->ci_addr_base, func_to_io->ci_dma_base);
444
445err_ci_base:
446 devm_kfree(&pdev->dev, func_to_io->sq_db);
447
448err_sq_db:
449 devm_kfree(&pdev->dev, func_to_io->rq_wq);
450
451err_rq_wq:
452 devm_kfree(&pdev->dev, func_to_io->sq_wq);
453
454err_sq_wq:
455 devm_kfree(&pdev->dev, func_to_io->qps);
456 return err;
457}
458
459
460
461
462
463
464void hinic_io_destroy_qps(struct hinic_func_to_io *func_to_io, int num_qps)
465{
466 struct hinic_hwif *hwif = func_to_io->hwif;
467 struct pci_dev *pdev = hwif->pdev;
468 size_t ci_table_size;
469 int i;
470
471 ci_table_size = CI_TABLE_SIZE(num_qps);
472
473 for (i = 0; i < num_qps; i++)
474 destroy_qp(func_to_io, &func_to_io->qps[i]);
475
476 dma_free_coherent(&pdev->dev, ci_table_size, func_to_io->ci_addr_base,
477 func_to_io->ci_dma_base);
478
479 devm_kfree(&pdev->dev, func_to_io->sq_db);
480
481 devm_kfree(&pdev->dev, func_to_io->rq_wq);
482 devm_kfree(&pdev->dev, func_to_io->sq_wq);
483
484 devm_kfree(&pdev->dev, func_to_io->qps);
485}
486
487
488
489
490
491
492
493
494
495
496
497int hinic_io_init(struct hinic_func_to_io *func_to_io,
498 struct hinic_hwif *hwif, u16 max_qps, int num_ceqs,
499 struct msix_entry *ceq_msix_entries)
500{
501 struct pci_dev *pdev = hwif->pdev;
502 enum hinic_cmdq_type cmdq, type;
503 void __iomem *db_area;
504 int err;
505
506 func_to_io->hwif = hwif;
507 func_to_io->qps = NULL;
508 func_to_io->max_qps = max_qps;
509
510 err = hinic_ceqs_init(&func_to_io->ceqs, hwif, num_ceqs,
511 HINIC_DEFAULT_CEQ_LEN, HINIC_EQ_PAGE_SIZE,
512 ceq_msix_entries);
513 if (err) {
514 dev_err(&pdev->dev, "Failed to init CEQs\n");
515 return err;
516 }
517
518 err = hinic_wqs_alloc(&func_to_io->wqs, 2 * max_qps, hwif);
519 if (err) {
520 dev_err(&pdev->dev, "Failed to allocate WQS for IO\n");
521 goto err_wqs_alloc;
522 }
523
524 func_to_io->db_base = pci_ioremap_bar(pdev, HINIC_PCI_DB_BAR);
525 if (!func_to_io->db_base) {
526 dev_err(&pdev->dev, "Failed to remap IO DB area\n");
527 err = -ENOMEM;
528 goto err_db_ioremap;
529 }
530
531 init_db_area_idx(&func_to_io->free_db_area);
532
533 for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++) {
534 db_area = get_db_area(func_to_io);
535 if (IS_ERR(db_area)) {
536 dev_err(&pdev->dev, "Failed to get cmdq db area\n");
537 err = PTR_ERR(db_area);
538 goto err_db_area;
539 }
540
541 func_to_io->cmdq_db_area[cmdq] = db_area;
542 }
543
544 err = hinic_init_cmdqs(&func_to_io->cmdqs, hwif,
545 func_to_io->cmdq_db_area);
546 if (err) {
547 dev_err(&pdev->dev, "Failed to initialize cmdqs\n");
548 goto err_init_cmdqs;
549 }
550
551 return 0;
552
553err_init_cmdqs:
554err_db_area:
555 for (type = HINIC_CMDQ_SYNC; type < cmdq; type++)
556 return_db_area(func_to_io, func_to_io->cmdq_db_area[type]);
557
558 iounmap(func_to_io->db_base);
559
560err_db_ioremap:
561 hinic_wqs_free(&func_to_io->wqs);
562
563err_wqs_alloc:
564 hinic_ceqs_free(&func_to_io->ceqs);
565 return err;
566}
567
568
569
570
571
572void hinic_io_free(struct hinic_func_to_io *func_to_io)
573{
574 enum hinic_cmdq_type cmdq;
575
576 hinic_free_cmdqs(&func_to_io->cmdqs);
577
578 for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++)
579 return_db_area(func_to_io, func_to_io->cmdq_db_area[cmdq]);
580
581 iounmap(func_to_io->db_base);
582 hinic_wqs_free(&func_to_io->wqs);
583 hinic_ceqs_free(&func_to_io->ceqs);
584}
585
