1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <asm-generic/kmap_types.h>
34#include <linux/module.h>
35#include <linux/init.h>
36#include <linux/errno.h>
37#include <linux/pci.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
40#include <linux/io-mapping.h>
41#include <linux/mlx5/driver.h>
42#include <linux/mlx5/cq.h>
43#include <linux/mlx5/qp.h>
44#include <linux/mlx5/srq.h>
45#include <linux/debugfs.h>
46#include "mlx5_core.h"
47
48#define DRIVER_NAME "mlx5_core"
49#define DRIVER_VERSION "2.2-1"
50#define DRIVER_RELDATE "Feb 2014"
51
52MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
53MODULE_DESCRIPTION("Mellanox ConnectX-IB HCA core library");
54MODULE_LICENSE("Dual BSD/GPL");
55MODULE_VERSION(DRIVER_VERSION);
56
57int mlx5_core_debug_mask;
58module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644);
59MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0");
60
61struct workqueue_struct *mlx5_core_wq;
62
63static int set_dma_caps(struct pci_dev *pdev)
64{
65 int err;
66
67 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
68 if (err) {
69 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
70 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
71 if (err) {
72 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
73 return err;
74 }
75 }
76
77 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
78 if (err) {
79 dev_warn(&pdev->dev,
80 "Warning: couldn't set 64-bit consistent PCI DMA mask.\n");
81 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
82 if (err) {
83 dev_err(&pdev->dev,
84 "Can't set consistent PCI DMA mask, aborting.\n");
85 return err;
86 }
87 }
88
89 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
90 return err;
91}
92
93static int request_bar(struct pci_dev *pdev)
94{
95 int err = 0;
96
97 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
98 dev_err(&pdev->dev, "Missing registers BAR, aborting.\n");
99 return -ENODEV;
100 }
101
102 err = pci_request_regions(pdev, DRIVER_NAME);
103 if (err)
104 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
105
106 return err;
107}
108
109static void release_bar(struct pci_dev *pdev)
110{
111 pci_release_regions(pdev);
112}
113
114static int mlx5_enable_msix(struct mlx5_core_dev *dev)
115{
116 struct mlx5_eq_table *table = &dev->priv.eq_table;
117 int num_eqs = 1 << dev->caps.log_max_eq;
118 int nvec;
119 int err;
120 int i;
121
122 nvec = dev->caps.num_ports * num_online_cpus() + MLX5_EQ_VEC_COMP_BASE;
123 nvec = min_t(int, nvec, num_eqs);
124 if (nvec <= MLX5_EQ_VEC_COMP_BASE)
125 return -ENOMEM;
126
127 table->msix_arr = kzalloc(nvec * sizeof(*table->msix_arr), GFP_KERNEL);
128 if (!table->msix_arr)
129 return -ENOMEM;
130
131 for (i = 0; i < nvec; i++)
132 table->msix_arr[i].entry = i;
133
134retry:
135 table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE;
136 err = pci_enable_msix(dev->pdev, table->msix_arr, nvec);
137 if (err <= 0) {
138 return err;
139 } else if (err > 2) {
140 nvec = err;
141 goto retry;
142 }
143
144 mlx5_core_dbg(dev, "received %d MSI vectors out of %d requested\n", err, nvec);
145
146 return 0;
147}
148
149static void mlx5_disable_msix(struct mlx5_core_dev *dev)
150{
151 struct mlx5_eq_table *table = &dev->priv.eq_table;
152
153 pci_disable_msix(dev->pdev);
154 kfree(table->msix_arr);
155}
156
157struct mlx5_reg_host_endianess {
158 u8 he;
159 u8 rsvd[15];
160};
161
162
163#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos))
164
165enum {
166 MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) |
167 CAP_MASK(MLX5_CAP_OFF_DCT, 1),
168};
169
170
171
172static void copy_rw_fields(struct mlx5_hca_cap *to, struct mlx5_hca_cap *from)
173{
174 u64 v64;
175
176 to->log_max_qp = from->log_max_qp & 0x1f;
177 to->log_max_ra_req_dc = from->log_max_ra_req_dc & 0x3f;
178 to->log_max_ra_res_dc = from->log_max_ra_res_dc & 0x3f;
179 to->log_max_ra_req_qp = from->log_max_ra_req_qp & 0x3f;
180 to->log_max_ra_res_qp = from->log_max_ra_res_qp & 0x3f;
181 to->log_max_atomic_size_qp = from->log_max_atomic_size_qp;
182 to->log_max_atomic_size_dc = from->log_max_atomic_size_dc;
183 v64 = be64_to_cpu(from->flags) & MLX5_CAP_BITS_RW_MASK;
184 to->flags = cpu_to_be64(v64);
185}
186
187enum {
188 HCA_CAP_OPMOD_GET_MAX = 0,
189 HCA_CAP_OPMOD_GET_CUR = 1,
190};
191
192static int handle_hca_cap(struct mlx5_core_dev *dev)
193{
194 struct mlx5_cmd_query_hca_cap_mbox_out *query_out = NULL;
195 struct mlx5_cmd_set_hca_cap_mbox_in *set_ctx = NULL;
196 struct mlx5_cmd_query_hca_cap_mbox_in query_ctx;
197 struct mlx5_cmd_set_hca_cap_mbox_out set_out;
198 u64 flags;
199 int err;
200
201 memset(&query_ctx, 0, sizeof(query_ctx));
202 query_out = kzalloc(sizeof(*query_out), GFP_KERNEL);
203 if (!query_out)
204 return -ENOMEM;
205
206 set_ctx = kzalloc(sizeof(*set_ctx), GFP_KERNEL);
207 if (!set_ctx) {
208 err = -ENOMEM;
209 goto query_ex;
210 }
211
212 query_ctx.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_HCA_CAP);
213 query_ctx.hdr.opmod = cpu_to_be16(HCA_CAP_OPMOD_GET_CUR);
214 err = mlx5_cmd_exec(dev, &query_ctx, sizeof(query_ctx),
215 query_out, sizeof(*query_out));
216 if (err)
217 goto query_ex;
218
219 err = mlx5_cmd_status_to_err(&query_out->hdr);
220 if (err) {
221 mlx5_core_warn(dev, "query hca cap failed, %d\n", err);
222 goto query_ex;
223 }
224
225 copy_rw_fields(&set_ctx->hca_cap, &query_out->hca_cap);
226
227 if (dev->profile->mask & MLX5_PROF_MASK_QP_SIZE)
228 set_ctx->hca_cap.log_max_qp = dev->profile->log_max_qp;
229
230 flags = be64_to_cpu(query_out->hca_cap.flags);
231
232 flags &= ~MLX5_DEV_CAP_FLAG_CMDIF_CSUM;
233
234 set_ctx->hca_cap.flags = cpu_to_be64(flags);
235 memset(&set_out, 0, sizeof(set_out));
236 set_ctx->hca_cap.log_uar_page_sz = cpu_to_be16(PAGE_SHIFT - 12);
237 set_ctx->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_SET_HCA_CAP);
238 err = mlx5_cmd_exec(dev, set_ctx, sizeof(*set_ctx),
239 &set_out, sizeof(set_out));
240 if (err) {
241 mlx5_core_warn(dev, "set hca cap failed, %d\n", err);
242 goto query_ex;
243 }
244
245 err = mlx5_cmd_status_to_err(&set_out.hdr);
246 if (err)
247 goto query_ex;
248
249query_ex:
250 kfree(query_out);
251 kfree(set_ctx);
252
253 return err;
254}
255
256static int set_hca_ctrl(struct mlx5_core_dev *dev)
257{
258 struct mlx5_reg_host_endianess he_in;
259 struct mlx5_reg_host_endianess he_out;
260 int err;
261
262 memset(&he_in, 0, sizeof(he_in));
263 he_in.he = MLX5_SET_HOST_ENDIANNESS;
264 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in),
265 &he_out, sizeof(he_out),
266 MLX5_REG_HOST_ENDIANNESS, 0, 1);
267 return err;
268}
269
270static int mlx5_core_enable_hca(struct mlx5_core_dev *dev)
271{
272 int err;
273 struct mlx5_enable_hca_mbox_in in;
274 struct mlx5_enable_hca_mbox_out out;
275
276 memset(&in, 0, sizeof(in));
277 memset(&out, 0, sizeof(out));
278 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ENABLE_HCA);
279 err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
280 if (err)
281 return err;
282
283 if (out.hdr.status)
284 return mlx5_cmd_status_to_err(&out.hdr);
285
286 return 0;
287}
288
289static int mlx5_core_disable_hca(struct mlx5_core_dev *dev)
290{
291 int err;
292 struct mlx5_disable_hca_mbox_in in;
293 struct mlx5_disable_hca_mbox_out out;
294
295 memset(&in, 0, sizeof(in));
296 memset(&out, 0, sizeof(out));
297 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DISABLE_HCA);
298 err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
299 if (err)
300 return err;
301
302 if (out.hdr.status)
303 return mlx5_cmd_status_to_err(&out.hdr);
304
305 return 0;
306}
307
308int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev)
309{
310 struct mlx5_priv *priv = &dev->priv;
311 int err;
312
313 dev->pdev = pdev;
314 pci_set_drvdata(dev->pdev, dev);
315 strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN);
316 priv->name[MLX5_MAX_NAME_LEN - 1] = 0;
317
318 mutex_init(&priv->pgdir_mutex);
319 INIT_LIST_HEAD(&priv->pgdir_list);
320 spin_lock_init(&priv->mkey_lock);
321
322 priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root);
323 if (!priv->dbg_root)
324 return -ENOMEM;
325
326 err = pci_enable_device(pdev);
327 if (err) {
328 dev_err(&pdev->dev, "Cannot enable PCI device, aborting.\n");
329 goto err_dbg;
330 }
331
332 err = request_bar(pdev);
333 if (err) {
334 dev_err(&pdev->dev, "error requesting BARs, aborting.\n");
335 goto err_disable;
336 }
337
338 pci_set_master(pdev);
339
340 err = set_dma_caps(pdev);
341 if (err) {
342 dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n");
343 goto err_clr_master;
344 }
345
346 dev->iseg_base = pci_resource_start(dev->pdev, 0);
347 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg));
348 if (!dev->iseg) {
349 err = -ENOMEM;
350 dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n");
351 goto err_clr_master;
352 }
353 dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev),
354 fw_rev_min(dev), fw_rev_sub(dev));
355
356 err = mlx5_cmd_init(dev);
357 if (err) {
358 dev_err(&pdev->dev, "Failed initializing command interface, aborting\n");
359 goto err_unmap;
360 }
361
362 mlx5_pagealloc_init(dev);
363
364 err = mlx5_core_enable_hca(dev);
365 if (err) {
366 dev_err(&pdev->dev, "enable hca failed\n");
367 goto err_pagealloc_cleanup;
368 }
369
370 err = mlx5_satisfy_startup_pages(dev, 1);
371 if (err) {
372 dev_err(&pdev->dev, "failed to allocate boot pages\n");
373 goto err_disable_hca;
374 }
375
376 err = set_hca_ctrl(dev);
377 if (err) {
378 dev_err(&pdev->dev, "set_hca_ctrl failed\n");
379 goto reclaim_boot_pages;
380 }
381
382 err = handle_hca_cap(dev);
383 if (err) {
384 dev_err(&pdev->dev, "handle_hca_cap failed\n");
385 goto reclaim_boot_pages;
386 }
387
388 err = mlx5_satisfy_startup_pages(dev, 0);
389 if (err) {
390 dev_err(&pdev->dev, "failed to allocate init pages\n");
391 goto reclaim_boot_pages;
392 }
393
394 err = mlx5_pagealloc_start(dev);
395 if (err) {
396 dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n");
397 goto reclaim_boot_pages;
398 }
399
400 err = mlx5_cmd_init_hca(dev);
401 if (err) {
402 dev_err(&pdev->dev, "init hca failed\n");
403 goto err_pagealloc_stop;
404 }
405
406 mlx5_start_health_poll(dev);
407
408 err = mlx5_cmd_query_hca_cap(dev, &dev->caps);
409 if (err) {
410 dev_err(&pdev->dev, "query hca failed\n");
411 goto err_stop_poll;
412 }
413
414 err = mlx5_cmd_query_adapter(dev);
415 if (err) {
416 dev_err(&pdev->dev, "query adapter failed\n");
417 goto err_stop_poll;
418 }
419
420 err = mlx5_enable_msix(dev);
421 if (err) {
422 dev_err(&pdev->dev, "enable msix failed\n");
423 goto err_stop_poll;
424 }
425
426 err = mlx5_eq_init(dev);
427 if (err) {
428 dev_err(&pdev->dev, "failed to initialize eq\n");
429 goto disable_msix;
430 }
431
432 err = mlx5_alloc_uuars(dev, &priv->uuari);
433 if (err) {
434 dev_err(&pdev->dev, "Failed allocating uar, aborting\n");
435 goto err_eq_cleanup;
436 }
437
438 err = mlx5_start_eqs(dev);
439 if (err) {
440 dev_err(&pdev->dev, "Failed to start pages and async EQs\n");
441 goto err_free_uar;
442 }
443
444 MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
445
446 mlx5_init_cq_table(dev);
447 mlx5_init_qp_table(dev);
448 mlx5_init_srq_table(dev);
449
450 return 0;
451
452err_free_uar:
453 mlx5_free_uuars(dev, &priv->uuari);
454
455err_eq_cleanup:
456 mlx5_eq_cleanup(dev);
457
458disable_msix:
459 mlx5_disable_msix(dev);
460
461err_stop_poll:
462 mlx5_stop_health_poll(dev);
463 if (mlx5_cmd_teardown_hca(dev)) {
464 dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n");
465 return err;
466 }
467
468err_pagealloc_stop:
469 mlx5_pagealloc_stop(dev);
470
471reclaim_boot_pages:
472 mlx5_reclaim_startup_pages(dev);
473
474err_disable_hca:
475 mlx5_core_disable_hca(dev);
476
477err_pagealloc_cleanup:
478 mlx5_pagealloc_cleanup(dev);
479 mlx5_cmd_cleanup(dev);
480
481err_unmap:
482 iounmap(dev->iseg);
483
484err_clr_master:
485 pci_clear_master(dev->pdev);
486 release_bar(dev->pdev);
487
488err_disable:
489 pci_disable_device(dev->pdev);
490
491err_dbg:
492 debugfs_remove(priv->dbg_root);
493 return err;
494}
495EXPORT_SYMBOL(mlx5_dev_init);
496
497void mlx5_dev_cleanup(struct mlx5_core_dev *dev)
498{
499 struct mlx5_priv *priv = &dev->priv;
500
501 mlx5_cleanup_srq_table(dev);
502 mlx5_cleanup_qp_table(dev);
503 mlx5_cleanup_cq_table(dev);
504 mlx5_stop_eqs(dev);
505 mlx5_free_uuars(dev, &priv->uuari);
506 mlx5_eq_cleanup(dev);
507 mlx5_disable_msix(dev);
508 mlx5_stop_health_poll(dev);
509 if (mlx5_cmd_teardown_hca(dev)) {
510 dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n");
511 return;
512 }
513 mlx5_pagealloc_stop(dev);
514 mlx5_reclaim_startup_pages(dev);
515 mlx5_core_disable_hca(dev);
516 mlx5_pagealloc_cleanup(dev);
517 mlx5_cmd_cleanup(dev);
518 iounmap(dev->iseg);
519 pci_clear_master(dev->pdev);
520 release_bar(dev->pdev);
521 pci_disable_device(dev->pdev);
522 debugfs_remove(priv->dbg_root);
523}
524EXPORT_SYMBOL(mlx5_dev_cleanup);
525
526static int __init init(void)
527{
528 int err;
529
530 mlx5_register_debugfs();
531 mlx5_core_wq = create_singlethread_workqueue("mlx5_core_wq");
532 if (!mlx5_core_wq) {
533 err = -ENOMEM;
534 goto err_debug;
535 }
536 mlx5_health_init();
537
538 return 0;
539
540 mlx5_health_cleanup();
541err_debug:
542 mlx5_unregister_debugfs();
543 return err;
544}
545
546static void __exit cleanup(void)
547{
548 mlx5_health_cleanup();
549 destroy_workqueue(mlx5_core_wq);
550 mlx5_unregister_debugfs();
551}
552
553module_init(init);
554module_exit(cleanup);
555
