1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48#include <linux/delay.h>
49#include "hfi.h"
50#include "qp.h"
51#include "trace.h"
52
53#define SC_CTXT_PACKET_EGRESS_TIMEOUT 350
54
55#define SC(name) SEND_CTXT_##name
56
57
58
59static void sc_wait_for_packet_egress(struct send_context *sc, int pause);
60
61
62
63
64
65void __cm_reset(struct hfi1_devdata *dd, u64 sendctrl)
66{
67 write_csr(dd, SEND_CTRL, sendctrl | SEND_CTRL_CM_RESET_SMASK);
68 while (1) {
69 udelay(1);
70 sendctrl = read_csr(dd, SEND_CTRL);
71 if ((sendctrl & SEND_CTRL_CM_RESET_SMASK) == 0)
72 break;
73 }
74}
75
76
77#ifndef SEND_CTRL_UNSUPPORTED_VL_SHIFT
78#define SEND_CTRL_UNSUPPORTED_VL_SHIFT 3
79#define SEND_CTRL_UNSUPPORTED_VL_MASK 0xffull
80#define SEND_CTRL_UNSUPPORTED_VL_SMASK (SEND_CTRL_UNSUPPORTED_VL_MASK \
81 << SEND_CTRL_UNSUPPORTED_VL_SHIFT)
82#endif
83
84
85void pio_send_control(struct hfi1_devdata *dd, int op)
86{
87 u64 reg, mask;
88 unsigned long flags;
89 int write = 1;
90 int flush = 0;
91
92 spin_lock_irqsave(&dd->sendctrl_lock, flags);
93
94 reg = read_csr(dd, SEND_CTRL);
95 switch (op) {
96 case PSC_GLOBAL_ENABLE:
97 reg |= SEND_CTRL_SEND_ENABLE_SMASK;
98
99 case PSC_DATA_VL_ENABLE:
100
101 mask = (((~0ull) << num_vls) & SEND_CTRL_UNSUPPORTED_VL_MASK) <<
102 SEND_CTRL_UNSUPPORTED_VL_SHIFT;
103 reg = (reg & ~SEND_CTRL_UNSUPPORTED_VL_SMASK) | mask;
104 break;
105 case PSC_GLOBAL_DISABLE:
106 reg &= ~SEND_CTRL_SEND_ENABLE_SMASK;
107 break;
108 case PSC_GLOBAL_VLARB_ENABLE:
109 reg |= SEND_CTRL_VL_ARBITER_ENABLE_SMASK;
110 break;
111 case PSC_GLOBAL_VLARB_DISABLE:
112 reg &= ~SEND_CTRL_VL_ARBITER_ENABLE_SMASK;
113 break;
114 case PSC_CM_RESET:
115 __cm_reset(dd, reg);
116 write = 0;
117 break;
118 case PSC_DATA_VL_DISABLE:
119 reg |= SEND_CTRL_UNSUPPORTED_VL_SMASK;
120 flush = 1;
121 break;
122 default:
123 dd_dev_err(dd, "%s: invalid control %d\n", __func__, op);
124 break;
125 }
126
127 if (write) {
128 write_csr(dd, SEND_CTRL, reg);
129 if (flush)
130 (void)read_csr(dd, SEND_CTRL);
131 }
132
133 spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
134}
135
136
137#define NUM_SC_POOLS 2
138
139
140#define SCS_POOL_0 -1
141#define SCS_POOL_1 -2
142
143#define SCC_PER_VL -1
144#define SCC_PER_CPU -2
145
146#define SCC_PER_KRCVQ -3
147#define SCC_ACK_CREDITS 32
148
149#define PIO_WAIT_BATCH_SIZE 5
150
151
152static struct sc_config_sizes sc_config_sizes[SC_MAX] = {
153 [SC_KERNEL] = { .size = SCS_POOL_0,
154 .count = SCC_PER_VL },
155 [SC_ACK] = { .size = SCC_ACK_CREDITS,
156 .count = SCC_PER_KRCVQ },
157 [SC_USER] = { .size = SCS_POOL_0,
158 .count = SCC_PER_CPU },
159
160};
161
162
163struct mem_pool_config {
164 int centipercent;
165 int absolute_blocks;
166};
167
168
169static struct mem_pool_config sc_mem_pool_config[NUM_SC_POOLS] = {
170
171 { 10000, -1 },
172 { 0, -1 },
173};
174
175
176struct mem_pool_info {
177 int centipercent;
178
179
180
181 int count;
182 int blocks;
183 int size;
184};
185
186
187
188
189
190
191
192
193
194
195static int wildcard_to_pool(int wc)
196{
197 if (wc >= 0)
198 return -1;
199 return -wc - 1;
200}
201
202static const char *sc_type_names[SC_MAX] = {
203 "kernel",
204 "ack",
205 "user"
206};
207
208static const char *sc_type_name(int index)
209{
210 if (index < 0 || index >= SC_MAX)
211 return "unknown";
212 return sc_type_names[index];
213}
214
215
216
217
218
219
220int init_sc_pools_and_sizes(struct hfi1_devdata *dd)
221{
222 struct mem_pool_info mem_pool_info[NUM_SC_POOLS] = { { 0 } };
223 int total_blocks = (dd->chip_pio_mem_size / PIO_BLOCK_SIZE) - 1;
224 int total_contexts = 0;
225 int fixed_blocks;
226 int pool_blocks;
227 int used_blocks;
228 int cp_total;
229 int ab_total;
230 int extra;
231 int i;
232
233
234
235
236
237
238
239
240 cp_total = 0;
241 ab_total = 0;
242 for (i = 0; i < NUM_SC_POOLS; i++) {
243 int cp = sc_mem_pool_config[i].centipercent;
244 int ab = sc_mem_pool_config[i].absolute_blocks;
245
246
247
248
249
250 if (cp >= 0) {
251 cp_total += cp;
252 } else if (ab >= 0) {
253 ab_total += ab;
254 } else {
255 dd_dev_err(
256 dd,
257 "Send context memory pool %d: both the block count and centipercent are invalid\n",
258 i);
259 return -EINVAL;
260 }
261
262 mem_pool_info[i].centipercent = cp;
263 mem_pool_info[i].blocks = ab;
264 }
265
266
267 if (cp_total != 0 && ab_total != 0) {
268 dd_dev_err(
269 dd,
270 "All send context memory pools must be described as either centipercent or blocks, no mixing between pools\n");
271 return -EINVAL;
272 }
273
274
275 if (cp_total != 0 && cp_total != 10000) {
276 dd_dev_err(
277 dd,
278 "Send context memory pool centipercent is %d, expecting 10000\n",
279 cp_total);
280 return -EINVAL;
281 }
282
283
284 if (ab_total > total_blocks) {
285 dd_dev_err(
286 dd,
287 "Send context memory pool absolute block count %d is larger than the memory size %d\n",
288 ab_total, total_blocks);
289 return -EINVAL;
290 }
291
292
293
294
295
296
297
298
299 fixed_blocks = 0;
300 for (i = 0; i < SC_MAX; i++) {
301 int count = sc_config_sizes[i].count;
302 int size = sc_config_sizes[i].size;
303 int pool;
304
305
306
307
308
309
310
311 if (i == SC_ACK) {
312 count = dd->n_krcv_queues;
313 } else if (i == SC_KERNEL) {
314 count = (INIT_SC_PER_VL * num_vls) + 1 ;
315 } else if (count == SCC_PER_CPU) {
316 count = dd->num_rcv_contexts - dd->n_krcv_queues;
317 } else if (count < 0) {
318 dd_dev_err(
319 dd,
320 "%s send context invalid count wildcard %d\n",
321 sc_type_name(i), count);
322 return -EINVAL;
323 }
324 if (total_contexts + count > dd->chip_send_contexts)
325 count = dd->chip_send_contexts - total_contexts;
326
327 total_contexts += count;
328
329
330
331
332
333
334
335 pool = wildcard_to_pool(size);
336 if (pool == -1) {
337 fixed_blocks += size * count;
338 } else if (pool < NUM_SC_POOLS) {
339 mem_pool_info[pool].count += count;
340 } else {
341 dd_dev_err(
342 dd,
343 "%s send context invalid pool wildcard %d\n",
344 sc_type_name(i), size);
345 return -EINVAL;
346 }
347
348 dd->sc_sizes[i].count = count;
349 dd->sc_sizes[i].size = size;
350 }
351 if (fixed_blocks > total_blocks) {
352 dd_dev_err(
353 dd,
354 "Send context fixed block count, %u, larger than total block count %u\n",
355 fixed_blocks, total_blocks);
356 return -EINVAL;
357 }
358
359
360 pool_blocks = total_blocks - fixed_blocks;
361 if (ab_total > pool_blocks) {
362 dd_dev_err(
363 dd,
364 "Send context fixed pool sizes, %u, larger than pool block count %u\n",
365 ab_total, pool_blocks);
366 return -EINVAL;
367 }
368
369 pool_blocks -= ab_total;
370
371 for (i = 0; i < NUM_SC_POOLS; i++) {
372 struct mem_pool_info *pi = &mem_pool_info[i];
373
374
375 if (pi->centipercent >= 0)
376 pi->blocks = (pool_blocks * pi->centipercent) / 10000;
377
378 if (pi->blocks == 0 && pi->count != 0) {
379 dd_dev_err(
380 dd,
381 "Send context memory pool %d has %u contexts, but no blocks\n",
382 i, pi->count);
383 return -EINVAL;
384 }
385 if (pi->count == 0) {
386
387 if (pi->blocks != 0)
388 dd_dev_err(
389 dd,
390 "Send context memory pool %d has %u blocks, but zero contexts\n",
391 i, pi->blocks);
392 pi->size = 0;
393 } else {
394 pi->size = pi->blocks / pi->count;
395 }
396 }
397
398
399 used_blocks = 0;
400 for (i = 0; i < SC_MAX; i++) {
401 if (dd->sc_sizes[i].size < 0) {
402 unsigned pool = wildcard_to_pool(dd->sc_sizes[i].size);
403
404 WARN_ON_ONCE(pool >= NUM_SC_POOLS);
405 dd->sc_sizes[i].size = mem_pool_info[pool].size;
406 }
407
408#define PIO_MAX_BLOCKS 1024
409 if (dd->sc_sizes[i].size > PIO_MAX_BLOCKS)
410 dd->sc_sizes[i].size = PIO_MAX_BLOCKS;
411
412
413 used_blocks += dd->sc_sizes[i].size * dd->sc_sizes[i].count;
414 }
415 extra = total_blocks - used_blocks;
416 if (extra != 0)
417 dd_dev_info(dd, "unused send context blocks: %d\n", extra);
418
419 return total_contexts;
420}
421
422int init_send_contexts(struct hfi1_devdata *dd)
423{
424 u16 base;
425 int ret, i, j, context;
426
427 ret = init_credit_return(dd);
428 if (ret)
429 return ret;
430
431 dd->hw_to_sw = kmalloc_array(TXE_NUM_CONTEXTS, sizeof(u8),
432 GFP_KERNEL);
433 dd->send_contexts = kcalloc(dd->num_send_contexts,
434 sizeof(struct send_context_info),
435 GFP_KERNEL);
436 if (!dd->send_contexts || !dd->hw_to_sw) {
437 kfree(dd->hw_to_sw);
438 kfree(dd->send_contexts);
439 free_credit_return(dd);
440 return -ENOMEM;
441 }
442
443
444 for (i = 0; i < TXE_NUM_CONTEXTS; i++)
445 dd->hw_to_sw[i] = INVALID_SCI;
446
447
448
449
450
451 context = 0;
452 base = 1;
453 for (i = 0; i < SC_MAX; i++) {
454 struct sc_config_sizes *scs = &dd->sc_sizes[i];
455
456 for (j = 0; j < scs->count; j++) {
457 struct send_context_info *sci =
458 &dd->send_contexts[context];
459 sci->type = i;
460 sci->base = base;
461 sci->credits = scs->size;
462
463 context++;
464 base += scs->size;
465 }
466 }
467
468 return 0;
469}
470
471
472
473
474
475
476static int sc_hw_alloc(struct hfi1_devdata *dd, int type, u32 *sw_index,
477 u32 *hw_context)
478{
479 struct send_context_info *sci;
480 u32 index;
481 u32 context;
482
483 for (index = 0, sci = &dd->send_contexts[0];
484 index < dd->num_send_contexts; index++, sci++) {
485 if (sci->type == type && sci->allocated == 0) {
486 sci->allocated = 1;
487
488 context = dd->chip_send_contexts - index - 1;
489 dd->hw_to_sw[context] = index;
490 *sw_index = index;
491 *hw_context = context;
492 return 0;
493 }
494 }
495 dd_dev_err(dd, "Unable to locate a free type %d send context\n", type);
496 return -ENOSPC;
497}
498
499
500
501
502
503
504static void sc_hw_free(struct hfi1_devdata *dd, u32 sw_index, u32 hw_context)
505{
506 struct send_context_info *sci;
507
508 sci = &dd->send_contexts[sw_index];
509 if (!sci->allocated) {
510 dd_dev_err(dd, "%s: sw_index %u not allocated? hw_context %u\n",
511 __func__, sw_index, hw_context);
512 }
513 sci->allocated = 0;
514 dd->hw_to_sw[hw_context] = INVALID_SCI;
515}
516
517
518static inline u32 group_context(u32 context, u32 group)
519{
520 return (context >> group) << group;
521}
522
523
524static inline u32 group_size(u32 group)
525{
526 return 1 << group;
527}
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542static void cr_group_addresses(struct send_context *sc, dma_addr_t *pa)
543{
544 u32 gc = group_context(sc->hw_context, sc->group);
545 u32 index = sc->hw_context & 0x7;
546
547 sc->hw_free = &sc->dd->cr_base[sc->node].va[gc].cr[index];
548 *pa = (unsigned long)
549 &((struct credit_return *)sc->dd->cr_base[sc->node].pa)[gc];
550}
551
552
553
554
555
556static void sc_halted(struct work_struct *work)
557{
558 struct send_context *sc;
559
560 sc = container_of(work, struct send_context, halt_work);
561 sc_restart(sc);
562}
563
564
565
566
567
568
569
570
571
572
573
574u32 sc_mtu_to_threshold(struct send_context *sc, u32 mtu, u32 hdrqentsize)
575{
576 u32 release_credits;
577 u32 threshold;
578
579
580 mtu += hdrqentsize << 2;
581 release_credits = DIV_ROUND_UP(mtu, PIO_BLOCK_SIZE);
582
583
584 if (sc->credits <= release_credits)
585 threshold = 1;
586 else
587 threshold = sc->credits - release_credits;
588
589 return threshold;
590}
591
592
593
594
595
596
597
598
599static u32 sc_percent_to_threshold(struct send_context *sc, u32 percent)
600{
601 return (sc->credits * percent) / 100;
602}
603
604
605
606
607void sc_set_cr_threshold(struct send_context *sc, u32 new_threshold)
608{
609 unsigned long flags;
610 u32 old_threshold;
611 int force_return = 0;
612
613 spin_lock_irqsave(&sc->credit_ctrl_lock, flags);
614
615 old_threshold = (sc->credit_ctrl >>
616 SC(CREDIT_CTRL_THRESHOLD_SHIFT))
617 & SC(CREDIT_CTRL_THRESHOLD_MASK);
618
619 if (new_threshold != old_threshold) {
620 sc->credit_ctrl =
621 (sc->credit_ctrl
622 & ~SC(CREDIT_CTRL_THRESHOLD_SMASK))
623 | ((new_threshold
624 & SC(CREDIT_CTRL_THRESHOLD_MASK))
625 << SC(CREDIT_CTRL_THRESHOLD_SHIFT));
626 write_kctxt_csr(sc->dd, sc->hw_context,
627 SC(CREDIT_CTRL), sc->credit_ctrl);
628
629
630 force_return = 1;
631 }
632
633 spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags);
634
635 if (force_return)
636 sc_return_credits(sc);
637}
638
639
640
641
642
643
644void set_pio_integrity(struct send_context *sc)
645{
646 struct hfi1_devdata *dd = sc->dd;
647 u64 reg = 0;
648 u32 hw_context = sc->hw_context;
649 int type = sc->type;
650
651
652
653
654
655 if (likely(!HFI1_CAP_IS_KSET(NO_INTEGRITY)) &&
656 dd->hfi1_snoop.mode_flag != HFI1_PORT_SNOOP_MODE)
657 reg = hfi1_pkt_default_send_ctxt_mask(dd, type);
658
659 write_kctxt_csr(dd, hw_context, SC(CHECK_ENABLE), reg);
660}
661
662static u32 get_buffers_allocated(struct send_context *sc)
663{
664 int cpu;
665 u32 ret = 0;
666
667 for_each_possible_cpu(cpu)
668 ret += *per_cpu_ptr(sc->buffers_allocated, cpu);
669 return ret;
670}
671
672static void reset_buffers_allocated(struct send_context *sc)
673{
674 int cpu;
675
676 for_each_possible_cpu(cpu)
677 (*per_cpu_ptr(sc->buffers_allocated, cpu)) = 0;
678}
679
680
681
682
683
684struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,
685 uint hdrqentsize, int numa)
686{
687 struct send_context_info *sci;
688 struct send_context *sc = NULL;
689 dma_addr_t pa;
690 unsigned long flags;
691 u64 reg;
692 u32 thresh;
693 u32 sw_index;
694 u32 hw_context;
695 int ret;
696 u8 opval, opmask;
697
698
699 if (dd->flags & HFI1_FROZEN)
700 return NULL;
701
702 sc = kzalloc_node(sizeof(*sc), GFP_KERNEL, numa);
703 if (!sc)
704 return NULL;
705
706 sc->buffers_allocated = alloc_percpu(u32);
707 if (!sc->buffers_allocated) {
708 kfree(sc);
709 dd_dev_err(dd,
710 "Cannot allocate buffers_allocated per cpu counters\n"
711 );
712 return NULL;
713 }
714
715 spin_lock_irqsave(&dd->sc_lock, flags);
716 ret = sc_hw_alloc(dd, type, &sw_index, &hw_context);
717 if (ret) {
718 spin_unlock_irqrestore(&dd->sc_lock, flags);
719 free_percpu(sc->buffers_allocated);
720 kfree(sc);
721 return NULL;
722 }
723
724 sci = &dd->send_contexts[sw_index];
725 sci->sc = sc;
726
727 sc->dd = dd;
728 sc->node = numa;
729 sc->type = type;
730 spin_lock_init(&sc->alloc_lock);
731 spin_lock_init(&sc->release_lock);
732 spin_lock_init(&sc->credit_ctrl_lock);
733 INIT_LIST_HEAD(&sc->piowait);
734 INIT_WORK(&sc->halt_work, sc_halted);
735 init_waitqueue_head(&sc->halt_wait);
736
737
738 sc->group = 0;
739
740 sc->sw_index = sw_index;
741 sc->hw_context = hw_context;
742 cr_group_addresses(sc, &pa);
743 sc->credits = sci->credits;
744
745
746#define PIO_ADDR_CONTEXT_MASK 0xfful
747#define PIO_ADDR_CONTEXT_SHIFT 16
748 sc->base_addr = dd->piobase + ((hw_context & PIO_ADDR_CONTEXT_MASK)
749 << PIO_ADDR_CONTEXT_SHIFT);
750
751
752 reg = ((sci->credits & SC(CTRL_CTXT_DEPTH_MASK))
753 << SC(CTRL_CTXT_DEPTH_SHIFT))
754 | ((sci->base & SC(CTRL_CTXT_BASE_MASK))
755 << SC(CTRL_CTXT_BASE_SHIFT));
756 write_kctxt_csr(dd, hw_context, SC(CTRL), reg);
757
758 set_pio_integrity(sc);
759
760
761 write_kctxt_csr(dd, hw_context, SC(ERR_MASK), (u64)-1);
762
763
764 write_kctxt_csr(dd, hw_context, SC(CHECK_PARTITION_KEY),
765 (SC(CHECK_PARTITION_KEY_VALUE_MASK) &
766 DEFAULT_PKEY) <<
767 SC(CHECK_PARTITION_KEY_VALUE_SHIFT));
768
769
770 if (type == SC_USER) {
771 opval = USER_OPCODE_CHECK_VAL;
772 opmask = USER_OPCODE_CHECK_MASK;
773 } else {
774 opval = OPCODE_CHECK_VAL_DISABLED;
775 opmask = OPCODE_CHECK_MASK_DISABLED;
776 }
777
778
779 write_kctxt_csr(dd, hw_context, SC(CHECK_OPCODE),
780 ((u64)opmask << SC(CHECK_OPCODE_MASK_SHIFT)) |
781 ((u64)opval << SC(CHECK_OPCODE_VALUE_SHIFT)));
782
783
784 reg = pa & SC(CREDIT_RETURN_ADDR_ADDRESS_SMASK);
785 write_kctxt_csr(dd, hw_context, SC(CREDIT_RETURN_ADDR), reg);
786
787
788
789
790
791
792
793
794
795 if (type == SC_ACK) {
796 thresh = sc_percent_to_threshold(sc, 50);
797 } else if (type == SC_USER) {
798 thresh = sc_percent_to_threshold(sc,
799 user_credit_return_threshold);
800 } else {
801 thresh = sc_mtu_to_threshold(sc, hfi1_max_mtu, hdrqentsize);
802 }
803 reg = thresh << SC(CREDIT_CTRL_THRESHOLD_SHIFT);
804
805 if (type == SC_USER && HFI1_CAP_IS_USET(EARLY_CREDIT_RETURN))
806 reg |= SC(CREDIT_CTRL_EARLY_RETURN_SMASK);
807 else if (HFI1_CAP_IS_KSET(EARLY_CREDIT_RETURN))
808 reg |= SC(CREDIT_CTRL_EARLY_RETURN_SMASK);
809
810
811 sc->credit_ctrl = reg;
812 write_kctxt_csr(dd, hw_context, SC(CREDIT_CTRL), reg);
813
814
815 if (type == SC_USER) {
816 reg = 1ULL << 15;
817 write_kctxt_csr(dd, hw_context, SC(CHECK_VL), reg);
818 }
819
820 spin_unlock_irqrestore(&dd->sc_lock, flags);
821
822
823
824
825
826
827
828
829
830 if (type != SC_USER) {
831
832
833
834
835 sc->sr_size = sci->credits + 1;
836 sc->sr = kzalloc_node(sizeof(union pio_shadow_ring) *
837 sc->sr_size, GFP_KERNEL, numa);
838 if (!sc->sr) {
839 sc_free(sc);
840 return NULL;
841 }
842 }
843
844 hfi1_cdbg(PIO,
845 "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u\n",
846 sw_index,
847 hw_context,
848 sc_type_name(type),
849 sc->group,
850 sc->credits,
851 sc->credit_ctrl,
852 thresh);
853
854 return sc;
855}
856
857
858void sc_free(struct send_context *sc)
859{
860 struct hfi1_devdata *dd;
861 unsigned long flags;
862 u32 sw_index;
863 u32 hw_context;
864
865 if (!sc)
866 return;
867
868 sc->flags |= SCF_IN_FREE;
869 dd = sc->dd;
870 if (!list_empty(&sc->piowait))
871 dd_dev_err(dd, "piowait list not empty!\n");
872 sw_index = sc->sw_index;
873 hw_context = sc->hw_context;
874 sc_disable(sc);
875 flush_work(&sc->halt_work);
876
877 spin_lock_irqsave(&dd->sc_lock, flags);
878 dd->send_contexts[sw_index].sc = NULL;
879
880
881 write_kctxt_csr(dd, hw_context, SC(CTRL), 0);
882 write_kctxt_csr(dd, hw_context, SC(CHECK_ENABLE), 0);
883 write_kctxt_csr(dd, hw_context, SC(ERR_MASK), 0);
884 write_kctxt_csr(dd, hw_context, SC(CHECK_PARTITION_KEY), 0);
885 write_kctxt_csr(dd, hw_context, SC(CHECK_OPCODE), 0);
886 write_kctxt_csr(dd, hw_context, SC(CREDIT_RETURN_ADDR), 0);
887 write_kctxt_csr(dd, hw_context, SC(CREDIT_CTRL), 0);
888
889
890 sc_hw_free(dd, sw_index, hw_context);
891 spin_unlock_irqrestore(&dd->sc_lock, flags);
892
893 kfree(sc->sr);
894 free_percpu(sc->buffers_allocated);
895 kfree(sc);
896}
897
898
899void sc_disable(struct send_context *sc)
900{
901 u64 reg;
902 unsigned long flags;
903 struct pio_buf *pbuf;
904
905 if (!sc)
906 return;
907
908
909 spin_lock_irqsave(&sc->alloc_lock, flags);
910 reg = read_kctxt_csr(sc->dd, sc->hw_context, SC(CTRL));
911 reg &= ~SC(CTRL_CTXT_ENABLE_SMASK);
912 sc->flags &= ~SCF_ENABLED;
913 sc_wait_for_packet_egress(sc, 1);
914 write_kctxt_csr(sc->dd, sc->hw_context, SC(CTRL), reg);
915 spin_unlock_irqrestore(&sc->alloc_lock, flags);
916
917
918
919
920
921
922
923
924 udelay(1);
925 spin_lock_irqsave(&sc->release_lock, flags);
926 if (sc->sr) {
927 while (sc->sr_tail != sc->sr_head) {
928 pbuf = &sc->sr[sc->sr_tail].pbuf;
929 if (pbuf->cb)
930 (*pbuf->cb)(pbuf->arg, PRC_SC_DISABLE);
931 sc->sr_tail++;
932 if (sc->sr_tail >= sc->sr_size)
933 sc->sr_tail = 0;
934 }
935 }
936 spin_unlock_irqrestore(&sc->release_lock, flags);
937}
938
939
940#define packet_occupancy(r) \
941 (((r) & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SMASK)\
942 >> SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SHIFT)
943
944
945#define egress_halted(r) \
946 ((r) & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_HALT_STATUS_SMASK)
947
948
949static void sc_wait_for_packet_egress(struct send_context *sc, int pause)
950{
951 struct hfi1_devdata *dd = sc->dd;
952 u64 reg = 0;
953 u64 reg_prev;
954 u32 loop = 0;
955
956 while (1) {
957 reg_prev = reg;
958 reg = read_csr(dd, sc->hw_context * 8 +
959 SEND_EGRESS_CTXT_STATUS);
960
961 if (egress_halted(reg))
962 break;
963 reg = packet_occupancy(reg);
964 if (reg == 0)
965 break;
966
967 if (reg != reg_prev)
968 loop = 0;
969 if (loop > 500) {
970
971 dd_dev_err(dd,
972 "%s: context %u(%u) timeout waiting for packets to egress, remaining count %u, bouncing link\n",
973 __func__, sc->sw_index,
974 sc->hw_context, (u32)reg);
975 queue_work(dd->pport->hfi1_wq,
976 &dd->pport->link_bounce_work);
977 break;
978 }
979 loop++;
980 udelay(1);
981 }
982
983 if (pause)
984
985 pause_for_credit_return(dd);
986}
987
988void sc_wait(struct hfi1_devdata *dd)
989{
990 int i;
991
992 for (i = 0; i < dd->num_send_contexts; i++) {
993 struct send_context *sc = dd->send_contexts[i].sc;
994
995 if (!sc)
996 continue;
997 sc_wait_for_packet_egress(sc, 0);
998 }
999}
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010int sc_restart(struct send_context *sc)
1011{
1012 struct hfi1_devdata *dd = sc->dd;
1013 u64 reg;
1014 u32 loop;
1015 int count;
1016
1017
1018 if (!(sc->flags & SCF_HALTED) || (sc->flags & SCF_IN_FREE))
1019 return -EINVAL;
1020
1021 dd_dev_info(dd, "restarting send context %u(%u)\n", sc->sw_index,
1022 sc->hw_context);
1023
1024
1025
1026
1027
1028
1029
1030 loop = 0;
1031 while (1) {
1032 reg = read_kctxt_csr(dd, sc->hw_context, SC(STATUS));
1033 if (reg & SC(STATUS_CTXT_HALTED_SMASK))
1034 break;
1035 if (loop > 100) {
1036 dd_dev_err(dd, "%s: context %u(%u) not halting, skipping\n",
1037 __func__, sc->sw_index, sc->hw_context);
1038 return -ETIME;
1039 }
1040 loop++;
1041 udelay(1);
1042 }
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054 if (sc->type != SC_USER) {
1055
1056 loop = 0;
1057 while (1) {
1058 count = get_buffers_allocated(sc);
1059 if (count == 0)
1060 break;
1061 if (loop > 100) {
1062 dd_dev_err(dd,
1063 "%s: context %u(%u) timeout waiting for PIO buffers to zero, remaining %d\n",
1064 __func__, sc->sw_index,
1065 sc->hw_context, count);
1066 }
1067 loop++;
1068 udelay(1);
1069 }
1070 }
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081 sc_disable(sc);
1082
1083
1084
1085
1086
1087
1088
1089 return sc_enable(sc);
1090}
1091
1092
1093
1094
1095
1096
1097void pio_freeze(struct hfi1_devdata *dd)
1098{
1099 struct send_context *sc;
1100 int i;
1101
1102 for (i = 0; i < dd->num_send_contexts; i++) {
1103 sc = dd->send_contexts[i].sc;
1104
1105
1106
1107
1108
1109 if (!sc || !(sc->flags & SCF_FROZEN) || sc->type == SC_USER)
1110 continue;
1111
1112
1113 sc_disable(sc);
1114 }
1115}
1116
1117
1118
1119
1120
1121
1122
1123
1124void pio_kernel_unfreeze(struct hfi1_devdata *dd)
1125{
1126 struct send_context *sc;
1127 int i;
1128
1129 for (i = 0; i < dd->num_send_contexts; i++) {
1130 sc = dd->send_contexts[i].sc;
1131 if (!sc || !(sc->flags & SCF_FROZEN) || sc->type == SC_USER)
1132 continue;
1133
1134 sc_enable(sc);
1135 }
1136}
1137
1138
1139
1140
1141
1142
1143
1144static int pio_init_wait_progress(struct hfi1_devdata *dd)
1145{
1146 u64 reg;
1147 int max, count = 0;
1148
1149
1150 max = (dd->icode == ICODE_FPGA_EMULATION) ? 120 : 5;
1151 while (1) {
1152 reg = read_csr(dd, SEND_PIO_INIT_CTXT);
1153 if (!(reg & SEND_PIO_INIT_CTXT_PIO_INIT_IN_PROGRESS_SMASK))
1154 break;
1155 if (count >= max)
1156 return -ETIMEDOUT;
1157 udelay(5);
1158 count++;
1159 }
1160
1161 return reg & SEND_PIO_INIT_CTXT_PIO_INIT_ERR_SMASK ? -EIO : 0;
1162}
1163
1164
1165
1166
1167
1168void pio_reset_all(struct hfi1_devdata *dd)
1169{
1170 int ret;
1171
1172
1173 ret = pio_init_wait_progress(dd);
1174
1175 if (ret == -EIO) {
1176
1177 write_csr(dd, SEND_PIO_ERR_CLEAR,
1178 SEND_PIO_ERR_CLEAR_PIO_INIT_SM_IN_ERR_SMASK);
1179 }
1180
1181
1182 write_csr(dd, SEND_PIO_INIT_CTXT,
1183 SEND_PIO_INIT_CTXT_PIO_ALL_CTXT_INIT_SMASK);
1184 udelay(2);
1185 ret = pio_init_wait_progress(dd);
1186 if (ret < 0) {
1187 dd_dev_err(dd,
1188 "PIO send context init %s while initializing all PIO blocks\n",
1189 ret == -ETIMEDOUT ? "is stuck" : "had an error");
1190 }
1191}
1192
1193
1194int sc_enable(struct send_context *sc)
1195{
1196 u64 sc_ctrl, reg, pio;
1197 struct hfi1_devdata *dd;
1198 unsigned long flags;
1199 int ret = 0;
1200
1201 if (!sc)
1202 return -EINVAL;
1203 dd = sc->dd;
1204
1205
1206
1207
1208
1209
1210
1211
1212 spin_lock_irqsave(&sc->alloc_lock, flags);
1213 sc_ctrl = read_kctxt_csr(dd, sc->hw_context, SC(CTRL));
1214 if ((sc_ctrl & SC(CTRL_CTXT_ENABLE_SMASK)))
1215 goto unlock;
1216
1217
1218
1219 *sc->hw_free = 0;
1220 sc->free = 0;
1221 sc->alloc_free = 0;
1222 sc->fill = 0;
1223 sc->sr_head = 0;
1224 sc->sr_tail = 0;
1225 sc->flags = 0;
1226
1227 reset_buffers_allocated(sc);
1228
1229
1230
1231
1232
1233
1234
1235 reg = read_kctxt_csr(dd, sc->hw_context, SC(ERR_STATUS));
1236 if (reg)
1237 write_kctxt_csr(dd, sc->hw_context, SC(ERR_CLEAR), reg);
1238
1239
1240
1241
1242
1243 spin_lock(&dd->sc_init_lock);
1244
1245
1246
1247
1248
1249
1250
1251 pio = ((sc->hw_context & SEND_PIO_INIT_CTXT_PIO_CTXT_NUM_MASK) <<
1252 SEND_PIO_INIT_CTXT_PIO_CTXT_NUM_SHIFT) |
1253 SEND_PIO_INIT_CTXT_PIO_SINGLE_CTXT_INIT_SMASK;
1254 write_csr(dd, SEND_PIO_INIT_CTXT, pio);
1255
1256
1257
1258
1259 udelay(2);
1260 ret = pio_init_wait_progress(dd);
1261 spin_unlock(&dd->sc_init_lock);
1262 if (ret) {
1263 dd_dev_err(dd,
1264 "sctxt%u(%u): Context not enabled due to init failure %d\n",
1265 sc->sw_index, sc->hw_context, ret);
1266 goto unlock;
1267 }
1268
1269
1270
1271
1272 sc_ctrl |= SC(CTRL_CTXT_ENABLE_SMASK);
1273 write_kctxt_csr(dd, sc->hw_context, SC(CTRL), sc_ctrl);
1274
1275
1276
1277
1278 read_kctxt_csr(dd, sc->hw_context, SC(CTRL));
1279 sc->flags |= SCF_ENABLED;
1280
1281unlock:
1282 spin_unlock_irqrestore(&sc->alloc_lock, flags);
1283
1284 return ret;
1285}
1286
1287
1288void sc_return_credits(struct send_context *sc)
1289{
1290 if (!sc)
1291 return;
1292
1293
1294 write_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE),
1295 SC(CREDIT_FORCE_FORCE_RETURN_SMASK));
1296
1297
1298
1299
1300 read_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE));
1301
1302 write_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE), 0);
1303}
1304
1305
1306void sc_flush(struct send_context *sc)
1307{
1308 if (!sc)
1309 return;
1310
1311 sc_wait_for_packet_egress(sc, 1);
1312}
1313
1314
1315void sc_drop(struct send_context *sc)
1316{
1317 if (!sc)
1318 return;
1319
1320 dd_dev_info(sc->dd, "%s: context %u(%u) - not implemented\n",
1321 __func__, sc->sw_index, sc->hw_context);
1322}
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332void sc_stop(struct send_context *sc, int flag)
1333{
1334 unsigned long flags;
1335
1336
1337 sc->flags |= flag;
1338
1339
1340 spin_lock_irqsave(&sc->alloc_lock, flags);
1341 sc->flags &= ~SCF_ENABLED;
1342 spin_unlock_irqrestore(&sc->alloc_lock, flags);
1343 wake_up(&sc->halt_wait);
1344}
1345
1346#define BLOCK_DWORDS (PIO_BLOCK_SIZE / sizeof(u32))
1347#define dwords_to_blocks(x) DIV_ROUND_UP(x, BLOCK_DWORDS)
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len,
1360 pio_release_cb cb, void *arg)
1361{
1362 struct pio_buf *pbuf = NULL;
1363 unsigned long flags;
1364 unsigned long avail;
1365 unsigned long blocks = dwords_to_blocks(dw_len);
1366 unsigned long start_fill;
1367 int trycount = 0;
1368 u32 head, next;
1369
1370 spin_lock_irqsave(&sc->alloc_lock, flags);
1371 if (!(sc->flags & SCF_ENABLED)) {
1372 spin_unlock_irqrestore(&sc->alloc_lock, flags);
1373 goto done;
1374 }
1375
1376retry:
1377 avail = (unsigned long)sc->credits - (sc->fill - sc->alloc_free);
1378 if (blocks > avail) {
1379
1380 if (unlikely(trycount)) {
1381 spin_unlock_irqrestore(&sc->alloc_lock, flags);
1382 goto done;
1383 }
1384
1385 sc->alloc_free = ACCESS_ONCE(sc->free);
1386 avail =
1387 (unsigned long)sc->credits -
1388 (sc->fill - sc->alloc_free);
1389 if (blocks > avail) {
1390
1391 spin_unlock_irqrestore(&sc->alloc_lock, flags);
1392 sc_release_update(sc);
1393 spin_lock_irqsave(&sc->alloc_lock, flags);
1394 sc->alloc_free = ACCESS_ONCE(sc->free);
1395 trycount++;
1396 goto retry;
1397 }
1398 }
1399
1400
1401
1402 preempt_disable();
1403 this_cpu_inc(*sc->buffers_allocated);
1404
1405
1406 head = sc->sr_head;
1407
1408
1409 start_fill = sc->fill;
1410 sc->fill += blocks;
1411
1412
1413
1414
1415
1416
1417
1418
1419 pbuf = &sc->sr[head].pbuf;
1420 pbuf->sent_at = sc->fill;
1421 pbuf->cb = cb;
1422 pbuf->arg = arg;
1423 pbuf->sc = sc;
1424
1425
1426
1427 next = head + 1;
1428 if (next >= sc->sr_size)
1429 next = 0;
1430
1431
1432
1433
1434 smp_wmb();
1435 sc->sr_head = next;
1436 spin_unlock_irqrestore(&sc->alloc_lock, flags);
1437
1438
1439 pbuf->start = sc->base_addr + ((start_fill % sc->credits)
1440 * PIO_BLOCK_SIZE);
1441 pbuf->size = sc->credits * PIO_BLOCK_SIZE;
1442 pbuf->end = sc->base_addr + pbuf->size;
1443 pbuf->block_count = blocks;
1444 pbuf->qw_written = 0;
1445 pbuf->carry_bytes = 0;
1446 pbuf->carry.val64 = 0;
1447done:
1448 return pbuf;
1449}
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462void sc_add_credit_return_intr(struct send_context *sc)
1463{
1464 unsigned long flags;
1465
1466
1467 spin_lock_irqsave(&sc->credit_ctrl_lock, flags);
1468 if (sc->credit_intr_count == 0) {
1469 sc->credit_ctrl |= SC(CREDIT_CTRL_CREDIT_INTR_SMASK);
1470 write_kctxt_csr(sc->dd, sc->hw_context,
1471 SC(CREDIT_CTRL), sc->credit_ctrl);
1472 }
1473 sc->credit_intr_count++;
1474 spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags);
1475}
1476
1477
1478
1479
1480
1481void sc_del_credit_return_intr(struct send_context *sc)
1482{
1483 unsigned long flags;
1484
1485 WARN_ON(sc->credit_intr_count == 0);
1486
1487
1488 spin_lock_irqsave(&sc->credit_ctrl_lock, flags);
1489 sc->credit_intr_count--;
1490 if (sc->credit_intr_count == 0) {
1491 sc->credit_ctrl &= ~SC(CREDIT_CTRL_CREDIT_INTR_SMASK);
1492 write_kctxt_csr(sc->dd, sc->hw_context,
1493 SC(CREDIT_CTRL), sc->credit_ctrl);
1494 }
1495 spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags);
1496}
1497
1498
1499
1500
1501
1502void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint)
1503{
1504 if (needint)
1505 sc_add_credit_return_intr(sc);
1506 else
1507 sc_del_credit_return_intr(sc);
1508 trace_hfi1_wantpiointr(sc, needint, sc->credit_ctrl);
1509 if (needint) {
1510 mmiowb();
1511 sc_return_credits(sc);
1512 }
1513}
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523static void sc_piobufavail(struct send_context *sc)
1524{
1525 struct hfi1_devdata *dd = sc->dd;
1526 struct hfi1_ibdev *dev = &dd->verbs_dev;
1527 struct list_head *list;
1528 struct rvt_qp *qps[PIO_WAIT_BATCH_SIZE];
1529 struct rvt_qp *qp;
1530 struct hfi1_qp_priv *priv;
1531 unsigned long flags;
1532 unsigned i, n = 0;
1533
1534 if (dd->send_contexts[sc->sw_index].type != SC_KERNEL)
1535 return;
1536 list = &sc->piowait;
1537
1538
1539
1540
1541
1542
1543 write_seqlock_irqsave(&dev->iowait_lock, flags);
1544 while (!list_empty(list)) {
1545 struct iowait *wait;
1546
1547 if (n == ARRAY_SIZE(qps))
1548 break;
1549 wait = list_first_entry(list, struct iowait, list);
1550 qp = iowait_to_qp(wait);
1551 priv = qp->priv;
1552 list_del_init(&priv->s_iowait.list);
1553
1554 qps[n++] = qp;
1555 }
1556
1557
1558
1559
1560 if (n) {
1561 hfi1_sc_wantpiobuf_intr(sc, 0);
1562 if (!list_empty(list))
1563 hfi1_sc_wantpiobuf_intr(sc, 1);
1564 }
1565 write_sequnlock_irqrestore(&dev->iowait_lock, flags);
1566
1567 for (i = 0; i < n; i++)
1568 hfi1_qp_wakeup(qps[i],
1569 RVT_S_WAIT_PIO | RVT_S_WAIT_PIO_DRAIN);
1570}
1571
1572
1573static inline int fill_code(u64 hw_free)
1574{
1575 int code = 0;
1576
1577 if (hw_free & CR_STATUS_SMASK)
1578 code |= PRC_STATUS_ERR;
1579 if (hw_free & CR_CREDIT_RETURN_DUE_TO_PBC_SMASK)
1580 code |= PRC_PBC;
1581 if (hw_free & CR_CREDIT_RETURN_DUE_TO_THRESHOLD_SMASK)
1582 code |= PRC_THRESHOLD;
1583 if (hw_free & CR_CREDIT_RETURN_DUE_TO_ERR_SMASK)
1584 code |= PRC_FILL_ERR;
1585 if (hw_free & CR_CREDIT_RETURN_DUE_TO_FORCE_SMASK)
1586 code |= PRC_SC_DISABLE;
1587 return code;
1588}
1589
1590
1591#define sent_before(a, b) time_before(a, b)
1592
1593
1594
1595
1596void sc_release_update(struct send_context *sc)
1597{
1598 struct pio_buf *pbuf;
1599 u64 hw_free;
1600 u32 head, tail;
1601 unsigned long old_free;
1602 unsigned long free;
1603 unsigned long extra;
1604 unsigned long flags;
1605 int code;
1606
1607 if (!sc)
1608 return;
1609
1610 spin_lock_irqsave(&sc->release_lock, flags);
1611
1612 hw_free = le64_to_cpu(*sc->hw_free);
1613 old_free = sc->free;
1614 extra = (((hw_free & CR_COUNTER_SMASK) >> CR_COUNTER_SHIFT)
1615 - (old_free & CR_COUNTER_MASK))
1616 & CR_COUNTER_MASK;
1617 free = old_free + extra;
1618 trace_hfi1_piofree(sc, extra);
1619
1620
1621 code = -1;
1622 head = ACCESS_ONCE(sc->sr_head);
1623 tail = sc->sr_tail;
1624 while (head != tail) {
1625 pbuf = &sc->sr[tail].pbuf;
1626
1627 if (sent_before(free, pbuf->sent_at)) {
1628
1629 break;
1630 }
1631 if (pbuf->cb) {
1632 if (code < 0)
1633 code = fill_code(hw_free);
1634 (*pbuf->cb)(pbuf->arg, code);
1635 }
1636
1637 tail++;
1638 if (tail >= sc->sr_size)
1639 tail = 0;
1640 }
1641 sc->sr_tail = tail;
1642
1643 smp_wmb();
1644 sc->free = free;
1645 spin_unlock_irqrestore(&sc->release_lock, flags);
1646 sc_piobufavail(sc);
1647}
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658void sc_group_release_update(struct hfi1_devdata *dd, u32 hw_context)
1659{
1660 struct send_context *sc;
1661 u32 sw_index;
1662 u32 gc, gc_end;
1663
1664 spin_lock(&dd->sc_lock);
1665 sw_index = dd->hw_to_sw[hw_context];
1666 if (unlikely(sw_index >= dd->num_send_contexts)) {
1667 dd_dev_err(dd, "%s: invalid hw (%u) to sw (%u) mapping\n",
1668 __func__, hw_context, sw_index);
1669 goto done;
1670 }
1671 sc = dd->send_contexts[sw_index].sc;
1672 if (unlikely(!sc))
1673 goto done;
1674
1675 gc = group_context(hw_context, sc->group);
1676 gc_end = gc + group_size(sc->group);
1677 for (; gc < gc_end; gc++) {
1678 sw_index = dd->hw_to_sw[gc];
1679 if (unlikely(sw_index >= dd->num_send_contexts)) {
1680 dd_dev_err(dd,
1681 "%s: invalid hw (%u) to sw (%u) mapping\n",
1682 __func__, hw_context, sw_index);
1683 continue;
1684 }
1685 sc_release_update(dd->send_contexts[sw_index].sc);
1686 }
1687done:
1688 spin_unlock(&dd->sc_lock);
1689}
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700struct send_context *pio_select_send_context_vl(struct hfi1_devdata *dd,
1701 u32 selector, u8 vl)
1702{
1703 struct pio_vl_map *m;
1704 struct pio_map_elem *e;
1705 struct send_context *rval;
1706
1707
1708
1709
1710
1711
1712 if (unlikely(vl >= num_vls)) {
1713 rval = NULL;
1714 goto done;
1715 }
1716
1717 rcu_read_lock();
1718 m = rcu_dereference(dd->pio_map);
1719 if (unlikely(!m)) {
1720 rcu_read_unlock();
1721 return dd->vld[0].sc;
1722 }
1723 e = m->map[vl & m->mask];
1724 rval = e->ksc[selector & e->mask];
1725 rcu_read_unlock();
1726
1727done:
1728 rval = !rval ? dd->vld[0].sc : rval;
1729 return rval;
1730}
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740struct send_context *pio_select_send_context_sc(struct hfi1_devdata *dd,
1741 u32 selector, u8 sc5)
1742{
1743 u8 vl = sc_to_vlt(dd, sc5);
1744
1745 return pio_select_send_context_vl(dd, selector, vl);
1746}
1747
1748
1749
1750
1751static void pio_map_free(struct pio_vl_map *m)
1752{
1753 int i;
1754
1755 for (i = 0; m && i < m->actual_vls; i++)
1756 kfree(m->map[i]);
1757 kfree(m);
1758}
1759
1760
1761
1762
1763static void pio_map_rcu_callback(struct rcu_head *list)
1764{
1765 struct pio_vl_map *m = container_of(list, struct pio_vl_map, list);
1766
1767 pio_map_free(m);
1768}
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_scontexts)
1799{
1800 int i, j;
1801 int extra, sc_per_vl;
1802 int scontext = 1;
1803 int num_kernel_send_contexts = 0;
1804 u8 lvl_scontexts[OPA_MAX_VLS];
1805 struct pio_vl_map *oldmap, *newmap;
1806
1807 if (!vl_scontexts) {
1808
1809 for (i = 1; i < dd->num_send_contexts; i++)
1810 if (dd->send_contexts[i].type == SC_KERNEL)
1811 num_kernel_send_contexts++;
1812
1813 sc_per_vl = num_kernel_send_contexts / num_vls;
1814
1815 extra = num_kernel_send_contexts % num_vls;
1816 vl_scontexts = lvl_scontexts;
1817
1818 for (i = num_vls - 1; i >= 0; i--, extra--)
1819 vl_scontexts[i] = sc_per_vl + (extra > 0 ? 1 : 0);
1820 }
1821
1822 newmap = kzalloc(sizeof(*newmap) +
1823 roundup_pow_of_two(num_vls) *
1824 sizeof(struct pio_map_elem *),
1825 GFP_KERNEL);
1826 if (!newmap)
1827 goto bail;
1828 newmap->actual_vls = num_vls;
1829 newmap->vls = roundup_pow_of_two(num_vls);
1830 newmap->mask = (1 << ilog2(newmap->vls)) - 1;
1831 for (i = 0; i < newmap->vls; i++) {
1832
1833 int first_scontext = scontext;
1834
1835 if (i < newmap->actual_vls) {
1836 int sz = roundup_pow_of_two(vl_scontexts[i]);
1837
1838
1839 newmap->map[i] = kzalloc(sizeof(*newmap->map[i]) +
1840 sz * sizeof(struct
1841 send_context *),
1842 GFP_KERNEL);
1843 if (!newmap->map[i])
1844 goto bail;
1845 newmap->map[i]->mask = (1 << ilog2(sz)) - 1;
1846
1847 for (j = 0; j < sz; j++) {
1848 if (dd->kernel_send_context[scontext])
1849 newmap->map[i]->ksc[j] =
1850 dd->kernel_send_context[scontext];
1851 if (++scontext >= first_scontext +
1852 vl_scontexts[i])
1853
1854 scontext = first_scontext;
1855 }
1856 } else {
1857
1858 newmap->map[i] = newmap->map[i % num_vls];
1859 }
1860 scontext = first_scontext + vl_scontexts[i];
1861 }
1862
1863 spin_lock_irq(&dd->pio_map_lock);
1864 oldmap = rcu_dereference_protected(dd->pio_map,
1865 lockdep_is_held(&dd->pio_map_lock));
1866
1867
1868 rcu_assign_pointer(dd->pio_map, newmap);
1869
1870 spin_unlock_irq(&dd->pio_map_lock);
1871
1872 if (oldmap)
1873 call_rcu(&oldmap->list, pio_map_rcu_callback);
1874 return 0;
1875bail:
1876
1877 pio_map_free(newmap);
1878 return -ENOMEM;
1879}
1880
1881void free_pio_map(struct hfi1_devdata *dd)
1882{
1883
1884 if (rcu_access_pointer(dd->pio_map)) {
1885 spin_lock_irq(&dd->pio_map_lock);
1886 pio_map_free(rcu_access_pointer(dd->pio_map));
1887 RCU_INIT_POINTER(dd->pio_map, NULL);
1888 spin_unlock_irq(&dd->pio_map_lock);
1889 synchronize_rcu();
1890 }
1891 kfree(dd->kernel_send_context);
1892 dd->kernel_send_context = NULL;
1893}
1894
1895int init_pervl_scs(struct hfi1_devdata *dd)
1896{
1897 int i;
1898 u64 mask, all_vl_mask = (u64)0x80ff;
1899 u64 data_vls_mask = (u64)0x00ff;
1900 u32 ctxt;
1901 struct hfi1_pportdata *ppd = dd->pport;
1902
1903 dd->vld[15].sc = sc_alloc(dd, SC_KERNEL,
1904 dd->rcd[0]->rcvhdrqentsize, dd->node);
1905 if (!dd->vld[15].sc)
1906 goto nomem;
1907 hfi1_init_ctxt(dd->vld[15].sc);
1908 dd->vld[15].mtu = enum_to_mtu(OPA_MTU_2048);
1909
1910 dd->kernel_send_context = kmalloc_node(dd->num_send_contexts *
1911 sizeof(struct send_context *),
1912 GFP_KERNEL, dd->node);
1913 dd->kernel_send_context[0] = dd->vld[15].sc;
1914
1915 for (i = 0; i < num_vls; i++) {
1916
1917
1918
1919
1920
1921
1922
1923 dd->vld[i].sc = sc_alloc(dd, SC_KERNEL,
1924 dd->rcd[0]->rcvhdrqentsize, dd->node);
1925 if (!dd->vld[i].sc)
1926 goto nomem;
1927 dd->kernel_send_context[i + 1] = dd->vld[i].sc;
1928 hfi1_init_ctxt(dd->vld[i].sc);
1929
1930 dd->vld[i].mtu = hfi1_max_mtu;
1931 }
1932 for (i = num_vls; i < INIT_SC_PER_VL * num_vls; i++) {
1933 dd->kernel_send_context[i + 1] =
1934 sc_alloc(dd, SC_KERNEL, dd->rcd[0]->rcvhdrqentsize, dd->node);
1935 if (!dd->kernel_send_context[i + 1])
1936 goto nomem;
1937 hfi1_init_ctxt(dd->kernel_send_context[i + 1]);
1938 }
1939
1940 sc_enable(dd->vld[15].sc);
1941 ctxt = dd->vld[15].sc->hw_context;
1942 mask = all_vl_mask & ~(1LL << 15);
1943 write_kctxt_csr(dd, ctxt, SC(CHECK_VL), mask);
1944 dd_dev_info(dd,
1945 "Using send context %u(%u) for VL15\n",
1946 dd->vld[15].sc->sw_index, ctxt);
1947
1948 for (i = 0; i < num_vls; i++) {
1949 sc_enable(dd->vld[i].sc);
1950 ctxt = dd->vld[i].sc->hw_context;
1951 mask = all_vl_mask & ~(data_vls_mask);
1952 write_kctxt_csr(dd, ctxt, SC(CHECK_VL), mask);
1953 }
1954 for (i = num_vls; i < INIT_SC_PER_VL * num_vls; i++) {
1955 sc_enable(dd->kernel_send_context[i + 1]);
1956 ctxt = dd->kernel_send_context[i + 1]->hw_context;
1957 mask = all_vl_mask & ~(data_vls_mask);
1958 write_kctxt_csr(dd, ctxt, SC(CHECK_VL), mask);
1959 }
1960
1961 if (pio_map_init(dd, ppd->port - 1, num_vls, NULL))
1962 goto nomem;
1963 return 0;
1964nomem:
1965 sc_free(dd->vld[15].sc);
1966 for (i = 0; i < num_vls; i++)
1967 sc_free(dd->vld[i].sc);
1968 for (i = num_vls; i < INIT_SC_PER_VL * num_vls; i++)
1969 sc_free(dd->kernel_send_context[i + 1]);
1970 return -ENOMEM;
1971}
1972
1973int init_credit_return(struct hfi1_devdata *dd)
1974{
1975 int ret;
1976 int num_numa;
1977 int i;
1978
1979 num_numa = num_online_nodes();
1980
1981 for (i = 0; i < num_numa; i++) {
1982 if (!node_online(i)) {
1983 dd_dev_err(dd, "NUMA nodes are not compact\n");
1984 ret = -EINVAL;
1985 goto done;
1986 }
1987 }
1988
1989 dd->cr_base = kcalloc(
1990 num_numa,
1991 sizeof(struct credit_return_base),
1992 GFP_KERNEL);
1993 if (!dd->cr_base) {
1994 dd_dev_err(dd, "Unable to allocate credit return base\n");
1995 ret = -ENOMEM;
1996 goto done;
1997 }
1998 for (i = 0; i < num_numa; i++) {
1999 int bytes = TXE_NUM_CONTEXTS * sizeof(struct credit_return);
2000
2001 set_dev_node(&dd->pcidev->dev, i);
2002 dd->cr_base[i].va = dma_zalloc_coherent(
2003 &dd->pcidev->dev,
2004 bytes,
2005 &dd->cr_base[i].pa,
2006 GFP_KERNEL);
2007 if (!dd->cr_base[i].va) {
2008 set_dev_node(&dd->pcidev->dev, dd->node);
2009 dd_dev_err(dd,
2010 "Unable to allocate credit return DMA range for NUMA %d\n",
2011 i);
2012 ret = -ENOMEM;
2013 goto done;
2014 }
2015 }
2016 set_dev_node(&dd->pcidev->dev, dd->node);
2017
2018 ret = 0;
2019done:
2020 return ret;
2021}
2022
2023void free_credit_return(struct hfi1_devdata *dd)
2024{
2025 int num_numa;
2026 int i;
2027
2028 if (!dd->cr_base)
2029 return;
2030
2031 num_numa = num_online_nodes();
2032 for (i = 0; i < num_numa; i++) {
2033 if (dd->cr_base[i].va) {
2034 dma_free_coherent(&dd->pcidev->dev,
2035 TXE_NUM_CONTEXTS *
2036 sizeof(struct credit_return),
2037 dd->cr_base[i].va,
2038 dd->cr_base[i].pa);
2039 }
2040 }
2041 kfree(dd->cr_base);
2042 dd->cr_base = NULL;
2043}
2044