1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/module.h>
16#include <linux/init.h>
17#include <linux/moduleparam.h>
18#include <linux/sched.h>
19#include <linux/kernel.h>
20#include <linux/slab.h>
21#include <linux/errno.h>
22#include <linux/types.h>
23#include <linux/interrupt.h>
24#include <linux/in.h>
25#include <linux/netdevice.h>
26#include <linux/etherdevice.h>
27#include <linux/skbuff.h>
28#include <linux/ioctl.h>
29#include <linux/cdev.h>
30#include <linux/hugetlb.h>
31#include <linux/in6.h>
32#include <linux/timer.h>
33#include <linux/io.h>
34#include <asm/checksum.h>
35#include <asm/homecache.h>
36
37#include <hv/drv_xgbe_intf.h>
38#include <hv/drv_xgbe_impl.h>
39#include <hv/hypervisor.h>
40#include <hv/netio_intf.h>
41
42
43#include <linux/ip.h>
44#include <linux/tcp.h>
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89#define TILE_NET_MTU 1500
90
91
92
93
94
95
96
97
98
99
100
101
102#define TILE_NET_TX_QUEUE_LEN 0
103
104
105
106
107
108
109
110
111
112
113
114
115#define TILE_NET_TIMEOUT (5 * HZ)
116
117
118#define TILE_NET_RETRY_INTERVAL (5 * HZ)
119
120
121#define TILE_NET_DEVS 4
122
123
124
125
126#if NET_IP_ALIGN != LIPP_PACKET_PADDING
127#error "NET_IP_ALIGN must match LIPP_PACKET_PADDING."
128#endif
129
130
131
132#ifdef TILE_NET_DEBUG
133#define PDEBUG(fmt, args...) net_printk(fmt, ## args)
134#else
135#define PDEBUG(fmt, args...)
136#endif
137
138
139MODULE_AUTHOR("Tilera");
140MODULE_LICENSE("GPL");
141
142
143
144
145
146
147
148struct tile_netio_queue {
149 netio_queue_impl_t *__system_part;
150 netio_queue_user_impl_t __user_part;
151
152};
153
154
155
156
157
158struct tile_net_stats_t {
159 u32 rx_packets;
160 u32 rx_bytes;
161 u32 tx_packets;
162 u32 tx_bytes;
163};
164
165
166
167
168
169
170
171struct tile_net_cpu {
172
173 struct napi_struct napi;
174
175 struct tile_netio_queue queue;
176
177 struct tile_net_stats_t stats;
178
179 bool napi_enabled;
180
181 bool registered;
182
183 bool link_down;
184
185 bool egress_timer_scheduled;
186
187 unsigned int num_needed_small_buffers;
188
189 unsigned int num_needed_large_buffers;
190
191 struct timer_list egress_timer;
192};
193
194
195
196
197
198struct tile_net_priv {
199
200 struct net_device *dev;
201
202 struct page *eq_pages;
203
204 lepp_queue_t *eq;
205
206 spinlock_t eq_lock;
207
208 int hv_devhdl;
209
210 u32 intr_id;
211
212 bool partly_opened;
213
214 bool active;
215
216 struct cpumask network_cpus_map;
217
218 int network_cpus_count;
219
220 int network_cpus_credits;
221
222 struct net_device_stats stats;
223
224 struct delayed_work retry_work;
225
226 struct tile_net_cpu *cpu[NR_CPUS];
227};
228
229
230#define EQ_ORDER get_order(sizeof(lepp_queue_t))
231
232#define EQ_SIZE (1 << (PAGE_SHIFT + EQ_ORDER))
233
234
235
236
237static struct net_device *tile_net_devs[TILE_NET_DEVS];
238
239
240
241
242static DEFINE_PER_CPU(struct tile_net_cpu, hv_xgbe0);
243static DEFINE_PER_CPU(struct tile_net_cpu, hv_xgbe1);
244static DEFINE_PER_CPU(struct tile_net_cpu, hv_gbe0);
245static DEFINE_PER_CPU(struct tile_net_cpu, hv_gbe1);
246
247
248
249
250
251static bool network_cpus_used;
252
253
254
255
256static struct cpumask network_cpus_map;
257
258
259
260#ifdef TILE_NET_DEBUG
261
262
263
264
265
266static void net_printk(char *fmt, ...)
267{
268 int i;
269 int len;
270 va_list args;
271 static char buf[256];
272
273 len = sprintf(buf, "tile_net[%2.2d]: ", smp_processor_id());
274 va_start(args, fmt);
275 i = vscnprintf(buf + len, sizeof(buf) - len - 1, fmt, args);
276 va_end(args);
277 buf[255] = '\0';
278 pr_notice(buf);
279}
280#endif
281
282
283#ifdef TILE_NET_DUMP_PACKETS
284
285
286
287static void dump_packet(unsigned char *data, unsigned long length, char *s)
288{
289 int my_cpu = smp_processor_id();
290
291 unsigned long i;
292 char buf[128];
293
294 static unsigned int count;
295
296 pr_info("dump_packet(data %p, length 0x%lx s %s count 0x%x)\n",
297 data, length, s, count++);
298
299 pr_info("\n");
300
301 for (i = 0; i < length; i++) {
302 if ((i & 0xf) == 0)
303 sprintf(buf, "[%02d] %8.8lx:", my_cpu, i);
304 sprintf(buf + strlen(buf), " %2.2x", data[i]);
305 if ((i & 0xf) == 0xf || i == length - 1) {
306 strcat(buf, "\n");
307 pr_info("%s", buf);
308 }
309 }
310}
311#endif
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328inline int __netio_fastio1(u32 fastio_index, u32 arg0)
329{
330 long result, clobber_r1, clobber_r10;
331 asm volatile("swint2"
332 : "=R00" (result),
333 "=R01" (clobber_r1), "=R10" (clobber_r10)
334 : "R10" (fastio_index), "R01" (arg0)
335 : "memory", "r2", "r3", "r4",
336 "r5", "r6", "r7", "r8", "r9",
337 "r11", "r12", "r13", "r14",
338 "r15", "r16", "r17", "r18", "r19",
339 "r20", "r21", "r22", "r23", "r24",
340 "r25", "r26", "r27", "r28", "r29");
341 return result;
342}
343
344
345static void tile_net_return_credit(struct tile_net_cpu *info)
346{
347 struct tile_netio_queue *queue = &info->queue;
348 netio_queue_user_impl_t *qup = &queue->__user_part;
349
350
351 if (--qup->__receive_credit_remaining == 0) {
352 u32 interval = qup->__receive_credit_interval;
353 qup->__receive_credit_remaining = interval;
354 __netio_fastio_return_credits(qup->__fastio_index, interval);
355 }
356}
357
358
359
360
361
362
363static void tile_net_provide_linux_buffer(struct tile_net_cpu *info,
364 void *va, bool small)
365{
366 struct tile_netio_queue *queue = &info->queue;
367
368
369 unsigned int buffer = ((unsigned int)(__pa(va) >> 7) << 1) + small;
370
371 __netio_fastio_free_buffer(queue->__user_part.__fastio_index, buffer);
372}
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425static bool tile_net_provide_needed_buffer(struct tile_net_cpu *info,
426 bool small)
427{
428#if TILE_NET_MTU <= 1536
429
430 unsigned int large_size = NET_IP_ALIGN + 1536;
431#else
432
433 unsigned int large_size = NET_IP_ALIGN + TILE_NET_MTU + 100;
434#endif
435
436
437
438 unsigned int len =
439 (((small ? LIPP_SMALL_PACKET_SIZE : large_size) +
440 CHIP_L2_LINE_SIZE() - 1) & -CHIP_L2_LINE_SIZE());
441
442 unsigned int padding = 128 - NET_SKB_PAD;
443 unsigned int align;
444
445 struct sk_buff *skb;
446 void *va;
447
448 struct sk_buff **skb_ptr;
449
450
451 skb = netdev_alloc_skb(info->napi.dev, len + padding);
452 if (skb == NULL)
453 return false;
454
455
456 align = -(long)skb->data & (128 - 1);
457 BUG_ON(align > padding);
458 skb_reserve(skb, align);
459
460
461 va = skb->data;
462
463
464 BUG_ON(((((long)va & ~HPAGE_MASK) + len) & HPAGE_MASK) != 0);
465
466#ifdef TILE_NET_PARANOIA
467#if CHIP_HAS_CBOX_HOME_MAP()
468 if (hash_default) {
469 HV_PTE pte = *virt_to_pte(current->mm, (unsigned long)va);
470 if (hv_pte_get_mode(pte) != HV_PTE_MODE_CACHE_HASH_L3)
471 panic("Non-HFH ingress buffer! VA=%p Mode=%d PTE=%llx",
472 va, hv_pte_get_mode(pte), hv_pte_val(pte));
473 }
474#endif
475#endif
476
477
478 if (!hash_default)
479 __inv_buffer(va, len);
480
481
482
483
484 skb_reserve(skb, NET_IP_ALIGN);
485
486
487 skb_ptr = va - sizeof(*skb_ptr);
488 *skb_ptr = skb;
489
490
491 __insn_mf();
492
493
494 tile_net_provide_linux_buffer(info, va, small);
495
496 return true;
497}
498
499
500
501
502
503static void tile_net_provide_needed_buffers(struct tile_net_cpu *info)
504{
505 while (info->num_needed_small_buffers != 0) {
506 if (!tile_net_provide_needed_buffer(info, true))
507 goto oops;
508 info->num_needed_small_buffers--;
509 }
510
511 while (info->num_needed_large_buffers != 0) {
512 if (!tile_net_provide_needed_buffer(info, false))
513 goto oops;
514 info->num_needed_large_buffers--;
515 }
516
517 return;
518
519oops:
520
521
522 pr_notice("Could not provide a linux buffer to LIPP.\n");
523}
524
525
526
527
528
529
530
531static unsigned int tile_net_lepp_grab_comps(lepp_queue_t *eq,
532 struct sk_buff *comps[],
533 unsigned int comps_size,
534 unsigned int min_size)
535{
536 unsigned int n = 0;
537
538 unsigned int comp_head = eq->comp_head;
539 unsigned int comp_busy = eq->comp_busy;
540
541 while (comp_head != comp_busy && n < comps_size) {
542 comps[n++] = eq->comps[comp_head];
543 LEPP_QINC(comp_head);
544 }
545
546 if (n < min_size)
547 return 0;
548
549 eq->comp_head = comp_head;
550
551 return n;
552}
553
554
555
556
557
558static bool tile_net_lepp_free_comps(struct net_device *dev, bool all)
559{
560 struct tile_net_priv *priv = netdev_priv(dev);
561
562 lepp_queue_t *eq = priv->eq;
563
564 struct sk_buff *olds[64];
565 unsigned int wanted = 64;
566 unsigned int i, n;
567 bool pending;
568
569 spin_lock(&priv->eq_lock);
570
571 if (all)
572 eq->comp_busy = eq->comp_tail;
573
574 n = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
575
576 pending = (eq->comp_head != eq->comp_tail);
577
578 spin_unlock(&priv->eq_lock);
579
580 for (i = 0; i < n; i++)
581 kfree_skb(olds[i]);
582
583 return pending;
584}
585
586
587
588
589
590
591
592
593static void tile_net_schedule_egress_timer(struct tile_net_cpu *info)
594{
595 if (!info->egress_timer_scheduled) {
596 mod_timer_pinned(&info->egress_timer, jiffies + 1);
597 info->egress_timer_scheduled = true;
598 }
599}
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616static void tile_net_handle_egress_timer(unsigned long arg)
617{
618 struct tile_net_cpu *info = (struct tile_net_cpu *)arg;
619 struct net_device *dev = info->napi.dev;
620
621
622 info->egress_timer_scheduled = false;
623
624
625 if (tile_net_lepp_free_comps(dev, false))
626 tile_net_schedule_egress_timer(info);
627}
628
629
630#ifdef IGNORE_DUP_ACKS
631
632
633
634
635
636
637
638
639
640
641
642static bool is_dup_ack(char *s1, char *s2, unsigned int len)
643{
644 int i;
645
646 unsigned long long ignorable = 0;
647
648
649 ignorable |= (1ULL << 0x12);
650 ignorable |= (1ULL << 0x13);
651
652
653 ignorable |= (1ULL << 0x18);
654 ignorable |= (1ULL << 0x19);
655
656
657 ignorable |= (1ULL << 0x2a);
658 ignorable |= (1ULL << 0x2b);
659 ignorable |= (1ULL << 0x2c);
660 ignorable |= (1ULL << 0x2d);
661
662
663 ignorable |= (1ULL << 0x30);
664 ignorable |= (1ULL << 0x31);
665
666
667 ignorable |= (1ULL << 0x32);
668 ignorable |= (1ULL << 0x33);
669
670 for (i = 0; i < len; i++, ignorable >>= 1) {
671
672 if ((ignorable & 1) || (s1[i] == s2[i]))
673 continue;
674
675#ifdef TILE_NET_DEBUG
676
677 if (i < 0x38 && i != 0x2f &&
678 net_ratelimit())
679 pr_info("Diff at 0x%x\n", i);
680#endif
681
682 return false;
683 }
684
685#ifdef TILE_NET_NO_SUPPRESS_DUP_ACKS
686
687
688 if (s1[0x2a] == s2[0x2a] &&
689 s1[0x2b] == s2[0x2b] &&
690 s1[0x2c] == s2[0x2c] &&
691 s1[0x2d] == s2[0x2d]) {
692 return false;
693 }
694#endif
695
696 return true;
697}
698
699#endif
700
701
702
703static void tile_net_discard_aux(struct tile_net_cpu *info, int index)
704{
705 struct tile_netio_queue *queue = &info->queue;
706 netio_queue_impl_t *qsp = queue->__system_part;
707 netio_queue_user_impl_t *qup = &queue->__user_part;
708
709 int index2_aux = index + sizeof(netio_pkt_t);
710 int index2 =
711 ((index2_aux ==
712 qsp->__packet_receive_queue.__last_packet_plus_one) ?
713 0 : index2_aux);
714
715 netio_pkt_t *pkt = (netio_pkt_t *)((unsigned long) &qsp[1] + index);
716
717
718 unsigned int buffer = pkt->__packet.word;
719
720
721 void *va = __va((phys_addr_t)(buffer >> 1) << 7);
722
723
724 struct sk_buff **skb_ptr = va - sizeof(*skb_ptr);
725 struct sk_buff *skb = *skb_ptr;
726
727 kfree_skb(skb);
728
729
730 qup->__packet_receive_read = index2;
731}
732
733
734
735
736
737static void tile_net_discard_packets(struct net_device *dev)
738{
739 struct tile_net_priv *priv = netdev_priv(dev);
740 int my_cpu = smp_processor_id();
741 struct tile_net_cpu *info = priv->cpu[my_cpu];
742 struct tile_netio_queue *queue = &info->queue;
743 netio_queue_impl_t *qsp = queue->__system_part;
744 netio_queue_user_impl_t *qup = &queue->__user_part;
745
746 while (qup->__packet_receive_read !=
747 qsp->__packet_receive_queue.__packet_write) {
748 int index = qup->__packet_receive_read;
749 tile_net_discard_aux(info, index);
750 }
751}
752
753
754
755
756
757static bool tile_net_poll_aux(struct tile_net_cpu *info, int index)
758{
759 struct net_device *dev = info->napi.dev;
760
761 struct tile_netio_queue *queue = &info->queue;
762 netio_queue_impl_t *qsp = queue->__system_part;
763 netio_queue_user_impl_t *qup = &queue->__user_part;
764 struct tile_net_stats_t *stats = &info->stats;
765
766 int filter;
767
768 int index2_aux = index + sizeof(netio_pkt_t);
769 int index2 =
770 ((index2_aux ==
771 qsp->__packet_receive_queue.__last_packet_plus_one) ?
772 0 : index2_aux);
773
774 netio_pkt_t *pkt = (netio_pkt_t *)((unsigned long) &qsp[1] + index);
775
776 netio_pkt_metadata_t *metadata = NETIO_PKT_METADATA(pkt);
777
778
779
780 unsigned long len =
781 (NETIO_PKT_CUSTOM_LENGTH(pkt) +
782 NET_IP_ALIGN - NETIO_PACKET_PADDING);
783
784
785 unsigned int buffer = pkt->__packet.word;
786
787
788 bool small = ((buffer & 1) != 0);
789
790
791 void *va = __va((phys_addr_t)(buffer >> 1) << 7);
792
793
794
795 unsigned char *buf = va + NET_IP_ALIGN;
796
797
798 if (!hash_default)
799 __inv_buffer(buf, len);
800
801
802 dev->last_rx = jiffies;
803
804#ifdef TILE_NET_DUMP_PACKETS
805 dump_packet(buf, len, "rx");
806#endif
807
808#ifdef TILE_NET_VERIFY_INGRESS
809 if (!NETIO_PKT_L4_CSUM_CORRECT_M(metadata, pkt) &&
810 NETIO_PKT_L4_CSUM_CALCULATED_M(metadata, pkt)) {
811
812 pr_warning("Bad L4 checksum on %d byte packet.\n", len);
813 }
814 if (!NETIO_PKT_L3_CSUM_CORRECT_M(metadata, pkt) &&
815 NETIO_PKT_L3_CSUM_CALCULATED_M(metadata, pkt)) {
816 dump_packet(buf, len, "rx");
817 panic("Bad L3 checksum.");
818 }
819 switch (NETIO_PKT_STATUS_M(metadata, pkt)) {
820 case NETIO_PKT_STATUS_OVERSIZE:
821 if (len >= 64) {
822 dump_packet(buf, len, "rx");
823 panic("Unexpected OVERSIZE.");
824 }
825 break;
826 case NETIO_PKT_STATUS_BAD:
827 pr_warning("Unexpected BAD %ld byte packet.\n", len);
828 }
829#endif
830
831 filter = 0;
832
833
834
835 if (!(dev->flags & IFF_UP)) {
836
837 filter = 1;
838 } else if (NETIO_PKT_STATUS_M(metadata, pkt) == NETIO_PKT_STATUS_BAD) {
839
840 filter = 1;
841 } else if (!(dev->flags & IFF_PROMISC)) {
842
843 if (!is_multicast_ether_addr(buf)) {
844
845 const u8 *mine = dev->dev_addr;
846 filter = !ether_addr_equal(mine, buf);
847 }
848 }
849
850 if (filter) {
851
852
853
854 tile_net_provide_linux_buffer(info, va, small);
855
856 } else {
857
858
859 struct sk_buff **skb_ptr = va - sizeof(*skb_ptr);
860 struct sk_buff *skb = *skb_ptr;
861
862
863 if (skb->data != buf)
864 panic("Corrupt linux buffer from LIPP! "
865 "VA=%p, skb=%p, skb->data=%p\n",
866 va, skb, skb->data);
867
868
869 skb_put(skb, len);
870
871
872 skb->protocol = eth_type_trans(skb, dev);
873
874
875 if (NETIO_PKT_L4_CSUM_CORRECT_M(metadata, pkt))
876 skb->ip_summed = CHECKSUM_UNNECESSARY;
877
878 netif_receive_skb(skb);
879
880 stats->rx_packets++;
881 stats->rx_bytes += len;
882 }
883
884
885
886 tile_net_return_credit(info);
887
888
889 qup->__packet_receive_read = index2;
890
891 return !filter;
892}
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907static int tile_net_poll(struct napi_struct *napi, int budget)
908{
909 struct net_device *dev = napi->dev;
910 struct tile_net_priv *priv = netdev_priv(dev);
911 int my_cpu = smp_processor_id();
912 struct tile_net_cpu *info = priv->cpu[my_cpu];
913 struct tile_netio_queue *queue = &info->queue;
914 netio_queue_impl_t *qsp = queue->__system_part;
915 netio_queue_user_impl_t *qup = &queue->__user_part;
916
917 unsigned int work = 0;
918
919 while (priv->active) {
920 int index = qup->__packet_receive_read;
921 if (index == qsp->__packet_receive_queue.__packet_write)
922 break;
923
924 if (tile_net_poll_aux(info, index)) {
925 if (++work >= budget)
926 goto done;
927 }
928 }
929
930 napi_complete(&info->napi);
931
932 if (!priv->active)
933 goto done;
934
935
936 enable_percpu_irq(priv->intr_id, 0);
937
938
939 if (qup->__packet_receive_read !=
940 qsp->__packet_receive_queue.__packet_write) {
941
942
943 (void)napi_reschedule(&info->napi);
944 }
945
946done:
947
948 if (priv->active)
949 tile_net_provide_needed_buffers(info);
950
951 return work;
952}
953
954
955
956
957
958
959
960
961
962
963
964static irqreturn_t tile_net_handle_ingress_interrupt(int irq, void *dev_ptr)
965{
966 struct net_device *dev = (struct net_device *)dev_ptr;
967 struct tile_net_priv *priv = netdev_priv(dev);
968 int my_cpu = smp_processor_id();
969 struct tile_net_cpu *info = priv->cpu[my_cpu];
970
971
972 disable_percpu_irq(priv->intr_id);
973
974
975 if (!priv->active)
976 return IRQ_HANDLED;
977
978
979
980 napi_schedule(&info->napi);
981
982 return IRQ_HANDLED;
983}
984
985
986
987
988
989static int tile_net_open_aux(struct net_device *dev)
990{
991 struct tile_net_priv *priv = netdev_priv(dev);
992
993 int ret;
994 int dummy;
995 unsigned int epp_lotar;
996
997
998
999
1000 ret = hv_dev_pread(priv->hv_devhdl, 0,
1001 (HV_VirtAddr)&epp_lotar, sizeof(epp_lotar),
1002 NETIO_EPP_SHM_OFF);
1003 if (ret < 0) {
1004 pr_err("could not read epp_shm_queue lotar.\n");
1005 return -EIO;
1006 }
1007
1008
1009
1010
1011 {
1012 int epp_home = hv_lotar_to_cpu(epp_lotar);
1013 homecache_change_page_home(priv->eq_pages, EQ_ORDER, epp_home);
1014 }
1015
1016
1017
1018
1019 {
1020 netio_ipp_address_t ea = {
1021 .va = 0,
1022 .pa = __pa(priv->eq),
1023 .pte = hv_pte(0),
1024 .size = EQ_SIZE,
1025 };
1026 ea.pte = hv_pte_set_lotar(ea.pte, epp_lotar);
1027 ea.pte = hv_pte_set_mode(ea.pte, HV_PTE_MODE_CACHE_TILE_L3);
1028 ret = hv_dev_pwrite(priv->hv_devhdl, 0,
1029 (HV_VirtAddr)&ea,
1030 sizeof(ea),
1031 NETIO_EPP_SHM_OFF);
1032 if (ret < 0)
1033 return -EIO;
1034 }
1035
1036
1037
1038
1039 if (hv_dev_pwrite(priv->hv_devhdl, 0, (HV_VirtAddr)&dummy,
1040 sizeof(dummy), NETIO_IPP_START_SHIM_OFF) < 0) {
1041 pr_warning("Failed to start LIPP/LEPP.\n");
1042 return -EIO;
1043 }
1044
1045 return 0;
1046}
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056static void tile_net_register(void *dev_ptr)
1057{
1058 struct net_device *dev = (struct net_device *)dev_ptr;
1059 struct tile_net_priv *priv = netdev_priv(dev);
1060 int my_cpu = smp_processor_id();
1061 struct tile_net_cpu *info;
1062
1063 struct tile_netio_queue *queue;
1064
1065
1066 int queue_id =
1067 cpumask_test_cpu(my_cpu, &priv->network_cpus_map) ? 0 : 255;
1068
1069 netio_input_config_t config = {
1070 .flags = 0,
1071 .num_receive_packets = priv->network_cpus_credits,
1072 .queue_id = queue_id
1073 };
1074
1075 int ret = 0;
1076 netio_queue_impl_t *queuep;
1077
1078 PDEBUG("tile_net_register(queue_id %d)\n", queue_id);
1079
1080 if (!strcmp(dev->name, "xgbe0"))
1081 info = &__get_cpu_var(hv_xgbe0);
1082 else if (!strcmp(dev->name, "xgbe1"))
1083 info = &__get_cpu_var(hv_xgbe1);
1084 else if (!strcmp(dev->name, "gbe0"))
1085 info = &__get_cpu_var(hv_gbe0);
1086 else if (!strcmp(dev->name, "gbe1"))
1087 info = &__get_cpu_var(hv_gbe1);
1088 else
1089 BUG();
1090
1091
1092 init_timer(&info->egress_timer);
1093 info->egress_timer.data = (long)info;
1094 info->egress_timer.function = tile_net_handle_egress_timer;
1095
1096 priv->cpu[my_cpu] = info;
1097
1098
1099
1100
1101
1102 ret = hv_dev_pwrite(priv->hv_devhdl, 0,
1103 (HV_VirtAddr)&config,
1104 sizeof(netio_input_config_t),
1105 NETIO_IPP_INPUT_REGISTER_OFF);
1106 PDEBUG("hv_dev_pwrite(NETIO_IPP_INPUT_REGISTER_OFF) returned %d\n",
1107 ret);
1108 if (ret < 0) {
1109 if (ret != NETIO_LINK_DOWN) {
1110 printk(KERN_DEBUG "hv_dev_pwrite "
1111 "NETIO_IPP_INPUT_REGISTER_OFF failure %d\n",
1112 ret);
1113 }
1114 info->link_down = (ret == NETIO_LINK_DOWN);
1115 return;
1116 }
1117
1118
1119
1120
1121
1122 ret = hv_dev_pread(priv->hv_devhdl, 0,
1123 (HV_VirtAddr)&queuep,
1124 sizeof(netio_queue_impl_t *),
1125 NETIO_IPP_INPUT_REGISTER_OFF);
1126 PDEBUG("hv_dev_pread(NETIO_IPP_INPUT_REGISTER_OFF) returned %d\n",
1127 ret);
1128 PDEBUG("queuep %p\n", queuep);
1129 if (ret <= 0) {
1130
1131 pr_err("hv_dev_pread NETIO_IPP_INPUT_REGISTER_OFF failure\n");
1132 return;
1133 }
1134
1135 queue = &info->queue;
1136
1137 queue->__system_part = queuep;
1138
1139 memset(&queue->__user_part, 0, sizeof(netio_queue_user_impl_t));
1140
1141
1142 queue->__user_part.__receive_credit_interval = 4;
1143 queue->__user_part.__receive_credit_remaining =
1144 queue->__user_part.__receive_credit_interval;
1145
1146
1147
1148
1149
1150 ret = hv_dev_pread(priv->hv_devhdl, 0,
1151 (HV_VirtAddr)&queue->__user_part.__fastio_index,
1152 sizeof(queue->__user_part.__fastio_index),
1153 NETIO_IPP_GET_FASTIO_OFF);
1154 PDEBUG("hv_dev_pread(NETIO_IPP_GET_FASTIO_OFF) returned %d\n", ret);
1155
1156
1157 info->registered = true;
1158}
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169static void tile_net_deregister(void *dev_ptr)
1170{
1171 struct net_device *dev = (struct net_device *)dev_ptr;
1172 struct tile_net_priv *priv = netdev_priv(dev);
1173 int my_cpu = smp_processor_id();
1174 struct tile_net_cpu *info = priv->cpu[my_cpu];
1175
1176
1177 disable_percpu_irq(priv->intr_id);
1178
1179
1180 if (info == NULL || !info->registered)
1181 return;
1182
1183 {
1184 struct tile_netio_queue *queue = &info->queue;
1185 netio_queue_user_impl_t *qup = &queue->__user_part;
1186
1187
1188 __netio_fastio_return_credits(qup->__fastio_index, -1);
1189 }
1190}
1191
1192
1193
1194
1195
1196
1197
1198static void tile_net_unregister(void *dev_ptr)
1199{
1200 struct net_device *dev = (struct net_device *)dev_ptr;
1201 struct tile_net_priv *priv = netdev_priv(dev);
1202 int my_cpu = smp_processor_id();
1203 struct tile_net_cpu *info = priv->cpu[my_cpu];
1204
1205 int ret;
1206 int dummy = 0;
1207
1208
1209 disable_percpu_irq(priv->intr_id);
1210
1211
1212 if (info == NULL || !info->registered)
1213 return;
1214
1215
1216 ret = hv_dev_pwrite(priv->hv_devhdl, 0, (HV_VirtAddr)&dummy,
1217 sizeof(dummy), NETIO_IPP_INPUT_UNREGISTER_OFF);
1218 if (ret < 0)
1219 panic("Failed to unregister with LIPP/LEPP!\n");
1220
1221
1222 tile_net_discard_packets(dev);
1223
1224
1225 info->num_needed_small_buffers = 0;
1226 info->num_needed_large_buffers = 0;
1227
1228
1229 del_timer(&info->egress_timer);
1230 info->egress_timer_scheduled = false;
1231}
1232
1233
1234
1235
1236
1237
1238
1239
1240static void tile_net_stop_aux(struct net_device *dev)
1241{
1242 struct tile_net_priv *priv = netdev_priv(dev);
1243 int i;
1244
1245 int dummy = 0;
1246
1247
1248
1249
1250
1251
1252 on_each_cpu(tile_net_unregister, (void *)dev, 1);
1253 for_each_online_cpu(i) {
1254 struct tile_net_cpu *info = priv->cpu[i];
1255 if (info != NULL && info->registered) {
1256 netif_napi_del(&info->napi);
1257 info->registered = false;
1258 }
1259 }
1260
1261
1262 if (hv_dev_pwrite(priv->hv_devhdl, 0, (HV_VirtAddr)&dummy,
1263 sizeof(dummy), NETIO_IPP_STOP_SHIM_OFF) < 0)
1264 panic("Failed to stop LIPP/LEPP!\n");
1265
1266 priv->partly_opened = false;
1267}
1268
1269
1270
1271
1272
1273static void tile_net_stop_disable(void *dev_ptr)
1274{
1275 struct net_device *dev = (struct net_device *)dev_ptr;
1276 struct tile_net_priv *priv = netdev_priv(dev);
1277 int my_cpu = smp_processor_id();
1278 struct tile_net_cpu *info = priv->cpu[my_cpu];
1279
1280
1281 if (info != NULL && info->napi_enabled) {
1282 napi_disable(&info->napi);
1283 info->napi_enabled = false;
1284 }
1285}
1286
1287
1288
1289
1290
1291
1292
1293
1294static void tile_net_open_enable(void *dev_ptr)
1295{
1296 struct net_device *dev = (struct net_device *)dev_ptr;
1297 struct tile_net_priv *priv = netdev_priv(dev);
1298 int my_cpu = smp_processor_id();
1299 struct tile_net_cpu *info = priv->cpu[my_cpu];
1300
1301
1302 napi_enable(&info->napi);
1303 info->napi_enabled = true;
1304
1305
1306 enable_percpu_irq(priv->intr_id, 0);
1307}
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318static int tile_net_open_inner(struct net_device *dev)
1319{
1320 struct tile_net_priv *priv = netdev_priv(dev);
1321 int my_cpu = smp_processor_id();
1322 struct tile_net_cpu *info;
1323 struct tile_netio_queue *queue;
1324 int result = 0;
1325 int i;
1326 int dummy = 0;
1327
1328
1329
1330
1331
1332
1333 tile_net_register(dev);
1334 info = priv->cpu[my_cpu];
1335 if (!info->registered) {
1336 if (info->link_down)
1337 return 1;
1338 return -EAGAIN;
1339 }
1340
1341
1342
1343
1344
1345
1346
1347
1348 smp_call_function(tile_net_register, (void *)dev, 1);
1349 for_each_online_cpu(i) {
1350 struct tile_net_cpu *info = priv->cpu[i];
1351 if (info->registered)
1352 netif_napi_add(dev, &info->napi, tile_net_poll, 64);
1353 else
1354 result = -EAGAIN;
1355 }
1356 if (result != 0) {
1357 tile_net_stop_aux(dev);
1358 return result;
1359 }
1360
1361 queue = &info->queue;
1362
1363 if (priv->intr_id == 0) {
1364 unsigned int irq;
1365
1366
1367
1368
1369
1370
1371 priv->intr_id = queue->__system_part->__intr_id;
1372 BUG_ON(priv->intr_id == 0);
1373 irq = __ffs(priv->intr_id);
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384 tile_irq_activate(irq, TILE_IRQ_PERCPU);
1385 BUG_ON(request_irq(irq, tile_net_handle_ingress_interrupt,
1386 0, dev->name, (void *)dev) != 0);
1387 }
1388
1389 {
1390
1391
1392 int max_buffers =
1393 priv->network_cpus_count * priv->network_cpus_credits;
1394
1395 info->num_needed_small_buffers =
1396 min(LIPP_SMALL_BUFFERS, max_buffers);
1397
1398 info->num_needed_large_buffers =
1399 min(LIPP_LARGE_BUFFERS, max_buffers);
1400
1401 tile_net_provide_needed_buffers(info);
1402
1403 if (info->num_needed_small_buffers != 0 ||
1404 info->num_needed_large_buffers != 0)
1405 panic("Insufficient memory for buffer stack!");
1406 }
1407
1408
1409 priv->active = true;
1410
1411
1412 mb();
1413
1414
1415 on_each_cpu(tile_net_open_enable, (void *)dev, 1);
1416
1417
1418 if (hv_dev_pwrite(priv->hv_devhdl, 0, (HV_VirtAddr)&dummy,
1419 sizeof(dummy), NETIO_IPP_INPUT_INIT_OFF) < 0)
1420 panic("Failed to activate the LIPP Shim!\n");
1421
1422
1423 netif_start_queue(dev);
1424
1425 return 0;
1426}
1427
1428
1429
1430
1431
1432
1433static void tile_net_open_retry(struct work_struct *w)
1434{
1435 struct delayed_work *dw =
1436 container_of(w, struct delayed_work, work);
1437
1438 struct tile_net_priv *priv =
1439 container_of(dw, struct tile_net_priv, retry_work);
1440
1441
1442
1443
1444
1445
1446 if (tile_net_open_inner(priv->dev) != 0)
1447 schedule_delayed_work(&priv->retry_work,
1448 TILE_NET_RETRY_INTERVAL);
1449 else
1450 netif_carrier_on(priv->dev);
1451}
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468static int tile_net_open(struct net_device *dev)
1469{
1470 int ret = 0;
1471 struct tile_net_priv *priv = netdev_priv(dev);
1472
1473
1474
1475
1476
1477
1478
1479 if (!priv->partly_opened) {
1480
1481 int count;
1482 int credits;
1483
1484
1485 ret = tile_net_open_aux(dev);
1486 if (ret < 0) {
1487 pr_err("tile_net_open_aux failed: %d\n", ret);
1488 return ret;
1489 }
1490
1491
1492
1493 if (network_cpus_used)
1494 cpumask_copy(&priv->network_cpus_map,
1495 &network_cpus_map);
1496 else
1497 cpumask_copy(&priv->network_cpus_map, cpu_online_mask);
1498
1499
1500 count = cpumask_weight(&priv->network_cpus_map);
1501
1502
1503 credits = max(16, (LIPP_LARGE_BUFFERS / count) & ~1);
1504
1505
1506
1507 credits = min(NETIO_MAX_RECEIVE_PKTS, credits);
1508
1509 priv->network_cpus_count = count;
1510 priv->network_cpus_credits = credits;
1511
1512#ifdef TILE_NET_DEBUG
1513 pr_info("Using %d network cpus, with %d credits each\n",
1514 priv->network_cpus_count, priv->network_cpus_credits);
1515#endif
1516
1517 priv->partly_opened = true;
1518
1519 } else {
1520
1521
1522 }
1523
1524
1525
1526
1527 ret = tile_net_open_inner(dev);
1528 if (ret <= 0) {
1529 if (ret == 0)
1530 netif_carrier_on(dev);
1531 return ret;
1532 }
1533
1534
1535
1536
1537
1538
1539
1540 netif_carrier_off(dev);
1541 schedule_delayed_work(&priv->retry_work, TILE_NET_RETRY_INTERVAL);
1542
1543 return 0;
1544}
1545
1546
1547static int tile_net_drain_lipp_buffers(struct tile_net_priv *priv)
1548{
1549 int n = 0;
1550
1551
1552 while (true) {
1553 unsigned int buffer;
1554
1555
1556 if (hv_dev_pread(priv->hv_devhdl, 0, (HV_VirtAddr)&buffer,
1557 sizeof(buffer), NETIO_IPP_DRAIN_OFF) < 0)
1558 break;
1559
1560
1561 if (buffer == 0)
1562 break;
1563
1564 {
1565
1566 void *va = __va((phys_addr_t)(buffer >> 1) << 7);
1567
1568
1569 struct sk_buff **skb_ptr = va - sizeof(*skb_ptr);
1570 struct sk_buff *skb = *skb_ptr;
1571
1572 kfree_skb(skb);
1573 }
1574
1575 n++;
1576 }
1577
1578 return n;
1579}
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616static int tile_net_stop(struct net_device *dev)
1617{
1618 struct tile_net_priv *priv = netdev_priv(dev);
1619
1620 PDEBUG("tile_net_stop()\n");
1621
1622
1623 priv->active = false;
1624
1625
1626 mb();
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636 on_each_cpu(tile_net_deregister, (void *)dev, 1);
1637
1638
1639 (void)tile_net_drain_lipp_buffers(priv);
1640
1641
1642 cancel_delayed_work_sync(&priv->retry_work);
1643
1644
1645 netif_stop_queue(dev);
1646
1647
1648 on_each_cpu(tile_net_stop_disable, (void *)dev, 1);
1649
1650
1651
1652
1653
1654 if (tile_net_drain_lipp_buffers(priv) != 0)
1655 printk("Had to drain some extra LIPP buffers!\n");
1656
1657
1658 tile_net_stop_aux(dev);
1659
1660
1661
1662
1663
1664
1665 while (tile_net_lepp_free_comps(dev, true))
1666 ;
1667
1668
1669 memset(priv->eq, 0, sizeof(lepp_queue_t));
1670 mb();
1671
1672 return 0;
1673}
1674
1675
1676
1677
1678
1679
1680
1681static unsigned int tile_net_tx_frags(lepp_frag_t *frags,
1682 struct sk_buff *skb,
1683 void *b_data, unsigned int b_len)
1684{
1685 unsigned int i, n = 0;
1686
1687 struct skb_shared_info *sh = skb_shinfo(skb);
1688
1689 phys_addr_t cpa;
1690
1691 if (b_len != 0) {
1692
1693 if (!hash_default)
1694 finv_buffer_remote(b_data, b_len, 0);
1695
1696 cpa = __pa(b_data);
1697 frags[n].cpa_lo = cpa;
1698 frags[n].cpa_hi = cpa >> 32;
1699 frags[n].length = b_len;
1700 frags[n].hash_for_home = hash_default;
1701 n++;
1702 }
1703
1704 for (i = 0; i < sh->nr_frags; i++) {
1705
1706 skb_frag_t *f = &sh->frags[i];
1707 unsigned long pfn = page_to_pfn(skb_frag_page(f));
1708
1709
1710
1711 int hash_for_home = hash_default;
1712
1713
1714 if (!hash_default) {
1715 void *va = pfn_to_kaddr(pfn) + f->page_offset;
1716 BUG_ON(PageHighMem(skb_frag_page(f)));
1717 finv_buffer_remote(va, skb_frag_size(f), 0);
1718 }
1719
1720 cpa = ((phys_addr_t)pfn << PAGE_SHIFT) + f->page_offset;
1721 frags[n].cpa_lo = cpa;
1722 frags[n].cpa_hi = cpa >> 32;
1723 frags[n].length = skb_frag_size(f);
1724 frags[n].hash_for_home = hash_for_home;
1725 n++;
1726 }
1727
1728 return n;
1729}
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753static int tile_net_tx_tso(struct sk_buff *skb, struct net_device *dev)
1754{
1755 struct tile_net_priv *priv = netdev_priv(dev);
1756 int my_cpu = smp_processor_id();
1757 struct tile_net_cpu *info = priv->cpu[my_cpu];
1758 struct tile_net_stats_t *stats = &info->stats;
1759
1760 struct skb_shared_info *sh = skb_shinfo(skb);
1761
1762 unsigned char *data = skb->data;
1763
1764
1765 struct iphdr *ih = ip_hdr(skb);
1766 unsigned int ih_len = ih->ihl * 4;
1767
1768
1769 unsigned char *nh = skb_network_header(skb);
1770 unsigned int eh_len = nh - data;
1771
1772
1773 struct tcphdr *th = (struct tcphdr *)(nh + ih_len);
1774 unsigned int th_len = th->doff * 4;
1775
1776
1777
1778 unsigned int sh_len = eh_len + ih_len + th_len;
1779
1780
1781
1782 unsigned int b_len = skb_headlen(skb) - sh_len;
1783
1784
1785 unsigned int d_len = b_len + skb->data_len;
1786
1787
1788 unsigned int p_len = sh->gso_size;
1789
1790
1791 unsigned int num_segs = sh->gso_segs;
1792
1793
1794 u32 cmd_body[(LEPP_MAX_CMD_SIZE + 3) / 4];
1795 lepp_tso_cmd_t *cmd = (lepp_tso_cmd_t *)cmd_body;
1796
1797
1798 unsigned int num_frags =
1799 tile_net_tx_frags(cmd->frags, skb, data + sh_len, b_len);
1800
1801
1802 size_t cmd_size = LEPP_TSO_CMD_SIZE(num_frags, sh_len);
1803
1804
1805 lepp_tso_cmd_t cmd_init = {
1806 .tso = true,
1807 .header_size = sh_len,
1808 .ip_offset = eh_len,
1809 .tcp_offset = eh_len + ih_len,
1810 .payload_size = p_len,
1811 .num_frags = num_frags,
1812 };
1813
1814 unsigned long irqflags;
1815
1816 lepp_queue_t *eq = priv->eq;
1817
1818 struct sk_buff *olds[8];
1819 unsigned int wanted = 8;
1820 unsigned int i, nolds = 0;
1821
1822 unsigned int cmd_head, cmd_tail, cmd_next;
1823 unsigned int comp_tail;
1824
1825
1826
1827 BUG_ON(skb->protocol != htons(ETH_P_IP));
1828 BUG_ON(ih->protocol != IPPROTO_TCP);
1829 BUG_ON(skb->ip_summed != CHECKSUM_PARTIAL);
1830 BUG_ON(num_frags > LEPP_MAX_FRAGS);
1831
1832 BUG_ON(num_segs <= 1);
1833
1834
1835
1836
1837
1838 *cmd = cmd_init;
1839
1840
1841 memcpy(&cmd->frags[num_frags], data, sh_len);
1842
1843
1844
1845 prefetch_L1(&eq->comp_tail);
1846 prefetch_L1(&eq->cmd_tail);
1847 mb();
1848
1849
1850
1851
1852 spin_lock_irqsave(&priv->eq_lock, irqflags);
1853
1854
1855
1856 if (lepp_num_free_comp_slots(eq) == 0) {
1857 nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
1858 if (nolds == 0) {
1859busy:
1860 spin_unlock_irqrestore(&priv->eq_lock, irqflags);
1861 return NETDEV_TX_BUSY;
1862 }
1863 }
1864
1865 cmd_head = eq->cmd_head;
1866 cmd_tail = eq->cmd_tail;
1867
1868
1869
1870 cmd_next = cmd_tail + cmd_size;
1871 if (cmd_tail < cmd_head && cmd_next >= cmd_head)
1872 goto busy;
1873 if (cmd_next > LEPP_CMD_LIMIT) {
1874 cmd_next = 0;
1875 if (cmd_next == cmd_head)
1876 goto busy;
1877 }
1878
1879
1880 memcpy(&eq->cmds[cmd_tail], cmd, cmd_size);
1881
1882
1883 cmd_tail = cmd_next;
1884
1885
1886 comp_tail = eq->comp_tail;
1887 eq->comps[comp_tail] = skb;
1888 LEPP_QINC(comp_tail);
1889 eq->comp_tail = comp_tail;
1890
1891
1892
1893 __insn_mf();
1894
1895 eq->cmd_tail = cmd_tail;
1896
1897
1898
1899
1900 if (nolds == 0)
1901 nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 4);
1902
1903 spin_unlock_irqrestore(&priv->eq_lock, irqflags);
1904
1905
1906 for (i = 0; i < nolds; i++)
1907 kfree_skb(olds[i]);
1908
1909
1910 stats->tx_packets += num_segs;
1911 stats->tx_bytes += (num_segs * sh_len) + d_len;
1912
1913
1914 tile_net_schedule_egress_timer(info);
1915
1916 return NETDEV_TX_OK;
1917}
1918
1919
1920
1921
1922
1923static int tile_net_tx(struct sk_buff *skb, struct net_device *dev)
1924{
1925 struct tile_net_priv *priv = netdev_priv(dev);
1926 int my_cpu = smp_processor_id();
1927 struct tile_net_cpu *info = priv->cpu[my_cpu];
1928 struct tile_net_stats_t *stats = &info->stats;
1929
1930 unsigned long irqflags;
1931
1932 struct skb_shared_info *sh = skb_shinfo(skb);
1933
1934 unsigned int len = skb->len;
1935 unsigned char *data = skb->data;
1936
1937 unsigned int csum_start = skb_checksum_start_offset(skb);
1938
1939 lepp_frag_t frags[LEPP_MAX_FRAGS];
1940
1941 unsigned int num_frags;
1942
1943 lepp_queue_t *eq = priv->eq;
1944
1945 struct sk_buff *olds[8];
1946 unsigned int wanted = 8;
1947 unsigned int i, nolds = 0;
1948
1949 unsigned int cmd_size = sizeof(lepp_cmd_t);
1950
1951 unsigned int cmd_head, cmd_tail, cmd_next;
1952 unsigned int comp_tail;
1953
1954 lepp_cmd_t cmds[LEPP_MAX_FRAGS];
1955
1956
1957
1958
1959
1960
1961
1962
1963 if (!info->registered)
1964 return NETDEV_TX_BUSY;
1965
1966
1967
1968 dev->trans_start = jiffies;
1969
1970
1971#ifdef TILE_NET_PARANOIA
1972#if CHIP_HAS_CBOX_HOME_MAP()
1973 if (hash_default) {
1974 HV_PTE pte = *virt_to_pte(current->mm, (unsigned long)data);
1975 if (hv_pte_get_mode(pte) != HV_PTE_MODE_CACHE_HASH_L3)
1976 panic("Non-HFH egress buffer! VA=%p Mode=%d PTE=%llx",
1977 data, hv_pte_get_mode(pte), hv_pte_val(pte));
1978 }
1979#endif
1980#endif
1981
1982
1983#ifdef TILE_NET_DUMP_PACKETS
1984
1985 dump_packet(data, skb_headlen(skb), "tx");
1986#endif
1987
1988
1989 if (sh->gso_size != 0)
1990 return tile_net_tx_tso(skb, dev);
1991
1992
1993
1994
1995 num_frags = tile_net_tx_frags(frags, skb, data, skb_headlen(skb));
1996
1997 for (i = 0; i < num_frags; i++) {
1998
1999 bool final = (i == num_frags - 1);
2000
2001 lepp_cmd_t cmd = {
2002 .cpa_lo = frags[i].cpa_lo,
2003 .cpa_hi = frags[i].cpa_hi,
2004 .length = frags[i].length,
2005 .hash_for_home = frags[i].hash_for_home,
2006 .send_completion = final,
2007 .end_of_packet = final
2008 };
2009
2010 if (i == 0 && skb->ip_summed == CHECKSUM_PARTIAL) {
2011 cmd.compute_checksum = 1;
2012 cmd.checksum_data.bits.start_byte = csum_start;
2013 cmd.checksum_data.bits.count = len - csum_start;
2014 cmd.checksum_data.bits.destination_byte =
2015 csum_start + skb->csum_offset;
2016 }
2017
2018 cmds[i] = cmd;
2019 }
2020
2021
2022
2023 prefetch_L1(&eq->comp_tail);
2024 prefetch_L1(&eq->cmd_tail);
2025 mb();
2026
2027
2028
2029
2030 spin_lock_irqsave(&priv->eq_lock, irqflags);
2031
2032
2033
2034 if (lepp_num_free_comp_slots(eq) == 0) {
2035 nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
2036 if (nolds == 0) {
2037busy:
2038 spin_unlock_irqrestore(&priv->eq_lock, irqflags);
2039 return NETDEV_TX_BUSY;
2040 }
2041 }
2042
2043 cmd_head = eq->cmd_head;
2044 cmd_tail = eq->cmd_tail;
2045
2046
2047
2048 for (i = 0; i < num_frags; i++) {
2049
2050
2051 cmd_next = cmd_tail + cmd_size;
2052 if (cmd_tail < cmd_head && cmd_next >= cmd_head)
2053 goto busy;
2054 if (cmd_next > LEPP_CMD_LIMIT) {
2055 cmd_next = 0;
2056 if (cmd_next == cmd_head)
2057 goto busy;
2058 }
2059
2060
2061 *(lepp_cmd_t *)&eq->cmds[cmd_tail] = cmds[i];
2062
2063
2064 cmd_tail = cmd_next;
2065 }
2066
2067
2068 comp_tail = eq->comp_tail;
2069 eq->comps[comp_tail] = skb;
2070 LEPP_QINC(comp_tail);
2071 eq->comp_tail = comp_tail;
2072
2073
2074
2075 __insn_mf();
2076
2077 eq->cmd_tail = cmd_tail;
2078
2079
2080
2081
2082 if (nolds == 0)
2083 nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 4);
2084
2085 spin_unlock_irqrestore(&priv->eq_lock, irqflags);
2086
2087
2088 for (i = 0; i < nolds; i++)
2089 kfree_skb(olds[i]);
2090
2091
2092 stats->tx_packets++;
2093 stats->tx_bytes += ((len >= ETH_ZLEN) ? len : ETH_ZLEN);
2094
2095
2096 tile_net_schedule_egress_timer(info);
2097
2098 return NETDEV_TX_OK;
2099}
2100
2101
2102
2103
2104
2105static void tile_net_tx_timeout(struct net_device *dev)
2106{
2107 PDEBUG("tile_net_tx_timeout()\n");
2108 PDEBUG("Transmit timeout at %ld, latency %ld\n", jiffies,
2109 jiffies - dev->trans_start);
2110
2111
2112 netif_wake_queue(dev);
2113}
2114
2115
2116
2117
2118
2119static int tile_net_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
2120{
2121 return -EOPNOTSUPP;
2122}
2123
2124
2125
2126
2127
2128
2129
2130static struct net_device_stats *tile_net_get_stats(struct net_device *dev)
2131{
2132 struct tile_net_priv *priv = netdev_priv(dev);
2133 u32 rx_packets = 0;
2134 u32 tx_packets = 0;
2135 u32 rx_bytes = 0;
2136 u32 tx_bytes = 0;
2137 int i;
2138
2139 for_each_online_cpu(i) {
2140 if (priv->cpu[i]) {
2141 rx_packets += priv->cpu[i]->stats.rx_packets;
2142 rx_bytes += priv->cpu[i]->stats.rx_bytes;
2143 tx_packets += priv->cpu[i]->stats.tx_packets;
2144 tx_bytes += priv->cpu[i]->stats.tx_bytes;
2145 }
2146 }
2147
2148 priv->stats.rx_packets = rx_packets;
2149 priv->stats.rx_bytes = rx_bytes;
2150 priv->stats.tx_packets = tx_packets;
2151 priv->stats.tx_bytes = tx_bytes;
2152
2153 return &priv->stats;
2154}
2155
2156
2157
2158
2159
2160
2161
2162
2163static int tile_net_change_mtu(struct net_device *dev, int new_mtu)
2164{
2165 PDEBUG("tile_net_change_mtu()\n");
2166
2167
2168 if ((new_mtu < 68) || (new_mtu > 1500))
2169 return -EINVAL;
2170
2171
2172 dev->mtu = new_mtu;
2173
2174 return 0;
2175}
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189static int tile_net_set_mac_address(struct net_device *dev, void *p)
2190{
2191 struct sockaddr *addr = p;
2192
2193 if (!is_valid_ether_addr(addr->sa_data))
2194 return -EADDRNOTAVAIL;
2195
2196
2197 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2198 dev->addr_assign_type &= ~NET_ADDR_RANDOM;
2199
2200 return 0;
2201}
2202
2203
2204
2205
2206
2207
2208static int tile_net_get_mac(struct net_device *dev)
2209{
2210 struct tile_net_priv *priv = netdev_priv(dev);
2211
2212 char hv_dev_name[32];
2213 int len;
2214
2215 __netio_getset_offset_t offset = { .word = NETIO_IPP_PARAM_OFF };
2216
2217 int ret;
2218
2219
2220 strcpy(hv_dev_name, dev->name);
2221 len = strlen(hv_dev_name);
2222
2223
2224 hv_dev_name[len] = hv_dev_name[len - 1];
2225 hv_dev_name[len - 1] = '/';
2226 len++;
2227
2228
2229 strcpy(hv_dev_name + len, hash_default ? "/native_hash" : "/native");
2230
2231
2232 priv->hv_devhdl = hv_dev_open((HV_VirtAddr)hv_dev_name, 0);
2233 PDEBUG("hv_dev_open(%s) returned %d %p\n",
2234 hv_dev_name, priv->hv_devhdl, &priv->hv_devhdl);
2235 if (priv->hv_devhdl < 0) {
2236 if (priv->hv_devhdl == HV_ENODEV)
2237 printk(KERN_DEBUG "Ignoring unconfigured device %s\n",
2238 hv_dev_name);
2239 else
2240 printk(KERN_DEBUG "hv_dev_open(%s) returned %d\n",
2241 hv_dev_name, priv->hv_devhdl);
2242 return -1;
2243 }
2244
2245
2246
2247
2248
2249 offset.bits.class = NETIO_PARAM;
2250 offset.bits.addr = NETIO_PARAM_MAC;
2251 ret = hv_dev_pread(priv->hv_devhdl, 0,
2252 (HV_VirtAddr)dev->dev_addr, dev->addr_len,
2253 offset.word);
2254 PDEBUG("hv_dev_pread(NETIO_PARAM_MAC) returned %d\n", ret);
2255 if (ret <= 0) {
2256 printk(KERN_DEBUG "hv_dev_pread(NETIO_PARAM_MAC) %s failed\n",
2257 dev->name);
2258
2259
2260
2261
2262
2263 eth_hw_addr_random(dev);
2264 }
2265
2266 return 0;
2267}
2268
2269
2270#ifdef CONFIG_NET_POLL_CONTROLLER
2271
2272
2273
2274
2275
2276static void tile_net_netpoll(struct net_device *dev)
2277{
2278 struct tile_net_priv *priv = netdev_priv(dev);
2279 disable_percpu_irq(priv->intr_id);
2280 tile_net_handle_ingress_interrupt(priv->intr_id, dev);
2281 enable_percpu_irq(priv->intr_id, 0);
2282}
2283#endif
2284
2285
2286static const struct net_device_ops tile_net_ops = {
2287 .ndo_open = tile_net_open,
2288 .ndo_stop = tile_net_stop,
2289 .ndo_start_xmit = tile_net_tx,
2290 .ndo_do_ioctl = tile_net_ioctl,
2291 .ndo_get_stats = tile_net_get_stats,
2292 .ndo_change_mtu = tile_net_change_mtu,
2293 .ndo_tx_timeout = tile_net_tx_timeout,
2294 .ndo_set_mac_address = tile_net_set_mac_address,
2295#ifdef CONFIG_NET_POLL_CONTROLLER
2296 .ndo_poll_controller = tile_net_netpoll,
2297#endif
2298};
2299
2300
2301
2302
2303
2304
2305
2306
2307static void tile_net_setup(struct net_device *dev)
2308{
2309 PDEBUG("tile_net_setup()\n");
2310
2311 ether_setup(dev);
2312
2313 dev->netdev_ops = &tile_net_ops;
2314
2315 dev->watchdog_timeo = TILE_NET_TIMEOUT;
2316
2317
2318 dev->features |= NETIF_F_LLTX;
2319
2320
2321 dev->features |= NETIF_F_HW_CSUM;
2322
2323
2324 dev->features |= NETIF_F_SG;
2325
2326
2327 dev->features |= NETIF_F_TSO;
2328
2329#ifdef TILE_NET_GSO
2330
2331 dev->features |= NETIF_F_GSO;
2332#endif
2333
2334 if (hash_default)
2335 dev->features |= NETIF_F_HIGHDMA;
2336
2337
2338
2339 dev->tx_queue_len = TILE_NET_TX_QUEUE_LEN;
2340
2341 dev->mtu = TILE_NET_MTU;
2342}
2343
2344
2345
2346
2347
2348
2349static struct net_device *tile_net_dev_init(const char *name)
2350{
2351 int ret;
2352 struct net_device *dev;
2353 struct tile_net_priv *priv;
2354
2355
2356
2357
2358
2359
2360 dev = alloc_netdev(sizeof(*priv), name, tile_net_setup);
2361 if (!dev) {
2362 pr_err("alloc_netdev(%s) failed\n", name);
2363 return NULL;
2364 }
2365
2366 priv = netdev_priv(dev);
2367
2368
2369
2370 memset(priv, 0, sizeof(*priv));
2371
2372
2373 priv->dev = dev;
2374
2375 INIT_DELAYED_WORK(&priv->retry_work, tile_net_open_retry);
2376
2377 spin_lock_init(&priv->eq_lock);
2378
2379
2380 priv->eq_pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, EQ_ORDER);
2381 if (!priv->eq_pages) {
2382 free_netdev(dev);
2383 return NULL;
2384 }
2385 priv->eq = page_address(priv->eq_pages);
2386
2387
2388 ret = register_netdev(dev);
2389 if (ret) {
2390 pr_err("register_netdev %s failed %d\n", dev->name, ret);
2391 __free_pages(priv->eq_pages, EQ_ORDER);
2392 free_netdev(dev);
2393 return NULL;
2394 }
2395
2396
2397 ret = tile_net_get_mac(dev);
2398 if (ret < 0) {
2399 unregister_netdev(dev);
2400 __free_pages(priv->eq_pages, EQ_ORDER);
2401 free_netdev(dev);
2402 return NULL;
2403 }
2404
2405 return dev;
2406}
2407
2408
2409
2410
2411
2412
2413
2414
2415static void tile_net_cleanup(void)
2416{
2417 int i;
2418
2419 for (i = 0; i < TILE_NET_DEVS; i++) {
2420 if (tile_net_devs[i]) {
2421 struct net_device *dev = tile_net_devs[i];
2422 struct tile_net_priv *priv = netdev_priv(dev);
2423 unregister_netdev(dev);
2424 finv_buffer_remote(priv->eq, EQ_SIZE, 0);
2425 __free_pages(priv->eq_pages, EQ_ORDER);
2426 free_netdev(dev);
2427 }
2428 }
2429}
2430
2431
2432
2433
2434
2435static int tile_net_init_module(void)
2436{
2437 pr_info("Tilera Network Driver\n");
2438
2439 tile_net_devs[0] = tile_net_dev_init("xgbe0");
2440 tile_net_devs[1] = tile_net_dev_init("xgbe1");
2441 tile_net_devs[2] = tile_net_dev_init("gbe0");
2442 tile_net_devs[3] = tile_net_dev_init("gbe1");
2443
2444 return 0;
2445}
2446
2447
2448module_init(tile_net_init_module);
2449module_exit(tile_net_cleanup);
2450
2451
2452#ifndef MODULE
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462static int __init network_cpus_setup(char *str)
2463{
2464 int rc = cpulist_parse_crop(str, &network_cpus_map);
2465 if (rc != 0) {
2466 pr_warning("network_cpus=%s: malformed cpu list\n",
2467 str);
2468 } else {
2469
2470
2471 cpumask_and(&network_cpus_map, &network_cpus_map,
2472 cpu_possible_mask);
2473
2474
2475 if (cpumask_empty(&network_cpus_map)) {
2476 pr_warning("Ignoring network_cpus='%s'.\n",
2477 str);
2478 } else {
2479 char buf[1024];
2480 cpulist_scnprintf(buf, sizeof(buf), &network_cpus_map);
2481 pr_info("Linux network CPUs: %s\n", buf);
2482 network_cpus_used = true;
2483 }
2484 }
2485
2486 return 0;
2487}
2488__setup("network_cpus=", network_cpus_setup);
2489
2490#endif
2491