1
2
3
4
5
6
7
8
9
10
11#ifndef _ASM_X86_UV_UV_BAU_H
12#define _ASM_X86_UV_UV_BAU_H
13
14#include <linux/bitmap.h>
15#define BITSPERBYTE 8
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36#define MAX_CPUS_PER_UVHUB 128
37#define MAX_CPUS_PER_SOCKET 64
38#define ADP_SZ 64
39#define UV_CPUS_PER_AS 32
40#define ITEMS_PER_DESC 8
41
42#define MAX_BAU_CONCURRENT 3
43#define UV_ACT_STATUS_MASK 0x3
44#define UV_ACT_STATUS_SIZE 2
45#define UV_DISTRIBUTION_SIZE 256
46#define UV_SW_ACK_NPENDING 8
47#define UV1_NET_ENDPOINT_INTD 0x38
48#define UV2_NET_ENDPOINT_INTD 0x28
49#define UV_NET_ENDPOINT_INTD (is_uv1_hub() ? \
50 UV1_NET_ENDPOINT_INTD : UV2_NET_ENDPOINT_INTD)
51#define UV_DESC_PSHIFT 49
52#define UV_PAYLOADQ_GNODE_SHIFT 49
53#define UV_PTC_BASENAME "sgi_uv/ptc_statistics"
54#define UV_BAU_BASENAME "sgi_uv/bau_tunables"
55#define UV_BAU_TUNABLES_DIR "sgi_uv"
56#define UV_BAU_TUNABLES_FILE "bau_tunables"
57#define WHITESPACE " \t\n"
58#define cpubit_isset(cpu, bau_local_cpumask) \
59 test_bit((cpu), (bau_local_cpumask).bits)
60
61
62
63
64
65
66
67
68#define UV1_INTD_SOFT_ACK_TIMEOUT_PERIOD (9UL)
69#define UV2_INTD_SOFT_ACK_TIMEOUT_PERIOD (15UL)
70
71#define UV_INTD_SOFT_ACK_TIMEOUT_PERIOD (is_uv1_hub() ? \
72 UV1_INTD_SOFT_ACK_TIMEOUT_PERIOD : \
73 UV2_INTD_SOFT_ACK_TIMEOUT_PERIOD)
74
75
76#define BAU_MISC_CONTROL_MULT_MASK 3
77
78#define UVH_AGING_PRESCALE_SEL 0x000000b000UL
79
80#define BAU_URGENCY_7_SHIFT 28
81#define BAU_URGENCY_7_MASK 7
82
83#define UVH_TRANSACTION_TIMEOUT 0x000000b200UL
84
85#define BAU_TRANS_SHIFT 40
86#define BAU_TRANS_MASK 0x3f
87
88
89
90
91#define AS_PUSH_SHIFT UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT
92#define SOFTACK_MSHIFT UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT
93#define SOFTACK_PSHIFT UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT
94#define SOFTACK_TIMEOUT_PERIOD UV_INTD_SOFT_ACK_TIMEOUT_PERIOD
95#define PREFETCH_HINT_SHFT UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_PREFETCH_HINT_SHFT
96#define SB_STATUS_SHFT UV3H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT
97#define write_gmmr uv_write_global_mmr64
98#define write_lmmr uv_write_local_mmr
99#define read_lmmr uv_read_local_mmr
100#define read_gmmr uv_read_global_mmr64
101
102
103
104
105#define DS_IDLE 0
106#define DS_ACTIVE 1
107#define DS_DESTINATION_TIMEOUT 2
108#define DS_SOURCE_TIMEOUT 3
109
110
111
112
113
114
115
116
117
118
119
120
121#define UV2H_DESC_IDLE 0
122#define UV2H_DESC_BUSY 2
123#define UV2H_DESC_DEST_TIMEOUT 4
124#define UV2H_DESC_DEST_STRONG_NACK 5
125#define UV2H_DESC_SOURCE_TIMEOUT 6
126#define UV2H_DESC_DEST_PUT_ERR 7
127
128
129
130
131#define PLUGGED_DELAY 10
132
133
134
135
136
137#define PLUGSB4RESET 100
138
139#define TIMEOUTSB4RESET 1
140
141#define IPI_RESET_LIMIT 1
142
143#define COMPLETE_THRESHOLD 5
144
145
146#define GIVEUP_LIMIT 100
147
148#define UV_LB_SUBNODEID 0x10
149
150
151#define UV_SA_SHFT UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT
152#define UV_SA_MASK UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK
153
154#define UV2_ACK_MASK 0x7UL
155#define UV2_ACK_UNITS_SHFT 3
156#define UV2_EXT_SHFT UV2H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT
157
158
159
160
161#define DEST_Q_SIZE 20
162
163
164
165#define DEST_NUM_RESOURCES 8
166
167
168
169#define FLUSH_RETRY_PLUGGED 1
170#define FLUSH_RETRY_TIMEOUT 2
171#define FLUSH_GIVEUP 3
172#define FLUSH_COMPLETE 4
173
174
175
176
177#define CONGESTED_RESPONSE_US 1000
178
179#define CONGESTED_REPS 10
180
181#define DISABLED_PERIOD 10
182
183
184#define MSG_NOOP 0
185#define MSG_REGULAR 1
186#define MSG_RETRY 2
187
188#define BAU_DESC_QUALIFIER 0x534749
189
190enum uv_bau_version {
191 UV_BAU_V1 = 1,
192 UV_BAU_V2,
193 UV_BAU_V3,
194 UV_BAU_V4,
195};
196
197
198
199
200
201
202
203
204
205
206
207struct pnmask {
208 unsigned long bits[BITS_TO_LONGS(UV_DISTRIBUTION_SIZE)];
209};
210
211
212
213
214
215
216struct bau_local_cpumask {
217 unsigned long bits;
218};
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240struct uv1_2_3_bau_msg_payload {
241 u64 address;
242 u16 sending_cpu;
243 u16 acknowledge_count;
244};
245
246
247
248
249
250
251
252
253struct uv4_bau_msg_payload {
254 u64 address;
255 u16 sending_cpu;
256 u16 acknowledge_count;
257 u32 reserved:8;
258 u32 qualifier:24;
259};
260
261
262
263
264
265struct uv1_bau_msg_header {
266 unsigned int dest_subnodeid:6;
267
268 unsigned int base_dest_nasid:15;
269
270 unsigned int command:8;
271
272
273 unsigned int rsvd_1:3;
274
275
276 unsigned int rsvd_2:9;
277
278
279 unsigned int sequence:16;
280
281
282
283
284
285
286 unsigned int rsvd_3:1;
287
288
289
290
291 unsigned int replied_to:1;
292
293
294 unsigned int msg_type:3;
295
296
297 unsigned int canceled:1;
298
299
300 unsigned int payload_1a:1;
301
302 unsigned int payload_1b:2;
303
304
305
306 unsigned int payload_1ca:6;
307
308 unsigned int payload_1c:2;
309
310
311
312 unsigned int payload_1d:6;
313
314 unsigned int payload_1e:2;
315
316
317 unsigned int rsvd_4:7;
318
319 unsigned int swack_flag:1;
320
321
322
323
324 unsigned int rsvd_5:6;
325
326 unsigned int rsvd_6:5;
327
328 unsigned int int_both:1;
329
330
331 unsigned int fairness:3;
332
333 unsigned int multilevel:1;
334
335
336
337 unsigned int chaining:1;
338
339
340 unsigned int rsvd_7:21;
341
342};
343
344
345
346
347
348
349struct uv2_3_bau_msg_header {
350 unsigned int base_dest_nasid:15;
351
352 unsigned int dest_subnodeid:5;
353
354 unsigned int rsvd_1:1;
355
356
357
358
359
360 unsigned int replied_to:1;
361
362
363 unsigned int msg_type:3;
364
365
366 unsigned int canceled:1;
367
368
369 unsigned int payload_1:3;
370
371
372
373 unsigned int payload_2a:3;
374 unsigned int payload_2b:5;
375
376
377
378 unsigned int payload_3:8;
379
380
381 unsigned int rsvd_2:7;
382
383 unsigned int swack_flag:1;
384
385 unsigned int rsvd_3a:3;
386 unsigned int rsvd_3b:8;
387 unsigned int rsvd_3c:8;
388 unsigned int rsvd_3d:3;
389
390 unsigned int fairness:3;
391
392
393 unsigned int sequence:16;
394
395 unsigned int chaining:1;
396
397
398 unsigned int multilevel:1;
399
400
401 unsigned int rsvd_4:24;
402
403
404
405 unsigned int command:8;
406
407};
408
409
410
411
412
413
414struct bau_desc {
415 struct pnmask distribution;
416
417
418
419 union bau_msg_header {
420 struct uv1_bau_msg_header uv1_hdr;
421 struct uv2_3_bau_msg_header uv2_3_hdr;
422 } header;
423
424 union bau_payload_header {
425 struct uv1_2_3_bau_msg_payload uv1_2_3;
426 struct uv4_bau_msg_payload uv4;
427 } payload;
428};
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461struct bau_pq_entry {
462 unsigned long address;
463
464
465 unsigned short sending_cpu;
466
467 unsigned short acknowledge_count;
468
469
470 unsigned short replied_to:1;
471 unsigned short msg_type:3;
472 unsigned short canceled:1;
473 unsigned short unused1:3;
474
475 unsigned char unused2a;
476
477 unsigned char unused2;
478
479 unsigned char swack_vec;
480
481 unsigned short sequence;
482
483 unsigned char unused4[2];
484
485 int number_of_cpus;
486
487 unsigned char unused5[8];
488
489};
490
491struct msg_desc {
492 struct bau_pq_entry *msg;
493 int msg_slot;
494 struct bau_pq_entry *queue_first;
495 struct bau_pq_entry *queue_last;
496};
497
498struct reset_args {
499 int sender;
500};
501
502
503
504
505struct ptc_stats {
506
507 unsigned long s_giveup;
508
509 unsigned long s_requestor;
510
511 unsigned long s_stimeout;
512 unsigned long s_dtimeout;
513 unsigned long s_strongnacks;
514 unsigned long s_time;
515 unsigned long s_retriesok;
516 unsigned long s_ntargcpu;
517
518 unsigned long s_ntargself;
519
520 unsigned long s_ntarglocals;
521
522 unsigned long s_ntargremotes;
523
524 unsigned long s_ntarglocaluvhub;
525 unsigned long s_ntargremoteuvhub;
526 unsigned long s_ntarguvhub;
527
528 unsigned long s_ntarguvhub16;
529
530 unsigned long s_ntarguvhub8;
531
532 unsigned long s_ntarguvhub4;
533
534 unsigned long s_ntarguvhub2;
535
536 unsigned long s_ntarguvhub1;
537
538 unsigned long s_resets_plug;
539
540 unsigned long s_resets_timeout;
541
542 unsigned long s_busy;
543
544 unsigned long s_throttles;
545 unsigned long s_retry_messages;
546 unsigned long s_bau_reenabled;
547 unsigned long s_bau_disabled;
548 unsigned long s_uv2_wars;
549 unsigned long s_uv2_wars_hw;
550 unsigned long s_uv2_war_waits;
551 unsigned long s_overipilimit;
552 unsigned long s_giveuplimit;
553 unsigned long s_enters;
554 unsigned long s_ipifordisabled;
555 unsigned long s_plugged;
556 unsigned long s_congested;
557
558 unsigned long d_alltlb;
559
560 unsigned long d_onetlb;
561
562 unsigned long d_multmsg;
563
564 unsigned long d_nomsg;
565 unsigned long d_time;
566
567 unsigned long d_requestee;
568
569 unsigned long d_retries;
570
571 unsigned long d_canceled;
572
573 unsigned long d_nocanceled;
574
575 unsigned long d_resets;
576
577 unsigned long d_rcanceled;
578
579};
580
581struct tunables {
582 int *tunp;
583 int deflt;
584};
585
586struct hub_and_pnode {
587 short uvhub;
588 short pnode;
589};
590
591struct socket_desc {
592 short num_cpus;
593 short cpu_number[MAX_CPUS_PER_SOCKET];
594};
595
596struct uvhub_desc {
597 unsigned short socket_mask;
598 short num_cpus;
599 short uvhub;
600 short pnode;
601 struct socket_desc socket[2];
602};
603
604
605
606
607
608
609
610
611struct bau_control {
612 struct bau_desc *descriptor_base;
613 struct bau_pq_entry *queue_first;
614 struct bau_pq_entry *queue_last;
615 struct bau_pq_entry *bau_msg_head;
616 struct bau_control *uvhub_master;
617 struct bau_control *socket_master;
618 struct ptc_stats *statp;
619 cpumask_t *cpumask;
620 unsigned long timeout_interval;
621 unsigned long set_bau_on_time;
622 atomic_t active_descriptor_count;
623 int plugged_tries;
624 int timeout_tries;
625 int ipi_attempts;
626 int conseccompletes;
627 u64 status_mmr;
628 int status_index;
629 bool nobau;
630 short baudisabled;
631 short cpu;
632 short osnode;
633 short uvhub_cpu;
634 short uvhub;
635 short uvhub_version;
636 short cpus_in_socket;
637 short cpus_in_uvhub;
638 short partition_base_pnode;
639 short busy;
640 unsigned short message_number;
641 unsigned short uvhub_quiesce;
642 short socket_acknowledge_count[DEST_Q_SIZE];
643 cycles_t send_message;
644 cycles_t period_end;
645 cycles_t period_time;
646 spinlock_t uvhub_lock;
647 spinlock_t queue_lock;
648 spinlock_t disable_lock;
649
650 int max_concurr;
651 int max_concurr_const;
652 int plugged_delay;
653 int plugsb4reset;
654 int timeoutsb4reset;
655 int ipi_reset_limit;
656 int complete_threshold;
657 int cong_response_us;
658 int cong_reps;
659 cycles_t disabled_period;
660 int period_giveups;
661 int giveup_limit;
662 long period_requests;
663 struct hub_and_pnode *thp;
664};
665
666
667struct bau_operations {
668 unsigned long (*read_l_sw_ack)(void);
669 unsigned long (*read_g_sw_ack)(int pnode);
670 unsigned long (*bau_gpa_to_offset)(unsigned long vaddr);
671 void (*write_l_sw_ack)(unsigned long mmr);
672 void (*write_g_sw_ack)(int pnode, unsigned long mmr);
673 void (*write_payload_first)(int pnode, unsigned long mmr);
674 void (*write_payload_last)(int pnode, unsigned long mmr);
675 int (*wait_completion)(struct bau_desc*,
676 struct bau_control*, long try);
677};
678
679static inline void write_mmr_data_broadcast(int pnode, unsigned long mmr_image)
680{
681 write_gmmr(pnode, UVH_BAU_DATA_BROADCAST, mmr_image);
682}
683
684static inline void write_mmr_descriptor_base(int pnode, unsigned long mmr_image)
685{
686 write_gmmr(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE, mmr_image);
687}
688
689static inline void write_mmr_activation(unsigned long index)
690{
691 write_lmmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
692}
693
694static inline void write_gmmr_activation(int pnode, unsigned long mmr_image)
695{
696 write_gmmr(pnode, UVH_LB_BAU_SB_ACTIVATION_CONTROL, mmr_image);
697}
698
699static inline void write_mmr_proc_payload_first(int pnode, unsigned long mmr_image)
700{
701 write_gmmr(pnode, UV4H_LB_PROC_INTD_QUEUE_FIRST, mmr_image);
702}
703
704static inline void write_mmr_proc_payload_last(int pnode, unsigned long mmr_image)
705{
706 write_gmmr(pnode, UV4H_LB_PROC_INTD_QUEUE_LAST, mmr_image);
707}
708
709static inline void write_mmr_payload_first(int pnode, unsigned long mmr_image)
710{
711 write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, mmr_image);
712}
713
714static inline void write_mmr_payload_tail(int pnode, unsigned long mmr_image)
715{
716 write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, mmr_image);
717}
718
719static inline void write_mmr_payload_last(int pnode, unsigned long mmr_image)
720{
721 write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, mmr_image);
722}
723
724static inline void write_mmr_misc_control(int pnode, unsigned long mmr_image)
725{
726 write_gmmr(pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image);
727}
728
729static inline unsigned long read_mmr_misc_control(int pnode)
730{
731 return read_gmmr(pnode, UVH_LB_BAU_MISC_CONTROL);
732}
733
734static inline void write_mmr_sw_ack(unsigned long mr)
735{
736 uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, mr);
737}
738
739static inline void write_gmmr_sw_ack(int pnode, unsigned long mr)
740{
741 write_gmmr(pnode, UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, mr);
742}
743
744static inline unsigned long read_mmr_sw_ack(void)
745{
746 return read_lmmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
747}
748
749static inline unsigned long read_gmmr_sw_ack(int pnode)
750{
751 return read_gmmr(pnode, UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
752}
753
754static inline void write_mmr_proc_sw_ack(unsigned long mr)
755{
756 uv_write_local_mmr(UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR, mr);
757}
758
759static inline void write_gmmr_proc_sw_ack(int pnode, unsigned long mr)
760{
761 write_gmmr(pnode, UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR, mr);
762}
763
764static inline unsigned long read_mmr_proc_sw_ack(void)
765{
766 return read_lmmr(UV4H_LB_PROC_INTD_SOFT_ACK_PENDING);
767}
768
769static inline unsigned long read_gmmr_proc_sw_ack(int pnode)
770{
771 return read_gmmr(pnode, UV4H_LB_PROC_INTD_SOFT_ACK_PENDING);
772}
773
774static inline void write_mmr_data_config(int pnode, unsigned long mr)
775{
776 uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, mr);
777}
778
779static inline int bau_uvhub_isset(int uvhub, struct pnmask *dstp)
780{
781 return constant_test_bit(uvhub, &dstp->bits[0]);
782}
783static inline void bau_uvhub_set(int pnode, struct pnmask *dstp)
784{
785 __set_bit(pnode, &dstp->bits[0]);
786}
787static inline void bau_uvhubs_clear(struct pnmask *dstp,
788 int nbits)
789{
790 bitmap_zero(&dstp->bits[0], nbits);
791}
792static inline int bau_uvhub_weight(struct pnmask *dstp)
793{
794 return bitmap_weight((unsigned long *)&dstp->bits[0],
795 UV_DISTRIBUTION_SIZE);
796}
797
798static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits)
799{
800 bitmap_zero(&dstp->bits, nbits);
801}
802
803extern void uv_bau_message_intr1(void);
804#ifdef CONFIG_TRACING
805#define trace_uv_bau_message_intr1 uv_bau_message_intr1
806#endif
807extern void uv_bau_timeout_intr1(void);
808
809struct atomic_short {
810 short counter;
811};
812
813
814
815
816
817
818
819static inline int atomic_read_short(const struct atomic_short *v)
820{
821 return v->counter;
822}
823
824
825
826
827
828
829
830
831static inline int atom_asr(short i, struct atomic_short *v)
832{
833 short __i = i;
834 asm volatile(LOCK_PREFIX "xaddw %0, %1"
835 : "+r" (i), "+m" (v->counter)
836 : : "memory");
837 return i + __i;
838}
839
840
841
842
843
844
845
846
847
848
849
850static inline int atomic_inc_unless_ge(spinlock_t *lock, atomic_t *v, int u)
851{
852 spin_lock(lock);
853 if (atomic_read(v) >= u) {
854 spin_unlock(lock);
855 return 0;
856 }
857 atomic_inc(v);
858 spin_unlock(lock);
859 return 1;
860}
861
862#endif
863