1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/delay.h>
17#include <linux/slab.h>
18#include <asm/uncached.h>
19#include <asm/sn/mspec.h>
20#include <asm/sn/sn_sal.h>
21#include "xpc.h"
22
23
24
25
26
27
28
29#define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2)
30#define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
31#define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46#define XPC_NOTIFY_IRQ_AMOS_SN2 0
47#define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \
48 XP_MAX_NPARTITIONS_SN2)
49#define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \
50 XP_NASID_MASK_WORDS_SN2)
51#define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
52
53
54
55
56
57static void *xpc_remote_copy_buffer_base_sn2;
58static char *xpc_remote_copy_buffer_sn2;
59
60static struct xpc_vars_sn2 *xpc_vars_sn2;
61static struct xpc_vars_part_sn2 *xpc_vars_part_sn2;
62
63static int
64xpc_setup_partitions_sn2(void)
65{
66
67 return 0;
68}
69
70static void
71xpc_teardown_partitions_sn2(void)
72{
73
74}
75
76
77static u64 xpc_sh1_IPI_access_sn2;
78static u64 xpc_sh2_IPI_access0_sn2;
79static u64 xpc_sh2_IPI_access1_sn2;
80static u64 xpc_sh2_IPI_access2_sn2;
81static u64 xpc_sh2_IPI_access3_sn2;
82
83
84
85
86static void
87xpc_allow_IPI_ops_sn2(void)
88{
89 int node;
90 int nasid;
91
92
93 if (is_shub2()) {
94 xpc_sh2_IPI_access0_sn2 =
95 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0));
96 xpc_sh2_IPI_access1_sn2 =
97 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1));
98 xpc_sh2_IPI_access2_sn2 =
99 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2));
100 xpc_sh2_IPI_access3_sn2 =
101 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3));
102
103 for_each_online_node(node) {
104 nasid = cnodeid_to_nasid(node);
105 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
106 -1UL);
107 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
108 -1UL);
109 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
110 -1UL);
111 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
112 -1UL);
113 }
114 } else {
115 xpc_sh1_IPI_access_sn2 =
116 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS));
117
118 for_each_online_node(node) {
119 nasid = cnodeid_to_nasid(node);
120 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
121 -1UL);
122 }
123 }
124}
125
126
127
128
129static void
130xpc_disallow_IPI_ops_sn2(void)
131{
132 int node;
133 int nasid;
134
135
136 if (is_shub2()) {
137 for_each_online_node(node) {
138 nasid = cnodeid_to_nasid(node);
139 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
140 xpc_sh2_IPI_access0_sn2);
141 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
142 xpc_sh2_IPI_access1_sn2);
143 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
144 xpc_sh2_IPI_access2_sn2);
145 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
146 xpc_sh2_IPI_access3_sn2);
147 }
148 } else {
149 for_each_online_node(node) {
150 nasid = cnodeid_to_nasid(node);
151 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
152 xpc_sh1_IPI_access_sn2);
153 }
154 }
155}
156
157
158
159
160
161
162
163
164static u64
165xpc_receive_IRQ_amo_sn2(struct amo *amo)
166{
167 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR);
168}
169
170static enum xp_retval
171xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid,
172 int vector)
173{
174 int ret = 0;
175 unsigned long irq_flags;
176
177 local_irq_save(irq_flags);
178
179 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag);
180 sn_send_IPI_phys(nasid, phys_cpuid, vector, 0);
181
182
183
184
185
186
187
188 ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable),
189 xp_nofault_PIOR_target));
190
191 local_irq_restore(irq_flags);
192
193 return (ret == 0) ? xpSuccess : xpPioReadError;
194}
195
196static struct amo *
197xpc_init_IRQ_amo_sn2(int index)
198{
199 struct amo *amo = xpc_vars_sn2->amos_page + index;
200
201 (void)xpc_receive_IRQ_amo_sn2(amo);
202 return amo;
203}
204
205
206
207
208
209
210
211
212static irqreturn_t
213xpc_handle_activate_IRQ_sn2(int irq, void *dev_id)
214{
215 unsigned long irq_flags;
216
217 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
218 xpc_activate_IRQ_rcvd++;
219 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
220
221 wake_up_interruptible(&xpc_activate_IRQ_wq);
222 return IRQ_HANDLED;
223}
224
225
226
227
228static void
229xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid,
230 int to_nasid, int to_phys_cpuid)
231{
232 struct amo *amos = (struct amo *)__va(amos_page_pa +
233 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
234 sizeof(struct amo)));
235
236 (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)],
237 BIT_MASK(from_nasid / 2), to_nasid,
238 to_phys_cpuid, SGI_XPC_ACTIVATE);
239}
240
241static void
242xpc_send_local_activate_IRQ_sn2(int from_nasid)
243{
244 unsigned long irq_flags;
245 struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa +
246 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
247 sizeof(struct amo)));
248
249
250 FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable),
251 FETCHOP_OR, BIT_MASK(from_nasid / 2));
252
253 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
254 xpc_activate_IRQ_rcvd++;
255 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
256
257 wake_up_interruptible(&xpc_activate_IRQ_wq);
258}
259
260
261
262
263
264
265
266
267static void
268xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part)
269{
270 union xpc_channel_ctl_flags chctl;
271 unsigned long irq_flags;
272
273 chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2.
274 local_chctl_amo_va);
275 if (chctl.all_flags == 0)
276 return;
277
278 spin_lock_irqsave(&part->chctl_lock, irq_flags);
279 part->chctl.all_flags |= chctl.all_flags;
280 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
281
282 dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags="
283 "0x%llx\n", XPC_PARTID(part), chctl.all_flags);
284
285 xpc_wakeup_channel_mgr(part);
286}
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304static irqreturn_t
305xpc_handle_notify_IRQ_sn2(int irq, void *dev_id)
306{
307 short partid = (short)(u64)dev_id;
308 struct xpc_partition *part = &xpc_partitions[partid];
309
310 DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2);
311
312 if (xpc_part_ref(part)) {
313 xpc_check_for_sent_chctl_flags_sn2(part);
314
315 xpc_part_deref(part);
316 }
317 return IRQ_HANDLED;
318}
319
320
321
322
323
324
325static void
326xpc_check_for_dropped_notify_IRQ_sn2(struct timer_list *t)
327{
328 struct xpc_partition *part =
329 from_timer(part, t, sn.sn2.dropped_notify_IRQ_timer);
330
331 if (xpc_part_ref(part)) {
332 xpc_check_for_sent_chctl_flags_sn2(part);
333
334 t->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
335 add_timer(t);
336 xpc_part_deref(part);
337 }
338}
339
340
341
342
343
344static void
345xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
346 char *chctl_flag_string, unsigned long *irq_flags)
347{
348 struct xpc_partition *part = &xpc_partitions[ch->partid];
349 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
350 union xpc_channel_ctl_flags chctl = { 0 };
351 enum xp_retval ret;
352
353 if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) {
354 chctl.flags[ch->number] = chctl_flag;
355 ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va,
356 chctl.all_flags,
357 part_sn2->notify_IRQ_nasid,
358 part_sn2->notify_IRQ_phys_cpuid,
359 SGI_XPC_NOTIFY);
360 dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n",
361 chctl_flag_string, ch->partid, ch->number, ret);
362 if (unlikely(ret != xpSuccess)) {
363 if (irq_flags != NULL)
364 spin_unlock_irqrestore(&ch->lock, *irq_flags);
365 XPC_DEACTIVATE_PARTITION(part, ret);
366 if (irq_flags != NULL)
367 spin_lock_irqsave(&ch->lock, *irq_flags);
368 }
369 }
370}
371
372#define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \
373 xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f)
374
375
376
377
378
379
380static void
381xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
382 char *chctl_flag_string)
383{
384 struct xpc_partition *part = &xpc_partitions[ch->partid];
385 union xpc_channel_ctl_flags chctl = { 0 };
386
387 chctl.flags[ch->number] = chctl_flag;
388 FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va->
389 variable), FETCHOP_OR, chctl.all_flags);
390 dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n",
391 chctl_flag_string, ch->partid, ch->number);
392}
393
394#define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \
395 xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f)
396
397static void
398xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch,
399 unsigned long *irq_flags)
400{
401 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
402
403 args->reason = ch->reason;
404 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags);
405}
406
407static void
408xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
409{
410 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags);
411}
412
413static void
414xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
415{
416 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
417
418 args->entry_size = ch->entry_size;
419 args->local_nentries = ch->local_nentries;
420 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags);
421}
422
423static void
424xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
425{
426 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
427
428 args->remote_nentries = ch->remote_nentries;
429 args->local_nentries = ch->local_nentries;
430 args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue);
431 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags);
432}
433
434static void
435xpc_send_chctl_opencomplete_sn2(struct xpc_channel *ch,
436 unsigned long *irq_flags)
437{
438 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENCOMPLETE, irq_flags);
439}
440
441static void
442xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch)
443{
444 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL);
445}
446
447static void
448xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch)
449{
450 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
451}
452
453static enum xp_retval
454xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
455 unsigned long msgqueue_pa)
456{
457 ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
458 return xpSuccess;
459}
460
461
462
463
464
465
466static void
467xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
468{
469 unsigned long irq_flags;
470 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
471 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
472 sizeof(struct amo)));
473
474 local_irq_save(irq_flags);
475
476
477 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
478 BIT(sn_partition_id));
479
480
481
482
483
484
485
486 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
487 variable),
488 xp_nofault_PIOR_target));
489
490 local_irq_restore(irq_flags);
491}
492
493static void
494xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
495{
496 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
497 unsigned long irq_flags;
498 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
499 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
500 sizeof(struct amo)));
501
502 local_irq_save(irq_flags);
503
504
505 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
506 ~BIT(sn_partition_id));
507
508
509
510
511
512
513
514 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
515 variable),
516 xp_nofault_PIOR_target));
517
518 local_irq_restore(irq_flags);
519
520
521
522
523
524 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
525 cnodeid_to_nasid(0),
526 part_sn2->activate_IRQ_nasid,
527 part_sn2->activate_IRQ_phys_cpuid);
528}
529
530static void
531xpc_assume_partition_disengaged_sn2(short partid)
532{
533 struct amo *amo = xpc_vars_sn2->amos_page +
534 XPC_ENGAGED_PARTITIONS_AMO_SN2;
535
536
537 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
538 ~BIT(partid));
539}
540
541static int
542xpc_partition_engaged_sn2(short partid)
543{
544 struct amo *amo = xpc_vars_sn2->amos_page +
545 XPC_ENGAGED_PARTITIONS_AMO_SN2;
546
547
548 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
549 BIT(partid)) != 0;
550}
551
552static int
553xpc_any_partition_engaged_sn2(void)
554{
555 struct amo *amo = xpc_vars_sn2->amos_page +
556 XPC_ENGAGED_PARTITIONS_AMO_SN2;
557
558
559 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
560}
561
562
563static u64 xpc_prot_vec_sn2[MAX_NUMNODES];
564
565
566
567
568static enum xp_retval
569xpc_allow_amo_ops_sn2(struct amo *amos_page)
570{
571 enum xp_retval ret = xpSuccess;
572
573
574
575
576
577
578 if (!enable_shub_wars_1_1())
579 ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE);
580
581 return ret;
582}
583
584
585
586
587static void
588xpc_allow_amo_ops_shub_wars_1_1_sn2(void)
589{
590 int node;
591 int nasid;
592
593 if (!enable_shub_wars_1_1())
594 return;
595
596 for_each_online_node(node) {
597 nasid = cnodeid_to_nasid(node);
598
599 xpc_prot_vec_sn2[node] =
600 (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid,
601 SH1_MD_DQLP_MMR_DIR_PRIVEC0));
602
603 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
604 SH1_MD_DQLP_MMR_DIR_PRIVEC0),
605 -1UL);
606 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
607 SH1_MD_DQRP_MMR_DIR_PRIVEC0),
608 -1UL);
609 }
610}
611
612static enum xp_retval
613xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa,
614 size_t *len)
615{
616 s64 status;
617 enum xp_retval ret;
618
619 status = sn_partition_reserved_page_pa((u64)buf, cookie,
620 (u64 *)rp_pa, (u64 *)len);
621 if (status == SALRET_OK)
622 ret = xpSuccess;
623 else if (status == SALRET_MORE_PASSES)
624 ret = xpNeedMoreInfo;
625 else
626 ret = xpSalError;
627
628 return ret;
629}
630
631
632static int
633xpc_setup_rsvd_page_sn2(struct xpc_rsvd_page *rp)
634{
635 struct amo *amos_page;
636 int i;
637 int ret;
638
639 xpc_vars_sn2 = XPC_RP_VARS(rp);
640
641 rp->sn.sn2.vars_pa = xp_pa(xpc_vars_sn2);
642
643
644 xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) +
645 XPC_RP_VARS_SIZE);
646
647
648
649
650
651
652
653
654
655
656
657
658
659 amos_page = xpc_vars_sn2->amos_page;
660 if (amos_page == NULL) {
661 amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1));
662 if (amos_page == NULL) {
663 dev_err(xpc_part, "can't allocate page of amos\n");
664 return -ENOMEM;
665 }
666
667
668
669
670
671 ret = xpc_allow_amo_ops_sn2(amos_page);
672 if (ret != xpSuccess) {
673 dev_err(xpc_part, "can't allow amo operations\n");
674 uncached_free_page(__IA64_UNCACHED_OFFSET |
675 TO_PHYS((u64)amos_page), 1);
676 return -EPERM;
677 }
678 }
679
680
681 memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2));
682
683 xpc_vars_sn2->version = XPC_V_VERSION;
684 xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0);
685 xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0);
686 xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2);
687 xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page);
688 xpc_vars_sn2->amos_page = amos_page;
689
690
691 memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) *
692 XP_MAX_NPARTITIONS_SN2);
693
694
695 for (i = 0; i < xpc_nasid_mask_nlongs; i++)
696 (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i);
697
698
699 (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2);
700 (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2);
701
702 return 0;
703}
704
705static int
706xpc_hb_allowed_sn2(short partid, void *heartbeating_to_mask)
707{
708 return test_bit(partid, heartbeating_to_mask);
709}
710
711static void
712xpc_allow_hb_sn2(short partid)
713{
714 DBUG_ON(xpc_vars_sn2 == NULL);
715 set_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
716}
717
718static void
719xpc_disallow_hb_sn2(short partid)
720{
721 DBUG_ON(xpc_vars_sn2 == NULL);
722 clear_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
723}
724
725static void
726xpc_disallow_all_hbs_sn2(void)
727{
728 DBUG_ON(xpc_vars_sn2 == NULL);
729 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, xp_max_npartitions);
730}
731
732static void
733xpc_increment_heartbeat_sn2(void)
734{
735 xpc_vars_sn2->heartbeat++;
736}
737
738static void
739xpc_offline_heartbeat_sn2(void)
740{
741 xpc_increment_heartbeat_sn2();
742 xpc_vars_sn2->heartbeat_offline = 1;
743}
744
745static void
746xpc_online_heartbeat_sn2(void)
747{
748 xpc_increment_heartbeat_sn2();
749 xpc_vars_sn2->heartbeat_offline = 0;
750}
751
752static void
753xpc_heartbeat_init_sn2(void)
754{
755 DBUG_ON(xpc_vars_sn2 == NULL);
756
757 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2);
758 xpc_online_heartbeat_sn2();
759}
760
761static void
762xpc_heartbeat_exit_sn2(void)
763{
764 xpc_offline_heartbeat_sn2();
765}
766
767static enum xp_retval
768xpc_get_remote_heartbeat_sn2(struct xpc_partition *part)
769{
770 struct xpc_vars_sn2 *remote_vars;
771 enum xp_retval ret;
772
773 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
774
775
776 ret = xp_remote_memcpy(xp_pa(remote_vars),
777 part->sn.sn2.remote_vars_pa,
778 XPC_RP_VARS_SIZE);
779 if (ret != xpSuccess)
780 return ret;
781
782 dev_dbg(xpc_part, "partid=%d, heartbeat=%lld, last_heartbeat=%lld, "
783 "heartbeat_offline=%lld, HB_mask[0]=0x%lx\n", XPC_PARTID(part),
784 remote_vars->heartbeat, part->last_heartbeat,
785 remote_vars->heartbeat_offline,
786 remote_vars->heartbeating_to_mask[0]);
787
788 if ((remote_vars->heartbeat == part->last_heartbeat &&
789 !remote_vars->heartbeat_offline) ||
790 !xpc_hb_allowed_sn2(sn_partition_id,
791 remote_vars->heartbeating_to_mask)) {
792 ret = xpNoHeartbeat;
793 } else {
794 part->last_heartbeat = remote_vars->heartbeat;
795 }
796
797 return ret;
798}
799
800
801
802
803
804
805
806static enum xp_retval
807xpc_get_remote_vars_sn2(unsigned long remote_vars_pa,
808 struct xpc_vars_sn2 *remote_vars)
809{
810 enum xp_retval ret;
811
812 if (remote_vars_pa == 0)
813 return xpVarsNotSet;
814
815
816 ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa,
817 XPC_RP_VARS_SIZE);
818 if (ret != xpSuccess)
819 return ret;
820
821 if (XPC_VERSION_MAJOR(remote_vars->version) !=
822 XPC_VERSION_MAJOR(XPC_V_VERSION)) {
823 return xpBadVersion;
824 }
825
826 return xpSuccess;
827}
828
829static void
830xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
831 unsigned long remote_rp_pa, int nasid)
832{
833 xpc_send_local_activate_IRQ_sn2(nasid);
834}
835
836static void
837xpc_request_partition_reactivation_sn2(struct xpc_partition *part)
838{
839 xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid);
840}
841
842static void
843xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
844{
845 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
846 unsigned long irq_flags;
847 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
848 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
849 sizeof(struct amo)));
850
851 local_irq_save(irq_flags);
852
853
854 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
855 BIT(sn_partition_id));
856
857
858
859
860
861
862
863 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
864 variable),
865 xp_nofault_PIOR_target));
866
867 local_irq_restore(irq_flags);
868
869
870
871
872
873 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
874 cnodeid_to_nasid(0),
875 part_sn2->activate_IRQ_nasid,
876 part_sn2->activate_IRQ_phys_cpuid);
877}
878
879static void
880xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
881{
882 unsigned long irq_flags;
883 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
884 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
885 sizeof(struct amo)));
886
887 local_irq_save(irq_flags);
888
889
890 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
891 ~BIT(sn_partition_id));
892
893
894
895
896
897
898
899 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
900 variable),
901 xp_nofault_PIOR_target));
902
903 local_irq_restore(irq_flags);
904}
905
906static int
907xpc_partition_deactivation_requested_sn2(short partid)
908{
909 struct amo *amo = xpc_vars_sn2->amos_page +
910 XPC_DEACTIVATE_REQUEST_AMO_SN2;
911
912
913 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
914 BIT(partid)) != 0;
915}
916
917
918
919
920static void
921xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
922 unsigned long *remote_rp_ts_jiffies,
923 unsigned long remote_rp_pa,
924 unsigned long remote_vars_pa,
925 struct xpc_vars_sn2 *remote_vars)
926{
927 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
928
929 part->remote_rp_version = remote_rp_version;
930 dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n",
931 part->remote_rp_version);
932
933 part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies;
934 dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n",
935 part->remote_rp_ts_jiffies);
936
937 part->remote_rp_pa = remote_rp_pa;
938 dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa);
939
940 part_sn2->remote_vars_pa = remote_vars_pa;
941 dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n",
942 part_sn2->remote_vars_pa);
943
944 part->last_heartbeat = remote_vars->heartbeat - 1;
945 dev_dbg(xpc_part, " last_heartbeat = 0x%016llx\n",
946 part->last_heartbeat);
947
948 part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa;
949 dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n",
950 part_sn2->remote_vars_part_pa);
951
952 part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid;
953 dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n",
954 part_sn2->activate_IRQ_nasid);
955
956 part_sn2->activate_IRQ_phys_cpuid =
957 remote_vars->activate_IRQ_phys_cpuid;
958 dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n",
959 part_sn2->activate_IRQ_phys_cpuid);
960
961 part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa;
962 dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n",
963 part_sn2->remote_amos_page_pa);
964
965 part_sn2->remote_vars_version = remote_vars->version;
966 dev_dbg(xpc_part, " remote_vars_version = 0x%x\n",
967 part_sn2->remote_vars_version);
968}
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984static void
985xpc_identify_activate_IRQ_req_sn2(int nasid)
986{
987 struct xpc_rsvd_page *remote_rp;
988 struct xpc_vars_sn2 *remote_vars;
989 unsigned long remote_rp_pa;
990 unsigned long remote_vars_pa;
991 int remote_rp_version;
992 int reactivate = 0;
993 unsigned long remote_rp_ts_jiffies = 0;
994 short partid;
995 struct xpc_partition *part;
996 struct xpc_partition_sn2 *part_sn2;
997 enum xp_retval ret;
998
999
1000
1001 remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2;
1002
1003 ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa);
1004 if (ret != xpSuccess) {
1005 dev_warn(xpc_part, "unable to get reserved page from nasid %d, "
1006 "which sent interrupt, reason=%d\n", nasid, ret);
1007 return;
1008 }
1009
1010 remote_vars_pa = remote_rp->sn.sn2.vars_pa;
1011 remote_rp_version = remote_rp->version;
1012 remote_rp_ts_jiffies = remote_rp->ts_jiffies;
1013
1014 partid = remote_rp->SAL_partid;
1015 part = &xpc_partitions[partid];
1016 part_sn2 = &part->sn.sn2;
1017
1018
1019
1020 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
1021
1022 ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
1023 if (ret != xpSuccess) {
1024 dev_warn(xpc_part, "unable to get XPC variables from nasid %d, "
1025 "which sent interrupt, reason=%d\n", nasid, ret);
1026
1027 XPC_DEACTIVATE_PARTITION(part, ret);
1028 return;
1029 }
1030
1031 part->activate_IRQ_rcvd++;
1032
1033 dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = "
1034 "%lld:0x%lx\n", (int)nasid, (int)partid,
1035 part->activate_IRQ_rcvd,
1036 remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]);
1037
1038 if (xpc_partition_disengaged(part) &&
1039 part->act_state == XPC_P_AS_INACTIVE) {
1040
1041 xpc_update_partition_info_sn2(part, remote_rp_version,
1042 &remote_rp_ts_jiffies,
1043 remote_rp_pa, remote_vars_pa,
1044 remote_vars);
1045
1046 if (xpc_partition_deactivation_requested_sn2(partid)) {
1047
1048
1049
1050
1051 return;
1052 }
1053
1054 xpc_activate_partition(part);
1055 return;
1056 }
1057
1058 DBUG_ON(part->remote_rp_version == 0);
1059 DBUG_ON(part_sn2->remote_vars_version == 0);
1060
1061 if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) {
1062
1063
1064
1065 DBUG_ON(xpc_partition_engaged_sn2(partid));
1066 DBUG_ON(xpc_partition_deactivation_requested_sn2(partid));
1067
1068 xpc_update_partition_info_sn2(part, remote_rp_version,
1069 &remote_rp_ts_jiffies,
1070 remote_rp_pa, remote_vars_pa,
1071 remote_vars);
1072 reactivate = 1;
1073 }
1074
1075 if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) {
1076
1077 return;
1078 }
1079
1080 if (reactivate)
1081 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
1082 else if (xpc_partition_deactivation_requested_sn2(partid))
1083 XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown);
1084}
1085
1086
1087
1088
1089
1090
1091
1092
1093int
1094xpc_identify_activate_IRQ_sender_sn2(void)
1095{
1096 int l;
1097 int b;
1098 unsigned long nasid_mask_long;
1099 u64 nasid;
1100 int n_IRQs_detected = 0;
1101 struct amo *act_amos;
1102
1103 act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2;
1104
1105
1106 for (l = 0; l < xpc_nasid_mask_nlongs; l++) {
1107
1108 if (xpc_exiting)
1109 break;
1110
1111 nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]);
1112
1113 b = find_first_bit(&nasid_mask_long, BITS_PER_LONG);
1114 if (b >= BITS_PER_LONG) {
1115
1116 continue;
1117 }
1118
1119 dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l,
1120 nasid_mask_long);
1121
1122
1123
1124
1125
1126
1127
1128 xpc_mach_nasids[l] |= nasid_mask_long;
1129
1130
1131
1132 do {
1133 n_IRQs_detected++;
1134 nasid = (l * BITS_PER_LONG + b) * 2;
1135 dev_dbg(xpc_part, "interrupt from nasid %lld\n", nasid);
1136 xpc_identify_activate_IRQ_req_sn2(nasid);
1137
1138 b = find_next_bit(&nasid_mask_long, BITS_PER_LONG,
1139 b + 1);
1140 } while (b < BITS_PER_LONG);
1141 }
1142 return n_IRQs_detected;
1143}
1144
1145static void
1146xpc_process_activate_IRQ_rcvd_sn2(void)
1147{
1148 unsigned long irq_flags;
1149 int n_IRQs_expected;
1150 int n_IRQs_detected;
1151
1152 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1153 n_IRQs_expected = xpc_activate_IRQ_rcvd;
1154 xpc_activate_IRQ_rcvd = 0;
1155 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1156
1157 n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2();
1158 if (n_IRQs_detected < n_IRQs_expected) {
1159
1160 (void)xpc_identify_activate_IRQ_sender_sn2();
1161 }
1162}
1163
1164
1165
1166
1167static enum xp_retval
1168xpc_setup_ch_structures_sn2(struct xpc_partition *part)
1169{
1170 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1171 struct xpc_channel_sn2 *ch_sn2;
1172 enum xp_retval retval;
1173 int ret;
1174 int cpuid;
1175 int ch_number;
1176 struct timer_list *timer;
1177 short partid = XPC_PARTID(part);
1178
1179
1180
1181 part_sn2->local_GPs =
1182 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1183 &part_sn2->local_GPs_base);
1184 if (part_sn2->local_GPs == NULL) {
1185 dev_err(xpc_chan, "can't get memory for local get/put "
1186 "values\n");
1187 return xpNoMemory;
1188 }
1189
1190 part_sn2->remote_GPs =
1191 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1192 &part_sn2->remote_GPs_base);
1193 if (part_sn2->remote_GPs == NULL) {
1194 dev_err(xpc_chan, "can't get memory for remote get/put "
1195 "values\n");
1196 retval = xpNoMemory;
1197 goto out_1;
1198 }
1199
1200 part_sn2->remote_GPs_pa = 0;
1201
1202
1203
1204 part_sn2->local_openclose_args =
1205 xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE,
1206 GFP_KERNEL, &part_sn2->
1207 local_openclose_args_base);
1208 if (part_sn2->local_openclose_args == NULL) {
1209 dev_err(xpc_chan, "can't get memory for local connect args\n");
1210 retval = xpNoMemory;
1211 goto out_2;
1212 }
1213
1214 part_sn2->remote_openclose_args_pa = 0;
1215
1216 part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid);
1217
1218 part_sn2->notify_IRQ_nasid = 0;
1219 part_sn2->notify_IRQ_phys_cpuid = 0;
1220 part_sn2->remote_chctl_amo_va = NULL;
1221
1222 sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid);
1223 ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2,
1224 IRQF_SHARED, part_sn2->notify_IRQ_owner,
1225 (void *)(u64)partid);
1226 if (ret != 0) {
1227 dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
1228 "errno=%d\n", -ret);
1229 retval = xpLackOfResources;
1230 goto out_3;
1231 }
1232
1233
1234 timer = &part_sn2->dropped_notify_IRQ_timer;
1235 timer_setup(timer, xpc_check_for_dropped_notify_IRQ_sn2, 0);
1236 timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
1237 add_timer(timer);
1238
1239 for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
1240 ch_sn2 = &part->channels[ch_number].sn.sn2;
1241
1242 ch_sn2->local_GP = &part_sn2->local_GPs[ch_number];
1243 ch_sn2->local_openclose_args =
1244 &part_sn2->local_openclose_args[ch_number];
1245
1246 mutex_init(&ch_sn2->msg_to_pull_mutex);
1247 }
1248
1249
1250
1251
1252
1253
1254
1255
1256 xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs);
1257 xpc_vars_part_sn2[partid].openclose_args_pa =
1258 xp_pa(part_sn2->local_openclose_args);
1259 xpc_vars_part_sn2[partid].chctl_amo_pa =
1260 xp_pa(part_sn2->local_chctl_amo_va);
1261 cpuid = raw_smp_processor_id();
1262 xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid);
1263 xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid =
1264 cpu_physical_id(cpuid);
1265 xpc_vars_part_sn2[partid].nchannels = part->nchannels;
1266 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2;
1267
1268 return xpSuccess;
1269
1270
1271out_3:
1272 kfree(part_sn2->local_openclose_args_base);
1273 part_sn2->local_openclose_args = NULL;
1274out_2:
1275 kfree(part_sn2->remote_GPs_base);
1276 part_sn2->remote_GPs = NULL;
1277out_1:
1278 kfree(part_sn2->local_GPs_base);
1279 part_sn2->local_GPs = NULL;
1280 return retval;
1281}
1282
1283
1284
1285
1286static void
1287xpc_teardown_ch_structures_sn2(struct xpc_partition *part)
1288{
1289 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1290 short partid = XPC_PARTID(part);
1291
1292
1293
1294
1295
1296 xpc_vars_part_sn2[partid].magic = 0;
1297
1298
1299 del_timer_sync(&part_sn2->dropped_notify_IRQ_timer);
1300 free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid);
1301
1302 kfree(part_sn2->local_openclose_args_base);
1303 part_sn2->local_openclose_args = NULL;
1304 kfree(part_sn2->remote_GPs_base);
1305 part_sn2->remote_GPs = NULL;
1306 kfree(part_sn2->local_GPs_base);
1307 part_sn2->local_GPs = NULL;
1308 part_sn2->local_chctl_amo_va = NULL;
1309}
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320static enum xp_retval
1321xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst,
1322 const unsigned long src_pa, size_t cnt)
1323{
1324 enum xp_retval ret;
1325
1326 DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa));
1327 DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst));
1328 DBUG_ON(cnt != L1_CACHE_ALIGN(cnt));
1329
1330 if (part->act_state == XPC_P_AS_DEACTIVATING)
1331 return part->reason;
1332
1333 ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt);
1334 if (ret != xpSuccess) {
1335 dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed,"
1336 " ret=%d\n", XPC_PARTID(part), ret);
1337 }
1338 return ret;
1339}
1340
1341
1342
1343
1344
1345static enum xp_retval
1346xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
1347{
1348 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1349 u8 buffer[L1_CACHE_BYTES * 2];
1350 struct xpc_vars_part_sn2 *pulled_entry_cacheline =
1351 (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer);
1352 struct xpc_vars_part_sn2 *pulled_entry;
1353 unsigned long remote_entry_cacheline_pa;
1354 unsigned long remote_entry_pa;
1355 short partid = XPC_PARTID(part);
1356 enum xp_retval ret;
1357
1358
1359
1360 DBUG_ON(part_sn2->remote_vars_part_pa !=
1361 L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa));
1362 DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2);
1363
1364 remote_entry_pa = part_sn2->remote_vars_part_pa +
1365 sn_partition_id * sizeof(struct xpc_vars_part_sn2);
1366
1367 remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
1368
1369 pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline
1370 + (remote_entry_pa &
1371 (L1_CACHE_BYTES - 1)));
1372
1373 ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline,
1374 remote_entry_cacheline_pa,
1375 L1_CACHE_BYTES);
1376 if (ret != xpSuccess) {
1377 dev_dbg(xpc_chan, "failed to pull XPC vars_part from "
1378 "partition %d, ret=%d\n", partid, ret);
1379 return ret;
1380 }
1381
1382
1383
1384 if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 &&
1385 pulled_entry->magic != XPC_VP_MAGIC2_SN2) {
1386
1387 if (pulled_entry->magic != 0) {
1388 dev_dbg(xpc_chan, "partition %d's XPC vars_part for "
1389 "partition %d has bad magic value (=0x%llx)\n",
1390 partid, sn_partition_id, pulled_entry->magic);
1391 return xpBadMagic;
1392 }
1393
1394
1395 return xpRetry;
1396 }
1397
1398 if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) {
1399
1400
1401
1402 if (pulled_entry->GPs_pa == 0 ||
1403 pulled_entry->openclose_args_pa == 0 ||
1404 pulled_entry->chctl_amo_pa == 0) {
1405
1406 dev_err(xpc_chan, "partition %d's XPC vars_part for "
1407 "partition %d are not valid\n", partid,
1408 sn_partition_id);
1409 return xpInvalidAddress;
1410 }
1411
1412
1413
1414 part_sn2->remote_GPs_pa = pulled_entry->GPs_pa;
1415 part_sn2->remote_openclose_args_pa =
1416 pulled_entry->openclose_args_pa;
1417 part_sn2->remote_chctl_amo_va =
1418 (struct amo *)__va(pulled_entry->chctl_amo_pa);
1419 part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid;
1420 part_sn2->notify_IRQ_phys_cpuid =
1421 pulled_entry->notify_IRQ_phys_cpuid;
1422
1423 if (part->nchannels > pulled_entry->nchannels)
1424 part->nchannels = pulled_entry->nchannels;
1425
1426
1427
1428 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2;
1429 }
1430
1431 if (pulled_entry->magic == XPC_VP_MAGIC1_SN2)
1432 return xpRetry;
1433
1434 return xpSuccess;
1435}
1436
1437
1438
1439
1440
1441
1442static enum xp_retval
1443xpc_make_first_contact_sn2(struct xpc_partition *part)
1444{
1445 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1446 enum xp_retval ret;
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458 if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa,
1459 PAGE_SIZE, 1) < 0) {
1460 dev_warn(xpc_part, "xpc_activating(%d) failed to register "
1461 "xp_addr region\n", XPC_PARTID(part));
1462
1463 ret = xpPhysAddrRegFailed;
1464 XPC_DEACTIVATE_PARTITION(part, ret);
1465 return ret;
1466 }
1467
1468
1469
1470
1471
1472 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
1473 cnodeid_to_nasid(0),
1474 part_sn2->activate_IRQ_nasid,
1475 part_sn2->activate_IRQ_phys_cpuid);
1476
1477 while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) {
1478 if (ret != xpRetry) {
1479 XPC_DEACTIVATE_PARTITION(part, ret);
1480 return ret;
1481 }
1482
1483 dev_dbg(xpc_part, "waiting to make first contact with "
1484 "partition %d\n", XPC_PARTID(part));
1485
1486
1487 (void)msleep_interruptible(250);
1488
1489 if (part->act_state == XPC_P_AS_DEACTIVATING)
1490 return part->reason;
1491 }
1492
1493 return xpSuccess;
1494}
1495
1496
1497
1498
1499static u64
1500xpc_get_chctl_all_flags_sn2(struct xpc_partition *part)
1501{
1502 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1503 unsigned long irq_flags;
1504 union xpc_channel_ctl_flags chctl;
1505 enum xp_retval ret;
1506
1507
1508
1509
1510
1511 spin_lock_irqsave(&part->chctl_lock, irq_flags);
1512 chctl = part->chctl;
1513 if (chctl.all_flags != 0)
1514 part->chctl.all_flags = 0;
1515
1516 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
1517
1518 if (xpc_any_openclose_chctl_flags_set(&chctl)) {
1519 ret = xpc_pull_remote_cachelines_sn2(part, part->
1520 remote_openclose_args,
1521 part_sn2->
1522 remote_openclose_args_pa,
1523 XPC_OPENCLOSE_ARGS_SIZE);
1524 if (ret != xpSuccess) {
1525 XPC_DEACTIVATE_PARTITION(part, ret);
1526
1527 dev_dbg(xpc_chan, "failed to pull openclose args from "
1528 "partition %d, ret=%d\n", XPC_PARTID(part),
1529 ret);
1530
1531
1532 chctl.all_flags = 0;
1533 }
1534 }
1535
1536 if (xpc_any_msg_chctl_flags_set(&chctl)) {
1537 ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs,
1538 part_sn2->remote_GPs_pa,
1539 XPC_GP_SIZE);
1540 if (ret != xpSuccess) {
1541 XPC_DEACTIVATE_PARTITION(part, ret);
1542
1543 dev_dbg(xpc_chan, "failed to pull GPs from partition "
1544 "%d, ret=%d\n", XPC_PARTID(part), ret);
1545
1546
1547 chctl.all_flags = 0;
1548 }
1549 }
1550
1551 return chctl.all_flags;
1552}
1553
1554
1555
1556
1557static enum xp_retval
1558xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch)
1559{
1560 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1561 unsigned long irq_flags;
1562 int nentries;
1563 size_t nbytes;
1564
1565 for (nentries = ch->local_nentries; nentries > 0; nentries--) {
1566
1567 nbytes = nentries * ch->entry_size;
1568 ch_sn2->local_msgqueue =
1569 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL,
1570 &ch_sn2->local_msgqueue_base);
1571 if (ch_sn2->local_msgqueue == NULL)
1572 continue;
1573
1574 nbytes = nentries * sizeof(struct xpc_notify_sn2);
1575 ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL);
1576 if (ch_sn2->notify_queue == NULL) {
1577 kfree(ch_sn2->local_msgqueue_base);
1578 ch_sn2->local_msgqueue = NULL;
1579 continue;
1580 }
1581
1582 spin_lock_irqsave(&ch->lock, irq_flags);
1583 if (nentries < ch->local_nentries) {
1584 dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, "
1585 "partid=%d, channel=%d\n", nentries,
1586 ch->local_nentries, ch->partid, ch->number);
1587
1588 ch->local_nentries = nentries;
1589 }
1590 spin_unlock_irqrestore(&ch->lock, irq_flags);
1591 return xpSuccess;
1592 }
1593
1594 dev_dbg(xpc_chan, "can't get memory for local message queue and notify "
1595 "queue, partid=%d, channel=%d\n", ch->partid, ch->number);
1596 return xpNoMemory;
1597}
1598
1599
1600
1601
1602static enum xp_retval
1603xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch)
1604{
1605 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1606 unsigned long irq_flags;
1607 int nentries;
1608 size_t nbytes;
1609
1610 DBUG_ON(ch->remote_nentries <= 0);
1611
1612 for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
1613
1614 nbytes = nentries * ch->entry_size;
1615 ch_sn2->remote_msgqueue =
1616 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2->
1617 remote_msgqueue_base);
1618 if (ch_sn2->remote_msgqueue == NULL)
1619 continue;
1620
1621 spin_lock_irqsave(&ch->lock, irq_flags);
1622 if (nentries < ch->remote_nentries) {
1623 dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, "
1624 "partid=%d, channel=%d\n", nentries,
1625 ch->remote_nentries, ch->partid, ch->number);
1626
1627 ch->remote_nentries = nentries;
1628 }
1629 spin_unlock_irqrestore(&ch->lock, irq_flags);
1630 return xpSuccess;
1631 }
1632
1633 dev_dbg(xpc_chan, "can't get memory for cached remote message queue, "
1634 "partid=%d, channel=%d\n", ch->partid, ch->number);
1635 return xpNoMemory;
1636}
1637
1638
1639
1640
1641
1642
1643static enum xp_retval
1644xpc_setup_msg_structures_sn2(struct xpc_channel *ch)
1645{
1646 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1647 enum xp_retval ret;
1648
1649 DBUG_ON(ch->flags & XPC_C_SETUP);
1650
1651 ret = xpc_allocate_local_msgqueue_sn2(ch);
1652 if (ret == xpSuccess) {
1653
1654 ret = xpc_allocate_remote_msgqueue_sn2(ch);
1655 if (ret != xpSuccess) {
1656 kfree(ch_sn2->local_msgqueue_base);
1657 ch_sn2->local_msgqueue = NULL;
1658 kfree(ch_sn2->notify_queue);
1659 ch_sn2->notify_queue = NULL;
1660 }
1661 }
1662 return ret;
1663}
1664
1665
1666
1667
1668
1669static void
1670xpc_teardown_msg_structures_sn2(struct xpc_channel *ch)
1671{
1672 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1673
1674 lockdep_assert_held(&ch->lock);
1675
1676 ch_sn2->remote_msgqueue_pa = 0;
1677
1678 ch_sn2->local_GP->get = 0;
1679 ch_sn2->local_GP->put = 0;
1680 ch_sn2->remote_GP.get = 0;
1681 ch_sn2->remote_GP.put = 0;
1682 ch_sn2->w_local_GP.get = 0;
1683 ch_sn2->w_local_GP.put = 0;
1684 ch_sn2->w_remote_GP.get = 0;
1685 ch_sn2->w_remote_GP.put = 0;
1686 ch_sn2->next_msg_to_pull = 0;
1687
1688 if (ch->flags & XPC_C_SETUP) {
1689 dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n",
1690 ch->flags, ch->partid, ch->number);
1691
1692 kfree(ch_sn2->local_msgqueue_base);
1693 ch_sn2->local_msgqueue = NULL;
1694 kfree(ch_sn2->remote_msgqueue_base);
1695 ch_sn2->remote_msgqueue = NULL;
1696 kfree(ch_sn2->notify_queue);
1697 ch_sn2->notify_queue = NULL;
1698 }
1699}
1700
1701
1702
1703
1704static void
1705xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put)
1706{
1707 struct xpc_notify_sn2 *notify;
1708 u8 notify_type;
1709 s64 get = ch->sn.sn2.w_remote_GP.get - 1;
1710
1711 while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
1712
1713 notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries];
1714
1715
1716
1717
1718
1719
1720
1721 notify_type = notify->type;
1722 if (notify_type == 0 ||
1723 cmpxchg(¬ify->type, notify_type, 0) != notify_type) {
1724 continue;
1725 }
1726
1727 DBUG_ON(notify_type != XPC_N_CALL);
1728
1729 atomic_dec(&ch->n_to_notify);
1730
1731 if (notify->func != NULL) {
1732 dev_dbg(xpc_chan, "notify->func() called, notify=0x%p "
1733 "msg_number=%lld partid=%d channel=%d\n",
1734 (void *)notify, get, ch->partid, ch->number);
1735
1736 notify->func(reason, ch->partid, ch->number,
1737 notify->key);
1738
1739 dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p"
1740 " msg_number=%lld partid=%d channel=%d\n",
1741 (void *)notify, get, ch->partid, ch->number);
1742 }
1743 }
1744}
1745
1746static void
1747xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch)
1748{
1749 xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put);
1750}
1751
1752
1753
1754
1755static inline void
1756xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch)
1757{
1758 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1759 struct xpc_msg_sn2 *msg;
1760 s64 get;
1761
1762 get = ch_sn2->w_remote_GP.get;
1763 do {
1764 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
1765 (get % ch->local_nentries) *
1766 ch->entry_size);
1767 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
1768 msg->flags = 0;
1769 } while (++get < ch_sn2->remote_GP.get);
1770}
1771
1772
1773
1774
1775static inline void
1776xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
1777{
1778 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1779 struct xpc_msg_sn2 *msg;
1780 s64 put, remote_nentries = ch->remote_nentries;
1781
1782
1783 if (ch_sn2->remote_GP.put < remote_nentries)
1784 return;
1785
1786 put = max(ch_sn2->w_remote_GP.put, remote_nentries);
1787 do {
1788 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
1789 (put % remote_nentries) *
1790 ch->entry_size);
1791 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
1792 DBUG_ON(!(msg->flags & XPC_M_SN2_DONE));
1793 DBUG_ON(msg->number != put - remote_nentries);
1794 msg->flags = 0;
1795 } while (++put < ch_sn2->remote_GP.put);
1796}
1797
1798static int
1799xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch)
1800{
1801 return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get;
1802}
1803
1804static void
1805xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number)
1806{
1807 struct xpc_channel *ch = &part->channels[ch_number];
1808 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1809 int npayloads_sent;
1810
1811 ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number];
1812
1813
1814
1815 xpc_msgqueue_ref(ch);
1816
1817 if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get &&
1818 ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) {
1819
1820 xpc_msgqueue_deref(ch);
1821 return;
1822 }
1823
1824 if (!(ch->flags & XPC_C_CONNECTED)) {
1825 xpc_msgqueue_deref(ch);
1826 return;
1827 }
1828
1829
1830
1831
1832
1833
1834
1835 if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) {
1836
1837
1838
1839
1840
1841
1842
1843
1844 if (atomic_read(&ch->n_to_notify) > 0) {
1845
1846
1847
1848
1849 xpc_notify_senders_sn2(ch, xpMsgDelivered,
1850 ch_sn2->remote_GP.get);
1851 }
1852
1853
1854
1855
1856
1857 xpc_clear_local_msgqueue_flags_sn2(ch);
1858
1859 ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get;
1860
1861 dev_dbg(xpc_chan, "w_remote_GP.get changed to %lld, partid=%d, "
1862 "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid,
1863 ch->number);
1864
1865
1866
1867
1868
1869 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
1870 wake_up(&ch->msg_allocate_wq);
1871 }
1872
1873
1874
1875
1876
1877
1878 if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) {
1879
1880
1881
1882
1883 xpc_clear_remote_msgqueue_flags_sn2(ch);
1884
1885 smp_wmb();
1886 ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put;
1887
1888 dev_dbg(xpc_chan, "w_remote_GP.put changed to %lld, partid=%d, "
1889 "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid,
1890 ch->number);
1891
1892 npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch);
1893 if (npayloads_sent > 0) {
1894 dev_dbg(xpc_chan, "msgs waiting to be copied and "
1895 "delivered=%d, partid=%d, channel=%d\n",
1896 npayloads_sent, ch->partid, ch->number);
1897
1898 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)
1899 xpc_activate_kthreads(ch, npayloads_sent);
1900 }
1901 }
1902
1903 xpc_msgqueue_deref(ch);
1904}
1905
1906static struct xpc_msg_sn2 *
1907xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
1908{
1909 struct xpc_partition *part = &xpc_partitions[ch->partid];
1910 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1911 unsigned long remote_msg_pa;
1912 struct xpc_msg_sn2 *msg;
1913 u32 msg_index;
1914 u32 nmsgs;
1915 u64 msg_offset;
1916 enum xp_retval ret;
1917
1918 if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) {
1919
1920 return NULL;
1921 }
1922
1923 while (get >= ch_sn2->next_msg_to_pull) {
1924
1925
1926
1927 msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries;
1928
1929 DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put);
1930 nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull;
1931 if (msg_index + nmsgs > ch->remote_nentries) {
1932
1933 nmsgs = ch->remote_nentries - msg_index;
1934 }
1935
1936 msg_offset = msg_index * ch->entry_size;
1937 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
1938 msg_offset);
1939 remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset;
1940
1941 ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa,
1942 nmsgs * ch->entry_size);
1943 if (ret != xpSuccess) {
1944
1945 dev_dbg(xpc_chan, "failed to pull %d msgs starting with"
1946 " msg %lld from partition %d, channel=%d, "
1947 "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull,
1948 ch->partid, ch->number, ret);
1949
1950 XPC_DEACTIVATE_PARTITION(part, ret);
1951
1952 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
1953 return NULL;
1954 }
1955
1956 ch_sn2->next_msg_to_pull += nmsgs;
1957 }
1958
1959 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
1960
1961
1962 msg_offset = (get % ch->remote_nentries) * ch->entry_size;
1963 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset);
1964
1965 return msg;
1966}
1967
1968
1969
1970
1971static void *
1972xpc_get_deliverable_payload_sn2(struct xpc_channel *ch)
1973{
1974 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1975 struct xpc_msg_sn2 *msg;
1976 void *payload = NULL;
1977 s64 get;
1978
1979 do {
1980 if (ch->flags & XPC_C_DISCONNECTING)
1981 break;
1982
1983 get = ch_sn2->w_local_GP.get;
1984 smp_rmb();
1985 if (get == ch_sn2->w_remote_GP.put)
1986 break;
1987
1988
1989
1990
1991
1992
1993
1994
1995 if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) {
1996
1997
1998 dev_dbg(xpc_chan, "w_local_GP.get changed to %lld, "
1999 "partid=%d, channel=%d\n", get + 1,
2000 ch->partid, ch->number);
2001
2002
2003
2004 msg = xpc_pull_remote_msg_sn2(ch, get);
2005
2006 if (msg != NULL) {
2007 DBUG_ON(msg->number != get);
2008 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
2009 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
2010
2011 payload = &msg->payload;
2012 }
2013 break;
2014 }
2015
2016 } while (1);
2017
2018 return payload;
2019}
2020
2021
2022
2023
2024
2025
2026static void
2027xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
2028{
2029 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2030 struct xpc_msg_sn2 *msg;
2031 s64 put = initial_put + 1;
2032 int send_msgrequest = 0;
2033
2034 while (1) {
2035
2036 while (1) {
2037 if (put == ch_sn2->w_local_GP.put)
2038 break;
2039
2040 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2041 local_msgqueue + (put %
2042 ch->local_nentries) *
2043 ch->entry_size);
2044
2045 if (!(msg->flags & XPC_M_SN2_READY))
2046 break;
2047
2048 put++;
2049 }
2050
2051 if (put == initial_put) {
2052
2053 break;
2054 }
2055
2056 if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) !=
2057 initial_put) {
2058
2059 DBUG_ON(ch_sn2->local_GP->put < initial_put);
2060 break;
2061 }
2062
2063
2064
2065 dev_dbg(xpc_chan, "local_GP->put changed to %lld, partid=%d, "
2066 "channel=%d\n", put, ch->partid, ch->number);
2067
2068 send_msgrequest = 1;
2069
2070
2071
2072
2073
2074
2075 initial_put = put;
2076 }
2077
2078 if (send_msgrequest)
2079 xpc_send_chctl_msgrequest_sn2(ch);
2080}
2081
2082
2083
2084
2085
2086static enum xp_retval
2087xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
2088 struct xpc_msg_sn2 **address_of_msg)
2089{
2090 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2091 struct xpc_msg_sn2 *msg;
2092 enum xp_retval ret;
2093 s64 put;
2094
2095
2096
2097
2098
2099
2100 ret = xpTimeout;
2101
2102 while (1) {
2103
2104 put = ch_sn2->w_local_GP.put;
2105 smp_rmb();
2106 if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) {
2107
2108
2109
2110
2111
2112
2113
2114 if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) ==
2115 put) {
2116
2117 break;
2118 }
2119 continue;
2120 }
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131 if (ret == xpTimeout)
2132 xpc_send_chctl_local_msgrequest_sn2(ch);
2133
2134 if (flags & XPC_NOWAIT)
2135 return xpNoWait;
2136
2137 ret = xpc_allocate_msg_wait(ch);
2138 if (ret != xpInterrupted && ret != xpTimeout)
2139 return ret;
2140 }
2141
2142
2143 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
2144 (put % ch->local_nentries) *
2145 ch->entry_size);
2146
2147 DBUG_ON(msg->flags != 0);
2148 msg->number = put;
2149
2150 dev_dbg(xpc_chan, "w_local_GP.put changed to %lld; msg=0x%p, "
2151 "msg_number=%lld, partid=%d, channel=%d\n", put + 1,
2152 (void *)msg, msg->number, ch->partid, ch->number);
2153
2154 *address_of_msg = msg;
2155 return xpSuccess;
2156}
2157
2158
2159
2160
2161
2162
2163static enum xp_retval
2164xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload,
2165 u16 payload_size, u8 notify_type, xpc_notify_func func,
2166 void *key)
2167{
2168 enum xp_retval ret = xpSuccess;
2169 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2170 struct xpc_msg_sn2 *msg = msg;
2171 struct xpc_notify_sn2 *notify = notify;
2172 s64 msg_number;
2173 s64 put;
2174
2175 DBUG_ON(notify_type == XPC_N_CALL && func == NULL);
2176
2177 if (XPC_MSG_SIZE(payload_size) > ch->entry_size)
2178 return xpPayloadTooBig;
2179
2180 xpc_msgqueue_ref(ch);
2181
2182 if (ch->flags & XPC_C_DISCONNECTING) {
2183 ret = ch->reason;
2184 goto out_1;
2185 }
2186 if (!(ch->flags & XPC_C_CONNECTED)) {
2187 ret = xpNotConnected;
2188 goto out_1;
2189 }
2190
2191 ret = xpc_allocate_msg_sn2(ch, flags, &msg);
2192 if (ret != xpSuccess)
2193 goto out_1;
2194
2195 msg_number = msg->number;
2196
2197 if (notify_type != 0) {
2198
2199
2200
2201
2202 msg->flags |= XPC_M_SN2_INTERRUPT;
2203
2204 atomic_inc(&ch->n_to_notify);
2205
2206 notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries];
2207 notify->func = func;
2208 notify->key = key;
2209 notify->type = notify_type;
2210
2211
2212
2213 if (ch->flags & XPC_C_DISCONNECTING) {
2214
2215
2216
2217
2218
2219
2220
2221 if (cmpxchg(¬ify->type, notify_type, 0) ==
2222 notify_type) {
2223 atomic_dec(&ch->n_to_notify);
2224 ret = ch->reason;
2225 }
2226 goto out_1;
2227 }
2228 }
2229
2230 memcpy(&msg->payload, payload, payload_size);
2231
2232 msg->flags |= XPC_M_SN2_READY;
2233
2234
2235
2236
2237
2238 smp_mb();
2239
2240
2241
2242 put = ch_sn2->local_GP->put;
2243 if (put == msg_number)
2244 xpc_send_msgs_sn2(ch, put);
2245
2246out_1:
2247 xpc_msgqueue_deref(ch);
2248 return ret;
2249}
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259static void
2260xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
2261{
2262 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2263 struct xpc_msg_sn2 *msg;
2264 s64 get = initial_get + 1;
2265 int send_msgrequest = 0;
2266
2267 while (1) {
2268
2269 while (1) {
2270 if (get == ch_sn2->w_local_GP.get)
2271 break;
2272
2273 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2274 remote_msgqueue + (get %
2275 ch->remote_nentries) *
2276 ch->entry_size);
2277
2278 if (!(msg->flags & XPC_M_SN2_DONE))
2279 break;
2280
2281 msg_flags |= msg->flags;
2282 get++;
2283 }
2284
2285 if (get == initial_get) {
2286
2287 break;
2288 }
2289
2290 if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) !=
2291 initial_get) {
2292
2293 DBUG_ON(ch_sn2->local_GP->get <= initial_get);
2294 break;
2295 }
2296
2297
2298
2299 dev_dbg(xpc_chan, "local_GP->get changed to %lld, partid=%d, "
2300 "channel=%d\n", get, ch->partid, ch->number);
2301
2302 send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT);
2303
2304
2305
2306
2307
2308
2309 initial_get = get;
2310 }
2311
2312 if (send_msgrequest)
2313 xpc_send_chctl_msgrequest_sn2(ch);
2314}
2315
2316static void
2317xpc_received_payload_sn2(struct xpc_channel *ch, void *payload)
2318{
2319 struct xpc_msg_sn2 *msg;
2320 s64 msg_number;
2321 s64 get;
2322
2323 msg = container_of(payload, struct xpc_msg_sn2, payload);
2324 msg_number = msg->number;
2325
2326 dev_dbg(xpc_chan, "msg=0x%p, msg_number=%lld, partid=%d, channel=%d\n",
2327 (void *)msg, msg_number, ch->partid, ch->number);
2328
2329 DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) !=
2330 msg_number % ch->remote_nentries);
2331 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
2332 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
2333
2334 msg->flags |= XPC_M_SN2_DONE;
2335
2336
2337
2338
2339
2340 smp_mb();
2341
2342
2343
2344
2345
2346 get = ch->sn.sn2.local_GP->get;
2347 if (get == msg_number)
2348 xpc_acknowledge_msgs_sn2(ch, get, msg->flags);
2349}
2350
2351static struct xpc_arch_operations xpc_arch_ops_sn2 = {
2352 .setup_partitions = xpc_setup_partitions_sn2,
2353 .teardown_partitions = xpc_teardown_partitions_sn2,
2354 .process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2,
2355 .get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2,
2356 .setup_rsvd_page = xpc_setup_rsvd_page_sn2,
2357
2358 .allow_hb = xpc_allow_hb_sn2,
2359 .disallow_hb = xpc_disallow_hb_sn2,
2360 .disallow_all_hbs = xpc_disallow_all_hbs_sn2,
2361 .increment_heartbeat = xpc_increment_heartbeat_sn2,
2362 .offline_heartbeat = xpc_offline_heartbeat_sn2,
2363 .online_heartbeat = xpc_online_heartbeat_sn2,
2364 .heartbeat_init = xpc_heartbeat_init_sn2,
2365 .heartbeat_exit = xpc_heartbeat_exit_sn2,
2366 .get_remote_heartbeat = xpc_get_remote_heartbeat_sn2,
2367
2368 .request_partition_activation =
2369 xpc_request_partition_activation_sn2,
2370 .request_partition_reactivation =
2371 xpc_request_partition_reactivation_sn2,
2372 .request_partition_deactivation =
2373 xpc_request_partition_deactivation_sn2,
2374 .cancel_partition_deactivation_request =
2375 xpc_cancel_partition_deactivation_request_sn2,
2376
2377 .setup_ch_structures = xpc_setup_ch_structures_sn2,
2378 .teardown_ch_structures = xpc_teardown_ch_structures_sn2,
2379
2380 .make_first_contact = xpc_make_first_contact_sn2,
2381
2382 .get_chctl_all_flags = xpc_get_chctl_all_flags_sn2,
2383 .send_chctl_closerequest = xpc_send_chctl_closerequest_sn2,
2384 .send_chctl_closereply = xpc_send_chctl_closereply_sn2,
2385 .send_chctl_openrequest = xpc_send_chctl_openrequest_sn2,
2386 .send_chctl_openreply = xpc_send_chctl_openreply_sn2,
2387 .send_chctl_opencomplete = xpc_send_chctl_opencomplete_sn2,
2388 .process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2,
2389
2390 .save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2,
2391
2392 .setup_msg_structures = xpc_setup_msg_structures_sn2,
2393 .teardown_msg_structures = xpc_teardown_msg_structures_sn2,
2394
2395 .indicate_partition_engaged = xpc_indicate_partition_engaged_sn2,
2396 .indicate_partition_disengaged = xpc_indicate_partition_disengaged_sn2,
2397 .partition_engaged = xpc_partition_engaged_sn2,
2398 .any_partition_engaged = xpc_any_partition_engaged_sn2,
2399 .assume_partition_disengaged = xpc_assume_partition_disengaged_sn2,
2400
2401 .n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2,
2402 .send_payload = xpc_send_payload_sn2,
2403 .get_deliverable_payload = xpc_get_deliverable_payload_sn2,
2404 .received_payload = xpc_received_payload_sn2,
2405 .notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2,
2406};
2407
2408int
2409xpc_init_sn2(void)
2410{
2411 int ret;
2412 size_t buf_size;
2413
2414 xpc_arch_ops = xpc_arch_ops_sn2;
2415
2416 if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) {
2417 dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is "
2418 "larger than %d\n", XPC_MSG_HDR_MAX_SIZE);
2419 return -E2BIG;
2420 }
2421
2422 buf_size = max(XPC_RP_VARS_SIZE,
2423 XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2);
2424 xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size,
2425 GFP_KERNEL,
2426 &xpc_remote_copy_buffer_base_sn2);
2427 if (xpc_remote_copy_buffer_sn2 == NULL) {
2428 dev_err(xpc_part, "can't get memory for remote copy buffer\n");
2429 return -ENOMEM;
2430 }
2431
2432
2433 xpc_allow_IPI_ops_sn2();
2434 xpc_allow_amo_ops_shub_wars_1_1_sn2();
2435
2436
2437
2438
2439
2440
2441
2442 ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0,
2443 "xpc hb", NULL);
2444 if (ret != 0) {
2445 dev_err(xpc_part, "can't register ACTIVATE IRQ handler, "
2446 "errno=%d\n", -ret);
2447 xpc_disallow_IPI_ops_sn2();
2448 kfree(xpc_remote_copy_buffer_base_sn2);
2449 }
2450 return ret;
2451}
2452
2453void
2454xpc_exit_sn2(void)
2455{
2456 free_irq(SGI_XPC_ACTIVATE, NULL);
2457 xpc_disallow_IPI_ops_sn2();
2458 kfree(xpc_remote_copy_buffer_base_sn2);
2459}
2460