2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
10 * Cross Partition Communication (XPC) sn2-based functions.
12 * Architecture specific implementation of common functions.
16 #include <linux/delay.h>
17 #include <asm/uncached.h>
18 #include <asm/sn/mspec.h>
19 #include <asm/sn/sn_sal.h>
23 * Define the number of u64s required to represent all the C-brick nasids
24 * as a bitmap. The cross-partition kernel modules deal only with
25 * C-brick nasids, thus the need for bitmaps which don't account for
26 * odd-numbered (non C-brick) nasids.
28 #define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2)
29 #define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
30 #define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
33 * Memory for XPC's amo variables is allocated by the MSPEC driver. These
34 * pages are located in the lowest granule. The lowest granule uses 4k pages
35 * for cached references and an alternate TLB handler to never provide a
36 * cacheable mapping for the entire region. This will prevent speculative
37 * reading of cached copies of our lines from being issued which will cause
38 * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64
39 * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of
40 * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify
41 * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote
42 * partitions (i.e., XPCs) consider themselves currently engaged with the
43 * local XPC and 1 amo variable to request partition deactivation.
45 #define XPC_NOTIFY_IRQ_AMOS_SN2 0
46 #define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \
47 XP_MAX_NPARTITIONS_SN2)
48 #define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \
49 XP_NASID_MASK_WORDS_SN2)
50 #define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
53 * Buffer used to store a local copy of portions of a remote partition's
54 * reserved page (either its header and part_nasids mask, or its vars).
56 static void *xpc_remote_copy_buffer_base_sn2
;
57 static char *xpc_remote_copy_buffer_sn2
;
59 static struct xpc_vars_sn2
*xpc_vars_sn2
;
60 static struct xpc_vars_part_sn2
*xpc_vars_part_sn2
;
63 xpc_setup_partitions_sn_sn2(void)
65 /* nothing needs to be done */
69 /* SH_IPI_ACCESS shub register value on startup */
70 static u64 xpc_sh1_IPI_access_sn2
;
71 static u64 xpc_sh2_IPI_access0_sn2
;
72 static u64 xpc_sh2_IPI_access1_sn2
;
73 static u64 xpc_sh2_IPI_access2_sn2
;
74 static u64 xpc_sh2_IPI_access3_sn2
;
77 * Change protections to allow IPI operations.
80 xpc_allow_IPI_ops_sn2(void)
85 /* !!! The following should get moved into SAL. */
87 xpc_sh2_IPI_access0_sn2
=
88 (u64
)HUB_L((u64
*)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0
));
89 xpc_sh2_IPI_access1_sn2
=
90 (u64
)HUB_L((u64
*)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1
));
91 xpc_sh2_IPI_access2_sn2
=
92 (u64
)HUB_L((u64
*)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2
));
93 xpc_sh2_IPI_access3_sn2
=
94 (u64
)HUB_L((u64
*)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3
));
96 for_each_online_node(node
) {
97 nasid
= cnodeid_to_nasid(node
);
98 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH2_IPI_ACCESS0
),
100 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH2_IPI_ACCESS1
),
102 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH2_IPI_ACCESS2
),
104 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH2_IPI_ACCESS3
),
108 xpc_sh1_IPI_access_sn2
=
109 (u64
)HUB_L((u64
*)LOCAL_MMR_ADDR(SH1_IPI_ACCESS
));
111 for_each_online_node(node
) {
112 nasid
= cnodeid_to_nasid(node
);
113 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH1_IPI_ACCESS
),
120 * Restrict protections to disallow IPI operations.
123 xpc_disallow_IPI_ops_sn2(void)
128 /* !!! The following should get moved into SAL. */
130 for_each_online_node(node
) {
131 nasid
= cnodeid_to_nasid(node
);
132 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH2_IPI_ACCESS0
),
133 xpc_sh2_IPI_access0_sn2
);
134 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH2_IPI_ACCESS1
),
135 xpc_sh2_IPI_access1_sn2
);
136 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH2_IPI_ACCESS2
),
137 xpc_sh2_IPI_access2_sn2
);
138 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH2_IPI_ACCESS3
),
139 xpc_sh2_IPI_access3_sn2
);
142 for_each_online_node(node
) {
143 nasid
= cnodeid_to_nasid(node
);
144 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
, SH1_IPI_ACCESS
),
145 xpc_sh1_IPI_access_sn2
);
151 * The following set of functions are used for the sending and receiving of
152 * IRQs (also known as IPIs). There are two flavors of IRQs, one that is
153 * associated with partition activity (SGI_XPC_ACTIVATE) and the other that
154 * is associated with channel activity (SGI_XPC_NOTIFY).
158 xpc_receive_IRQ_amo_sn2(struct amo
*amo
)
160 return FETCHOP_LOAD_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_CLEAR
);
163 static enum xp_retval
164 xpc_send_IRQ_sn2(struct amo
*amo
, u64 flag
, int nasid
, int phys_cpuid
,
168 unsigned long irq_flags
;
170 local_irq_save(irq_flags
);
172 FETCHOP_STORE_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_OR
, flag
);
173 sn_send_IPI_phys(nasid
, phys_cpuid
, vector
, 0);
176 * We must always use the nofault function regardless of whether we
177 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
178 * didn't, we'd never know that the other partition is down and would
179 * keep sending IRQs and amos to it until the heartbeat times out.
181 ret
= xp_nofault_PIOR((u64
*)GLOBAL_MMR_ADDR(NASID_GET(&amo
->variable
),
182 xp_nofault_PIOR_target
));
184 local_irq_restore(irq_flags
);
186 return (ret
== 0) ? xpSuccess
: xpPioReadError
;
190 xpc_init_IRQ_amo_sn2(int index
)
192 struct amo
*amo
= xpc_vars_sn2
->amos_page
+ index
;
194 (void)xpc_receive_IRQ_amo_sn2(amo
); /* clear amo variable */
199 * Functions associated with SGI_XPC_ACTIVATE IRQ.
203 * Notify the heartbeat check thread that an activate IRQ has been received.
206 xpc_handle_activate_IRQ_sn2(int irq
, void *dev_id
)
208 unsigned long irq_flags
;
210 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock
, irq_flags
);
211 xpc_activate_IRQ_rcvd
++;
212 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock
, irq_flags
);
214 wake_up_interruptible(&xpc_activate_IRQ_wq
);
219 * Flag the appropriate amo variable and send an IRQ to the specified node.
222 xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa
, int from_nasid
,
223 int to_nasid
, int to_phys_cpuid
)
225 struct amo
*amos
= (struct amo
*)__va(amos_page_pa
+
226 (XPC_ACTIVATE_IRQ_AMOS_SN2
*
227 sizeof(struct amo
)));
229 (void)xpc_send_IRQ_sn2(&amos
[BIT_WORD(from_nasid
/ 2)],
230 BIT_MASK(from_nasid
/ 2), to_nasid
,
231 to_phys_cpuid
, SGI_XPC_ACTIVATE
);
235 xpc_send_local_activate_IRQ_sn2(int from_nasid
)
237 unsigned long irq_flags
;
238 struct amo
*amos
= (struct amo
*)__va(xpc_vars_sn2
->amos_page_pa
+
239 (XPC_ACTIVATE_IRQ_AMOS_SN2
*
240 sizeof(struct amo
)));
242 /* fake the sending and receipt of an activate IRQ from remote nasid */
243 FETCHOP_STORE_OP(TO_AMO((u64
)&amos
[BIT_WORD(from_nasid
/ 2)].variable
),
244 FETCHOP_OR
, BIT_MASK(from_nasid
/ 2));
246 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock
, irq_flags
);
247 xpc_activate_IRQ_rcvd
++;
248 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock
, irq_flags
);
250 wake_up_interruptible(&xpc_activate_IRQ_wq
);
254 * Functions associated with SGI_XPC_NOTIFY IRQ.
258 * Check to see if any chctl flags were sent from the specified partition.
261 xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition
*part
)
263 union xpc_channel_ctl_flags chctl
;
264 unsigned long irq_flags
;
266 chctl
.all_flags
= xpc_receive_IRQ_amo_sn2(part
->sn
.sn2
.
268 if (chctl
.all_flags
== 0)
271 spin_lock_irqsave(&part
->chctl_lock
, irq_flags
);
272 part
->chctl
.all_flags
|= chctl
.all_flags
;
273 spin_unlock_irqrestore(&part
->chctl_lock
, irq_flags
);
275 dev_dbg(xpc_chan
, "received notify IRQ from partid=%d, chctl.all_flags="
276 "0x%lx\n", XPC_PARTID(part
), chctl
.all_flags
);
278 xpc_wakeup_channel_mgr(part
);
282 * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified
283 * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more
284 * than one partition, we use an amo structure per partition to indicate
285 * whether a partition has sent an IRQ or not. If it has, then wake up the
286 * associated kthread to handle it.
288 * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC
289 * running on other partitions.
291 * Noteworthy Arguments:
293 * irq - Interrupt ReQuest number. NOT USED.
295 * dev_id - partid of IRQ's potential sender.
298 xpc_handle_notify_IRQ_sn2(int irq
, void *dev_id
)
300 short partid
= (short)(u64
)dev_id
;
301 struct xpc_partition
*part
= &xpc_partitions
[partid
];
303 DBUG_ON(partid
< 0 || partid
>= XP_MAX_NPARTITIONS_SN2
);
305 if (xpc_part_ref(part
)) {
306 xpc_check_for_sent_chctl_flags_sn2(part
);
308 xpc_part_deref(part
);
314 * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor
315 * because the write to their associated amo variable completed after the IRQ
319 xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition
*part
)
321 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
323 if (xpc_part_ref(part
)) {
324 xpc_check_for_sent_chctl_flags_sn2(part
);
326 part_sn2
->dropped_notify_IRQ_timer
.expires
= jiffies
+
327 XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL
;
328 add_timer(&part_sn2
->dropped_notify_IRQ_timer
);
329 xpc_part_deref(part
);
334 * Send a notify IRQ to the remote partition that is associated with the
338 xpc_send_notify_IRQ_sn2(struct xpc_channel
*ch
, u8 chctl_flag
,
339 char *chctl_flag_string
, unsigned long *irq_flags
)
341 struct xpc_partition
*part
= &xpc_partitions
[ch
->partid
];
342 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
343 union xpc_channel_ctl_flags chctl
= { 0 };
346 if (likely(part
->act_state
!= XPC_P_AS_DEACTIVATING
)) {
347 chctl
.flags
[ch
->number
] = chctl_flag
;
348 ret
= xpc_send_IRQ_sn2(part_sn2
->remote_chctl_amo_va
,
350 part_sn2
->notify_IRQ_nasid
,
351 part_sn2
->notify_IRQ_phys_cpuid
,
353 dev_dbg(xpc_chan
, "%s sent to partid=%d, channel=%d, ret=%d\n",
354 chctl_flag_string
, ch
->partid
, ch
->number
, ret
);
355 if (unlikely(ret
!= xpSuccess
)) {
356 if (irq_flags
!= NULL
)
357 spin_unlock_irqrestore(&ch
->lock
, *irq_flags
);
358 XPC_DEACTIVATE_PARTITION(part
, ret
);
359 if (irq_flags
!= NULL
)
360 spin_lock_irqsave(&ch
->lock
, *irq_flags
);
365 #define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \
366 xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f)
369 * Make it look like the remote partition, which is associated with the
370 * specified channel, sent us a notify IRQ. This faked IRQ will be handled
371 * by xpc_check_for_dropped_notify_IRQ_sn2().
374 xpc_send_local_notify_IRQ_sn2(struct xpc_channel
*ch
, u8 chctl_flag
,
375 char *chctl_flag_string
)
377 struct xpc_partition
*part
= &xpc_partitions
[ch
->partid
];
378 union xpc_channel_ctl_flags chctl
= { 0 };
380 chctl
.flags
[ch
->number
] = chctl_flag
;
381 FETCHOP_STORE_OP(TO_AMO((u64
)&part
->sn
.sn2
.local_chctl_amo_va
->
382 variable
), FETCHOP_OR
, chctl
.all_flags
);
383 dev_dbg(xpc_chan
, "%s sent local from partid=%d, channel=%d\n",
384 chctl_flag_string
, ch
->partid
, ch
->number
);
387 #define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \
388 xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f)
391 xpc_send_chctl_closerequest_sn2(struct xpc_channel
*ch
,
392 unsigned long *irq_flags
)
394 struct xpc_openclose_args
*args
= ch
->sn
.sn2
.local_openclose_args
;
396 args
->reason
= ch
->reason
;
397 XPC_SEND_NOTIFY_IRQ_SN2(ch
, XPC_CHCTL_CLOSEREQUEST
, irq_flags
);
401 xpc_send_chctl_closereply_sn2(struct xpc_channel
*ch
, unsigned long *irq_flags
)
403 XPC_SEND_NOTIFY_IRQ_SN2(ch
, XPC_CHCTL_CLOSEREPLY
, irq_flags
);
407 xpc_send_chctl_openrequest_sn2(struct xpc_channel
*ch
, unsigned long *irq_flags
)
409 struct xpc_openclose_args
*args
= ch
->sn
.sn2
.local_openclose_args
;
411 args
->entry_size
= ch
->entry_size
;
412 args
->local_nentries
= ch
->local_nentries
;
413 XPC_SEND_NOTIFY_IRQ_SN2(ch
, XPC_CHCTL_OPENREQUEST
, irq_flags
);
417 xpc_send_chctl_openreply_sn2(struct xpc_channel
*ch
, unsigned long *irq_flags
)
419 struct xpc_openclose_args
*args
= ch
->sn
.sn2
.local_openclose_args
;
421 args
->remote_nentries
= ch
->remote_nentries
;
422 args
->local_nentries
= ch
->local_nentries
;
423 args
->local_msgqueue_pa
= xp_pa(ch
->sn
.sn2
.local_msgqueue
);
424 XPC_SEND_NOTIFY_IRQ_SN2(ch
, XPC_CHCTL_OPENREPLY
, irq_flags
);
428 xpc_send_chctl_msgrequest_sn2(struct xpc_channel
*ch
)
430 XPC_SEND_NOTIFY_IRQ_SN2(ch
, XPC_CHCTL_MSGREQUEST
, NULL
);
434 xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel
*ch
)
436 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch
, XPC_CHCTL_MSGREQUEST
);
440 xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel
*ch
,
441 unsigned long msgqueue_pa
)
443 ch
->sn
.sn2
.remote_msgqueue_pa
= msgqueue_pa
;
447 * This next set of functions are used to keep track of when a partition is
448 * potentially engaged in accessing memory belonging to another partition.
452 xpc_indicate_partition_engaged_sn2(struct xpc_partition
*part
)
454 unsigned long irq_flags
;
455 struct amo
*amo
= (struct amo
*)__va(part
->sn
.sn2
.remote_amos_page_pa
+
456 (XPC_ENGAGED_PARTITIONS_AMO_SN2
*
457 sizeof(struct amo
)));
459 local_irq_save(irq_flags
);
461 /* set bit corresponding to our partid in remote partition's amo */
462 FETCHOP_STORE_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_OR
,
463 BIT(sn_partition_id
));
466 * We must always use the nofault function regardless of whether we
467 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
468 * didn't, we'd never know that the other partition is down and would
469 * keep sending IRQs and amos to it until the heartbeat times out.
471 (void)xp_nofault_PIOR((u64
*)GLOBAL_MMR_ADDR(NASID_GET(&amo
->
473 xp_nofault_PIOR_target
));
475 local_irq_restore(irq_flags
);
479 xpc_indicate_partition_disengaged_sn2(struct xpc_partition
*part
)
481 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
482 unsigned long irq_flags
;
483 struct amo
*amo
= (struct amo
*)__va(part_sn2
->remote_amos_page_pa
+
484 (XPC_ENGAGED_PARTITIONS_AMO_SN2
*
485 sizeof(struct amo
)));
487 local_irq_save(irq_flags
);
489 /* clear bit corresponding to our partid in remote partition's amo */
490 FETCHOP_STORE_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_AND
,
491 ~BIT(sn_partition_id
));
494 * We must always use the nofault function regardless of whether we
495 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
496 * didn't, we'd never know that the other partition is down and would
497 * keep sending IRQs and amos to it until the heartbeat times out.
499 (void)xp_nofault_PIOR((u64
*)GLOBAL_MMR_ADDR(NASID_GET(&amo
->
501 xp_nofault_PIOR_target
));
503 local_irq_restore(irq_flags
);
506 * Send activate IRQ to get other side to see that we've cleared our
507 * bit in their engaged partitions amo.
509 xpc_send_activate_IRQ_sn2(part_sn2
->remote_amos_page_pa
,
511 part_sn2
->activate_IRQ_nasid
,
512 part_sn2
->activate_IRQ_phys_cpuid
);
516 xpc_assume_partition_disengaged_sn2(short partid
)
518 struct amo
*amo
= xpc_vars_sn2
->amos_page
+
519 XPC_ENGAGED_PARTITIONS_AMO_SN2
;
521 /* clear bit(s) based on partid mask in our partition's amo */
522 FETCHOP_STORE_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_AND
,
527 xpc_partition_engaged_sn2(short partid
)
529 struct amo
*amo
= xpc_vars_sn2
->amos_page
+
530 XPC_ENGAGED_PARTITIONS_AMO_SN2
;
532 /* our partition's amo variable ANDed with partid mask */
533 return (FETCHOP_LOAD_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_LOAD
) &
538 xpc_any_partition_engaged_sn2(void)
540 struct amo
*amo
= xpc_vars_sn2
->amos_page
+
541 XPC_ENGAGED_PARTITIONS_AMO_SN2
;
543 /* our partition's amo variable */
544 return FETCHOP_LOAD_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_LOAD
) != 0;
547 /* original protection values for each node */
548 static u64 xpc_prot_vec_sn2
[MAX_NUMNODES
];
551 * Change protections to allow amo operations on non-Shub 1.1 systems.
553 static enum xp_retval
554 xpc_allow_amo_ops_sn2(struct amo
*amos_page
)
556 enum xp_retval ret
= xpSuccess
;
559 * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST
560 * collides with memory operations. On those systems we call
561 * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead.
563 if (!enable_shub_wars_1_1())
564 ret
= xp_expand_memprotect(ia64_tpa((u64
)amos_page
), PAGE_SIZE
);
570 * Change protections to allow amo operations on Shub 1.1 systems.
573 xpc_allow_amo_ops_shub_wars_1_1_sn2(void)
578 if (!enable_shub_wars_1_1())
581 for_each_online_node(node
) {
582 nasid
= cnodeid_to_nasid(node
);
583 /* save current protection values */
584 xpc_prot_vec_sn2
[node
] =
585 (u64
)HUB_L((u64
*)GLOBAL_MMR_ADDR(nasid
,
586 SH1_MD_DQLP_MMR_DIR_PRIVEC0
));
587 /* open up everything */
588 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
,
589 SH1_MD_DQLP_MMR_DIR_PRIVEC0
),
591 HUB_S((u64
*)GLOBAL_MMR_ADDR(nasid
,
592 SH1_MD_DQRP_MMR_DIR_PRIVEC0
),
597 static enum xp_retval
598 xpc_get_partition_rsvd_page_pa_sn2(void *buf
, u64
*cookie
, unsigned long *rp_pa
,
604 status
= sn_partition_reserved_page_pa((u64
)buf
, cookie
, rp_pa
, len
);
605 if (status
== SALRET_OK
)
607 else if (status
== SALRET_MORE_PASSES
)
608 ret
= xpNeedMoreInfo
;
617 xpc_setup_rsvd_page_sn_sn2(struct xpc_rsvd_page
*rp
)
619 struct amo
*amos_page
;
623 xpc_vars_sn2
= XPC_RP_VARS(rp
);
625 rp
->sn
.vars_pa
= xp_pa(xpc_vars_sn2
);
627 /* vars_part array follows immediately after vars */
628 xpc_vars_part_sn2
= (struct xpc_vars_part_sn2
*)((u8
*)XPC_RP_VARS(rp
) +
632 * Before clearing xpc_vars_sn2, see if a page of amos had been
633 * previously allocated. If not we'll need to allocate one and set
634 * permissions so that cross-partition amos are allowed.
636 * The allocated amo page needs MCA reporting to remain disabled after
637 * XPC has unloaded. To make this work, we keep a copy of the pointer
638 * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure,
639 * which is pointed to by the reserved page, and re-use that saved copy
640 * on subsequent loads of XPC. This amo page is never freed, and its
641 * memory protections are never restricted.
643 amos_page
= xpc_vars_sn2
->amos_page
;
644 if (amos_page
== NULL
) {
645 amos_page
= (struct amo
*)TO_AMO(uncached_alloc_page(0, 1));
646 if (amos_page
== NULL
) {
647 dev_err(xpc_part
, "can't allocate page of amos\n");
652 * Open up amo-R/W to cpu. This is done on Shub 1.1 systems
653 * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called.
655 ret
= xpc_allow_amo_ops_sn2(amos_page
);
656 if (ret
!= xpSuccess
) {
657 dev_err(xpc_part
, "can't allow amo operations\n");
658 uncached_free_page(__IA64_UNCACHED_OFFSET
|
659 TO_PHYS((u64
)amos_page
), 1);
664 /* clear xpc_vars_sn2 */
665 memset(xpc_vars_sn2
, 0, sizeof(struct xpc_vars_sn2
));
667 xpc_vars_sn2
->version
= XPC_V_VERSION
;
668 xpc_vars_sn2
->activate_IRQ_nasid
= cpuid_to_nasid(0);
669 xpc_vars_sn2
->activate_IRQ_phys_cpuid
= cpu_physical_id(0);
670 xpc_vars_sn2
->vars_part_pa
= xp_pa(xpc_vars_part_sn2
);
671 xpc_vars_sn2
->amos_page_pa
= ia64_tpa((u64
)amos_page
);
672 xpc_vars_sn2
->amos_page
= amos_page
; /* save for next load of XPC */
674 /* clear xpc_vars_part_sn2 */
675 memset((u64
*)xpc_vars_part_sn2
, 0, sizeof(struct xpc_vars_part_sn2
) *
676 XP_MAX_NPARTITIONS_SN2
);
678 /* initialize the activate IRQ related amo variables */
679 for (i
= 0; i
< xpc_nasid_mask_nlongs
; i
++)
680 (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2
+ i
);
682 /* initialize the engaged remote partitions related amo variables */
683 (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2
);
684 (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2
);
690 xpc_increment_heartbeat_sn2(void)
692 xpc_vars_sn2
->heartbeat
++;
696 xpc_offline_heartbeat_sn2(void)
698 xpc_increment_heartbeat_sn2();
699 xpc_vars_sn2
->heartbeat_offline
= 1;
703 xpc_online_heartbeat_sn2(void)
705 xpc_increment_heartbeat_sn2();
706 xpc_vars_sn2
->heartbeat_offline
= 0;
710 xpc_heartbeat_init_sn2(void)
712 DBUG_ON(xpc_vars_sn2
== NULL
);
714 bitmap_zero(xpc_vars_sn2
->heartbeating_to_mask
, XP_MAX_NPARTITIONS_SN2
);
715 xpc_heartbeating_to_mask
= &xpc_vars_sn2
->heartbeating_to_mask
[0];
716 xpc_online_heartbeat_sn2();
720 xpc_heartbeat_exit_sn2(void)
722 xpc_offline_heartbeat_sn2();
725 static enum xp_retval
726 xpc_get_remote_heartbeat_sn2(struct xpc_partition
*part
)
728 struct xpc_vars_sn2
*remote_vars
;
731 remote_vars
= (struct xpc_vars_sn2
*)xpc_remote_copy_buffer_sn2
;
733 /* pull the remote vars structure that contains the heartbeat */
734 ret
= xp_remote_memcpy(xp_pa(remote_vars
),
735 part
->sn
.sn2
.remote_vars_pa
,
737 if (ret
!= xpSuccess
)
740 dev_dbg(xpc_part
, "partid=%d, heartbeat=%ld, last_heartbeat=%ld, "
741 "heartbeat_offline=%ld, HB_mask[0]=0x%lx\n", XPC_PARTID(part
),
742 remote_vars
->heartbeat
, part
->last_heartbeat
,
743 remote_vars
->heartbeat_offline
,
744 remote_vars
->heartbeating_to_mask
[0]);
746 if ((remote_vars
->heartbeat
== part
->last_heartbeat
&&
747 remote_vars
->heartbeat_offline
== 0) ||
748 !xpc_hb_allowed(sn_partition_id
,
749 &remote_vars
->heartbeating_to_mask
)) {
752 part
->last_heartbeat
= remote_vars
->heartbeat
;
759 * Get a copy of the remote partition's XPC variables from the reserved page.
761 * remote_vars points to a buffer that is cacheline aligned for BTE copies and
762 * assumed to be of size XPC_RP_VARS_SIZE.
764 static enum xp_retval
765 xpc_get_remote_vars_sn2(unsigned long remote_vars_pa
,
766 struct xpc_vars_sn2
*remote_vars
)
770 if (remote_vars_pa
== 0)
773 /* pull over the cross partition variables */
774 ret
= xp_remote_memcpy(xp_pa(remote_vars
), remote_vars_pa
,
776 if (ret
!= xpSuccess
)
779 if (XPC_VERSION_MAJOR(remote_vars
->version
) !=
780 XPC_VERSION_MAJOR(XPC_V_VERSION
)) {
788 xpc_request_partition_activation_sn2(struct xpc_rsvd_page
*remote_rp
,
789 unsigned long remote_rp_pa
, int nasid
)
791 xpc_send_local_activate_IRQ_sn2(nasid
);
795 xpc_request_partition_reactivation_sn2(struct xpc_partition
*part
)
797 xpc_send_local_activate_IRQ_sn2(part
->sn
.sn2
.activate_IRQ_nasid
);
801 xpc_request_partition_deactivation_sn2(struct xpc_partition
*part
)
803 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
804 unsigned long irq_flags
;
805 struct amo
*amo
= (struct amo
*)__va(part_sn2
->remote_amos_page_pa
+
806 (XPC_DEACTIVATE_REQUEST_AMO_SN2
*
807 sizeof(struct amo
)));
809 local_irq_save(irq_flags
);
811 /* set bit corresponding to our partid in remote partition's amo */
812 FETCHOP_STORE_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_OR
,
813 BIT(sn_partition_id
));
816 * We must always use the nofault function regardless of whether we
817 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
818 * didn't, we'd never know that the other partition is down and would
819 * keep sending IRQs and amos to it until the heartbeat times out.
821 (void)xp_nofault_PIOR((u64
*)GLOBAL_MMR_ADDR(NASID_GET(&amo
->
823 xp_nofault_PIOR_target
));
825 local_irq_restore(irq_flags
);
828 * Send activate IRQ to get other side to see that we've set our
829 * bit in their deactivate request amo.
831 xpc_send_activate_IRQ_sn2(part_sn2
->remote_amos_page_pa
,
833 part_sn2
->activate_IRQ_nasid
,
834 part_sn2
->activate_IRQ_phys_cpuid
);
838 xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition
*part
)
840 unsigned long irq_flags
;
841 struct amo
*amo
= (struct amo
*)__va(part
->sn
.sn2
.remote_amos_page_pa
+
842 (XPC_DEACTIVATE_REQUEST_AMO_SN2
*
843 sizeof(struct amo
)));
845 local_irq_save(irq_flags
);
847 /* clear bit corresponding to our partid in remote partition's amo */
848 FETCHOP_STORE_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_AND
,
849 ~BIT(sn_partition_id
));
852 * We must always use the nofault function regardless of whether we
853 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
854 * didn't, we'd never know that the other partition is down and would
855 * keep sending IRQs and amos to it until the heartbeat times out.
857 (void)xp_nofault_PIOR((u64
*)GLOBAL_MMR_ADDR(NASID_GET(&amo
->
859 xp_nofault_PIOR_target
));
861 local_irq_restore(irq_flags
);
865 xpc_partition_deactivation_requested_sn2(short partid
)
867 struct amo
*amo
= xpc_vars_sn2
->amos_page
+
868 XPC_DEACTIVATE_REQUEST_AMO_SN2
;
870 /* our partition's amo variable ANDed with partid mask */
871 return (FETCHOP_LOAD_OP(TO_AMO((u64
)&amo
->variable
), FETCHOP_LOAD
) &
876 * Update the remote partition's info.
879 xpc_update_partition_info_sn2(struct xpc_partition
*part
, u8 remote_rp_version
,
880 unsigned long *remote_rp_ts_jiffies
,
881 unsigned long remote_rp_pa
,
882 unsigned long remote_vars_pa
,
883 struct xpc_vars_sn2
*remote_vars
)
885 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
887 part
->remote_rp_version
= remote_rp_version
;
888 dev_dbg(xpc_part
, " remote_rp_version = 0x%016x\n",
889 part
->remote_rp_version
);
891 part
->remote_rp_ts_jiffies
= *remote_rp_ts_jiffies
;
892 dev_dbg(xpc_part
, " remote_rp_ts_jiffies = 0x%016lx\n",
893 part
->remote_rp_ts_jiffies
);
895 part
->remote_rp_pa
= remote_rp_pa
;
896 dev_dbg(xpc_part
, " remote_rp_pa = 0x%016lx\n", part
->remote_rp_pa
);
898 part_sn2
->remote_vars_pa
= remote_vars_pa
;
899 dev_dbg(xpc_part
, " remote_vars_pa = 0x%016lx\n",
900 part_sn2
->remote_vars_pa
);
902 part
->last_heartbeat
= remote_vars
->heartbeat
- 1;
903 dev_dbg(xpc_part
, " last_heartbeat = 0x%016lx\n",
904 part
->last_heartbeat
);
906 part_sn2
->remote_vars_part_pa
= remote_vars
->vars_part_pa
;
907 dev_dbg(xpc_part
, " remote_vars_part_pa = 0x%016lx\n",
908 part_sn2
->remote_vars_part_pa
);
910 part_sn2
->activate_IRQ_nasid
= remote_vars
->activate_IRQ_nasid
;
911 dev_dbg(xpc_part
, " activate_IRQ_nasid = 0x%x\n",
912 part_sn2
->activate_IRQ_nasid
);
914 part_sn2
->activate_IRQ_phys_cpuid
=
915 remote_vars
->activate_IRQ_phys_cpuid
;
916 dev_dbg(xpc_part
, " activate_IRQ_phys_cpuid = 0x%x\n",
917 part_sn2
->activate_IRQ_phys_cpuid
);
919 part_sn2
->remote_amos_page_pa
= remote_vars
->amos_page_pa
;
920 dev_dbg(xpc_part
, " remote_amos_page_pa = 0x%lx\n",
921 part_sn2
->remote_amos_page_pa
);
923 part_sn2
->remote_vars_version
= remote_vars
->version
;
924 dev_dbg(xpc_part
, " remote_vars_version = 0x%x\n",
925 part_sn2
->remote_vars_version
);
929 * Prior code has determined the nasid which generated a activate IRQ.
930 * Inspect that nasid to determine if its partition needs to be activated
933 * A partition is considered "awaiting activation" if our partition
934 * flags indicate it is not active and it has a heartbeat. A
935 * partition is considered "awaiting deactivation" if our partition
936 * flags indicate it is active but it has no heartbeat or it is not
937 * sending its heartbeat to us.
939 * To determine the heartbeat, the remote nasid must have a properly
940 * initialized reserved page.
943 xpc_identify_activate_IRQ_req_sn2(int nasid
)
945 struct xpc_rsvd_page
*remote_rp
;
946 struct xpc_vars_sn2
*remote_vars
;
947 unsigned long remote_rp_pa
;
948 unsigned long remote_vars_pa
;
949 int remote_rp_version
;
951 unsigned long remote_rp_ts_jiffies
= 0;
953 struct xpc_partition
*part
;
954 struct xpc_partition_sn2
*part_sn2
;
957 /* pull over the reserved page structure */
959 remote_rp
= (struct xpc_rsvd_page
*)xpc_remote_copy_buffer_sn2
;
961 ret
= xpc_get_remote_rp(nasid
, NULL
, remote_rp
, &remote_rp_pa
);
962 if (ret
!= xpSuccess
) {
963 dev_warn(xpc_part
, "unable to get reserved page from nasid %d, "
964 "which sent interrupt, reason=%d\n", nasid
, ret
);
968 remote_vars_pa
= remote_rp
->sn
.vars_pa
;
969 remote_rp_version
= remote_rp
->version
;
970 remote_rp_ts_jiffies
= remote_rp
->ts_jiffies
;
972 partid
= remote_rp
->SAL_partid
;
973 part
= &xpc_partitions
[partid
];
974 part_sn2
= &part
->sn
.sn2
;
976 /* pull over the cross partition variables */
978 remote_vars
= (struct xpc_vars_sn2
*)xpc_remote_copy_buffer_sn2
;
980 ret
= xpc_get_remote_vars_sn2(remote_vars_pa
, remote_vars
);
981 if (ret
!= xpSuccess
) {
982 dev_warn(xpc_part
, "unable to get XPC variables from nasid %d, "
983 "which sent interrupt, reason=%d\n", nasid
, ret
);
985 XPC_DEACTIVATE_PARTITION(part
, ret
);
989 part
->activate_IRQ_rcvd
++;
991 dev_dbg(xpc_part
, "partid for nasid %d is %d; IRQs = %d; HB = "
992 "%ld:0x%lx\n", (int)nasid
, (int)partid
, part
->activate_IRQ_rcvd
,
993 remote_vars
->heartbeat
, remote_vars
->heartbeating_to_mask
[0]);
995 if (xpc_partition_disengaged(part
) &&
996 part
->act_state
== XPC_P_AS_INACTIVE
) {
998 xpc_update_partition_info_sn2(part
, remote_rp_version
,
999 &remote_rp_ts_jiffies
,
1000 remote_rp_pa
, remote_vars_pa
,
1003 if (xpc_partition_deactivation_requested_sn2(partid
)) {
1005 * Other side is waiting on us to deactivate even though
1011 xpc_activate_partition(part
);
1015 DBUG_ON(part
->remote_rp_version
== 0);
1016 DBUG_ON(part_sn2
->remote_vars_version
== 0);
1018 if (remote_rp_ts_jiffies
!= part
->remote_rp_ts_jiffies
) {
1020 /* the other side rebooted */
1022 DBUG_ON(xpc_partition_engaged_sn2(partid
));
1023 DBUG_ON(xpc_partition_deactivation_requested_sn2(partid
));
1025 xpc_update_partition_info_sn2(part
, remote_rp_version
,
1026 &remote_rp_ts_jiffies
,
1027 remote_rp_pa
, remote_vars_pa
,
1032 if (part
->disengage_timeout
> 0 && !xpc_partition_disengaged(part
)) {
1033 /* still waiting on other side to disengage from us */
1038 XPC_DEACTIVATE_PARTITION(part
, xpReactivating
);
1039 else if (xpc_partition_deactivation_requested_sn2(partid
))
1040 XPC_DEACTIVATE_PARTITION(part
, xpOtherGoingDown
);
1044 * Loop through the activation amo variables and process any bits
1045 * which are set. Each bit indicates a nasid sending a partition
1046 * activation or deactivation request.
1048 * Return #of IRQs detected.
1051 xpc_identify_activate_IRQ_sender_sn2(void)
1055 unsigned long nasid_mask_long
;
1056 u64 nasid
; /* remote nasid */
1057 int n_IRQs_detected
= 0;
1058 struct amo
*act_amos
;
1060 act_amos
= xpc_vars_sn2
->amos_page
+ XPC_ACTIVATE_IRQ_AMOS_SN2
;
1062 /* scan through activate amo variables looking for non-zero entries */
1063 for (l
= 0; l
< xpc_nasid_mask_nlongs
; l
++) {
1068 nasid_mask_long
= xpc_receive_IRQ_amo_sn2(&act_amos
[l
]);
1070 b
= find_first_bit(&nasid_mask_long
, BITS_PER_LONG
);
1071 if (b
>= BITS_PER_LONG
) {
1072 /* no IRQs from nasids in this amo variable */
1076 dev_dbg(xpc_part
, "amo[%d] gave back 0x%lx\n", l
,
1080 * If this nasid has been added to the machine since
1081 * our partition was reset, this will retain the
1082 * remote nasid in our reserved pages machine mask.
1083 * This is used in the event of module reload.
1085 xpc_mach_nasids
[l
] |= nasid_mask_long
;
1087 /* locate the nasid(s) which sent interrupts */
1091 nasid
= (l
* BITS_PER_LONG
+ b
) * 2;
1092 dev_dbg(xpc_part
, "interrupt from nasid %ld\n", nasid
);
1093 xpc_identify_activate_IRQ_req_sn2(nasid
);
1095 b
= find_next_bit(&nasid_mask_long
, BITS_PER_LONG
,
1097 } while (b
< BITS_PER_LONG
);
1099 return n_IRQs_detected
;
1103 xpc_process_activate_IRQ_rcvd_sn2(void)
1105 unsigned long irq_flags
;
1106 int n_IRQs_expected
;
1107 int n_IRQs_detected
;
1109 DBUG_ON(xpc_activate_IRQ_rcvd
== 0);
1111 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock
, irq_flags
);
1112 n_IRQs_expected
= xpc_activate_IRQ_rcvd
;
1113 xpc_activate_IRQ_rcvd
= 0;
1114 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock
, irq_flags
);
1116 n_IRQs_detected
= xpc_identify_activate_IRQ_sender_sn2();
1117 if (n_IRQs_detected
< n_IRQs_expected
) {
1118 /* retry once to help avoid missing amo */
1119 (void)xpc_identify_activate_IRQ_sender_sn2();
1124 * Setup the channel structures that are sn2 specific.
1126 static enum xp_retval
1127 xpc_setup_ch_structures_sn_sn2(struct xpc_partition
*part
)
1129 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
1130 struct xpc_channel_sn2
*ch_sn2
;
1131 enum xp_retval retval
;
1135 struct timer_list
*timer
;
1136 short partid
= XPC_PARTID(part
);
1138 /* allocate all the required GET/PUT values */
1140 part_sn2
->local_GPs
=
1141 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE
, GFP_KERNEL
,
1142 &part_sn2
->local_GPs_base
);
1143 if (part_sn2
->local_GPs
== NULL
) {
1144 dev_err(xpc_chan
, "can't get memory for local get/put "
1149 part_sn2
->remote_GPs
=
1150 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE
, GFP_KERNEL
,
1151 &part_sn2
->remote_GPs_base
);
1152 if (part_sn2
->remote_GPs
== NULL
) {
1153 dev_err(xpc_chan
, "can't get memory for remote get/put "
1155 retval
= xpNoMemory
;
1159 part_sn2
->remote_GPs_pa
= 0;
1161 /* allocate all the required open and close args */
1163 part_sn2
->local_openclose_args
=
1164 xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE
,
1165 GFP_KERNEL
, &part_sn2
->
1166 local_openclose_args_base
);
1167 if (part_sn2
->local_openclose_args
== NULL
) {
1168 dev_err(xpc_chan
, "can't get memory for local connect args\n");
1169 retval
= xpNoMemory
;
1173 part_sn2
->remote_openclose_args_pa
= 0;
1175 part_sn2
->local_chctl_amo_va
= xpc_init_IRQ_amo_sn2(partid
);
1177 part_sn2
->notify_IRQ_nasid
= 0;
1178 part_sn2
->notify_IRQ_phys_cpuid
= 0;
1179 part_sn2
->remote_chctl_amo_va
= NULL
;
1181 sprintf(part_sn2
->notify_IRQ_owner
, "xpc%02d", partid
);
1182 ret
= request_irq(SGI_XPC_NOTIFY
, xpc_handle_notify_IRQ_sn2
,
1183 IRQF_SHARED
, part_sn2
->notify_IRQ_owner
,
1184 (void *)(u64
)partid
);
1186 dev_err(xpc_chan
, "can't register NOTIFY IRQ handler, "
1187 "errno=%d\n", -ret
);
1188 retval
= xpLackOfResources
;
1192 /* Setup a timer to check for dropped notify IRQs */
1193 timer
= &part_sn2
->dropped_notify_IRQ_timer
;
1196 (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2
;
1197 timer
->data
= (unsigned long)part
;
1198 timer
->expires
= jiffies
+ XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL
;
1201 for (ch_number
= 0; ch_number
< part
->nchannels
; ch_number
++) {
1202 ch_sn2
= &part
->channels
[ch_number
].sn
.sn2
;
1204 ch_sn2
->local_GP
= &part_sn2
->local_GPs
[ch_number
];
1205 ch_sn2
->local_openclose_args
=
1206 &part_sn2
->local_openclose_args
[ch_number
];
1208 mutex_init(&ch_sn2
->msg_to_pull_mutex
);
1212 * Setup the per partition specific variables required by the
1213 * remote partition to establish channel connections with us.
1215 * The setting of the magic # indicates that these per partition
1216 * specific variables are ready to be used.
1218 xpc_vars_part_sn2
[partid
].GPs_pa
= xp_pa(part_sn2
->local_GPs
);
1219 xpc_vars_part_sn2
[partid
].openclose_args_pa
=
1220 xp_pa(part_sn2
->local_openclose_args
);
1221 xpc_vars_part_sn2
[partid
].chctl_amo_pa
=
1222 xp_pa(part_sn2
->local_chctl_amo_va
);
1223 cpuid
= raw_smp_processor_id(); /* any CPU in this partition will do */
1224 xpc_vars_part_sn2
[partid
].notify_IRQ_nasid
= cpuid_to_nasid(cpuid
);
1225 xpc_vars_part_sn2
[partid
].notify_IRQ_phys_cpuid
=
1226 cpu_physical_id(cpuid
);
1227 xpc_vars_part_sn2
[partid
].nchannels
= part
->nchannels
;
1228 xpc_vars_part_sn2
[partid
].magic
= XPC_VP_MAGIC1_SN2
;
1232 /* setup of ch structures failed */
1234 kfree(part_sn2
->local_openclose_args_base
);
1235 part_sn2
->local_openclose_args
= NULL
;
1237 kfree(part_sn2
->remote_GPs_base
);
1238 part_sn2
->remote_GPs
= NULL
;
1240 kfree(part_sn2
->local_GPs_base
);
1241 part_sn2
->local_GPs
= NULL
;
1246 * Teardown the channel structures that are sn2 specific.
1249 xpc_teardown_ch_structures_sn_sn2(struct xpc_partition
*part
)
1251 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
1252 short partid
= XPC_PARTID(part
);
1255 * Indicate that the variables specific to the remote partition are no
1256 * longer available for its use.
1258 xpc_vars_part_sn2
[partid
].magic
= 0;
1260 /* in case we've still got outstanding timers registered... */
1261 del_timer_sync(&part_sn2
->dropped_notify_IRQ_timer
);
1262 free_irq(SGI_XPC_NOTIFY
, (void *)(u64
)partid
);
1264 kfree(part_sn2
->local_openclose_args_base
);
1265 part_sn2
->local_openclose_args
= NULL
;
1266 kfree(part_sn2
->remote_GPs_base
);
1267 part_sn2
->remote_GPs
= NULL
;
1268 kfree(part_sn2
->local_GPs_base
);
1269 part_sn2
->local_GPs
= NULL
;
1270 part_sn2
->local_chctl_amo_va
= NULL
;
1274 * Create a wrapper that hides the underlying mechanism for pulling a cacheline
1275 * (or multiple cachelines) from a remote partition.
1277 * src_pa must be a cacheline aligned physical address on the remote partition.
1278 * dst must be a cacheline aligned virtual address on this partition.
1279 * cnt must be cacheline sized
1281 /* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */
1282 static enum xp_retval
1283 xpc_pull_remote_cachelines_sn2(struct xpc_partition
*part
, void *dst
,
1284 const unsigned long src_pa
, size_t cnt
)
1288 DBUG_ON(src_pa
!= L1_CACHE_ALIGN(src_pa
));
1289 DBUG_ON((unsigned long)dst
!= L1_CACHE_ALIGN((unsigned long)dst
));
1290 DBUG_ON(cnt
!= L1_CACHE_ALIGN(cnt
));
1292 if (part
->act_state
== XPC_P_AS_DEACTIVATING
)
1293 return part
->reason
;
1295 ret
= xp_remote_memcpy(xp_pa(dst
), src_pa
, cnt
);
1296 if (ret
!= xpSuccess
) {
1297 dev_dbg(xpc_chan
, "xp_remote_memcpy() from partition %d failed,"
1298 " ret=%d\n", XPC_PARTID(part
), ret
);
1304 * Pull the remote per partition specific variables from the specified
1307 static enum xp_retval
1308 xpc_pull_remote_vars_part_sn2(struct xpc_partition
*part
)
1310 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
1311 u8 buffer
[L1_CACHE_BYTES
* 2];
1312 struct xpc_vars_part_sn2
*pulled_entry_cacheline
=
1313 (struct xpc_vars_part_sn2
*)L1_CACHE_ALIGN((u64
)buffer
);
1314 struct xpc_vars_part_sn2
*pulled_entry
;
1315 unsigned long remote_entry_cacheline_pa
;
1316 unsigned long remote_entry_pa
;
1317 short partid
= XPC_PARTID(part
);
1320 /* pull the cacheline that contains the variables we're interested in */
1322 DBUG_ON(part_sn2
->remote_vars_part_pa
!=
1323 L1_CACHE_ALIGN(part_sn2
->remote_vars_part_pa
));
1324 DBUG_ON(sizeof(struct xpc_vars_part_sn2
) != L1_CACHE_BYTES
/ 2);
1326 remote_entry_pa
= part_sn2
->remote_vars_part_pa
+
1327 sn_partition_id
* sizeof(struct xpc_vars_part_sn2
);
1329 remote_entry_cacheline_pa
= (remote_entry_pa
& ~(L1_CACHE_BYTES
- 1));
1331 pulled_entry
= (struct xpc_vars_part_sn2
*)((u64
)pulled_entry_cacheline
1332 + (remote_entry_pa
&
1333 (L1_CACHE_BYTES
- 1)));
1335 ret
= xpc_pull_remote_cachelines_sn2(part
, pulled_entry_cacheline
,
1336 remote_entry_cacheline_pa
,
1338 if (ret
!= xpSuccess
) {
1339 dev_dbg(xpc_chan
, "failed to pull XPC vars_part from "
1340 "partition %d, ret=%d\n", partid
, ret
);
1344 /* see if they've been set up yet */
1346 if (pulled_entry
->magic
!= XPC_VP_MAGIC1_SN2
&&
1347 pulled_entry
->magic
!= XPC_VP_MAGIC2_SN2
) {
1349 if (pulled_entry
->magic
!= 0) {
1350 dev_dbg(xpc_chan
, "partition %d's XPC vars_part for "
1351 "partition %d has bad magic value (=0x%lx)\n",
1352 partid
, sn_partition_id
, pulled_entry
->magic
);
1356 /* they've not been initialized yet */
1360 if (xpc_vars_part_sn2
[partid
].magic
== XPC_VP_MAGIC1_SN2
) {
1362 /* validate the variables */
1364 if (pulled_entry
->GPs_pa
== 0 ||
1365 pulled_entry
->openclose_args_pa
== 0 ||
1366 pulled_entry
->chctl_amo_pa
== 0) {
1368 dev_err(xpc_chan
, "partition %d's XPC vars_part for "
1369 "partition %d are not valid\n", partid
,
1371 return xpInvalidAddress
;
1374 /* the variables we imported look to be valid */
1376 part_sn2
->remote_GPs_pa
= pulled_entry
->GPs_pa
;
1377 part_sn2
->remote_openclose_args_pa
=
1378 pulled_entry
->openclose_args_pa
;
1379 part_sn2
->remote_chctl_amo_va
=
1380 (struct amo
*)__va(pulled_entry
->chctl_amo_pa
);
1381 part_sn2
->notify_IRQ_nasid
= pulled_entry
->notify_IRQ_nasid
;
1382 part_sn2
->notify_IRQ_phys_cpuid
=
1383 pulled_entry
->notify_IRQ_phys_cpuid
;
1385 if (part
->nchannels
> pulled_entry
->nchannels
)
1386 part
->nchannels
= pulled_entry
->nchannels
;
1388 /* let the other side know that we've pulled their variables */
1390 xpc_vars_part_sn2
[partid
].magic
= XPC_VP_MAGIC2_SN2
;
1393 if (pulled_entry
->magic
== XPC_VP_MAGIC1_SN2
)
1400 * Establish first contact with the remote partititon. This involves pulling
1401 * the XPC per partition variables from the remote partition and waiting for
1402 * the remote partition to pull ours.
1404 static enum xp_retval
1405 xpc_make_first_contact_sn2(struct xpc_partition
*part
)
1407 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
1411 * Register the remote partition's amos with SAL so it can handle
1412 * and cleanup errors within that address range should the remote
1413 * partition go down. We don't unregister this range because it is
1414 * difficult to tell when outstanding writes to the remote partition
1415 * are finished and thus when it is safe to unregister. This should
1416 * not result in wasted space in the SAL xp_addr_region table because
1417 * we should get the same page for remote_amos_page_pa after module
1418 * reloads and system reboots.
1420 if (sn_register_xp_addr_region(part_sn2
->remote_amos_page_pa
,
1421 PAGE_SIZE
, 1) < 0) {
1422 dev_warn(xpc_part
, "xpc_activating(%d) failed to register "
1423 "xp_addr region\n", XPC_PARTID(part
));
1425 ret
= xpPhysAddrRegFailed
;
1426 XPC_DEACTIVATE_PARTITION(part
, ret
);
1431 * Send activate IRQ to get other side to activate if they've not
1432 * already begun to do so.
1434 xpc_send_activate_IRQ_sn2(part_sn2
->remote_amos_page_pa
,
1435 cnodeid_to_nasid(0),
1436 part_sn2
->activate_IRQ_nasid
,
1437 part_sn2
->activate_IRQ_phys_cpuid
);
1439 while ((ret
= xpc_pull_remote_vars_part_sn2(part
)) != xpSuccess
) {
1440 if (ret
!= xpRetry
) {
1441 XPC_DEACTIVATE_PARTITION(part
, ret
);
1445 dev_dbg(xpc_part
, "waiting to make first contact with "
1446 "partition %d\n", XPC_PARTID(part
));
1448 /* wait a 1/4 of a second or so */
1449 (void)msleep_interruptible(250);
1451 if (part
->act_state
== XPC_P_AS_DEACTIVATING
)
1452 return part
->reason
;
1459 * Get the chctl flags and pull the openclose args and/or remote GPs as needed.
1462 xpc_get_chctl_all_flags_sn2(struct xpc_partition
*part
)
1464 struct xpc_partition_sn2
*part_sn2
= &part
->sn
.sn2
;
1465 unsigned long irq_flags
;
1466 union xpc_channel_ctl_flags chctl
;
1470 * See if there are any chctl flags to be handled.
1473 spin_lock_irqsave(&part
->chctl_lock
, irq_flags
);
1474 chctl
= part
->chctl
;
1475 if (chctl
.all_flags
!= 0)
1476 part
->chctl
.all_flags
= 0;
1478 spin_unlock_irqrestore(&part
->chctl_lock
, irq_flags
);
1480 if (xpc_any_openclose_chctl_flags_set(&chctl
)) {
1481 ret
= xpc_pull_remote_cachelines_sn2(part
, part
->
1482 remote_openclose_args
,
1484 remote_openclose_args_pa
,
1485 XPC_OPENCLOSE_ARGS_SIZE
);
1486 if (ret
!= xpSuccess
) {
1487 XPC_DEACTIVATE_PARTITION(part
, ret
);
1489 dev_dbg(xpc_chan
, "failed to pull openclose args from "
1490 "partition %d, ret=%d\n", XPC_PARTID(part
),
1493 /* don't bother processing chctl flags anymore */
1494 chctl
.all_flags
= 0;
1498 if (xpc_any_msg_chctl_flags_set(&chctl
)) {
1499 ret
= xpc_pull_remote_cachelines_sn2(part
, part_sn2
->remote_GPs
,
1500 part_sn2
->remote_GPs_pa
,
1502 if (ret
!= xpSuccess
) {
1503 XPC_DEACTIVATE_PARTITION(part
, ret
);
1505 dev_dbg(xpc_chan
, "failed to pull GPs from partition "
1506 "%d, ret=%d\n", XPC_PARTID(part
), ret
);
1508 /* don't bother processing chctl flags anymore */
1509 chctl
.all_flags
= 0;
1513 return chctl
.all_flags
;
1517 * Allocate the local message queue and the notify queue.
1519 static enum xp_retval
1520 xpc_allocate_local_msgqueue_sn2(struct xpc_channel
*ch
)
1522 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1523 unsigned long irq_flags
;
1527 for (nentries
= ch
->local_nentries
; nentries
> 0; nentries
--) {
1529 nbytes
= nentries
* ch
->entry_size
;
1530 ch_sn2
->local_msgqueue
=
1531 xpc_kzalloc_cacheline_aligned(nbytes
, GFP_KERNEL
,
1532 &ch_sn2
->local_msgqueue_base
);
1533 if (ch_sn2
->local_msgqueue
== NULL
)
1536 nbytes
= nentries
* sizeof(struct xpc_notify_sn2
);
1537 ch_sn2
->notify_queue
= kzalloc(nbytes
, GFP_KERNEL
);
1538 if (ch_sn2
->notify_queue
== NULL
) {
1539 kfree(ch_sn2
->local_msgqueue_base
);
1540 ch_sn2
->local_msgqueue
= NULL
;
1544 spin_lock_irqsave(&ch
->lock
, irq_flags
);
1545 if (nentries
< ch
->local_nentries
) {
1546 dev_dbg(xpc_chan
, "nentries=%d local_nentries=%d, "
1547 "partid=%d, channel=%d\n", nentries
,
1548 ch
->local_nentries
, ch
->partid
, ch
->number
);
1550 ch
->local_nentries
= nentries
;
1552 spin_unlock_irqrestore(&ch
->lock
, irq_flags
);
1556 dev_dbg(xpc_chan
, "can't get memory for local message queue and notify "
1557 "queue, partid=%d, channel=%d\n", ch
->partid
, ch
->number
);
1562 * Allocate the cached remote message queue.
1564 static enum xp_retval
1565 xpc_allocate_remote_msgqueue_sn2(struct xpc_channel
*ch
)
1567 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1568 unsigned long irq_flags
;
1572 DBUG_ON(ch
->remote_nentries
<= 0);
1574 for (nentries
= ch
->remote_nentries
; nentries
> 0; nentries
--) {
1576 nbytes
= nentries
* ch
->entry_size
;
1577 ch_sn2
->remote_msgqueue
=
1578 xpc_kzalloc_cacheline_aligned(nbytes
, GFP_KERNEL
, &ch_sn2
->
1579 remote_msgqueue_base
);
1580 if (ch_sn2
->remote_msgqueue
== NULL
)
1583 spin_lock_irqsave(&ch
->lock
, irq_flags
);
1584 if (nentries
< ch
->remote_nentries
) {
1585 dev_dbg(xpc_chan
, "nentries=%d remote_nentries=%d, "
1586 "partid=%d, channel=%d\n", nentries
,
1587 ch
->remote_nentries
, ch
->partid
, ch
->number
);
1589 ch
->remote_nentries
= nentries
;
1591 spin_unlock_irqrestore(&ch
->lock
, irq_flags
);
1595 dev_dbg(xpc_chan
, "can't get memory for cached remote message queue, "
1596 "partid=%d, channel=%d\n", ch
->partid
, ch
->number
);
1601 * Allocate message queues and other stuff associated with a channel.
1603 * Note: Assumes all of the channel sizes are filled in.
1605 static enum xp_retval
1606 xpc_setup_msg_structures_sn2(struct xpc_channel
*ch
)
1608 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1611 DBUG_ON(ch
->flags
& XPC_C_SETUP
);
1613 ret
= xpc_allocate_local_msgqueue_sn2(ch
);
1614 if (ret
== xpSuccess
) {
1616 ret
= xpc_allocate_remote_msgqueue_sn2(ch
);
1617 if (ret
!= xpSuccess
) {
1618 kfree(ch_sn2
->local_msgqueue_base
);
1619 ch_sn2
->local_msgqueue
= NULL
;
1620 kfree(ch_sn2
->notify_queue
);
1621 ch_sn2
->notify_queue
= NULL
;
1628 * Free up message queues and other stuff that were allocated for the specified
1632 xpc_teardown_msg_structures_sn2(struct xpc_channel
*ch
)
1634 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1636 DBUG_ON(!spin_is_locked(&ch
->lock
));
1638 ch_sn2
->remote_msgqueue_pa
= 0;
1640 ch_sn2
->local_GP
->get
= 0;
1641 ch_sn2
->local_GP
->put
= 0;
1642 ch_sn2
->remote_GP
.get
= 0;
1643 ch_sn2
->remote_GP
.put
= 0;
1644 ch_sn2
->w_local_GP
.get
= 0;
1645 ch_sn2
->w_local_GP
.put
= 0;
1646 ch_sn2
->w_remote_GP
.get
= 0;
1647 ch_sn2
->w_remote_GP
.put
= 0;
1648 ch_sn2
->next_msg_to_pull
= 0;
1650 if (ch
->flags
& XPC_C_SETUP
) {
1651 dev_dbg(xpc_chan
, "ch->flags=0x%x, partid=%d, channel=%d\n",
1652 ch
->flags
, ch
->partid
, ch
->number
);
1654 kfree(ch_sn2
->local_msgqueue_base
);
1655 ch_sn2
->local_msgqueue
= NULL
;
1656 kfree(ch_sn2
->remote_msgqueue_base
);
1657 ch_sn2
->remote_msgqueue
= NULL
;
1658 kfree(ch_sn2
->notify_queue
);
1659 ch_sn2
->notify_queue
= NULL
;
1664 * Notify those who wanted to be notified upon delivery of their message.
1667 xpc_notify_senders_sn2(struct xpc_channel
*ch
, enum xp_retval reason
, s64 put
)
1669 struct xpc_notify_sn2
*notify
;
1671 s64 get
= ch
->sn
.sn2
.w_remote_GP
.get
- 1;
1673 while (++get
< put
&& atomic_read(&ch
->n_to_notify
) > 0) {
1675 notify
= &ch
->sn
.sn2
.notify_queue
[get
% ch
->local_nentries
];
1678 * See if the notify entry indicates it was associated with
1679 * a message who's sender wants to be notified. It is possible
1680 * that it is, but someone else is doing or has done the
1683 notify_type
= notify
->type
;
1684 if (notify_type
== 0 ||
1685 cmpxchg(¬ify
->type
, notify_type
, 0) != notify_type
) {
1689 DBUG_ON(notify_type
!= XPC_N_CALL
);
1691 atomic_dec(&ch
->n_to_notify
);
1693 if (notify
->func
!= NULL
) {
1694 dev_dbg(xpc_chan
, "notify->func() called, notify=0x%p "
1695 "msg_number=%ld partid=%d channel=%d\n",
1696 (void *)notify
, get
, ch
->partid
, ch
->number
);
1698 notify
->func(reason
, ch
->partid
, ch
->number
,
1701 dev_dbg(xpc_chan
, "notify->func() returned, notify=0x%p"
1702 " msg_number=%ld partid=%d channel=%d\n",
1703 (void *)notify
, get
, ch
->partid
, ch
->number
);
1709 xpc_notify_senders_of_disconnect_sn2(struct xpc_channel
*ch
)
1711 xpc_notify_senders_sn2(ch
, ch
->reason
, ch
->sn
.sn2
.w_local_GP
.put
);
1715 * Clear some of the msg flags in the local message queue.
1718 xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel
*ch
)
1720 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1721 struct xpc_msg_sn2
*msg
;
1724 get
= ch_sn2
->w_remote_GP
.get
;
1726 msg
= (struct xpc_msg_sn2
*)((u64
)ch_sn2
->local_msgqueue
+
1727 (get
% ch
->local_nentries
) *
1730 } while (++get
< ch_sn2
->remote_GP
.get
);
1734 * Clear some of the msg flags in the remote message queue.
1737 xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel
*ch
)
1739 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1740 struct xpc_msg_sn2
*msg
;
1743 put
= ch_sn2
->w_remote_GP
.put
;
1745 msg
= (struct xpc_msg_sn2
*)((u64
)ch_sn2
->remote_msgqueue
+
1746 (put
% ch
->remote_nentries
) *
1749 } while (++put
< ch_sn2
->remote_GP
.put
);
1753 xpc_n_of_deliverable_payloads_sn2(struct xpc_channel
*ch
)
1755 return ch
->sn
.sn2
.w_remote_GP
.put
- ch
->sn
.sn2
.w_local_GP
.get
;
1759 xpc_process_msg_chctl_flags_sn2(struct xpc_partition
*part
, int ch_number
)
1761 struct xpc_channel
*ch
= &part
->channels
[ch_number
];
1762 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1765 ch_sn2
->remote_GP
= part
->sn
.sn2
.remote_GPs
[ch_number
];
1767 /* See what, if anything, has changed for each connected channel */
1769 xpc_msgqueue_ref(ch
);
1771 if (ch_sn2
->w_remote_GP
.get
== ch_sn2
->remote_GP
.get
&&
1772 ch_sn2
->w_remote_GP
.put
== ch_sn2
->remote_GP
.put
) {
1773 /* nothing changed since GPs were last pulled */
1774 xpc_msgqueue_deref(ch
);
1778 if (!(ch
->flags
& XPC_C_CONNECTED
)) {
1779 xpc_msgqueue_deref(ch
);
1784 * First check to see if messages recently sent by us have been
1785 * received by the other side. (The remote GET value will have
1786 * changed since we last looked at it.)
1789 if (ch_sn2
->w_remote_GP
.get
!= ch_sn2
->remote_GP
.get
) {
1792 * We need to notify any senders that want to be notified
1793 * that their sent messages have been received by their
1794 * intended recipients. We need to do this before updating
1795 * w_remote_GP.get so that we don't allocate the same message
1796 * queue entries prematurely (see xpc_allocate_msg()).
1798 if (atomic_read(&ch
->n_to_notify
) > 0) {
1800 * Notify senders that messages sent have been
1801 * received and delivered by the other side.
1803 xpc_notify_senders_sn2(ch
, xpMsgDelivered
,
1804 ch_sn2
->remote_GP
.get
);
1808 * Clear msg->flags in previously sent messages, so that
1809 * they're ready for xpc_allocate_msg().
1811 xpc_clear_local_msgqueue_flags_sn2(ch
);
1813 ch_sn2
->w_remote_GP
.get
= ch_sn2
->remote_GP
.get
;
1815 dev_dbg(xpc_chan
, "w_remote_GP.get changed to %ld, partid=%d, "
1816 "channel=%d\n", ch_sn2
->w_remote_GP
.get
, ch
->partid
,
1820 * If anyone was waiting for message queue entries to become
1821 * available, wake them up.
1823 if (atomic_read(&ch
->n_on_msg_allocate_wq
) > 0)
1824 wake_up(&ch
->msg_allocate_wq
);
1828 * Now check for newly sent messages by the other side. (The remote
1829 * PUT value will have changed since we last looked at it.)
1832 if (ch_sn2
->w_remote_GP
.put
!= ch_sn2
->remote_GP
.put
) {
1834 * Clear msg->flags in previously received messages, so that
1835 * they're ready for xpc_get_deliverable_payload_sn2().
1837 xpc_clear_remote_msgqueue_flags_sn2(ch
);
1839 ch_sn2
->w_remote_GP
.put
= ch_sn2
->remote_GP
.put
;
1841 dev_dbg(xpc_chan
, "w_remote_GP.put changed to %ld, partid=%d, "
1842 "channel=%d\n", ch_sn2
->w_remote_GP
.put
, ch
->partid
,
1845 npayloads_sent
= xpc_n_of_deliverable_payloads_sn2(ch
);
1846 if (npayloads_sent
> 0) {
1847 dev_dbg(xpc_chan
, "msgs waiting to be copied and "
1848 "delivered=%d, partid=%d, channel=%d\n",
1849 npayloads_sent
, ch
->partid
, ch
->number
);
1851 if (ch
->flags
& XPC_C_CONNECTEDCALLOUT_MADE
)
1852 xpc_activate_kthreads(ch
, npayloads_sent
);
1856 xpc_msgqueue_deref(ch
);
1859 static struct xpc_msg_sn2
*
1860 xpc_pull_remote_msg_sn2(struct xpc_channel
*ch
, s64 get
)
1862 struct xpc_partition
*part
= &xpc_partitions
[ch
->partid
];
1863 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1864 unsigned long remote_msg_pa
;
1865 struct xpc_msg_sn2
*msg
;
1871 if (mutex_lock_interruptible(&ch_sn2
->msg_to_pull_mutex
) != 0) {
1872 /* we were interrupted by a signal */
1876 while (get
>= ch_sn2
->next_msg_to_pull
) {
1878 /* pull as many messages as are ready and able to be pulled */
1880 msg_index
= ch_sn2
->next_msg_to_pull
% ch
->remote_nentries
;
1882 DBUG_ON(ch_sn2
->next_msg_to_pull
>= ch_sn2
->w_remote_GP
.put
);
1883 nmsgs
= ch_sn2
->w_remote_GP
.put
- ch_sn2
->next_msg_to_pull
;
1884 if (msg_index
+ nmsgs
> ch
->remote_nentries
) {
1885 /* ignore the ones that wrap the msg queue for now */
1886 nmsgs
= ch
->remote_nentries
- msg_index
;
1889 msg_offset
= msg_index
* ch
->entry_size
;
1890 msg
= (struct xpc_msg_sn2
*)((u64
)ch_sn2
->remote_msgqueue
+
1892 remote_msg_pa
= ch_sn2
->remote_msgqueue_pa
+ msg_offset
;
1894 ret
= xpc_pull_remote_cachelines_sn2(part
, msg
, remote_msg_pa
,
1895 nmsgs
* ch
->entry_size
);
1896 if (ret
!= xpSuccess
) {
1898 dev_dbg(xpc_chan
, "failed to pull %d msgs starting with"
1899 " msg %ld from partition %d, channel=%d, "
1900 "ret=%d\n", nmsgs
, ch_sn2
->next_msg_to_pull
,
1901 ch
->partid
, ch
->number
, ret
);
1903 XPC_DEACTIVATE_PARTITION(part
, ret
);
1905 mutex_unlock(&ch_sn2
->msg_to_pull_mutex
);
1909 ch_sn2
->next_msg_to_pull
+= nmsgs
;
1912 mutex_unlock(&ch_sn2
->msg_to_pull_mutex
);
1914 /* return the message we were looking for */
1915 msg_offset
= (get
% ch
->remote_nentries
) * ch
->entry_size
;
1916 msg
= (struct xpc_msg_sn2
*)((u64
)ch_sn2
->remote_msgqueue
+ msg_offset
);
1922 * Get the next deliverable message's payload.
1925 xpc_get_deliverable_payload_sn2(struct xpc_channel
*ch
)
1927 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1928 struct xpc_msg_sn2
*msg
;
1929 void *payload
= NULL
;
1933 if (ch
->flags
& XPC_C_DISCONNECTING
)
1936 get
= ch_sn2
->w_local_GP
.get
;
1937 rmb(); /* guarantee that .get loads before .put */
1938 if (get
== ch_sn2
->w_remote_GP
.put
)
1941 /* There are messages waiting to be pulled and delivered.
1942 * We need to try to secure one for ourselves. We'll do this
1943 * by trying to increment w_local_GP.get and hope that no one
1944 * else beats us to it. If they do, we'll we'll simply have
1945 * to try again for the next one.
1948 if (cmpxchg(&ch_sn2
->w_local_GP
.get
, get
, get
+ 1) == get
) {
1949 /* we got the entry referenced by get */
1951 dev_dbg(xpc_chan
, "w_local_GP.get changed to %ld, "
1952 "partid=%d, channel=%d\n", get
+ 1,
1953 ch
->partid
, ch
->number
);
1955 /* pull the message from the remote partition */
1957 msg
= xpc_pull_remote_msg_sn2(ch
, get
);
1959 DBUG_ON(msg
!= NULL
&& msg
->number
!= get
);
1960 DBUG_ON(msg
!= NULL
&& (msg
->flags
& XPC_M_SN2_DONE
));
1961 DBUG_ON(msg
!= NULL
&& !(msg
->flags
& XPC_M_SN2_READY
));
1963 payload
= &msg
->payload
;
1973 * Now we actually send the messages that are ready to be sent by advancing
1974 * the local message queue's Put value and then send a chctl msgrequest to the
1975 * recipient partition.
1978 xpc_send_msgs_sn2(struct xpc_channel
*ch
, s64 initial_put
)
1980 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
1981 struct xpc_msg_sn2
*msg
;
1982 s64 put
= initial_put
+ 1;
1983 int send_msgrequest
= 0;
1988 if (put
== ch_sn2
->w_local_GP
.put
)
1991 msg
= (struct xpc_msg_sn2
*)((u64
)ch_sn2
->
1992 local_msgqueue
+ (put
%
1993 ch
->local_nentries
) *
1996 if (!(msg
->flags
& XPC_M_SN2_READY
))
2002 if (put
== initial_put
) {
2003 /* nothing's changed */
2007 if (cmpxchg_rel(&ch_sn2
->local_GP
->put
, initial_put
, put
) !=
2009 /* someone else beat us to it */
2010 DBUG_ON(ch_sn2
->local_GP
->put
< initial_put
);
2014 /* we just set the new value of local_GP->put */
2016 dev_dbg(xpc_chan
, "local_GP->put changed to %ld, partid=%d, "
2017 "channel=%d\n", put
, ch
->partid
, ch
->number
);
2019 send_msgrequest
= 1;
2022 * We need to ensure that the message referenced by
2023 * local_GP->put is not XPC_M_SN2_READY or that local_GP->put
2024 * equals w_local_GP.put, so we'll go have a look.
2029 if (send_msgrequest
)
2030 xpc_send_chctl_msgrequest_sn2(ch
);
2034 * Allocate an entry for a message from the message queue associated with the
2035 * specified channel.
2037 static enum xp_retval
2038 xpc_allocate_msg_sn2(struct xpc_channel
*ch
, u32 flags
,
2039 struct xpc_msg_sn2
**address_of_msg
)
2041 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
2042 struct xpc_msg_sn2
*msg
;
2047 * Get the next available message entry from the local message queue.
2048 * If none are available, we'll make sure that we grab the latest
2055 put
= ch_sn2
->w_local_GP
.put
;
2056 rmb(); /* guarantee that .put loads before .get */
2057 if (put
- ch_sn2
->w_remote_GP
.get
< ch
->local_nentries
) {
2059 /* There are available message entries. We need to try
2060 * to secure one for ourselves. We'll do this by trying
2061 * to increment w_local_GP.put as long as someone else
2062 * doesn't beat us to it. If they do, we'll have to
2065 if (cmpxchg(&ch_sn2
->w_local_GP
.put
, put
, put
+ 1) ==
2067 /* we got the entry referenced by put */
2070 continue; /* try again */
2074 * There aren't any available msg entries at this time.
2076 * In waiting for a message entry to become available,
2077 * we set a timeout in case the other side is not sending
2078 * completion interrupts. This lets us fake a notify IRQ
2079 * that will cause the notify IRQ handler to fetch the latest
2080 * GP values as if an interrupt was sent by the other side.
2082 if (ret
== xpTimeout
)
2083 xpc_send_chctl_local_msgrequest_sn2(ch
);
2085 if (flags
& XPC_NOWAIT
)
2088 ret
= xpc_allocate_msg_wait(ch
);
2089 if (ret
!= xpInterrupted
&& ret
!= xpTimeout
)
2093 /* get the message's address and initialize it */
2094 msg
= (struct xpc_msg_sn2
*)((u64
)ch_sn2
->local_msgqueue
+
2095 (put
% ch
->local_nentries
) *
2098 DBUG_ON(msg
->flags
!= 0);
2101 dev_dbg(xpc_chan
, "w_local_GP.put changed to %ld; msg=0x%p, "
2102 "msg_number=%ld, partid=%d, channel=%d\n", put
+ 1,
2103 (void *)msg
, msg
->number
, ch
->partid
, ch
->number
);
2105 *address_of_msg
= msg
;
2110 * Common code that does the actual sending of the message by advancing the
2111 * local message queue's Put value and sends a chctl msgrequest to the
2112 * partition the message is being sent to.
2114 static enum xp_retval
2115 xpc_send_payload_sn2(struct xpc_channel
*ch
, u32 flags
, void *payload
,
2116 u16 payload_size
, u8 notify_type
, xpc_notify_func func
,
2119 enum xp_retval ret
= xpSuccess
;
2120 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
2121 struct xpc_msg_sn2
*msg
= msg
;
2122 struct xpc_notify_sn2
*notify
= notify
;
2126 DBUG_ON(notify_type
== XPC_N_CALL
&& func
== NULL
);
2128 if (XPC_MSG_SIZE(payload_size
) > ch
->entry_size
)
2129 return xpPayloadTooBig
;
2131 xpc_msgqueue_ref(ch
);
2133 if (ch
->flags
& XPC_C_DISCONNECTING
) {
2137 if (!(ch
->flags
& XPC_C_CONNECTED
)) {
2138 ret
= xpNotConnected
;
2142 ret
= xpc_allocate_msg_sn2(ch
, flags
, &msg
);
2143 if (ret
!= xpSuccess
)
2146 msg_number
= msg
->number
;
2148 if (notify_type
!= 0) {
2150 * Tell the remote side to send an ACK interrupt when the
2151 * message has been delivered.
2153 msg
->flags
|= XPC_M_SN2_INTERRUPT
;
2155 atomic_inc(&ch
->n_to_notify
);
2157 notify
= &ch_sn2
->notify_queue
[msg_number
% ch
->local_nentries
];
2158 notify
->func
= func
;
2160 notify
->type
= notify_type
;
2162 /* ??? Is a mb() needed here? */
2164 if (ch
->flags
& XPC_C_DISCONNECTING
) {
2166 * An error occurred between our last error check and
2167 * this one. We will try to clear the type field from
2168 * the notify entry. If we succeed then
2169 * xpc_disconnect_channel() didn't already process
2172 if (cmpxchg(¬ify
->type
, notify_type
, 0) ==
2174 atomic_dec(&ch
->n_to_notify
);
2181 memcpy(&msg
->payload
, payload
, payload_size
);
2183 msg
->flags
|= XPC_M_SN2_READY
;
2186 * The preceding store of msg->flags must occur before the following
2187 * load of local_GP->put.
2191 /* see if the message is next in line to be sent, if so send it */
2193 put
= ch_sn2
->local_GP
->put
;
2194 if (put
== msg_number
)
2195 xpc_send_msgs_sn2(ch
, put
);
2198 xpc_msgqueue_deref(ch
);
2203 * Now we actually acknowledge the messages that have been delivered and ack'd
2204 * by advancing the cached remote message queue's Get value and if requested
2205 * send a chctl msgrequest to the message sender's partition.
2207 * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition
2208 * that sent the message.
2211 xpc_acknowledge_msgs_sn2(struct xpc_channel
*ch
, s64 initial_get
, u8 msg_flags
)
2213 struct xpc_channel_sn2
*ch_sn2
= &ch
->sn
.sn2
;
2214 struct xpc_msg_sn2
*msg
;
2215 s64 get
= initial_get
+ 1;
2216 int send_msgrequest
= 0;
2221 if (get
== ch_sn2
->w_local_GP
.get
)
2224 msg
= (struct xpc_msg_sn2
*)((u64
)ch_sn2
->
2225 remote_msgqueue
+ (get
%
2226 ch
->remote_nentries
) *
2229 if (!(msg
->flags
& XPC_M_SN2_DONE
))
2232 msg_flags
|= msg
->flags
;
2236 if (get
== initial_get
) {
2237 /* nothing's changed */
2241 if (cmpxchg_rel(&ch_sn2
->local_GP
->get
, initial_get
, get
) !=
2243 /* someone else beat us to it */
2244 DBUG_ON(ch_sn2
->local_GP
->get
<= initial_get
);
2248 /* we just set the new value of local_GP->get */
2250 dev_dbg(xpc_chan
, "local_GP->get changed to %ld, partid=%d, "
2251 "channel=%d\n", get
, ch
->partid
, ch
->number
);
2253 send_msgrequest
= (msg_flags
& XPC_M_SN2_INTERRUPT
);
2256 * We need to ensure that the message referenced by
2257 * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get
2258 * equals w_local_GP.get, so we'll go have a look.
2263 if (send_msgrequest
)
2264 xpc_send_chctl_msgrequest_sn2(ch
);
2268 xpc_received_payload_sn2(struct xpc_channel
*ch
, void *payload
)
2270 struct xpc_msg_sn2
*msg
;
2274 msg
= container_of(payload
, struct xpc_msg_sn2
, payload
);
2275 msg_number
= msg
->number
;
2277 dev_dbg(xpc_chan
, "msg=0x%p, msg_number=%ld, partid=%d, channel=%d\n",
2278 (void *)msg
, msg_number
, ch
->partid
, ch
->number
);
2280 DBUG_ON((((u64
)msg
- (u64
)ch
->remote_msgqueue
) / ch
->entry_size
) !=
2281 msg_number
% ch
->remote_nentries
);
2282 DBUG_ON(msg
->flags
& XPC_M_SN2_DONE
);
2284 msg
->flags
|= XPC_M_SN2_DONE
;
2287 * The preceding store of msg->flags must occur before the following
2288 * load of local_GP->get.
2293 * See if this message is next in line to be acknowledged as having
2296 get
= ch
->sn
.sn2
.local_GP
->get
;
2297 if (get
== msg_number
)
2298 xpc_acknowledge_msgs_sn2(ch
, get
, msg
->flags
);
2307 xpc_setup_partitions_sn
= xpc_setup_partitions_sn_sn2
;
2308 xpc_get_partition_rsvd_page_pa
= xpc_get_partition_rsvd_page_pa_sn2
;
2309 xpc_setup_rsvd_page_sn
= xpc_setup_rsvd_page_sn_sn2
;
2310 xpc_increment_heartbeat
= xpc_increment_heartbeat_sn2
;
2311 xpc_offline_heartbeat
= xpc_offline_heartbeat_sn2
;
2312 xpc_online_heartbeat
= xpc_online_heartbeat_sn2
;
2313 xpc_heartbeat_init
= xpc_heartbeat_init_sn2
;
2314 xpc_heartbeat_exit
= xpc_heartbeat_exit_sn2
;
2315 xpc_get_remote_heartbeat
= xpc_get_remote_heartbeat_sn2
;
2317 xpc_request_partition_activation
= xpc_request_partition_activation_sn2
;
2318 xpc_request_partition_reactivation
=
2319 xpc_request_partition_reactivation_sn2
;
2320 xpc_request_partition_deactivation
=
2321 xpc_request_partition_deactivation_sn2
;
2322 xpc_cancel_partition_deactivation_request
=
2323 xpc_cancel_partition_deactivation_request_sn2
;
2325 xpc_process_activate_IRQ_rcvd
= xpc_process_activate_IRQ_rcvd_sn2
;
2326 xpc_setup_ch_structures_sn
= xpc_setup_ch_structures_sn_sn2
;
2327 xpc_teardown_ch_structures_sn
= xpc_teardown_ch_structures_sn_sn2
;
2328 xpc_make_first_contact
= xpc_make_first_contact_sn2
;
2330 xpc_get_chctl_all_flags
= xpc_get_chctl_all_flags_sn2
;
2331 xpc_send_chctl_closerequest
= xpc_send_chctl_closerequest_sn2
;
2332 xpc_send_chctl_closereply
= xpc_send_chctl_closereply_sn2
;
2333 xpc_send_chctl_openrequest
= xpc_send_chctl_openrequest_sn2
;
2334 xpc_send_chctl_openreply
= xpc_send_chctl_openreply_sn2
;
2336 xpc_save_remote_msgqueue_pa
= xpc_save_remote_msgqueue_pa_sn2
;
2338 xpc_setup_msg_structures
= xpc_setup_msg_structures_sn2
;
2339 xpc_teardown_msg_structures
= xpc_teardown_msg_structures_sn2
;
2341 xpc_notify_senders_of_disconnect
= xpc_notify_senders_of_disconnect_sn2
;
2342 xpc_process_msg_chctl_flags
= xpc_process_msg_chctl_flags_sn2
;
2343 xpc_n_of_deliverable_payloads
= xpc_n_of_deliverable_payloads_sn2
;
2344 xpc_get_deliverable_payload
= xpc_get_deliverable_payload_sn2
;
2346 xpc_indicate_partition_engaged
= xpc_indicate_partition_engaged_sn2
;
2347 xpc_indicate_partition_disengaged
=
2348 xpc_indicate_partition_disengaged_sn2
;
2349 xpc_partition_engaged
= xpc_partition_engaged_sn2
;
2350 xpc_any_partition_engaged
= xpc_any_partition_engaged_sn2
;
2351 xpc_assume_partition_disengaged
= xpc_assume_partition_disengaged_sn2
;
2353 xpc_send_payload
= xpc_send_payload_sn2
;
2354 xpc_received_payload
= xpc_received_payload_sn2
;
2356 if (offsetof(struct xpc_msg_sn2
, payload
) > XPC_MSG_HDR_MAX_SIZE
) {
2357 dev_err(xpc_part
, "header portion of struct xpc_msg_sn2 is "
2358 "larger than %d\n", XPC_MSG_HDR_MAX_SIZE
);
2362 buf_size
= max(XPC_RP_VARS_SIZE
,
2363 XPC_RP_HEADER_SIZE
+ XP_NASID_MASK_BYTES_SN2
);
2364 xpc_remote_copy_buffer_sn2
= xpc_kmalloc_cacheline_aligned(buf_size
,
2366 &xpc_remote_copy_buffer_base_sn2
);
2367 if (xpc_remote_copy_buffer_sn2
== NULL
) {
2368 dev_err(xpc_part
, "can't get memory for remote copy buffer\n");
2372 /* open up protections for IPI and [potentially] amo operations */
2373 xpc_allow_IPI_ops_sn2();
2374 xpc_allow_amo_ops_shub_wars_1_1_sn2();
2377 * This is safe to do before the xpc_hb_checker thread has started
2378 * because the handler releases a wait queue. If an interrupt is
2379 * received before the thread is waiting, it will not go to sleep,
2380 * but rather immediately process the interrupt.
2382 ret
= request_irq(SGI_XPC_ACTIVATE
, xpc_handle_activate_IRQ_sn2
, 0,
2385 dev_err(xpc_part
, "can't register ACTIVATE IRQ handler, "
2386 "errno=%d\n", -ret
);
2387 xpc_disallow_IPI_ops_sn2();
2388 kfree(xpc_remote_copy_buffer_base_sn2
);
2396 free_irq(SGI_XPC_ACTIVATE
, NULL
);
2397 xpc_disallow_IPI_ops_sn2();
2398 kfree(xpc_remote_copy_buffer_base_sn2
);