?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
sgi-xp: incoming XPC channel messages can come in after the channel's partition struc...
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / misc / sgi-xp / xpc_uv.c
blobf2cee6e364b6975499c7e4543a1b2c2cca0c2a53
1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved.
7 */
8
9 /*
10 * Cross Partition Communication (XPC) uv-based functions.
11 *
12 * Architecture specific implementation of common functions.
13 *
14 */
15
16 #include <linux/kernel.h>
17 #include <linux/mm.h>
18 #include <linux/interrupt.h>
19 #include <linux/delay.h>
20 #include <linux/device.h>
21 #include <linux/err.h>
22 #include <asm/uv/uv_hub.h>
23 #if defined CONFIG_X86_64
24 #include <asm/uv/bios.h>
25 #include <asm/uv/uv_irq.h>
26 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
27 #include <asm/sn/intr.h>
28 #include <asm/sn/sn_sal.h>
29 #endif
30 #include "../sgi-gru/gru.h"
31 #include "../sgi-gru/grukservices.h"
32 #include "xpc.h"
33
34 #if defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
35 struct uv_IO_APIC_route_entry {
36 __u64 vector : 8,
37 delivery_mode : 3,
38 dest_mode : 1,
39 delivery_status : 1,
40 polarity : 1,
41 __reserved_1 : 1,
42 trigger : 1,
43 mask : 1,
44 __reserved_2 : 15,
45 dest : 32;
46 };
47 #endif
48
49 static struct xpc_heartbeat_uv *xpc_heartbeat_uv;
50
51 #define XPC_ACTIVATE_MSG_SIZE_UV (1 * GRU_CACHE_LINE_BYTES)
52 #define XPC_ACTIVATE_MQ_SIZE_UV (4 * XP_MAX_NPARTITIONS_UV * \
53 XPC_ACTIVATE_MSG_SIZE_UV)
54 #define XPC_ACTIVATE_IRQ_NAME "xpc_activate"
55
56 #define XPC_NOTIFY_MSG_SIZE_UV (2 * GRU_CACHE_LINE_BYTES)
57 #define XPC_NOTIFY_MQ_SIZE_UV (4 * XP_MAX_NPARTITIONS_UV * \
58 XPC_NOTIFY_MSG_SIZE_UV)
59 #define XPC_NOTIFY_IRQ_NAME "xpc_notify"
60
61 static struct xpc_gru_mq_uv *xpc_activate_mq_uv;
62 static struct xpc_gru_mq_uv *xpc_notify_mq_uv;
63
64 static int
65 xpc_setup_partitions_uv(void)
66 {
67 short partid;
68 struct xpc_partition_uv *part_uv;
69
70 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
71 part_uv = &xpc_partitions[partid].sn.uv;
72
73 mutex_init(&part_uv->cached_activate_gru_mq_desc_mutex);
74 spin_lock_init(&part_uv->flags_lock);
75 part_uv->remote_act_state = XPC_P_AS_INACTIVE;
76 }
77 return 0;
78 }
79
80 static void
81 xpc_teardown_partitions_uv(void)
82 {
83 short partid;
84 struct xpc_partition_uv *part_uv;
85 unsigned long irq_flags;
86
87 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
88 part_uv = &xpc_partitions[partid].sn.uv;
89
90 if (part_uv->cached_activate_gru_mq_desc != NULL) {
91 mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex);
92 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
93 part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
94 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
95 kfree(part_uv->cached_activate_gru_mq_desc);
96 part_uv->cached_activate_gru_mq_desc = NULL;
97 mutex_unlock(&part_uv->
98 cached_activate_gru_mq_desc_mutex);
99 }
100 }
101 }
102
103 static int
104 xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name)
105 {
106 int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
107
108 #if defined CONFIG_X86_64
109 mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset);
110 if (mq->irq < 0) {
111 dev_err(xpc_part, "uv_setup_irq() returned error=%d\n",
112 -mq->irq);
113 return mq->irq;
114 }
115
116 mq->mmr_value = uv_read_global_mmr64(mmr_pnode, mq->mmr_offset);
117
118 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
119 if (strcmp(irq_name, XPC_ACTIVATE_IRQ_NAME) == 0)
120 mq->irq = SGI_XPC_ACTIVATE;
121 else if (strcmp(irq_name, XPC_NOTIFY_IRQ_NAME) == 0)
122 mq->irq = SGI_XPC_NOTIFY;
123 else
124 return -EINVAL;
125
126 mq->mmr_value = (unsigned long)cpu_physical_id(cpu) << 32 | mq->irq;
127 uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mq->mmr_value);
128 #else
129 #error not a supported configuration
130 #endif
131
132 return 0;
133 }
134
135 static void
136 xpc_release_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq)
137 {
138 #if defined CONFIG_X86_64
139 uv_teardown_irq(mq->irq, mq->mmr_blade, mq->mmr_offset);
140
141 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
142 int mmr_pnode;
143 unsigned long mmr_value;
144
145 mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
146 mmr_value = 1UL << 16;
147
148 uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mmr_value);
149 #else
150 #error not a supported configuration
151 #endif
152 }
153
154 static int
155 xpc_gru_mq_watchlist_alloc_uv(struct xpc_gru_mq_uv *mq)
156 {
157 int ret;
158
159 #if defined CONFIG_X86_64
160 ret = uv_bios_mq_watchlist_alloc(mq->mmr_blade, uv_gpa(mq->address),
161 mq->order, &mq->mmr_offset);
162 if (ret < 0) {
163 dev_err(xpc_part, "uv_bios_mq_watchlist_alloc() failed, "
164 "ret=%d\n", ret);
165 return ret;
166 }
167 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
168 ret = sn_mq_watchlist_alloc(mq->mmr_blade, (void *)uv_gpa(mq->address),
169 mq->order, &mq->mmr_offset);
170 if (ret < 0) {
171 dev_err(xpc_part, "sn_mq_watchlist_alloc() failed, ret=%d\n",
172 ret);
173 return -EBUSY;
174 }
175 #else
176 #error not a supported configuration
177 #endif
178
179 mq->watchlist_num = ret;
180 return 0;
181 }
182
183 static void
184 xpc_gru_mq_watchlist_free_uv(struct xpc_gru_mq_uv *mq)
185 {
186 int ret;
187
188 #if defined CONFIG_X86_64
189 ret = uv_bios_mq_watchlist_free(mq->mmr_blade, mq->watchlist_num);
190 BUG_ON(ret != BIOS_STATUS_SUCCESS);
191 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
192 ret = sn_mq_watchlist_free(mq->mmr_blade, mq->watchlist_num);
193 BUG_ON(ret != SALRET_OK);
194 #else
195 #error not a supported configuration
196 #endif
197 }
198
199 static struct xpc_gru_mq_uv *
200 xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name,
201 irq_handler_t irq_handler)
202 {
203 enum xp_retval xp_ret;
204 int ret;
205 int nid;
206 int pg_order;
207 struct page *page;
208 struct xpc_gru_mq_uv *mq;
209 struct uv_IO_APIC_route_entry *mmr_value;
210
211 mq = kmalloc(sizeof(struct xpc_gru_mq_uv), GFP_KERNEL);
212 if (mq == NULL) {
213 dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
214 "a xpc_gru_mq_uv structure\n");
215 ret = -ENOMEM;
216 goto out_0;
217 }
218
219 mq->gru_mq_desc = kzalloc(sizeof(struct gru_message_queue_desc),
220 GFP_KERNEL);
221 if (mq->gru_mq_desc == NULL) {
222 dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
223 "a gru_message_queue_desc structure\n");
224 ret = -ENOMEM;
225 goto out_1;
226 }
227
228 pg_order = get_order(mq_size);
229 mq->order = pg_order + PAGE_SHIFT;
230 mq_size = 1UL << mq->order;
231
232 mq->mmr_blade = uv_cpu_to_blade_id(cpu);
233
234 nid = cpu_to_node(cpu);
235 page = alloc_pages_exact_node(nid, GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
236 pg_order);
237 if (page == NULL) {
238 dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d "
239 "bytes of memory on nid=%d for GRU mq\n", mq_size, nid);
240 ret = -ENOMEM;
241 goto out_2;
242 }
243 mq->address = page_address(page);
244
245 /* enable generation of irq when GRU mq operation occurs to this mq */
246 ret = xpc_gru_mq_watchlist_alloc_uv(mq);
247 if (ret != 0)
248 goto out_3;
249
250 ret = xpc_get_gru_mq_irq_uv(mq, cpu, irq_name);
251 if (ret != 0)
252 goto out_4;
253
254 ret = request_irq(mq->irq, irq_handler, 0, irq_name, NULL);
255 if (ret != 0) {
256 dev_err(xpc_part, "request_irq(irq=%d) returned error=%d\n",
257 mq->irq, -ret);
258 goto out_5;
259 }
260
261 mmr_value = (struct uv_IO_APIC_route_entry *)&mq->mmr_value;
262 ret = gru_create_message_queue(mq->gru_mq_desc, mq->address, mq_size,
263 nid, mmr_value->vector, mmr_value->dest);
264 if (ret != 0) {
265 dev_err(xpc_part, "gru_create_message_queue() returned "
266 "error=%d\n", ret);
267 ret = -EINVAL;
268 goto out_6;
269 }
270
271 /* allow other partitions to access this GRU mq */
272 xp_ret = xp_expand_memprotect(xp_pa(mq->address), mq_size);
273 if (xp_ret != xpSuccess) {
274 ret = -EACCES;
275 goto out_6;
276 }
277
278 return mq;
279
280 /* something went wrong */
281 out_6:
282 free_irq(mq->irq, NULL);
283 out_5:
284 xpc_release_gru_mq_irq_uv(mq);
285 out_4:
286 xpc_gru_mq_watchlist_free_uv(mq);
287 out_3:
288 free_pages((unsigned long)mq->address, pg_order);
289 out_2:
290 kfree(mq->gru_mq_desc);
291 out_1:
292 kfree(mq);
293 out_0:
294 return ERR_PTR(ret);
295 }
296
297 static void
298 xpc_destroy_gru_mq_uv(struct xpc_gru_mq_uv *mq)
299 {
300 unsigned int mq_size;
301 int pg_order;
302 int ret;
303
304 /* disallow other partitions to access GRU mq */
305 mq_size = 1UL << mq->order;
306 ret = xp_restrict_memprotect(xp_pa(mq->address), mq_size);
307 BUG_ON(ret != xpSuccess);
308
309 /* unregister irq handler and release mq irq/vector mapping */
310 free_irq(mq->irq, NULL);
311 xpc_release_gru_mq_irq_uv(mq);
312
313 /* disable generation of irq when GRU mq op occurs to this mq */
314 xpc_gru_mq_watchlist_free_uv(mq);
315
316 pg_order = mq->order - PAGE_SHIFT;
317 free_pages((unsigned long)mq->address, pg_order);
318
319 kfree(mq);
320 }
321
322 static enum xp_retval
323 xpc_send_gru_msg(struct gru_message_queue_desc *gru_mq_desc, void *msg,
324 size_t msg_size)
325 {
326 enum xp_retval xp_ret;
327 int ret;
328
329 while (1) {
330 ret = gru_send_message_gpa(gru_mq_desc, msg, msg_size);
331 if (ret == MQE_OK) {
332 xp_ret = xpSuccess;
333 break;
334 }
335
336 if (ret == MQE_QUEUE_FULL) {
337 dev_dbg(xpc_chan, "gru_send_message_gpa() returned "
338 "error=MQE_QUEUE_FULL\n");
339 /* !!! handle QLimit reached; delay & try again */
340 /* ??? Do we add a limit to the number of retries? */
341 (void)msleep_interruptible(10);
342 } else if (ret == MQE_CONGESTION) {
343 dev_dbg(xpc_chan, "gru_send_message_gpa() returned "
344 "error=MQE_CONGESTION\n");
345 /* !!! handle LB Overflow; simply try again */
346 /* ??? Do we add a limit to the number of retries? */
347 } else {
348 /* !!! Currently this is MQE_UNEXPECTED_CB_ERR */
349 dev_err(xpc_chan, "gru_send_message_gpa() returned "
350 "error=%d\n", ret);
351 xp_ret = xpGruSendMqError;
352 break;
353 }
354 }
355 return xp_ret;
356 }
357
358 static void
359 xpc_process_activate_IRQ_rcvd_uv(void)
360 {
361 unsigned long irq_flags;
362 short partid;
363 struct xpc_partition *part;
364 u8 act_state_req;
365
366 DBUG_ON(xpc_activate_IRQ_rcvd == 0);
367
368 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
369 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
370 part = &xpc_partitions[partid];
371
372 if (part->sn.uv.act_state_req == 0)
373 continue;
374
375 xpc_activate_IRQ_rcvd--;
376 BUG_ON(xpc_activate_IRQ_rcvd < 0);
377
378 act_state_req = part->sn.uv.act_state_req;
379 part->sn.uv.act_state_req = 0;
380 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
381
382 if (act_state_req == XPC_P_ASR_ACTIVATE_UV) {
383 if (part->act_state == XPC_P_AS_INACTIVE)
384 xpc_activate_partition(part);
385 else if (part->act_state == XPC_P_AS_DEACTIVATING)
386 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
387
388 } else if (act_state_req == XPC_P_ASR_REACTIVATE_UV) {
389 if (part->act_state == XPC_P_AS_INACTIVE)
390 xpc_activate_partition(part);
391 else
392 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
393
394 } else if (act_state_req == XPC_P_ASR_DEACTIVATE_UV) {
395 XPC_DEACTIVATE_PARTITION(part, part->sn.uv.reason);
396
397 } else {
398 BUG();
399 }
400
401 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
402 if (xpc_activate_IRQ_rcvd == 0)
403 break;
404 }
405 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
406
407 }
408
409 static void
410 xpc_handle_activate_mq_msg_uv(struct xpc_partition *part,
411 struct xpc_activate_mq_msghdr_uv *msg_hdr,
412 int part_setup,
413 int *wakeup_hb_checker)
414 {
415 unsigned long irq_flags;
416 struct xpc_partition_uv *part_uv = &part->sn.uv;
417 struct xpc_openclose_args *args;
418
419 part_uv->remote_act_state = msg_hdr->act_state;
420
421 switch (msg_hdr->type) {
422 case XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV:
423 /* syncing of remote_act_state was just done above */
424 break;
425
426 case XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV: {
427 struct xpc_activate_mq_msg_activate_req_uv *msg;
428
429 /*
430 * ??? Do we deal here with ts_jiffies being different
431 * ??? if act_state != XPC_P_AS_INACTIVE instead of
432 * ??? below?
433 */
434 msg = container_of(msg_hdr, struct
435 xpc_activate_mq_msg_activate_req_uv, hdr);
436
437 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
438 if (part_uv->act_state_req == 0)
439 xpc_activate_IRQ_rcvd++;
440 part_uv->act_state_req = XPC_P_ASR_ACTIVATE_UV;
441 part->remote_rp_pa = msg->rp_gpa; /* !!! _pa is _gpa */
442 part->remote_rp_ts_jiffies = msg_hdr->rp_ts_jiffies;
443 part_uv->heartbeat_gpa = msg->heartbeat_gpa;
444
445 if (msg->activate_gru_mq_desc_gpa !=
446 part_uv->activate_gru_mq_desc_gpa) {
447 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
448 part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
449 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
450 part_uv->activate_gru_mq_desc_gpa =
451 msg->activate_gru_mq_desc_gpa;
452 }
453 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
454
455 (*wakeup_hb_checker)++;
456 break;
457 }
458 case XPC_ACTIVATE_MQ_MSG_DEACTIVATE_REQ_UV: {
459 struct xpc_activate_mq_msg_deactivate_req_uv *msg;
460
461 msg = container_of(msg_hdr, struct
462 xpc_activate_mq_msg_deactivate_req_uv, hdr);
463
464 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
465 if (part_uv->act_state_req == 0)
466 xpc_activate_IRQ_rcvd++;
467 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
468 part_uv->reason = msg->reason;
469 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
470
471 (*wakeup_hb_checker)++;
472 return;
473 }
474 case XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREQUEST_UV: {
475 struct xpc_activate_mq_msg_chctl_closerequest_uv *msg;
476
477 if (!part_setup)
478 break;
479
480 msg = container_of(msg_hdr, struct
481 xpc_activate_mq_msg_chctl_closerequest_uv,
482 hdr);
483 args = &part->remote_openclose_args[msg->ch_number];
484 args->reason = msg->reason;
485
486 spin_lock_irqsave(&part->chctl_lock, irq_flags);
487 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREQUEST;
488 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
489
490 xpc_wakeup_channel_mgr(part);
491 break;
492 }
493 case XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREPLY_UV: {
494 struct xpc_activate_mq_msg_chctl_closereply_uv *msg;
495
496 if (!part_setup)
497 break;
498
499 msg = container_of(msg_hdr, struct
500 xpc_activate_mq_msg_chctl_closereply_uv,
501 hdr);
502
503 spin_lock_irqsave(&part->chctl_lock, irq_flags);
504 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREPLY;
505 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
506
507 xpc_wakeup_channel_mgr(part);
508 break;
509 }
510 case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREQUEST_UV: {
511 struct xpc_activate_mq_msg_chctl_openrequest_uv *msg;
512
513 if (!part_setup)
514 break;
515
516 msg = container_of(msg_hdr, struct
517 xpc_activate_mq_msg_chctl_openrequest_uv,
518 hdr);
519 args = &part->remote_openclose_args[msg->ch_number];
520 args->entry_size = msg->entry_size;
521 args->local_nentries = msg->local_nentries;
522
523 spin_lock_irqsave(&part->chctl_lock, irq_flags);
524 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREQUEST;
525 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
526
527 xpc_wakeup_channel_mgr(part);
528 break;
529 }
530 case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV: {
531 struct xpc_activate_mq_msg_chctl_openreply_uv *msg;
532
533 if (!part_setup)
534 break;
535
536 msg = container_of(msg_hdr, struct
537 xpc_activate_mq_msg_chctl_openreply_uv, hdr);
538 args = &part->remote_openclose_args[msg->ch_number];
539 args->remote_nentries = msg->remote_nentries;
540 args->local_nentries = msg->local_nentries;
541 args->local_msgqueue_pa = msg->notify_gru_mq_desc_gpa;
542
543 spin_lock_irqsave(&part->chctl_lock, irq_flags);
544 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREPLY;
545 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
546
547 xpc_wakeup_channel_mgr(part);
548 break;
549 }
550 case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENCOMPLETE_UV: {
551 struct xpc_activate_mq_msg_chctl_opencomplete_uv *msg;
552
553 if (!part_setup)
554 break;
555
556 msg = container_of(msg_hdr, struct
557 xpc_activate_mq_msg_chctl_opencomplete_uv, hdr);
558 spin_lock_irqsave(&part->chctl_lock, irq_flags);
559 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENCOMPLETE;
560 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
561
562 xpc_wakeup_channel_mgr(part);
563 }
564 case XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV:
565 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
566 part_uv->flags |= XPC_P_ENGAGED_UV;
567 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
568 break;
569
570 case XPC_ACTIVATE_MQ_MSG_MARK_DISENGAGED_UV:
571 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
572 part_uv->flags &= ~XPC_P_ENGAGED_UV;
573 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
574 break;
575
576 default:
577 dev_err(xpc_part, "received unknown activate_mq msg type=%d "
578 "from partition=%d\n", msg_hdr->type, XPC_PARTID(part));
579
580 /* get hb checker to deactivate from the remote partition */
581 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
582 if (part_uv->act_state_req == 0)
583 xpc_activate_IRQ_rcvd++;
584 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
585 part_uv->reason = xpBadMsgType;
586 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
587
588 (*wakeup_hb_checker)++;
589 return;
590 }
591
592 if (msg_hdr->rp_ts_jiffies != part->remote_rp_ts_jiffies &&
593 part->remote_rp_ts_jiffies != 0) {
594 /*
595 * ??? Does what we do here need to be sensitive to
596 * ??? act_state or remote_act_state?
597 */
598 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
599 if (part_uv->act_state_req == 0)
600 xpc_activate_IRQ_rcvd++;
601 part_uv->act_state_req = XPC_P_ASR_REACTIVATE_UV;
602 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
603
604 (*wakeup_hb_checker)++;
605 }
606 }
607
608 static irqreturn_t
609 xpc_handle_activate_IRQ_uv(int irq, void *dev_id)
610 {
611 struct xpc_activate_mq_msghdr_uv *msg_hdr;
612 short partid;
613 struct xpc_partition *part;
614 int wakeup_hb_checker = 0;
615 int part_referenced;
616
617 while (1) {
618 msg_hdr = gru_get_next_message(xpc_activate_mq_uv->gru_mq_desc);
619 if (msg_hdr == NULL)
620 break;
621
622 partid = msg_hdr->partid;
623 if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) {
624 dev_err(xpc_part, "xpc_handle_activate_IRQ_uv() "
625 "received invalid partid=0x%x in message\n",
626 partid);
627 } else {
628 part = &xpc_partitions[partid];
629
630 part_referenced = xpc_part_ref(part);
631 xpc_handle_activate_mq_msg_uv(part, msg_hdr,
632 part_referenced,
633 &wakeup_hb_checker);
634 if (part_referenced)
635 xpc_part_deref(part);
636 }
637
638 gru_free_message(xpc_activate_mq_uv->gru_mq_desc, msg_hdr);
639 }
640
641 if (wakeup_hb_checker)
642 wake_up_interruptible(&xpc_activate_IRQ_wq);
643
644 return IRQ_HANDLED;
645 }
646
647 static enum xp_retval
648 xpc_cache_remote_gru_mq_desc_uv(struct gru_message_queue_desc *gru_mq_desc,
649 unsigned long gru_mq_desc_gpa)
650 {
651 enum xp_retval ret;
652
653 ret = xp_remote_memcpy(uv_gpa(gru_mq_desc), gru_mq_desc_gpa,
654 sizeof(struct gru_message_queue_desc));
655 if (ret == xpSuccess)
656 gru_mq_desc->mq = NULL;
657
658 return ret;
659 }
660
661 static enum xp_retval
662 xpc_send_activate_IRQ_uv(struct xpc_partition *part, void *msg, size_t msg_size,
663 int msg_type)
664 {
665 struct xpc_activate_mq_msghdr_uv *msg_hdr = msg;
666 struct xpc_partition_uv *part_uv = &part->sn.uv;
667 struct gru_message_queue_desc *gru_mq_desc;
668 unsigned long irq_flags;
669 enum xp_retval ret;
670
671 DBUG_ON(msg_size > XPC_ACTIVATE_MSG_SIZE_UV);
672
673 msg_hdr->type = msg_type;
674 msg_hdr->partid = xp_partition_id;
675 msg_hdr->act_state = part->act_state;
676 msg_hdr->rp_ts_jiffies = xpc_rsvd_page->ts_jiffies;
677
678 mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex);
679 again:
680 if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV)) {
681 gru_mq_desc = part_uv->cached_activate_gru_mq_desc;
682 if (gru_mq_desc == NULL) {
683 gru_mq_desc = kmalloc(sizeof(struct
684 gru_message_queue_desc),
685 GFP_KERNEL);
686 if (gru_mq_desc == NULL) {
687 ret = xpNoMemory;
688 goto done;
689 }
690 part_uv->cached_activate_gru_mq_desc = gru_mq_desc;
691 }
692
693 ret = xpc_cache_remote_gru_mq_desc_uv(gru_mq_desc,
694 part_uv->
695 activate_gru_mq_desc_gpa);
696 if (ret != xpSuccess)
697 goto done;
698
699 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
700 part_uv->flags |= XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
701 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
702 }
703
704 /* ??? Is holding a spin_lock (ch->lock) during this call a bad idea? */
705 ret = xpc_send_gru_msg(part_uv->cached_activate_gru_mq_desc, msg,
706 msg_size);
707 if (ret != xpSuccess) {
708 smp_rmb(); /* ensure a fresh copy of part_uv->flags */
709 if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV))
710 goto again;
711 }
712 done:
713 mutex_unlock(&part_uv->cached_activate_gru_mq_desc_mutex);
714 return ret;
715 }
716
717 static void
718 xpc_send_activate_IRQ_part_uv(struct xpc_partition *part, void *msg,
719 size_t msg_size, int msg_type)
720 {
721 enum xp_retval ret;
722
723 ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type);
724 if (unlikely(ret != xpSuccess))
725 XPC_DEACTIVATE_PARTITION(part, ret);
726 }
727
728 static void
729 xpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags,
730 void *msg, size_t msg_size, int msg_type)
731 {
732 struct xpc_partition *part = &xpc_partitions[ch->partid];
733 enum xp_retval ret;
734
735 ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type);
736 if (unlikely(ret != xpSuccess)) {
737 if (irq_flags != NULL)
738 spin_unlock_irqrestore(&ch->lock, *irq_flags);
739
740 XPC_DEACTIVATE_PARTITION(part, ret);
741
742 if (irq_flags != NULL)
743 spin_lock_irqsave(&ch->lock, *irq_flags);
744 }
745 }
746
747 static void
748 xpc_send_local_activate_IRQ_uv(struct xpc_partition *part, int act_state_req)
749 {
750 unsigned long irq_flags;
751 struct xpc_partition_uv *part_uv = &part->sn.uv;
752
753 /*
754 * !!! Make our side think that the remote partition sent an activate
755 * !!! mq message our way by doing what the activate IRQ handler would
756 * !!! do had one really been sent.
757 */
758
759 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
760 if (part_uv->act_state_req == 0)
761 xpc_activate_IRQ_rcvd++;
762 part_uv->act_state_req = act_state_req;
763 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
764
765 wake_up_interruptible(&xpc_activate_IRQ_wq);
766 }
767
768 static enum xp_retval
769 xpc_get_partition_rsvd_page_pa_uv(void *buf, u64 *cookie, unsigned long *rp_pa,
770 size_t *len)
771 {
772 s64 status;
773 enum xp_retval ret;
774
775 #if defined CONFIG_X86_64
776 status = uv_bios_reserved_page_pa((u64)buf, cookie, (u64 *)rp_pa,
777 (u64 *)len);
778 if (status == BIOS_STATUS_SUCCESS)
779 ret = xpSuccess;
780 else if (status == BIOS_STATUS_MORE_PASSES)
781 ret = xpNeedMoreInfo;
782 else
783 ret = xpBiosError;
784
785 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
786 status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len);
787 if (status == SALRET_OK)
788 ret = xpSuccess;
789 else if (status == SALRET_MORE_PASSES)
790 ret = xpNeedMoreInfo;
791 else
792 ret = xpSalError;
793
794 #else
795 #error not a supported configuration
796 #endif
797
798 return ret;
799 }
800
801 static int
802 xpc_setup_rsvd_page_uv(struct xpc_rsvd_page *rp)
803 {
804 xpc_heartbeat_uv =
805 &xpc_partitions[sn_partition_id].sn.uv.cached_heartbeat;
806 rp->sn.uv.heartbeat_gpa = uv_gpa(xpc_heartbeat_uv);
807 rp->sn.uv.activate_gru_mq_desc_gpa =
808 uv_gpa(xpc_activate_mq_uv->gru_mq_desc);
809 return 0;
810 }
811
812 static void
813 xpc_allow_hb_uv(short partid)
814 {
815 }
816
817 static void
818 xpc_disallow_hb_uv(short partid)
819 {
820 }
821
822 static void
823 xpc_disallow_all_hbs_uv(void)
824 {
825 }
826
827 static void
828 xpc_increment_heartbeat_uv(void)
829 {
830 xpc_heartbeat_uv->value++;
831 }
832
833 static void
834 xpc_offline_heartbeat_uv(void)
835 {
836 xpc_increment_heartbeat_uv();
837 xpc_heartbeat_uv->offline = 1;
838 }
839
840 static void
841 xpc_online_heartbeat_uv(void)
842 {
843 xpc_increment_heartbeat_uv();
844 xpc_heartbeat_uv->offline = 0;
845 }
846
847 static void
848 xpc_heartbeat_init_uv(void)
849 {
850 xpc_heartbeat_uv->value = 1;
851 xpc_heartbeat_uv->offline = 0;
852 }
853
854 static void
855 xpc_heartbeat_exit_uv(void)
856 {
857 xpc_offline_heartbeat_uv();
858 }
859
860 static enum xp_retval
861 xpc_get_remote_heartbeat_uv(struct xpc_partition *part)
862 {
863 struct xpc_partition_uv *part_uv = &part->sn.uv;
864 enum xp_retval ret;
865
866 ret = xp_remote_memcpy(uv_gpa(&part_uv->cached_heartbeat),
867 part_uv->heartbeat_gpa,
868 sizeof(struct xpc_heartbeat_uv));
869 if (ret != xpSuccess)
870 return ret;
871
872 if (part_uv->cached_heartbeat.value == part->last_heartbeat &&
873 !part_uv->cached_heartbeat.offline) {
874
875 ret = xpNoHeartbeat;
876 } else {
877 part->last_heartbeat = part_uv->cached_heartbeat.value;
878 }
879 return ret;
880 }
881
882 static void
883 xpc_request_partition_activation_uv(struct xpc_rsvd_page *remote_rp,
884 unsigned long remote_rp_gpa, int nasid)
885 {
886 short partid = remote_rp->SAL_partid;
887 struct xpc_partition *part = &xpc_partitions[partid];
888 struct xpc_activate_mq_msg_activate_req_uv msg;
889
890 part->remote_rp_pa = remote_rp_gpa; /* !!! _pa here is really _gpa */
891 part->remote_rp_ts_jiffies = remote_rp->ts_jiffies;
892 part->sn.uv.heartbeat_gpa = remote_rp->sn.uv.heartbeat_gpa;
893 part->sn.uv.activate_gru_mq_desc_gpa =
894 remote_rp->sn.uv.activate_gru_mq_desc_gpa;
895
896 /*
897 * ??? Is it a good idea to make this conditional on what is
898 * ??? potentially stale state information?
899 */
900 if (part->sn.uv.remote_act_state == XPC_P_AS_INACTIVE) {
901 msg.rp_gpa = uv_gpa(xpc_rsvd_page);
902 msg.heartbeat_gpa = xpc_rsvd_page->sn.uv.heartbeat_gpa;
903 msg.activate_gru_mq_desc_gpa =
904 xpc_rsvd_page->sn.uv.activate_gru_mq_desc_gpa;
905 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
906 XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV);
907 }
908
909 if (part->act_state == XPC_P_AS_INACTIVE)
910 xpc_send_local_activate_IRQ_uv(part, XPC_P_ASR_ACTIVATE_UV);
911 }
912
913 static void
914 xpc_request_partition_reactivation_uv(struct xpc_partition *part)
915 {
916 xpc_send_local_activate_IRQ_uv(part, XPC_P_ASR_ACTIVATE_UV);
917 }
918
919 static void
920 xpc_request_partition_deactivation_uv(struct xpc_partition *part)
921 {
922 struct xpc_activate_mq_msg_deactivate_req_uv msg;
923
924 /*
925 * ??? Is it a good idea to make this conditional on what is
926 * ??? potentially stale state information?
927 */
928 if (part->sn.uv.remote_act_state != XPC_P_AS_DEACTIVATING &&
929 part->sn.uv.remote_act_state != XPC_P_AS_INACTIVE) {
930
931 msg.reason = part->reason;
932 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
933 XPC_ACTIVATE_MQ_MSG_DEACTIVATE_REQ_UV);
934 }
935 }
936
937 static void
938 xpc_cancel_partition_deactivation_request_uv(struct xpc_partition *part)
939 {
940 /* nothing needs to be done */
941 return;
942 }
943
944 static void
945 xpc_init_fifo_uv(struct xpc_fifo_head_uv *head)
946 {
947 head->first = NULL;
948 head->last = NULL;
949 spin_lock_init(&head->lock);
950 head->n_entries = 0;
951 }
952
953 static void *
954 xpc_get_fifo_entry_uv(struct xpc_fifo_head_uv *head)
955 {
956 unsigned long irq_flags;
957 struct xpc_fifo_entry_uv *first;
958
959 spin_lock_irqsave(&head->lock, irq_flags);
960 first = head->first;
961 if (head->first != NULL) {
962 head->first = first->next;
963 if (head->first == NULL)
964 head->last = NULL;
965 }
966 head->n_entries--;
967 BUG_ON(head->n_entries < 0);
968 spin_unlock_irqrestore(&head->lock, irq_flags);
969 first->next = NULL;
970 return first;
971 }
972
973 static void
974 xpc_put_fifo_entry_uv(struct xpc_fifo_head_uv *head,
975 struct xpc_fifo_entry_uv *last)
976 {
977 unsigned long irq_flags;
978
979 last->next = NULL;
980 spin_lock_irqsave(&head->lock, irq_flags);
981 if (head->last != NULL)
982 head->last->next = last;
983 else
984 head->first = last;
985 head->last = last;
986 head->n_entries++;
987 spin_unlock_irqrestore(&head->lock, irq_flags);
988 }
989
990 static int
991 xpc_n_of_fifo_entries_uv(struct xpc_fifo_head_uv *head)
992 {
993 return head->n_entries;
994 }
995
996 /*
997 * Setup the channel structures that are uv specific.
998 */
999 static enum xp_retval
1000 xpc_setup_ch_structures_uv(struct xpc_partition *part)
1001 {
1002 struct xpc_channel_uv *ch_uv;
1003 int ch_number;
1004
1005 for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
1006 ch_uv = &part->channels[ch_number].sn.uv;
1007
1008 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
1009 xpc_init_fifo_uv(&ch_uv->recv_msg_list);
1010 }
1011
1012 return xpSuccess;
1013 }
1014
1015 /*
1016 * Teardown the channel structures that are uv specific.
1017 */
1018 static void
1019 xpc_teardown_ch_structures_uv(struct xpc_partition *part)
1020 {
1021 /* nothing needs to be done */
1022 return;
1023 }
1024
1025 static enum xp_retval
1026 xpc_make_first_contact_uv(struct xpc_partition *part)
1027 {
1028 struct xpc_activate_mq_msg_uv msg;
1029
1030 /*
1031 * We send a sync msg to get the remote partition's remote_act_state
1032 * updated to our current act_state which at this point should
1033 * be XPC_P_AS_ACTIVATING.
1034 */
1035 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
1036 XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV);
1037
1038 while (part->sn.uv.remote_act_state != XPC_P_AS_ACTIVATING) {
1039
1040 dev_dbg(xpc_part, "waiting to make first contact with "
1041 "partition %d\n", XPC_PARTID(part));
1042
1043 /* wait a 1/4 of a second or so */
1044 (void)msleep_interruptible(250);
1045
1046 if (part->act_state == XPC_P_AS_DEACTIVATING)
1047 return part->reason;
1048 }
1049
1050 return xpSuccess;
1051 }
1052
1053 static u64
1054 xpc_get_chctl_all_flags_uv(struct xpc_partition *part)
1055 {
1056 unsigned long irq_flags;
1057 union xpc_channel_ctl_flags chctl;
1058
1059 spin_lock_irqsave(&part->chctl_lock, irq_flags);
1060 chctl = part->chctl;
1061 if (chctl.all_flags != 0)
1062 part->chctl.all_flags = 0;
1063
1064 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
1065 return chctl.all_flags;
1066 }
1067
1068 static enum xp_retval
1069 xpc_allocate_send_msg_slot_uv(struct xpc_channel *ch)
1070 {
1071 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
1072 struct xpc_send_msg_slot_uv *msg_slot;
1073 unsigned long irq_flags;
1074 int nentries;
1075 int entry;
1076 size_t nbytes;
1077
1078 for (nentries = ch->local_nentries; nentries > 0; nentries--) {
1079 nbytes = nentries * sizeof(struct xpc_send_msg_slot_uv);
1080 ch_uv->send_msg_slots = kzalloc(nbytes, GFP_KERNEL);
1081 if (ch_uv->send_msg_slots == NULL)
1082 continue;
1083
1084 for (entry = 0; entry < nentries; entry++) {
1085 msg_slot = &ch_uv->send_msg_slots[entry];
1086
1087 msg_slot->msg_slot_number = entry;
1088 xpc_put_fifo_entry_uv(&ch_uv->msg_slot_free_list,
1089 &msg_slot->next);
1090 }
1091
1092 spin_lock_irqsave(&ch->lock, irq_flags);
1093 if (nentries < ch->local_nentries)
1094 ch->local_nentries = nentries;
1095 spin_unlock_irqrestore(&ch->lock, irq_flags);
1096 return xpSuccess;
1097 }
1098
1099 return xpNoMemory;
1100 }
1101
1102 static enum xp_retval
1103 xpc_allocate_recv_msg_slot_uv(struct xpc_channel *ch)
1104 {
1105 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
1106 struct xpc_notify_mq_msg_uv *msg_slot;
1107 unsigned long irq_flags;
1108 int nentries;
1109 int entry;
1110 size_t nbytes;
1111
1112 for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
1113 nbytes = nentries * ch->entry_size;
1114 ch_uv->recv_msg_slots = kzalloc(nbytes, GFP_KERNEL);
1115 if (ch_uv->recv_msg_slots == NULL)
1116 continue;
1117
1118 for (entry = 0; entry < nentries; entry++) {
1119 msg_slot = ch_uv->recv_msg_slots +
1120 entry * ch->entry_size;
1121
1122 msg_slot->hdr.msg_slot_number = entry;
1123 }
1124
1125 spin_lock_irqsave(&ch->lock, irq_flags);
1126 if (nentries < ch->remote_nentries)
1127 ch->remote_nentries = nentries;
1128 spin_unlock_irqrestore(&ch->lock, irq_flags);
1129 return xpSuccess;
1130 }
1131
1132 return xpNoMemory;
1133 }
1134
1135 /*
1136 * Allocate msg_slots associated with the channel.
1137 */
1138 static enum xp_retval
1139 xpc_setup_msg_structures_uv(struct xpc_channel *ch)
1140 {
1141 static enum xp_retval ret;
1142 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
1143
1144 DBUG_ON(ch->flags & XPC_C_SETUP);
1145
1146 ch_uv->cached_notify_gru_mq_desc = kmalloc(sizeof(struct
1147 gru_message_queue_desc),
1148 GFP_KERNEL);
1149 if (ch_uv->cached_notify_gru_mq_desc == NULL)
1150 return xpNoMemory;
1151
1152 ret = xpc_allocate_send_msg_slot_uv(ch);
1153 if (ret == xpSuccess) {
1154
1155 ret = xpc_allocate_recv_msg_slot_uv(ch);
1156 if (ret != xpSuccess) {
1157 kfree(ch_uv->send_msg_slots);
1158 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
1159 }
1160 }
1161 return ret;
1162 }
1163
1164 /*
1165 * Free up msg_slots and clear other stuff that were setup for the specified
1166 * channel.
1167 */
1168 static void
1169 xpc_teardown_msg_structures_uv(struct xpc_channel *ch)
1170 {
1171 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
1172
1173 DBUG_ON(!spin_is_locked(&ch->lock));
1174
1175 kfree(ch_uv->cached_notify_gru_mq_desc);
1176 ch_uv->cached_notify_gru_mq_desc = NULL;
1177
1178 if (ch->flags & XPC_C_SETUP) {
1179 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
1180 kfree(ch_uv->send_msg_slots);
1181 xpc_init_fifo_uv(&ch_uv->recv_msg_list);
1182 kfree(ch_uv->recv_msg_slots);
1183 }
1184 }
1185
1186 static void
1187 xpc_send_chctl_closerequest_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1188 {
1189 struct xpc_activate_mq_msg_chctl_closerequest_uv msg;
1190
1191 msg.ch_number = ch->number;
1192 msg.reason = ch->reason;
1193 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1194 XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREQUEST_UV);
1195 }
1196
1197 static void
1198 xpc_send_chctl_closereply_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1199 {
1200 struct xpc_activate_mq_msg_chctl_closereply_uv msg;
1201
1202 msg.ch_number = ch->number;
1203 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1204 XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREPLY_UV);
1205 }
1206
1207 static void
1208 xpc_send_chctl_openrequest_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1209 {
1210 struct xpc_activate_mq_msg_chctl_openrequest_uv msg;
1211
1212 msg.ch_number = ch->number;
1213 msg.entry_size = ch->entry_size;
1214 msg.local_nentries = ch->local_nentries;
1215 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1216 XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREQUEST_UV);
1217 }
1218
1219 static void
1220 xpc_send_chctl_openreply_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1221 {
1222 struct xpc_activate_mq_msg_chctl_openreply_uv msg;
1223
1224 msg.ch_number = ch->number;
1225 msg.local_nentries = ch->local_nentries;
1226 msg.remote_nentries = ch->remote_nentries;
1227 msg.notify_gru_mq_desc_gpa = uv_gpa(xpc_notify_mq_uv->gru_mq_desc);
1228 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1229 XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV);
1230 }
1231
1232 static void
1233 xpc_send_chctl_opencomplete_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1234 {
1235 struct xpc_activate_mq_msg_chctl_opencomplete_uv msg;
1236
1237 msg.ch_number = ch->number;
1238 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1239 XPC_ACTIVATE_MQ_MSG_CHCTL_OPENCOMPLETE_UV);
1240 }
1241
1242 static void
1243 xpc_send_chctl_local_msgrequest_uv(struct xpc_partition *part, int ch_number)
1244 {
1245 unsigned long irq_flags;
1246
1247 spin_lock_irqsave(&part->chctl_lock, irq_flags);
1248 part->chctl.flags[ch_number] |= XPC_CHCTL_MSGREQUEST;
1249 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
1250
1251 xpc_wakeup_channel_mgr(part);
1252 }
1253
1254 static enum xp_retval
1255 xpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch,
1256 unsigned long gru_mq_desc_gpa)
1257 {
1258 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
1259
1260 DBUG_ON(ch_uv->cached_notify_gru_mq_desc == NULL);
1261 return xpc_cache_remote_gru_mq_desc_uv(ch_uv->cached_notify_gru_mq_desc,
1262 gru_mq_desc_gpa);
1263 }
1264
1265 static void
1266 xpc_indicate_partition_engaged_uv(struct xpc_partition *part)
1267 {
1268 struct xpc_activate_mq_msg_uv msg;
1269
1270 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
1271 XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV);
1272 }
1273
1274 static void
1275 xpc_indicate_partition_disengaged_uv(struct xpc_partition *part)
1276 {
1277 struct xpc_activate_mq_msg_uv msg;
1278
1279 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
1280 XPC_ACTIVATE_MQ_MSG_MARK_DISENGAGED_UV);
1281 }
1282
1283 static void
1284 xpc_assume_partition_disengaged_uv(short partid)
1285 {
1286 struct xpc_partition_uv *part_uv = &xpc_partitions[partid].sn.uv;
1287 unsigned long irq_flags;
1288
1289 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
1290 part_uv->flags &= ~XPC_P_ENGAGED_UV;
1291 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
1292 }
1293
1294 static int
1295 xpc_partition_engaged_uv(short partid)
1296 {
1297 return (xpc_partitions[partid].sn.uv.flags & XPC_P_ENGAGED_UV) != 0;
1298 }
1299
1300 static int
1301 xpc_any_partition_engaged_uv(void)
1302 {
1303 struct xpc_partition_uv *part_uv;
1304 short partid;
1305
1306 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
1307 part_uv = &xpc_partitions[partid].sn.uv;
1308 if ((part_uv->flags & XPC_P_ENGAGED_UV) != 0)
1309 return 1;
1310 }
1311 return 0;
1312 }
1313
1314 static enum xp_retval
1315 xpc_allocate_msg_slot_uv(struct xpc_channel *ch, u32 flags,
1316 struct xpc_send_msg_slot_uv **address_of_msg_slot)
1317 {
1318 enum xp_retval ret;
1319 struct xpc_send_msg_slot_uv *msg_slot;
1320 struct xpc_fifo_entry_uv *entry;
1321
1322 while (1) {
1323 entry = xpc_get_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list);
1324 if (entry != NULL)
1325 break;
1326
1327 if (flags & XPC_NOWAIT)
1328 return xpNoWait;
1329
1330 ret = xpc_allocate_msg_wait(ch);
1331 if (ret != xpInterrupted && ret != xpTimeout)
1332 return ret;
1333 }
1334
1335 msg_slot = container_of(entry, struct xpc_send_msg_slot_uv, next);
1336 *address_of_msg_slot = msg_slot;
1337 return xpSuccess;
1338 }
1339
1340 static void
1341 xpc_free_msg_slot_uv(struct xpc_channel *ch,
1342 struct xpc_send_msg_slot_uv *msg_slot)
1343 {
1344 xpc_put_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list, &msg_slot->next);
1345
1346 /* wakeup anyone waiting for a free msg slot */
1347 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
1348 wake_up(&ch->msg_allocate_wq);
1349 }
1350
1351 static void
1352 xpc_notify_sender_uv(struct xpc_channel *ch,
1353 struct xpc_send_msg_slot_uv *msg_slot,
1354 enum xp_retval reason)
1355 {
1356 xpc_notify_func func = msg_slot->func;
1357
1358 if (func != NULL && cmpxchg(&msg_slot->func, func, NULL) == func) {
1359
1360 atomic_dec(&ch->n_to_notify);
1361
1362 dev_dbg(xpc_chan, "msg_slot->func() called, msg_slot=0x%p "
1363 "msg_slot_number=%d partid=%d channel=%d\n", msg_slot,
1364 msg_slot->msg_slot_number, ch->partid, ch->number);
1365
1366 func(reason, ch->partid, ch->number, msg_slot->key);
1367
1368 dev_dbg(xpc_chan, "msg_slot->func() returned, msg_slot=0x%p "
1369 "msg_slot_number=%d partid=%d channel=%d\n", msg_slot,
1370 msg_slot->msg_slot_number, ch->partid, ch->number);
1371 }
1372 }
1373
1374 static void
1375 xpc_handle_notify_mq_ack_uv(struct xpc_channel *ch,
1376 struct xpc_notify_mq_msg_uv *msg)
1377 {
1378 struct xpc_send_msg_slot_uv *msg_slot;
1379 int entry = msg->hdr.msg_slot_number % ch->local_nentries;
1380
1381 msg_slot = &ch->sn.uv.send_msg_slots[entry];
1382
1383 BUG_ON(msg_slot->msg_slot_number != msg->hdr.msg_slot_number);
1384 msg_slot->msg_slot_number += ch->local_nentries;
1385
1386 if (msg_slot->func != NULL)
1387 xpc_notify_sender_uv(ch, msg_slot, xpMsgDelivered);
1388
1389 xpc_free_msg_slot_uv(ch, msg_slot);
1390 }
1391
1392 static void
1393 xpc_handle_notify_mq_msg_uv(struct xpc_partition *part,
1394 struct xpc_notify_mq_msg_uv *msg)
1395 {
1396 struct xpc_partition_uv *part_uv = &part->sn.uv;
1397 struct xpc_channel *ch;
1398 struct xpc_channel_uv *ch_uv;
1399 struct xpc_notify_mq_msg_uv *msg_slot;
1400 unsigned long irq_flags;
1401 int ch_number = msg->hdr.ch_number;
1402
1403 if (unlikely(ch_number >= part->nchannels)) {
1404 dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received invalid "
1405 "channel number=0x%x in message from partid=%d\n",
1406 ch_number, XPC_PARTID(part));
1407
1408 /* get hb checker to deactivate from the remote partition */
1409 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1410 if (part_uv->act_state_req == 0)
1411 xpc_activate_IRQ_rcvd++;
1412 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
1413 part_uv->reason = xpBadChannelNumber;
1414 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1415
1416 wake_up_interruptible(&xpc_activate_IRQ_wq);
1417 return;
1418 }
1419
1420 ch = &part->channels[ch_number];
1421 xpc_msgqueue_ref(ch);
1422
1423 if (!(ch->flags & XPC_C_CONNECTED)) {
1424 xpc_msgqueue_deref(ch);
1425 return;
1426 }
1427
1428 /* see if we're really dealing with an ACK for a previously sent msg */
1429 if (msg->hdr.size == 0) {
1430 xpc_handle_notify_mq_ack_uv(ch, msg);
1431 xpc_msgqueue_deref(ch);
1432 return;
1433 }
1434
1435 /* we're dealing with a normal message sent via the notify_mq */
1436 ch_uv = &ch->sn.uv;
1437
1438 msg_slot = ch_uv->recv_msg_slots +
1439 (msg->hdr.msg_slot_number % ch->remote_nentries) * ch->entry_size;
1440
1441 BUG_ON(msg->hdr.msg_slot_number != msg_slot->hdr.msg_slot_number);
1442 BUG_ON(msg_slot->hdr.size != 0);
1443
1444 memcpy(msg_slot, msg, msg->hdr.size);
1445
1446 xpc_put_fifo_entry_uv(&ch_uv->recv_msg_list, &msg_slot->hdr.u.next);
1447
1448 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) {
1449 /*
1450 * If there is an existing idle kthread get it to deliver
1451 * the payload, otherwise we'll have to get the channel mgr
1452 * for this partition to create a kthread to do the delivery.
1453 */
1454 if (atomic_read(&ch->kthreads_idle) > 0)
1455 wake_up_nr(&ch->idle_wq, 1);
1456 else
1457 xpc_send_chctl_local_msgrequest_uv(part, ch->number);
1458 }
1459 xpc_msgqueue_deref(ch);
1460 }
1461
1462 static irqreturn_t
1463 xpc_handle_notify_IRQ_uv(int irq, void *dev_id)
1464 {
1465 struct xpc_notify_mq_msg_uv *msg;
1466 short partid;
1467 struct xpc_partition *part;
1468
1469 while ((msg = gru_get_next_message(xpc_notify_mq_uv->gru_mq_desc)) !=
1470 NULL) {
1471
1472 partid = msg->hdr.partid;
1473 if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) {
1474 dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received "
1475 "invalid partid=0x%x in message\n", partid);
1476 } else {
1477 part = &xpc_partitions[partid];
1478
1479 if (xpc_part_ref(part)) {
1480 xpc_handle_notify_mq_msg_uv(part, msg);
1481 xpc_part_deref(part);
1482 }
1483 }
1484
1485 gru_free_message(xpc_notify_mq_uv->gru_mq_desc, msg);
1486 }
1487
1488 return IRQ_HANDLED;
1489 }
1490
1491 static int
1492 xpc_n_of_deliverable_payloads_uv(struct xpc_channel *ch)
1493 {
1494 return xpc_n_of_fifo_entries_uv(&ch->sn.uv.recv_msg_list);
1495 }
1496
1497 static void
1498 xpc_process_msg_chctl_flags_uv(struct xpc_partition *part, int ch_number)
1499 {
1500 struct xpc_channel *ch = &part->channels[ch_number];
1501 int ndeliverable_payloads;
1502
1503 xpc_msgqueue_ref(ch);
1504
1505 ndeliverable_payloads = xpc_n_of_deliverable_payloads_uv(ch);
1506
1507 if (ndeliverable_payloads > 0 &&
1508 (ch->flags & XPC_C_CONNECTED) &&
1509 (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)) {
1510
1511 xpc_activate_kthreads(ch, ndeliverable_payloads);
1512 }
1513
1514 xpc_msgqueue_deref(ch);
1515 }
1516
1517 static enum xp_retval
1518 xpc_send_payload_uv(struct xpc_channel *ch, u32 flags, void *payload,
1519 u16 payload_size, u8 notify_type, xpc_notify_func func,
1520 void *key)
1521 {
1522 enum xp_retval ret = xpSuccess;
1523 struct xpc_send_msg_slot_uv *msg_slot = NULL;
1524 struct xpc_notify_mq_msg_uv *msg;
1525 u8 msg_buffer[XPC_NOTIFY_MSG_SIZE_UV];
1526 size_t msg_size;
1527
1528 DBUG_ON(notify_type != XPC_N_CALL);
1529
1530 msg_size = sizeof(struct xpc_notify_mq_msghdr_uv) + payload_size;
1531 if (msg_size > ch->entry_size)
1532 return xpPayloadTooBig;
1533
1534 xpc_msgqueue_ref(ch);
1535
1536 if (ch->flags & XPC_C_DISCONNECTING) {
1537 ret = ch->reason;
1538 goto out_1;
1539 }
1540 if (!(ch->flags & XPC_C_CONNECTED)) {
1541 ret = xpNotConnected;
1542 goto out_1;
1543 }
1544
1545 ret = xpc_allocate_msg_slot_uv(ch, flags, &msg_slot);
1546 if (ret != xpSuccess)
1547 goto out_1;
1548
1549 if (func != NULL) {
1550 atomic_inc(&ch->n_to_notify);
1551
1552 msg_slot->key = key;
1553 smp_wmb(); /* a non-NULL func must hit memory after the key */
1554 msg_slot->func = func;
1555
1556 if (ch->flags & XPC_C_DISCONNECTING) {
1557 ret = ch->reason;
1558 goto out_2;
1559 }
1560 }
1561
1562 msg = (struct xpc_notify_mq_msg_uv *)&msg_buffer;
1563 msg->hdr.partid = xp_partition_id;
1564 msg->hdr.ch_number = ch->number;
1565 msg->hdr.size = msg_size;
1566 msg->hdr.msg_slot_number = msg_slot->msg_slot_number;
1567 memcpy(&msg->payload, payload, payload_size);
1568
1569 ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,
1570 msg_size);
1571 if (ret == xpSuccess)
1572 goto out_1;
1573
1574 XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);
1575 out_2:
1576 if (func != NULL) {
1577 /*
1578 * Try to NULL the msg_slot's func field. If we fail, then
1579 * xpc_notify_senders_of_disconnect_uv() beat us to it, in which
1580 * case we need to pretend we succeeded to send the message
1581 * since the user will get a callout for the disconnect error
1582 * by xpc_notify_senders_of_disconnect_uv(), and to also get an
1583 * error returned here will confuse them. Additionally, since
1584 * in this case the channel is being disconnected we don't need
1585 * to put the the msg_slot back on the free list.
1586 */
1587 if (cmpxchg(&msg_slot->func, func, NULL) != func) {
1588 ret = xpSuccess;
1589 goto out_1;
1590 }
1591
1592 msg_slot->key = NULL;
1593 atomic_dec(&ch->n_to_notify);
1594 }
1595 xpc_free_msg_slot_uv(ch, msg_slot);
1596 out_1:
1597 xpc_msgqueue_deref(ch);
1598 return ret;
1599 }
1600
1601 /*
1602 * Tell the callers of xpc_send_notify() that the status of their payloads
1603 * is unknown because the channel is now disconnecting.
1604 *
1605 * We don't worry about putting these msg_slots on the free list since the
1606 * msg_slots themselves are about to be kfree'd.
1607 */
1608 static void
1609 xpc_notify_senders_of_disconnect_uv(struct xpc_channel *ch)
1610 {
1611 struct xpc_send_msg_slot_uv *msg_slot;
1612 int entry;
1613
1614 DBUG_ON(!(ch->flags & XPC_C_DISCONNECTING));
1615
1616 for (entry = 0; entry < ch->local_nentries; entry++) {
1617
1618 if (atomic_read(&ch->n_to_notify) == 0)
1619 break;
1620
1621 msg_slot = &ch->sn.uv.send_msg_slots[entry];
1622 if (msg_slot->func != NULL)
1623 xpc_notify_sender_uv(ch, msg_slot, ch->reason);
1624 }
1625 }
1626
1627 /*
1628 * Get the next deliverable message's payload.
1629 */
1630 static void *
1631 xpc_get_deliverable_payload_uv(struct xpc_channel *ch)
1632 {
1633 struct xpc_fifo_entry_uv *entry;
1634 struct xpc_notify_mq_msg_uv *msg;
1635 void *payload = NULL;
1636
1637 if (!(ch->flags & XPC_C_DISCONNECTING)) {
1638 entry = xpc_get_fifo_entry_uv(&ch->sn.uv.recv_msg_list);
1639 if (entry != NULL) {
1640 msg = container_of(entry, struct xpc_notify_mq_msg_uv,
1641 hdr.u.next);
1642 payload = &msg->payload;
1643 }
1644 }
1645 return payload;
1646 }
1647
1648 static void
1649 xpc_received_payload_uv(struct xpc_channel *ch, void *payload)
1650 {
1651 struct xpc_notify_mq_msg_uv *msg;
1652 enum xp_retval ret;
1653
1654 msg = container_of(payload, struct xpc_notify_mq_msg_uv, payload);
1655
1656 /* return an ACK to the sender of this message */
1657
1658 msg->hdr.partid = xp_partition_id;
1659 msg->hdr.size = 0; /* size of zero indicates this is an ACK */
1660
1661 ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,
1662 sizeof(struct xpc_notify_mq_msghdr_uv));
1663 if (ret != xpSuccess)
1664 XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);
1665
1666 msg->hdr.msg_slot_number += ch->remote_nentries;
1667 }
1668
1669 static struct xpc_arch_operations xpc_arch_ops_uv = {
1670 .setup_partitions = xpc_setup_partitions_uv,
1671 .teardown_partitions = xpc_teardown_partitions_uv,
1672 .process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_uv,
1673 .get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_uv,
1674 .setup_rsvd_page = xpc_setup_rsvd_page_uv,
1675
1676 .allow_hb = xpc_allow_hb_uv,
1677 .disallow_hb = xpc_disallow_hb_uv,
1678 .disallow_all_hbs = xpc_disallow_all_hbs_uv,
1679 .increment_heartbeat = xpc_increment_heartbeat_uv,
1680 .offline_heartbeat = xpc_offline_heartbeat_uv,
1681 .online_heartbeat = xpc_online_heartbeat_uv,
1682 .heartbeat_init = xpc_heartbeat_init_uv,
1683 .heartbeat_exit = xpc_heartbeat_exit_uv,
1684 .get_remote_heartbeat = xpc_get_remote_heartbeat_uv,
1685
1686 .request_partition_activation =
1687 xpc_request_partition_activation_uv,
1688 .request_partition_reactivation =
1689 xpc_request_partition_reactivation_uv,
1690 .request_partition_deactivation =
1691 xpc_request_partition_deactivation_uv,
1692 .cancel_partition_deactivation_request =
1693 xpc_cancel_partition_deactivation_request_uv,
1694
1695 .setup_ch_structures = xpc_setup_ch_structures_uv,
1696 .teardown_ch_structures = xpc_teardown_ch_structures_uv,
1697
1698 .make_first_contact = xpc_make_first_contact_uv,
1699
1700 .get_chctl_all_flags = xpc_get_chctl_all_flags_uv,
1701 .send_chctl_closerequest = xpc_send_chctl_closerequest_uv,
1702 .send_chctl_closereply = xpc_send_chctl_closereply_uv,
1703 .send_chctl_openrequest = xpc_send_chctl_openrequest_uv,
1704 .send_chctl_openreply = xpc_send_chctl_openreply_uv,
1705 .send_chctl_opencomplete = xpc_send_chctl_opencomplete_uv,
1706 .process_msg_chctl_flags = xpc_process_msg_chctl_flags_uv,
1707
1708 .save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_uv,
1709
1710 .setup_msg_structures = xpc_setup_msg_structures_uv,
1711 .teardown_msg_structures = xpc_teardown_msg_structures_uv,
1712
1713 .indicate_partition_engaged = xpc_indicate_partition_engaged_uv,
1714 .indicate_partition_disengaged = xpc_indicate_partition_disengaged_uv,
1715 .assume_partition_disengaged = xpc_assume_partition_disengaged_uv,
1716 .partition_engaged = xpc_partition_engaged_uv,
1717 .any_partition_engaged = xpc_any_partition_engaged_uv,
1718
1719 .n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_uv,
1720 .send_payload = xpc_send_payload_uv,
1721 .get_deliverable_payload = xpc_get_deliverable_payload_uv,
1722 .received_payload = xpc_received_payload_uv,
1723 .notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_uv,
1724 };
1725
1726 int
1727 xpc_init_uv(void)
1728 {
1729 xpc_arch_ops = xpc_arch_ops_uv;
1730
1731 if (sizeof(struct xpc_notify_mq_msghdr_uv) > XPC_MSG_HDR_MAX_SIZE) {
1732 dev_err(xpc_part, "xpc_notify_mq_msghdr_uv is larger than %d\n",
1733 XPC_MSG_HDR_MAX_SIZE);
1734 return -E2BIG;
1735 }
1736
1737 xpc_activate_mq_uv = xpc_create_gru_mq_uv(XPC_ACTIVATE_MQ_SIZE_UV, 0,
1738 XPC_ACTIVATE_IRQ_NAME,
1739 xpc_handle_activate_IRQ_uv);
1740 if (IS_ERR(xpc_activate_mq_uv))
1741 return PTR_ERR(xpc_activate_mq_uv);
1742
1743 xpc_notify_mq_uv = xpc_create_gru_mq_uv(XPC_NOTIFY_MQ_SIZE_UV, 0,
1744 XPC_NOTIFY_IRQ_NAME,
1745 xpc_handle_notify_IRQ_uv);
1746 if (IS_ERR(xpc_notify_mq_uv)) {
1747 xpc_destroy_gru_mq_uv(xpc_activate_mq_uv);
1748 return PTR_ERR(xpc_notify_mq_uv);
1749 }
1750
1751 return 0;
1752 }
1753
1754 void
1755 xpc_exit_uv(void)
1756 {
1757 xpc_destroy_gru_mq_uv(xpc_notify_mq_uv);
1758 xpc_destroy_gru_mq_uv(xpc_activate_mq_uv);
1759 }
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree