| blob | ea34a585ce88da36d47a7f7d6fa9603c4aec4edf |
1 /* Process the definitions file with autogen to produce upc_atomic.upc:
3 autogen -L .. upc_atomic.def
5 Copyright (C) 2013
6 Free Software Foundation, Inc.
7 This file is part of the UPC runtime Library.
8 Written by Gary Funck <gary@intrepid.com>
9 and Nenad Vukicevic <nenad@intrepid.com>
11 This file is part of GCC.
13 GCC is free software; you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation; either version 3, or (at your option)
16 any later version.
18 GCC is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 Under Section 7 of GPL version 3, you are granted additional
24 permissions described in the GCC Runtime Library Exception, version
25 3.1, as published by the Free Software Foundation.
27 You should have received a copy of the GNU General Public License and
28 a copy of the GCC Runtime Library Exception along with this program;
29 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
30 <http://www.gnu.org/licenses/>. */
32 #include <upc.h>
33 #include <assert.h>
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <stdint.h>
37 #include <stdbool.h>
38 #include <upc_atomic.h>
39 #include "upc_config.h"
41 /**
42 * @file __upc_atomic.upc
43 * GUPC Portals4 UPC atomics implementation.
44 */
46 /**
47 * @addtogroup ATOMIC GUPCR Atomics Functions
48 * @{
49 */
51 /** Atomic domain representation */
52 struct upc_atomicdomain_struct
53 {
54 upc_op_t ops;
55 upc_type_t optype;
56 };
58 /* Represent a bit-encoded operation as an integer. */
59 typedef unsigned int upc_op_num_t;
62 typedef int I_type;
63 typedef unsigned int UI_type;
64 typedef long L_type;
65 typedef unsigned long UL_type;
66 typedef long long LL_type;
67 typedef unsigned long long ULL_type;
68 typedef int32_t I32_type;
69 typedef uint32_t UI32_type;
70 typedef int64_t I64_type;
71 typedef uint64_t UI64_type;
72 typedef float F_type;
73 typedef double D_type;
74 typedef shared void * PTS_type;
77 #define ATOMIC_ACCESS_OPS (UPC_GET | UPC_SET | UPC_CSWAP)
79 #define ATOMIC_NUM_OPS (UPC_ADD | UPC_MULT | UPC_MIN | UPC_MAX | UPC_SUB | UPC_INC | UPC_DEC)
81 #define ATOMIC_BIT_OPS (UPC_AND | UPC_OR | UPC_XOR)
82 #define ATOMIC_ALL_OPS (ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS \
83 | ATOMIC_BIT_OPS)
85 /**
86 * Check if OP is a valid atomic operation type.
87 *
88 * @param [in] op UPC atomic operation
89 * @retval TRUE if op is a valid atomic operation
90 */
91 static inline bool
92 __upc_atomic_is_valid_op (upc_op_t op)
93 {
94 return !((op & ~(-op)) || (op & ~ATOMIC_ALL_OPS));
95 }
97 /**
98 * Convert the bit-encoded OP into an integer.
99 *
100 * @param [in] op UPC atomic operation
101 * @retval op represented as integer index
102 * (UPC_ADD_OP, UPC_MULT_OP ...)
103 */
104 static inline upc_op_num_t
105 __upc_atomic_op_num (upc_op_t op)
106 {
107 return (LONG_LONG_BITS - 1) - __builtin_clzll ((long long) op);
108 }
110 /**
111 * Check if UPC_TYPE is a valid atomic operation type.
112 *
113 * @param [in] upc_type UPC atomic type
114 * @retval TRUE if atomic operations are supported on UPC_TYPE
115 */
116 static bool
117 __upc_atomic_is_valid_type (upc_type_t upc_type)
118 {
119 switch (upc_type)
120 {
121 case UPC_INT:
122 case UPC_UINT:
123 case UPC_LONG:
124 case UPC_ULONG:
125 case UPC_LLONG:
126 case UPC_ULLONG:
127 case UPC_INT32:
128 case UPC_UINT32:
129 case UPC_INT64:
130 case UPC_UINT64:
131 case UPC_FLOAT:
132 case UPC_DOUBLE:
133 case UPC_PTS:
134 return true;
135 default: break;
136 }
137 return false;
138 }
140 /**
141 * Return the atomic operations supported for type UPC_TYPE.
142 *
143 * @param [in] upc_type UPC atomic type
144 * @retval bit vector of supported atomic operations.
145 */
146 static upc_op_t
147 __upc_atomic_supported_ops (upc_type_t upc_type)
148 {
149 switch (upc_type)
150 {
151 case UPC_INT:
152 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
153 case UPC_UINT:
154 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
155 case UPC_LONG:
156 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
157 case UPC_ULONG:
158 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
159 case UPC_LLONG:
160 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
161 case UPC_ULLONG:
162 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
163 case UPC_INT32:
164 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
165 case UPC_UINT32:
166 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
167 case UPC_INT64:
168 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
169 case UPC_UINT64:
170 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS | ATOMIC_BIT_OPS;
171 case UPC_FLOAT:
172 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS;
173 case UPC_DOUBLE:
174 return ATOMIC_ACCESS_OPS | ATOMIC_NUM_OPS;
175 case UPC_PTS:
176 return ATOMIC_ACCESS_OPS;
177 }
178 return 0;
179 }
181 /**
182 * Convert UPC atomic operation into a string.
183 *
184 * @param [in] upc_op UPC atomic operation
185 * @retval Character string
186 */
187 static const char *
188 __upc_atomic_op_name (upc_op_num_t op_num)
189 {
190 switch (op_num)
191 {
192 case UPC_ADD_OP:
193 return "UPC_ADD";
194 case UPC_MULT_OP:
195 return "UPC_MULT";
196 case UPC_AND_OP:
197 return "UPC_AND";
198 case UPC_OR_OP:
199 return "UPC_OR";
200 case UPC_XOR_OP:
201 return "UPC_XOR";
202 case UPC_MIN_OP:
203 return "UPC_MIN";
204 case UPC_MAX_OP:
205 return "UPC_MAX";
206 case UPC_GET_OP:
207 return "UPC_GET";
208 case UPC_SET_OP:
209 return "UPC_SET";
210 case UPC_CSWAP_OP:
211 return "UPC_CSWAP";
212 case UPC_SUB_OP:
213 return "UPC_SUB";
214 case UPC_INC_OP:
215 return "UPC_INC";
216 case UPC_DEC_OP:
217 return "UPC_DEC";
218 }
219 return NULL;
220 }
222 /**
223 * Convert UPC atomic type into a string.
224 *
225 * @param [in] upc_type UPC atomic type
226 * @retval Character string
227 */
228 static const char *
229 __upc_atomic_type_name (upc_type_t upc_type)
230 {
231 switch (upc_type)
232 {
233 case UPC_INT:
234 return "UPC_INT";
235 case UPC_UINT:
236 return "UPC_UINT";
237 case UPC_LONG:
238 return "UPC_LONG";
239 case UPC_ULONG:
240 return "UPC_ULONG";
241 case UPC_LLONG:
242 return "UPC_LLONG";
243 case UPC_ULLONG:
244 return "UPC_ULLONG";
245 case UPC_INT32:
246 return "UPC_INT32";
247 case UPC_UINT32:
248 return "UPC_UINT32";
249 case UPC_INT64:
250 return "UPC_INT64";
251 case UPC_UINT64:
252 return "UPC_UINT64";
253 case UPC_FLOAT:
254 return "UPC_FLOAT";
255 case UPC_DOUBLE:
256 return "UPC_DOUBLE";
257 case UPC_PTS:
258 return "UPC_PTS";
259 }
260 return NULL;
261 }
263 #define REQ_FETCH_PTR 0b00000001
264 #define REQ_OPERAND1 0b00000010
265 #define REQ_OPERAND2 0b00000100
266 #define NULL_OPERAND1 0b00001000
267 #define NULL_OPERAND2 0b00010000
269 static const unsigned int operand_check[] =
270 {
271 /* UPC_ADD_OP */ REQ_OPERAND1 | NULL_OPERAND2,
272 /* UPC_MULT_OP */ REQ_OPERAND1 | NULL_OPERAND2,
273 /* UPC_AND_OP */ REQ_OPERAND1 | NULL_OPERAND2,
274 /* UPC_OR_OP */ REQ_OPERAND1 | NULL_OPERAND2,
275 /* UPC_XOR_OP */ REQ_OPERAND1 | NULL_OPERAND2,
276 /* UPC_LOGAND_OP */ 0,
277 /* UPC_LOGOR_OP */ 0,
278 /* UPC_MIN_OP */ REQ_OPERAND1 | NULL_OPERAND2,
279 /* UPC_MAX_OP */ REQ_OPERAND1 | NULL_OPERAND2,
280 /* UPC_GET_OP */ REQ_FETCH_PTR | NULL_OPERAND1 | NULL_OPERAND2,
281 /* UPC_SET_OP */ REQ_OPERAND1 | NULL_OPERAND2,
282 /* UPC_CSWAP_OP */ REQ_OPERAND1 | REQ_OPERAND2,
283 /* UPC_SUB_OP */ REQ_OPERAND1 | NULL_OPERAND2,
284 /* UPC_INC_OP */ NULL_OPERAND1 | NULL_OPERAND2,
285 /* UPC_DEC_OP */ NULL_OPERAND1 | NULL_OPERAND2,
286 };
288 static inline void
289 __upc_atomic_check_operands (upc_op_num_t op_num,
290 void * restrict fetch_ptr,
291 const void * restrict operand1,
292 const void * restrict operand2)
293 {
294 const unsigned int check = operand_check[op_num];
295 if ((check & REQ_FETCH_PTR) && fetch_ptr == NULL)
296 __upc_fatal ("atomic operation `%s' "
297 "requires a non-NULL fetch pointer",
298 __upc_atomic_op_name (op_num));
299 if ((check & REQ_OPERAND1) && operand1 == NULL)
300 __upc_fatal ("atomic operation `%s' "
301 "requires a non-NULL operand1 pointer",
302 __upc_atomic_op_name (op_num));
303 if ((check & REQ_OPERAND2) && operand2 == NULL)
304 __upc_fatal ("atomic operation `%s' "
305 "requires a non-NULL operand2 pointer",
306 __upc_atomic_op_name (op_num));
307 if ((check & NULL_OPERAND1) && operand1 != NULL)
308 __upc_fatal ("atomic operation `%s' "
309 "requires a NULL operand1 pointer",
310 __upc_atomic_op_name (op_num));
311 if ((check & NULL_OPERAND2) && operand2 != NULL)
312 __upc_fatal ("atomic operation `%s' "
313 "requires a NULL operand2 pointer",
314 __upc_atomic_op_name (op_num));
315 }
317 static void
318 __upc_atomic_I (
319 I_type * restrict fetch_ptr,
320 upc_op_num_t op_num,
321 shared I_type * restrict target,
322 I_type * restrict operand1 __attribute__((unused)),
323 I_type * restrict operand2 __attribute__((unused)))
324 {
325 I_type orig_value __attribute__((unused));
326 I_type new_value __attribute__((unused));
327 I_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
328 switch (op_num)
329 {
330 case UPC_ADD_OP:
331 orig_value = __atomic_fetch_add (target_ptr, *operand1,
332 __ATOMIC_SEQ_CST);
333 break;
334 case UPC_MULT_OP:
335 do
336 {
337 orig_value = *target_ptr;
338 new_value = orig_value * *operand1;
339 }
340 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
341 /* weak */ 0,
342 /* success_memmodel */ __ATOMIC_SEQ_CST,
343 /* failure_memmodel */ __ATOMIC_SEQ_CST));
344 break;
345 case UPC_AND_OP:
346 orig_value = __atomic_fetch_and (target_ptr, *operand1,
347 __ATOMIC_SEQ_CST);
348 break;
349 case UPC_OR_OP:
350 orig_value = __atomic_fetch_or (target_ptr, *operand1,
351 __ATOMIC_SEQ_CST);
352 break;
353 case UPC_XOR_OP:
354 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
355 __ATOMIC_SEQ_CST);
356 break;
357 case UPC_MIN_OP:
358 do
359 {
360 orig_value = *target_ptr;
361 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
362 }
363 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
364 /* weak */ 0,
365 /* success_memmodel */ __ATOMIC_SEQ_CST,
366 /* failure_memmodel */ __ATOMIC_SEQ_CST));
367 break;
368 case UPC_MAX_OP:
369 do
370 {
371 orig_value = *target_ptr;
372 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
373 }
374 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
375 /* weak */ 0,
376 /* success_memmodel */ __ATOMIC_SEQ_CST,
377 /* failure_memmodel */ __ATOMIC_SEQ_CST));
378 break;
379 case UPC_GET_OP:
380 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
381 break;
382 case UPC_SET_OP:
383 if (fetch_ptr == NULL)
384 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
385 else
386 __atomic_exchange (target_ptr, operand1, &orig_value,
387 /* memmodel */ __ATOMIC_SEQ_CST);
388 break;
389 case UPC_CSWAP_OP:
390 do
391 {
392 orig_value = *target_ptr;
393 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
394 }
395 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
396 /* weak */ 0,
397 /* success_memmodel */ __ATOMIC_SEQ_CST,
398 /* failure_memmodel */ __ATOMIC_SEQ_CST));
399 break;
400 case UPC_SUB_OP:
401 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
402 __ATOMIC_SEQ_CST);
403 break;
404 case UPC_INC_OP:
405 orig_value = __atomic_fetch_add (target_ptr, (int) 1,
406 __ATOMIC_SEQ_CST);
407 break;
408 case UPC_DEC_OP:
409 orig_value = __atomic_fetch_sub (target_ptr, (int) 1,
410 __ATOMIC_SEQ_CST);
411 break;
412 default: break;
413 }
414 if (fetch_ptr != NULL)
415 *fetch_ptr = orig_value;
416 }
418 static void
419 __upc_atomic_UI (
420 UI_type * restrict fetch_ptr,
421 upc_op_num_t op_num,
422 shared UI_type * restrict target,
423 UI_type * restrict operand1 __attribute__((unused)),
424 UI_type * restrict operand2 __attribute__((unused)))
425 {
426 UI_type orig_value __attribute__((unused));
427 UI_type new_value __attribute__((unused));
428 UI_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
429 switch (op_num)
430 {
431 case UPC_ADD_OP:
432 orig_value = __atomic_fetch_add (target_ptr, *operand1,
433 __ATOMIC_SEQ_CST);
434 break;
435 case UPC_MULT_OP:
436 do
437 {
438 orig_value = *target_ptr;
439 new_value = orig_value * *operand1;
440 }
441 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
442 /* weak */ 0,
443 /* success_memmodel */ __ATOMIC_SEQ_CST,
444 /* failure_memmodel */ __ATOMIC_SEQ_CST));
445 break;
446 case UPC_AND_OP:
447 orig_value = __atomic_fetch_and (target_ptr, *operand1,
448 __ATOMIC_SEQ_CST);
449 break;
450 case UPC_OR_OP:
451 orig_value = __atomic_fetch_or (target_ptr, *operand1,
452 __ATOMIC_SEQ_CST);
453 break;
454 case UPC_XOR_OP:
455 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
456 __ATOMIC_SEQ_CST);
457 break;
458 case UPC_MIN_OP:
459 do
460 {
461 orig_value = *target_ptr;
462 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
463 }
464 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
465 /* weak */ 0,
466 /* success_memmodel */ __ATOMIC_SEQ_CST,
467 /* failure_memmodel */ __ATOMIC_SEQ_CST));
468 break;
469 case UPC_MAX_OP:
470 do
471 {
472 orig_value = *target_ptr;
473 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
474 }
475 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
476 /* weak */ 0,
477 /* success_memmodel */ __ATOMIC_SEQ_CST,
478 /* failure_memmodel */ __ATOMIC_SEQ_CST));
479 break;
480 case UPC_GET_OP:
481 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
482 break;
483 case UPC_SET_OP:
484 if (fetch_ptr == NULL)
485 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
486 else
487 __atomic_exchange (target_ptr, operand1, &orig_value,
488 /* memmodel */ __ATOMIC_SEQ_CST);
489 break;
490 case UPC_CSWAP_OP:
491 do
492 {
493 orig_value = *target_ptr;
494 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
495 }
496 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
497 /* weak */ 0,
498 /* success_memmodel */ __ATOMIC_SEQ_CST,
499 /* failure_memmodel */ __ATOMIC_SEQ_CST));
500 break;
501 case UPC_SUB_OP:
502 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
503 __ATOMIC_SEQ_CST);
504 break;
505 case UPC_INC_OP:
506 orig_value = __atomic_fetch_add (target_ptr, (unsigned int) 1,
507 __ATOMIC_SEQ_CST);
508 break;
509 case UPC_DEC_OP:
510 orig_value = __atomic_fetch_sub (target_ptr, (unsigned int) 1,
511 __ATOMIC_SEQ_CST);
512 break;
513 default: break;
514 }
515 if (fetch_ptr != NULL)
516 *fetch_ptr = orig_value;
517 }
519 static void
520 __upc_atomic_L (
521 L_type * restrict fetch_ptr,
522 upc_op_num_t op_num,
523 shared L_type * restrict target,
524 L_type * restrict operand1 __attribute__((unused)),
525 L_type * restrict operand2 __attribute__((unused)))
526 {
527 L_type orig_value __attribute__((unused));
528 L_type new_value __attribute__((unused));
529 L_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
530 switch (op_num)
531 {
532 case UPC_ADD_OP:
533 orig_value = __atomic_fetch_add (target_ptr, *operand1,
534 __ATOMIC_SEQ_CST);
535 break;
536 case UPC_MULT_OP:
537 do
538 {
539 orig_value = *target_ptr;
540 new_value = orig_value * *operand1;
541 }
542 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
543 /* weak */ 0,
544 /* success_memmodel */ __ATOMIC_SEQ_CST,
545 /* failure_memmodel */ __ATOMIC_SEQ_CST));
546 break;
547 case UPC_AND_OP:
548 orig_value = __atomic_fetch_and (target_ptr, *operand1,
549 __ATOMIC_SEQ_CST);
550 break;
551 case UPC_OR_OP:
552 orig_value = __atomic_fetch_or (target_ptr, *operand1,
553 __ATOMIC_SEQ_CST);
554 break;
555 case UPC_XOR_OP:
556 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
557 __ATOMIC_SEQ_CST);
558 break;
559 case UPC_MIN_OP:
560 do
561 {
562 orig_value = *target_ptr;
563 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
564 }
565 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
566 /* weak */ 0,
567 /* success_memmodel */ __ATOMIC_SEQ_CST,
568 /* failure_memmodel */ __ATOMIC_SEQ_CST));
569 break;
570 case UPC_MAX_OP:
571 do
572 {
573 orig_value = *target_ptr;
574 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
575 }
576 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
577 /* weak */ 0,
578 /* success_memmodel */ __ATOMIC_SEQ_CST,
579 /* failure_memmodel */ __ATOMIC_SEQ_CST));
580 break;
581 case UPC_GET_OP:
582 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
583 break;
584 case UPC_SET_OP:
585 if (fetch_ptr == NULL)
586 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
587 else
588 __atomic_exchange (target_ptr, operand1, &orig_value,
589 /* memmodel */ __ATOMIC_SEQ_CST);
590 break;
591 case UPC_CSWAP_OP:
592 do
593 {
594 orig_value = *target_ptr;
595 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
596 }
597 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
598 /* weak */ 0,
599 /* success_memmodel */ __ATOMIC_SEQ_CST,
600 /* failure_memmodel */ __ATOMIC_SEQ_CST));
601 break;
602 case UPC_SUB_OP:
603 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
604 __ATOMIC_SEQ_CST);
605 break;
606 case UPC_INC_OP:
607 orig_value = __atomic_fetch_add (target_ptr, (long) 1,
608 __ATOMIC_SEQ_CST);
609 break;
610 case UPC_DEC_OP:
611 orig_value = __atomic_fetch_sub (target_ptr, (long) 1,
612 __ATOMIC_SEQ_CST);
613 break;
614 default: break;
615 }
616 if (fetch_ptr != NULL)
617 *fetch_ptr = orig_value;
618 }
620 static void
621 __upc_atomic_UL (
622 UL_type * restrict fetch_ptr,
623 upc_op_num_t op_num,
624 shared UL_type * restrict target,
625 UL_type * restrict operand1 __attribute__((unused)),
626 UL_type * restrict operand2 __attribute__((unused)))
627 {
628 UL_type orig_value __attribute__((unused));
629 UL_type new_value __attribute__((unused));
630 UL_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
631 switch (op_num)
632 {
633 case UPC_ADD_OP:
634 orig_value = __atomic_fetch_add (target_ptr, *operand1,
635 __ATOMIC_SEQ_CST);
636 break;
637 case UPC_MULT_OP:
638 do
639 {
640 orig_value = *target_ptr;
641 new_value = orig_value * *operand1;
642 }
643 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
644 /* weak */ 0,
645 /* success_memmodel */ __ATOMIC_SEQ_CST,
646 /* failure_memmodel */ __ATOMIC_SEQ_CST));
647 break;
648 case UPC_AND_OP:
649 orig_value = __atomic_fetch_and (target_ptr, *operand1,
650 __ATOMIC_SEQ_CST);
651 break;
652 case UPC_OR_OP:
653 orig_value = __atomic_fetch_or (target_ptr, *operand1,
654 __ATOMIC_SEQ_CST);
655 break;
656 case UPC_XOR_OP:
657 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
658 __ATOMIC_SEQ_CST);
659 break;
660 case UPC_MIN_OP:
661 do
662 {
663 orig_value = *target_ptr;
664 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
665 }
666 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
667 /* weak */ 0,
668 /* success_memmodel */ __ATOMIC_SEQ_CST,
669 /* failure_memmodel */ __ATOMIC_SEQ_CST));
670 break;
671 case UPC_MAX_OP:
672 do
673 {
674 orig_value = *target_ptr;
675 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
676 }
677 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
678 /* weak */ 0,
679 /* success_memmodel */ __ATOMIC_SEQ_CST,
680 /* failure_memmodel */ __ATOMIC_SEQ_CST));
681 break;
682 case UPC_GET_OP:
683 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
684 break;
685 case UPC_SET_OP:
686 if (fetch_ptr == NULL)
687 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
688 else
689 __atomic_exchange (target_ptr, operand1, &orig_value,
690 /* memmodel */ __ATOMIC_SEQ_CST);
691 break;
692 case UPC_CSWAP_OP:
693 do
694 {
695 orig_value = *target_ptr;
696 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
697 }
698 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
699 /* weak */ 0,
700 /* success_memmodel */ __ATOMIC_SEQ_CST,
701 /* failure_memmodel */ __ATOMIC_SEQ_CST));
702 break;
703 case UPC_SUB_OP:
704 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
705 __ATOMIC_SEQ_CST);
706 break;
707 case UPC_INC_OP:
708 orig_value = __atomic_fetch_add (target_ptr, (unsigned long) 1,
709 __ATOMIC_SEQ_CST);
710 break;
711 case UPC_DEC_OP:
712 orig_value = __atomic_fetch_sub (target_ptr, (unsigned long) 1,
713 __ATOMIC_SEQ_CST);
714 break;
715 default: break;
716 }
717 if (fetch_ptr != NULL)
718 *fetch_ptr = orig_value;
719 }
721 static void
722 __upc_atomic_LL (
723 LL_type * restrict fetch_ptr,
724 upc_op_num_t op_num,
725 shared LL_type * restrict target,
726 LL_type * restrict operand1 __attribute__((unused)),
727 LL_type * restrict operand2 __attribute__((unused)))
728 {
729 LL_type orig_value __attribute__((unused));
730 LL_type new_value __attribute__((unused));
731 LL_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
732 switch (op_num)
733 {
734 case UPC_ADD_OP:
735 orig_value = __atomic_fetch_add (target_ptr, *operand1,
736 __ATOMIC_SEQ_CST);
737 break;
738 case UPC_MULT_OP:
739 do
740 {
741 orig_value = *target_ptr;
742 new_value = orig_value * *operand1;
743 }
744 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
745 /* weak */ 0,
746 /* success_memmodel */ __ATOMIC_SEQ_CST,
747 /* failure_memmodel */ __ATOMIC_SEQ_CST));
748 break;
749 case UPC_AND_OP:
750 orig_value = __atomic_fetch_and (target_ptr, *operand1,
751 __ATOMIC_SEQ_CST);
752 break;
753 case UPC_OR_OP:
754 orig_value = __atomic_fetch_or (target_ptr, *operand1,
755 __ATOMIC_SEQ_CST);
756 break;
757 case UPC_XOR_OP:
758 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
759 __ATOMIC_SEQ_CST);
760 break;
761 case UPC_MIN_OP:
762 do
763 {
764 orig_value = *target_ptr;
765 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
766 }
767 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
768 /* weak */ 0,
769 /* success_memmodel */ __ATOMIC_SEQ_CST,
770 /* failure_memmodel */ __ATOMIC_SEQ_CST));
771 break;
772 case UPC_MAX_OP:
773 do
774 {
775 orig_value = *target_ptr;
776 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
777 }
778 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
779 /* weak */ 0,
780 /* success_memmodel */ __ATOMIC_SEQ_CST,
781 /* failure_memmodel */ __ATOMIC_SEQ_CST));
782 break;
783 case UPC_GET_OP:
784 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
785 break;
786 case UPC_SET_OP:
787 if (fetch_ptr == NULL)
788 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
789 else
790 __atomic_exchange (target_ptr, operand1, &orig_value,
791 /* memmodel */ __ATOMIC_SEQ_CST);
792 break;
793 case UPC_CSWAP_OP:
794 do
795 {
796 orig_value = *target_ptr;
797 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
798 }
799 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
800 /* weak */ 0,
801 /* success_memmodel */ __ATOMIC_SEQ_CST,
802 /* failure_memmodel */ __ATOMIC_SEQ_CST));
803 break;
804 case UPC_SUB_OP:
805 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
806 __ATOMIC_SEQ_CST);
807 break;
808 case UPC_INC_OP:
809 orig_value = __atomic_fetch_add (target_ptr, (long long) 1,
810 __ATOMIC_SEQ_CST);
811 break;
812 case UPC_DEC_OP:
813 orig_value = __atomic_fetch_sub (target_ptr, (long long) 1,
814 __ATOMIC_SEQ_CST);
815 break;
816 default: break;
817 }
818 if (fetch_ptr != NULL)
819 *fetch_ptr = orig_value;
820 }
822 static void
823 __upc_atomic_ULL (
824 ULL_type * restrict fetch_ptr,
825 upc_op_num_t op_num,
826 shared ULL_type * restrict target,
827 ULL_type * restrict operand1 __attribute__((unused)),
828 ULL_type * restrict operand2 __attribute__((unused)))
829 {
830 ULL_type orig_value __attribute__((unused));
831 ULL_type new_value __attribute__((unused));
832 ULL_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
833 switch (op_num)
834 {
835 case UPC_ADD_OP:
836 orig_value = __atomic_fetch_add (target_ptr, *operand1,
837 __ATOMIC_SEQ_CST);
838 break;
839 case UPC_MULT_OP:
840 do
841 {
842 orig_value = *target_ptr;
843 new_value = orig_value * *operand1;
844 }
845 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
846 /* weak */ 0,
847 /* success_memmodel */ __ATOMIC_SEQ_CST,
848 /* failure_memmodel */ __ATOMIC_SEQ_CST));
849 break;
850 case UPC_AND_OP:
851 orig_value = __atomic_fetch_and (target_ptr, *operand1,
852 __ATOMIC_SEQ_CST);
853 break;
854 case UPC_OR_OP:
855 orig_value = __atomic_fetch_or (target_ptr, *operand1,
856 __ATOMIC_SEQ_CST);
857 break;
858 case UPC_XOR_OP:
859 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
860 __ATOMIC_SEQ_CST);
861 break;
862 case UPC_MIN_OP:
863 do
864 {
865 orig_value = *target_ptr;
866 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
867 }
868 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
869 /* weak */ 0,
870 /* success_memmodel */ __ATOMIC_SEQ_CST,
871 /* failure_memmodel */ __ATOMIC_SEQ_CST));
872 break;
873 case UPC_MAX_OP:
874 do
875 {
876 orig_value = *target_ptr;
877 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
878 }
879 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
880 /* weak */ 0,
881 /* success_memmodel */ __ATOMIC_SEQ_CST,
882 /* failure_memmodel */ __ATOMIC_SEQ_CST));
883 break;
884 case UPC_GET_OP:
885 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
886 break;
887 case UPC_SET_OP:
888 if (fetch_ptr == NULL)
889 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
890 else
891 __atomic_exchange (target_ptr, operand1, &orig_value,
892 /* memmodel */ __ATOMIC_SEQ_CST);
893 break;
894 case UPC_CSWAP_OP:
895 do
896 {
897 orig_value = *target_ptr;
898 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
899 }
900 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
901 /* weak */ 0,
902 /* success_memmodel */ __ATOMIC_SEQ_CST,
903 /* failure_memmodel */ __ATOMIC_SEQ_CST));
904 break;
905 case UPC_SUB_OP:
906 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
907 __ATOMIC_SEQ_CST);
908 break;
909 case UPC_INC_OP:
910 orig_value = __atomic_fetch_add (target_ptr, (unsigned long long) 1,
911 __ATOMIC_SEQ_CST);
912 break;
913 case UPC_DEC_OP:
914 orig_value = __atomic_fetch_sub (target_ptr, (unsigned long long) 1,
915 __ATOMIC_SEQ_CST);
916 break;
917 default: break;
918 }
919 if (fetch_ptr != NULL)
920 *fetch_ptr = orig_value;
921 }
923 static void
924 __upc_atomic_I32 (
925 I32_type * restrict fetch_ptr,
926 upc_op_num_t op_num,
927 shared I32_type * restrict target,
928 I32_type * restrict operand1 __attribute__((unused)),
929 I32_type * restrict operand2 __attribute__((unused)))
930 {
931 I32_type orig_value __attribute__((unused));
932 I32_type new_value __attribute__((unused));
933 I32_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
934 switch (op_num)
935 {
936 case UPC_ADD_OP:
937 orig_value = __atomic_fetch_add (target_ptr, *operand1,
938 __ATOMIC_SEQ_CST);
939 break;
940 case UPC_MULT_OP:
941 do
942 {
943 orig_value = *target_ptr;
944 new_value = orig_value * *operand1;
945 }
946 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
947 /* weak */ 0,
948 /* success_memmodel */ __ATOMIC_SEQ_CST,
949 /* failure_memmodel */ __ATOMIC_SEQ_CST));
950 break;
951 case UPC_AND_OP:
952 orig_value = __atomic_fetch_and (target_ptr, *operand1,
953 __ATOMIC_SEQ_CST);
954 break;
955 case UPC_OR_OP:
956 orig_value = __atomic_fetch_or (target_ptr, *operand1,
957 __ATOMIC_SEQ_CST);
958 break;
959 case UPC_XOR_OP:
960 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
961 __ATOMIC_SEQ_CST);
962 break;
963 case UPC_MIN_OP:
964 do
965 {
966 orig_value = *target_ptr;
967 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
968 }
969 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
970 /* weak */ 0,
971 /* success_memmodel */ __ATOMIC_SEQ_CST,
972 /* failure_memmodel */ __ATOMIC_SEQ_CST));
973 break;
974 case UPC_MAX_OP:
975 do
976 {
977 orig_value = *target_ptr;
978 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
979 }
980 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
981 /* weak */ 0,
982 /* success_memmodel */ __ATOMIC_SEQ_CST,
983 /* failure_memmodel */ __ATOMIC_SEQ_CST));
984 break;
985 case UPC_GET_OP:
986 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
987 break;
988 case UPC_SET_OP:
989 if (fetch_ptr == NULL)
990 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
991 else
992 __atomic_exchange (target_ptr, operand1, &orig_value,
993 /* memmodel */ __ATOMIC_SEQ_CST);
994 break;
995 case UPC_CSWAP_OP:
996 do
997 {
998 orig_value = *target_ptr;
999 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
1000 }
1001 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1002 /* weak */ 0,
1003 /* success_memmodel */ __ATOMIC_SEQ_CST,
1004 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1005 break;
1006 case UPC_SUB_OP:
1007 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
1008 __ATOMIC_SEQ_CST);
1009 break;
1010 case UPC_INC_OP:
1011 orig_value = __atomic_fetch_add (target_ptr, (int32_t) 1,
1012 __ATOMIC_SEQ_CST);
1013 break;
1014 case UPC_DEC_OP:
1015 orig_value = __atomic_fetch_sub (target_ptr, (int32_t) 1,
1016 __ATOMIC_SEQ_CST);
1017 break;
1018 default: break;
1019 }
1020 if (fetch_ptr != NULL)
1021 *fetch_ptr = orig_value;
1022 }
1024 static void
1025 __upc_atomic_UI32 (
1026 UI32_type * restrict fetch_ptr,
1027 upc_op_num_t op_num,
1028 shared UI32_type * restrict target,
1029 UI32_type * restrict operand1 __attribute__((unused)),
1030 UI32_type * restrict operand2 __attribute__((unused)))
1031 {
1032 UI32_type orig_value __attribute__((unused));
1033 UI32_type new_value __attribute__((unused));
1034 UI32_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
1035 switch (op_num)
1036 {
1037 case UPC_ADD_OP:
1038 orig_value = __atomic_fetch_add (target_ptr, *operand1,
1039 __ATOMIC_SEQ_CST);
1040 break;
1041 case UPC_MULT_OP:
1042 do
1043 {
1044 orig_value = *target_ptr;
1045 new_value = orig_value * *operand1;
1046 }
1047 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1048 /* weak */ 0,
1049 /* success_memmodel */ __ATOMIC_SEQ_CST,
1050 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1051 break;
1052 case UPC_AND_OP:
1053 orig_value = __atomic_fetch_and (target_ptr, *operand1,
1054 __ATOMIC_SEQ_CST);
1055 break;
1056 case UPC_OR_OP:
1057 orig_value = __atomic_fetch_or (target_ptr, *operand1,
1058 __ATOMIC_SEQ_CST);
1059 break;
1060 case UPC_XOR_OP:
1061 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
1062 __ATOMIC_SEQ_CST);
1063 break;
1064 case UPC_MIN_OP:
1065 do
1066 {
1067 orig_value = *target_ptr;
1068 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
1069 }
1070 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1071 /* weak */ 0,
1072 /* success_memmodel */ __ATOMIC_SEQ_CST,
1073 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1074 break;
1075 case UPC_MAX_OP:
1076 do
1077 {
1078 orig_value = *target_ptr;
1079 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
1080 }
1081 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1082 /* weak */ 0,
1083 /* success_memmodel */ __ATOMIC_SEQ_CST,
1084 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1085 break;
1086 case UPC_GET_OP:
1087 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
1088 break;
1089 case UPC_SET_OP:
1090 if (fetch_ptr == NULL)
1091 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
1092 else
1093 __atomic_exchange (target_ptr, operand1, &orig_value,
1094 /* memmodel */ __ATOMIC_SEQ_CST);
1095 break;
1096 case UPC_CSWAP_OP:
1097 do
1098 {
1099 orig_value = *target_ptr;
1100 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
1101 }
1102 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1103 /* weak */ 0,
1104 /* success_memmodel */ __ATOMIC_SEQ_CST,
1105 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1106 break;
1107 case UPC_SUB_OP:
1108 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
1109 __ATOMIC_SEQ_CST);
1110 break;
1111 case UPC_INC_OP:
1112 orig_value = __atomic_fetch_add (target_ptr, (uint32_t) 1,
1113 __ATOMIC_SEQ_CST);
1114 break;
1115 case UPC_DEC_OP:
1116 orig_value = __atomic_fetch_sub (target_ptr, (uint32_t) 1,
1117 __ATOMIC_SEQ_CST);
1118 break;
1119 default: break;
1120 }
1121 if (fetch_ptr != NULL)
1122 *fetch_ptr = orig_value;
1123 }
1125 static void
1126 __upc_atomic_I64 (
1127 I64_type * restrict fetch_ptr,
1128 upc_op_num_t op_num,
1129 shared I64_type * restrict target,
1130 I64_type * restrict operand1 __attribute__((unused)),
1131 I64_type * restrict operand2 __attribute__((unused)))
1132 {
1133 I64_type orig_value __attribute__((unused));
1134 I64_type new_value __attribute__((unused));
1135 I64_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
1136 switch (op_num)
1137 {
1138 case UPC_ADD_OP:
1139 orig_value = __atomic_fetch_add (target_ptr, *operand1,
1140 __ATOMIC_SEQ_CST);
1141 break;
1142 case UPC_MULT_OP:
1143 do
1144 {
1145 orig_value = *target_ptr;
1146 new_value = orig_value * *operand1;
1147 }
1148 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1149 /* weak */ 0,
1150 /* success_memmodel */ __ATOMIC_SEQ_CST,
1151 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1152 break;
1153 case UPC_AND_OP:
1154 orig_value = __atomic_fetch_and (target_ptr, *operand1,
1155 __ATOMIC_SEQ_CST);
1156 break;
1157 case UPC_OR_OP:
1158 orig_value = __atomic_fetch_or (target_ptr, *operand1,
1159 __ATOMIC_SEQ_CST);
1160 break;
1161 case UPC_XOR_OP:
1162 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
1163 __ATOMIC_SEQ_CST);
1164 break;
1165 case UPC_MIN_OP:
1166 do
1167 {
1168 orig_value = *target_ptr;
1169 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
1170 }
1171 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1172 /* weak */ 0,
1173 /* success_memmodel */ __ATOMIC_SEQ_CST,
1174 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1175 break;
1176 case UPC_MAX_OP:
1177 do
1178 {
1179 orig_value = *target_ptr;
1180 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
1181 }
1182 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1183 /* weak */ 0,
1184 /* success_memmodel */ __ATOMIC_SEQ_CST,
1185 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1186 break;
1187 case UPC_GET_OP:
1188 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
1189 break;
1190 case UPC_SET_OP:
1191 if (fetch_ptr == NULL)
1192 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
1193 else
1194 __atomic_exchange (target_ptr, operand1, &orig_value,
1195 /* memmodel */ __ATOMIC_SEQ_CST);
1196 break;
1197 case UPC_CSWAP_OP:
1198 do
1199 {
1200 orig_value = *target_ptr;
1201 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
1202 }
1203 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1204 /* weak */ 0,
1205 /* success_memmodel */ __ATOMIC_SEQ_CST,
1206 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1207 break;
1208 case UPC_SUB_OP:
1209 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
1210 __ATOMIC_SEQ_CST);
1211 break;
1212 case UPC_INC_OP:
1213 orig_value = __atomic_fetch_add (target_ptr, (int64_t) 1,
1214 __ATOMIC_SEQ_CST);
1215 break;
1216 case UPC_DEC_OP:
1217 orig_value = __atomic_fetch_sub (target_ptr, (int64_t) 1,
1218 __ATOMIC_SEQ_CST);
1219 break;
1220 default: break;
1221 }
1222 if (fetch_ptr != NULL)
1223 *fetch_ptr = orig_value;
1224 }
1226 static void
1227 __upc_atomic_UI64 (
1228 UI64_type * restrict fetch_ptr,
1229 upc_op_num_t op_num,
1230 shared UI64_type * restrict target,
1231 UI64_type * restrict operand1 __attribute__((unused)),
1232 UI64_type * restrict operand2 __attribute__((unused)))
1233 {
1234 UI64_type orig_value __attribute__((unused));
1235 UI64_type new_value __attribute__((unused));
1236 UI64_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
1237 switch (op_num)
1238 {
1239 case UPC_ADD_OP:
1240 orig_value = __atomic_fetch_add (target_ptr, *operand1,
1241 __ATOMIC_SEQ_CST);
1242 break;
1243 case UPC_MULT_OP:
1244 do
1245 {
1246 orig_value = *target_ptr;
1247 new_value = orig_value * *operand1;
1248 }
1249 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1250 /* weak */ 0,
1251 /* success_memmodel */ __ATOMIC_SEQ_CST,
1252 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1253 break;
1254 case UPC_AND_OP:
1255 orig_value = __atomic_fetch_and (target_ptr, *operand1,
1256 __ATOMIC_SEQ_CST);
1257 break;
1258 case UPC_OR_OP:
1259 orig_value = __atomic_fetch_or (target_ptr, *operand1,
1260 __ATOMIC_SEQ_CST);
1261 break;
1262 case UPC_XOR_OP:
1263 orig_value = __atomic_fetch_xor (target_ptr, *operand1,
1264 __ATOMIC_SEQ_CST);
1265 break;
1266 case UPC_MIN_OP:
1267 do
1268 {
1269 orig_value = *target_ptr;
1270 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
1271 }
1272 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1273 /* weak */ 0,
1274 /* success_memmodel */ __ATOMIC_SEQ_CST,
1275 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1276 break;
1277 case UPC_MAX_OP:
1278 do
1279 {
1280 orig_value = *target_ptr;
1281 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
1282 }
1283 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1284 /* weak */ 0,
1285 /* success_memmodel */ __ATOMIC_SEQ_CST,
1286 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1287 break;
1288 case UPC_GET_OP:
1289 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
1290 break;
1291 case UPC_SET_OP:
1292 if (fetch_ptr == NULL)
1293 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
1294 else
1295 __atomic_exchange (target_ptr, operand1, &orig_value,
1296 /* memmodel */ __ATOMIC_SEQ_CST);
1297 break;
1298 case UPC_CSWAP_OP:
1299 do
1300 {
1301 orig_value = *target_ptr;
1302 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
1303 }
1304 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1305 /* weak */ 0,
1306 /* success_memmodel */ __ATOMIC_SEQ_CST,
1307 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1308 break;
1309 case UPC_SUB_OP:
1310 orig_value = __atomic_fetch_sub (target_ptr, *operand1,
1311 __ATOMIC_SEQ_CST);
1312 break;
1313 case UPC_INC_OP:
1314 orig_value = __atomic_fetch_add (target_ptr, (uint64_t) 1,
1315 __ATOMIC_SEQ_CST);
1316 break;
1317 case UPC_DEC_OP:
1318 orig_value = __atomic_fetch_sub (target_ptr, (uint64_t) 1,
1319 __ATOMIC_SEQ_CST);
1320 break;
1321 default: break;
1322 }
1323 if (fetch_ptr != NULL)
1324 *fetch_ptr = orig_value;
1325 }
1327 static void
1328 __upc_atomic_F (
1329 F_type * restrict fetch_ptr,
1330 upc_op_num_t op_num,
1331 shared F_type * restrict target,
1332 F_type * restrict operand1 __attribute__((unused)),
1333 F_type * restrict operand2 __attribute__((unused)))
1334 {
1335 F_type orig_value __attribute__((unused));
1336 F_type new_value __attribute__((unused));
1337 F_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
1338 switch (op_num)
1339 {
1340 case UPC_ADD_OP:
1341 do
1342 {
1343 orig_value = *target_ptr;
1344 new_value = orig_value + *operand1;
1345 }
1346 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1347 /* weak */ 0,
1348 /* success_memmodel */ __ATOMIC_SEQ_CST,
1349 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1350 break;
1351 case UPC_MULT_OP:
1352 do
1353 {
1354 orig_value = *target_ptr;
1355 new_value = orig_value * *operand1;
1356 }
1357 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1358 /* weak */ 0,
1359 /* success_memmodel */ __ATOMIC_SEQ_CST,
1360 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1361 break;
1362 case UPC_MIN_OP:
1363 do
1364 {
1365 orig_value = *target_ptr;
1366 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
1367 }
1368 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1369 /* weak */ 0,
1370 /* success_memmodel */ __ATOMIC_SEQ_CST,
1371 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1372 break;
1373 case UPC_MAX_OP:
1374 do
1375 {
1376 orig_value = *target_ptr;
1377 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
1378 }
1379 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1380 /* weak */ 0,
1381 /* success_memmodel */ __ATOMIC_SEQ_CST,
1382 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1383 break;
1384 case UPC_GET_OP:
1385 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
1386 break;
1387 case UPC_SET_OP:
1388 if (fetch_ptr == NULL)
1389 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
1390 else
1391 __atomic_exchange (target_ptr, operand1, &orig_value,
1392 /* memmodel */ __ATOMIC_SEQ_CST);
1393 break;
1394 case UPC_CSWAP_OP:
1395 do
1396 {
1397 orig_value = *target_ptr;
1398 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
1399 }
1400 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1401 /* weak */ 0,
1402 /* success_memmodel */ __ATOMIC_SEQ_CST,
1403 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1404 break;
1405 case UPC_SUB_OP:
1406 do
1407 {
1408 orig_value = *target_ptr;
1409 new_value = orig_value - *operand1;
1410 }
1411 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1412 /* weak */ 0,
1413 /* success_memmodel */ __ATOMIC_SEQ_CST,
1414 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1415 break;
1416 case UPC_INC_OP:
1417 do
1418 {
1419 orig_value = *target_ptr;
1420 new_value = orig_value + (float) 1;
1421 }
1422 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1423 /* weak */ 0,
1424 /* success_memmodel */ __ATOMIC_SEQ_CST,
1425 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1426 break;
1427 case UPC_DEC_OP:
1428 do
1429 {
1430 orig_value = *target_ptr;
1431 new_value = orig_value - (float) 1;
1432 }
1433 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1434 /* weak */ 0,
1435 /* success_memmodel */ __ATOMIC_SEQ_CST,
1436 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1437 break;
1438 default: break;
1439 }
1440 if (fetch_ptr != NULL)
1441 *fetch_ptr = orig_value;
1442 }
1444 static void
1445 __upc_atomic_D (
1446 D_type * restrict fetch_ptr,
1447 upc_op_num_t op_num,
1448 shared D_type * restrict target,
1449 D_type * restrict operand1 __attribute__((unused)),
1450 D_type * restrict operand2 __attribute__((unused)))
1451 {
1452 D_type orig_value __attribute__((unused));
1453 D_type new_value __attribute__((unused));
1454 D_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
1455 switch (op_num)
1456 {
1457 case UPC_ADD_OP:
1458 do
1459 {
1460 orig_value = *target_ptr;
1461 new_value = orig_value + *operand1;
1462 }
1463 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1464 /* weak */ 0,
1465 /* success_memmodel */ __ATOMIC_SEQ_CST,
1466 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1467 break;
1468 case UPC_MULT_OP:
1469 do
1470 {
1471 orig_value = *target_ptr;
1472 new_value = orig_value * *operand1;
1473 }
1474 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1475 /* weak */ 0,
1476 /* success_memmodel */ __ATOMIC_SEQ_CST,
1477 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1478 break;
1479 case UPC_MIN_OP:
1480 do
1481 {
1482 orig_value = *target_ptr;
1483 new_value = (*operand1 < orig_value) ? *operand1 : orig_value;
1484 }
1485 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1486 /* weak */ 0,
1487 /* success_memmodel */ __ATOMIC_SEQ_CST,
1488 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1489 break;
1490 case UPC_MAX_OP:
1491 do
1492 {
1493 orig_value = *target_ptr;
1494 new_value = (*operand1 > orig_value) ? *operand1 : orig_value;
1495 }
1496 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1497 /* weak */ 0,
1498 /* success_memmodel */ __ATOMIC_SEQ_CST,
1499 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1500 break;
1501 case UPC_GET_OP:
1502 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
1503 break;
1504 case UPC_SET_OP:
1505 if (fetch_ptr == NULL)
1506 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
1507 else
1508 __atomic_exchange (target_ptr, operand1, &orig_value,
1509 /* memmodel */ __ATOMIC_SEQ_CST);
1510 break;
1511 case UPC_CSWAP_OP:
1512 do
1513 {
1514 orig_value = *target_ptr;
1515 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
1516 }
1517 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1518 /* weak */ 0,
1519 /* success_memmodel */ __ATOMIC_SEQ_CST,
1520 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1521 break;
1522 case UPC_SUB_OP:
1523 do
1524 {
1525 orig_value = *target_ptr;
1526 new_value = orig_value - *operand1;
1527 }
1528 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1529 /* weak */ 0,
1530 /* success_memmodel */ __ATOMIC_SEQ_CST,
1531 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1532 break;
1533 case UPC_INC_OP:
1534 do
1535 {
1536 orig_value = *target_ptr;
1537 new_value = orig_value + (double) 1;
1538 }
1539 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1540 /* weak */ 0,
1541 /* success_memmodel */ __ATOMIC_SEQ_CST,
1542 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1543 break;
1544 case UPC_DEC_OP:
1545 do
1546 {
1547 orig_value = *target_ptr;
1548 new_value = orig_value - (double) 1;
1549 }
1550 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1551 /* weak */ 0,
1552 /* success_memmodel */ __ATOMIC_SEQ_CST,
1553 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1554 break;
1555 default: break;
1556 }
1557 if (fetch_ptr != NULL)
1558 *fetch_ptr = orig_value;
1559 }
1561 static void
1562 __upc_atomic_PTS (
1563 PTS_type * restrict fetch_ptr,
1564 upc_op_num_t op_num,
1565 shared PTS_type * restrict target,
1566 PTS_type * restrict operand1 __attribute__((unused)),
1567 PTS_type * restrict operand2 __attribute__((unused)))
1568 {
1569 PTS_type orig_value __attribute__((unused));
1570 PTS_type new_value __attribute__((unused));
1571 PTS_type *target_ptr = __cvtaddr (*(upc_shared_ptr_t *)&target);
1572 switch (op_num)
1573 {
1574 case UPC_GET_OP:
1575 __atomic_load (target_ptr, &orig_value, __ATOMIC_SEQ_CST);
1576 break;
1577 case UPC_SET_OP:
1578 if (fetch_ptr == NULL)
1579 __atomic_store (target_ptr, operand1, __ATOMIC_SEQ_CST);
1580 else
1581 __atomic_exchange (target_ptr, operand1, &orig_value,
1582 /* memmodel */ __ATOMIC_SEQ_CST);
1583 break;
1584 case UPC_CSWAP_OP:
1585 do
1586 {
1587 orig_value = *target_ptr;
1588 new_value = (orig_value == *operand1) ? *operand2 : orig_value;
1589 }
1590 while (!__atomic_compare_exchange (target_ptr, &orig_value, &new_value,
1591 /* weak */ 0,
1592 /* success_memmodel */ __ATOMIC_SEQ_CST,
1593 /* failure_memmodel */ __ATOMIC_SEQ_CST));
1594 break;
1595 default: break;
1596 }
1597 if (fetch_ptr != NULL)
1598 *fetch_ptr = orig_value;
1599 }
1601 /**
1602 * UPC atomic relaxed operation.
1603 *
1604 * @param [in] domain Atomic domain
1605 * @param [in] fetch_ptr Target of the update
1606 * @param [in] op Atomic operation
1607 * @param [in] target Target address of the operation
1608 * @param [in] operand1 Operation required argument
1609 * @param [in] operand2 Operation required argument
1610 *
1611 * @ingroup UPCATOMIC UPC Atomic Functions
1612 */
1613 void
1614 upc_atomic_relaxed (upc_atomicdomain_t *domain,
1615 void * restrict fetch_ptr,
1616 upc_op_t op,
1617 shared void * restrict target,
1618 const void * restrict operand1,
1619 const void * restrict operand2)
1620 {
1621 struct upc_atomicdomain_struct *ldomain =
1622 (struct upc_atomicdomain_struct *) &domain[MYTHREAD];
1623 upc_op_num_t op_num;
1624 if (op & ~(-op))
1625 __upc_fatal ("atomic operation (0x%llx) may have only "
1626 "a single bit set", (long long)op);
1627 if (!__upc_atomic_is_valid_op (op))
1628 __upc_fatal ("invalid atomic operation (0x%llx)",
1629 (long long)op);
1630 op_num = __upc_atomic_op_num (op);
1631 if (op & ~ldomain->ops)
1632 __upc_fatal ("invalid operation (%s) for specified domain",
1633 __upc_atomic_op_name (op_num));
1634 __upc_atomic_check_operands (op_num, fetch_ptr, operand1, operand2);
1635 switch (ldomain->optype)
1636 {
1637 case UPC_INT:
1638 __upc_atomic_I (
1639 (I_type *) fetch_ptr,
1640 op_num,
1641 (shared I_type *) target,
1642 (I_type *) operand1,
1643 (I_type *) operand2);
1644 break;
1645 case UPC_UINT:
1646 __upc_atomic_UI (
1647 (UI_type *) fetch_ptr,
1648 op_num,
1649 (shared UI_type *) target,
1650 (UI_type *) operand1,
1651 (UI_type *) operand2);
1652 break;
1653 case UPC_LONG:
1654 __upc_atomic_L (
1655 (L_type *) fetch_ptr,
1656 op_num,
1657 (shared L_type *) target,
1658 (L_type *) operand1,
1659 (L_type *) operand2);
1660 break;
1661 case UPC_ULONG:
1662 __upc_atomic_UL (
1663 (UL_type *) fetch_ptr,
1664 op_num,
1665 (shared UL_type *) target,
1666 (UL_type *) operand1,
1667 (UL_type *) operand2);
1668 break;
1669 case UPC_LLONG:
1670 __upc_atomic_LL (
1671 (LL_type *) fetch_ptr,
1672 op_num,
1673 (shared LL_type *) target,
1674 (LL_type *) operand1,
1675 (LL_type *) operand2);
1676 break;
1677 case UPC_ULLONG:
1678 __upc_atomic_ULL (
1679 (ULL_type *) fetch_ptr,
1680 op_num,
1681 (shared ULL_type *) target,
1682 (ULL_type *) operand1,
1683 (ULL_type *) operand2);
1684 break;
1685 case UPC_INT32:
1686 __upc_atomic_I32 (
1687 (I32_type *) fetch_ptr,
1688 op_num,
1689 (shared I32_type *) target,
1690 (I32_type *) operand1,
1691 (I32_type *) operand2);
1692 break;
1693 case UPC_UINT32:
1694 __upc_atomic_UI32 (
1695 (UI32_type *) fetch_ptr,
1696 op_num,
1697 (shared UI32_type *) target,
1698 (UI32_type *) operand1,
1699 (UI32_type *) operand2);
1700 break;
1701 case UPC_INT64:
1702 __upc_atomic_I64 (
1703 (I64_type *) fetch_ptr,
1704 op_num,
1705 (shared I64_type *) target,
1706 (I64_type *) operand1,
1707 (I64_type *) operand2);
1708 break;
1709 case UPC_UINT64:
1710 __upc_atomic_UI64 (
1711 (UI64_type *) fetch_ptr,
1712 op_num,
1713 (shared UI64_type *) target,
1714 (UI64_type *) operand1,
1715 (UI64_type *) operand2);
1716 break;
1717 case UPC_FLOAT:
1718 __upc_atomic_F (
1719 (F_type *) fetch_ptr,
1720 op_num,
1721 (shared F_type *) target,
1722 (F_type *) operand1,
1723 (F_type *) operand2);
1724 break;
1725 case UPC_DOUBLE:
1726 __upc_atomic_D (
1727 (D_type *) fetch_ptr,
1728 op_num,
1729 (shared D_type *) target,
1730 (D_type *) operand1,
1731 (D_type *) operand2);
1732 break;
1733 case UPC_PTS:
1734 __upc_atomic_PTS (
1735 (PTS_type *) fetch_ptr,
1736 op_num,
1737 (shared PTS_type *) target,
1738 (PTS_type *) operand1,
1739 (PTS_type *) operand2);
1740 break;
1741 }
1742 }
1744 /**
1745 * UPC atomic strict operation.
1746 *
1747 * @param [in] domain Atomic domain
1748 * @param [in] fetch_ptr Target of the update
1749 * @param [in] op Atomic operation
1750 * @param [in] target Target address of the operation
1751 * @param [in] operand1 Operation required argument
1752 * @param [in] operand2 Operation required argument
1753 *
1754 * @ingroup UPCATOMIC UPC Atomic Functions
1755 */
1756 void
1757 upc_atomic_strict (upc_atomicdomain_t *domain,
1758 void * restrict fetch_ptr,
1759 upc_op_t op,
1760 shared void * restrict target,
1761 const void * restrict operand1,
1762 const void * restrict operand2)
1763 {
1764 upc_fence;
1765 upc_atomic_relaxed (domain, fetch_ptr, op, target, operand1, operand2);
1766 upc_fence;
1767 }
1769 /**
1770 * Collective allocation of atomic domain.
1771 *
1772 * Implementation uses native Portals4 atomic functions and the
1773 * hint field is ignored.
1774 *
1775 * @parm [in] type Atomic operation type
1776 * @parm [in] ops Atomic domain operations
1777 * @parm [in] hints Atomic operation hint
1778 * @retval Allocated atomic domain pointer
1779 *
1780 * @ingroup UPCATOMIC UPC Atomic Functions
1781 */
1782 upc_atomicdomain_t *
1783 upc_all_atomicdomain_alloc (upc_type_t type,
1784 upc_op_t ops,
1785 __attribute__((unused)) upc_atomichint_t hints)
1786 {
1787 upc_atomicdomain_t *domain;
1788 struct upc_atomicdomain_struct *ldomain;
1789 upc_op_t supported_ops;
1790 if (!__upc_atomic_is_valid_type (type))
1791 __upc_fatal ("unsupported atomic type: 0x%llx",
1792 (long long) type);
1793 supported_ops = __upc_atomic_supported_ops (type);
1794 if ((ops & ~supported_ops) != 0)
1795 __upc_fatal ("one/more requested atomic operations (0x%llx) unsupported "
1796 "for type `%s'", (long long) ops,
1797 __upc_atomic_type_name (type));
1798 domain = (upc_atomicdomain_t *)
1799 upc_all_alloc (THREADS, sizeof (struct upc_atomicdomain_struct));
1800 if (domain == NULL)
1801 __upc_fatal ("unable to allocate atomic domain");
1802 ldomain = (struct upc_atomicdomain_struct *)&domain[MYTHREAD];
1803 ldomain->ops = ops;
1804 ldomain->optype = type;
1805 return domain;
1806 }
1808 /**
1809 * Collective free of the atomic domain.
1810 *
1811 * @param [in] domain Pointer to atomic domain
1812 *
1813 * @ingroup UPCATOMIC UPC Atomic Functions
1814 */
1815 void
1816 upc_all_atomicdomain_free (upc_atomicdomain_t * domain)
1817 {
1818 assert (domain != NULL);
1819 upc_barrier;
1820 if (MYTHREAD == 0)
1821 {
1822 upc_free (domain);
1823 }
1824 upc_barrier;
1825 }
1827 /**
1828 * Query implementation for expected performance.
1829 *
1830 * @parm [in] ops Atomic domain operations
1831 * @parm [in] optype Atomic operation type
1832 * @parm [in] addr Atomic address
1833 * @retval Expected performance
1834 *
1835 * @ingroup UPCATOMIC UPC Atomic Functions
1836 */
1837 int
1838 upc_atomic_query (__attribute__((unused)) upc_type_t optype,
1839 __attribute__((unused)) upc_op_t ops,
1840 __attribute__((unused)) shared void *addr)
1841 {
1842 /* We could make the distinction that only operations
1843 directly supported by the builtin atomics are "fast",
1844 but for now ... everything in the SMP runtime is
1845 defined to be fast. */
1846 return UPC_ATOMIC_PERFORMANCE_FAST;
1847 }
1849 /** @} */