?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
build: Fix dtc-checkout race condition in Makefile
[qemu/ar7.git] / target / s390x / mem_helper.c
blobbbbe1c62b3485f5986cc6eba4ef37d1d061c5b5a
1 /*
2 * S/390 memory access helper routines
3 *
4 * Copyright (c) 2009 Ulrich Hecht
5 * Copyright (c) 2009 Alexander Graf
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/osdep.h"
22 #include "cpu.h"
23 #include "internal.h"
24 #include "exec/address-spaces.h"
25 #include "exec/helper-proto.h"
26 #include "exec/exec-all.h"
27 #include "exec/cpu_ldst.h"
28 #include "qemu/int128.h"
29
30 #if !defined(CONFIG_USER_ONLY)
31 #include "hw/s390x/storage-keys.h"
32 #endif
33
34 /*****************************************************************************/
35 /* Softmmu support */
36 #if !defined(CONFIG_USER_ONLY)
37
38 /* try to fill the TLB and return an exception if error. If retaddr is
39 NULL, it means that the function was called in C code (i.e. not
40 from generated code or from helper.c) */
41 /* XXX: fix it to restore all registers */
42 void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
43 int mmu_idx, uintptr_t retaddr)
44 {
45 int ret = s390_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
46 if (unlikely(ret != 0)) {
47 cpu_loop_exit_restore(cs, retaddr);
48 }
49 }
50
51 #endif
52
53 /* #define DEBUG_HELPER */
54 #ifdef DEBUG_HELPER
55 #define HELPER_LOG(x...) qemu_log(x)
56 #else
57 #define HELPER_LOG(x...)
58 #endif
59
60 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key)
61 {
62 uint16_t pkm = env->cregs[3] >> 16;
63
64 if (env->psw.mask & PSW_MASK_PSTATE) {
65 /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */
66 return pkm & (0x80 >> psw_key);
67 }
68 return true;
69 }
70
71 /* Reduce the length so that addr + len doesn't cross a page boundary. */
72 static inline uint32_t adj_len_to_page(uint32_t len, uint64_t addr)
73 {
74 #ifndef CONFIG_USER_ONLY
75 if ((addr & ~TARGET_PAGE_MASK) + len - 1 >= TARGET_PAGE_SIZE) {
76 return -(addr | TARGET_PAGE_MASK);
77 }
78 #endif
79 return len;
80 }
81
82 /* Trigger a SPECIFICATION exception if an address or a length is not
83 naturally aligned. */
84 static inline void check_alignment(CPUS390XState *env, uint64_t v,
85 int wordsize, uintptr_t ra)
86 {
87 if (v % wordsize) {
88 CPUState *cs = CPU(s390_env_get_cpu(env));
89 cpu_restore_state(cs, ra);
90 program_interrupt(env, PGM_SPECIFICATION, 6);
91 }
92 }
93
94 /* Load a value from memory according to its size. */
95 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr,
96 int wordsize, uintptr_t ra)
97 {
98 switch (wordsize) {
99 case 1:
100 return cpu_ldub_data_ra(env, addr, ra);
101 case 2:
102 return cpu_lduw_data_ra(env, addr, ra);
103 default:
104 abort();
105 }
106 }
107
108 /* Store a to memory according to its size. */
109 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr,
110 uint64_t value, int wordsize,
111 uintptr_t ra)
112 {
113 switch (wordsize) {
114 case 1:
115 cpu_stb_data_ra(env, addr, value, ra);
116 break;
117 case 2:
118 cpu_stw_data_ra(env, addr, value, ra);
119 break;
120 default:
121 abort();
122 }
123 }
124
125 static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte,
126 uint32_t l, uintptr_t ra)
127 {
128 int mmu_idx = cpu_mmu_index(env, false);
129
130 while (l > 0) {
131 void *p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
132 if (p) {
133 /* Access to the whole page in write mode granted. */
134 uint32_t l_adj = adj_len_to_page(l, dest);
135 memset(p, byte, l_adj);
136 dest += l_adj;
137 l -= l_adj;
138 } else {
139 /* We failed to get access to the whole page. The next write
140 access will likely fill the QEMU TLB for the next iteration. */
141 cpu_stb_data_ra(env, dest, byte, ra);
142 dest++;
143 l--;
144 }
145 }
146 }
147
148 #ifndef CONFIG_USER_ONLY
149 static void fast_memmove_idx(CPUS390XState *env, uint64_t dest, uint64_t src,
150 uint32_t len, int dest_idx, int src_idx,
151 uintptr_t ra)
152 {
153 TCGMemOpIdx oi_dest = make_memop_idx(MO_UB, dest_idx);
154 TCGMemOpIdx oi_src = make_memop_idx(MO_UB, src_idx);
155 uint32_t len_adj;
156 void *src_p;
157 void *dest_p;
158 uint8_t x;
159
160 while (len > 0) {
161 src = wrap_address(env, src);
162 dest = wrap_address(env, dest);
163 src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, src_idx);
164 dest_p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, dest_idx);
165
166 if (src_p && dest_p) {
167 /* Access to both whole pages granted. */
168 len_adj = adj_len_to_page(adj_len_to_page(len, src), dest);
169 memmove(dest_p, src_p, len_adj);
170 } else {
171 /* We failed to get access to one or both whole pages. The next
172 read or write access will likely fill the QEMU TLB for the
173 next iteration. */
174 len_adj = 1;
175 x = helper_ret_ldub_mmu(env, src, oi_src, ra);
176 helper_ret_stb_mmu(env, dest, x, oi_dest, ra);
177 }
178 src += len_adj;
179 dest += len_adj;
180 len -= len_adj;
181 }
182 }
183
184 static int mmu_idx_from_as(uint8_t as)
185 {
186 switch (as) {
187 case AS_PRIMARY:
188 return MMU_PRIMARY_IDX;
189 case AS_SECONDARY:
190 return MMU_SECONDARY_IDX;
191 case AS_HOME:
192 return MMU_HOME_IDX;
193 default:
194 /* FIXME AS_ACCREG */
195 g_assert_not_reached();
196 }
197 }
198
199 static void fast_memmove_as(CPUS390XState *env, uint64_t dest, uint64_t src,
200 uint32_t len, uint8_t dest_as, uint8_t src_as,
201 uintptr_t ra)
202 {
203 int src_idx = mmu_idx_from_as(src_as);
204 int dest_idx = mmu_idx_from_as(dest_as);
205
206 fast_memmove_idx(env, dest, src, len, dest_idx, src_idx, ra);
207 }
208 #endif
209
210 static void fast_memmove(CPUS390XState *env, uint64_t dest, uint64_t src,
211 uint32_t l, uintptr_t ra)
212 {
213 int mmu_idx = cpu_mmu_index(env, false);
214
215 while (l > 0) {
216 void *src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, mmu_idx);
217 void *dest_p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
218 if (src_p && dest_p) {
219 /* Access to both whole pages granted. */
220 uint32_t l_adj = adj_len_to_page(l, src);
221 l_adj = adj_len_to_page(l_adj, dest);
222 memmove(dest_p, src_p, l_adj);
223 src += l_adj;
224 dest += l_adj;
225 l -= l_adj;
226 } else {
227 /* We failed to get access to one or both whole pages. The next
228 read or write access will likely fill the QEMU TLB for the
229 next iteration. */
230 cpu_stb_data_ra(env, dest, cpu_ldub_data_ra(env, src, ra), ra);
231 src++;
232 dest++;
233 l--;
234 }
235 }
236 }
237
238 /* and on array */
239 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest,
240 uint64_t src, uintptr_t ra)
241 {
242 uint32_t i;
243 uint8_t c = 0;
244
245 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
246 __func__, l, dest, src);
247
248 for (i = 0; i <= l; i++) {
249 uint8_t x = cpu_ldub_data_ra(env, src + i, ra);
250 x &= cpu_ldub_data_ra(env, dest + i, ra);
251 c |= x;
252 cpu_stb_data_ra(env, dest + i, x, ra);
253 }
254 return c != 0;
255 }
256
257 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
258 uint64_t src)
259 {
260 return do_helper_nc(env, l, dest, src, GETPC());
261 }
262
263 /* xor on array */
264 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest,
265 uint64_t src, uintptr_t ra)
266 {
267 uint32_t i;
268 uint8_t c = 0;
269
270 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
271 __func__, l, dest, src);
272
273 /* xor with itself is the same as memset(0) */
274 if (src == dest) {
275 fast_memset(env, dest, 0, l + 1, ra);
276 return 0;
277 }
278
279 for (i = 0; i <= l; i++) {
280 uint8_t x = cpu_ldub_data_ra(env, src + i, ra);
281 x ^= cpu_ldub_data_ra(env, dest + i, ra);
282 c |= x;
283 cpu_stb_data_ra(env, dest + i, x, ra);
284 }
285 return c != 0;
286 }
287
288 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
289 uint64_t src)
290 {
291 return do_helper_xc(env, l, dest, src, GETPC());
292 }
293
294 /* or on array */
295 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest,
296 uint64_t src, uintptr_t ra)
297 {
298 uint32_t i;
299 uint8_t c = 0;
300
301 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
302 __func__, l, dest, src);
303
304 for (i = 0; i <= l; i++) {
305 uint8_t x = cpu_ldub_data_ra(env, src + i, ra);
306 x |= cpu_ldub_data_ra(env, dest + i, ra);
307 c |= x;
308 cpu_stb_data_ra(env, dest + i, x, ra);
309 }
310 return c != 0;
311 }
312
313 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
314 uint64_t src)
315 {
316 return do_helper_oc(env, l, dest, src, GETPC());
317 }
318
319 /* memmove */
320 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest,
321 uint64_t src, uintptr_t ra)
322 {
323 uint32_t i;
324
325 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
326 __func__, l, dest, src);
327
328 /* mvc and memmove do not behave the same when areas overlap! */
329 /* mvc with source pointing to the byte after the destination is the
330 same as memset with the first source byte */
331 if (dest == src + 1) {
332 fast_memset(env, dest, cpu_ldub_data_ra(env, src, ra), l + 1, ra);
333 } else if (dest < src || src + l < dest) {
334 fast_memmove(env, dest, src, l + 1, ra);
335 } else {
336 /* slow version with byte accesses which always work */
337 for (i = 0; i <= l; i++) {
338 uint8_t x = cpu_ldub_data_ra(env, src + i, ra);
339 cpu_stb_data_ra(env, dest + i, x, ra);
340 }
341 }
342
343 return env->cc_op;
344 }
345
346 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
347 {
348 do_helper_mvc(env, l, dest, src, GETPC());
349 }
350
351 /* move inverse */
352 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
353 {
354 uintptr_t ra = GETPC();
355 int i;
356
357 for (i = 0; i <= l; i++) {
358 uint8_t v = cpu_ldub_data_ra(env, src - i, ra);
359 cpu_stb_data_ra(env, dest + i, v, ra);
360 }
361 }
362
363 /* move numerics */
364 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
365 {
366 uintptr_t ra = GETPC();
367 int i;
368
369 for (i = 0; i <= l; i++) {
370 uint8_t v = cpu_ldub_data_ra(env, dest + i, ra) & 0xf0;
371 v |= cpu_ldub_data_ra(env, src + i, ra) & 0x0f;
372 cpu_stb_data_ra(env, dest + i, v, ra);
373 }
374 }
375
376 /* move with offset */
377 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
378 {
379 uintptr_t ra = GETPC();
380 int len_dest = l >> 4;
381 int len_src = l & 0xf;
382 uint8_t byte_dest, byte_src;
383 int i;
384
385 src += len_src;
386 dest += len_dest;
387
388 /* Handle rightmost byte */
389 byte_src = cpu_ldub_data_ra(env, src, ra);
390 byte_dest = cpu_ldub_data_ra(env, dest, ra);
391 byte_dest = (byte_dest & 0x0f) | (byte_src << 4);
392 cpu_stb_data_ra(env, dest, byte_dest, ra);
393
394 /* Process remaining bytes from right to left */
395 for (i = 1; i <= len_dest; i++) {
396 byte_dest = byte_src >> 4;
397 if (len_src - i >= 0) {
398 byte_src = cpu_ldub_data_ra(env, src - i, ra);
399 } else {
400 byte_src = 0;
401 }
402 byte_dest |= byte_src << 4;
403 cpu_stb_data_ra(env, dest - i, byte_dest, ra);
404 }
405 }
406
407 /* move zones */
408 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
409 {
410 uintptr_t ra = GETPC();
411 int i;
412
413 for (i = 0; i <= l; i++) {
414 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra) & 0x0f;
415 b |= cpu_ldub_data_ra(env, src + i, ra) & 0xf0;
416 cpu_stb_data_ra(env, dest + i, b, ra);
417 }
418 }
419
420 /* compare unsigned byte arrays */
421 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1,
422 uint64_t s2, uintptr_t ra)
423 {
424 uint32_t i;
425 uint32_t cc = 0;
426
427 HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
428 __func__, l, s1, s2);
429
430 for (i = 0; i <= l; i++) {
431 uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra);
432 uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra);
433 HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
434 if (x < y) {
435 cc = 1;
436 break;
437 } else if (x > y) {
438 cc = 2;
439 break;
440 }
441 }
442
443 HELPER_LOG("\n");
444 return cc;
445 }
446
447 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
448 {
449 return do_helper_clc(env, l, s1, s2, GETPC());
450 }
451
452 /* compare logical under mask */
453 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
454 uint64_t addr)
455 {
456 uintptr_t ra = GETPC();
457 uint32_t cc = 0;
458
459 HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
460 mask, addr);
461
462 while (mask) {
463 if (mask & 8) {
464 uint8_t d = cpu_ldub_data_ra(env, addr, ra);
465 uint8_t r = extract32(r1, 24, 8);
466 HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
467 addr);
468 if (r < d) {
469 cc = 1;
470 break;
471 } else if (r > d) {
472 cc = 2;
473 break;
474 }
475 addr++;
476 }
477 mask = (mask << 1) & 0xf;
478 r1 <<= 8;
479 }
480
481 HELPER_LOG("\n");
482 return cc;
483 }
484
485 static inline uint64_t get_address(CPUS390XState *env, int reg)
486 {
487 return wrap_address(env, env->regs[reg]);
488 }
489
490 static inline void set_address(CPUS390XState *env, int reg, uint64_t address)
491 {
492 if (env->psw.mask & PSW_MASK_64) {
493 /* 64-Bit mode */
494 env->regs[reg] = address;
495 } else {
496 if (!(env->psw.mask & PSW_MASK_32)) {
497 /* 24-Bit mode. According to the PoO it is implementation
498 dependent if bits 32-39 remain unchanged or are set to
499 zeros. Choose the former so that the function can also be
500 used for TRT. */
501 env->regs[reg] = deposit64(env->regs[reg], 0, 24, address);
502 } else {
503 /* 31-Bit mode. According to the PoO it is implementation
504 dependent if bit 32 remains unchanged or is set to zero.
505 Choose the latter so that the function can also be used for
506 TRT. */
507 address &= 0x7fffffff;
508 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
509 }
510 }
511 }
512
513 static inline uint64_t wrap_length(CPUS390XState *env, uint64_t length)
514 {
515 if (!(env->psw.mask & PSW_MASK_64)) {
516 /* 24-Bit and 31-Bit mode */
517 length &= 0x7fffffff;
518 }
519 return length;
520 }
521
522 static inline uint64_t get_length(CPUS390XState *env, int reg)
523 {
524 return wrap_length(env, env->regs[reg]);
525 }
526
527 static inline void set_length(CPUS390XState *env, int reg, uint64_t length)
528 {
529 if (env->psw.mask & PSW_MASK_64) {
530 /* 64-Bit mode */
531 env->regs[reg] = length;
532 } else {
533 /* 24-Bit and 31-Bit mode */
534 env->regs[reg] = deposit64(env->regs[reg], 0, 32, length);
535 }
536 }
537
538 /* search string (c is byte to search, r2 is string, r1 end of string) */
539 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
540 {
541 uintptr_t ra = GETPC();
542 uint64_t end, str;
543 uint32_t len;
544 uint8_t v, c = env->regs[0];
545
546 /* Bits 32-55 must contain all 0. */
547 if (env->regs[0] & 0xffffff00u) {
548 cpu_restore_state(ENV_GET_CPU(env), ra);
549 program_interrupt(env, PGM_SPECIFICATION, 6);
550 }
551
552 str = get_address(env, r2);
553 end = get_address(env, r1);
554
555 /* Lest we fail to service interrupts in a timely manner, limit the
556 amount of work we're willing to do. For now, let's cap at 8k. */
557 for (len = 0; len < 0x2000; ++len) {
558 if (str + len == end) {
559 /* Character not found. R1 & R2 are unmodified. */
560 env->cc_op = 2;
561 return;
562 }
563 v = cpu_ldub_data_ra(env, str + len, ra);
564 if (v == c) {
565 /* Character found. Set R1 to the location; R2 is unmodified. */
566 env->cc_op = 1;
567 set_address(env, r1, str + len);
568 return;
569 }
570 }
571
572 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
573 env->cc_op = 3;
574 set_address(env, r2, str + len);
575 }
576
577 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2)
578 {
579 uintptr_t ra = GETPC();
580 uint32_t len;
581 uint16_t v, c = env->regs[0];
582 uint64_t end, str, adj_end;
583
584 /* Bits 32-47 of R0 must be zero. */
585 if (env->regs[0] & 0xffff0000u) {
586 cpu_restore_state(ENV_GET_CPU(env), ra);
587 program_interrupt(env, PGM_SPECIFICATION, 6);
588 }
589
590 str = get_address(env, r2);
591 end = get_address(env, r1);
592
593 /* If the LSB of the two addresses differ, use one extra byte. */
594 adj_end = end + ((str ^ end) & 1);
595
596 /* Lest we fail to service interrupts in a timely manner, limit the
597 amount of work we're willing to do. For now, let's cap at 8k. */
598 for (len = 0; len < 0x2000; len += 2) {
599 if (str + len == adj_end) {
600 /* End of input found. */
601 env->cc_op = 2;
602 return;
603 }
604 v = cpu_lduw_data_ra(env, str + len, ra);
605 if (v == c) {
606 /* Character found. Set R1 to the location; R2 is unmodified. */
607 env->cc_op = 1;
608 set_address(env, r1, str + len);
609 return;
610 }
611 }
612
613 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
614 env->cc_op = 3;
615 set_address(env, r2, str + len);
616 }
617
618 /* unsigned string compare (c is string terminator) */
619 uint64_t HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
620 {
621 uintptr_t ra = GETPC();
622 uint32_t len;
623
624 c = c & 0xff;
625 s1 = wrap_address(env, s1);
626 s2 = wrap_address(env, s2);
627
628 /* Lest we fail to service interrupts in a timely manner, limit the
629 amount of work we're willing to do. For now, let's cap at 8k. */
630 for (len = 0; len < 0x2000; ++len) {
631 uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra);
632 uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra);
633 if (v1 == v2) {
634 if (v1 == c) {
635 /* Equal. CC=0, and don't advance the registers. */
636 env->cc_op = 0;
637 env->retxl = s2;
638 return s1;
639 }
640 } else {
641 /* Unequal. CC={1,2}, and advance the registers. Note that
642 the terminator need not be zero, but the string that contains
643 the terminator is by definition "low". */
644 env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
645 env->retxl = s2 + len;
646 return s1 + len;
647 }
648 }
649
650 /* CPU-determined bytes equal; advance the registers. */
651 env->cc_op = 3;
652 env->retxl = s2 + len;
653 return s1 + len;
654 }
655
656 /* move page */
657 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint64_t r1, uint64_t r2)
658 {
659 /* ??? missing r0 handling, which includes access keys, but more
660 importantly optional suppression of the exception! */
661 fast_memmove(env, r1, r2, TARGET_PAGE_SIZE, GETPC());
662 return 0; /* data moved */
663 }
664
665 /* string copy (c is string terminator) */
666 uint64_t HELPER(mvst)(CPUS390XState *env, uint64_t c, uint64_t d, uint64_t s)
667 {
668 uintptr_t ra = GETPC();
669 uint32_t len;
670
671 c = c & 0xff;
672 d = wrap_address(env, d);
673 s = wrap_address(env, s);
674
675 /* Lest we fail to service interrupts in a timely manner, limit the
676 amount of work we're willing to do. For now, let's cap at 8k. */
677 for (len = 0; len < 0x2000; ++len) {
678 uint8_t v = cpu_ldub_data_ra(env, s + len, ra);
679 cpu_stb_data_ra(env, d + len, v, ra);
680 if (v == c) {
681 /* Complete. Set CC=1 and advance R1. */
682 env->cc_op = 1;
683 env->retxl = s;
684 return d + len;
685 }
686 }
687
688 /* Incomplete. Set CC=3 and signal to advance R1 and R2. */
689 env->cc_op = 3;
690 env->retxl = s + len;
691 return d + len;
692 }
693
694 /* load access registers r1 to r3 from memory at a2 */
695 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
696 {
697 uintptr_t ra = GETPC();
698 int i;
699
700 for (i = r1;; i = (i + 1) % 16) {
701 env->aregs[i] = cpu_ldl_data_ra(env, a2, ra);
702 a2 += 4;
703
704 if (i == r3) {
705 break;
706 }
707 }
708 }
709
710 /* store access registers r1 to r3 in memory at a2 */
711 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
712 {
713 uintptr_t ra = GETPC();
714 int i;
715
716 for (i = r1;; i = (i + 1) % 16) {
717 cpu_stl_data_ra(env, a2, env->aregs[i], ra);
718 a2 += 4;
719
720 if (i == r3) {
721 break;
722 }
723 }
724 }
725
726 /* move long helper */
727 static inline uint32_t do_mvcl(CPUS390XState *env,
728 uint64_t *dest, uint64_t *destlen,
729 uint64_t *src, uint64_t *srclen,
730 uint16_t pad, int wordsize, uintptr_t ra)
731 {
732 uint64_t len = MIN(*srclen, *destlen);
733 uint32_t cc;
734
735 if (*destlen == *srclen) {
736 cc = 0;
737 } else if (*destlen < *srclen) {
738 cc = 1;
739 } else {
740 cc = 2;
741 }
742
743 /* Copy the src array */
744 fast_memmove(env, *dest, *src, len, ra);
745 *src += len;
746 *srclen -= len;
747 *dest += len;
748 *destlen -= len;
749
750 /* Pad the remaining area */
751 if (wordsize == 1) {
752 fast_memset(env, *dest, pad, *destlen, ra);
753 *dest += *destlen;
754 *destlen = 0;
755 } else {
756 /* If remaining length is odd, pad with odd byte first. */
757 if (*destlen & 1) {
758 cpu_stb_data_ra(env, *dest, pad & 0xff, ra);
759 *dest += 1;
760 *destlen -= 1;
761 }
762 /* The remaining length is even, pad using words. */
763 for (; *destlen; *dest += 2, *destlen -= 2) {
764 cpu_stw_data_ra(env, *dest, pad, ra);
765 }
766 }
767
768 return cc;
769 }
770
771 /* move long */
772 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
773 {
774 uintptr_t ra = GETPC();
775 uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
776 uint64_t dest = get_address(env, r1);
777 uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
778 uint64_t src = get_address(env, r2);
779 uint8_t pad = env->regs[r2 + 1] >> 24;
780 uint32_t cc;
781
782 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra);
783
784 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen);
785 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen);
786 set_address(env, r1, dest);
787 set_address(env, r2, src);
788
789 return cc;
790 }
791
792 /* move long extended */
793 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
794 uint32_t r3)
795 {
796 uintptr_t ra = GETPC();
797 uint64_t destlen = get_length(env, r1 + 1);
798 uint64_t dest = get_address(env, r1);
799 uint64_t srclen = get_length(env, r3 + 1);
800 uint64_t src = get_address(env, r3);
801 uint8_t pad = a2;
802 uint32_t cc;
803
804 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra);
805
806 set_length(env, r1 + 1, destlen);
807 set_length(env, r3 + 1, srclen);
808 set_address(env, r1, dest);
809 set_address(env, r3, src);
810
811 return cc;
812 }
813
814 /* move long unicode */
815 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
816 uint32_t r3)
817 {
818 uintptr_t ra = GETPC();
819 uint64_t destlen = get_length(env, r1 + 1);
820 uint64_t dest = get_address(env, r1);
821 uint64_t srclen = get_length(env, r3 + 1);
822 uint64_t src = get_address(env, r3);
823 uint16_t pad = a2;
824 uint32_t cc;
825
826 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra);
827
828 set_length(env, r1 + 1, destlen);
829 set_length(env, r3 + 1, srclen);
830 set_address(env, r1, dest);
831 set_address(env, r3, src);
832
833 return cc;
834 }
835
836 /* compare logical long helper */
837 static inline uint32_t do_clcl(CPUS390XState *env,
838 uint64_t *src1, uint64_t *src1len,
839 uint64_t *src3, uint64_t *src3len,
840 uint16_t pad, uint64_t limit,
841 int wordsize, uintptr_t ra)
842 {
843 uint64_t len = MAX(*src1len, *src3len);
844 uint32_t cc = 0;
845
846 check_alignment(env, *src1len | *src3len, wordsize, ra);
847
848 if (!len) {
849 return cc;
850 }
851
852 /* Lest we fail to service interrupts in a timely manner, limit the
853 amount of work we're willing to do. */
854 if (len > limit) {
855 len = limit;
856 cc = 3;
857 }
858
859 for (; len; len -= wordsize) {
860 uint16_t v1 = pad;
861 uint16_t v3 = pad;
862
863 if (*src1len) {
864 v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra);
865 }
866 if (*src3len) {
867 v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra);
868 }
869
870 if (v1 != v3) {
871 cc = (v1 < v3) ? 1 : 2;
872 break;
873 }
874
875 if (*src1len) {
876 *src1 += wordsize;
877 *src1len -= wordsize;
878 }
879 if (*src3len) {
880 *src3 += wordsize;
881 *src3len -= wordsize;
882 }
883 }
884
885 return cc;
886 }
887
888
889 /* compare logical long */
890 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
891 {
892 uintptr_t ra = GETPC();
893 uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24);
894 uint64_t src1 = get_address(env, r1);
895 uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24);
896 uint64_t src3 = get_address(env, r2);
897 uint8_t pad = env->regs[r2 + 1] >> 24;
898 uint32_t cc;
899
900 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra);
901
902 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len);
903 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len);
904 set_address(env, r1, src1);
905 set_address(env, r2, src3);
906
907 return cc;
908 }
909
910 /* compare logical long extended memcompare insn with padding */
911 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
912 uint32_t r3)
913 {
914 uintptr_t ra = GETPC();
915 uint64_t src1len = get_length(env, r1 + 1);
916 uint64_t src1 = get_address(env, r1);
917 uint64_t src3len = get_length(env, r3 + 1);
918 uint64_t src3 = get_address(env, r3);
919 uint8_t pad = a2;
920 uint32_t cc;
921
922 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra);
923
924 set_length(env, r1 + 1, src1len);
925 set_length(env, r3 + 1, src3len);
926 set_address(env, r1, src1);
927 set_address(env, r3, src3);
928
929 return cc;
930 }
931
932 /* compare logical long unicode memcompare insn with padding */
933 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
934 uint32_t r3)
935 {
936 uintptr_t ra = GETPC();
937 uint64_t src1len = get_length(env, r1 + 1);
938 uint64_t src1 = get_address(env, r1);
939 uint64_t src3len = get_length(env, r3 + 1);
940 uint64_t src3 = get_address(env, r3);
941 uint16_t pad = a2;
942 uint32_t cc = 0;
943
944 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra);
945
946 set_length(env, r1 + 1, src1len);
947 set_length(env, r3 + 1, src3len);
948 set_address(env, r1, src1);
949 set_address(env, r3, src3);
950
951 return cc;
952 }
953
954 /* checksum */
955 uint64_t HELPER(cksm)(CPUS390XState *env, uint64_t r1,
956 uint64_t src, uint64_t src_len)
957 {
958 uintptr_t ra = GETPC();
959 uint64_t max_len, len;
960 uint64_t cksm = (uint32_t)r1;
961
962 /* Lest we fail to service interrupts in a timely manner, limit the
963 amount of work we're willing to do. For now, let's cap at 8k. */
964 max_len = (src_len > 0x2000 ? 0x2000 : src_len);
965
966 /* Process full words as available. */
967 for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
968 cksm += (uint32_t)cpu_ldl_data_ra(env, src, ra);
969 }
970
971 switch (max_len - len) {
972 case 1:
973 cksm += cpu_ldub_data_ra(env, src, ra) << 24;
974 len += 1;
975 break;
976 case 2:
977 cksm += cpu_lduw_data_ra(env, src, ra) << 16;
978 len += 2;
979 break;
980 case 3:
981 cksm += cpu_lduw_data_ra(env, src, ra) << 16;
982 cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8;
983 len += 3;
984 break;
985 }
986
987 /* Fold the carry from the checksum. Note that we can see carry-out
988 during folding more than once (but probably not more than twice). */
989 while (cksm > 0xffffffffull) {
990 cksm = (uint32_t)cksm + (cksm >> 32);
991 }
992
993 /* Indicate whether or not we've processed everything. */
994 env->cc_op = (len == src_len ? 0 : 3);
995
996 /* Return both cksm and processed length. */
997 env->retxl = cksm;
998 return len;
999 }
1000
1001 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src)
1002 {
1003 uintptr_t ra = GETPC();
1004 int len_dest = len >> 4;
1005 int len_src = len & 0xf;
1006 uint8_t b;
1007
1008 dest += len_dest;
1009 src += len_src;
1010
1011 /* last byte is special, it only flips the nibbles */
1012 b = cpu_ldub_data_ra(env, src, ra);
1013 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1014 src--;
1015 len_src--;
1016
1017 /* now pack every value */
1018 while (len_dest >= 0) {
1019 b = 0;
1020
1021 if (len_src > 0) {
1022 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1023 src--;
1024 len_src--;
1025 }
1026 if (len_src > 0) {
1027 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1028 src--;
1029 len_src--;
1030 }
1031
1032 len_dest--;
1033 dest--;
1034 cpu_stb_data_ra(env, dest, b, ra);
1035 }
1036 }
1037
1038 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src,
1039 uint32_t srclen, int ssize, uintptr_t ra)
1040 {
1041 int i;
1042 /* The destination operand is always 16 bytes long. */
1043 const int destlen = 16;
1044
1045 /* The operands are processed from right to left. */
1046 src += srclen - 1;
1047 dest += destlen - 1;
1048
1049 for (i = 0; i < destlen; i++) {
1050 uint8_t b = 0;
1051
1052 /* Start with a positive sign */
1053 if (i == 0) {
1054 b = 0xc;
1055 } else if (srclen > ssize) {
1056 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1057 src -= ssize;
1058 srclen -= ssize;
1059 }
1060
1061 if (srclen > ssize) {
1062 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1063 src -= ssize;
1064 srclen -= ssize;
1065 }
1066
1067 cpu_stb_data_ra(env, dest, b, ra);
1068 dest--;
1069 }
1070 }
1071
1072
1073 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src,
1074 uint32_t srclen)
1075 {
1076 do_pkau(env, dest, src, srclen, 1, GETPC());
1077 }
1078
1079 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src,
1080 uint32_t srclen)
1081 {
1082 do_pkau(env, dest, src, srclen, 2, GETPC());
1083 }
1084
1085 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
1086 uint64_t src)
1087 {
1088 uintptr_t ra = GETPC();
1089 int len_dest = len >> 4;
1090 int len_src = len & 0xf;
1091 uint8_t b;
1092 int second_nibble = 0;
1093
1094 dest += len_dest;
1095 src += len_src;
1096
1097 /* last byte is special, it only flips the nibbles */
1098 b = cpu_ldub_data_ra(env, src, ra);
1099 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1100 src--;
1101 len_src--;
1102
1103 /* now pad every nibble with 0xf0 */
1104
1105 while (len_dest > 0) {
1106 uint8_t cur_byte = 0;
1107
1108 if (len_src > 0) {
1109 cur_byte = cpu_ldub_data_ra(env, src, ra);
1110 }
1111
1112 len_dest--;
1113 dest--;
1114
1115 /* only advance one nibble at a time */
1116 if (second_nibble) {
1117 cur_byte >>= 4;
1118 len_src--;
1119 src--;
1120 }
1121 second_nibble = !second_nibble;
1122
1123 /* digit */
1124 cur_byte = (cur_byte & 0xf);
1125 /* zone bits */
1126 cur_byte |= 0xf0;
1127
1128 cpu_stb_data_ra(env, dest, cur_byte, ra);
1129 }
1130 }
1131
1132 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest,
1133 uint32_t destlen, int dsize, uint64_t src,
1134 uintptr_t ra)
1135 {
1136 int i;
1137 uint32_t cc;
1138 uint8_t b;
1139 /* The source operand is always 16 bytes long. */
1140 const int srclen = 16;
1141
1142 /* The operands are processed from right to left. */
1143 src += srclen - 1;
1144 dest += destlen - dsize;
1145
1146 /* Check for the sign. */
1147 b = cpu_ldub_data_ra(env, src, ra);
1148 src--;
1149 switch (b & 0xf) {
1150 case 0xa:
1151 case 0xc:
1152 case 0xe ... 0xf:
1153 cc = 0; /* plus */
1154 break;
1155 case 0xb:
1156 case 0xd:
1157 cc = 1; /* minus */
1158 break;
1159 default:
1160 case 0x0 ... 0x9:
1161 cc = 3; /* invalid */
1162 break;
1163 }
1164
1165 /* Now pad every nibble with 0x30, advancing one nibble at a time. */
1166 for (i = 0; i < destlen; i += dsize) {
1167 if (i == (31 * dsize)) {
1168 /* If length is 32/64 bytes, the leftmost byte is 0. */
1169 b = 0;
1170 } else if (i % (2 * dsize)) {
1171 b = cpu_ldub_data_ra(env, src, ra);
1172 src--;
1173 } else {
1174 b >>= 4;
1175 }
1176 cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra);
1177 dest -= dsize;
1178 }
1179
1180 return cc;
1181 }
1182
1183 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1184 uint64_t src)
1185 {
1186 return do_unpkau(env, dest, destlen, 1, src, GETPC());
1187 }
1188
1189 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1190 uint64_t src)
1191 {
1192 return do_unpkau(env, dest, destlen, 2, src, GETPC());
1193 }
1194
1195 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen)
1196 {
1197 uintptr_t ra = GETPC();
1198 uint32_t cc = 0;
1199 int i;
1200
1201 for (i = 0; i < destlen; i++) {
1202 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra);
1203 /* digit */
1204 cc |= (b & 0xf0) > 0x90 ? 2 : 0;
1205
1206 if (i == (destlen - 1)) {
1207 /* sign */
1208 cc |= (b & 0xf) < 0xa ? 1 : 0;
1209 } else {
1210 /* digit */
1211 cc |= (b & 0xf) > 0x9 ? 2 : 0;
1212 }
1213 }
1214
1215 return cc;
1216 }
1217
1218 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array,
1219 uint64_t trans, uintptr_t ra)
1220 {
1221 uint32_t i;
1222
1223 for (i = 0; i <= len; i++) {
1224 uint8_t byte = cpu_ldub_data_ra(env, array + i, ra);
1225 uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1226 cpu_stb_data_ra(env, array + i, new_byte, ra);
1227 }
1228
1229 return env->cc_op;
1230 }
1231
1232 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
1233 uint64_t trans)
1234 {
1235 do_helper_tr(env, len, array, trans, GETPC());
1236 }
1237
1238 uint64_t HELPER(tre)(CPUS390XState *env, uint64_t array,
1239 uint64_t len, uint64_t trans)
1240 {
1241 uintptr_t ra = GETPC();
1242 uint8_t end = env->regs[0] & 0xff;
1243 uint64_t l = len;
1244 uint64_t i;
1245 uint32_t cc = 0;
1246
1247 if (!(env->psw.mask & PSW_MASK_64)) {
1248 array &= 0x7fffffff;
1249 l = (uint32_t)l;
1250 }
1251
1252 /* Lest we fail to service interrupts in a timely manner, limit the
1253 amount of work we're willing to do. For now, let's cap at 8k. */
1254 if (l > 0x2000) {
1255 l = 0x2000;
1256 cc = 3;
1257 }
1258
1259 for (i = 0; i < l; i++) {
1260 uint8_t byte, new_byte;
1261
1262 byte = cpu_ldub_data_ra(env, array + i, ra);
1263
1264 if (byte == end) {
1265 cc = 1;
1266 break;
1267 }
1268
1269 new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1270 cpu_stb_data_ra(env, array + i, new_byte, ra);
1271 }
1272
1273 env->cc_op = cc;
1274 env->retxl = len - i;
1275 return array + i;
1276 }
1277
1278 static inline uint32_t do_helper_trt(CPUS390XState *env, int len,
1279 uint64_t array, uint64_t trans,
1280 int inc, uintptr_t ra)
1281 {
1282 int i;
1283
1284 for (i = 0; i <= len; i++) {
1285 uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra);
1286 uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra);
1287
1288 if (sbyte != 0) {
1289 set_address(env, 1, array + i * inc);
1290 env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte);
1291 return (i == len) ? 2 : 1;
1292 }
1293 }
1294
1295 return 0;
1296 }
1297
1298 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array,
1299 uint64_t trans)
1300 {
1301 return do_helper_trt(env, len, array, trans, 1, GETPC());
1302 }
1303
1304 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array,
1305 uint64_t trans)
1306 {
1307 return do_helper_trt(env, len, array, trans, -1, GETPC());
1308 }
1309
1310 /* Translate one/two to one/two */
1311 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2,
1312 uint32_t tst, uint32_t sizes)
1313 {
1314 uintptr_t ra = GETPC();
1315 int dsize = (sizes & 1) ? 1 : 2;
1316 int ssize = (sizes & 2) ? 1 : 2;
1317 uint64_t tbl = get_address(env, 1);
1318 uint64_t dst = get_address(env, r1);
1319 uint64_t len = get_length(env, r1 + 1);
1320 uint64_t src = get_address(env, r2);
1321 uint32_t cc = 3;
1322 int i;
1323
1324 /* The lower address bits of TBL are ignored. For TROO, TROT, it's
1325 the low 3 bits (double-word aligned). For TRTO, TRTT, it's either
1326 the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */
1327 if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) {
1328 tbl &= -4096;
1329 } else {
1330 tbl &= -8;
1331 }
1332
1333 check_alignment(env, len, ssize, ra);
1334
1335 /* Lest we fail to service interrupts in a timely manner, */
1336 /* limit the amount of work we're willing to do. */
1337 for (i = 0; i < 0x2000; i++) {
1338 uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra);
1339 uint64_t tble = tbl + (sval * dsize);
1340 uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra);
1341 if (dval == tst) {
1342 cc = 1;
1343 break;
1344 }
1345 cpu_stsize_data_ra(env, dst, dval, dsize, ra);
1346
1347 len -= ssize;
1348 src += ssize;
1349 dst += dsize;
1350
1351 if (len == 0) {
1352 cc = 0;
1353 break;
1354 }
1355 }
1356
1357 set_address(env, r1, dst);
1358 set_length(env, r1 + 1, len);
1359 set_address(env, r2, src);
1360
1361 return cc;
1362 }
1363
1364 void HELPER(cdsg)(CPUS390XState *env, uint64_t addr,
1365 uint32_t r1, uint32_t r3)
1366 {
1367 uintptr_t ra = GETPC();
1368 Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]);
1369 Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1370 Int128 oldv;
1371 bool fail;
1372
1373 if (parallel_cpus) {
1374 #ifndef CONFIG_ATOMIC128
1375 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
1376 #else
1377 int mem_idx = cpu_mmu_index(env, false);
1378 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
1379 oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
1380 fail = !int128_eq(oldv, cmpv);
1381 #endif
1382 } else {
1383 uint64_t oldh, oldl;
1384
1385 check_alignment(env, addr, 16, ra);
1386
1387 oldh = cpu_ldq_data_ra(env, addr + 0, ra);
1388 oldl = cpu_ldq_data_ra(env, addr + 8, ra);
1389
1390 oldv = int128_make128(oldl, oldh);
1391 fail = !int128_eq(oldv, cmpv);
1392 if (fail) {
1393 newv = oldv;
1394 }
1395
1396 cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra);
1397 cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra);
1398 }
1399
1400 env->cc_op = fail;
1401 env->regs[r1] = int128_gethi(oldv);
1402 env->regs[r1 + 1] = int128_getlo(oldv);
1403 }
1404
1405 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2)
1406 {
1407 #if !defined(CONFIG_USER_ONLY) || defined(CONFIG_ATOMIC128)
1408 uint32_t mem_idx = cpu_mmu_index(env, false);
1409 #endif
1410 uintptr_t ra = GETPC();
1411 uint32_t fc = extract32(env->regs[0], 0, 8);
1412 uint32_t sc = extract32(env->regs[0], 8, 8);
1413 uint64_t pl = get_address(env, 1) & -16;
1414 uint64_t svh, svl;
1415 uint32_t cc;
1416
1417 /* Sanity check the function code and storage characteristic. */
1418 if (fc > 1 || sc > 3) {
1419 if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) {
1420 goto spec_exception;
1421 }
1422 if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) {
1423 goto spec_exception;
1424 }
1425 }
1426
1427 /* Sanity check the alignments. */
1428 if (extract32(a1, 0, 4 << fc) || extract32(a2, 0, 1 << sc)) {
1429 goto spec_exception;
1430 }
1431
1432 /* Sanity check writability of the store address. */
1433 #ifndef CONFIG_USER_ONLY
1434 probe_write(env, a2, mem_idx, ra);
1435 #endif
1436
1437 /* Note that the compare-and-swap is atomic, and the store is atomic, but
1438 the complete operation is not. Therefore we do not need to assert serial
1439 context in order to implement this. That said, restart early if we can't
1440 support either operation that is supposed to be atomic. */
1441 if (parallel_cpus) {
1442 int mask = 0;
1443 #if !defined(CONFIG_ATOMIC64)
1444 mask = -8;
1445 #elif !defined(CONFIG_ATOMIC128)
1446 mask = -16;
1447 #endif
1448 if (((4 << fc) | (1 << sc)) & mask) {
1449 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
1450 }
1451 }
1452
1453 /* All loads happen before all stores. For simplicity, load the entire
1454 store value area from the parameter list. */
1455 svh = cpu_ldq_data_ra(env, pl + 16, ra);
1456 svl = cpu_ldq_data_ra(env, pl + 24, ra);
1457
1458 switch (fc) {
1459 case 0:
1460 {
1461 uint32_t nv = cpu_ldl_data_ra(env, pl, ra);
1462 uint32_t cv = env->regs[r3];
1463 uint32_t ov;
1464
1465 if (parallel_cpus) {
1466 #ifdef CONFIG_USER_ONLY
1467 uint32_t *haddr = g2h(a1);
1468 ov = atomic_cmpxchg__nocheck(haddr, cv, nv);
1469 #else
1470 TCGMemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx);
1471 ov = helper_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra);
1472 #endif
1473 } else {
1474 ov = cpu_ldl_data_ra(env, a1, ra);
1475 cpu_stl_data_ra(env, a1, (ov == cv ? nv : ov), ra);
1476 }
1477 cc = (ov != cv);
1478 env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov);
1479 }
1480 break;
1481
1482 case 1:
1483 {
1484 uint64_t nv = cpu_ldq_data_ra(env, pl, ra);
1485 uint64_t cv = env->regs[r3];
1486 uint64_t ov;
1487
1488 if (parallel_cpus) {
1489 #ifdef CONFIG_ATOMIC64
1490 # ifdef CONFIG_USER_ONLY
1491 uint64_t *haddr = g2h(a1);
1492 ov = atomic_cmpxchg__nocheck(haddr, cv, nv);
1493 # else
1494 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN, mem_idx);
1495 ov = helper_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra);
1496 # endif
1497 #else
1498 /* Note that we asserted !parallel_cpus above. */
1499 g_assert_not_reached();
1500 #endif
1501 } else {
1502 ov = cpu_ldq_data_ra(env, a1, ra);
1503 cpu_stq_data_ra(env, a1, (ov == cv ? nv : ov), ra);
1504 }
1505 cc = (ov != cv);
1506 env->regs[r3] = ov;
1507 }
1508 break;
1509
1510 case 2:
1511 {
1512 uint64_t nvh = cpu_ldq_data_ra(env, pl, ra);
1513 uint64_t nvl = cpu_ldq_data_ra(env, pl + 8, ra);
1514 Int128 nv = int128_make128(nvl, nvh);
1515 Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1516 Int128 ov;
1517
1518 if (parallel_cpus) {
1519 #ifdef CONFIG_ATOMIC128
1520 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
1521 ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
1522 cc = !int128_eq(ov, cv);
1523 #else
1524 /* Note that we asserted !parallel_cpus above. */
1525 g_assert_not_reached();
1526 #endif
1527 } else {
1528 uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra);
1529 uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra);
1530
1531 ov = int128_make128(ol, oh);
1532 cc = !int128_eq(ov, cv);
1533 if (cc) {
1534 nv = ov;
1535 }
1536
1537 cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra);
1538 cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra);
1539 }
1540
1541 env->regs[r3 + 0] = int128_gethi(ov);
1542 env->regs[r3 + 1] = int128_getlo(ov);
1543 }
1544 break;
1545
1546 default:
1547 g_assert_not_reached();
1548 }
1549
1550 /* Store only if the comparison succeeded. Note that above we use a pair
1551 of 64-bit big-endian loads, so for sc < 3 we must extract the value
1552 from the most-significant bits of svh. */
1553 if (cc == 0) {
1554 switch (sc) {
1555 case 0:
1556 cpu_stb_data_ra(env, a2, svh >> 56, ra);
1557 break;
1558 case 1:
1559 cpu_stw_data_ra(env, a2, svh >> 48, ra);
1560 break;
1561 case 2:
1562 cpu_stl_data_ra(env, a2, svh >> 32, ra);
1563 break;
1564 case 3:
1565 cpu_stq_data_ra(env, a2, svh, ra);
1566 break;
1567 case 4:
1568 if (parallel_cpus) {
1569 #ifdef CONFIG_ATOMIC128
1570 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
1571 Int128 sv = int128_make128(svl, svh);
1572 helper_atomic_sto_be_mmu(env, a2, sv, oi, ra);
1573 #else
1574 /* Note that we asserted !parallel_cpus above. */
1575 g_assert_not_reached();
1576 #endif
1577 } else {
1578 cpu_stq_data_ra(env, a2 + 0, svh, ra);
1579 cpu_stq_data_ra(env, a2 + 8, svl, ra);
1580 }
1581 break;
1582 default:
1583 g_assert_not_reached();
1584 }
1585 }
1586
1587 return cc;
1588
1589 spec_exception:
1590 cpu_restore_state(ENV_GET_CPU(env), ra);
1591 program_interrupt(env, PGM_SPECIFICATION, 6);
1592 g_assert_not_reached();
1593 }
1594
1595 #if !defined(CONFIG_USER_ONLY)
1596 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1597 {
1598 uintptr_t ra = GETPC();
1599 S390CPU *cpu = s390_env_get_cpu(env);
1600 bool PERchanged = false;
1601 uint64_t src = a2;
1602 uint32_t i;
1603
1604 for (i = r1;; i = (i + 1) % 16) {
1605 uint64_t val = cpu_ldq_data_ra(env, src, ra);
1606 if (env->cregs[i] != val && i >= 9 && i <= 11) {
1607 PERchanged = true;
1608 }
1609 env->cregs[i] = val;
1610 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
1611 i, src, val);
1612 src += sizeof(uint64_t);
1613
1614 if (i == r3) {
1615 break;
1616 }
1617 }
1618
1619 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
1620 s390_cpu_recompute_watchpoints(CPU(cpu));
1621 }
1622
1623 tlb_flush(CPU(cpu));
1624 }
1625
1626 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1627 {
1628 uintptr_t ra = GETPC();
1629 S390CPU *cpu = s390_env_get_cpu(env);
1630 bool PERchanged = false;
1631 uint64_t src = a2;
1632 uint32_t i;
1633
1634 for (i = r1;; i = (i + 1) % 16) {
1635 uint32_t val = cpu_ldl_data_ra(env, src, ra);
1636 if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) {
1637 PERchanged = true;
1638 }
1639 env->cregs[i] = deposit64(env->cregs[i], 0, 32, val);
1640 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val);
1641 src += sizeof(uint32_t);
1642
1643 if (i == r3) {
1644 break;
1645 }
1646 }
1647
1648 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
1649 s390_cpu_recompute_watchpoints(CPU(cpu));
1650 }
1651
1652 tlb_flush(CPU(cpu));
1653 }
1654
1655 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1656 {
1657 uintptr_t ra = GETPC();
1658 uint64_t dest = a2;
1659 uint32_t i;
1660
1661 for (i = r1;; i = (i + 1) % 16) {
1662 cpu_stq_data_ra(env, dest, env->cregs[i], ra);
1663 dest += sizeof(uint64_t);
1664
1665 if (i == r3) {
1666 break;
1667 }
1668 }
1669 }
1670
1671 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1672 {
1673 uintptr_t ra = GETPC();
1674 uint64_t dest = a2;
1675 uint32_t i;
1676
1677 for (i = r1;; i = (i + 1) % 16) {
1678 cpu_stl_data_ra(env, dest, env->cregs[i], ra);
1679 dest += sizeof(uint32_t);
1680
1681 if (i == r3) {
1682 break;
1683 }
1684 }
1685 }
1686
1687 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr)
1688 {
1689 uintptr_t ra = GETPC();
1690 CPUState *cs = CPU(s390_env_get_cpu(env));
1691 int i;
1692
1693 real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK;
1694
1695 /* Check low-address protection */
1696 if ((env->cregs[0] & CR0_LOWPROT) && real_addr < 0x2000) {
1697 cpu_restore_state(cs, ra);
1698 program_interrupt(env, PGM_PROTECTION, 4);
1699 return 1;
1700 }
1701
1702 for (i = 0; i < TARGET_PAGE_SIZE; i += 8) {
1703 cpu_stq_real_ra(env, real_addr + i, 0, ra);
1704 }
1705
1706 return 0;
1707 }
1708
1709 uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2)
1710 {
1711 /* XXX implement */
1712 return 0;
1713 }
1714
1715 /* insert storage key extended */
1716 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
1717 {
1718 static S390SKeysState *ss;
1719 static S390SKeysClass *skeyclass;
1720 uint64_t addr = wrap_address(env, r2);
1721 uint8_t key;
1722
1723 if (addr > ram_size) {
1724 return 0;
1725 }
1726
1727 if (unlikely(!ss)) {
1728 ss = s390_get_skeys_device();
1729 skeyclass = S390_SKEYS_GET_CLASS(ss);
1730 }
1731
1732 if (skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key)) {
1733 return 0;
1734 }
1735 return key;
1736 }
1737
1738 /* set storage key extended */
1739 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
1740 {
1741 static S390SKeysState *ss;
1742 static S390SKeysClass *skeyclass;
1743 uint64_t addr = wrap_address(env, r2);
1744 uint8_t key;
1745
1746 if (addr > ram_size) {
1747 return;
1748 }
1749
1750 if (unlikely(!ss)) {
1751 ss = s390_get_skeys_device();
1752 skeyclass = S390_SKEYS_GET_CLASS(ss);
1753 }
1754
1755 key = (uint8_t) r1;
1756 skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
1757 }
1758
1759 /* reset reference bit extended */
1760 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
1761 {
1762 static S390SKeysState *ss;
1763 static S390SKeysClass *skeyclass;
1764 uint8_t re, key;
1765
1766 if (r2 > ram_size) {
1767 return 0;
1768 }
1769
1770 if (unlikely(!ss)) {
1771 ss = s390_get_skeys_device();
1772 skeyclass = S390_SKEYS_GET_CLASS(ss);
1773 }
1774
1775 if (skeyclass->get_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
1776 return 0;
1777 }
1778
1779 re = key & (SK_R | SK_C);
1780 key &= ~SK_R;
1781
1782 if (skeyclass->set_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
1783 return 0;
1784 }
1785
1786 /*
1787 * cc
1788 *
1789 * 0 Reference bit zero; change bit zero
1790 * 1 Reference bit zero; change bit one
1791 * 2 Reference bit one; change bit zero
1792 * 3 Reference bit one; change bit one
1793 */
1794
1795 return re >> 1;
1796 }
1797
1798 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
1799 {
1800 uintptr_t ra = GETPC();
1801 int cc = 0, i;
1802
1803 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
1804 __func__, l, a1, a2);
1805
1806 if (l > 256) {
1807 /* max 256 */
1808 l = 256;
1809 cc = 3;
1810 }
1811
1812 /* XXX replace w/ memcpy */
1813 for (i = 0; i < l; i++) {
1814 uint8_t x = cpu_ldub_primary_ra(env, a2 + i, ra);
1815 cpu_stb_secondary_ra(env, a1 + i, x, ra);
1816 }
1817
1818 return cc;
1819 }
1820
1821 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
1822 {
1823 uintptr_t ra = GETPC();
1824 int cc = 0, i;
1825
1826 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
1827 __func__, l, a1, a2);
1828
1829 if (l > 256) {
1830 /* max 256 */
1831 l = 256;
1832 cc = 3;
1833 }
1834
1835 /* XXX replace w/ memcpy */
1836 for (i = 0; i < l; i++) {
1837 uint8_t x = cpu_ldub_secondary_ra(env, a2 + i, ra);
1838 cpu_stb_primary_ra(env, a1 + i, x, ra);
1839 }
1840
1841 return cc;
1842 }
1843
1844 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4)
1845 {
1846 CPUState *cs = CPU(s390_env_get_cpu(env));
1847 const uintptr_t ra = GETPC();
1848 uint64_t table, entry, raddr;
1849 uint16_t entries, i, index = 0;
1850
1851 if (r2 & 0xff000) {
1852 cpu_restore_state(cs, ra);
1853 program_interrupt(env, PGM_SPECIFICATION, 4);
1854 }
1855
1856 if (!(r2 & 0x800)) {
1857 /* invalidation-and-clearing operation */
1858 table = r1 & _ASCE_ORIGIN;
1859 entries = (r2 & 0x7ff) + 1;
1860
1861 switch (r1 & _ASCE_TYPE_MASK) {
1862 case _ASCE_TYPE_REGION1:
1863 index = (r2 >> 53) & 0x7ff;
1864 break;
1865 case _ASCE_TYPE_REGION2:
1866 index = (r2 >> 42) & 0x7ff;
1867 break;
1868 case _ASCE_TYPE_REGION3:
1869 index = (r2 >> 31) & 0x7ff;
1870 break;
1871 case _ASCE_TYPE_SEGMENT:
1872 index = (r2 >> 20) & 0x7ff;
1873 break;
1874 }
1875 for (i = 0; i < entries; i++) {
1876 /* addresses are not wrapped in 24/31bit mode but table index is */
1877 raddr = table + ((index + i) & 0x7ff) * sizeof(entry);
1878 entry = cpu_ldq_real_ra(env, raddr, ra);
1879 if (!(entry & _REGION_ENTRY_INV)) {
1880 /* we are allowed to not store if already invalid */
1881 entry |= _REGION_ENTRY_INV;
1882 cpu_stq_real_ra(env, raddr, entry, ra);
1883 }
1884 }
1885 }
1886
1887 /* We simply flush the complete tlb, therefore we can ignore r3. */
1888 if (m4 & 1) {
1889 tlb_flush(cs);
1890 } else {
1891 tlb_flush_all_cpus_synced(cs);
1892 }
1893 }
1894
1895 /* invalidate pte */
1896 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr,
1897 uint32_t m4)
1898 {
1899 CPUState *cs = CPU(s390_env_get_cpu(env));
1900 const uintptr_t ra = GETPC();
1901 uint64_t page = vaddr & TARGET_PAGE_MASK;
1902 uint64_t pte_addr, pte;
1903
1904 /* Compute the page table entry address */
1905 pte_addr = (pto & _SEGMENT_ENTRY_ORIGIN);
1906 pte_addr += (vaddr & VADDR_PX) >> 9;
1907
1908 /* Mark the page table entry as invalid */
1909 pte = cpu_ldq_real_ra(env, pte_addr, ra);
1910 pte |= _PAGE_INVALID;
1911 cpu_stq_real_ra(env, pte_addr, pte, ra);
1912
1913 /* XXX we exploit the fact that Linux passes the exact virtual
1914 address here - it's not obliged to! */
1915 if (m4 & 1) {
1916 if (vaddr & ~VADDR_PX) {
1917 tlb_flush_page(cs, page);
1918 /* XXX 31-bit hack */
1919 tlb_flush_page(cs, page ^ 0x80000000);
1920 } else {
1921 /* looks like we don't have a valid virtual address */
1922 tlb_flush(cs);
1923 }
1924 } else {
1925 if (vaddr & ~VADDR_PX) {
1926 tlb_flush_page_all_cpus_synced(cs, page);
1927 /* XXX 31-bit hack */
1928 tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000);
1929 } else {
1930 /* looks like we don't have a valid virtual address */
1931 tlb_flush_all_cpus_synced(cs);
1932 }
1933 }
1934 }
1935
1936 /* flush local tlb */
1937 void HELPER(ptlb)(CPUS390XState *env)
1938 {
1939 S390CPU *cpu = s390_env_get_cpu(env);
1940
1941 tlb_flush(CPU(cpu));
1942 }
1943
1944 /* flush global tlb */
1945 void HELPER(purge)(CPUS390XState *env)
1946 {
1947 S390CPU *cpu = s390_env_get_cpu(env);
1948
1949 tlb_flush_all_cpus_synced(CPU(cpu));
1950 }
1951
1952 /* load using real address */
1953 uint64_t HELPER(lura)(CPUS390XState *env, uint64_t addr)
1954 {
1955 return cpu_ldl_real_ra(env, wrap_address(env, addr), GETPC());
1956 }
1957
1958 uint64_t HELPER(lurag)(CPUS390XState *env, uint64_t addr)
1959 {
1960 return cpu_ldq_real_ra(env, wrap_address(env, addr), GETPC());
1961 }
1962
1963 /* store using real address */
1964 void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint64_t v1)
1965 {
1966 cpu_stl_real_ra(env, wrap_address(env, addr), (uint32_t)v1, GETPC());
1967
1968 if ((env->psw.mask & PSW_MASK_PER) &&
1969 (env->cregs[9] & PER_CR9_EVENT_STORE) &&
1970 (env->cregs[9] & PER_CR9_EVENT_STORE_REAL)) {
1971 /* PSW is saved just before calling the helper. */
1972 env->per_address = env->psw.addr;
1973 env->per_perc_atmid = PER_CODE_EVENT_STORE_REAL | get_per_atmid(env);
1974 }
1975 }
1976
1977 void HELPER(sturg)(CPUS390XState *env, uint64_t addr, uint64_t v1)
1978 {
1979 cpu_stq_real_ra(env, wrap_address(env, addr), v1, GETPC());
1980
1981 if ((env->psw.mask & PSW_MASK_PER) &&
1982 (env->cregs[9] & PER_CR9_EVENT_STORE) &&
1983 (env->cregs[9] & PER_CR9_EVENT_STORE_REAL)) {
1984 /* PSW is saved just before calling the helper. */
1985 env->per_address = env->psw.addr;
1986 env->per_perc_atmid = PER_CODE_EVENT_STORE_REAL | get_per_atmid(env);
1987 }
1988 }
1989
1990 /* load real address */
1991 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr)
1992 {
1993 CPUState *cs = CPU(s390_env_get_cpu(env));
1994 uint32_t cc = 0;
1995 uint64_t asc = env->psw.mask & PSW_MASK_ASC;
1996 uint64_t ret;
1997 int old_exc, flags;
1998
1999 /* XXX incomplete - has more corner cases */
2000 if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
2001 cpu_restore_state(cs, GETPC());
2002 program_interrupt(env, PGM_SPECIAL_OP, 2);
2003 }
2004
2005 old_exc = cs->exception_index;
2006 if (mmu_translate(env, addr, 0, asc, &ret, &flags, true)) {
2007 cc = 3;
2008 }
2009 if (cs->exception_index == EXCP_PGM) {
2010 ret = env->int_pgm_code | 0x80000000;
2011 } else {
2012 ret |= addr & ~TARGET_PAGE_MASK;
2013 }
2014 cs->exception_index = old_exc;
2015
2016 env->cc_op = cc;
2017 return ret;
2018 }
2019 #endif
2020
2021 /* load pair from quadword */
2022 uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr)
2023 {
2024 uintptr_t ra = GETPC();
2025 uint64_t hi, lo;
2026
2027 if (parallel_cpus) {
2028 #ifndef CONFIG_ATOMIC128
2029 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
2030 #else
2031 int mem_idx = cpu_mmu_index(env, false);
2032 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
2033 Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
2034 hi = int128_gethi(v);
2035 lo = int128_getlo(v);
2036 #endif
2037 } else {
2038 check_alignment(env, addr, 16, ra);
2039
2040 hi = cpu_ldq_data_ra(env, addr + 0, ra);
2041 lo = cpu_ldq_data_ra(env, addr + 8, ra);
2042 }
2043
2044 env->retxl = lo;
2045 return hi;
2046 }
2047
2048 /* store pair to quadword */
2049 void HELPER(stpq)(CPUS390XState *env, uint64_t addr,
2050 uint64_t low, uint64_t high)
2051 {
2052 uintptr_t ra = GETPC();
2053
2054 if (parallel_cpus) {
2055 #ifndef CONFIG_ATOMIC128
2056 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
2057 #else
2058 int mem_idx = cpu_mmu_index(env, false);
2059 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
2060
2061 Int128 v = int128_make128(low, high);
2062 helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
2063 #endif
2064 } else {
2065 check_alignment(env, addr, 16, ra);
2066
2067 cpu_stq_data_ra(env, addr + 0, high, ra);
2068 cpu_stq_data_ra(env, addr + 8, low, ra);
2069 }
2070 }
2071
2072 /* Execute instruction. This instruction executes an insn modified with
2073 the contents of r1. It does not change the executed instruction in memory;
2074 it does not change the program counter.
2075
2076 Perform this by recording the modified instruction in env->ex_value.
2077 This will be noticed by cpu_get_tb_cpu_state and thus tb translation.
2078 */
2079 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr)
2080 {
2081 uint64_t insn = cpu_lduw_code(env, addr);
2082 uint8_t opc = insn >> 8;
2083
2084 /* Or in the contents of R1[56:63]. */
2085 insn |= r1 & 0xff;
2086
2087 /* Load the rest of the instruction. */
2088 insn <<= 48;
2089 switch (get_ilen(opc)) {
2090 case 2:
2091 break;
2092 case 4:
2093 insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32;
2094 break;
2095 case 6:
2096 insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16;
2097 break;
2098 default:
2099 g_assert_not_reached();
2100 }
2101
2102 /* The very most common cases can be sped up by avoiding a new TB. */
2103 if ((opc & 0xf0) == 0xd0) {
2104 typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t,
2105 uint64_t, uintptr_t);
2106 static const dx_helper dx[16] = {
2107 [0x2] = do_helper_mvc,
2108 [0x4] = do_helper_nc,
2109 [0x5] = do_helper_clc,
2110 [0x6] = do_helper_oc,
2111 [0x7] = do_helper_xc,
2112 [0xc] = do_helper_tr,
2113 };
2114 dx_helper helper = dx[opc & 0xf];
2115
2116 if (helper) {
2117 uint32_t l = extract64(insn, 48, 8);
2118 uint32_t b1 = extract64(insn, 44, 4);
2119 uint32_t d1 = extract64(insn, 32, 12);
2120 uint32_t b2 = extract64(insn, 28, 4);
2121 uint32_t d2 = extract64(insn, 16, 12);
2122 uint64_t a1 = wrap_address(env, env->regs[b1] + d1);
2123 uint64_t a2 = wrap_address(env, env->regs[b2] + d2);
2124
2125 env->cc_op = helper(env, l, a1, a2, 0);
2126 env->psw.addr += ilen;
2127 return;
2128 }
2129 } else if (opc == 0x0a) {
2130 env->int_svc_code = extract64(insn, 48, 8);
2131 env->int_svc_ilen = ilen;
2132 helper_exception(env, EXCP_SVC);
2133 g_assert_not_reached();
2134 }
2135
2136 /* Record the insn we want to execute as well as the ilen to use
2137 during the execution of the target insn. This will also ensure
2138 that ex_value is non-zero, which flags that we are in a state
2139 that requires such execution. */
2140 env->ex_value = insn | ilen;
2141 }
2142
2143 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src,
2144 uint64_t len)
2145 {
2146 const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY;
2147 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2148 const uint64_t r0 = env->regs[0];
2149 const uintptr_t ra = GETPC();
2150 CPUState *cs = CPU(s390_env_get_cpu(env));
2151 uint8_t dest_key, dest_as, dest_k, dest_a;
2152 uint8_t src_key, src_as, src_k, src_a;
2153 uint64_t val;
2154 int cc = 0;
2155
2156 HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n",
2157 __func__, dest, src, len);
2158
2159 if (!(env->psw.mask & PSW_MASK_DAT)) {
2160 cpu_restore_state(cs, ra);
2161 program_interrupt(env, PGM_SPECIAL_OP, 6);
2162 }
2163
2164 /* OAC (operand access control) for the first operand -> dest */
2165 val = (r0 & 0xffff0000ULL) >> 16;
2166 dest_key = (val >> 12) & 0xf;
2167 dest_as = (val >> 6) & 0x3;
2168 dest_k = (val >> 1) & 0x1;
2169 dest_a = val & 0x1;
2170
2171 /* OAC (operand access control) for the second operand -> src */
2172 val = (r0 & 0x0000ffffULL);
2173 src_key = (val >> 12) & 0xf;
2174 src_as = (val >> 6) & 0x3;
2175 src_k = (val >> 1) & 0x1;
2176 src_a = val & 0x1;
2177
2178 if (!dest_k) {
2179 dest_key = psw_key;
2180 }
2181 if (!src_k) {
2182 src_key = psw_key;
2183 }
2184 if (!dest_a) {
2185 dest_as = psw_as;
2186 }
2187 if (!src_a) {
2188 src_as = psw_as;
2189 }
2190
2191 if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) {
2192 cpu_restore_state(cs, ra);
2193 program_interrupt(env, PGM_SPECIAL_OP, 6);
2194 }
2195 if (!(env->cregs[0] & CR0_SECONDARY) &&
2196 (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) {
2197 cpu_restore_state(cs, ra);
2198 program_interrupt(env, PGM_SPECIAL_OP, 6);
2199 }
2200 if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) {
2201 cpu_restore_state(cs, ra);
2202 program_interrupt(env, PGM_PRIVILEGED, 6);
2203 }
2204
2205 len = wrap_length(env, len);
2206 if (len > 4096) {
2207 cc = 3;
2208 len = 4096;
2209 }
2210
2211 /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */
2212 if (src_as == AS_ACCREG || dest_as == AS_ACCREG ||
2213 (env->psw.mask & PSW_MASK_PSTATE)) {
2214 qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n",
2215 __func__);
2216 cpu_restore_state(cs, ra);
2217 program_interrupt(env, PGM_ADDRESSING, 6);
2218 }
2219
2220 /* FIXME: a) LAP
2221 * b) Access using correct keys
2222 * c) AR-mode
2223 */
2224 #ifdef CONFIG_USER_ONLY
2225 /* psw keys are never valid in user mode, we will never reach this */
2226 g_assert_not_reached();
2227 #else
2228 fast_memmove_as(env, dest, src, len, dest_as, src_as, ra);
2229 #endif
2230
2231 return cc;
2232 }
2233
2234 /* Decode a Unicode character. A return value < 0 indicates success, storing
2235 the UTF-32 result into OCHAR and the input length into OLEN. A return
2236 value >= 0 indicates failure, and the CC value to be returned. */
2237 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2238 uint64_t ilen, bool enh_check, uintptr_t ra,
2239 uint32_t *ochar, uint32_t *olen);
2240
2241 /* Encode a Unicode character. A return value < 0 indicates success, storing
2242 the bytes into ADDR and the output length into OLEN. A return value >= 0
2243 indicates failure, and the CC value to be returned. */
2244 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2245 uint64_t ilen, uintptr_t ra, uint32_t c,
2246 uint32_t *olen);
2247
2248 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2249 bool enh_check, uintptr_t ra,
2250 uint32_t *ochar, uint32_t *olen)
2251 {
2252 uint8_t s0, s1, s2, s3;
2253 uint32_t c, l;
2254
2255 if (ilen < 1) {
2256 return 0;
2257 }
2258 s0 = cpu_ldub_data_ra(env, addr, ra);
2259 if (s0 <= 0x7f) {
2260 /* one byte character */
2261 l = 1;
2262 c = s0;
2263 } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) {
2264 /* invalid character */
2265 return 2;
2266 } else if (s0 <= 0xdf) {
2267 /* two byte character */
2268 l = 2;
2269 if (ilen < 2) {
2270 return 0;
2271 }
2272 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2273 c = s0 & 0x1f;
2274 c = (c << 6) | (s1 & 0x3f);
2275 if (enh_check && (s1 & 0xc0) != 0x80) {
2276 return 2;
2277 }
2278 } else if (s0 <= 0xef) {
2279 /* three byte character */
2280 l = 3;
2281 if (ilen < 3) {
2282 return 0;
2283 }
2284 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2285 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2286 c = s0 & 0x0f;
2287 c = (c << 6) | (s1 & 0x3f);
2288 c = (c << 6) | (s2 & 0x3f);
2289 /* Fold the byte-by-byte range descriptions in the PoO into
2290 tests against the complete value. It disallows encodings
2291 that could be smaller, and the UTF-16 surrogates. */
2292 if (enh_check
2293 && ((s1 & 0xc0) != 0x80
2294 || (s2 & 0xc0) != 0x80
2295 || c < 0x1000
2296 || (c >= 0xd800 && c <= 0xdfff))) {
2297 return 2;
2298 }
2299 } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) {
2300 /* four byte character */
2301 l = 4;
2302 if (ilen < 4) {
2303 return 0;
2304 }
2305 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2306 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2307 s3 = cpu_ldub_data_ra(env, addr + 3, ra);
2308 c = s0 & 0x07;
2309 c = (c << 6) | (s1 & 0x3f);
2310 c = (c << 6) | (s2 & 0x3f);
2311 c = (c << 6) | (s3 & 0x3f);
2312 /* See above. */
2313 if (enh_check
2314 && ((s1 & 0xc0) != 0x80
2315 || (s2 & 0xc0) != 0x80
2316 || (s3 & 0xc0) != 0x80
2317 || c < 0x010000
2318 || c > 0x10ffff)) {
2319 return 2;
2320 }
2321 } else {
2322 /* invalid character */
2323 return 2;
2324 }
2325
2326 *ochar = c;
2327 *olen = l;
2328 return -1;
2329 }
2330
2331 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2332 bool enh_check, uintptr_t ra,
2333 uint32_t *ochar, uint32_t *olen)
2334 {
2335 uint16_t s0, s1;
2336 uint32_t c, l;
2337
2338 if (ilen < 2) {
2339 return 0;
2340 }
2341 s0 = cpu_lduw_data_ra(env, addr, ra);
2342 if ((s0 & 0xfc00) != 0xd800) {
2343 /* one word character */
2344 l = 2;
2345 c = s0;
2346 } else {
2347 /* two word character */
2348 l = 4;
2349 if (ilen < 4) {
2350 return 0;
2351 }
2352 s1 = cpu_lduw_data_ra(env, addr + 2, ra);
2353 c = extract32(s0, 6, 4) + 1;
2354 c = (c << 6) | (s0 & 0x3f);
2355 c = (c << 10) | (s1 & 0x3ff);
2356 if (enh_check && (s1 & 0xfc00) != 0xdc00) {
2357 /* invalid surrogate character */
2358 return 2;
2359 }
2360 }
2361
2362 *ochar = c;
2363 *olen = l;
2364 return -1;
2365 }
2366
2367 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2368 bool enh_check, uintptr_t ra,
2369 uint32_t *ochar, uint32_t *olen)
2370 {
2371 uint32_t c;
2372
2373 if (ilen < 4) {
2374 return 0;
2375 }
2376 c = cpu_ldl_data_ra(env, addr, ra);
2377 if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) {
2378 /* invalid unicode character */
2379 return 2;
2380 }
2381
2382 *ochar = c;
2383 *olen = 4;
2384 return -1;
2385 }
2386
2387 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2388 uintptr_t ra, uint32_t c, uint32_t *olen)
2389 {
2390 uint8_t d[4];
2391 uint32_t l, i;
2392
2393 if (c <= 0x7f) {
2394 /* one byte character */
2395 l = 1;
2396 d[0] = c;
2397 } else if (c <= 0x7ff) {
2398 /* two byte character */
2399 l = 2;
2400 d[1] = 0x80 | extract32(c, 0, 6);
2401 d[0] = 0xc0 | extract32(c, 6, 5);
2402 } else if (c <= 0xffff) {
2403 /* three byte character */
2404 l = 3;
2405 d[2] = 0x80 | extract32(c, 0, 6);
2406 d[1] = 0x80 | extract32(c, 6, 6);
2407 d[0] = 0xe0 | extract32(c, 12, 4);
2408 } else {
2409 /* four byte character */
2410 l = 4;
2411 d[3] = 0x80 | extract32(c, 0, 6);
2412 d[2] = 0x80 | extract32(c, 6, 6);
2413 d[1] = 0x80 | extract32(c, 12, 6);
2414 d[0] = 0xf0 | extract32(c, 18, 3);
2415 }
2416
2417 if (ilen < l) {
2418 return 1;
2419 }
2420 for (i = 0; i < l; ++i) {
2421 cpu_stb_data_ra(env, addr + i, d[i], ra);
2422 }
2423
2424 *olen = l;
2425 return -1;
2426 }
2427
2428 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2429 uintptr_t ra, uint32_t c, uint32_t *olen)
2430 {
2431 uint16_t d0, d1;
2432
2433 if (c <= 0xffff) {
2434 /* one word character */
2435 if (ilen < 2) {
2436 return 1;
2437 }
2438 cpu_stw_data_ra(env, addr, c, ra);
2439 *olen = 2;
2440 } else {
2441 /* two word character */
2442 if (ilen < 4) {
2443 return 1;
2444 }
2445 d1 = 0xdc00 | extract32(c, 0, 10);
2446 d0 = 0xd800 | extract32(c, 10, 6);
2447 d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1);
2448 cpu_stw_data_ra(env, addr + 0, d0, ra);
2449 cpu_stw_data_ra(env, addr + 2, d1, ra);
2450 *olen = 4;
2451 }
2452
2453 return -1;
2454 }
2455
2456 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2457 uintptr_t ra, uint32_t c, uint32_t *olen)
2458 {
2459 if (ilen < 4) {
2460 return 1;
2461 }
2462 cpu_stl_data_ra(env, addr, c, ra);
2463 *olen = 4;
2464 return -1;
2465 }
2466
2467 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1,
2468 uint32_t r2, uint32_t m3, uintptr_t ra,
2469 decode_unicode_fn decode,
2470 encode_unicode_fn encode)
2471 {
2472 uint64_t dst = get_address(env, r1);
2473 uint64_t dlen = get_length(env, r1 + 1);
2474 uint64_t src = get_address(env, r2);
2475 uint64_t slen = get_length(env, r2 + 1);
2476 bool enh_check = m3 & 1;
2477 int cc, i;
2478
2479 /* Lest we fail to service interrupts in a timely manner, limit the
2480 amount of work we're willing to do. For now, let's cap at 256. */
2481 for (i = 0; i < 256; ++i) {
2482 uint32_t c, ilen, olen;
2483
2484 cc = decode(env, src, slen, enh_check, ra, &c, &ilen);
2485 if (unlikely(cc >= 0)) {
2486 break;
2487 }
2488 cc = encode(env, dst, dlen, ra, c, &olen);
2489 if (unlikely(cc >= 0)) {
2490 break;
2491 }
2492
2493 src += ilen;
2494 slen -= ilen;
2495 dst += olen;
2496 dlen -= olen;
2497 cc = 3;
2498 }
2499
2500 set_address(env, r1, dst);
2501 set_length(env, r1 + 1, dlen);
2502 set_address(env, r2, src);
2503 set_length(env, r2 + 1, slen);
2504
2505 return cc;
2506 }
2507
2508 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2509 {
2510 return convert_unicode(env, r1, r2, m3, GETPC(),
2511 decode_utf8, encode_utf16);
2512 }
2513
2514 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2515 {
2516 return convert_unicode(env, r1, r2, m3, GETPC(),
2517 decode_utf8, encode_utf32);
2518 }
2519
2520 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2521 {
2522 return convert_unicode(env, r1, r2, m3, GETPC(),
2523 decode_utf16, encode_utf8);
2524 }
2525
2526 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2527 {
2528 return convert_unicode(env, r1, r2, m3, GETPC(),
2529 decode_utf16, encode_utf32);
2530 }
2531
2532 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2533 {
2534 return convert_unicode(env, r1, r2, m3, GETPC(),
2535 decode_utf32, encode_utf8);
2536 }
2537
2538 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2539 {
2540 return convert_unicode(env, r1, r2, m3, GETPC(),
2541 decode_utf32, encode_utf16);
2542 }
AR7 emulation for QEMU