2 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU Lesser General Public License as published by
6 * the Free Software Foundation; either version 2.1 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 #ifndef __GRU_INSTRUCTIONS_H__
20 #define __GRU_INSTRUCTIONS_H__
22 extern int gru_check_status_proc(void *cb
);
23 extern int gru_wait_proc(void *cb
);
24 extern void gru_wait_abort_proc(void *cb
);
29 * Architecture dependent functions
32 #if defined(CONFIG_IA64)
33 #include <linux/compiler.h>
34 #include <asm/intrinsics.h>
35 #define __flush_cache(p) ia64_fc((unsigned long)p)
36 /* Use volatile on IA64 to ensure ordering via st4.rel */
37 #define gru_ordered_store_int(p, v) \
40 *((volatile int *)(p)) = v; /* force st.rel */ \
42 #elif defined(CONFIG_X86_64)
43 #define __flush_cache(p) clflush(p)
44 #define gru_ordered_store_int(p, v) \
50 #error "Unsupported architecture"
54 * Control block status and exception codes
57 #define CBS_EXCEPTION 1
61 /* CB substatus bitmasks */
62 #define CBSS_MSG_QUEUE_MASK 7
63 #define CBSS_IMPLICIT_ABORT_ACTIVE_MASK 8
65 /* CB substatus message queue values (low 3 bits of substatus) */
66 #define CBSS_NO_ERROR 0
67 #define CBSS_LB_OVERFLOWED 1
68 #define CBSS_QLIMIT_REACHED 2
69 #define CBSS_PAGE_OVERFLOW 3
70 #define CBSS_AMO_NACKED 4
71 #define CBSS_PUT_NACKED 5
74 * Structure used to fetch exception detail for CBs that terminate with
77 struct control_block_extended_exc_detail
{
91 * Generic instruction format.
92 * This definition has precise bit field definitions.
94 struct gru_instruction_bits
{
97 unsigned char ima
: 3; /* CB_DelRep, unmapped mode */
98 unsigned char reserved0
: 4;
99 unsigned int xtype
: 3;
100 unsigned int iaa0
: 2;
101 unsigned int iaa1
: 2;
102 unsigned char reserved1
: 1;
103 unsigned char opc
: 8; /* opcode */
104 unsigned char exopc
: 8; /* extended opcode */
106 unsigned int idef2
: 22; /* TRi0 */
107 unsigned char reserved2
: 2;
108 unsigned char istatus
: 2;
109 unsigned char isubstatus
:4;
110 unsigned char reserved3
: 2;
112 unsigned long idef4
; /* 42 bits: TRi1, BufSize */
114 unsigned long idef1
; /* BAddr0 */
115 unsigned long idef5
; /* Nelem */
116 unsigned long idef6
; /* Stride, Operand1 */
117 unsigned long idef3
; /* BAddr1, Value, Operand2 */
118 unsigned long reserved4
;
120 unsigned long avalue
; /* AValue */
124 * Generic instruction with friendlier names. This format is used
125 * for inline instructions.
127 struct gru_instruction
{
129 unsigned int op32
; /* icmd,xtype,iaa0,ima,opc */
131 unsigned long tri1_bufsize
; /* DW 1 */
132 unsigned long baddr0
; /* DW 2 */
133 unsigned long nelem
; /* DW 3 */
134 unsigned long op1_stride
; /* DW 4 */
135 unsigned long op2_value_baddr1
; /* DW 5 */
136 unsigned long reserved0
; /* DW 6 */
137 unsigned long avalue
; /* DW 7 */
140 /* Some shifts and masks for the low 32 bits of a GRU command */
141 #define GRU_CB_ICMD_SHFT 0
142 #define GRU_CB_ICMD_MASK 0x1
143 #define GRU_CB_XTYPE_SHFT 8
144 #define GRU_CB_XTYPE_MASK 0x7
145 #define GRU_CB_IAA0_SHFT 11
146 #define GRU_CB_IAA0_MASK 0x3
147 #define GRU_CB_IAA1_SHFT 13
148 #define GRU_CB_IAA1_MASK 0x3
149 #define GRU_CB_IMA_SHFT 1
150 #define GRU_CB_IMA_MASK 0x3
151 #define GRU_CB_OPC_SHFT 16
152 #define GRU_CB_OPC_MASK 0xff
153 #define GRU_CB_EXOPC_SHFT 24
154 #define GRU_CB_EXOPC_MASK 0xff
156 /* GRU instruction opcodes (opc field) */
158 #define OP_BCOPY 0x01
159 #define OP_VLOAD 0x02
160 #define OP_IVLOAD 0x03
161 #define OP_VSTORE 0x04
162 #define OP_IVSTORE 0x05
164 #define OP_IVSET 0x07
166 #define OP_GAMXR 0x09
167 #define OP_GAMIR 0x0a
168 #define OP_GAMIRR 0x0b
169 #define OP_GAMER 0x0c
170 #define OP_GAMERR 0x0d
171 #define OP_BSTORE 0x0e
172 #define OP_VFLUSH 0x0f
175 /* Extended opcodes values (exopc field) */
177 /* GAMIR - AMOs with implicit operands */
178 #define EOP_IR_FETCH 0x01 /* Plain fetch of memory */
179 #define EOP_IR_CLR 0x02 /* Fetch and clear */
180 #define EOP_IR_INC 0x05 /* Fetch and increment */
181 #define EOP_IR_DEC 0x07 /* Fetch and decrement */
182 #define EOP_IR_QCHK1 0x0d /* Queue check, 64 byte msg */
183 #define EOP_IR_QCHK2 0x0e /* Queue check, 128 byte msg */
185 /* GAMIRR - Registered AMOs with implicit operands */
186 #define EOP_IRR_FETCH 0x01 /* Registered fetch of memory */
187 #define EOP_IRR_CLR 0x02 /* Registered fetch and clear */
188 #define EOP_IRR_INC 0x05 /* Registered fetch and increment */
189 #define EOP_IRR_DEC 0x07 /* Registered fetch and decrement */
190 #define EOP_IRR_DECZ 0x0f /* Registered fetch and decrement, update on zero*/
192 /* GAMER - AMOs with explicit operands */
193 #define EOP_ER_SWAP 0x00 /* Exchange argument and memory */
194 #define EOP_ER_OR 0x01 /* Logical OR with memory */
195 #define EOP_ER_AND 0x02 /* Logical AND with memory */
196 #define EOP_ER_XOR 0x03 /* Logical XOR with memory */
197 #define EOP_ER_ADD 0x04 /* Add value to memory */
198 #define EOP_ER_CSWAP 0x08 /* Compare with operand2, write operand1 if match*/
199 #define EOP_ER_CADD 0x0c /* Queue check, operand1*64 byte msg */
201 /* GAMERR - Registered AMOs with explicit operands */
202 #define EOP_ERR_SWAP 0x00 /* Exchange argument and memory */
203 #define EOP_ERR_OR 0x01 /* Logical OR with memory */
204 #define EOP_ERR_AND 0x02 /* Logical AND with memory */
205 #define EOP_ERR_XOR 0x03 /* Logical XOR with memory */
206 #define EOP_ERR_ADD 0x04 /* Add value to memory */
207 #define EOP_ERR_CSWAP 0x08 /* Compare with operand2, write operand1 if match*/
208 #define EOP_ERR_EPOLL 0x09 /* Poll for equality */
209 #define EOP_ERR_NPOLL 0x0a /* Poll for inequality */
211 /* GAMXR - SGI Arithmetic unit */
212 #define EOP_XR_CSWAP 0x0b /* Masked compare exchange */
215 /* Transfer types (xtype field) */
216 #define XTYPE_B 0x0 /* byte */
217 #define XTYPE_S 0x1 /* short (2-byte) */
218 #define XTYPE_W 0x2 /* word (4-byte) */
219 #define XTYPE_DW 0x3 /* doubleword (8-byte) */
220 #define XTYPE_CL 0x6 /* cacheline (64-byte) */
223 /* Instruction access attributes (iaa0, iaa1 fields) */
224 #define IAA_RAM 0x0 /* normal cached RAM access */
225 #define IAA_NCRAM 0x2 /* noncoherent RAM access */
226 #define IAA_MMIO 0x1 /* noncoherent memory-mapped I/O space */
227 #define IAA_REGISTER 0x3 /* memory-mapped registers, etc. */
230 /* Instruction mode attributes (ima field) */
231 #define IMA_MAPPED 0x0 /* Virtual mode */
232 #define IMA_CB_DELAY 0x1 /* hold read responses until status changes */
233 #define IMA_UNMAPPED 0x2 /* bypass the TLBs (OS only) */
234 #define IMA_INTERRUPT 0x4 /* Interrupt when instruction completes */
236 /* CBE ecause bits */
237 #define CBE_CAUSE_RI (1 << 0)
238 #define CBE_CAUSE_INVALID_INSTRUCTION (1 << 1)
239 #define CBE_CAUSE_UNMAPPED_MODE_FORBIDDEN (1 << 2)
240 #define CBE_CAUSE_PE_CHECK_DATA_ERROR (1 << 3)
241 #define CBE_CAUSE_IAA_GAA_MISMATCH (1 << 4)
242 #define CBE_CAUSE_DATA_SEGMENT_LIMIT_EXCEPTION (1 << 5)
243 #define CBE_CAUSE_OS_FATAL_TLB_FAULT (1 << 6)
244 #define CBE_CAUSE_EXECUTION_HW_ERROR (1 << 7)
245 #define CBE_CAUSE_TLBHW_ERROR (1 << 8)
246 #define CBE_CAUSE_RA_REQUEST_TIMEOUT (1 << 9)
247 #define CBE_CAUSE_HA_REQUEST_TIMEOUT (1 << 10)
248 #define CBE_CAUSE_RA_RESPONSE_FATAL (1 << 11)
249 #define CBE_CAUSE_RA_RESPONSE_NON_FATAL (1 << 12)
250 #define CBE_CAUSE_HA_RESPONSE_FATAL (1 << 13)
251 #define CBE_CAUSE_HA_RESPONSE_NON_FATAL (1 << 14)
252 #define CBE_CAUSE_ADDRESS_SPACE_DECODE_ERROR (1 << 15)
253 #define CBE_CAUSE_RESPONSE_DATA_ERROR (1 << 16)
254 #define CBE_CAUSE_PROTOCOL_STATE_DATA_ERROR (1 << 17)
257 * Exceptions are retried for the following cases. If any OTHER bits are set
258 * in ecause, the exception is not retryable.
260 #define EXCEPTION_RETRY_BITS (CBE_CAUSE_RESPONSE_DATA_ERROR | \
261 CBE_CAUSE_RA_REQUEST_TIMEOUT | \
262 CBE_CAUSE_TLBHW_ERROR | \
263 CBE_CAUSE_HA_REQUEST_TIMEOUT)
265 /* Message queue head structure */
275 /* Generate the low word of a GRU instruction */
276 static inline unsigned int
277 __opword(unsigned char opcode
, unsigned char exopc
, unsigned char xtype
,
278 unsigned char iaa0
, unsigned char iaa1
,
281 return (1 << GRU_CB_ICMD_SHFT
) |
282 (iaa0
<< GRU_CB_IAA0_SHFT
) |
283 (iaa1
<< GRU_CB_IAA1_SHFT
) |
284 (ima
<< GRU_CB_IMA_SHFT
) |
285 (xtype
<< GRU_CB_XTYPE_SHFT
) |
286 (opcode
<< GRU_CB_OPC_SHFT
) |
287 (exopc
<< GRU_CB_EXOPC_SHFT
);
291 * Architecture specific intrinsics
293 static inline void gru_flush_cache(void *p
)
299 * Store the lower 32 bits of the command including the "start" bit. Then
300 * start the instruction executing.
302 static inline void gru_start_instruction(struct gru_instruction
*ins
, int op32
)
304 gru_ordered_store_int(ins
, op32
);
305 gru_flush_cache(ins
);
309 /* Convert "hints" to IMA */
310 #define CB_IMA(h) ((h) | IMA_UNMAPPED)
312 /* Convert data segment cache line index into TRI0 / TRI1 value */
313 #define GRU_DINDEX(i) ((i) * GRU_CACHE_LINE_BYTES)
315 /* Inline functions for GRU instructions.
317 * - nelem and stride are in elements
318 * - tri0/tri1 is in bytes for the beginning of the data segment.
320 static inline void gru_vload(void *cb
, unsigned long mem_addr
,
321 unsigned int tri0
, unsigned char xtype
, unsigned long nelem
,
322 unsigned long stride
, unsigned long hints
)
324 struct gru_instruction
*ins
= (struct gru_instruction
*)cb
;
326 ins
->baddr0
= (long)mem_addr
;
329 ins
->op1_stride
= stride
;
330 gru_start_instruction(ins
, __opword(OP_VLOAD
, 0, xtype
, IAA_RAM
, 0,
334 static inline void gru_vstore(void *cb
, unsigned long mem_addr
,
335 unsigned int tri0
, unsigned char xtype
, unsigned long nelem
,
336 unsigned long stride
, unsigned long hints
)
338 struct gru_instruction
*ins
= (void *)cb
;
340 ins
->baddr0
= (long)mem_addr
;
343 ins
->op1_stride
= stride
;
344 gru_start_instruction(ins
, __opword(OP_VSTORE
, 0, xtype
, IAA_RAM
, 0,
348 static inline void gru_ivload(void *cb
, unsigned long mem_addr
,
349 unsigned int tri0
, unsigned int tri1
, unsigned char xtype
,
350 unsigned long nelem
, unsigned long hints
)
352 struct gru_instruction
*ins
= (void *)cb
;
354 ins
->baddr0
= (long)mem_addr
;
357 ins
->tri1_bufsize
= tri1
;
358 gru_start_instruction(ins
, __opword(OP_IVLOAD
, 0, xtype
, IAA_RAM
, 0,
362 static inline void gru_ivstore(void *cb
, unsigned long mem_addr
,
363 unsigned int tri0
, unsigned int tri1
,
364 unsigned char xtype
, unsigned long nelem
, unsigned long hints
)
366 struct gru_instruction
*ins
= (void *)cb
;
368 ins
->baddr0
= (long)mem_addr
;
371 ins
->tri1_bufsize
= tri1
;
372 gru_start_instruction(ins
, __opword(OP_IVSTORE
, 0, xtype
, IAA_RAM
, 0,
376 static inline void gru_vset(void *cb
, unsigned long mem_addr
,
377 unsigned long value
, unsigned char xtype
, unsigned long nelem
,
378 unsigned long stride
, unsigned long hints
)
380 struct gru_instruction
*ins
= (void *)cb
;
382 ins
->baddr0
= (long)mem_addr
;
383 ins
->op2_value_baddr1
= value
;
385 ins
->op1_stride
= stride
;
386 gru_start_instruction(ins
, __opword(OP_VSET
, 0, xtype
, IAA_RAM
, 0,
390 static inline void gru_ivset(void *cb
, unsigned long mem_addr
,
391 unsigned int tri1
, unsigned long value
, unsigned char xtype
,
392 unsigned long nelem
, unsigned long hints
)
394 struct gru_instruction
*ins
= (void *)cb
;
396 ins
->baddr0
= (long)mem_addr
;
397 ins
->op2_value_baddr1
= value
;
399 ins
->tri1_bufsize
= tri1
;
400 gru_start_instruction(ins
, __opword(OP_IVSET
, 0, xtype
, IAA_RAM
, 0,
404 static inline void gru_vflush(void *cb
, unsigned long mem_addr
,
405 unsigned long nelem
, unsigned char xtype
, unsigned long stride
,
408 struct gru_instruction
*ins
= (void *)cb
;
410 ins
->baddr0
= (long)mem_addr
;
411 ins
->op1_stride
= stride
;
413 gru_start_instruction(ins
, __opword(OP_VFLUSH
, 0, xtype
, IAA_RAM
, 0,
417 static inline void gru_nop(void *cb
, int hints
)
419 struct gru_instruction
*ins
= (void *)cb
;
421 gru_start_instruction(ins
, __opword(OP_NOP
, 0, 0, 0, 0, CB_IMA(hints
)));
425 static inline void gru_bcopy(void *cb
, const unsigned long src
,
427 unsigned int tri0
, unsigned int xtype
, unsigned long nelem
,
428 unsigned int bufsize
, unsigned long hints
)
430 struct gru_instruction
*ins
= (void *)cb
;
432 ins
->baddr0
= (long)src
;
433 ins
->op2_value_baddr1
= (long)dest
;
436 ins
->tri1_bufsize
= bufsize
;
437 gru_start_instruction(ins
, __opword(OP_BCOPY
, 0, xtype
, IAA_RAM
,
438 IAA_RAM
, CB_IMA(hints
)));
441 static inline void gru_bstore(void *cb
, const unsigned long src
,
442 unsigned long dest
, unsigned int tri0
, unsigned int xtype
,
443 unsigned long nelem
, unsigned long hints
)
445 struct gru_instruction
*ins
= (void *)cb
;
447 ins
->baddr0
= (long)src
;
448 ins
->op2_value_baddr1
= (long)dest
;
451 gru_start_instruction(ins
, __opword(OP_BSTORE
, 0, xtype
, 0, IAA_RAM
,
455 static inline void gru_gamir(void *cb
, int exopc
, unsigned long src
,
456 unsigned int xtype
, unsigned long hints
)
458 struct gru_instruction
*ins
= (void *)cb
;
460 ins
->baddr0
= (long)src
;
461 gru_start_instruction(ins
, __opword(OP_GAMIR
, exopc
, xtype
, IAA_RAM
, 0,
465 static inline void gru_gamirr(void *cb
, int exopc
, unsigned long src
,
466 unsigned int xtype
, unsigned long hints
)
468 struct gru_instruction
*ins
= (void *)cb
;
470 ins
->baddr0
= (long)src
;
471 gru_start_instruction(ins
, __opword(OP_GAMIRR
, exopc
, xtype
, IAA_RAM
, 0,
475 static inline void gru_gamer(void *cb
, int exopc
, unsigned long src
,
477 unsigned long operand1
, unsigned long operand2
,
480 struct gru_instruction
*ins
= (void *)cb
;
482 ins
->baddr0
= (long)src
;
483 ins
->op1_stride
= operand1
;
484 ins
->op2_value_baddr1
= operand2
;
485 gru_start_instruction(ins
, __opword(OP_GAMER
, exopc
, xtype
, IAA_RAM
, 0,
489 static inline void gru_gamerr(void *cb
, int exopc
, unsigned long src
,
490 unsigned int xtype
, unsigned long operand1
,
491 unsigned long operand2
, unsigned long hints
)
493 struct gru_instruction
*ins
= (void *)cb
;
495 ins
->baddr0
= (long)src
;
496 ins
->op1_stride
= operand1
;
497 ins
->op2_value_baddr1
= operand2
;
498 gru_start_instruction(ins
, __opword(OP_GAMERR
, exopc
, xtype
, IAA_RAM
, 0,
502 static inline void gru_gamxr(void *cb
, unsigned long src
,
503 unsigned int tri0
, unsigned long hints
)
505 struct gru_instruction
*ins
= (void *)cb
;
507 ins
->baddr0
= (long)src
;
509 gru_start_instruction(ins
, __opword(OP_GAMXR
, EOP_XR_CSWAP
, XTYPE_DW
,
510 IAA_RAM
, 0, CB_IMA(hints
)));
513 static inline void gru_mesq(void *cb
, unsigned long queue
,
514 unsigned long tri0
, unsigned long nelem
,
517 struct gru_instruction
*ins
= (void *)cb
;
519 ins
->baddr0
= (long)queue
;
522 gru_start_instruction(ins
, __opword(OP_MESQ
, 0, XTYPE_CL
, IAA_RAM
, 0,
526 static inline unsigned long gru_get_amo_value(void *cb
)
528 struct gru_instruction
*ins
= (void *)cb
;
533 static inline int gru_get_amo_value_head(void *cb
)
535 struct gru_instruction
*ins
= (void *)cb
;
537 return ins
->avalue
& 0xffffffff;
540 static inline int gru_get_amo_value_limit(void *cb
)
542 struct gru_instruction
*ins
= (void *)cb
;
544 return ins
->avalue
>> 32;
547 static inline union gru_mesqhead
gru_mesq_head(int head
, int limit
)
549 union gru_mesqhead mqh
;
557 * Get struct control_block_extended_exc_detail for CB.
559 extern int gru_get_cb_exception_detail(void *cb
,
560 struct control_block_extended_exc_detail
*excdet
);
562 #define GRU_EXC_STR_SIZE 256
566 * Control block definition for checking status
568 struct gru_control_block_status
{
569 unsigned int icmd
:1;
571 unsigned int reserved0
:4;
572 unsigned int unused1
:24;
573 unsigned int unused2
:24;
574 unsigned int istatus
:2;
575 unsigned int isubstatus
:4;
576 unsigned int unused3
:2;
580 static inline int gru_get_cb_status(void *cb
)
582 struct gru_control_block_status
*cbs
= (void *)cb
;
587 /* Get CB message queue substatus */
588 static inline int gru_get_cb_message_queue_substatus(void *cb
)
590 struct gru_control_block_status
*cbs
= (void *)cb
;
592 return cbs
->isubstatus
& CBSS_MSG_QUEUE_MASK
;
595 /* Get CB substatus */
596 static inline int gru_get_cb_substatus(void *cb
)
598 struct gru_control_block_status
*cbs
= (void *)cb
;
600 return cbs
->isubstatus
;
603 /* Check the status of a CB. If the CB is in UPM mode, call the
604 * OS to handle the UPM status.
605 * Returns the CB status field value (0 for normal completion)
607 static inline int gru_check_status(void *cb
)
609 struct gru_control_block_status
*cbs
= (void *)cb
;
613 if (ret
== CBS_CALL_OS
)
614 ret
= gru_check_status_proc(cb
);
618 /* Wait for CB to complete.
619 * Returns the CB status field value (0 for normal completion)
621 static inline int gru_wait(void *cb
)
623 struct gru_control_block_status
*cbs
= (void *)cb
;
624 int ret
= cbs
->istatus
;
627 ret
= gru_wait_proc(cb
);
631 /* Wait for CB to complete. Aborts program if error. (Note: error does NOT
632 * mean TLB mis - only fatal errors such as memory parity error or user
633 * bugs will cause termination.
635 static inline void gru_wait_abort(void *cb
)
637 struct gru_control_block_status
*cbs
= (void *)cb
;
639 if (cbs
->istatus
!= CBS_IDLE
)
640 gru_wait_abort_proc(cb
);
645 * Get a pointer to a control block
646 * gseg - GSeg address returned from gru_get_thread_gru_segment()
647 * index - index of desired CB
649 static inline void *gru_get_cb_pointer(void *gseg
,
652 return gseg
+ GRU_CB_BASE
+ index
* GRU_HANDLE_STRIDE
;
656 * Get a pointer to a cacheline in the data segment portion of a GSeg
657 * gseg - GSeg address returned from gru_get_thread_gru_segment()
658 * index - index of desired cache line
660 static inline void *gru_get_data_pointer(void *gseg
, int index
)
662 return gseg
+ GRU_DS_BASE
+ index
* GRU_CACHE_LINE_BYTES
;
666 * Convert a vaddr into the tri index within the GSEG
667 * vaddr - virtual address of within gseg
669 static inline int gru_get_tri(void *vaddr
)
671 return ((unsigned long)vaddr
& (GRU_GSEG_PAGESIZE
- 1)) - GRU_DS_BASE
;
673 #endif /* __GRU_INSTRUCTIONS_H__ */