2 * SN Platform GRU Driver
4 * FILE OPERATIONS & DRIVER INITIALIZATION
6 * This file supports the user system call for file open, close, mmap, etc.
7 * This also incudes the driver initialization code.
9 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 #include <linux/module.h>
27 #include <linux/kernel.h>
28 #include <linux/errno.h>
29 #include <linux/slab.h>
32 #include <linux/spinlock.h>
33 #include <linux/device.h>
34 #include <linux/miscdevice.h>
35 #include <linux/interrupt.h>
36 #include <linux/proc_fs.h>
37 #include <linux/uaccess.h>
38 #include <asm/uv/uv.h>
41 #include "grutables.h"
43 #include <asm/uv/uv_hub.h>
44 #include <asm/uv/uv_mmrs.h>
46 struct gru_blade_state
*gru_base
[GRU_MAX_BLADES
] __read_mostly
;
47 unsigned long gru_start_paddr __read_mostly
;
48 void *gru_start_vaddr __read_mostly
;
49 unsigned long gru_end_paddr __read_mostly
;
50 unsigned int gru_max_gids __read_mostly
;
51 struct gru_stats_s gru_stats
;
53 /* Guaranteed user available resources on each node */
54 static int max_user_cbrs
, max_user_dsr_bytes
;
56 static struct miscdevice gru_miscdev
;
62 * Called when unmapping a device mapping. Frees all gru resources
63 * and tables belonging to the vma.
65 static void gru_vma_close(struct vm_area_struct
*vma
)
67 struct gru_vma_data
*vdata
;
68 struct gru_thread_state
*gts
;
69 struct list_head
*entry
, *next
;
71 if (!vma
->vm_private_data
)
74 vdata
= vma
->vm_private_data
;
75 vma
->vm_private_data
= NULL
;
76 gru_dbg(grudev
, "vma %p, file %p, vdata %p\n", vma
, vma
->vm_file
,
78 list_for_each_safe(entry
, next
, &vdata
->vd_head
) {
80 list_entry(entry
, struct gru_thread_state
, ts_next
);
81 list_del(>s
->ts_next
);
82 mutex_lock(>s
->ts_ctxlock
);
84 gru_unload_context(gts
, 0);
85 mutex_unlock(>s
->ts_ctxlock
);
95 * Called when mmapping the device. Initializes the vma with a fault handler
96 * and private data structure necessary to allocate, track, and free the
99 static int gru_file_mmap(struct file
*file
, struct vm_area_struct
*vma
)
101 if ((vma
->vm_flags
& (VM_SHARED
| VM_WRITE
)) != (VM_SHARED
| VM_WRITE
))
104 if (vma
->vm_start
& (GRU_GSEG_PAGESIZE
- 1) ||
105 vma
->vm_end
& (GRU_GSEG_PAGESIZE
- 1))
109 (VM_IO
| VM_DONTCOPY
| VM_LOCKED
| VM_DONTEXPAND
| VM_PFNMAP
|
111 vma
->vm_page_prot
= PAGE_SHARED
;
112 vma
->vm_ops
= &gru_vm_ops
;
114 vma
->vm_private_data
= gru_alloc_vma_data(vma
, 0);
115 if (!vma
->vm_private_data
)
118 gru_dbg(grudev
, "file %p, vaddr 0x%lx, vma %p, vdata %p\n",
119 file
, vma
->vm_start
, vma
, vma
->vm_private_data
);
124 * Create a new GRU context
126 static int gru_create_new_context(unsigned long arg
)
128 struct gru_create_context_req req
;
129 struct vm_area_struct
*vma
;
130 struct gru_vma_data
*vdata
;
134 if (copy_from_user(&req
, (void __user
*)arg
, sizeof(req
)))
137 if (req
.data_segment_bytes
> max_user_dsr_bytes
)
139 if (req
.control_blocks
> max_user_cbrs
|| !req
.maximum_thread_count
)
142 if (!(req
.options
& GRU_OPT_MISS_MASK
))
143 req
.options
|= GRU_OPT_MISS_FMM_INTR
;
145 down_write(¤t
->mm
->mmap_sem
);
146 vma
= gru_find_vma(req
.gseg
);
148 vdata
= vma
->vm_private_data
;
149 vdata
->vd_user_options
= req
.options
;
150 vdata
->vd_dsr_au_count
=
151 GRU_DS_BYTES_TO_AU(req
.data_segment_bytes
);
152 vdata
->vd_cbr_au_count
= GRU_CB_COUNT_TO_AU(req
.control_blocks
);
155 up_write(¤t
->mm
->mmap_sem
);
161 * Get GRU configuration info (temp - for emulator testing)
163 static long gru_get_config_info(unsigned long arg
)
165 struct gru_config_info info
;
168 if (num_online_nodes() > 1 &&
169 (uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
173 info
.cpus
= num_online_cpus();
174 info
.nodes
= num_online_nodes();
175 info
.blades
= info
.nodes
/ nodesperblade
;
176 info
.chiplets
= GRU_CHIPLETS_PER_BLADE
* info
.blades
;
178 if (copy_to_user((void __user
*)arg
, &info
, sizeof(info
)))
184 * gru_file_unlocked_ioctl
186 * Called to update file attributes via IOCTL calls.
188 static long gru_file_unlocked_ioctl(struct file
*file
, unsigned int req
,
193 gru_dbg(grudev
, "file %p\n", file
);
196 case GRU_CREATE_CONTEXT
:
197 err
= gru_create_new_context(arg
);
199 case GRU_SET_CONTEXT_OPTION
:
200 err
= gru_set_context_option(arg
);
202 case GRU_USER_GET_EXCEPTION_DETAIL
:
203 err
= gru_get_exception_detail(arg
);
205 case GRU_USER_UNLOAD_CONTEXT
:
206 err
= gru_user_unload_context(arg
);
208 case GRU_USER_FLUSH_TLB
:
209 err
= gru_user_flush_tlb(arg
);
211 case GRU_USER_CALL_OS
:
212 err
= gru_handle_user_call_os(arg
);
214 case GRU_GET_GSEG_STATISTICS
:
215 err
= gru_get_gseg_statistics(arg
);
218 err
= gru_ktest(arg
);
220 case GRU_GET_CONFIG_INFO
:
221 err
= gru_get_config_info(arg
);
223 case GRU_DUMP_CHIPLET_STATE
:
224 err
= gru_dump_chiplet_request(arg
);
231 * Called at init time to build tables for all GRUs that are present in the
234 static void gru_init_chiplet(struct gru_state
*gru
, unsigned long paddr
,
235 void *vaddr
, int nid
, int bid
, int grunum
)
237 spin_lock_init(&gru
->gs_lock
);
238 spin_lock_init(&gru
->gs_asid_lock
);
239 gru
->gs_gru_base_paddr
= paddr
;
240 gru
->gs_gru_base_vaddr
= vaddr
;
241 gru
->gs_gid
= bid
* GRU_CHIPLETS_PER_BLADE
+ grunum
;
242 gru
->gs_blade
= gru_base
[bid
];
243 gru
->gs_blade_id
= bid
;
244 gru
->gs_cbr_map
= (GRU_CBR_AU
== 64) ? ~0 : (1UL << GRU_CBR_AU
) - 1;
245 gru
->gs_dsr_map
= (1UL << GRU_DSR_AU
) - 1;
246 gru
->gs_asid_limit
= MAX_ASID
;
247 gru_tgh_flush_init(gru
);
248 if (gru
->gs_gid
>= gru_max_gids
)
249 gru_max_gids
= gru
->gs_gid
+ 1;
250 gru_dbg(grudev
, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n",
251 bid
, nid
, gru
->gs_gid
, gru
->gs_gru_base_vaddr
,
252 gru
->gs_gru_base_paddr
);
255 static int gru_init_tables(unsigned long gru_base_paddr
, void *gru_base_vaddr
)
257 int pnode
, nid
, bid
, chip
;
258 int cbrs
, dsrbytes
, n
;
259 int order
= get_order(sizeof(struct gru_blade_state
));
261 struct gru_state
*gru
;
265 max_user_cbrs
= GRU_NUM_CB
;
266 max_user_dsr_bytes
= GRU_NUM_DSR_BYTES
;
267 for_each_possible_blade(bid
) {
268 pnode
= uv_blade_to_pnode(bid
);
269 nid
= uv_blade_to_memory_nid(bid
);
270 page
= alloc_pages_exact_node(nid
, GFP_KERNEL
, order
);
273 gru_base
[bid
] = page_address(page
);
274 memset(gru_base
[bid
], 0, sizeof(struct gru_blade_state
));
275 gru_base
[bid
]->bs_lru_gru
= &gru_base
[bid
]->bs_grus
[0];
276 spin_lock_init(&gru_base
[bid
]->bs_lock
);
277 init_rwsem(&gru_base
[bid
]->bs_kgts_sema
);
281 for (gru
= gru_base
[bid
]->bs_grus
, chip
= 0;
282 chip
< GRU_CHIPLETS_PER_BLADE
;
284 paddr
= gru_chiplet_paddr(gru_base_paddr
, pnode
, chip
);
285 vaddr
= gru_chiplet_vaddr(gru_base_vaddr
, pnode
, chip
);
286 gru_init_chiplet(gru
, paddr
, vaddr
, nid
, bid
, chip
);
287 n
= hweight64(gru
->gs_cbr_map
) * GRU_CBR_AU_SIZE
;
289 n
= hweight64(gru
->gs_dsr_map
) * GRU_DSR_AU_BYTES
;
290 dsrbytes
= max(dsrbytes
, n
);
292 max_user_cbrs
= min(max_user_cbrs
, cbrs
);
293 max_user_dsr_bytes
= min(max_user_dsr_bytes
, dsrbytes
);
299 for (bid
--; bid
>= 0; bid
--)
300 free_pages((unsigned long)gru_base
[bid
], order
);
306 static int get_base_irq(void)
311 #elif defined CONFIG_X86_64
313 static void noop(unsigned int irq
)
317 static struct irq_chip gru_chip
= {
324 static int get_base_irq(void)
326 set_irq_chip(IRQ_GRU
, &gru_chip
);
327 set_irq_chip(IRQ_GRU
+ 1, &gru_chip
);
335 * Called at boot or module load time to initialize the GRUs.
337 static int __init
gru_init(void)
345 #if defined CONFIG_IA64
346 gru_start_paddr
= 0xd000000000UL
; /* ZZZZZZZZZZZZZZZZZZZ fixme */
348 gru_start_paddr
= uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR
) &
351 gru_start_vaddr
= __va(gru_start_paddr
);
352 gru_end_paddr
= gru_start_paddr
+ GRU_MAX_BLADES
* GRU_SIZE
;
353 printk(KERN_INFO
"GRU space: 0x%lx - 0x%lx\n",
354 gru_start_paddr
, gru_end_paddr
);
355 irq
= get_base_irq();
356 for (chip
= 0; chip
< GRU_CHIPLETS_PER_BLADE
; chip
++) {
357 ret
= request_irq(irq
+ chip
, gru_intr
, 0, id
, NULL
);
358 /* TODO: fix irq handling on x86. For now ignore failure because
359 * interrupts are not required & not yet fully supported */
362 "!!!WARNING: GRU ignoring request failure!!!\n");
366 printk(KERN_ERR
"%s: request_irq failed\n",
372 ret
= misc_register(&gru_miscdev
);
374 printk(KERN_ERR
"%s: misc_register failed\n",
379 ret
= gru_proc_init();
381 printk(KERN_ERR
"%s: proc init failed\n", GRU_DRIVER_ID_STR
);
385 ret
= gru_init_tables(gru_start_paddr
, gru_start_vaddr
);
387 printk(KERN_ERR
"%s: init tables failed\n", GRU_DRIVER_ID_STR
);
390 gru_kservices_init();
392 printk(KERN_INFO
"%s: v%s\n", GRU_DRIVER_ID_STR
,
393 GRU_DRIVER_VERSION_STR
);
399 misc_deregister(&gru_miscdev
);
401 for (--chip
; chip
>= 0; chip
--)
402 free_irq(irq
+ chip
, NULL
);
407 static void __exit
gru_exit(void)
410 int order
= get_order(sizeof(struct gru_state
) *
411 GRU_CHIPLETS_PER_BLADE
);
416 for (i
= 0; i
< GRU_CHIPLETS_PER_BLADE
; i
++)
417 free_irq(IRQ_GRU
+ i
, NULL
);
418 gru_kservices_exit();
419 for (bid
= 0; bid
< GRU_MAX_BLADES
; bid
++)
420 free_pages((unsigned long)gru_base
[bid
], order
);
422 misc_deregister(&gru_miscdev
);
426 static const struct file_operations gru_fops
= {
427 .owner
= THIS_MODULE
,
428 .unlocked_ioctl
= gru_file_unlocked_ioctl
,
429 .mmap
= gru_file_mmap
,
432 static struct miscdevice gru_miscdev
= {
433 .minor
= MISC_DYNAMIC_MINOR
,
438 const struct vm_operations_struct gru_vm_ops
= {
439 .close
= gru_vma_close
,
444 fs_initcall(gru_init
);
446 module_init(gru_init
);
448 module_exit(gru_exit
);
450 module_param(gru_options
, ulong
, 0644);
451 MODULE_PARM_DESC(gru_options
, "Various debug options");
453 MODULE_AUTHOR("Silicon Graphics, Inc.");
454 MODULE_LICENSE("GPL");
455 MODULE_DESCRIPTION(GRU_DRIVER_ID_STR GRU_DRIVER_VERSION_STR
);
456 MODULE_VERSION(GRU_DRIVER_VERSION_STR
);