2 * SGI UltraViolet TLB flush routines.
4 * (c) 2008 Cliff Wickman <cpw@sgi.com>, SGI.
6 * This code is released under the GNU General Public License version 2 or
9 #include <linux/seq_file.h>
10 #include <linux/proc_fs.h>
11 #include <linux/kernel.h>
13 #include <asm/mmu_context.h>
14 #include <asm/uv/uv.h>
15 #include <asm/uv/uv_mmrs.h>
16 #include <asm/uv/uv_hub.h>
17 #include <asm/uv/uv_bau.h>
21 #include <asm/irq_vectors.h>
23 static struct bau_control
**uv_bau_table_bases __read_mostly
;
24 static int uv_bau_retry_limit __read_mostly
;
26 /* position of pnode (which is nasid>>1): */
27 static int uv_nshift __read_mostly
;
29 static unsigned long uv_mmask __read_mostly
;
31 static DEFINE_PER_CPU(struct ptc_stats
, ptcstats
);
32 static DEFINE_PER_CPU(struct bau_control
, bau_control
);
35 * Determine the first node on a blade.
37 static int __init
blade_to_first_node(int blade
)
41 for_each_online_node(node
) {
42 b
= uv_node_to_blade_id(node
);
50 * Determine the apicid of the first cpu on a blade.
52 static int __init
blade_to_first_apicid(int blade
)
56 for_each_present_cpu(cpu
)
57 if (blade
== uv_cpu_to_blade_id(cpu
))
58 return per_cpu(x86_cpu_to_apicid
, cpu
);
63 * Free a software acknowledge hardware resource by clearing its Pending
64 * bit. This will return a reply to the sender.
65 * If the message has timed out, a reply has already been sent by the
66 * hardware but the resource has not been released. In that case our
67 * clear of the Timeout bit (as well) will free the resource. No reply will
68 * be sent (the hardware will only do one reply per message).
70 static void uv_reply_to_message(int resource
,
71 struct bau_payload_queue_entry
*msg
,
72 struct bau_msg_status
*msp
)
76 dw
= (1 << (resource
+ UV_SW_ACK_NPENDING
)) | (1 << resource
);
78 msg
->sw_ack_vector
= 0;
80 msp
->seen_by
.bits
= 0;
81 uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS
, dw
);
85 * Do all the things a cpu should do for a TLB shootdown message.
86 * Other cpu's may come here at the same time for this message.
88 static void uv_bau_process_message(struct bau_payload_queue_entry
*msg
,
89 int msg_slot
, int sw_ack_slot
)
91 unsigned long this_cpu_mask
;
92 struct bau_msg_status
*msp
;
95 msp
= __get_cpu_var(bau_control
).msg_statuses
+ msg_slot
;
96 cpu
= uv_blade_processor_id();
98 uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));
99 this_cpu_mask
= 1UL << cpu
;
100 if (msp
->seen_by
.bits
& this_cpu_mask
)
102 atomic_or_long(&msp
->seen_by
.bits
, this_cpu_mask
);
104 if (msg
->replied_to
== 1)
107 if (msg
->address
== TLB_FLUSH_ALL
) {
109 __get_cpu_var(ptcstats
).alltlb
++;
111 __flush_tlb_one(msg
->address
);
112 __get_cpu_var(ptcstats
).onetlb
++;
115 __get_cpu_var(ptcstats
).requestee
++;
117 atomic_inc_short(&msg
->acknowledge_count
);
118 if (msg
->number_of_cpus
== msg
->acknowledge_count
)
119 uv_reply_to_message(sw_ack_slot
, msg
, msp
);
123 * Examine the payload queue on one distribution node to see
124 * which messages have not been seen, and which cpu(s) have not seen them.
126 * Returns the number of cpu's that have not responded.
128 static int uv_examine_destination(struct bau_control
*bau_tablesp
, int sender
)
130 struct bau_payload_queue_entry
*msg
;
131 struct bau_msg_status
*msp
;
136 for (msg
= bau_tablesp
->va_queue_first
, i
= 0; i
< DEST_Q_SIZE
;
138 if ((msg
->sending_cpu
== sender
) && (!msg
->replied_to
)) {
139 msp
= bau_tablesp
->msg_statuses
+ i
;
141 "blade %d: address:%#lx %d of %d, not cpu(s): ",
142 i
, msg
->address
, msg
->acknowledge_count
,
143 msg
->number_of_cpus
);
144 for (j
= 0; j
< msg
->number_of_cpus
; j
++) {
145 if (!((1L << j
) & msp
->seen_by
.bits
)) {
157 * Examine the payload queue on all the distribution nodes to see
158 * which messages have not been seen, and which cpu(s) have not seen them.
160 * Returns the number of cpu's that have not responded.
162 static int uv_examine_destinations(struct bau_target_nodemask
*distribution
)
168 sender
= smp_processor_id();
169 for (i
= 0; i
< sizeof(struct bau_target_nodemask
) * BITSPERBYTE
; i
++) {
170 if (!bau_node_isset(i
, distribution
))
172 count
+= uv_examine_destination(uv_bau_table_bases
[i
], sender
);
178 * wait for completion of a broadcast message
180 * return COMPLETE, RETRY or GIVEUP
182 static int uv_wait_completion(struct bau_desc
*bau_desc
,
183 unsigned long mmr_offset
, int right_shift
)
186 long destination_timeouts
= 0;
187 long source_timeouts
= 0;
188 unsigned long descriptor_status
;
190 while ((descriptor_status
= (((unsigned long)
191 uv_read_local_mmr(mmr_offset
) >>
192 right_shift
) & UV_ACT_STATUS_MASK
)) !=
194 if (descriptor_status
== DESC_STATUS_SOURCE_TIMEOUT
) {
196 if (source_timeouts
> SOURCE_TIMEOUT_LIMIT
)
198 __get_cpu_var(ptcstats
).s_retry
++;
202 * spin here looking for progress at the destinations
204 if (descriptor_status
== DESC_STATUS_DESTINATION_TIMEOUT
) {
205 destination_timeouts
++;
206 if (destination_timeouts
> DESTINATION_TIMEOUT_LIMIT
) {
208 * returns number of cpus not responding
210 if (uv_examine_destinations
211 (&bau_desc
->distribution
) == 0) {
212 __get_cpu_var(ptcstats
).d_retry
++;
216 if (exams
>= uv_bau_retry_limit
) {
218 "uv_flush_tlb_others");
219 printk("giving up on cpu %d\n",
224 * delays can hang the simulator
227 destination_timeouts
= 0;
232 return FLUSH_COMPLETE
;
236 * uv_flush_send_and_wait
238 * Send a broadcast and wait for a broadcast message to complete.
240 * The flush_mask contains the cpus the broadcast was sent to.
242 * Returns NULL if all remote flushing was done. The mask is zeroed.
243 * Returns @flush_mask if some remote flushing remains to be done. The
244 * mask will have some bits still set.
246 const struct cpumask
*uv_flush_send_and_wait(int cpu
, int this_pnode
,
247 struct bau_desc
*bau_desc
,
248 struct cpumask
*flush_mask
)
250 int completion_status
= 0;
255 unsigned long mmr_offset
;
260 if (cpu
< UV_CPUS_PER_ACT_STATUS
) {
261 mmr_offset
= UVH_LB_BAU_SB_ACTIVATION_STATUS_0
;
262 right_shift
= cpu
* UV_ACT_STATUS_SIZE
;
264 mmr_offset
= UVH_LB_BAU_SB_ACTIVATION_STATUS_1
;
266 ((cpu
- UV_CPUS_PER_ACT_STATUS
) * UV_ACT_STATUS_SIZE
);
268 time1
= get_cycles();
271 index
= (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT
) |
273 uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL
, index
);
274 completion_status
= uv_wait_completion(bau_desc
, mmr_offset
,
276 } while (completion_status
== FLUSH_RETRY
);
277 time2
= get_cycles();
278 __get_cpu_var(ptcstats
).sflush
+= (time2
- time1
);
280 __get_cpu_var(ptcstats
).retriesok
++;
282 if (completion_status
== FLUSH_GIVEUP
) {
284 * Cause the caller to do an IPI-style TLB shootdown on
285 * the cpu's, all of which are still in the mask.
287 __get_cpu_var(ptcstats
).ptc_i
++;
292 * Success, so clear the remote cpu's from the mask so we don't
293 * use the IPI method of shootdown on them.
295 for_each_cpu(bit
, flush_mask
) {
296 pnode
= uv_cpu_to_pnode(bit
);
297 if (pnode
== this_pnode
)
299 cpumask_clear_cpu(bit
, flush_mask
);
301 if (!cpumask_empty(flush_mask
))
307 * uv_flush_tlb_others - globally purge translation cache of a virtual
308 * address or all TLB's
309 * @cpumask: mask of all cpu's in which the address is to be removed
310 * @mm: mm_struct containing virtual address range
311 * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu)
312 * @cpu: the current cpu
314 * This is the entry point for initiating any UV global TLB shootdown.
316 * Purges the translation caches of all specified processors of the given
317 * virtual address, or purges all TLB's on specified processors.
319 * The caller has derived the cpumask from the mm_struct. This function
320 * is called only if there are bits set in the mask. (e.g. flush_tlb_page())
322 * The cpumask is converted into a nodemask of the nodes containing
325 * Note that this function should be called with preemption disabled.
327 * Returns NULL if all remote flushing was done.
328 * Returns pointer to cpumask if some remote flushing remains to be
329 * done. The returned pointer is valid till preemption is re-enabled.
331 const struct cpumask
*uv_flush_tlb_others(const struct cpumask
*cpumask
,
332 struct mm_struct
*mm
,
333 unsigned long va
, unsigned int cpu
)
335 static DEFINE_PER_CPU(cpumask_t
, flush_tlb_mask
);
336 struct cpumask
*flush_mask
= &__get_cpu_var(flush_tlb_mask
);
343 struct bau_desc
*bau_desc
;
345 cpumask_andnot(flush_mask
, cpumask
, cpumask_of(cpu
));
347 uv_cpu
= uv_blade_processor_id();
348 this_pnode
= uv_hub_info
->pnode
;
349 bau_desc
= __get_cpu_var(bau_control
).descriptor_base
;
350 bau_desc
+= UV_ITEMS_PER_DESCRIPTOR
* uv_cpu
;
352 bau_nodes_clear(&bau_desc
->distribution
, UV_DISTRIBUTION_SIZE
);
355 for_each_cpu(bit
, flush_mask
) {
356 pnode
= uv_cpu_to_pnode(bit
);
357 BUG_ON(pnode
> (UV_DISTRIBUTION_SIZE
- 1));
358 if (pnode
== this_pnode
) {
362 bau_node_set(pnode
, &bau_desc
->distribution
);
367 * no off_node flushing; return status for local node
374 __get_cpu_var(ptcstats
).requestor
++;
375 __get_cpu_var(ptcstats
).ntargeted
+= i
;
377 bau_desc
->payload
.address
= va
;
378 bau_desc
->payload
.sending_cpu
= cpu
;
380 return uv_flush_send_and_wait(uv_cpu
, this_pnode
, bau_desc
, flush_mask
);
384 * The BAU message interrupt comes here. (registered by set_intr_gate)
387 * We received a broadcast assist message.
389 * Interrupts may have been disabled; this interrupt could represent
390 * the receipt of several messages.
392 * All cores/threads on this node get this interrupt.
393 * The last one to see it does the s/w ack.
394 * (the resource will not be freed until noninterruptable cpus see this
395 * interrupt; hardware will timeout the s/w ack and reply ERROR)
397 void uv_bau_message_interrupt(struct pt_regs
*regs
)
399 struct bau_payload_queue_entry
*va_queue_first
;
400 struct bau_payload_queue_entry
*va_queue_last
;
401 struct bau_payload_queue_entry
*msg
;
402 struct pt_regs
*old_regs
= set_irq_regs(regs
);
409 unsigned long local_pnode
;
415 time1
= get_cycles();
417 local_pnode
= uv_blade_to_pnode(uv_numa_blade_id());
419 va_queue_first
= __get_cpu_var(bau_control
).va_queue_first
;
420 va_queue_last
= __get_cpu_var(bau_control
).va_queue_last
;
422 msg
= __get_cpu_var(bau_control
).bau_msg_head
;
423 while (msg
->sw_ack_vector
) {
425 fw
= msg
->sw_ack_vector
;
426 msg_slot
= msg
- va_queue_first
;
427 sw_ack_slot
= ffs(fw
) - 1;
429 uv_bau_process_message(msg
, msg_slot
, sw_ack_slot
);
432 if (msg
> va_queue_last
)
433 msg
= va_queue_first
;
434 __get_cpu_var(bau_control
).bau_msg_head
= msg
;
437 __get_cpu_var(ptcstats
).nomsg
++;
439 __get_cpu_var(ptcstats
).multmsg
++;
441 time2
= get_cycles();
442 __get_cpu_var(ptcstats
).dflush
+= (time2
- time1
);
445 set_irq_regs(old_regs
);
448 static void uv_enable_timeouts(void)
455 unsigned long apicid
;
458 for_each_online_node(i
) {
459 blade
= uv_node_to_blade_id(i
);
460 if (blade
== last_blade
)
463 apicid
= per_cpu(x86_cpu_to_apicid
, cur_cpu
);
464 pnode
= uv_blade_to_pnode(blade
);
465 cur_cpu
+= uv_blade_nr_possible_cpus(i
);
469 static void *uv_ptc_seq_start(struct seq_file
*file
, loff_t
*offset
)
471 if (*offset
< num_possible_cpus())
476 static void *uv_ptc_seq_next(struct seq_file
*file
, void *data
, loff_t
*offset
)
479 if (*offset
< num_possible_cpus())
484 static void uv_ptc_seq_stop(struct seq_file
*file
, void *data
)
489 * Display the statistics thru /proc
490 * data points to the cpu number
492 static int uv_ptc_seq_show(struct seq_file
*file
, void *data
)
494 struct ptc_stats
*stat
;
497 cpu
= *(loff_t
*)data
;
501 "# cpu requestor requestee one all sretry dretry ptc_i ");
503 "sw_ack sflush dflush sok dnomsg dmult starget\n");
505 if (cpu
< num_possible_cpus() && cpu_online(cpu
)) {
506 stat
= &per_cpu(ptcstats
, cpu
);
507 seq_printf(file
, "cpu %d %ld %ld %ld %ld %ld %ld %ld ",
508 cpu
, stat
->requestor
,
509 stat
->requestee
, stat
->onetlb
, stat
->alltlb
,
510 stat
->s_retry
, stat
->d_retry
, stat
->ptc_i
);
511 seq_printf(file
, "%lx %ld %ld %ld %ld %ld %ld\n",
512 uv_read_global_mmr64(uv_cpu_to_pnode(cpu
),
513 UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE
),
514 stat
->sflush
, stat
->dflush
,
515 stat
->retriesok
, stat
->nomsg
,
516 stat
->multmsg
, stat
->ntargeted
);
523 * 0: display meaning of the statistics
526 static ssize_t
uv_ptc_proc_write(struct file
*file
, const char __user
*user
,
527 size_t count
, loff_t
*data
)
532 if (count
== 0 || count
> sizeof(optstr
))
534 if (copy_from_user(optstr
, user
, count
))
536 optstr
[count
- 1] = '\0';
537 if (strict_strtoul(optstr
, 10, &newmode
) < 0) {
538 printk(KERN_DEBUG
"%s is invalid\n", optstr
);
543 printk(KERN_DEBUG
"# cpu: cpu number\n");
545 "requestor: times this cpu was the flush requestor\n");
547 "requestee: times this cpu was requested to flush its TLBs\n");
549 "one: times requested to flush a single address\n");
551 "all: times requested to flush all TLB's\n");
553 "sretry: number of retries of source-side timeouts\n");
555 "dretry: number of retries of destination-side timeouts\n");
557 "ptc_i: times UV fell through to IPI-style flushes\n");
559 "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n");
561 "sflush_us: cycles spent in uv_flush_tlb_others()\n");
563 "dflush_us: cycles spent in handling flush requests\n");
564 printk(KERN_DEBUG
"sok: successes on retry\n");
565 printk(KERN_DEBUG
"dnomsg: interrupts with no message\n");
567 "dmult: interrupts with multiple messages\n");
568 printk(KERN_DEBUG
"starget: nodes targeted\n");
570 uv_bau_retry_limit
= newmode
;
571 printk(KERN_DEBUG
"timeout retry limit:%d\n",
578 static const struct seq_operations uv_ptc_seq_ops
= {
579 .start
= uv_ptc_seq_start
,
580 .next
= uv_ptc_seq_next
,
581 .stop
= uv_ptc_seq_stop
,
582 .show
= uv_ptc_seq_show
585 static int uv_ptc_proc_open(struct inode
*inode
, struct file
*file
)
587 return seq_open(file
, &uv_ptc_seq_ops
);
590 static const struct file_operations proc_uv_ptc_operations
= {
591 .open
= uv_ptc_proc_open
,
593 .write
= uv_ptc_proc_write
,
595 .release
= seq_release
,
598 static int __init
uv_ptc_init(void)
600 struct proc_dir_entry
*proc_uv_ptc
;
605 proc_uv_ptc
= create_proc_entry(UV_PTC_BASENAME
, 0444, NULL
);
607 printk(KERN_ERR
"unable to create %s proc entry\n",
611 proc_uv_ptc
->proc_fops
= &proc_uv_ptc_operations
;
616 * begin the initialization of the per-blade control structures
618 static struct bau_control
* __init
uv_table_bases_init(int blade
, int node
)
621 struct bau_msg_status
*msp
;
622 struct bau_control
*bau_tabp
;
625 kmalloc_node(sizeof(struct bau_control
), GFP_KERNEL
, node
);
628 bau_tabp
->msg_statuses
=
629 kmalloc_node(sizeof(struct bau_msg_status
) *
630 DEST_Q_SIZE
, GFP_KERNEL
, node
);
631 BUG_ON(!bau_tabp
->msg_statuses
);
633 for (i
= 0, msp
= bau_tabp
->msg_statuses
; i
< DEST_Q_SIZE
; i
++, msp
++)
634 bau_cpubits_clear(&msp
->seen_by
, (int)
635 uv_blade_nr_possible_cpus(blade
));
637 uv_bau_table_bases
[blade
] = bau_tabp
;
643 * finish the initialization of the per-blade control structures
646 uv_table_bases_finish(int blade
,
647 struct bau_control
*bau_tablesp
,
648 struct bau_desc
*adp
)
650 struct bau_control
*bcp
;
653 for_each_present_cpu(cpu
) {
654 if (blade
!= uv_cpu_to_blade_id(cpu
))
657 bcp
= (struct bau_control
*)&per_cpu(bau_control
, cpu
);
658 bcp
->bau_msg_head
= bau_tablesp
->va_queue_first
;
659 bcp
->va_queue_first
= bau_tablesp
->va_queue_first
;
660 bcp
->va_queue_last
= bau_tablesp
->va_queue_last
;
661 bcp
->msg_statuses
= bau_tablesp
->msg_statuses
;
662 bcp
->descriptor_base
= adp
;
667 * initialize the sending side's sending buffers
669 static struct bau_desc
* __init
670 uv_activation_descriptor_init(int node
, int pnode
)
676 unsigned long mmr_image
;
677 struct bau_desc
*adp
;
678 struct bau_desc
*ad2
;
680 adp
= (struct bau_desc
*)kmalloc_node(16384, GFP_KERNEL
, node
);
683 pa
= __pa((unsigned long)adp
);
687 mmr_image
= uv_read_global_mmr64(pnode
, UVH_LB_BAU_SB_DESCRIPTOR_BASE
);
689 uv_write_global_mmr64(pnode
, (unsigned long)
690 UVH_LB_BAU_SB_DESCRIPTOR_BASE
,
691 (n
<< UV_DESC_BASE_PNODE_SHIFT
| m
));
694 for (i
= 0, ad2
= adp
; i
< UV_ACTIVATION_DESCRIPTOR_SIZE
; i
++, ad2
++) {
695 memset(ad2
, 0, sizeof(struct bau_desc
));
696 ad2
->header
.sw_ack_flag
= 1;
697 ad2
->header
.base_dest_nodeid
= uv_cpu_to_pnode(0);
698 ad2
->header
.command
= UV_NET_ENDPOINT_INTD
;
699 ad2
->header
.int_both
= 1;
701 * all others need to be set to zero:
702 * fairness chaining multilevel count replied_to
709 * initialize the destination side's receiving buffers
711 static struct bau_payload_queue_entry
* __init
712 uv_payload_queue_init(int node
, int pnode
, struct bau_control
*bau_tablesp
)
714 struct bau_payload_queue_entry
*pqp
;
717 pqp
= (struct bau_payload_queue_entry
*) kmalloc_node(
718 (DEST_Q_SIZE
+ 1) * sizeof(struct bau_payload_queue_entry
),
722 cp
= (char *)pqp
+ 31;
723 pqp
= (struct bau_payload_queue_entry
*)(((unsigned long)cp
>> 5) << 5);
724 bau_tablesp
->va_queue_first
= pqp
;
725 uv_write_global_mmr64(pnode
,
726 UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST
,
727 ((unsigned long)pnode
<<
728 UV_PAYLOADQ_PNODE_SHIFT
) |
729 uv_physnodeaddr(pqp
));
730 uv_write_global_mmr64(pnode
, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL
,
731 uv_physnodeaddr(pqp
));
732 bau_tablesp
->va_queue_last
= pqp
+ (DEST_Q_SIZE
- 1);
733 uv_write_global_mmr64(pnode
, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST
,
735 uv_physnodeaddr(bau_tablesp
->va_queue_last
));
736 memset(pqp
, 0, sizeof(struct bau_payload_queue_entry
) * DEST_Q_SIZE
);
742 * Initialization of each UV blade's structures
744 static int __init
uv_init_blade(int blade
)
749 unsigned long apicid
;
750 struct bau_desc
*adp
;
751 struct bau_payload_queue_entry
*pqp
;
752 struct bau_control
*bau_tablesp
;
754 node
= blade_to_first_node(blade
);
755 bau_tablesp
= uv_table_bases_init(blade
, node
);
756 pnode
= uv_blade_to_pnode(blade
);
757 adp
= uv_activation_descriptor_init(node
, pnode
);
758 pqp
= uv_payload_queue_init(node
, pnode
, bau_tablesp
);
759 uv_table_bases_finish(blade
, bau_tablesp
, adp
);
761 * the below initialization can't be in firmware because the
762 * messaging IRQ will be determined by the OS
764 apicid
= blade_to_first_apicid(blade
);
765 pa
= uv_read_global_mmr64(pnode
, UVH_BAU_DATA_CONFIG
);
766 if ((pa
& 0xff) != UV_BAU_MESSAGE
) {
767 uv_write_global_mmr64(pnode
, UVH_BAU_DATA_CONFIG
,
768 ((apicid
<< 32) | UV_BAU_MESSAGE
));
774 * Initialization of BAU-related structures
776 static int __init
uv_bau_init(void)
785 uv_bau_retry_limit
= 1;
786 uv_nshift
= uv_hub_info
->n_val
;
787 uv_mmask
= (1UL << uv_hub_info
->n_val
) - 1;
788 nblades
= uv_num_possible_blades();
790 uv_bau_table_bases
= (struct bau_control
**)
791 kmalloc(nblades
* sizeof(struct bau_control
*), GFP_KERNEL
);
792 BUG_ON(!uv_bau_table_bases
);
794 for (blade
= 0; blade
< nblades
; blade
++)
795 if (uv_blade_nr_possible_cpus(blade
))
796 uv_init_blade(blade
);
798 alloc_intr_gate(UV_BAU_MESSAGE
, uv_bau_message_intr1
);
799 uv_enable_timeouts();
803 __initcall(uv_bau_init
);
804 __initcall(uv_ptc_init
);