From bd28a477274db2b836577dfd6f223c696ad00720 Mon Sep 17 00:00:00 2001 From: Prashanth Sreenivasa Date: Fri, 23 Apr 2010 14:55:01 -0700 Subject: [PATCH] 6910795 Use the invariant %stick for sun4v system tick --- usr/src/uts/sun4/ml/interrupt.s | 25 ++++++++++++------------ usr/src/uts/sun4/ml/swtch.s | 7 +++---- usr/src/uts/sun4/os/machdep.c | 8 ++++---- usr/src/uts/sun4u/sys/machclock.h | 22 +++++++++++++++++++-- usr/src/uts/sun4v/ml/mach_subr_asm.s | 17 +++++----------- usr/src/uts/sun4v/sys/machclock.h | 38 ++++++++++++++++++++++++++++++++++-- 6 files changed, 80 insertions(+), 37 deletions(-) diff --git a/usr/src/uts/sun4/ml/interrupt.s b/usr/src/uts/sun4/ml/interrupt.s index 3d7ff2de29..021fcc5747 100644 --- a/usr/src/uts/sun4/ml/interrupt.s +++ b/usr/src/uts/sun4/ml/interrupt.s @@ -19,8 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ #if defined(lint) @@ -389,7 +388,7 @@ intr_thread(struct regs *regs, uint64_t iv_p, uint_t pil) ! resume() hasn't yet stored a timestamp for it. Or, it could be in ! swtch() after its slice has been accounted for. ! Only account for the time slice if the starting timestamp is non-zero. - RD_TICK(%o4,%l2,%l3,__LINE__) + RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__) sub %o4, %o3, %o4 ! o4 has interval ! A high-level interrupt in current_thread() interrupting here @@ -518,7 +517,7 @@ intr_thread(struct regs *regs, uint64_t iv_p, uint_t pil) add THREAD_REG, T_INTR_START, %o3 1: ldx [%o3], %o5 - RD_TICK(%o4,%l2,%l3,__LINE__) + RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__) casx [%o3], %o5, %o4 cmp %o4, %o5 ! If a high-level interrupt occurred while we were attempting to store @@ -583,7 +582,7 @@ intr_thread(struct regs *regs, uint64_t iv_p, uint_t pil) or %o0, %lo(intr_thread_t_intr_start_zero), %o0 9: #endif /* DEBUG */ - RD_TICK(%o1,%l2,%l3,__LINE__) + RD_CLOCK_TICK(%o1,%l2,%l3,__LINE__) sub %o1, %o0, %l2 ! l2 has interval ! ! The general outline of what the code here does is: @@ -773,7 +772,7 @@ intr_thread(struct regs *regs, uint64_t iv_p, uint_t pil) add THREAD_REG, T_INTR_START, %o3 ! o3 has &curthread->t_intr_star 0: ldx [%o3], %o4 ! o4 = t_intr_start before - RD_TICK(%o5,%l2,%l3,__LINE__) + RD_CLOCK_TICK(%o5,%l2,%l3,__LINE__) casx [%o3], %o4, %o5 ! put o5 in ts if o4 == ts after cmp %o4, %o5 ! If a high-level interrupt occurred while we were attempting to store @@ -1046,7 +1045,7 @@ no_onfault: or %o0, %lo(current_thread_nested_pil_zero), %o0 9: #endif /* DEBUG */ - RD_TICK_NO_SUSPEND_CHECK(%l1, %l2) + RD_CLOCK_TICK_NO_SUSPEND_CHECK(%l1, %l2) sub %l1, %l3, %l3 ! interval in %l3 ! ! Check for Energy Star mode @@ -1101,7 +1100,7 @@ no_onfault: nop stx %g0, [THREAD_REG + T_INTR_START] - RD_TICK_NO_SUSPEND_CHECK(%o4, %l2) + RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2) sub %o4, %o5, %o5 ! o5 has the interval ! Check for Energy Star mode @@ -1154,7 +1153,7 @@ no_onfault: sllx %o4, 3, %o4 ! index to byte offset add %o4, CPU_MCPU, %o4 ! CPU_PIL_HIGH_START is too large add %o4, MCPU_PIL_HIGH_START, %o4 - RD_TICK_NO_SUSPEND_CHECK(%o5, %l2) + RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %l2) stx %o5, [%o3 + %o4] wrpr %g0, %o2, %pil ! enable interrupts @@ -1244,7 +1243,7 @@ current_thread_complete: sllx %o4, 3, %o4 ! index to byte offset add %o4, CPU_MCPU, %o4 ! CPU_PIL_HIGH_START is too large add %o4, MCPU_PIL_HIGH_START, %o4 - RD_TICK_NO_SUSPEND_CHECK(%o5, %o0) + RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %o0) ldx [%o3 + %o4], %o0 #ifdef DEBUG ! ASSERT(cpu.cpu_m.pil_high_start[pil - (LOCK_LEVEL + 1)] != 0) @@ -1334,7 +1333,7 @@ current_thread_complete: sll %o5, 3, %o5 ! convert array index to byte offset add %o5, CPU_MCPU, %o5 ! CPU_PIL_HIGH_START is too large add %o5, MCPU_PIL_HIGH_START, %o5 - RD_TICK_NO_SUSPEND_CHECK(%o4, %l2) + RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2) ! Another high-level interrupt is active below this one, so ! there is no need to check for an interrupt thread. That will be ! done by the lowest priority high-level interrupt active. @@ -1349,7 +1348,7 @@ current_thread_complete: bz,pt %xcc, 7f nop - RD_TICK_NO_SUSPEND_CHECK(%o4, %l2) + RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2) stx %o4, [THREAD_REG + T_INTR_START] 7: @@ -2191,7 +2190,7 @@ intr_get_time(void) ! Calculate elapsed time since t_intr_start. Update t_intr_start, ! get delta, and multiply by cpu_divisor if necessary. ! - RD_TICK_NO_SUSPEND_CHECK(%o2, %o0) + RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o2, %o0) stx %o2, [THREAD_REG + T_INTR_START] sub %o2, %o3, %o0 diff --git a/usr/src/uts/sun4/ml/swtch.s b/usr/src/uts/sun4/ml/swtch.s index d88d2bdc67..7c688ec551 100644 --- a/usr/src/uts/sun4/ml/swtch.s +++ b/usr/src/uts/sun4/ml/swtch.s @@ -19,8 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved. */ /* @@ -434,7 +433,7 @@ resume(kthread_id_t t) add THREAD_REG, T_INTR_START, %o2 1: ldx [%o2], %o1 - RD_TICK(%o0,%o3,%g5,__LINE__) + RD_CLOCK_TICK(%o0,%o3,%g5,__LINE__) casx [%o2], %o1, %o0 cmp %o0, %o1 be,pt %xcc, 5b @@ -594,7 +593,7 @@ resume_from_intr(kthread_id_t t) add THREAD_REG, T_INTR_START, %o2 2: ldx [%o2], %o1 - RD_TICK(%o0,%o3,%l1,__LINE__) + RD_CLOCK_TICK(%o0,%o3,%l1,__LINE__) casx [%o2], %o1, %o0 cmp %o0, %o1 bne,pn %xcc, 2b diff --git a/usr/src/uts/sun4/os/machdep.c b/usr/src/uts/sun4/os/machdep.c index 07a1f185da..2136550797 100644 --- a/usr/src/uts/sun4/os/machdep.c +++ b/usr/src/uts/sun4/os/machdep.c @@ -19,8 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved. */ #include @@ -61,6 +60,7 @@ #include #include #include +#include int maxphys = MMU_PAGESIZE * 16; /* 128k */ int klustsize = MMU_PAGESIZE * 16; /* 128k */ @@ -549,7 +549,7 @@ cpu_intr_swtch_enter(kthread_id_t t) if (t->t_intr_start) { do { start = t->t_intr_start; - interval = gettick_counter() - start; + interval = CLOCK_TICK_COUNTER() - start; } while (cas64(&t->t_intr_start, start, 0) != start); cpu = CPU; if (cpu->cpu_m.divisor > 1) @@ -577,7 +577,7 @@ cpu_intr_swtch_exit(kthread_id_t t) do { ts = t->t_intr_start; - } while (cas64(&t->t_intr_start, ts, gettick_counter()) != ts); + } while (cas64(&t->t_intr_start, ts, CLOCK_TICK_COUNTER()) != ts); } diff --git a/usr/src/uts/sun4u/sys/machclock.h b/usr/src/uts/sun4u/sys/machclock.h index 0fe85cfefd..fd3f427c5f 100644 --- a/usr/src/uts/sun4u/sys/machclock.h +++ b/usr/src/uts/sun4u/sys/machclock.h @@ -19,8 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. */ #ifndef _SYS_MACHCLOCK_H @@ -48,6 +47,22 @@ extern "C" { #define RD_TICK(out, scr1, scr2, label) \ RD_TICK_NO_SUSPEND_CHECK(out, scr1); +/* + * These macros on sun4u read the %tick register, due to : + * - %stick does not have enough precision, it's very low frequency + * - %stick accesses are very slow on UltraSPARC IIe + * Instead, consumers read %tick and scale it by the current stick/tick ratio. + * This only works because all cpus in a system change clock ratios + * synchronously and the changes are all initiated by the kernel. + */ +#define RD_CLOCK_TICK(out, scr1, scr2, label) \ +/* CSTYLED */ \ + RD_TICK(out,scr1,scr2,label) + +#define RD_CLOCK_TICK_NO_SUSPEND_CHECK(out, scr1) \ +/* CSTYLED */ \ + RD_TICK_NO_SUSPEND_CHECK(out,scr1) + #endif /* _ASM */ #if defined(CPU_MODULE) @@ -173,6 +188,9 @@ struct tod_ops { extern struct tod_ops tod_ops; extern char *tod_module_name; +extern uint64_t gettick_counter(void); +#define CLOCK_TICK_COUNTER() gettick_counter() + /* * These defines allow common code to use TOD functions independant * of hardware platform. diff --git a/usr/src/uts/sun4v/ml/mach_subr_asm.s b/usr/src/uts/sun4v/ml/mach_subr_asm.s index 18d902b393..89830cb988 100644 --- a/usr/src/uts/sun4v/ml/mach_subr_asm.s +++ b/usr/src/uts/sun4v/ml/mach_subr_asm.s @@ -19,8 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. */ /* @@ -34,6 +33,8 @@ #include "assym.h" #endif /* lint */ +#define CPU_MODULE /* need it for NSEC_SHIFT used by NATIVE_TIME_TO_NSEC() */ + #include #include #include @@ -202,18 +203,10 @@ tick2ns(hrtime_t tick, uint_t cpuid) #else /* lint */ ENTRY_NP(tick2ns) - sethi %hi(cpunodes), %o4 - or %o4, %lo(cpunodes), %o4 ! %o4 = &cpunodes - ! Register usage: ! - ! o0 = timestamp - ! o2 = byte offset into cpunodes for tick_nsec_scale of this CPU - ! o4 = &cpunodes + ! Use nsec_scale for sun4v which is based on %stick ! - mulx %o1, CPU_NODE_SIZE, %o2 ! %o2 = byte offset into cpunodes - add %o2, TICK_NSEC_SCALE, %o2 - ld [%o4 + %o2], %o2 ! %o2 = cpunodes[cpuid].tick_nsec_scale - NATIVE_TIME_TO_NSEC_SCALE(%o0, %o2, %o3, TICK_NSEC_SHIFT) + NATIVE_TIME_TO_NSEC(%o0, %o2, %o3) retl nop SET_SIZE(tick2ns) diff --git a/usr/src/uts/sun4v/sys/machclock.h b/usr/src/uts/sun4v/sys/machclock.h index 3a8e8fd618..fa0e8be98c 100644 --- a/usr/src/uts/sun4v/sys/machclock.h +++ b/usr/src/uts/sun4v/sys/machclock.h @@ -19,8 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. */ #ifndef _SYS_MACHCLOCK_H @@ -66,6 +65,41 @@ extern "C" { srlx out, 1, out; \ add out, scr1, out +/* + * These macros on sun4v read the %stick register, because : + * + * For sun4v platforms %tick can change dynamically *without* kernel + * knowledge, due to SP side power & thermal management cases, + * which is triggered externally by SP and handled by Hypervisor. + * + * The frequency of %tick cannot be relied upon by kernel code, + * since it changes dynamically without the kernel being aware. + * So, always use the constant-frequency %stick on sun4v. + */ +#define RD_CLOCK_TICK(out, scr1, scr2, label) \ +/* CSTYLED */ \ + RD_STICK(out,scr1,scr2,label) + +#define RD_STICK_NO_SUSPEND_CHECK(out, scr1) \ + sethi %hi(native_stick_offset), scr1; \ + ldx [scr1 + %lo(native_stick_offset)], scr1; \ + rd STICK, out; \ + sllx out, 1, out; \ + srlx out, 1, out; \ + add out, scr1, out + +#define RD_CLOCK_TICK_NO_SUSPEND_CHECK(out, scr1) \ +/* CSTYLED */ \ + RD_STICK_NO_SUSPEND_CHECK(out,scr1) + +#ifndef _ASM +#ifdef _KERNEL +extern u_longlong_t gettick(void); +#define CLOCK_TICK_COUNTER() gettick() /* returns %stick */ +#endif /* _KERNEL */ +#endif /* _ASM */ + + #define RD_TICK(out, scr1, scr2, label) \ .rd_tick.label: \ sethi %hi(native_tick_offset), scr1; \ -- 2.11.4.GIT