1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 // Implement TimeStamp::Now() with QueryPerformanceCounter() controlled with
8 // values of GetTickCount64().
10 #include "mozilla/DynamicallyLinkedFunctionPtr.h"
11 #include "mozilla/MathAlgorithms.h"
12 #include "mozilla/TimeStamp.h"
13 #include "mozilla/Uptime.h"
20 // To enable logging define to your favorite logging API
23 class AutoCriticalSection
{
25 explicit AutoCriticalSection(LPCRITICAL_SECTION aSection
)
26 : mSection(aSection
) {
27 ::EnterCriticalSection(mSection
);
29 ~AutoCriticalSection() { ::LeaveCriticalSection(mSection
); }
32 LPCRITICAL_SECTION mSection
;
35 // Estimate of the smallest duration of time we can measure.
36 static volatile ULONGLONG sResolution
;
37 static volatile ULONGLONG sResolutionSigDigs
;
38 static const double kNsPerSecd
= 1000000000.0;
39 static const LONGLONG kNsPerMillisec
= 1000000;
41 // ----------------------------------------------------------------------------
43 // ----------------------------------------------------------------------------
45 // Tolerance to failures settings.
47 // What is the interval we want to have failure free.
49 static const uint32_t kFailureFreeInterval
= 5000;
50 // How many failures we are willing to tolerate in the interval.
51 static const uint32_t kMaxFailuresPerInterval
= 4;
52 // What is the threshold to treat fluctuations as actual failures.
54 static const uint32_t kFailureThreshold
= 50;
56 // If we are not able to get the value of GTC time increment, use this value
57 // which is the most usual increment.
58 static const DWORD kDefaultTimeIncrement
= 156001;
60 // ----------------------------------------------------------------------------
61 // Global variables, not changing at runtime
62 // ----------------------------------------------------------------------------
64 // Result of QueryPerformanceFrequency
65 // We use default of 1 for the case we can't use QueryPerformanceCounter
66 // to make mt/ms conversions work despite that.
67 static uint64_t sFrequencyPerSec
= 1;
71 MFBT_API
uint64_t GetQueryPerformanceFrequencyPerSec() {
72 return sFrequencyPerSec
;
75 } // namespace mozilla
77 // How much we are tolerant to GTC occasional loose of resoltion.
78 // This number says how many multiples of the minimal GTC resolution
79 // detected on the system are acceptable. This number is empirical.
80 static const LONGLONG kGTCTickLeapTolerance
= 4;
82 // Base tolerance (more: "inability of detection" range) threshold is calculated
83 // dynamically, and kept in sGTCResolutionThreshold.
85 // Schematically, QPC worked "100%" correctly if ((GTC_now - GTC_epoch) -
86 // (QPC_now - QPC_epoch)) was in [-sGTCResolutionThreshold,
87 // sGTCResolutionThreshold] interval every time we'd compared two time stamps.
88 // If not, then we check the overflow behind this basic threshold
89 // is in kFailureThreshold. If not, we condider it as a QPC failure. If too
90 // many failures in short time are detected, QPC is considered faulty and
94 static LONGLONG sGTCResolutionThreshold
;
96 // If QPC is found faulty for two stamps in this interval, we engage
97 // the fault detection algorithm. For duration larger then this limit
98 // we bypass using durations calculated from QPC when jitter is detected,
99 // but don't touch the sUseQPC flag.
102 static const uint32_t kHardFailureLimit
= 2000;
103 // Conversion to [mt]
104 static LONGLONG sHardFailureLimit
;
106 // Conversion of kFailureFreeInterval and kFailureThreshold to [mt]
107 static LONGLONG sFailureFreeInterval
;
108 static LONGLONG sFailureThreshold
;
110 // ----------------------------------------------------------------------------
111 // Systemm status flags
112 // ----------------------------------------------------------------------------
114 // Flag for stable TSC that indicates platform where QPC is stable.
115 static bool sHasStableTSC
= false;
117 // ----------------------------------------------------------------------------
118 // Global state variables, changing at runtime
119 // ----------------------------------------------------------------------------
121 // Initially true, set to false when QPC is found unstable and never
122 // returns back to true since that time.
123 static bool volatile sUseQPC
= true;
125 // ----------------------------------------------------------------------------
127 // ----------------------------------------------------------------------------
129 // Thread spin count before entering the full wait state for sTimeStampLock.
130 // Inspired by Rob Arnold's work on PRMJ_Now().
131 static const DWORD kLockSpinCount
= 4096;
133 // Common mutex (thanks the relative complexity of the logic, this is better
134 // then using CMPXCHG8B.)
135 // It is protecting the globals bellow.
136 static CRITICAL_SECTION sTimeStampLock
;
138 // ----------------------------------------------------------------------------
139 // Global lock protected variables
140 // ----------------------------------------------------------------------------
142 // Timestamp in future until QPC must behave correctly.
143 // Set to now + kFailureFreeInterval on first QPC failure detection.
144 // Set to now + E * kFailureFreeInterval on following errors,
145 // where E is number of errors detected during last kFailureFreeInterval
146 // milliseconds, calculated simply as:
147 // E = (sFaultIntoleranceCheckpoint - now) / kFailureFreeInterval + 1.
148 // When E > kMaxFailuresPerInterval -> disable QPC.
151 static ULONGLONG sFaultIntoleranceCheckpoint
= 0;
156 static inline ULONGLONG
PerformanceCounter() {
158 ::QueryPerformanceCounter(&pc
);
160 // QueryPerformanceCounter may slightly jitter (not be 100% monotonic.)
161 // This is a simple go-backward protection for such a faulty hardware.
162 AutoCriticalSection
lock(&sTimeStampLock
);
164 static decltype(LARGE_INTEGER::QuadPart
) last
;
165 if (last
> pc
.QuadPart
) {
166 return last
* 1000ULL;
169 return pc
.QuadPart
* 1000ULL;
172 static void InitThresholds() {
173 DWORD timeAdjustment
= 0, timeIncrement
= 0;
174 BOOL timeAdjustmentDisabled
;
175 GetSystemTimeAdjustment(&timeAdjustment
, &timeIncrement
,
176 &timeAdjustmentDisabled
);
178 LOG(("TimeStamp: timeIncrement=%d [100ns]", timeIncrement
));
180 if (!timeIncrement
) {
181 timeIncrement
= kDefaultTimeIncrement
;
184 // Ceiling to a millisecond
185 // Example values: 156001, 210000
186 DWORD timeIncrementCeil
= timeIncrement
;
187 // Don't want to round up if already rounded, values will be: 156000, 209999
188 timeIncrementCeil
-= 1;
189 // Convert to ms, values will be: 15, 20
190 timeIncrementCeil
/= 10000;
191 // Round up, values will be: 16, 21
192 timeIncrementCeil
+= 1;
193 // Convert back to 100ns, values will be: 160000, 210000
194 timeIncrementCeil
*= 10000;
196 // How many milli-ticks has the interval rounded up
197 LONGLONG ticksPerGetTickCountResolutionCeiling
=
198 (int64_t(timeIncrementCeil
) * sFrequencyPerSec
) / 10000LL;
200 // GTC may jump by 32 (2*16) ms in two steps, therefor use the ceiling value.
201 sGTCResolutionThreshold
=
202 LONGLONG(kGTCTickLeapTolerance
* ticksPerGetTickCountResolutionCeiling
);
204 sHardFailureLimit
= ms2mt(kHardFailureLimit
);
205 sFailureFreeInterval
= ms2mt(kFailureFreeInterval
);
206 sFailureThreshold
= ms2mt(kFailureThreshold
);
209 static void InitResolution() {
210 // 10 total trials is arbitrary: what we're trying to avoid by
211 // looping is getting unlucky and being interrupted by a context
212 // switch or signal, or being bitten by paging/cache effects
214 ULONGLONG minres
= ~0ULL;
218 ULONGLONG start
= PerformanceCounter();
219 ULONGLONG end
= PerformanceCounter();
221 ULONGLONG candidate
= (end
- start
);
222 if (candidate
< minres
) {
225 } while (--loops
&& minres
);
231 // GetTickCount has only ~16ms known resolution
235 // Converting minres that is in [mt] to nanosecods, multiplicating
236 // the argument to preserve resolution.
237 ULONGLONG result
= mt2ms(minres
* kNsPerMillisec
);
242 sResolution
= result
;
244 // find the number of significant digits in mResolution, for the
245 // sake of ToSecondsSigDigits()
247 for (sigDigs
= 1; !(sigDigs
== result
|| 10 * sigDigs
> result
);
251 sResolutionSigDigs
= sigDigs
;
254 // ----------------------------------------------------------------------------
255 // TimeStampValue implementation
256 // ----------------------------------------------------------------------------
257 MFBT_API TimeStampValue
& TimeStampValue::operator+=(const int64_t aOther
) {
263 MFBT_API TimeStampValue
& TimeStampValue::operator-=(const int64_t aOther
) {
269 // If the duration is less then two seconds, perform check of QPC stability
270 // by comparing both GTC and QPC calculated durations of this and aOther.
271 MFBT_API
uint64_t TimeStampValue::CheckQPC(const TimeStampValue
& aOther
) const {
272 uint64_t deltaGTC
= mGTC
- aOther
.mGTC
;
274 if (!mHasQPC
|| !aOther
.mHasQPC
) { // Both not holding QPC
278 uint64_t deltaQPC
= mQPC
- aOther
.mQPC
;
280 if (sHasStableTSC
) { // For stable TSC there is no need to check
284 // Check QPC is sane before using it.
285 int64_t diff
= DeprecatedAbs(int64_t(deltaQPC
) - int64_t(deltaGTC
));
286 if (diff
<= sGTCResolutionThreshold
) {
290 // Treat absolutely for calibration purposes
291 int64_t duration
= DeprecatedAbs(int64_t(deltaGTC
));
292 int64_t overflow
= diff
- sGTCResolutionThreshold
;
294 LOG(("TimeStamp: QPC check after %llums with overflow %1.4fms",
295 mt2ms(duration
), mt2ms_f(overflow
)));
297 if (overflow
<= sFailureThreshold
) { // We are in the limit, let go.
301 // QPC deviates, don't use it, since now this method may only return deltaGTC.
303 if (!sUseQPC
) { // QPC already disabled, no need to run the fault tolerance
308 LOG(("TimeStamp: QPC jittered over failure threshold"));
310 if (duration
< sHardFailureLimit
) {
311 // Interval between the two time stamps is very short, consider
312 // QPC as unstable and record a failure.
313 uint64_t now
= ms2mt(GetTickCount64());
315 AutoCriticalSection
lock(&sTimeStampLock
);
317 if (sFaultIntoleranceCheckpoint
&& sFaultIntoleranceCheckpoint
> now
) {
318 // There's already been an error in the last fault intollerant interval.
319 // Time since now to the checkpoint actually holds information on how many
320 // failures there were in the failure free interval we have defined.
321 uint64_t failureCount
=
322 (sFaultIntoleranceCheckpoint
- now
+ sFailureFreeInterval
- 1) /
323 sFailureFreeInterval
;
324 if (failureCount
> kMaxFailuresPerInterval
) {
326 LOG(("TimeStamp: QPC disabled"));
328 // Move the fault intolerance checkpoint more to the future, prolong it
329 // to reflect the number of detected failures.
331 sFaultIntoleranceCheckpoint
= now
+ failureCount
* sFailureFreeInterval
;
332 LOG(("TimeStamp: recording %dth QPC failure", failureCount
));
335 // Setup fault intolerance checkpoint in the future for first detected
337 sFaultIntoleranceCheckpoint
= now
+ sFailureFreeInterval
;
338 LOG(("TimeStamp: recording 1st QPC failure"));
346 TimeStampValue::operator-(const TimeStampValue
& aOther
) const {
347 if (IsNull() && aOther
.IsNull()) {
351 return CheckQPC(aOther
);
354 class TimeStampValueTests
{
355 // Check that nullity is set/not set correctly.
356 static_assert(TimeStampValue
{0}.IsNull());
357 static_assert(!TimeStampValue
{1}.IsNull());
359 // Check that we ignore GTC when both TimeStampValues have QPC. (In each of
360 // these tests, looking at GTC would give a different result.)
361 static_assert(TimeStampValue
{1, 2, true} < TimeStampValue
{1, 3, true});
362 static_assert(!(TimeStampValue
{1, 2, true} == TimeStampValue
{1, 3, true}));
364 static_assert(TimeStampValue
{2, 2, true} < TimeStampValue
{1, 3, true});
365 static_assert(TimeStampValue
{2, 2, true} <= TimeStampValue
{1, 3, true});
366 static_assert(!(TimeStampValue
{2, 2, true} > TimeStampValue
{1, 3, true}));
368 static_assert(TimeStampValue
{1, 3, true} > TimeStampValue
{1, 2, true});
369 static_assert(!(TimeStampValue
{1, 3, true} == TimeStampValue
{1, 2, true}));
371 static_assert(TimeStampValue
{1, 3, true} > TimeStampValue
{2, 2, true});
372 static_assert(TimeStampValue
{1, 3, true} >= TimeStampValue
{2, 2, true});
373 static_assert(!(TimeStampValue
{1, 3, true} < TimeStampValue
{2, 2, true}));
375 static_assert(TimeStampValue
{1, 3, true} == TimeStampValue
{2, 3, true});
376 static_assert(!(TimeStampValue
{1, 3, true} < TimeStampValue
{2, 3, true}));
378 static_assert(TimeStampValue
{1, 2, true} != TimeStampValue
{1, 3, true});
379 static_assert(!(TimeStampValue
{1, 2, true} == TimeStampValue
{1, 3, true}));
381 // Check that, if either TimeStampValue doesn't have QPC, we only look at the
382 // GTC values. These are the same cases as above, except that we accept the
383 // opposite results because we turn off QPC on one or both of the
385 static_assert(TimeStampValue
{1, 2, false} == TimeStampValue
{1, 3, true});
386 static_assert(TimeStampValue
{1, 2, true} == TimeStampValue
{1, 3, false});
387 static_assert(TimeStampValue
{1, 2, false} == TimeStampValue
{1, 3, false});
389 static_assert(TimeStampValue
{2, 2, false} > TimeStampValue
{1, 3, true});
390 static_assert(TimeStampValue
{2, 2, true} > TimeStampValue
{1, 3, false});
391 static_assert(TimeStampValue
{2, 2, false} > TimeStampValue
{1, 3, false});
393 static_assert(TimeStampValue
{1, 3, false} == TimeStampValue
{1, 2, true});
394 static_assert(TimeStampValue
{1, 3, true} == TimeStampValue
{1, 2, false});
395 static_assert(TimeStampValue
{1, 3, false} == TimeStampValue
{1, 2, false});
397 static_assert(TimeStampValue
{1, 3, false} < TimeStampValue
{2, 2, true});
398 static_assert(TimeStampValue
{1, 3, true} < TimeStampValue
{2, 2, false});
399 static_assert(TimeStampValue
{1, 3, false} < TimeStampValue
{2, 2, false});
401 static_assert(TimeStampValue
{1, 3, false} < TimeStampValue
{2, 3, true});
402 static_assert(TimeStampValue
{1, 3, true} < TimeStampValue
{2, 3, false});
403 static_assert(TimeStampValue
{1, 3, false} < TimeStampValue
{2, 3, false});
405 static_assert(TimeStampValue
{1, 2, false} == TimeStampValue
{1, 3, true});
406 static_assert(TimeStampValue
{1, 2, true} == TimeStampValue
{1, 3, false});
407 static_assert(TimeStampValue
{1, 2, false} == TimeStampValue
{1, 3, false});
410 // ----------------------------------------------------------------------------
411 // TimeDuration and TimeStamp implementation
412 // ----------------------------------------------------------------------------
414 MFBT_API
double BaseTimeDurationPlatformUtils::ToSeconds(int64_t aTicks
) {
415 // Converting before arithmetic avoids blocked store forward
416 return double(aTicks
) / (double(sFrequencyPerSec
) * 1000.0);
419 MFBT_API
double BaseTimeDurationPlatformUtils::ToSecondsSigDigits(
421 // don't report a value < mResolution ...
422 LONGLONG resolution
= sResolution
;
423 LONGLONG resolutionSigDigs
= sResolutionSigDigs
;
424 LONGLONG valueSigDigs
= resolution
* (aTicks
/ resolution
);
425 // and chop off insignificant digits
426 valueSigDigs
= resolutionSigDigs
* (valueSigDigs
/ resolutionSigDigs
);
427 return double(valueSigDigs
) / kNsPerSecd
;
431 BaseTimeDurationPlatformUtils::TicksFromMilliseconds(double aMilliseconds
) {
432 double result
= ms2mt(aMilliseconds
);
433 if (result
> double(INT64_MAX
)) {
435 } else if (result
< double(INT64_MIN
)) {
442 MFBT_API
int64_t BaseTimeDurationPlatformUtils::ResolutionInTicks() {
443 return static_cast<int64_t>(sResolution
);
446 static bool HasStableTSC() {
447 #if defined(_M_ARM64)
448 // AArch64 defines that its system counter run at a constant rate
449 // regardless of the current clock frequency of the system. See "The
450 // Generic Timer", section D7, in the ARMARM for ARMv8.
461 __cpuid(cpuInfo
.regs
, 0);
462 // Only allow Intel or AMD CPUs for now.
463 // The order of the registers is reg[1], reg[3], reg[2]. We just adjust the
464 // string so that we can compare in one go.
465 if (_strnicmp(cpuInfo
.cpuString
, "GenuntelineI", sizeof(cpuInfo
.cpuString
)) &&
466 _strnicmp(cpuInfo
.cpuString
, "AuthcAMDenti", sizeof(cpuInfo
.cpuString
))) {
472 // detect if the Advanced Power Management feature is supported
473 __cpuid(regs
, 0x80000000);
474 if ((unsigned int)regs
[0] < 0x80000007) {
475 // XXX should we return true here? If there is no APM there may be
476 // no way how TSC can run out of sync among cores.
480 __cpuid(regs
, 0x80000007);
481 // if bit 8 is set than TSC will run at a constant rate
482 // in all ACPI P-states, C-states and T-states
483 return regs
[3] & (1 << 8);
487 static bool gInitialized
= false;
489 MFBT_API
void TimeStamp::Startup() {
496 // Decide which implementation to use for the high-performance timer.
498 InitializeCriticalSectionAndSpinCount(&sTimeStampLock
, kLockSpinCount
);
500 bool forceGTC
= false;
501 bool forceQPC
= false;
503 char* modevar
= getenv("MOZ_TIMESTAMP_MODE");
505 if (!strcmp(modevar
, "QPC")) {
507 } else if (!strcmp(modevar
, "GTC")) {
513 sUseQPC
= !forceGTC
&& ::QueryPerformanceFrequency(&freq
);
515 // No Performance Counter. Fall back to use GetTickCount64.
518 LOG(("TimeStamp: using GetTickCount64"));
522 sHasStableTSC
= forceQPC
|| HasStableTSC();
523 LOG(("TimeStamp: HasStableTSC=%d", sHasStableTSC
));
525 sFrequencyPerSec
= freq
.QuadPart
;
526 LOG(("TimeStamp: QPC frequency=%llu", sFrequencyPerSec
));
534 MFBT_API
void TimeStamp::Shutdown() { DeleteCriticalSection(&sTimeStampLock
); }
536 TimeStampValue
NowInternal(bool aHighResolution
) {
537 // sUseQPC is volatile
538 bool useQPC
= (aHighResolution
&& sUseQPC
);
540 // Both values are in [mt] units.
541 ULONGLONG QPC
= useQPC
? PerformanceCounter() : uint64_t(0);
542 ULONGLONG GTC
= ms2mt(GetTickCount64());
543 return TimeStampValue(GTC
, QPC
, useQPC
);
546 MFBT_API TimeStamp
TimeStamp::Now(bool aHighResolution
) {
547 return TimeStamp(NowInternal(aHighResolution
));
550 // Computes and returns the process uptime in microseconds.
551 // Returns 0 if an error was encountered.
553 MFBT_API
uint64_t TimeStamp::ComputeProcessUptime() {
554 FILETIME start
, foo
, bar
, baz
;
555 bool success
= GetProcessTimes(GetCurrentProcess(), &start
, &foo
, &bar
, &baz
);
560 static const StaticDynamicallyLinkedFunctionPtr
<void(WINAPI
*)(LPFILETIME
)>
561 pGetSystemTimePreciseAsFileTime(L
"kernel32.dll",
562 "GetSystemTimePreciseAsFileTime");
565 if (pGetSystemTimePreciseAsFileTime
) {
566 pGetSystemTimePreciseAsFileTime(&now
);
568 GetSystemTimeAsFileTime(&now
);
571 ULARGE_INTEGER startUsec
= {{start
.dwLowDateTime
, start
.dwHighDateTime
}};
572 ULARGE_INTEGER nowUsec
= {{now
.dwLowDateTime
, now
.dwHighDateTime
}};
574 return (nowUsec
.QuadPart
- startUsec
.QuadPart
) / 10ULL;
577 } // namespace mozilla