arch/i386/kernel/tsc.c

   1 /*
   2  * This code largely moved from arch/i386/kernel/timer/timer_tsc.c
   3  * which was originally moved from arch/i386/kernel/time.c.
   4  * See comments there for proper credits.
   5  */
   6
   7 #include <linux/clocksource.h>
   8 #include <linux/workqueue.h>
   9 #include <linux/cpufreq.h>
  10 #include <linux/jiffies.h>
  11 #include <linux/init.h>
  12 #include <linux/dmi.h>
  13
  14 #include <asm/delay.h>
  15 #include <asm/tsc.h>
  16 #include <asm/io.h>
  17
  18 #include "mach_timer.h"
  19
  20 /*
  21  * On some systems the TSC frequency does not
  22  * change with the cpu frequency. So we need
  23  * an extra value to store the TSC freq
  24  */
  25 unsigned int tsc_khz;
  26 unsigned long long (*custom_sched_clock)(void);
  27
  28 int tsc_disable;
  29
  30 #ifdef CONFIG_X86_TSC
  31 static int __init tsc_setup(char *str)
  32 {
  33         printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
  34                                 "cannot disable TSC.\n");
  35         return 1;
  36 }
  37 #else
  38 /*
  39  * disable flag for tsc. Takes effect by clearing the TSC cpu flag
  40  * in cpu/common.c
  41  */
  42 static int __init tsc_setup(char *str)
  43 {
  44         tsc_disable = 1;
  45
  46         return 1;
  47 }
  48 #endif
  49
  50 __setup("notsc", tsc_setup);
  51
  52 /*
  53  * code to mark and check if the TSC is unstable
  54  * due to cpufreq or due to unsynced TSCs
  55  */
  56 static int tsc_unstable;
  57
  58 static inline int check_tsc_unstable(void)
  59 {
  60         return tsc_unstable;
  61 }
  62
  63 /* Accellerators for sched_clock()
  64  * convert from cycles(64bits) => nanoseconds (64bits)
  65  *  basic equation:
  66  *              ns = cycles / (freq / ns_per_sec)
  67  *              ns = cycles * (ns_per_sec / freq)
  68  *              ns = cycles * (10^9 / (cpu_khz * 10^3))
  69  *              ns = cycles * (10^6 / cpu_khz)
  70  *
  71  *      Then we use scaling math (suggested by george@mvista.com) to get:
  72  *              ns = cycles * (10^6 * SC / cpu_khz) / SC
  73  *              ns = cycles * cyc2ns_scale / SC
  74  *
  75  *      And since SC is a constant power of two, we can convert the div
  76  *  into a shift.
  77  *
  78  *  We can use khz divisor instead of mhz to keep a better percision, since
  79  *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
  80  *  (mathieu.desnoyers@polymtl.ca)
  81  *
  82  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
  83  */
  84 static unsigned long cyc2ns_scale __read_mostly;
  85
  86 #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
  87
  88 static inline void set_cyc2ns_scale(unsigned long cpu_khz)
  89 {
  90         cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;
  91 }
  92
  93 static inline unsigned long long cycles_2_ns(unsigned long long cyc)
  94 {
  95         return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
  96 }
  97
  98 /*
  99  * Scheduler clock - returns current time in nanosec units.
 100  */
 101 unsigned long long sched_clock(void)
 102 {
 103         unsigned long long this_offset;
 104
 105         if (unlikely(custom_sched_clock))
 106                 return (*custom_sched_clock)();
 107
 108         /*
 109          * Fall back to jiffies if there's no TSC available:
 110          */
 111         if (unlikely(tsc_disable))
 112                 /* No locking but a rare wrong value is not a big deal: */
 113                 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
 114
 115         /* read the Time Stamp Counter: */
 116         rdtscll(this_offset);
 117
 118         /* return the value in ns */
 119         return cycles_2_ns(this_offset);
 120 }
 121
 122 static unsigned long calculate_cpu_khz(void)
 123 {
 124         unsigned long long start, end;
 125         unsigned long count;
 126         u64 delta64;
 127         int i;
 128         unsigned long flags;
 129
 130         local_irq_save(flags);
 131
 132         /* run 3 times to ensure the cache is warm */
 133         for (i = 0; i < 3; i++) {
 134                 mach_prepare_counter();
 135                 rdtscll(start);
 136                 mach_countup(&count);
 137                 rdtscll(end);
 138         }
 139         /*
 140          * Error: ECTCNEVERSET
 141          * The CTC wasn't reliable: we got a hit on the very first read,
 142          * or the CPU was so fast/slow that the quotient wouldn't fit in
 143          * 32 bits..
 144          */
 145         if (count <= 1)
 146                 goto err;
 147
 148         delta64 = end - start;
 149
 150         /* cpu freq too fast: */
 151         if (delta64 > (1ULL<<32))
 152                 goto err;
 153
 154         /* cpu freq too slow: */
 155         if (delta64 <= CALIBRATE_TIME_MSEC)
 156                 goto err;
 157
 158         delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
 159         do_div(delta64,CALIBRATE_TIME_MSEC);
 160
 161         local_irq_restore(flags);
 162         return (unsigned long)delta64;
 163 err:
 164         local_irq_restore(flags);
 165         return 0;
 166 }
 167
 168 int recalibrate_cpu_khz(void)
 169 {
 170 #ifndef CONFIG_SMP
 171         unsigned long cpu_khz_old = cpu_khz;
 172
 173         if (cpu_has_tsc) {
 174                 cpu_khz = calculate_cpu_khz();
 175                 tsc_khz = cpu_khz;
 176                 cpu_data[0].loops_per_jiffy =
 177                         cpufreq_scale(cpu_data[0].loops_per_jiffy,
 178                                         cpu_khz_old, cpu_khz);
 179                 return 0;
 180         } else
 181                 return -ENODEV;
 182 #else
 183         return -ENODEV;
 184 #endif
 185 }
 186
 187 EXPORT_SYMBOL(recalibrate_cpu_khz);
 188
 189 void __init tsc_init(void)
 190 {
 191         if (!cpu_has_tsc || tsc_disable)
 192                 goto out_no_tsc;
 193
 194         cpu_khz = calculate_cpu_khz();
 195         tsc_khz = cpu_khz;
 196
 197         if (!cpu_khz)
 198                 goto out_no_tsc;
 199
 200         printk("Detected %lu.%03lu MHz processor.\n",
 201                                 (unsigned long)cpu_khz / 1000,
 202                                 (unsigned long)cpu_khz % 1000);
 203
 204         set_cyc2ns_scale(cpu_khz);
 205         use_tsc_delay();
 206         return;
 207
 208 out_no_tsc:
 209         /*
 210          * Set the tsc_disable flag if there's no TSC support, this
 211          * makes it a fast flag for the kernel to see whether it
 212          * should be using the TSC.
 213          */
 214         tsc_disable = 1;
 215 }
 216
 217 #ifdef CONFIG_CPU_FREQ
 218
 219 /*
 220  * if the CPU frequency is scaled, TSC-based delays will need a different
 221  * loops_per_jiffy value to function properly.
 222  */
 223 static unsigned int ref_freq = 0;
 224 static unsigned long loops_per_jiffy_ref = 0;
 225 static unsigned long cpu_khz_ref = 0;
 226
 227 static int
 228 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
 229 {
 230         struct cpufreq_freqs *freq = data;
 231
 232         if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)
 233                 write_seqlock_irq(&xtime_lock);
 234
 235         if (!ref_freq) {
 236                 if (!freq->old){
 237                         ref_freq = freq->new;
 238                         goto end;
 239                 }
 240                 ref_freq = freq->old;
 241                 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
 242                 cpu_khz_ref = cpu_khz;
 243         }
 244
 245         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
 246             (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
 247             (val == CPUFREQ_RESUMECHANGE)) {
 248                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
 249                         cpu_data[freq->cpu].loops_per_jiffy =
 250                                 cpufreq_scale(loops_per_jiffy_ref,
 251                                                 ref_freq, freq->new);
 252
 253                 if (cpu_khz) {
 254
 255                         if (num_online_cpus() == 1)
 256                                 cpu_khz = cpufreq_scale(cpu_khz_ref,
 257                                                 ref_freq, freq->new);
 258                         if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
 259                                 tsc_khz = cpu_khz;
 260                                 set_cyc2ns_scale(cpu_khz);
 261                                 /*
 262                                  * TSC based sched_clock turns
 263                                  * to junk w/ cpufreq
 264                                  */
 265                                 mark_tsc_unstable();
 266                         }
 267                 }
 268         }
 269 end:
 270         if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)
 271                 write_sequnlock_irq(&xtime_lock);
 272
 273         return 0;
 274 }
 275
 276 static struct notifier_block time_cpufreq_notifier_block = {
 277         .notifier_call  = time_cpufreq_notifier
 278 };
 279
 280 static int __init cpufreq_tsc(void)
 281 {
 282         return cpufreq_register_notifier(&time_cpufreq_notifier_block,
 283                                          CPUFREQ_TRANSITION_NOTIFIER);
 284 }
 285 core_initcall(cpufreq_tsc);
 286
 287 #endif
 288
 289 /* clock source code */
 290
 291 static unsigned long current_tsc_khz = 0;
 292
 293 static cycle_t read_tsc(void)
 294 {
 295         cycle_t ret;
 296
 297         rdtscll(ret);
 298
 299         return ret;
 300 }
 301
 302 static struct clocksource clocksource_tsc = {
 303         .name                   = "tsc",
 304         .rating                 = 300,
 305         .read                   = read_tsc,
 306         .mask                   = CLOCKSOURCE_MASK(64),
 307         .mult                   = 0, /* to be set */
 308         .shift                  = 22,
 309         .flags                  = CLOCK_SOURCE_IS_CONTINUOUS |
 310                                   CLOCK_SOURCE_MUST_VERIFY,
 311 };
 312
 313 void mark_tsc_unstable(void)
 314 {
 315         if (!tsc_unstable) {
 316                 tsc_unstable = 1;
 317                 /* Can be called before registration */
 318                 if (clocksource_tsc.mult)
 319                         clocksource_change_rating(&clocksource_tsc, 0);
 320                 else
 321                         clocksource_tsc.rating = 0;
 322         }
 323 }
 324 EXPORT_SYMBOL_GPL(mark_tsc_unstable);
 325
 326 static int __init dmi_mark_tsc_unstable(struct dmi_system_id *d)
 327 {
 328         printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
 329                        d->ident);
 330         tsc_unstable = 1;
 331         return 0;
 332 }
 333
 334 /* List of systems that have known TSC problems */
 335 static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
 336         {
 337          .callback = dmi_mark_tsc_unstable,
 338          .ident = "IBM Thinkpad 380XD",
 339          .matches = {
 340                      DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
 341                      DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
 342                      },
 343          },
 344          {}
 345 };
 346
 347 #define TSC_FREQ_CHECK_INTERVAL (10*MSEC_PER_SEC) /* 10sec in MS */
 348 static struct timer_list verify_tsc_freq_timer;
 349
 350 /* XXX - Probably should add locking */
 351 static void verify_tsc_freq(unsigned long unused)
 352 {
 353         static u64 last_tsc;
 354         static unsigned long last_jiffies;
 355
 356         u64 now_tsc, interval_tsc;
 357         unsigned long now_jiffies, interval_jiffies;
 358
 359
 360         if (check_tsc_unstable())
 361                 return;
 362
 363         rdtscll(now_tsc);
 364         now_jiffies = jiffies;
 365
 366         if (!last_jiffies) {
 367                 goto out;
 368         }
 369
 370         interval_jiffies = now_jiffies - last_jiffies;
 371         interval_tsc = now_tsc - last_tsc;
 372         interval_tsc *= HZ;
 373         do_div(interval_tsc, cpu_khz*1000);
 374
 375         if (interval_tsc < (interval_jiffies * 3 / 4)) {
 376                 printk("TSC appears to be running slowly. "
 377                         "Marking it as unstable\n");
 378                 mark_tsc_unstable();
 379                 return;
 380         }
 381
 382 out:
 383         last_tsc = now_tsc;
 384         last_jiffies = now_jiffies;
 385         /* set us up to go off on the next interval: */
 386         mod_timer(&verify_tsc_freq_timer,
 387                 jiffies + msecs_to_jiffies(TSC_FREQ_CHECK_INTERVAL));
 388 }
 389
 390 /*
 391  * Make an educated guess if the TSC is trustworthy and synchronized
 392  * over all CPUs.
 393  */
 394 __cpuinit int unsynchronized_tsc(void)
 395 {
 396         if (!cpu_has_tsc || tsc_unstable)
 397                 return 1;
 398         /*
 399          * Intel systems are normally all synchronized.
 400          * Exceptions must mark TSC as unstable:
 401          */
 402         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
 403                 /* assume multi socket systems are not synchronized: */
 404                 if (num_possible_cpus() > 1)
 405                         tsc_unstable = 1;
 406         }
 407         return tsc_unstable;
 408 }
 409
 410 static int __init init_tsc_clocksource(void)
 411 {
 412
 413         if (cpu_has_tsc && tsc_khz && !tsc_disable) {
 414                 /* check blacklist */
 415                 dmi_check_system(bad_tsc_dmi_table);
 416
 417                 unsynchronized_tsc();
 418                 current_tsc_khz = tsc_khz;
 419                 clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
 420                                                         clocksource_tsc.shift);
 421                 /* lower the rating if we already know its unstable: */
 422                 if (check_tsc_unstable()) {
 423                         clocksource_tsc.rating = 0;
 424                         clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
 425                 }
 426
 427                 init_timer(&verify_tsc_freq_timer);
 428                 verify_tsc_freq_timer.function = verify_tsc_freq;
 429                 verify_tsc_freq_timer.expires =
 430                         jiffies + msecs_to_jiffies(TSC_FREQ_CHECK_INTERVAL);
 431                 add_timer(&verify_tsc_freq_timer);
 432
 433                 return clocksource_register(&clocksource_tsc);
 434         }
 435
 436         return 0;
 437 }
 438
 439 module_init(init_tsc_clocksource);