/******************************************************************************
* arch/x86/time.c
*
* Per-CPU time calibration and management.
*
* Copyright (c) 2002-2005, K A Fraser
*
* Portions from Linux are:
* Copyright (c) 1991, 1992, 1995 Linus Torvalds
*/
#include <xen/config.h>
#include <xen/errno.h>
#include <xen/event.h>
#include <xen/sched.h>
#include <xen/lib.h>
#include <xen/config.h>
#include <xen/init.h>
#include <xen/time.h>
#include <xen/timer.h>
#include <xen/smp.h>
#include <xen/irq.h>
#include <xen/softirq.h>
#include <xen/efi.h>
#include <xen/cpuidle.h>
#include <xen/symbols.h>
#include <xen/keyhandler.h>
#include <xen/guest_access.h>
#include <asm/io.h>
#include <asm/msr.h>
#include <asm/mpspec.h>
#include <asm/processor.h>
#include <asm/fixmap.h>
#include <asm/mc146818rtc.h>
#include <asm/div64.h>
#include <asm/acpi.h>
#include <asm/hpet.h>
#include <io_ports.h>
#include <asm/setup.h> /* for early_time_init */
#include <public/arch-x86/cpuid.h>
/* opt_clocksource: Force clocksource to one of: pit, hpet, cyclone, acpi. */
static char __initdata opt_clocksource[10];
string_param("clocksource", opt_clocksource);
unsigned long __read_mostly cpu_khz; /* CPU clock frequency in kHz. */
DEFINE_SPINLOCK(rtc_lock);
unsigned long pit0_ticks;
static u32 wc_sec, wc_nsec; /* UTC time at last 'time update'. */
static DEFINE_SPINLOCK(wc_lock);
struct cpu_time {
u64 local_tsc_stamp;
s_time_t stime_local_stamp;
s_time_t stime_master_stamp;
struct time_scale tsc_scale;
};
struct platform_timesource {
char *id;
char *name;
u64 frequency;
u64 (*read_counter)(void);
int (*init)(struct platform_timesource *);
void (*resume)(struct platform_timesource *);
int counter_bits;
};
static DEFINE_PER_CPU(struct cpu_time, cpu_time);
/* Calibrate all CPUs to platform timer every EPOCH. */
#define EPOCH MILLISECS(1000)
static struct timer calibration_timer;
/*
* We simulate a 32-bit platform timer from the 16-bit PIT ch2 counter.
* Otherwise overflow happens too quickly (~50ms) for us to guarantee that
* softirq handling will happen in time.
*
* The pit_lock protects the 16- and 32-bit stamp fields as well as the
*/
static DEFINE_SPINLOCK(pit_lock);
static u16 pit_stamp16;
static u32 pit_stamp32;
static bool_t __read_mostly using_pit;
/*
* 32-bit division of integer dividend and integer divisor yielding
* 32-bit fractional quotient.
*/
static inline u32 div_frac(u32 dividend, u32 divisor)
{
u32 quotient, remainder;
ASSERT(dividend < divisor);
asm (
"divl %4"
: "=a" (quotient), "=d" (remainder)
: "0" (0), "1" (dividend), "r" (divisor) );
return quotient;
}
/*
* 32-bit multiplication of multiplicand and fractional multiplier
* yielding 32-bit product (radix point at same position as in multiplicand).
*/
static inline u32 mul_frac(u32 multiplicand, u32 multiplier)
{
u32 product_int, product_frac;
asm (
"mul %3"
: "=a" (product_frac), "=d" (product_int)
: "0" (multiplicand), "r" (multiplier) );
return product_int;
}
/*
* Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction,
* yielding a 64-bit result.
*/
static inline u64 scale_delta(u64 delta, struct time_scale *scale)
{
u64 product;
if ( scale->shift < 0 )
delta >>= -scale->shift;
else
delta <<= scale->shift;
asm (
"mul %%rdx ; shrd $32,%%rdx,%%rax"
: "=a" (product) : "0" (delta), "d" ((u64)scale->mul_frac) );
return product;
}
#define _TS_MUL_FRAC_IDENTITY 0x80000000UL
/* Compute the reciprocal of the given time_scale. */
static inline struct time_scale scale_reciprocal(struct time_scale scale)
{
struct time_scale reciprocal;
u32 dividend;
ASSERT(scale.mul_frac != 0);
dividend = _TS_MUL_FRAC_IDENTITY;
reciprocal.shift = 1 - scale.shift;
while ( unlikely(dividend >= scale.mul_frac) )
{
dividend >>= 1;
reciprocal.shift++;
}
asm (
"divl %4"
: "=a" (reciprocal.mul_frac), "=d" (dividend)
: "0" (0), "1" (dividend), "r" (scale.mul_frac) );
return reciprocal;
}
/*
* cpu_mask that denotes the CPUs that needs timer interrupt coming in as
* IPIs in place of local APIC timers
*/
static cpumask_t pit_broadcast_mask;
static void smp_send_timer_broadcast_ipi(void)
{
int cpu = smp_processor_id();
cpumask_t mask;
cpumask_and(&mask, &cpu_online_map, &pit_broadcast_mask);
if ( cpumask_test_cpu(cpu, &mask) )
{
cpumask_clear_cpu(cpu, &mask);
raise_softirq(TIMER_SOFTIRQ);
}
if ( !cpumask_empty(&mask) )
{
cpumask_raise_softirq(&mask, TIMER_SOFTIRQ);
}
}
static void timer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
ASSERT(local_irq_is_enabled());
if ( hpet_legacy_irq_tick() )
return;
/* Only for start-of-day interruopt tests in io_apic.c. */
(*(volatile unsigned long *)&pit0_ticks)++;
/* Rough hack to allow accurate timers to sort-of-work with no APIC. */
if ( !cpu_has_apic )
raise_softirq(TIMER_SOFTIRQ);
if ( xen_cpuidle )
smp_send_timer_broadcast_ipi();
/* Emulate a 32-bit PIT counter. */
if ( using_pit )
{
u16 count;
spin_lock_irq(&pit_lock);
outb(0x80, PIT_MODE);
count = inb(PIT_CH2);
count |= inb(PIT_CH2) << 8;
pit_stamp32 += (u16)(pit_stamp16 - count);
pit_stamp16 = count;
spin_unlock_irq(&pit_lock);
}
}
static struct irqaction __read_mostly irq0 = { timer_interrupt, "timer", NULL };
/* ------ Calibrate the TSC -------
* Return processor ticks per second / CALIBRATE_FRAC.
*/
#define CLOCK_TICK_RATE 1193182 /* system crystal frequency (Hz) */
#define CALIBRATE_FRAC 20 /* calibrate over 50ms */
#define CALIBRATE_LATCH ((CLOCK_TICK_RATE+(CALIBRATE_FRAC/2))/CALIBRATE_FRAC)
static u64 init_pit_and_calibrate_tsc(void)
{
u64 start, end;
unsigned long count;
/* Set PIT channel 0 to HZ Hz. */
#define LATCH (((CLOCK_TICK_RATE)+(HZ/2))/HZ)
outb_p(0x34, PIT_MODE); /* binary, mode 2, LSB/MSB, ch 0 */
outb_p(LATCH & 0xff, PIT_CH0); /* LSB */
outb(LATCH >> 8, PIT_CH0); /* MSB */
/* Set the Gate high, disable speaker */
outb((inb(0x61) & ~0x02) | 0x01, 0x61);
/*
* Now let's take care of CTC channel 2
*
* Set the Gate high, program CTC channel 2 for mode 0, (interrupt on
* terminal count mode), binary count, load 5 * LATCH count, (LSB and MSB)
* to begin countdown.
*/
outb(0xb0, PIT_MODE); /* binary, mode 0, LSB/MSB, Ch 2 */
outb(CALIBRATE_LATCH & 0xff, PIT_CH2); /* LSB of count */
outb(CALIBRATE_LATCH >> 8, PIT_CH2); /* MSB of count */
rdtscll(start);
for ( count = 0; (inb(0x61) & 0x20) == 0; count++ )
continue;
rdtscll(end);
/* Error if the CTC doesn't behave itself. */
if ( count == 0 )
return 0;
return ((end - start) * (u64)CALIBRATE_FRAC);
}
static void set_time_scale(struct time_scale *ts, u64 ticks_per_sec)