4.3 时钟虚拟化

kvm支持的时钟有8254,local apic timer,kvmclock等，本节将分析8254和kvmclock
4.3.1 8254时钟虚拟化
structkvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags)
{
a. 建立内核线程
pit->worker_task = kthread_run(kthread_worker_fn,&pit->worker,
"kvm-pit/%d", pid_nr);
b. 准备workqueue
init_kthread_work(&pit->expired,pit_do_work);
c. 始化一个高精准定时器，这个定时器就作为我们虚拟时钟的时钟源
hrtimer_init(&pit_state->timer,CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
d. pit_state->irq_ack_notifier.irq_acked= kvm_pit_ack_irq;时钟中断模拟ack
.kvm_register_irq_mask_notifier(kvm, 0,&pit->mask_notifier);
e. kvm_pit_reset(pit);
f. kvm_iodevice_init(&pit->dev,&pit_dev_ops); //注册io虚拟化操作
ret = kvm_io_bus_register_dev(kvm,KVM_PIO_BUS, KVM_PIT_BASE_ADDRESS,
KVM_PIT_MEM_LENGTH, &pit->dev);
}

当guestos需要通过寄存器操作启动一个时钟控制器时，pit_load_count==》create_pit_timer， 会启动一个hr timer来模拟时钟中断源

hrtimer_cancel(&ps->timer);
flush_kthread_work(&ps->pit->expired);
ps->period = interval;
ps->is_periodic = is_period;
ps->timer.function = pit_timer_fn;
ps->kvm = ps->pit->kvm;
atomic_set(&ps->pending, 0);
ps->irq_ack = 1;
.......
hrtimer_start(&ps->timer,ktime_add_ns(ktime_get(), interval),
HRTIMER_MODE_ABS);

pit_tiemr_fn完成时钟的累加
staticenum hrtimer_restart pit_timer_fn(struct hrtimer *data)
{
struct kvm_kpit_state *ps =container_of(data, struct kvm_kpit_state, timer);
struct kvm_pit *pt =ps->kvm->arch.vpit;

//如果时钟中断需要重新注入，就直接累加;否则那么不进行累加，直接合并时钟中断
if (ps->reinject ||!atomic_read(&ps->pending)) {
atomic_inc(&ps->pending);
queue_kthread_work(&pt->worker,&pt->expired);
}

if (ps->is_periodic) {
// 如果定时器周期触发，则再次启动定时器，否则销毁
hrtimer_add_expires_ns(&ps->timer,ps->period);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}

当定时器将时钟中断pending增加，并且添加完工作队列以后，接着就触发下面的时钟中断注入，如果上一个中断被接收，接着触发下一个。代码如下：
static void pit_do_work(struct kthread_work *work)
{
.......
spin_lock(&ps->inject_lock);
if (ps->irq_ack){
ps->irq_ack= 0;
inject = 1;
}
spin_unlock(&ps->inject_lock);
if (inject) {
///*模拟一个高电平和一个低电平，发送给PIC，触发时钟中断。*/
kvm_set_irq(kvm,kvm->arch.vpit->irq_source_id, 0, 1, false);
kvm_set_irq(kvm,kvm->arch.vpit->irq_source_id, 0, 0, false);

//nmi watchdog support

if(kvm->arch.vapics_in_nmi_mode > 0)
kvm_for_each_vcpu(i,vcpu, kvm)
kvm_apic_nmi_wd_deliver(vcpu);
}
}

kvm_pit_ack_irq实现中断的ack应答虚拟化
staticvoid kvm_pit_ack_irq(struct kvm_irq_ack_notifier *kian)
{
struct kvm_kpit_state *ps =container_of(kian, struct kvm_kpit_state,
irq_ack_notifier);
int value;

spin_lock(&ps->inject_lock);
value =atomic_dec_return(&ps->pending); //注入成功，则中断累加器减一
if (value < 0) // 异常情况，pending本来就是0还减少，说明是无效的ack
atomic_inc(&ps->pending);
else if (value > 0) / > 0还需要重新注入积累的中断
queue_kthread_work(&ps->pit->worker,&ps->pit->expired);
ps->irq_ack = 1;//设置ack位
spin_unlock(&ps->inject_lock);
}

4.3.2 kvmclock时钟虚拟化
时间虚拟化的一种实现方式是通过时钟中断计数，进而换算得到，这种方式在虚拟机里存在问题，因为有时运行vpcu的cpu被调度出来使 时钟中断不能准时到达guest os。另外一种方式，如模拟HPET，guest
os当需要的时候会去读当前的时间，这种方式会使得虚拟机频繁的VM-exit，影响性能。为此kvm引入了基于半虚拟化的时钟kvmclock,这种方式需要在guest上实现一个kvmclock驱动， 建立guest os 到VMM的通道， 这样通过这个通道guest os 向vmm 查询时间。

(1) guest os kvmclock 驱动
源码路径： arch\x86\kernel\ kvmclock.cpvclock.c.
kvmclock_init ==>
a. kvm_register_clock
src =&hv_clock[cpu].pvti;
low =(int)slow_virt_to_phys(src) | 1;
high =((u64)slow_virt_to_phys(src) >> 32);
ret =native_write_msr_safe(msr_kvm_system_time, low, high);
通过msr寄存write的方式将hv_clock[cpu].pvti的gpa通知给vmm.
b. 写改x86的函数指针
pv_time_ops.sched_clock = kvm_clock_read;
x86_platform.calibrate_tsc =kvm_get_tsc_khz;
x86_platform.get_wallclock =kvm_get_wallclock;
x86_platform.set_wallclock = kvm_set_wallclock;
例如x86_platform.get_wallclock 默认为mach_get_cmos_time（从cmos取得wallclock).
wallclock指的是操作系统从开机开始的绝对时间。
c. clocksource_register_hz(&kvm_clock,NSEC_PER_SEC); //注册系统时钟源
static struct clocksource kvm_clock = {
.name ="kvm-clock",
.read = kvm_clock_get_cycles,
.rating = 400, //rating400为理想时钟源
.mask =CLOCKSOURCE_MASK(64),
.flags =CLOCK_SOURCE_IS_CONTINUOUS,
};
由于kvm-clock将rating 设为400，这样会使clocksource_register_hz==》__clocksource_register_scale==》 clocksource_select==》__clocksource_select将guest os 的curr_clocksource设为kvmclock

下面重点分析kvm_clock_read和kvm_get_wallclock
static void kvm_get_wallclock(struct timespec *now)
{
structpvclock_vcpu_time_info *vcpu_time;
int low, high;
int cpu;
low =(int)__pa_symbol(&wall_clock);
high = ((u64)__pa_symbol(&wall_clock)>> 32);
native_write_msr(msr_kvm_wall_clock,low, high);
preempt_disable();
cpu =smp_processor_id();
vcpu_time =&hv_clock[cpu].pvti;
pvclock_read_wallclock(&wall_clock,vcpu_time, now);
preempt_enable();
}
a. native_write_msr(msr_kvm_wall_clock, low, high);通知vmm要取wall_clock并将wall_clock的gpa告诉vmm.

b. pvclock_read_wallclock 返回vmm设置号的wallclock. wall_clock在返回前相当于是guest 与vmm间的共享内存.

pvclock_read_wallclock的访问
void pvclock_read_wallclock(struct pvclock_wall_clock*wall_clock,
{

//等待vmm设置好wall_clock, 用version来标记数据是否更新
do {
version =wall_clock->version;
rmb(); /* fetch version before time */
now.tv_sec = wall_clock->sec;
now.tv_nsec =wall_clock->nsec;
rmb(); /* fetch time before checking version */
} while ((wall_clock->version& 1) || (version != wall_clock->version)); /
//这时wall_clock记录的是系统开机时的时间

//取得系统运行的时间, vcpu_time作为共享内存，其地址在kvm_register_clock通知了vmm
delta =pvclock_clocksource_read(vcpu_time);

//两者相加为wall_clock,
delta += now.tv_sec *(u64)NSEC_PER_SEC + now.tv_nsec;

now.tv_nsec =do_div(delta, NSEC_PER_SEC);
now.tv_sec = delta;

set_normalized_timespec(ts,now.tv_sec, now.tv_nsec);
}

static cycle_t kvm_clock_read(void) ==> pvclock_clocksource_read
cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info*src)
{

do {
version =__pvclock_read_cycles(src, &ret, &flags); // __native_read_tsc
} while((src->version & 1) || version != src->version);

......
if ((valid_flags &PVCLOCK_TSC_STABLE_BIT) &&
(flags & PVCLOCK_TSC_STABLE_BIT))
return ret;
last =atomic64_read(&last_value);
do {
if (ret < last)
return last;
last =atomic64_cmpxchg(&last_value, last, ret);
} while (unlikely(last!= ret));

return ret;
}

(2) VMM kvmclock实现
msr的实现：
kvm_set_msr_common ==> case MSR_KVM_WALL_CLOCK ==> kvm_write_wall_clock
static void kvm_write_wall_clock(struct kvm *kvm, gpa_twall_clock)
{
.......
//a. 读guest version
r = kvm_read_guest(kvm,wall_clock, &version, sizeof(version));
if (r)
return;

if (version & 1)
++version; /* first time write, random junk */

++version;

kvm_write_guest(kvm,wall_clock, &version, sizeof(version));//更新version

getboottime(&boot);//得到系统的boot时间

if(kvm->arch.kvmclock_offset) {
struct timespec ts =ns_to_timespec(kvm->arch.kvmclock_offset);
boot =timespec_sub(boot, ts);
}
wc.sec = boot.tv_sec;
wc.nsec = boot.tv_nsec;
wc.version = version;

kvm_write_guest(kvm,wall_clock, &wc, sizeof(wc)); //更新guest wall_clock

version++;
kvm_write_guest(kvm,wall_clock, &version, sizeof(version)); //更新version，完成通讯
}

kvm_read_guest/kvm_write_guest 的工作原理是通过gpa得到对应page 的hva和页内偏移，然后就能读写内存了
int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data,unsigned long len)
{
gfn_t gfn = gpa>> PAGE_SHIFT;
int seg;
int offset =offset_in_page(gpa);
int ret;

while ((seg =next_segment(len, offset)) != 0) {
ret =kvm_read_guest_page(kvm, gfn, data, offset, seg);
if (ret < 0)
return ret;
offset = 0;
len -= seg;
data += seg;
++gfn;
}
return 0;
}
int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data,int offset,
int len)
{
int r;
unsigned long addr;

addr =gfn_to_hva_prot(kvm, gfn, NULL);
if(kvm_is_error_hva(addr))
return -EFAULT;
r = kvm_read_hva(data,(void __user *)addr + offset, len); // call __copy_from_user
if (r)
return -EFAULT;
return 0;
}

kvm_set_msr_common ==> case MSR_KVM_SYSTEM_TIME
a. kvmclock_reset //vcpu->arch.pv_time_enabled = false
b. kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE,vcpu);
c. if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
&vcpu->arch.pv_time, data &~1ULL,
sizeof(struct pvclock_vcpu_time_info)))
vcpu->arch.pv_time_enabled= false;
else
vcpu->arch.pv_time_enabled= true;
kvm_gfn_to_hva_cache_init会得到guest os 的hv_clock[cpu].pvti

vcpu_enter_guest==> KVM_REQ_GLOBAL_CLOCK_UPDATE kvm_gen_kvmclock_update(vcpu);
set_bit(KVM_REQ_CLOCK_UPDATE,&v->requests);
schedule_delayed_work(&kvm->arch.kvmclock_update_work,
KVMCLOCK_UPDATE_DELAY);

由于在kvm_arch_init_vm时：
INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work,kvmclock_update_fn);
INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work,kvmclock_sync_fn);

所以kvm->arch.kvmclock_update_work==》
static void kvmclock_update_fn(struct work_struct *work)
{
。。。。。。
//对每个vcpu设置KVM_REQ_CLOCK_UPDATE
kvm_for_each_vcpu(i,vcpu, kvm) {
set_bit(KVM_REQ_CLOCK_UPDATE,&vcpu->requests);
kvm_vcpu_kick(vcpu);
}
}

vcpu_enter_guest==>KVM_REQ_CLOCK_UPDATE kvm_guest_time_update(vcpu);
kvm_guest_time_update会将时间更新到vcpu->pv_time

4.3.3 Cpu Steal time
Cpu Steal time指的是vcpu 等待 real cpu 的时间, 因为vcpu会发生vm-exit而进入vmm;进入vmm 后到重新vm-entry的时间就是一次cpu steal time. 该指标是衡量vm性能的重要指标。 通过半虚拟化技术guest os能得到cpu steal time. VMM与guest通讯机制与上一节类似，本节就不讨论了。

（1） Guest os 实现
1. kvm_guest_init注册函数指针pv_time_ops.steal_clock =kvm_steal_clock; 对非guest而言
该函数为native_steal_clock， 直接返回0

2. Guest os 通过kvm_register_steal_time 通知vmm 共享内存地址：
wrmsrl(MSR_KVM_STEAL_TIME,(slow_virt_to_phys(st) | KVM_MSR_ENABLED));

内核kernel\core.c update_rq_clock ==> update_rq_clock_task ==>
paravirt_steal_clock(cpu_of(rq))==> pv_time_ops.steal_clock;

（2） vmm 实现
kvm_set_msr_common ==》　case MSR_KVM_STEAL_TIME
a. kvm_gfn_to_hva_cache_init得到guest os gpa -> hva
b. vcpu->arch.st.last_steal= current->sched_info.run_delay;
c. accumulate_steal_time(vcpu);
static void accumulate_steal_time(struct kvm_vcpu *vcpu)
{
.......
delta =current->sched_info.run_delay - vcpu->arch.st.last_steal;
vcpu->arch.st.last_steal= current->sched_info.run_delay;
vcpu->arch.st.accum_steal= delta;
}
第一调用时delta会为0， 但当以后vcpu_load时kvm_arch_vcpu_load会重新调用accumulate_steal_time

d. kvm_make_request(KVM_REQ_STEAL_UPDATE,vcpu);

vcpu_enter_guest ==> record_steal_time(vcpu);
static void record_steal_time(struct kvm_vcpu *vcpu)
{

............ //kvm_read_guest_cached
vcpu->arch.st.steal.steal+= vcpu->arch.st.accum_steal;
vcpu->arch.st.steal.version+= 2;
vcpu->arch.st.accum_steal= 0;
......... //kvm_write_guest_cached
}