1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
|
/*
* linux/arch/i386/kernel/i387.c
*
* Copyright (C) 1994 Linus Torvalds
*
* Pentium III FXSR, SSE support
* General FPU state handling cleanups
* Gareth Hughes <gareth@valinux.com>, May 2000
*/
#include <xen/config.h>
#include <xen/sched.h>
#include <asm/current.h>
#include <asm/processor.h>
#include <asm/hvm/support.h>
#include <asm/i387.h>
#include <asm/xstate.h>
#include <asm/asm_defns.h>
static void fpu_init(void)
{
unsigned long val;
asm volatile ( "fninit" );
if ( cpu_has_xmm )
{
/* load default value into MXCSR control/status register */
val = MXCSR_DEFAULT;
asm volatile ( "ldmxcsr %0" : : "m" (val) );
}
}
/*******************************/
/* FPU Restore Functions */
/*******************************/
/* Restore x87 extended state */
static inline void fpu_xrstor(struct vcpu *v, uint64_t mask)
{
bool_t ok;
ASSERT(v->arch.xsave_area);
/*
* XCR0 normally represents what guest OS set. In case of Xen itself,
* we set the accumulated feature mask before doing save/restore.
*/
ok = set_xcr0(v->arch.xcr0_accum | XSTATE_FP_SSE);
ASSERT(ok);
xrstor(v, mask);
ok = set_xcr0(v->arch.xcr0 ?: XSTATE_FP_SSE);
ASSERT(ok);
}
/* Restor x87 FPU, MMX, SSE and SSE2 state */
static inline void fpu_fxrstor(struct vcpu *v)
{
const typeof(v->arch.xsave_area->fpu_sse) *fpu_ctxt = v->arch.fpu_ctxt;
/*
* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
* is pending. Clear the x87 state here by setting it to fixed
* values. The hypervisor data segment can be sometimes 0 and
* sometimes new user value. Both should be ok. Use the FPU saved
* data block as a safe address because it should be in L1.
*/
if ( !(fpu_ctxt->fsw & 0x0080) &&
boot_cpu_data.x86_vendor == X86_VENDOR_AMD )
{
asm volatile ( "fnclex\n\t"
"ffree %%st(7)\n\t" /* clear stack tag */
"fildl %0" /* load to clear state */
: : "m" (*fpu_ctxt) );
}
/*
* FXRSTOR can fault if passed a corrupted data block. We handle this
* possibility, which may occur if the block was passed to us by control
* tools or through VCPUOP_initialise, by silently clearing the block.
*/
switch ( __builtin_expect(fpu_ctxt->x[FPU_WORD_SIZE_OFFSET], 8) )
{
default:
asm volatile (
/* See below for why the operands/constraints are this way. */
"1: " REX64_PREFIX "fxrstor (%2)\n"
".section .fixup,\"ax\" \n"
"2: push %%"__OP"ax \n"
" push %%"__OP"cx \n"
" push %%"__OP"di \n"
" mov %2,%%"__OP"di \n"
" mov %1,%%ecx \n"
" xor %%eax,%%eax \n"
" rep ; stosl \n"
" pop %%"__OP"di \n"
" pop %%"__OP"cx \n"
" pop %%"__OP"ax \n"
" jmp 1b \n"
".previous \n"
_ASM_EXTABLE(1b, 2b)
:
: "m" (*fpu_ctxt), "i" (sizeof(*fpu_ctxt) / 4), "R" (fpu_ctxt) );
break;
case 4: case 2:
asm volatile (
"1: fxrstor %0 \n"
".section .fixup,\"ax\"\n"
"2: push %%"__OP"ax \n"
" push %%"__OP"cx \n"
" push %%"__OP"di \n"
" lea %0,%%"__OP"di \n"
" mov %1,%%ecx \n"
" xor %%eax,%%eax \n"
" rep ; stosl \n"
" pop %%"__OP"di \n"
" pop %%"__OP"cx \n"
" pop %%"__OP"ax \n"
" jmp 1b \n"
".previous \n"
_ASM_EXTABLE(1b, 2b)
:
: "m" (*fpu_ctxt), "i" (sizeof(*fpu_ctxt) / 4) );
break;
}
}
/* Restore x87 extended state */
static inline void fpu_frstor(struct vcpu *v)
{
const char *fpu_ctxt = v->arch.fpu_ctxt;
asm volatile ( "frstor %0" : : "m" (*fpu_ctxt) );
}
/*******************************/
/* FPU Save Functions */
/*******************************/
/* Save x87 extended state */
static inline void fpu_xsave(struct vcpu *v)
{
bool_t ok;
ASSERT(v->arch.xsave_area);
/*
* XCR0 normally represents what guest OS set. In case of Xen itself,
* we set the accumulated feature mask before doing save/restore.
*/
ok = set_xcr0(v->arch.xcr0_accum | XSTATE_FP_SSE);
ASSERT(ok);
xsave(v, v->arch.nonlazy_xstate_used ? XSTATE_ALL : XSTATE_LAZY);
ok = set_xcr0(v->arch.xcr0 ?: XSTATE_FP_SSE);
ASSERT(ok);
}
/* Save x87 FPU, MMX, SSE and SSE2 state */
static inline void fpu_fxsave(struct vcpu *v)
{
typeof(v->arch.xsave_area->fpu_sse) *fpu_ctxt = v->arch.fpu_ctxt;
int word_size = cpu_has_fpu_sel ? 8 : 0;
if ( !is_pv_32bit_vcpu(v) )
{
/*
* The only way to force fxsaveq on a wide range of gas versions.
* On older versions the rex64 prefix works only if we force an
* addressing mode that doesn't require extended registers.
*/
asm volatile ( REX64_PREFIX "fxsave (%1)"
: "=m" (*fpu_ctxt) : "R" (fpu_ctxt) );
/*
* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
* is pending.
*/
if ( !(fpu_ctxt->fsw & 0x0080) &&
boot_cpu_data.x86_vendor == X86_VENDOR_AMD )
return;
if ( word_size > 0 &&
!((fpu_ctxt->fip.addr | fpu_ctxt->fdp.addr) >> 32) )
{
struct ix87_env fpu_env;
asm volatile ( "fnstenv %0" : "=m" (fpu_env) );
fpu_ctxt->fip.sel = fpu_env.fcs;
fpu_ctxt->fdp.sel = fpu_env.fds;
word_size = 4;
}
}
else
{
asm volatile ( "fxsave %0" : "=m" (*fpu_ctxt) );
word_size = 4;
}
if ( word_size >= 0 )
fpu_ctxt->x[FPU_WORD_SIZE_OFFSET] = word_size;
}
/* Save x87 FPU state */
static inline void fpu_fsave(struct vcpu *v)
{
char *fpu_ctxt = v->arch.fpu_ctxt;
/* FWAIT is required to make FNSAVE synchronous. */
asm volatile ( "fnsave %0 ; fwait" : "=m" (*fpu_ctxt) );
}
/*******************************/
/* VCPU FPU Functions */
/*******************************/
/* Restore FPU state whenever VCPU is schduled in. */
void vcpu_restore_fpu_eager(struct vcpu *v)
{
ASSERT(!is_idle_vcpu(v));
/* save the nonlazy extended state which is not tracked by CR0.TS bit */
if ( v->arch.nonlazy_xstate_used )
{
/* Avoid recursion */
clts();
fpu_xrstor(v, XSTATE_NONLAZY);
stts();
}
}
/*
* Restore FPU state when #NM is triggered.
*/
void vcpu_restore_fpu_lazy(struct vcpu *v)
{
ASSERT(!is_idle_vcpu(v));
/* Avoid recursion. */
clts();
if ( v->fpu_dirtied )
return;
if ( cpu_has_xsave )
fpu_xrstor(v, XSTATE_LAZY);
else if ( v->fpu_initialised )
{
if ( cpu_has_fxsr )
fpu_fxrstor(v);
else
fpu_frstor(v);
}
else
fpu_init();
v->fpu_initialised = 1;
v->fpu_dirtied = 1;
}
/*
* On each context switch, save the necessary FPU info of VCPU being switch
* out. It dispatches saving operation based on CPU's capability.
*/
void vcpu_save_fpu(struct vcpu *v)
{
if ( !v->fpu_dirtied )
return;
ASSERT(!is_idle_vcpu(v));
/* This can happen, if a paravirtualised guest OS has set its CR0.TS. */
clts();
if ( cpu_has_xsave )
fpu_xsave(v);
else if ( cpu_has_fxsr )
fpu_fxsave(v);
else
fpu_fsave(v);
v->fpu_dirtied = 0;
stts();
}
/* Initialize FPU's context save area */
int vcpu_init_fpu(struct vcpu *v)
{
int rc = 0;
/* Idle domain doesn't have FPU state allocated */
if ( is_idle_vcpu(v) )
goto done;
if ( (rc = xstate_alloc_save_area(v)) != 0 )
return rc;
if ( v->arch.xsave_area )
v->arch.fpu_ctxt = &v->arch.xsave_area->fpu_sse;
else
{
v->arch.fpu_ctxt = _xzalloc(sizeof(v->arch.xsave_area->fpu_sse), 16);
if ( !v->arch.fpu_ctxt )
{
rc = -ENOMEM;
goto done;
}
}
done:
return rc;
}
/* Free FPU's context save area */
void vcpu_destroy_fpu(struct vcpu *v)
{
if ( v->arch.xsave_area )
xstate_free_save_area(v);
else
xfree(v->arch.fpu_ctxt);
}
/*
* Local variables:
* mode: C
* c-file-style: "BSD"
* c-basic-offset: 4
* tab-width: 4
* indent-tabs-mode: nil
* End:
*/
|
