Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/ml/fb_swtch_src.s
+++ new/usr/src/uts/i86pc/ml/fb_swtch_src.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 + * Copyright 2018 Joyent, Inc.
25 26 */
26 27
27 28
28 29 #if defined(__lint)
29 30
30 31 int fb_swtch_silence_lint = 0;
31 32
32 33 #else
33 34
34 35 #include <sys/asm_linkage.h>
35 36 #include <sys/segments.h>
36 37 #include <sys/controlregs.h>
37 38 #include <sys/machparam.h>
38 39 #include <sys/multiboot.h>
39 40 #include <sys/fastboot.h>
40 41 #include "assym.h"
41 42
42 43 /*
43 44 * This code is to switch from 64-bit or 32-bit to protected mode.
44 45 */
|
↓ open down ↓ |
10 lines elided |
↑ open up ↑ |
45 46
46 47 /*
47 48 * For debugging with LEDs
48 49 */
49 50 #define FB_OUTB_ASM(val) \
50 51 movb val, %al; \
51 52 outb $0x80;
52 53
53 54
54 55 #define DISABLE_PAGING \
56 + movl %cr4, %eax ;\
57 + btrl $17, %eax /* clear PCIDE bit */ ;\
58 + movl %eax, %cr4 ;\
55 59 movl %cr0, %eax ;\
56 60 btrl $31, %eax /* clear PG bit */ ;\
57 61 movl %eax, %cr0
58 62
59 63 /*
60 64 * This macro contains common code for 64/32-bit versions of copy_sections().
61 65 * On entry:
62 66 * fbf points to the fboot_file_t
63 67 * snum contains the number of sections
64 68 * Registers that would be clobbered:
65 69 * fbs, snum, %eax, %ecx, %edi, %esi.
66 70 * NOTE: fb_dest_pa is supposed to be in the first 1GB,
67 71 * therefore it is safe to use 32-bit register to hold it's value
68 72 * even for 64-bit code.
69 73 */
70 74
71 75 #define COPY_SECT(fbf, fbs, snum) \
72 76 lea FB_SECTIONS(fbf), fbs; \
73 77 xorl %eax, %eax; \
74 78 1: movl FB_DEST_PA(fbf), %esi; \
75 79 addl FB_SEC_OFFSET(fbs), %esi; \
76 80 movl FB_SEC_PADDR(fbs), %edi; \
77 81 movl FB_SEC_SIZE(fbs), %ecx; \
78 82 rep \
79 83 movsb; \
80 84 /* Zero BSS */ \
81 85 movl FB_SEC_BSS_SIZE(fbs), %ecx; \
82 86 rep \
83 87 stosb; \
84 88 add $FB_SECTIONS_INCR, fbs; \
85 89 dec snum; \
86 90 jnz 1b
87 91
88 92
89 93 .globl _start
90 94 _start:
91 95
92 96 /* Disable interrupts */
93 97 cli
94 98
95 99 #if defined(__amd64)
96 100 /* Switch to a low memory stack */
97 101 movq $_start, %rsp
98 102 addq $FASTBOOT_STACK_OFFSET, %rsp
99 103
100 104 /*
101 105 * Copy from old stack to new stack
102 106 * If the content before fi_valid gets bigger than 0x200 bytes,
103 107 * the reserved stack size above will need to be changed.
104 108 */
105 109 movq %rdi, %rsi /* source from old stack */
106 110 movq %rsp, %rdi /* destination on the new stack */
107 111 movq $FI_VALID, %rcx /* size to copy */
108 112 rep
109 113 smovb
110 114
111 115 #elif defined(__i386)
112 116 movl 0x4(%esp), %esi /* address of fastboot info struct */
113 117
114 118 /* Switch to a low memory stack */
115 119 movl $_start, %esp
116 120 addl $FASTBOOT_STACK_OFFSET, %esp
117 121
118 122 /* Copy struct to stack */
119 123 movl %esp, %edi /* destination on the new stack */
120 124 movl $FI_VALID, %ecx /* size to copy */
121 125 rep
122 126 smovb
123 127
124 128 #endif
125 129
126 130 #if defined(__amd64)
127 131
128 132 xorl %eax, %eax
129 133 xorl %edx, %edx
130 134
131 135 movl $MSR_AMD_FSBASE, %ecx
132 136 wrmsr
133 137
134 138 movl $MSR_AMD_GSBASE, %ecx
135 139 wrmsr
136 140
137 141 movl $MSR_AMD_KGSBASE, %ecx
138 142 wrmsr
139 143
140 144 #endif
141 145 /*
142 146 * zero out all the registers to make sure they're 16 bit clean
143 147 */
144 148 #if defined(__amd64)
145 149 xorq %r8, %r8
146 150 xorq %r9, %r9
147 151 xorq %r10, %r10
148 152 xorq %r11, %r11
149 153 xorq %r12, %r12
150 154 xorq %r13, %r13
151 155 xorq %r14, %r14
152 156 xorq %r15, %r15
153 157 #endif
154 158 xorl %eax, %eax
155 159 xorl %ebx, %ebx
156 160 xorl %ecx, %ecx
157 161 xorl %edx, %edx
158 162 xorl %ebp, %ebp
159 163
160 164 #if defined(__amd64)
161 165 /*
162 166 * Load our own GDT
163 167 */
164 168 lgdt gdt_info
165 169 #endif
166 170 /*
167 171 * Load our own IDT
168 172 */
169 173 lidt idt_info
170 174
171 175 #if defined(__amd64)
172 176 /*
173 177 * Invalidate all TLB entries.
174 178 * Load temporary pagetables to copy kernel and boot-archive
175 179 */
176 180 movq %cr4, %rax
177 181 andq $_BITNOT(CR4_PGE), %rax
178 182 movq %rax, %cr4
179 183 movq FI_PAGETABLE_PA(%rsp), %rax
180 184 movq %rax, %cr3
181 185
182 186 leaq FI_FILES(%rsp), %rbx /* offset to the files */
183 187
184 188 /* copy unix to final destination */
185 189 movq FI_LAST_TABLE_PA(%rsp), %rsi /* page table PA */
186 190 leaq _MUL(FASTBOOT_UNIX, FI_FILES_INCR)(%rbx), %rdi
187 191 call map_copy
188 192
189 193 /* copy boot archive to final destination */
190 194 movq FI_LAST_TABLE_PA(%rsp), %rsi /* page table PA */
191 195 leaq _MUL(FASTBOOT_BOOTARCHIVE, FI_FILES_INCR)(%rbx), %rdi
192 196 call map_copy
193 197
194 198 /* Copy sections if there are any */
195 199 leaq _MUL(FASTBOOT_UNIX, FI_FILES_INCR)(%rbx), %rdi
196 200 movl FB_SECTCNT(%rdi), %esi
197 201 cmpl $0, %esi
198 202 je 1f
199 203 call copy_sections
200 204 1:
201 205 /*
202 206 * Shut down 64 bit mode. First get into compatiblity mode.
203 207 */
204 208 movq %rsp, %rax
205 209 pushq $B32DATA_SEL
206 210 pushq %rax
207 211 pushf
208 212 pushq $B32CODE_SEL
209 213 pushq $1f
210 214 iretq
211 215
212 216 .code32
213 217 1:
214 218 movl $B32DATA_SEL, %eax
|
↓ open down ↓ |
150 lines elided |
↑ open up ↑ |
215 219 movw %ax, %ss
216 220 movw %ax, %ds
217 221 movw %ax, %es
218 222 movw %ax, %fs
219 223 movw %ax, %gs
220 224
221 225 /*
222 226 * Disable long mode by:
223 227 * - shutting down paging (bit 31 of cr0). This will flush the
224 228 * TLBs.
229 + * - turning off PCID in cr4
225 230 * - disabling LME (long mode enable) in EFER (extended feature reg)
226 231 */
227 232 #endif
228 233 DISABLE_PAGING /* clobbers %eax */
229 234
230 235 #if defined(__amd64)
231 236 ljmp $B32CODE_SEL, $1f
232 237 1:
233 238 #endif
234 239
235 240 /*
236 241 * Clear PGE, PAE and PSE flags as dboot expects them to be
237 242 * cleared.
238 243 */
239 244 movl %cr4, %eax
240 245 andl $_BITNOT(CR4_PGE | CR4_PAE | CR4_PSE), %eax
241 246 movl %eax, %cr4
242 247
243 248 #if defined(__amd64)
244 249 movl $MSR_AMD_EFER, %ecx /* Extended Feature Enable */
245 250 rdmsr
246 251 btcl $8, %eax /* bit 8 Long Mode Enable bit */
247 252 wrmsr
248 253
249 254 #elif defined(__i386)
250 255 /*
251 256 * If fi_has_pae is set, re-enable paging with PAE.
252 257 */
253 258 leal FI_FILES(%esp), %ebx /* offset to the files */
254 259 movl FI_HAS_PAE(%esp), %edi /* need to enable paging or not */
255 260 cmpl $0, %edi
256 261 je paging_on /* no need to enable paging */
257 262
258 263 movl FI_LAST_TABLE_PA(%esp), %esi /* page table PA */
259 264
260 265 /*
261 266 * Turn on PAE
262 267 */
263 268 movl %cr4, %eax
264 269 orl $CR4_PAE, %eax
265 270 movl %eax, %cr4
266 271
267 272 /*
268 273 * Load top pagetable base address into cr3
269 274 */
270 275 movl FI_PAGETABLE_PA(%esp), %eax
271 276 movl %eax, %cr3
272 277
273 278 movl %cr0, %eax
274 279 orl $_CONST(CR0_PG | CR0_WP | CR0_AM), %eax
275 280 andl $_BITNOT(CR0_NW | CR0_CD), %eax
276 281 movl %eax, %cr0
277 282 jmp paging_on
278 283 paging_on:
279 284
280 285 /* copy unix to final destination */
281 286 leal _MUL(FASTBOOT_UNIX, FI_FILES_INCR)(%ebx), %edx
282 287 call map_copy
283 288
284 289 /* copy boot archive to final destination */
285 290 leal _MUL(FASTBOOT_BOOTARCHIVE, FI_FILES_INCR)(%ebx), %edx
286 291 call map_copy
287 292
288 293 /* Disable paging one more time */
289 294 DISABLE_PAGING
290 295
291 296 /* Copy sections if there are any */
292 297 leal _MUL(FASTBOOT_UNIX, FI_FILES_INCR)(%ebx), %edx
293 298 movl FB_SECTCNT(%edx), %eax
294 299 cmpl $0, %eax
295 300 je 1f
296 301 call copy_sections
297 302 1:
298 303
299 304 /* Whatever flags we turn on we need to turn off */
300 305 movl %cr4, %eax
301 306 andl $_BITNOT(CR4_PAE), %eax
302 307 movl %eax, %cr4
303 308 #endif /* __i386 */
304 309
305 310 dboot_jump:
306 311 /* Jump to dboot */
307 312 movl $DBOOT_ENTRY_ADDRESS, %edi
308 313 movl FI_NEW_MBI_PA(%esp), %ebx
309 314 movl $MB_BOOTLOADER_MAGIC, %eax
310 315 jmp *%edi
311 316
312 317 #if defined(__amd64)
313 318
314 319 .code64
315 320 ENTRY_NP(copy_sections)
316 321 /*
317 322 * On entry
318 323 * %rdi points to the fboot_file_t
319 324 * %rsi contains number of sections
320 325 */
321 326 movq %rdi, %rdx
322 327 movq %rsi, %r9
323 328
324 329 COPY_SECT(%rdx, %r8, %r9)
325 330 ret
326 331 SET_SIZE(copy_sections)
327 332
328 333 ENTRY_NP(map_copy)
329 334 /*
330 335 * On entry
331 336 * %rdi points to the fboot_file_t
332 337 * %rsi has FI_LAST_TABLE_PA(%rsp)
333 338 */
334 339
335 340 movq %rdi, %rdx
336 341 movq %rsi, %r8
337 342 movq FB_PTE_LIST_PA(%rdx), %rax /* PA list of the source */
338 343 movq FB_DEST_PA(%rdx), %rdi /* PA of the destination */
339 344
340 345 2:
341 346 movq (%rax), %rcx /* Are we done? */
342 347 cmpl $FASTBOOT_TERMINATE, %ecx
343 348 je 1f
344 349
345 350 movq %rcx, (%r8)
346 351 movq %cr3, %rsi /* Reload cr3 */
347 352 movq %rsi, %cr3
348 353 movq FB_VA(%rdx), %rsi /* Load from VA */
349 354 movq $PAGESIZE, %rcx
350 355 shrq $3, %rcx /* 8-byte at a time */
351 356 rep
352 357 smovq
353 358 addq $8, %rax /* Go to next PTE */
354 359 jmp 2b
355 360 1:
356 361 ret
357 362 SET_SIZE(map_copy)
358 363
359 364 #elif defined(__i386)
360 365
361 366 ENTRY_NP(copy_sections)
362 367 /*
363 368 * On entry
364 369 * %edx points to the fboot_file_t
365 370 * %eax contains the number of sections
366 371 */
367 372 pushl %ebp
368 373 pushl %ebx
369 374 pushl %esi
370 375 pushl %edi
371 376
372 377 movl %eax, %ebp
373 378
374 379 COPY_SECT(%edx, %ebx, %ebp)
375 380
376 381 popl %edi
377 382 popl %esi
378 383 popl %ebx
379 384 popl %ebp
380 385 ret
381 386 SET_SIZE(copy_sections)
382 387
383 388 ENTRY_NP(map_copy)
384 389 /*
385 390 * On entry
386 391 * %edx points to the fboot_file_t
387 392 * %edi has FB_HAS_PAE(%esp)
388 393 * %esi has FI_LAST_TABLE_PA(%esp)
389 394 */
390 395 pushl %eax
391 396 pushl %ebx
392 397 pushl %ecx
393 398 pushl %edx
394 399 pushl %ebp
395 400 pushl %esi
396 401 pushl %edi
397 402 movl %esi, %ebp /* Save page table PA in %ebp */
398 403
399 404 movl FB_PTE_LIST_PA(%edx), %eax /* PA list of the source */
400 405 movl FB_DEST_PA(%edx), %ebx /* PA of the destination */
401 406
402 407 loop:
403 408 movl (%eax), %esi /* Are we done? */
404 409 cmpl $FASTBOOT_TERMINATE, %esi
405 410 je done
406 411
407 412 cmpl $1, (%esp) /* Is paging on? */
408 413 jne no_paging /* Nope */
409 414
410 415 movl %ebp, %edi /* Page table PA */
411 416 movl %esi, (%edi) /* Program low 32-bit */
412 417 movl 4(%eax), %esi /* high bits of the table */
413 418 movl %esi, 4(%edi) /* Program high 32-bit */
414 419 movl %cr3, %esi /* Reload cr3 */
415 420 movl %esi, %cr3
416 421 movl FB_VA(%edx), %esi /* Load from VA */
417 422 jmp do_copy
418 423 no_paging:
419 424 andl $_BITNOT(MMU_PAGEOFFSET), %esi /* clear lower 12-bit */
420 425 do_copy:
421 426 movl %ebx, %edi
422 427 movl $PAGESIZE, %ecx
423 428 shrl $2, %ecx /* 4-byte at a time */
424 429 rep
425 430 smovl
426 431 addl $8, %eax /* We built the PTEs as 8-byte entries */
427 432 addl $PAGESIZE, %ebx
428 433 jmp loop
429 434 done:
430 435 popl %edi
431 436 popl %esi
432 437 popl %ebp
433 438 popl %edx
434 439 popl %ecx
435 440 popl %ebx
436 441 popl %eax
437 442 ret
438 443 SET_SIZE(map_copy)
439 444 #endif /* __i386 */
440 445
441 446
442 447 idt_info:
443 448 .value 0x3ff
444 449 .quad 0
445 450
446 451 /*
447 452 * We need to trampoline thru a gdt we have in low memory.
448 453 */
449 454 #include "../boot/boot_gdt.s"
450 455 #endif /* __lint */
|
↓ open down ↓ |
216 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX