5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #if !defined(lint)
26 #include "assym.h"
27 #endif
28
29 /*
30 * General assembly language routines.
31 * It is the intent of this file to contain routines that are
32 * specific to cpu architecture.
33 */
34
35 /*
36 * WARNING: If you add a fast trap handler which can be invoked by a
37 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
38 * instead of "done" instruction to return back to the user mode. See
39 * comments for the "fast_trap_done" entry point for more information.
40 */
41 #define FAST_TRAP_DONE \
42 ba,a fast_trap_done
43
44 #include <sys/machclock.h>
45 #include <sys/clock.h>
46
47 #if defined(lint)
48 #include <sys/types.h>
49 #include <sys/scb.h>
50 #include <sys/systm.h>
51 #include <sys/regset.h>
52 #include <sys/sunddi.h>
53 #include <sys/lockstat.h>
54 #endif /* lint */
55
56
57 #include <sys/asm_linkage.h>
58 #include <sys/privregs.h>
59 #include <vm/hat_sfmmu.h>
60 #include <sys/machparam.h> /* To get SYSBASE and PAGESIZE */
61 #include <sys/machthread.h>
62 #include <sys/clock.h>
63 #include <sys/intreg.h>
64 #include <sys/psr_compat.h>
65 #include <sys/isa_defs.h>
66 #include <sys/dditypes.h>
67 #include <sys/intr.h>
68 #include <sys/hypervisor_api.h>
69
70 #if !defined(lint)
71 #include "assym.h"
72 #endif
73
74 #define ICACHE_FLUSHSZ 0x20
75
76 #if defined(lint)
77 /*
78 * Softint generated when counter field of tick reg matches value field
79 * of tick_cmpr reg
80 */
81 /*ARGSUSED*/
82 void
83 tickcmpr_set(uint64_t clock_cycles)
84 {}
85
86 #else /* lint */
87
88 ENTRY_NP(tickcmpr_set)
89 ! get 64-bit clock_cycles interval
90 mov %o0, %o2
91 mov 8, %o3 ! A reasonable initial step size
92 1:
93 WR_TICKCMPR(%o2,%o4,%o5,__LINE__) ! Write to TICK_CMPR
94
95 GET_NATIVE_TIME(%o0,%o4,%o5,__LINE__) ! Read %tick to confirm the
96 ! value we wrote was in the
97 ! future.
98
99 cmp %o2, %o0 ! If the value we wrote was in the
100 bg,pt %xcc, 2f ! future, then blow out of here.
101 sllx %o3, 1, %o3 ! If not, then double our step size,
102 ba,pt %xcc, 1b ! and take another lap.
103 add %o0, %o3, %o2 !
104 2:
105 retl
106 nop
107 SET_SIZE(tickcmpr_set)
108
109 #endif /* lint */
110
111 #if defined(lint)
112
113 void
114 tickcmpr_disable(void)
115 {}
116
117 #else
118
119 ENTRY_NP(tickcmpr_disable)
120 mov 1, %g1
121 sllx %g1, TICKINT_DIS_SHFT, %o0
122 WR_TICKCMPR(%o0,%o4,%o5,__LINE__) ! Write to TICK_CMPR
123 retl
124 nop
125 SET_SIZE(tickcmpr_disable)
126
127 #endif
128
129 #if defined(lint)
130
131 /*
132 * tick_write_delta() is intended to increment %stick by the specified delta,
133 * but %stick is only writeable in hyperprivileged mode and at present there
134 * is no provision for this. tick_write_delta is called by the cylic subsystem
135 * if a negative %stick delta is observed after cyclic processing is resumed
136 * after an event such as an OS suspend/resume. On sun4v, the suspend/resume
137 * routines should adjust the %stick offset preventing the cyclic subsystem
138 * from detecting a negative delta. If a negative delta is detected, panic the
139 * system. The negative delta could be caused by improper %stick
140 * synchronization after a suspend/resume.
141 */
142
143 /*ARGSUSED*/
144 void
145 tick_write_delta(uint64_t delta)
146 {}
147
148 #else /* lint */
149
150 .seg ".text"
151 tick_write_delta_panic:
152 .asciz "tick_write_delta: not supported, delta: 0x%lx"
153
154 ENTRY_NP(tick_write_delta)
155 sethi %hi(tick_write_delta_panic), %o1
156 save %sp, -SA(MINFRAME), %sp ! get a new window to preserve caller
157 mov %i0, %o1
158 call panic
159 or %i1, %lo(tick_write_delta_panic), %o0
160 /*NOTREACHED*/
161 retl
162 nop
163 #endif
164
165 #if defined(lint)
166 /*
167 * return 1 if disabled
168 */
169
170 int
171 tickcmpr_disabled(void)
172 { return (0); }
173
174 #else /* lint */
175
176 ENTRY_NP(tickcmpr_disabled)
177 RD_TICKCMPR(%g1,%o0,%o1,__LINE__)
178 retl
179 srlx %g1, TICKINT_DIS_SHFT, %o0
180 SET_SIZE(tickcmpr_disabled)
181
182 #endif /* lint */
183
184 /*
185 * Get current tick
186 */
187 #if defined(lint)
188
189 u_longlong_t
190 gettick(void)
191 { return (0); }
192
193 u_longlong_t
194 randtick(void)
195 { return (0); }
196
197 #else /* lint */
198
199 ENTRY(gettick)
200 ALTENTRY(randtick)
201 GET_NATIVE_TIME(%o0,%o2,%o3,__LINE__)
202 retl
203 nop
204 SET_SIZE(randtick)
205 SET_SIZE(gettick)
206
207 #endif /* lint */
208
209 /*
210 * Get current tick. For trapstat use only.
211 */
212 #if defined (lint)
213
214 hrtime_t
215 rdtick()
216 { return (0); }
217
218 #else
219 ENTRY(rdtick)
220 retl
221 RD_TICK_PHYSICAL(%o0)
222 SET_SIZE(rdtick)
223 #endif /* lint */
224
225
226 /*
227 * Return the counter portion of the tick register.
228 */
229
230 #if defined(lint)
231
232 uint64_t
233 gettick_counter(void)
234 { return(0); }
235
236 uint64_t
237 gettick_npt(void)
238 { return(0); }
239
240 uint64_t
241 getstick_npt(void)
242 { return(0); }
243
244 #else /* lint */
245
246 ENTRY_NP(gettick_counter)
247 RD_TICK(%o0,%o1,%o2,__LINE__)
248 retl
249 nop
250 SET_SIZE(gettick_counter)
251
252 ENTRY_NP(gettick_npt)
253 RD_TICK_PHYSICAL(%o0)
254 retl
255 srlx %o0, 63, %o0
256 SET_SIZE(gettick_npt)
257
258 ENTRY_NP(getstick_npt)
259 RD_STICK_PHYSICAL(%o0)
260 retl
261 srlx %o0, 63, %o0
262 SET_SIZE(getstick_npt)
263 #endif /* lint */
264
265 /*
266 * Provide a C callable interface to the trap that reads the hi-res timer.
267 * Returns 64-bit nanosecond timestamp in %o0 and %o1.
268 */
269
270 #if defined(lint)
271
272 hrtime_t
273 gethrtime(void)
274 {
275 return ((hrtime_t)0);
276 }
277
278 hrtime_t
279 gethrtime_unscaled(void)
280 {
281 return ((hrtime_t)0);
282 }
283
284 hrtime_t
285 gethrtime_max(void)
286 {
287 return ((hrtime_t)0);
288 }
289
290 void
291 scalehrtime(hrtime_t *hrt)
292 {
293 *hrt = 0;
294 }
295
296 void
297 gethrestime(timespec_t *tp)
298 {
299 tp->tv_sec = 0;
300 tp->tv_nsec = 0;
301 }
302
303 time_t
304 gethrestime_sec(void)
305 {
306 return (0);
307 }
308
309 void
310 gethrestime_lasttick(timespec_t *tp)
311 {
312 tp->tv_sec = 0;
313 tp->tv_nsec = 0;
314 }
315
316 /*ARGSUSED*/
317 void
318 hres_tick(void)
319 {
320 }
321
322 void
323 panic_hres_tick(void)
324 {
325 }
326
327 #else /* lint */
328
329 ENTRY_NP(gethrtime)
330 GET_HRTIME(%g1,%o0,%o1,%o2,%o3,%o4,%o5,%g2,__LINE__)
331 ! %g1 = hrtime
332 retl
333 mov %g1, %o0
334 SET_SIZE(gethrtime)
335
336 ENTRY_NP(gethrtime_unscaled)
337 GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__) ! %g1 = native time
338 retl
339 mov %g1, %o0
340 SET_SIZE(gethrtime_unscaled)
341
342 ENTRY_NP(gethrtime_waitfree)
343 ALTENTRY(dtrace_gethrtime)
344 GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__) ! %g1 = native time
345 NATIVE_TIME_TO_NSEC(%g1, %o2, %o3)
346 retl
347 mov %g1, %o0
348 SET_SIZE(dtrace_gethrtime)
601 inc %i1 ! release lock
602 st %i1, [%l4 + %lo(hres_lock)] ! clear hres_lock
603
604 ret
605 restore
606
607 9:
608 !
609 ! release hres_lock
610 !
611 ld [%l4 + %lo(hres_lock)], %i1
612 inc %i1
613 st %i1, [%l4 + %lo(hres_lock)]
614
615 sethi %hi(hrtime_base_panic), %o0
616 call panic
617 or %o0, %lo(hrtime_base_panic), %o0
618
619 SET_SIZE(hres_tick)
620
621 #endif /* lint */
622
623 #if !defined(lint) && !defined(__lint)
624
625 .seg ".text"
626 kstat_q_panic_msg:
627 .asciz "kstat_q_exit: qlen == 0"
628
629 ENTRY(kstat_q_panic)
630 save %sp, -SA(MINFRAME), %sp
631 sethi %hi(kstat_q_panic_msg), %o0
632 call panic
633 or %o0, %lo(kstat_q_panic_msg), %o0
634 /*NOTREACHED*/
635 SET_SIZE(kstat_q_panic)
636
637 #define BRZPN brz,pn
638 #define BRZPT brz,pt
639
640 #define KSTAT_Q_UPDATE(QOP, QBR, QZERO, QRETURN, QTYPE) \
641 ld [%o0 + QTYPE/**/CNT], %o1; /* %o1 = old qlen */ \
642 QOP %o1, 1, %o2; /* %o2 = new qlen */ \
643 QBR %o1, QZERO; /* done if qlen == 0 */ \
644 st %o2, [%o0 + QTYPE/**/CNT]; /* delay: save qlen */ \
714 GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
715 #if defined(DEBUG)
716 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_W)
717 #else
718 KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_W)
719 #endif
720 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R)
721 SET_SIZE(kstat_waitq_to_runq)
722
723 .align 16
724 ENTRY(kstat_runq_back_to_waitq)
725 GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
726 #if defined(DEBUG)
727 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_R)
728 #else
729 KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_R)
730 #endif
731 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W)
732 SET_SIZE(kstat_runq_back_to_waitq)
733
734 #endif /* lint */
735
736 #ifdef lint
737
738 int64_t timedelta;
739 hrtime_t hres_last_tick;
740 volatile timestruc_t hrestime;
741 int64_t hrestime_adj;
742 volatile int hres_lock;
743 uint_t nsec_scale;
744 hrtime_t hrtime_base;
745 int traptrace_use_stick;
746
747 #else
748 /*
749 * -- WARNING --
750 *
751 * The following variables MUST be together on a 128-byte boundary.
752 * In addition to the primary performance motivation (having them all
753 * on the same cache line(s)), code here and in the GET*TIME() macros
754 * assumes that they all have the same high 22 address bits (so
755 * there's only one sethi).
756 */
757 .seg ".data"
758 .global timedelta, hres_last_tick, hrestime, hrestime_adj
759 .global hres_lock, nsec_scale, hrtime_base, traptrace_use_stick
760 .global nsec_shift, adj_shift, native_tick_offset, native_stick_offset
761
762 /* XXX - above comment claims 128-bytes is necessary */
763 .align 64
764 timedelta:
765 .word 0, 0 /* int64_t */
766 hres_last_tick:
767 .word 0, 0 /* hrtime_t */
771 .word 0, 0 /* int64_t */
772 hres_lock:
773 .word 0
774 nsec_scale:
775 .word 0
776 hrtime_base:
777 .word 0, 0
778 traptrace_use_stick:
779 .word 0
780 nsec_shift:
781 .word NSEC_SHIFT
782 adj_shift:
783 .word ADJ_SHIFT
784 .align 8
785 native_tick_offset:
786 .word 0, 0
787 .align 8
788 native_stick_offset:
789 .word 0, 0
790
791 #endif
792
793
794 /*
795 * drv_usecwait(clock_t n) [DDI/DKI - section 9F]
796 * usec_delay(int n) [compatibility - should go one day]
797 * Delay by spinning.
798 *
799 * delay for n microseconds. numbers <= 0 delay 1 usec
800 *
801 * With UltraSPARC-III the combination of supporting mixed-speed CPUs
802 * and variable clock rate for power management requires that we
803 * use %stick to implement this routine.
804 */
805
806 #if defined(lint)
807
808 /*ARGSUSED*/
809 void
810 drv_usecwait(clock_t n)
811 {}
812
813 /*ARGSUSED*/
814 void
815 usec_delay(int n)
816 {}
817
818 #else /* lint */
819
820 ENTRY(drv_usecwait)
821 ALTENTRY(usec_delay)
822 brlez,a,pn %o0, 0f
823 mov 1, %o0
824 0:
825 sethi %hi(sticks_per_usec), %o1
826 lduw [%o1 + %lo(sticks_per_usec)], %o1
827 mulx %o1, %o0, %o1 ! Scale usec to ticks
828 inc %o1 ! We don't start on a tick edge
829 GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
830 add %o1, %o2, %o1
831
832 1: cmp %o1, %o2
833 GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
834 bgeu,pt %xcc, 1b
835 nop
836 retl
837 nop
838 SET_SIZE(usec_delay)
839 SET_SIZE(drv_usecwait)
840 #endif /* lint */
841
842 #if defined(lint)
843
844 /* ARGSUSED */
845 void
846 pil14_interrupt(int level)
847 {}
848
849 #else
850
851 /*
852 * Level-14 interrupt prologue.
853 */
854 ENTRY_NP(pil14_interrupt)
855 CPU_ADDR(%g1, %g2)
856 rdpr %pil, %g6 ! %g6 = interrupted PIL
857 stn %g6, [%g1 + CPU_PROFILE_PIL] ! record interrupted PIL
858 rdpr %tstate, %g6
859 rdpr %tpc, %g5
860 btst TSTATE_PRIV, %g6 ! trap from supervisor mode?
861 bnz,a,pt %xcc, 1f
862 stn %g5, [%g1 + CPU_PROFILE_PC] ! if so, record kernel PC
863 stn %g5, [%g1 + CPU_PROFILE_UPC] ! if not, record user PC
864 ba pil_interrupt_common ! must be large-disp branch
865 stn %g0, [%g1 + CPU_PROFILE_PC] ! zero kernel PC
866 1: ba pil_interrupt_common ! must be large-disp branch
867 stn %g0, [%g1 + CPU_PROFILE_UPC] ! zero user PC
868 SET_SIZE(pil14_interrupt)
869
870 ENTRY_NP(tick_rtt)
916
917 !
918 ! If we're here, then we have programmed TICK_COMPARE with a %tick
919 ! which is in the past; we'll now load an initial step size, and loop
920 ! until we've managed to program TICK_COMPARE to fire in the future.
921 !
922 mov 8, %o4 ! 8 = arbitrary inital step
923 1: add %o0, %o4, %o5 ! Add the step
924 WR_TICKCMPR(%o5,%g1,%g2,__LINE__) ! Write to TICK_CMPR
925 GET_NATIVE_TIME(%o0,%g1,%g2,__LINE__) ! %o0 = tick
926 cmp %o5, %o0 ! In the future?
927 bg,a,pt %xcc, 2f ! Yes, drive on.
928 wrpr %g0, %g5, %pstate ! delay: enable vec intr
929 ba 1b ! No, try again.
930 sllx %o4, 1, %o4 ! delay: double step size
931
932 2: ba current_thread_complete
933 nop
934 SET_SIZE(tick_rtt)
935
936 #endif /* lint */
937
938 #if defined(lint)
939
940 /* ARGSUSED */
941 void
942 pil15_interrupt(int level)
943 {}
944
945 #else /* lint */
946
947 /*
948 * Level-15 interrupt prologue.
949 */
950 ENTRY_NP(pil15_interrupt)
951 CPU_ADDR(%g1, %g2)
952 rdpr %tstate, %g6
953 rdpr %tpc, %g5
954 btst TSTATE_PRIV, %g6 ! trap from supervisor mode?
955 bnz,a,pt %xcc, 1f
956 stn %g5, [%g1 + CPU_CPCPROFILE_PC] ! if so, record kernel PC
957 stn %g5, [%g1 + CPU_CPCPROFILE_UPC] ! if not, record user PC
958 ba pil15_epilogue ! must be large-disp branch
959 stn %g0, [%g1 + CPU_CPCPROFILE_PC] ! zero kernel PC
960 1: ba pil15_epilogue ! must be large-disp branch
961 stn %g0, [%g1 + CPU_CPCPROFILE_UPC] ! zero user PC
962 SET_SIZE(pil15_interrupt)
963
964 #endif /* lint */
965
966 #if defined(lint)
967 /*
968 * Prefetch a page_t for write or read, this assumes a linear
969 * scan of sequential page_t's.
970 */
971 /*ARGSUSED*/
972 void
973 prefetch_page_w(void *pp)
974 {}
975
976 /*ARGSUSED*/
977 void
978 prefetch_page_r(void *pp)
979 {}
980 #else /* lint */
981
982 /* XXXQ These should be inline templates, not functions */
983 ENTRY(prefetch_page_w)
984 retl
985 nop
986 SET_SIZE(prefetch_page_w)
987
988 ENTRY(prefetch_page_r)
989 retl
990 nop
991 SET_SIZE(prefetch_page_r)
992
993 #endif /* lint */
994
995 #if defined(lint)
996 /*
997 * Prefetch struct smap for write.
998 */
999 /*ARGSUSED*/
1000 void
1001 prefetch_smap_w(void *smp)
1002 {}
1003 #else /* lint */
1004
1005 /* XXXQ These should be inline templates, not functions */
1006 ENTRY(prefetch_smap_w)
1007 retl
1008 nop
1009 SET_SIZE(prefetch_smap_w)
1010
1011 #endif /* lint */
1012
1013 /*
1014 * Generic sun4v MMU and Cache operations.
1015 */
1016
1017 #if defined(lint)
1018
1019 /*ARGSUSED*/
1020 void
1021 vtag_flushpage(caddr_t vaddr, uint64_t sfmmup)
1022 {}
1023
1024 /*ARGSUSED*/
1025 void
1026 vtag_flushall(void)
1027 {}
1028
1029 /*ARGSUSED*/
1030 void
1031 vtag_unmap_perm_tl1(uint64_t vaddr, uint64_t ctxnum)
1032 {}
1033
1034 /*ARGSUSED*/
1035 void
1036 vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup)
1037 {}
1038
1039 /*ARGSUSED*/
1040 void
1041 vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt)
1042 {}
1043
1044 /*ARGSUSED*/
1045 void
1046 vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2)
1047 {}
1048
1049 /*ARGSUSED*/
1050 void
1051 vac_flushpage(pfn_t pfnum, int vcolor)
1052 {}
1053
1054 /*ARGSUSED*/
1055 void
1056 vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor)
1057 {}
1058
1059 /*ARGSUSED*/
1060 void
1061 flush_instr_mem(caddr_t vaddr, size_t len)
1062 {}
1063
1064 #else /* lint */
1065
1066 ENTRY_NP(vtag_flushpage)
1067 /*
1068 * flush page from the tlb
1069 *
1070 * %o0 = vaddr
1071 * %o1 = sfmmup
1072 */
1073 SFMMU_CPU_CNUM(%o1, %g1, %g2) /* %g1 = sfmmu cnum on this CPU */
1074
1075 mov %g1, %o1
1076 mov MAP_ITLB | MAP_DTLB, %o2
1077 ta MMU_UNMAP_ADDR
1078 brz,pt %o0, 1f
1079 nop
1080 ba panic_bad_hcall
1081 mov MMU_UNMAP_ADDR, %o1
1082 1:
1083 retl
1084 nop
1085 SET_SIZE(vtag_flushpage)
1245 /*
1246 * flush_instr_mem:
1247 * Flush a portion of the I-$ starting at vaddr
1248 * %o0 vaddr
1249 * %o1 bytes to be flushed
1250 */
1251
1252 ENTRY(flush_instr_mem)
1253 membar #StoreStore ! Ensure the stores
1254 ! are globally visible
1255 1:
1256 flush %o0
1257 subcc %o1, ICACHE_FLUSHSZ, %o1 ! bytes = bytes-0x20
1258 bgu,pt %ncc, 1b
1259 add %o0, ICACHE_FLUSHSZ, %o0 ! vaddr = vaddr+0x20
1260
1261 retl
1262 nop
1263 SET_SIZE(flush_instr_mem)
1264
1265 #endif /* !lint */
1266
1267 #if !defined(CUSTOM_FPZERO)
1268
1269 /*
1270 * fp_zero() - clear all fp data registers and the fsr
1271 */
1272
1273 #if defined(lint) || defined(__lint)
1274
1275 void
1276 fp_zero(void)
1277 {}
1278
1279 #else /* lint */
1280
1281 .global fp_zero_zero
1282 .align 8
1283 fp_zero_zero:
1284 .xword 0
1285
1286 ENTRY_NP(fp_zero)
1287 sethi %hi(fp_zero_zero), %o0
1288 ldx [%o0 + %lo(fp_zero_zero)], %fsr
1289 ldd [%o0 + %lo(fp_zero_zero)], %f0
1290 fmovd %f0, %f2
1291 fmovd %f0, %f4
1292 fmovd %f0, %f6
1293 fmovd %f0, %f8
1294 fmovd %f0, %f10
1295 fmovd %f0, %f12
1296 fmovd %f0, %f14
1297 fmovd %f0, %f16
1298 fmovd %f0, %f18
1299 fmovd %f0, %f20
1300 fmovd %f0, %f22
1304 fmovd %f0, %f30
1305 fmovd %f0, %f32
1306 fmovd %f0, %f34
1307 fmovd %f0, %f36
1308 fmovd %f0, %f38
1309 fmovd %f0, %f40
1310 fmovd %f0, %f42
1311 fmovd %f0, %f44
1312 fmovd %f0, %f46
1313 fmovd %f0, %f48
1314 fmovd %f0, %f50
1315 fmovd %f0, %f52
1316 fmovd %f0, %f54
1317 fmovd %f0, %f56
1318 fmovd %f0, %f58
1319 fmovd %f0, %f60
1320 retl
1321 fmovd %f0, %f62
1322 SET_SIZE(fp_zero)
1323
1324 #endif /* lint */
1325 #endif /* CUSTOM_FPZERO */
|
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include "assym.h"
26
27 /*
28 * General assembly language routines.
29 * It is the intent of this file to contain routines that are
30 * specific to cpu architecture.
31 */
32
33 /*
34 * WARNING: If you add a fast trap handler which can be invoked by a
35 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
36 * instead of "done" instruction to return back to the user mode. See
37 * comments for the "fast_trap_done" entry point for more information.
38 */
39 #define FAST_TRAP_DONE \
40 ba,a fast_trap_done
41
42 #include <sys/machclock.h>
43 #include <sys/clock.h>
44
45
46 #include <sys/asm_linkage.h>
47 #include <sys/privregs.h>
48 #include <vm/hat_sfmmu.h>
49 #include <sys/machparam.h> /* To get SYSBASE and PAGESIZE */
50 #include <sys/machthread.h>
51 #include <sys/clock.h>
52 #include <sys/intreg.h>
53 #include <sys/psr_compat.h>
54 #include <sys/isa_defs.h>
55 #include <sys/dditypes.h>
56 #include <sys/intr.h>
57 #include <sys/hypervisor_api.h>
58
59 #include "assym.h"
60
61 #define ICACHE_FLUSHSZ 0x20
62
63 /*
64 * Softint generated when counter field of tick reg matches value field
65 * of tick_cmpr reg
66 */
67 ENTRY_NP(tickcmpr_set)
68 ! get 64-bit clock_cycles interval
69 mov %o0, %o2
70 mov 8, %o3 ! A reasonable initial step size
71 1:
72 WR_TICKCMPR(%o2,%o4,%o5,__LINE__) ! Write to TICK_CMPR
73
74 GET_NATIVE_TIME(%o0,%o4,%o5,__LINE__) ! Read %tick to confirm the
75 ! value we wrote was in the
76 ! future.
77
78 cmp %o2, %o0 ! If the value we wrote was in the
79 bg,pt %xcc, 2f ! future, then blow out of here.
80 sllx %o3, 1, %o3 ! If not, then double our step size,
81 ba,pt %xcc, 1b ! and take another lap.
82 add %o0, %o3, %o2 !
83 2:
84 retl
85 nop
86 SET_SIZE(tickcmpr_set)
87
88 ENTRY_NP(tickcmpr_disable)
89 mov 1, %g1
90 sllx %g1, TICKINT_DIS_SHFT, %o0
91 WR_TICKCMPR(%o0,%o4,%o5,__LINE__) ! Write to TICK_CMPR
92 retl
93 nop
94 SET_SIZE(tickcmpr_disable)
95
96 .seg ".text"
97 tick_write_delta_panic:
98 .asciz "tick_write_delta: not supported, delta: 0x%lx"
99
100 /*
101 * tick_write_delta() is intended to increment %stick by the specified delta,
102 * but %stick is only writeable in hyperprivileged mode and at present there
103 * is no provision for this. tick_write_delta is called by the cylic subsystem
104 * if a negative %stick delta is observed after cyclic processing is resumed
105 * after an event such as an OS suspend/resume. On sun4v, the suspend/resume
106 * routines should adjust the %stick offset preventing the cyclic subsystem
107 * from detecting a negative delta. If a negative delta is detected, panic the
108 * system. The negative delta could be caused by improper %stick
109 * synchronization after a suspend/resume.
110 */
111 ENTRY_NP(tick_write_delta)
112 sethi %hi(tick_write_delta_panic), %o1
113 save %sp, -SA(MINFRAME), %sp ! get a new window to preserve caller
114 mov %i0, %o1
115 call panic
116 or %i1, %lo(tick_write_delta_panic), %o0
117 /*NOTREACHED*/
118 retl
119 nop
120
121 ENTRY_NP(tickcmpr_disabled)
122 RD_TICKCMPR(%g1,%o0,%o1,__LINE__)
123 retl
124 srlx %g1, TICKINT_DIS_SHFT, %o0
125 SET_SIZE(tickcmpr_disabled)
126
127 /*
128 * Get current tick
129 */
130
131 ENTRY(gettick)
132 ALTENTRY(randtick)
133 GET_NATIVE_TIME(%o0,%o2,%o3,__LINE__)
134 retl
135 nop
136 SET_SIZE(randtick)
137 SET_SIZE(gettick)
138
139 /*
140 * Get current tick. For trapstat use only.
141 */
142 ENTRY(rdtick)
143 retl
144 RD_TICK_PHYSICAL(%o0)
145 SET_SIZE(rdtick)
146
147
148 /*
149 * Return the counter portion of the tick register.
150 */
151
152 ENTRY_NP(gettick_counter)
153 RD_TICK(%o0,%o1,%o2,__LINE__)
154 retl
155 nop
156 SET_SIZE(gettick_counter)
157
158 ENTRY_NP(gettick_npt)
159 RD_TICK_PHYSICAL(%o0)
160 retl
161 srlx %o0, 63, %o0
162 SET_SIZE(gettick_npt)
163
164 ENTRY_NP(getstick_npt)
165 RD_STICK_PHYSICAL(%o0)
166 retl
167 srlx %o0, 63, %o0
168 SET_SIZE(getstick_npt)
169
170 /*
171 * Provide a C callable interface to the trap that reads the hi-res timer.
172 * Returns 64-bit nanosecond timestamp in %o0 and %o1.
173 */
174
175 ENTRY_NP(gethrtime)
176 GET_HRTIME(%g1,%o0,%o1,%o2,%o3,%o4,%o5,%g2,__LINE__)
177 ! %g1 = hrtime
178 retl
179 mov %g1, %o0
180 SET_SIZE(gethrtime)
181
182 ENTRY_NP(gethrtime_unscaled)
183 GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__) ! %g1 = native time
184 retl
185 mov %g1, %o0
186 SET_SIZE(gethrtime_unscaled)
187
188 ENTRY_NP(gethrtime_waitfree)
189 ALTENTRY(dtrace_gethrtime)
190 GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__) ! %g1 = native time
191 NATIVE_TIME_TO_NSEC(%g1, %o2, %o3)
192 retl
193 mov %g1, %o0
194 SET_SIZE(dtrace_gethrtime)
447 inc %i1 ! release lock
448 st %i1, [%l4 + %lo(hres_lock)] ! clear hres_lock
449
450 ret
451 restore
452
453 9:
454 !
455 ! release hres_lock
456 !
457 ld [%l4 + %lo(hres_lock)], %i1
458 inc %i1
459 st %i1, [%l4 + %lo(hres_lock)]
460
461 sethi %hi(hrtime_base_panic), %o0
462 call panic
463 or %o0, %lo(hrtime_base_panic), %o0
464
465 SET_SIZE(hres_tick)
466
467 .seg ".text"
468 kstat_q_panic_msg:
469 .asciz "kstat_q_exit: qlen == 0"
470
471 ENTRY(kstat_q_panic)
472 save %sp, -SA(MINFRAME), %sp
473 sethi %hi(kstat_q_panic_msg), %o0
474 call panic
475 or %o0, %lo(kstat_q_panic_msg), %o0
476 /*NOTREACHED*/
477 SET_SIZE(kstat_q_panic)
478
479 #define BRZPN brz,pn
480 #define BRZPT brz,pt
481
482 #define KSTAT_Q_UPDATE(QOP, QBR, QZERO, QRETURN, QTYPE) \
483 ld [%o0 + QTYPE/**/CNT], %o1; /* %o1 = old qlen */ \
484 QOP %o1, 1, %o2; /* %o2 = new qlen */ \
485 QBR %o1, QZERO; /* done if qlen == 0 */ \
486 st %o2, [%o0 + QTYPE/**/CNT]; /* delay: save qlen */ \
556 GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
557 #if defined(DEBUG)
558 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_W)
559 #else
560 KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_W)
561 #endif
562 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R)
563 SET_SIZE(kstat_waitq_to_runq)
564
565 .align 16
566 ENTRY(kstat_runq_back_to_waitq)
567 GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
568 #if defined(DEBUG)
569 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_R)
570 #else
571 KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_R)
572 #endif
573 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W)
574 SET_SIZE(kstat_runq_back_to_waitq)
575
576 /*
577 * -- WARNING --
578 *
579 * The following variables MUST be together on a 128-byte boundary.
580 * In addition to the primary performance motivation (having them all
581 * on the same cache line(s)), code here and in the GET*TIME() macros
582 * assumes that they all have the same high 22 address bits (so
583 * there's only one sethi).
584 */
585 .seg ".data"
586 .global timedelta, hres_last_tick, hrestime, hrestime_adj
587 .global hres_lock, nsec_scale, hrtime_base, traptrace_use_stick
588 .global nsec_shift, adj_shift, native_tick_offset, native_stick_offset
589
590 /* XXX - above comment claims 128-bytes is necessary */
591 .align 64
592 timedelta:
593 .word 0, 0 /* int64_t */
594 hres_last_tick:
595 .word 0, 0 /* hrtime_t */
599 .word 0, 0 /* int64_t */
600 hres_lock:
601 .word 0
602 nsec_scale:
603 .word 0
604 hrtime_base:
605 .word 0, 0
606 traptrace_use_stick:
607 .word 0
608 nsec_shift:
609 .word NSEC_SHIFT
610 adj_shift:
611 .word ADJ_SHIFT
612 .align 8
613 native_tick_offset:
614 .word 0, 0
615 .align 8
616 native_stick_offset:
617 .word 0, 0
618
619
620 /*
621 * drv_usecwait(clock_t n) [DDI/DKI - section 9F]
622 * usec_delay(int n) [compatibility - should go one day]
623 * Delay by spinning.
624 *
625 * delay for n microseconds. numbers <= 0 delay 1 usec
626 *
627 * With UltraSPARC-III the combination of supporting mixed-speed CPUs
628 * and variable clock rate for power management requires that we
629 * use %stick to implement this routine.
630 */
631
632 ENTRY(drv_usecwait)
633 ALTENTRY(usec_delay)
634 brlez,a,pn %o0, 0f
635 mov 1, %o0
636 0:
637 sethi %hi(sticks_per_usec), %o1
638 lduw [%o1 + %lo(sticks_per_usec)], %o1
639 mulx %o1, %o0, %o1 ! Scale usec to ticks
640 inc %o1 ! We don't start on a tick edge
641 GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
642 add %o1, %o2, %o1
643
644 1: cmp %o1, %o2
645 GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
646 bgeu,pt %xcc, 1b
647 nop
648 retl
649 nop
650 SET_SIZE(usec_delay)
651 SET_SIZE(drv_usecwait)
652
653 /*
654 * Level-14 interrupt prologue.
655 */
656 ENTRY_NP(pil14_interrupt)
657 CPU_ADDR(%g1, %g2)
658 rdpr %pil, %g6 ! %g6 = interrupted PIL
659 stn %g6, [%g1 + CPU_PROFILE_PIL] ! record interrupted PIL
660 rdpr %tstate, %g6
661 rdpr %tpc, %g5
662 btst TSTATE_PRIV, %g6 ! trap from supervisor mode?
663 bnz,a,pt %xcc, 1f
664 stn %g5, [%g1 + CPU_PROFILE_PC] ! if so, record kernel PC
665 stn %g5, [%g1 + CPU_PROFILE_UPC] ! if not, record user PC
666 ba pil_interrupt_common ! must be large-disp branch
667 stn %g0, [%g1 + CPU_PROFILE_PC] ! zero kernel PC
668 1: ba pil_interrupt_common ! must be large-disp branch
669 stn %g0, [%g1 + CPU_PROFILE_UPC] ! zero user PC
670 SET_SIZE(pil14_interrupt)
671
672 ENTRY_NP(tick_rtt)
718
719 !
720 ! If we're here, then we have programmed TICK_COMPARE with a %tick
721 ! which is in the past; we'll now load an initial step size, and loop
722 ! until we've managed to program TICK_COMPARE to fire in the future.
723 !
724 mov 8, %o4 ! 8 = arbitrary inital step
725 1: add %o0, %o4, %o5 ! Add the step
726 WR_TICKCMPR(%o5,%g1,%g2,__LINE__) ! Write to TICK_CMPR
727 GET_NATIVE_TIME(%o0,%g1,%g2,__LINE__) ! %o0 = tick
728 cmp %o5, %o0 ! In the future?
729 bg,a,pt %xcc, 2f ! Yes, drive on.
730 wrpr %g0, %g5, %pstate ! delay: enable vec intr
731 ba 1b ! No, try again.
732 sllx %o4, 1, %o4 ! delay: double step size
733
734 2: ba current_thread_complete
735 nop
736 SET_SIZE(tick_rtt)
737
738 /*
739 * Level-15 interrupt prologue.
740 */
741 ENTRY_NP(pil15_interrupt)
742 CPU_ADDR(%g1, %g2)
743 rdpr %tstate, %g6
744 rdpr %tpc, %g5
745 btst TSTATE_PRIV, %g6 ! trap from supervisor mode?
746 bnz,a,pt %xcc, 1f
747 stn %g5, [%g1 + CPU_CPCPROFILE_PC] ! if so, record kernel PC
748 stn %g5, [%g1 + CPU_CPCPROFILE_UPC] ! if not, record user PC
749 ba pil15_epilogue ! must be large-disp branch
750 stn %g0, [%g1 + CPU_CPCPROFILE_PC] ! zero kernel PC
751 1: ba pil15_epilogue ! must be large-disp branch
752 stn %g0, [%g1 + CPU_CPCPROFILE_UPC] ! zero user PC
753 SET_SIZE(pil15_interrupt)
754
755 /*
756 * Prefetch a page_t for write or read, this assumes a linear
757 * scan of sequential page_t's.
758 */
759 /* XXXQ These should be inline templates, not functions */
760 ENTRY(prefetch_page_w)
761 retl
762 nop
763 SET_SIZE(prefetch_page_w)
764
765 ENTRY(prefetch_page_r)
766 retl
767 nop
768 SET_SIZE(prefetch_page_r)
769
770 /*
771 * Prefetch struct smap for write.
772 */
773 /* XXXQ These should be inline templates, not functions */
774 ENTRY(prefetch_smap_w)
775 retl
776 nop
777 SET_SIZE(prefetch_smap_w)
778
779 /*
780 * Generic sun4v MMU and Cache operations.
781 */
782
783 ENTRY_NP(vtag_flushpage)
784 /*
785 * flush page from the tlb
786 *
787 * %o0 = vaddr
788 * %o1 = sfmmup
789 */
790 SFMMU_CPU_CNUM(%o1, %g1, %g2) /* %g1 = sfmmu cnum on this CPU */
791
792 mov %g1, %o1
793 mov MAP_ITLB | MAP_DTLB, %o2
794 ta MMU_UNMAP_ADDR
795 brz,pt %o0, 1f
796 nop
797 ba panic_bad_hcall
798 mov MMU_UNMAP_ADDR, %o1
799 1:
800 retl
801 nop
802 SET_SIZE(vtag_flushpage)
962 /*
963 * flush_instr_mem:
964 * Flush a portion of the I-$ starting at vaddr
965 * %o0 vaddr
966 * %o1 bytes to be flushed
967 */
968
969 ENTRY(flush_instr_mem)
970 membar #StoreStore ! Ensure the stores
971 ! are globally visible
972 1:
973 flush %o0
974 subcc %o1, ICACHE_FLUSHSZ, %o1 ! bytes = bytes-0x20
975 bgu,pt %ncc, 1b
976 add %o0, ICACHE_FLUSHSZ, %o0 ! vaddr = vaddr+0x20
977
978 retl
979 nop
980 SET_SIZE(flush_instr_mem)
981
982 #if !defined(CUSTOM_FPZERO)
983
984 /*
985 * fp_zero() - clear all fp data registers and the fsr
986 */
987
988 .global fp_zero_zero
989 .align 8
990 fp_zero_zero:
991 .xword 0
992
993 ENTRY_NP(fp_zero)
994 sethi %hi(fp_zero_zero), %o0
995 ldx [%o0 + %lo(fp_zero_zero)], %fsr
996 ldd [%o0 + %lo(fp_zero_zero)], %f0
997 fmovd %f0, %f2
998 fmovd %f0, %f4
999 fmovd %f0, %f6
1000 fmovd %f0, %f8
1001 fmovd %f0, %f10
1002 fmovd %f0, %f12
1003 fmovd %f0, %f14
1004 fmovd %f0, %f16
1005 fmovd %f0, %f18
1006 fmovd %f0, %f20
1007 fmovd %f0, %f22
1011 fmovd %f0, %f30
1012 fmovd %f0, %f32
1013 fmovd %f0, %f34
1014 fmovd %f0, %f36
1015 fmovd %f0, %f38
1016 fmovd %f0, %f40
1017 fmovd %f0, %f42
1018 fmovd %f0, %f44
1019 fmovd %f0, %f46
1020 fmovd %f0, %f48
1021 fmovd %f0, %f50
1022 fmovd %f0, %f52
1023 fmovd %f0, %f54
1024 fmovd %f0, %f56
1025 fmovd %f0, %f58
1026 fmovd %f0, %f60
1027 retl
1028 fmovd %f0, %f62
1029 SET_SIZE(fp_zero)
1030
1031 #endif /* CUSTOM_FPZERO */
|