5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #if !defined(lint)
26 #include "assym.h"
27 #endif
28
29 /*
30 * General assembly language routines.
31 * It is the intent of this file to contain routines that are
32 * specific to cpu architecture.
33 */
34
35 /*
36 * WARNING: If you add a fast trap handler which can be invoked by a
37 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
38 * instead of "done" instruction to return back to the user mode. See
39 * comments for the "fast_trap_done" entry point for more information.
40 */
41 #define FAST_TRAP_DONE \
42 ba,a fast_trap_done
43
44 #include <sys/machclock.h>
45 #include <sys/clock.h>
46
47 #if defined(lint)
48 #include <sys/types.h>
49 #include <sys/scb.h>
50 #include <sys/systm.h>
51 #include <sys/regset.h>
52 #include <sys/sunddi.h>
53 #include <sys/lockstat.h>
54 #endif /* lint */
55
56
57 #include <sys/asm_linkage.h>
58 #include <sys/privregs.h>
59 #include <vm/hat_sfmmu.h>
60 #include <sys/machparam.h> /* To get SYSBASE and PAGESIZE */
61 #include <sys/machthread.h>
62 #include <sys/clock.h>
63 #include <sys/intreg.h>
64 #include <sys/psr_compat.h>
65 #include <sys/isa_defs.h>
66 #include <sys/dditypes.h>
67 #include <sys/intr.h>
68 #include <sys/hypervisor_api.h>
69
70 #if !defined(lint)
71 #include "assym.h"
72 #endif
73
74 #define ICACHE_FLUSHSZ 0x20
75
76 #if defined(lint)
77 /*
78 * Softint generated when counter field of tick reg matches value field
79 * of tick_cmpr reg
80 */
81 /*ARGSUSED*/
82 void
83 tickcmpr_set(uint64_t clock_cycles)
84 {}
85
86 #else /* lint */
87
88 ENTRY_NP(tickcmpr_set)
89 ! get 64-bit clock_cycles interval
90 mov %o0, %o2
91 mov 8, %o3 ! A reasonable initial step size
92 1:
93 WR_TICKCMPR(%o2,%o4,%o5,__LINE__) ! Write to TICK_CMPR
94
95 GET_NATIVE_TIME(%o0,%o4,%o5,__LINE__) ! Read %tick to confirm the
96 ! value we wrote was in the
97 ! future.
98
99 cmp %o2, %o0 ! If the value we wrote was in the
100 bg,pt %xcc, 2f ! future, then blow out of here.
101 sllx %o3, 1, %o3 ! If not, then double our step size,
102 ba,pt %xcc, 1b ! and take another lap.
103 add %o0, %o3, %o2 !
104 2:
105 retl
106 nop
107 SET_SIZE(tickcmpr_set)
108
109 #endif /* lint */
110
111 #if defined(lint)
112
113 void
114 tickcmpr_disable(void)
115 {}
116
117 #else
118
119 ENTRY_NP(tickcmpr_disable)
120 mov 1, %g1
121 sllx %g1, TICKINT_DIS_SHFT, %o0
122 WR_TICKCMPR(%o0,%o4,%o5,__LINE__) ! Write to TICK_CMPR
123 retl
124 nop
125 SET_SIZE(tickcmpr_disable)
126
127 #endif
128
129 #if defined(lint)
130
131 /*
132 * tick_write_delta() is intended to increment %stick by the specified delta,
133 * but %stick is only writeable in hyperprivileged mode and at present there
134 * is no provision for this. tick_write_delta is called by the cylic subsystem
135 * if a negative %stick delta is observed after cyclic processing is resumed
136 * after an event such as an OS suspend/resume. On sun4v, the suspend/resume
137 * routines should adjust the %stick offset preventing the cyclic subsystem
138 * from detecting a negative delta. If a negative delta is detected, panic the
139 * system. The negative delta could be caused by improper %stick
140 * synchronization after a suspend/resume.
141 */
142
143 /*ARGSUSED*/
144 void
145 tick_write_delta(uint64_t delta)
146 {}
147
148 #else /* lint */
149
150 .seg ".text"
151 tick_write_delta_panic:
152 .asciz "tick_write_delta: not supported, delta: 0x%lx"
153
154 ENTRY_NP(tick_write_delta)
155 sethi %hi(tick_write_delta_panic), %o1
156 save %sp, -SA(MINFRAME), %sp ! get a new window to preserve caller
157 mov %i0, %o1
158 call panic
159 or %i1, %lo(tick_write_delta_panic), %o0
160 /*NOTREACHED*/
161 retl
162 nop
163 #endif
164
165 #if defined(lint)
166 /*
167 * return 1 if disabled
168 */
169
170 int
171 tickcmpr_disabled(void)
172 { return (0); }
173
174 #else /* lint */
175
176 ENTRY_NP(tickcmpr_disabled)
177 RD_TICKCMPR(%g1,%o0,%o1,__LINE__)
178 retl
179 srlx %g1, TICKINT_DIS_SHFT, %o0
180 SET_SIZE(tickcmpr_disabled)
181
182 #endif /* lint */
183
184 /*
185 * Get current tick
186 */
187 #if defined(lint)
188
189 u_longlong_t
190 gettick(void)
191 { return (0); }
192
193 u_longlong_t
194 randtick(void)
195 { return (0); }
196
197 #else /* lint */
198
199 ENTRY(gettick)
200 ALTENTRY(randtick)
201 GET_NATIVE_TIME(%o0,%o2,%o3,__LINE__)
202 retl
203 nop
204 SET_SIZE(randtick)
205 SET_SIZE(gettick)
206
207 #endif /* lint */
208
209 /*
210 * Get current tick. For trapstat use only.
211 */
212 #if defined (lint)
213
214 hrtime_t
215 rdtick()
216 { return (0); }
217
218 #else
219 ENTRY(rdtick)
220 retl
221 RD_TICK_PHYSICAL(%o0)
222 SET_SIZE(rdtick)
223 #endif /* lint */
224
225
226 /*
227 * Return the counter portion of the tick register.
228 */
229
230 #if defined(lint)
231
232 uint64_t
233 gettick_counter(void)
234 { return(0); }
235
236 uint64_t
237 gettick_npt(void)
238 { return(0); }
239
240 uint64_t
241 getstick_npt(void)
242 { return(0); }
243
244 #else /* lint */
245
246 ENTRY_NP(gettick_counter)
247 RD_TICK(%o0,%o1,%o2,__LINE__)
248 retl
249 nop
250 SET_SIZE(gettick_counter)
251
252 ENTRY_NP(gettick_npt)
253 RD_TICK_PHYSICAL(%o0)
254 retl
255 srlx %o0, 63, %o0
256 SET_SIZE(gettick_npt)
257
258 ENTRY_NP(getstick_npt)
259 RD_STICK_PHYSICAL(%o0)
260 retl
261 srlx %o0, 63, %o0
262 SET_SIZE(getstick_npt)
263 #endif /* lint */
264
265 /*
266 * Provide a C callable interface to the trap that reads the hi-res timer.
267 * Returns 64-bit nanosecond timestamp in %o0 and %o1.
268 */
269
270 #if defined(lint)
271
272 hrtime_t
273 gethrtime(void)
274 {
275 return ((hrtime_t)0);
276 }
277
278 hrtime_t
279 gethrtime_unscaled(void)
280 {
281 return ((hrtime_t)0);
282 }
283
284 hrtime_t
285 gethrtime_max(void)
286 {
287 return ((hrtime_t)0);
288 }
289
290 void
291 scalehrtime(hrtime_t *hrt)
292 {
293 *hrt = 0;
294 }
295
296 void
297 gethrestime(timespec_t *tp)
298 {
299 tp->tv_sec = 0;
300 tp->tv_nsec = 0;
301 }
302
303 time_t
304 gethrestime_sec(void)
305 {
306 return (0);
307 }
308
309 void
310 gethrestime_lasttick(timespec_t *tp)
311 {
312 tp->tv_sec = 0;
313 tp->tv_nsec = 0;
314 }
315
316 /*ARGSUSED*/
317 void
318 hres_tick(void)
319 {
320 }
321
322 void
323 panic_hres_tick(void)
324 {
325 }
326
327 #else /* lint */
328
329 ENTRY_NP(gethrtime)
330 GET_HRTIME(%g1,%o0,%o1,%o2,%o3,%o4,%o5,%g2,__LINE__)
331 ! %g1 = hrtime
332 retl
333 mov %g1, %o0
334 SET_SIZE(gethrtime)
335
336 ENTRY_NP(gethrtime_unscaled)
337 GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__) ! %g1 = native time
338 retl
339 mov %g1, %o0
340 SET_SIZE(gethrtime_unscaled)
341
342 ENTRY_NP(gethrtime_waitfree)
343 ALTENTRY(dtrace_gethrtime)
344 GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__) ! %g1 = native time
345 NATIVE_TIME_TO_NSEC(%g1, %o2, %o3)
346 retl
347 mov %g1, %o0
348 SET_SIZE(dtrace_gethrtime)
601 inc %i1 ! release lock
602 st %i1, [%l4 + %lo(hres_lock)] ! clear hres_lock
603
604 ret
605 restore
606
607 9:
608 !
609 ! release hres_lock
610 !
611 ld [%l4 + %lo(hres_lock)], %i1
612 inc %i1
613 st %i1, [%l4 + %lo(hres_lock)]
614
615 sethi %hi(hrtime_base_panic), %o0
616 call panic
617 or %o0, %lo(hrtime_base_panic), %o0
618
619 SET_SIZE(hres_tick)
620
621 #endif /* lint */
622
623 #if !defined(lint) && !defined(__lint)
624
625 .seg ".text"
626 kstat_q_panic_msg:
627 .asciz "kstat_q_exit: qlen == 0"
628
629 ENTRY(kstat_q_panic)
630 save %sp, -SA(MINFRAME), %sp
631 sethi %hi(kstat_q_panic_msg), %o0
632 call panic
633 or %o0, %lo(kstat_q_panic_msg), %o0
634 /*NOTREACHED*/
635 SET_SIZE(kstat_q_panic)
636
637 #define BRZPN brz,pn
638 #define BRZPT brz,pt
639
640 #define KSTAT_Q_UPDATE(QOP, QBR, QZERO, QRETURN, QTYPE) \
641 ld [%o0 + QTYPE/**/CNT], %o1; /* %o1 = old qlen */ \
642 QOP %o1, 1, %o2; /* %o2 = new qlen */ \
643 QBR %o1, QZERO; /* done if qlen == 0 */ \
644 st %o2, [%o0 + QTYPE/**/CNT]; /* delay: save qlen */ \
714 GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
715 #if defined(DEBUG)
716 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_W)
717 #else
718 KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_W)
719 #endif
720 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R)
721 SET_SIZE(kstat_waitq_to_runq)
722
723 .align 16
724 ENTRY(kstat_runq_back_to_waitq)
725 GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
726 #if defined(DEBUG)
727 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_R)
728 #else
729 KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_R)
730 #endif
731 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W)
732 SET_SIZE(kstat_runq_back_to_waitq)
733
734 #endif /* lint */
735
736 #ifdef lint
737
738 int64_t timedelta;
739 hrtime_t hres_last_tick;
740 volatile timestruc_t hrestime;
741 int64_t hrestime_adj;
742 volatile int hres_lock;
743 uint_t nsec_scale;
744 hrtime_t hrtime_base;
745 int traptrace_use_stick;
746
747 #else
748 /*
749 * -- WARNING --
750 *
751 * The following variables MUST be together on a 128-byte boundary.
752 * In addition to the primary performance motivation (having them all
753 * on the same cache line(s)), code here and in the GET*TIME() macros
754 * assumes that they all have the same high 22 address bits (so
755 * there's only one sethi).
756 */
757 .seg ".data"
758 .global timedelta, hres_last_tick, hrestime, hrestime_adj
759 .global hres_lock, nsec_scale, hrtime_base, traptrace_use_stick
760 .global nsec_shift, adj_shift, native_tick_offset, native_stick_offset
761
762 /* XXX - above comment claims 128-bytes is necessary */
763 .align 64
764 timedelta:
765 .word 0, 0 /* int64_t */
766 hres_last_tick:
767 .word 0, 0 /* hrtime_t */
771 .word 0, 0 /* int64_t */
772 hres_lock:
773 .word 0
774 nsec_scale:
775 .word 0
776 hrtime_base:
777 .word 0, 0
778 traptrace_use_stick:
779 .word 0
780 nsec_shift:
781 .word NSEC_SHIFT
782 adj_shift:
783 .word ADJ_SHIFT
784 .align 8
785 native_tick_offset:
786 .word 0, 0
787 .align 8
788 native_stick_offset:
789 .word 0, 0
790
791 #endif
792
793
794 /*
795 * drv_usecwait(clock_t n) [DDI/DKI - section 9F]
796 * usec_delay(int n) [compatibility - should go one day]
797 * Delay by spinning.
798 *
799 * delay for n microseconds. numbers <= 0 delay 1 usec
800 *
801 * With UltraSPARC-III the combination of supporting mixed-speed CPUs
802 * and variable clock rate for power management requires that we
803 * use %stick to implement this routine.
804 */
805
806 #if defined(lint)
807
808 /*ARGSUSED*/
809 void
810 drv_usecwait(clock_t n)
811 {}
812
813 /*ARGSUSED*/
814 void
815 usec_delay(int n)
816 {}
817
818 #else /* lint */
819
820 ENTRY(drv_usecwait)
821 ALTENTRY(usec_delay)
822 brlez,a,pn %o0, 0f
823 mov 1, %o0
824 0:
825 sethi %hi(sticks_per_usec), %o1
826 lduw [%o1 + %lo(sticks_per_usec)], %o1
827 mulx %o1, %o0, %o1 ! Scale usec to ticks
828 inc %o1 ! We don't start on a tick edge
829 GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
830 add %o1, %o2, %o1
831
832 1: cmp %o1, %o2
833 GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
834 bgeu,pt %xcc, 1b
835 nop
836 retl
837 nop
838 SET_SIZE(usec_delay)
839 SET_SIZE(drv_usecwait)
840 #endif /* lint */
841
842 #if defined(lint)
843
844 /* ARGSUSED */
845 void
846 pil14_interrupt(int level)
847 {}
848
849 #else
850
851 /*
852 * Level-14 interrupt prologue.
853 */
854 ENTRY_NP(pil14_interrupt)
855 CPU_ADDR(%g1, %g2)
856 rdpr %pil, %g6 ! %g6 = interrupted PIL
857 stn %g6, [%g1 + CPU_PROFILE_PIL] ! record interrupted PIL
858 rdpr %tstate, %g6
859 rdpr %tpc, %g5
860 btst TSTATE_PRIV, %g6 ! trap from supervisor mode?
861 bnz,a,pt %xcc, 1f
862 stn %g5, [%g1 + CPU_PROFILE_PC] ! if so, record kernel PC
863 stn %g5, [%g1 + CPU_PROFILE_UPC] ! if not, record user PC
864 ba pil_interrupt_common ! must be large-disp branch
865 stn %g0, [%g1 + CPU_PROFILE_PC] ! zero kernel PC
866 1: ba pil_interrupt_common ! must be large-disp branch
867 stn %g0, [%g1 + CPU_PROFILE_UPC] ! zero user PC
868 SET_SIZE(pil14_interrupt)
869
870 ENTRY_NP(tick_rtt)
916
917 !
918 ! If we're here, then we have programmed TICK_COMPARE with a %tick
919 ! which is in the past; we'll now load an initial step size, and loop
920 ! until we've managed to program TICK_COMPARE to fire in the future.
921 !
922 mov 8, %o4 ! 8 = arbitrary inital step
923 1: add %o0, %o4, %o5 ! Add the step
924 WR_TICKCMPR(%o5,%g1,%g2,__LINE__) ! Write to TICK_CMPR
925 GET_NATIVE_TIME(%o0,%g1,%g2,__LINE__) ! %o0 = tick
926 cmp %o5, %o0 ! In the future?
927 bg,a,pt %xcc, 2f ! Yes, drive on.
928 wrpr %g0, %g5, %pstate ! delay: enable vec intr
929 ba 1b ! No, try again.
930 sllx %o4, 1, %o4 ! delay: double step size
931
932 2: ba current_thread_complete
933 nop
934 SET_SIZE(tick_rtt)
935
936 #endif /* lint */
937
938 #if defined(lint)
939
940 /* ARGSUSED */
941 void
942 pil15_interrupt(int level)
943 {}
944
945 #else /* lint */
946
947 /*
948 * Level-15 interrupt prologue.
949 */
950 ENTRY_NP(pil15_interrupt)
951 CPU_ADDR(%g1, %g2)
952 rdpr %tstate, %g6
953 rdpr %tpc, %g5
954 btst TSTATE_PRIV, %g6 ! trap from supervisor mode?
955 bnz,a,pt %xcc, 1f
956 stn %g5, [%g1 + CPU_CPCPROFILE_PC] ! if so, record kernel PC
957 stn %g5, [%g1 + CPU_CPCPROFILE_UPC] ! if not, record user PC
958 ba pil15_epilogue ! must be large-disp branch
959 stn %g0, [%g1 + CPU_CPCPROFILE_PC] ! zero kernel PC
960 1: ba pil15_epilogue ! must be large-disp branch
961 stn %g0, [%g1 + CPU_CPCPROFILE_UPC] ! zero user PC
962 SET_SIZE(pil15_interrupt)
963
964 #endif /* lint */
965
966 #if defined(lint)
967 /*
968 * Prefetch a page_t for write or read, this assumes a linear
969 * scan of sequential page_t's.
970 */
971 /*ARGSUSED*/
972 void
973 prefetch_page_w(void *pp)
974 {}
975
976 /*ARGSUSED*/
977 void
978 prefetch_page_r(void *pp)
979 {}
980 #else /* lint */
981
982 /* XXXQ These should be inline templates, not functions */
983 ENTRY(prefetch_page_w)
984 retl
985 nop
986 SET_SIZE(prefetch_page_w)
987
988 ENTRY(prefetch_page_r)
989 retl
990 nop
991 SET_SIZE(prefetch_page_r)
992
993 #endif /* lint */
994
995 #if defined(lint)
996 /*
997 * Prefetch struct smap for write.
998 */
999 /*ARGSUSED*/
1000 void
1001 prefetch_smap_w(void *smp)
1002 {}
1003 #else /* lint */
1004
1005 /* XXXQ These should be inline templates, not functions */
1006 ENTRY(prefetch_smap_w)
1007 retl
1008 nop
1009 SET_SIZE(prefetch_smap_w)
1010
1011 #endif /* lint */
1012
1013 /*
1014 * Generic sun4v MMU and Cache operations.
1015 */
1016
1017 #if defined(lint)
1018
1019 /*ARGSUSED*/
1020 void
1021 vtag_flushpage(caddr_t vaddr, uint64_t sfmmup)
1022 {}
1023
1024 /*ARGSUSED*/
1025 void
1026 vtag_flushall(void)
1027 {}
1028
1029 /*ARGSUSED*/
1030 void
1031 vtag_unmap_perm_tl1(uint64_t vaddr, uint64_t ctxnum)
1032 {}
1033
1034 /*ARGSUSED*/
1035 void
1036 vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup)
1037 {}
1038
1039 /*ARGSUSED*/
1040 void
1041 vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt)
1042 {}
1043
1044 /*ARGSUSED*/
1045 void
1046 vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2)
1047 {}
1048
1049 /*ARGSUSED*/
1050 void
1051 vac_flushpage(pfn_t pfnum, int vcolor)
1052 {}
1053
1054 /*ARGSUSED*/
1055 void
1056 vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor)
1057 {}
1058
1059 /*ARGSUSED*/
1060 void
1061 flush_instr_mem(caddr_t vaddr, size_t len)
1062 {}
1063
1064 #else /* lint */
1065
1066 ENTRY_NP(vtag_flushpage)
1067 /*
1068 * flush page from the tlb
1069 *
1070 * %o0 = vaddr
1071 * %o1 = sfmmup
1072 */
1073 SFMMU_CPU_CNUM(%o1, %g1, %g2) /* %g1 = sfmmu cnum on this CPU */
1074
1075 mov %g1, %o1
1076 mov MAP_ITLB | MAP_DTLB, %o2
1077 ta MMU_UNMAP_ADDR
1078 brz,pt %o0, 1f
1079 nop
1080 ba panic_bad_hcall
1081 mov MMU_UNMAP_ADDR, %o1
1082 1:
1083 retl
1084 nop
1085 SET_SIZE(vtag_flushpage)
1245 /*
1246 * flush_instr_mem:
1247 * Flush a portion of the I-$ starting at vaddr
1248 * %o0 vaddr
1249 * %o1 bytes to be flushed
1250 */
1251
1252 ENTRY(flush_instr_mem)
1253 membar #StoreStore ! Ensure the stores
1254 ! are globally visible
1255 1:
1256 flush %o0
1257 subcc %o1, ICACHE_FLUSHSZ, %o1 ! bytes = bytes-0x20
1258 bgu,pt %ncc, 1b
1259 add %o0, ICACHE_FLUSHSZ, %o0 ! vaddr = vaddr+0x20
1260
1261 retl
1262 nop
1263 SET_SIZE(flush_instr_mem)
1264
1265 #endif /* !lint */
1266
1267 #if !defined(CUSTOM_FPZERO)
1268
1269 /*
1270 * fp_zero() - clear all fp data registers and the fsr
1271 */
1272
1273 #if defined(lint) || defined(__lint)
1274
1275 void
1276 fp_zero(void)
1277 {}
1278
1279 #else /* lint */
1280
1281 .global fp_zero_zero
1282 .align 8
1283 fp_zero_zero:
1284 .xword 0
1285
1286 ENTRY_NP(fp_zero)
1287 sethi %hi(fp_zero_zero), %o0
1288 ldx [%o0 + %lo(fp_zero_zero)], %fsr
1289 ldd [%o0 + %lo(fp_zero_zero)], %f0
1290 fmovd %f0, %f2
1291 fmovd %f0, %f4
1292 fmovd %f0, %f6
1293 fmovd %f0, %f8
1294 fmovd %f0, %f10
1295 fmovd %f0, %f12
1296 fmovd %f0, %f14
1297 fmovd %f0, %f16
1298 fmovd %f0, %f18
1299 fmovd %f0, %f20
1300 fmovd %f0, %f22
1304 fmovd %f0, %f30
1305 fmovd %f0, %f32
1306 fmovd %f0, %f34
1307 fmovd %f0, %f36
1308 fmovd %f0, %f38
1309 fmovd %f0, %f40
1310 fmovd %f0, %f42
1311 fmovd %f0, %f44
1312 fmovd %f0, %f46
1313 fmovd %f0, %f48
1314 fmovd %f0, %f50
1315 fmovd %f0, %f52
1316 fmovd %f0, %f54
1317 fmovd %f0, %f56
1318 fmovd %f0, %f58
1319 fmovd %f0, %f60
1320 retl
1321 fmovd %f0, %f62
1322 SET_SIZE(fp_zero)
1323
1324 #endif /* lint */
1325 #endif /* CUSTOM_FPZERO */
|
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include "assym.h"
26
27 /*
28 * General assembly language routines.
29 * It is the intent of this file to contain routines that are
30 * specific to cpu architecture.
31 */
32
33 /*
34 * WARNING: If you add a fast trap handler which can be invoked by a
35 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
36 * instead of "done" instruction to return back to the user mode. See
37 * comments for the "fast_trap_done" entry point for more information.
38 */
39 #define FAST_TRAP_DONE \
40 ba,a fast_trap_done
41
42 #include <sys/machclock.h>
43 #include <sys/clock.h>
44
45
46 #include <sys/asm_linkage.h>
47 #include <sys/privregs.h>
48 #include <vm/hat_sfmmu.h>
49 #include <sys/machparam.h> /* To get SYSBASE and PAGESIZE */
50 #include <sys/machthread.h>
51 #include <sys/clock.h>
52 #include <sys/intreg.h>
53 #include <sys/psr_compat.h>
54 #include <sys/isa_defs.h>
55 #include <sys/dditypes.h>
56 #include <sys/intr.h>
57 #include <sys/hypervisor_api.h>
58
59 #include "assym.h"
60
61 #define ICACHE_FLUSHSZ 0x20
62
63 ENTRY_NP(tickcmpr_set)
64 ! get 64-bit clock_cycles interval
65 mov %o0, %o2
66 mov 8, %o3 ! A reasonable initial step size
67 1:
68 WR_TICKCMPR(%o2,%o4,%o5,__LINE__) ! Write to TICK_CMPR
69
70 GET_NATIVE_TIME(%o0,%o4,%o5,__LINE__) ! Read %tick to confirm the
71 ! value we wrote was in the
72 ! future.
73
74 cmp %o2, %o0 ! If the value we wrote was in the
75 bg,pt %xcc, 2f ! future, then blow out of here.
76 sllx %o3, 1, %o3 ! If not, then double our step size,
77 ba,pt %xcc, 1b ! and take another lap.
78 add %o0, %o3, %o2 !
79 2:
80 retl
81 nop
82 SET_SIZE(tickcmpr_set)
83
84 ENTRY_NP(tickcmpr_disable)
85 mov 1, %g1
86 sllx %g1, TICKINT_DIS_SHFT, %o0
87 WR_TICKCMPR(%o0,%o4,%o5,__LINE__) ! Write to TICK_CMPR
88 retl
89 nop
90 SET_SIZE(tickcmpr_disable)
91
92 .seg ".text"
93 tick_write_delta_panic:
94 .asciz "tick_write_delta: not supported, delta: 0x%lx"
95
96 ENTRY_NP(tick_write_delta)
97 sethi %hi(tick_write_delta_panic), %o1
98 save %sp, -SA(MINFRAME), %sp ! get a new window to preserve caller
99 mov %i0, %o1
100 call panic
101 or %i1, %lo(tick_write_delta_panic), %o0
102 /*NOTREACHED*/
103 retl
104 nop
105
106 ENTRY_NP(tickcmpr_disabled)
107 RD_TICKCMPR(%g1,%o0,%o1,__LINE__)
108 retl
109 srlx %g1, TICKINT_DIS_SHFT, %o0
110 SET_SIZE(tickcmpr_disabled)
111
112 /*
113 * Get current tick
114 */
115
116 ENTRY(gettick)
117 ALTENTRY(randtick)
118 GET_NATIVE_TIME(%o0,%o2,%o3,__LINE__)
119 retl
120 nop
121 SET_SIZE(randtick)
122 SET_SIZE(gettick)
123
124 /*
125 * Get current tick. For trapstat use only.
126 */
127 ENTRY(rdtick)
128 retl
129 RD_TICK_PHYSICAL(%o0)
130 SET_SIZE(rdtick)
131
132
133 /*
134 * Return the counter portion of the tick register.
135 */
136
137 ENTRY_NP(gettick_counter)
138 RD_TICK(%o0,%o1,%o2,__LINE__)
139 retl
140 nop
141 SET_SIZE(gettick_counter)
142
143 ENTRY_NP(gettick_npt)
144 RD_TICK_PHYSICAL(%o0)
145 retl
146 srlx %o0, 63, %o0
147 SET_SIZE(gettick_npt)
148
149 ENTRY_NP(getstick_npt)
150 RD_STICK_PHYSICAL(%o0)
151 retl
152 srlx %o0, 63, %o0
153 SET_SIZE(getstick_npt)
154
155 /*
156 * Provide a C callable interface to the trap that reads the hi-res timer.
157 * Returns 64-bit nanosecond timestamp in %o0 and %o1.
158 */
159
160 ENTRY_NP(gethrtime)
161 GET_HRTIME(%g1,%o0,%o1,%o2,%o3,%o4,%o5,%g2,__LINE__)
162 ! %g1 = hrtime
163 retl
164 mov %g1, %o0
165 SET_SIZE(gethrtime)
166
167 ENTRY_NP(gethrtime_unscaled)
168 GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__) ! %g1 = native time
169 retl
170 mov %g1, %o0
171 SET_SIZE(gethrtime_unscaled)
172
173 ENTRY_NP(gethrtime_waitfree)
174 ALTENTRY(dtrace_gethrtime)
175 GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__) ! %g1 = native time
176 NATIVE_TIME_TO_NSEC(%g1, %o2, %o3)
177 retl
178 mov %g1, %o0
179 SET_SIZE(dtrace_gethrtime)
432 inc %i1 ! release lock
433 st %i1, [%l4 + %lo(hres_lock)] ! clear hres_lock
434
435 ret
436 restore
437
438 9:
439 !
440 ! release hres_lock
441 !
442 ld [%l4 + %lo(hres_lock)], %i1
443 inc %i1
444 st %i1, [%l4 + %lo(hres_lock)]
445
446 sethi %hi(hrtime_base_panic), %o0
447 call panic
448 or %o0, %lo(hrtime_base_panic), %o0
449
450 SET_SIZE(hres_tick)
451
452 .seg ".text"
453 kstat_q_panic_msg:
454 .asciz "kstat_q_exit: qlen == 0"
455
456 ENTRY(kstat_q_panic)
457 save %sp, -SA(MINFRAME), %sp
458 sethi %hi(kstat_q_panic_msg), %o0
459 call panic
460 or %o0, %lo(kstat_q_panic_msg), %o0
461 /*NOTREACHED*/
462 SET_SIZE(kstat_q_panic)
463
464 #define BRZPN brz,pn
465 #define BRZPT brz,pt
466
467 #define KSTAT_Q_UPDATE(QOP, QBR, QZERO, QRETURN, QTYPE) \
468 ld [%o0 + QTYPE/**/CNT], %o1; /* %o1 = old qlen */ \
469 QOP %o1, 1, %o2; /* %o2 = new qlen */ \
470 QBR %o1, QZERO; /* done if qlen == 0 */ \
471 st %o2, [%o0 + QTYPE/**/CNT]; /* delay: save qlen */ \
541 GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
542 #if defined(DEBUG)
543 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_W)
544 #else
545 KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_W)
546 #endif
547 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R)
548 SET_SIZE(kstat_waitq_to_runq)
549
550 .align 16
551 ENTRY(kstat_runq_back_to_waitq)
552 GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
553 #if defined(DEBUG)
554 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_R)
555 #else
556 KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_R)
557 #endif
558 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W)
559 SET_SIZE(kstat_runq_back_to_waitq)
560
561 /*
562 * -- WARNING --
563 *
564 * The following variables MUST be together on a 128-byte boundary.
565 * In addition to the primary performance motivation (having them all
566 * on the same cache line(s)), code here and in the GET*TIME() macros
567 * assumes that they all have the same high 22 address bits (so
568 * there's only one sethi).
569 */
570 .seg ".data"
571 .global timedelta, hres_last_tick, hrestime, hrestime_adj
572 .global hres_lock, nsec_scale, hrtime_base, traptrace_use_stick
573 .global nsec_shift, adj_shift, native_tick_offset, native_stick_offset
574
575 /* XXX - above comment claims 128-bytes is necessary */
576 .align 64
577 timedelta:
578 .word 0, 0 /* int64_t */
579 hres_last_tick:
580 .word 0, 0 /* hrtime_t */
584 .word 0, 0 /* int64_t */
585 hres_lock:
586 .word 0
587 nsec_scale:
588 .word 0
589 hrtime_base:
590 .word 0, 0
591 traptrace_use_stick:
592 .word 0
593 nsec_shift:
594 .word NSEC_SHIFT
595 adj_shift:
596 .word ADJ_SHIFT
597 .align 8
598 native_tick_offset:
599 .word 0, 0
600 .align 8
601 native_stick_offset:
602 .word 0, 0
603
604
605 /*
606 * drv_usecwait(clock_t n) [DDI/DKI - section 9F]
607 * usec_delay(int n) [compatibility - should go one day]
608 * Delay by spinning.
609 *
610 * delay for n microseconds. numbers <= 0 delay 1 usec
611 *
612 * With UltraSPARC-III the combination of supporting mixed-speed CPUs
613 * and variable clock rate for power management requires that we
614 * use %stick to implement this routine.
615 */
616
617 ENTRY(drv_usecwait)
618 ALTENTRY(usec_delay)
619 brlez,a,pn %o0, 0f
620 mov 1, %o0
621 0:
622 sethi %hi(sticks_per_usec), %o1
623 lduw [%o1 + %lo(sticks_per_usec)], %o1
624 mulx %o1, %o0, %o1 ! Scale usec to ticks
625 inc %o1 ! We don't start on a tick edge
626 GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
627 add %o1, %o2, %o1
628
629 1: cmp %o1, %o2
630 GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
631 bgeu,pt %xcc, 1b
632 nop
633 retl
634 nop
635 SET_SIZE(usec_delay)
636 SET_SIZE(drv_usecwait)
637
638 /*
639 * Level-14 interrupt prologue.
640 */
641 ENTRY_NP(pil14_interrupt)
642 CPU_ADDR(%g1, %g2)
643 rdpr %pil, %g6 ! %g6 = interrupted PIL
644 stn %g6, [%g1 + CPU_PROFILE_PIL] ! record interrupted PIL
645 rdpr %tstate, %g6
646 rdpr %tpc, %g5
647 btst TSTATE_PRIV, %g6 ! trap from supervisor mode?
648 bnz,a,pt %xcc, 1f
649 stn %g5, [%g1 + CPU_PROFILE_PC] ! if so, record kernel PC
650 stn %g5, [%g1 + CPU_PROFILE_UPC] ! if not, record user PC
651 ba pil_interrupt_common ! must be large-disp branch
652 stn %g0, [%g1 + CPU_PROFILE_PC] ! zero kernel PC
653 1: ba pil_interrupt_common ! must be large-disp branch
654 stn %g0, [%g1 + CPU_PROFILE_UPC] ! zero user PC
655 SET_SIZE(pil14_interrupt)
656
657 ENTRY_NP(tick_rtt)
703
704 !
705 ! If we're here, then we have programmed TICK_COMPARE with a %tick
706 ! which is in the past; we'll now load an initial step size, and loop
707 ! until we've managed to program TICK_COMPARE to fire in the future.
708 !
709 mov 8, %o4 ! 8 = arbitrary inital step
710 1: add %o0, %o4, %o5 ! Add the step
711 WR_TICKCMPR(%o5,%g1,%g2,__LINE__) ! Write to TICK_CMPR
712 GET_NATIVE_TIME(%o0,%g1,%g2,__LINE__) ! %o0 = tick
713 cmp %o5, %o0 ! In the future?
714 bg,a,pt %xcc, 2f ! Yes, drive on.
715 wrpr %g0, %g5, %pstate ! delay: enable vec intr
716 ba 1b ! No, try again.
717 sllx %o4, 1, %o4 ! delay: double step size
718
719 2: ba current_thread_complete
720 nop
721 SET_SIZE(tick_rtt)
722
723 /*
724 * Level-15 interrupt prologue.
725 */
726 ENTRY_NP(pil15_interrupt)
727 CPU_ADDR(%g1, %g2)
728 rdpr %tstate, %g6
729 rdpr %tpc, %g5
730 btst TSTATE_PRIV, %g6 ! trap from supervisor mode?
731 bnz,a,pt %xcc, 1f
732 stn %g5, [%g1 + CPU_CPCPROFILE_PC] ! if so, record kernel PC
733 stn %g5, [%g1 + CPU_CPCPROFILE_UPC] ! if not, record user PC
734 ba pil15_epilogue ! must be large-disp branch
735 stn %g0, [%g1 + CPU_CPCPROFILE_PC] ! zero kernel PC
736 1: ba pil15_epilogue ! must be large-disp branch
737 stn %g0, [%g1 + CPU_CPCPROFILE_UPC] ! zero user PC
738 SET_SIZE(pil15_interrupt)
739
740 /* XXXQ These should be inline templates, not functions */
741 ENTRY(prefetch_page_w)
742 retl
743 nop
744 SET_SIZE(prefetch_page_w)
745
746 ENTRY(prefetch_page_r)
747 retl
748 nop
749 SET_SIZE(prefetch_page_r)
750
751 /* XXXQ These should be inline templates, not functions */
752 ENTRY(prefetch_smap_w)
753 retl
754 nop
755 SET_SIZE(prefetch_smap_w)
756
757 /*
758 * Generic sun4v MMU and Cache operations.
759 */
760
761 ENTRY_NP(vtag_flushpage)
762 /*
763 * flush page from the tlb
764 *
765 * %o0 = vaddr
766 * %o1 = sfmmup
767 */
768 SFMMU_CPU_CNUM(%o1, %g1, %g2) /* %g1 = sfmmu cnum on this CPU */
769
770 mov %g1, %o1
771 mov MAP_ITLB | MAP_DTLB, %o2
772 ta MMU_UNMAP_ADDR
773 brz,pt %o0, 1f
774 nop
775 ba panic_bad_hcall
776 mov MMU_UNMAP_ADDR, %o1
777 1:
778 retl
779 nop
780 SET_SIZE(vtag_flushpage)
940 /*
941 * flush_instr_mem:
942 * Flush a portion of the I-$ starting at vaddr
943 * %o0 vaddr
944 * %o1 bytes to be flushed
945 */
946
947 ENTRY(flush_instr_mem)
948 membar #StoreStore ! Ensure the stores
949 ! are globally visible
950 1:
951 flush %o0
952 subcc %o1, ICACHE_FLUSHSZ, %o1 ! bytes = bytes-0x20
953 bgu,pt %ncc, 1b
954 add %o0, ICACHE_FLUSHSZ, %o0 ! vaddr = vaddr+0x20
955
956 retl
957 nop
958 SET_SIZE(flush_instr_mem)
959
960 #if !defined(CUSTOM_FPZERO)
961
962 /*
963 * fp_zero() - clear all fp data registers and the fsr
964 */
965
966 .global fp_zero_zero
967 .align 8
968 fp_zero_zero:
969 .xword 0
970
971 ENTRY_NP(fp_zero)
972 sethi %hi(fp_zero_zero), %o0
973 ldx [%o0 + %lo(fp_zero_zero)], %fsr
974 ldd [%o0 + %lo(fp_zero_zero)], %f0
975 fmovd %f0, %f2
976 fmovd %f0, %f4
977 fmovd %f0, %f6
978 fmovd %f0, %f8
979 fmovd %f0, %f10
980 fmovd %f0, %f12
981 fmovd %f0, %f14
982 fmovd %f0, %f16
983 fmovd %f0, %f18
984 fmovd %f0, %f20
985 fmovd %f0, %f22
989 fmovd %f0, %f30
990 fmovd %f0, %f32
991 fmovd %f0, %f34
992 fmovd %f0, %f36
993 fmovd %f0, %f38
994 fmovd %f0, %f40
995 fmovd %f0, %f42
996 fmovd %f0, %f44
997 fmovd %f0, %f46
998 fmovd %f0, %f48
999 fmovd %f0, %f50
1000 fmovd %f0, %f52
1001 fmovd %f0, %f54
1002 fmovd %f0, %f56
1003 fmovd %f0, %f58
1004 fmovd %f0, %f60
1005 retl
1006 fmovd %f0, %f62
1007 SET_SIZE(fp_zero)
1008
1009 #endif /* CUSTOM_FPZERO */
|