123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446 |
- // The content of this file is x86_64-only:
- #if defined(__x86_64__)
- #include "sanitizer_common/sanitizer_asm.h"
- #if !defined(__APPLE__)
- .section .text
- #else
- .section __TEXT,__text
- #endif
- ASM_HIDDEN(__tsan_trace_switch)
- .globl ASM_SYMBOL(__tsan_trace_switch_thunk)
- ASM_SYMBOL(__tsan_trace_switch_thunk):
- CFI_STARTPROC
- _CET_ENDBR
- # Save scratch registers.
- push %rax
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rax, 0)
- push %rcx
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rcx, 0)
- push %rdx
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdx, 0)
- push %rsi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rsi, 0)
- push %rdi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdi, 0)
- push %r8
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r8, 0)
- push %r9
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r9, 0)
- push %r10
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r10, 0)
- push %r11
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r11, 0)
- # All XMM registers are caller-saved.
- sub $0x100, %rsp
- CFI_ADJUST_CFA_OFFSET(0x100)
- movdqu %xmm0, 0x0(%rsp)
- movdqu %xmm1, 0x10(%rsp)
- movdqu %xmm2, 0x20(%rsp)
- movdqu %xmm3, 0x30(%rsp)
- movdqu %xmm4, 0x40(%rsp)
- movdqu %xmm5, 0x50(%rsp)
- movdqu %xmm6, 0x60(%rsp)
- movdqu %xmm7, 0x70(%rsp)
- movdqu %xmm8, 0x80(%rsp)
- movdqu %xmm9, 0x90(%rsp)
- movdqu %xmm10, 0xa0(%rsp)
- movdqu %xmm11, 0xb0(%rsp)
- movdqu %xmm12, 0xc0(%rsp)
- movdqu %xmm13, 0xd0(%rsp)
- movdqu %xmm14, 0xe0(%rsp)
- movdqu %xmm15, 0xf0(%rsp)
- # Align stack frame.
- push %rbx # non-scratch
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rbx, 0)
- mov %rsp, %rbx # save current rsp
- CFI_DEF_CFA_REGISTER(%rbx)
- shr $4, %rsp # clear 4 lsb, align to 16
- shl $4, %rsp
- call ASM_SYMBOL(__tsan_trace_switch)
- # Unalign stack frame back.
- mov %rbx, %rsp # restore the original rsp
- CFI_DEF_CFA_REGISTER(%rsp)
- pop %rbx
- CFI_ADJUST_CFA_OFFSET(-8)
- # Restore scratch registers.
- movdqu 0x0(%rsp), %xmm0
- movdqu 0x10(%rsp), %xmm1
- movdqu 0x20(%rsp), %xmm2
- movdqu 0x30(%rsp), %xmm3
- movdqu 0x40(%rsp), %xmm4
- movdqu 0x50(%rsp), %xmm5
- movdqu 0x60(%rsp), %xmm6
- movdqu 0x70(%rsp), %xmm7
- movdqu 0x80(%rsp), %xmm8
- movdqu 0x90(%rsp), %xmm9
- movdqu 0xa0(%rsp), %xmm10
- movdqu 0xb0(%rsp), %xmm11
- movdqu 0xc0(%rsp), %xmm12
- movdqu 0xd0(%rsp), %xmm13
- movdqu 0xe0(%rsp), %xmm14
- movdqu 0xf0(%rsp), %xmm15
- add $0x100, %rsp
- CFI_ADJUST_CFA_OFFSET(-0x100)
- pop %r11
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r10
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r9
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r8
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rdi
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rsi
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rdx
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rcx
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rax
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rax)
- CFI_RESTORE(%rbx)
- CFI_RESTORE(%rcx)
- CFI_RESTORE(%rdx)
- CFI_RESTORE(%rsi)
- CFI_RESTORE(%rdi)
- CFI_RESTORE(%r8)
- CFI_RESTORE(%r9)
- CFI_RESTORE(%r10)
- CFI_RESTORE(%r11)
- ret
- CFI_ENDPROC
- ASM_HIDDEN(__tsan_report_race)
- .globl ASM_SYMBOL(__tsan_report_race_thunk)
- ASM_SYMBOL(__tsan_report_race_thunk):
- CFI_STARTPROC
- _CET_ENDBR
- # Save scratch registers.
- push %rax
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rax, 0)
- push %rcx
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rcx, 0)
- push %rdx
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdx, 0)
- push %rsi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rsi, 0)
- push %rdi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdi, 0)
- push %r8
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r8, 0)
- push %r9
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r9, 0)
- push %r10
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r10, 0)
- push %r11
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r11, 0)
- # All XMM registers are caller-saved.
- sub $0x100, %rsp
- CFI_ADJUST_CFA_OFFSET(0x100)
- movdqu %xmm0, 0x0(%rsp)
- movdqu %xmm1, 0x10(%rsp)
- movdqu %xmm2, 0x20(%rsp)
- movdqu %xmm3, 0x30(%rsp)
- movdqu %xmm4, 0x40(%rsp)
- movdqu %xmm5, 0x50(%rsp)
- movdqu %xmm6, 0x60(%rsp)
- movdqu %xmm7, 0x70(%rsp)
- movdqu %xmm8, 0x80(%rsp)
- movdqu %xmm9, 0x90(%rsp)
- movdqu %xmm10, 0xa0(%rsp)
- movdqu %xmm11, 0xb0(%rsp)
- movdqu %xmm12, 0xc0(%rsp)
- movdqu %xmm13, 0xd0(%rsp)
- movdqu %xmm14, 0xe0(%rsp)
- movdqu %xmm15, 0xf0(%rsp)
- # Align stack frame.
- push %rbx # non-scratch
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rbx, 0)
- mov %rsp, %rbx # save current rsp
- CFI_DEF_CFA_REGISTER(%rbx)
- shr $4, %rsp # clear 4 lsb, align to 16
- shl $4, %rsp
- call ASM_SYMBOL(__tsan_report_race)
- # Unalign stack frame back.
- mov %rbx, %rsp # restore the original rsp
- CFI_DEF_CFA_REGISTER(%rsp)
- pop %rbx
- CFI_ADJUST_CFA_OFFSET(-8)
- # Restore scratch registers.
- movdqu 0x0(%rsp), %xmm0
- movdqu 0x10(%rsp), %xmm1
- movdqu 0x20(%rsp), %xmm2
- movdqu 0x30(%rsp), %xmm3
- movdqu 0x40(%rsp), %xmm4
- movdqu 0x50(%rsp), %xmm5
- movdqu 0x60(%rsp), %xmm6
- movdqu 0x70(%rsp), %xmm7
- movdqu 0x80(%rsp), %xmm8
- movdqu 0x90(%rsp), %xmm9
- movdqu 0xa0(%rsp), %xmm10
- movdqu 0xb0(%rsp), %xmm11
- movdqu 0xc0(%rsp), %xmm12
- movdqu 0xd0(%rsp), %xmm13
- movdqu 0xe0(%rsp), %xmm14
- movdqu 0xf0(%rsp), %xmm15
- add $0x100, %rsp
- CFI_ADJUST_CFA_OFFSET(-0x100)
- pop %r11
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r10
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r9
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r8
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rdi
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rsi
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rdx
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rcx
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rax
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rax)
- CFI_RESTORE(%rbx)
- CFI_RESTORE(%rcx)
- CFI_RESTORE(%rdx)
- CFI_RESTORE(%rsi)
- CFI_RESTORE(%rdi)
- CFI_RESTORE(%r8)
- CFI_RESTORE(%r9)
- CFI_RESTORE(%r10)
- CFI_RESTORE(%r11)
- ret
- CFI_ENDPROC
- ASM_HIDDEN(__tsan_setjmp)
- #if defined(__NetBSD__)
- .comm _ZN14__interception15real___setjmp14E,8,8
- #elif !defined(__APPLE__)
- .comm _ZN14__interception11real_setjmpE,8,8
- #endif
- #if defined(__NetBSD__)
- .globl ASM_SYMBOL_INTERCEPTOR(__setjmp14)
- ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
- ASM_SYMBOL_INTERCEPTOR(__setjmp14):
- #else
- .globl ASM_SYMBOL_INTERCEPTOR(setjmp)
- ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
- ASM_SYMBOL_INTERCEPTOR(setjmp):
- #endif
- CFI_STARTPROC
- _CET_ENDBR
- // save env parameter
- push %rdi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdi, 0)
- // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
- #if defined(__FreeBSD__) || defined(__NetBSD__)
- lea 8(%rsp), %rdi
- #elif defined(__linux__) || defined(__APPLE__)
- lea 16(%rsp), %rdi
- #else
- # error "Unknown platform"
- #endif
- // call tsan interceptor
- call ASM_SYMBOL(__tsan_setjmp)
- // restore env parameter
- pop %rdi
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rdi)
- // tail jump to libc setjmp
- movl $0, %eax
- #if defined(__NetBSD__)
- movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx
- jmp *(%rdx)
- #elif !defined(__APPLE__)
- movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
- jmp *(%rdx)
- #else
- jmp ASM_SYMBOL(setjmp)
- #endif
- CFI_ENDPROC
- #if defined(__NetBSD__)
- ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
- #else
- ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
- #endif
- .comm _ZN14__interception12real__setjmpE,8,8
- .globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
- ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
- ASM_SYMBOL_INTERCEPTOR(_setjmp):
- CFI_STARTPROC
- _CET_ENDBR
- // save env parameter
- push %rdi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdi, 0)
- // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
- #if defined(__FreeBSD__) || defined(__NetBSD__)
- lea 8(%rsp), %rdi
- #elif defined(__linux__) || defined(__APPLE__)
- lea 16(%rsp), %rdi
- #else
- # error "Unknown platform"
- #endif
- // call tsan interceptor
- call ASM_SYMBOL(__tsan_setjmp)
- // restore env parameter
- pop %rdi
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rdi)
- // tail jump to libc setjmp
- movl $0, %eax
- #if !defined(__APPLE__)
- movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
- jmp *(%rdx)
- #else
- jmp ASM_SYMBOL(_setjmp)
- #endif
- CFI_ENDPROC
- ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
- #if defined(__NetBSD__)
- .comm _ZN14__interception18real___sigsetjmp14E,8,8
- .globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)
- ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
- ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14):
- #else
- .comm _ZN14__interception14real_sigsetjmpE,8,8
- .globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
- ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
- ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
- #endif
- CFI_STARTPROC
- _CET_ENDBR
- // save env parameter
- push %rdi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdi, 0)
- // save savesigs parameter
- push %rsi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rsi, 0)
- // align stack frame
- sub $8, %rsp
- CFI_ADJUST_CFA_OFFSET(8)
- // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
- #if defined(__FreeBSD__) || defined(__NetBSD__)
- lea 24(%rsp), %rdi
- #elif defined(__linux__) || defined(__APPLE__)
- lea 32(%rsp), %rdi
- #else
- # error "Unknown platform"
- #endif
- // call tsan interceptor
- call ASM_SYMBOL(__tsan_setjmp)
- // unalign stack frame
- add $8, %rsp
- CFI_ADJUST_CFA_OFFSET(-8)
- // restore savesigs parameter
- pop %rsi
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rsi)
- // restore env parameter
- pop %rdi
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rdi)
- // tail jump to libc sigsetjmp
- movl $0, %eax
- #if defined(__NetBSD__)
- movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx
- jmp *(%rdx)
- #elif !defined(__APPLE__)
- movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
- jmp *(%rdx)
- #else
- jmp ASM_SYMBOL(sigsetjmp)
- #endif
- CFI_ENDPROC
- #if defined(__NetBSD__)
- ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
- #else
- ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
- #endif
- #if !defined(__APPLE__) && !defined(__NetBSD__)
- .comm _ZN14__interception16real___sigsetjmpE,8,8
- .globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
- ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
- ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
- CFI_STARTPROC
- _CET_ENDBR
- // save env parameter
- push %rdi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdi, 0)
- // save savesigs parameter
- push %rsi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rsi, 0)
- // align stack frame
- sub $8, %rsp
- CFI_ADJUST_CFA_OFFSET(8)
- // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
- #if defined(__FreeBSD__)
- lea 24(%rsp), %rdi
- #else
- lea 32(%rsp), %rdi
- #endif
- // call tsan interceptor
- call ASM_SYMBOL(__tsan_setjmp)
- // unalign stack frame
- add $8, %rsp
- CFI_ADJUST_CFA_OFFSET(-8)
- // restore savesigs parameter
- pop %rsi
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rsi)
- // restore env parameter
- pop %rdi
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rdi)
- // tail jump to libc sigsetjmp
- movl $0, %eax
- movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
- jmp *(%rdx)
- CFI_ENDPROC
- ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
- #endif // !defined(__APPLE__) && !defined(__NetBSD__)
- NO_EXEC_STACK_DIRECTIVE
- #endif
|