| // The content of this file is x86_64-only: |
| #if defined(__x86_64__) |
| |
| #include "sanitizer_common/sanitizer_asm.h" |
| |
| #if !defined(__APPLE__) |
| .section .text |
| #else |
| .section __TEXT,__text |
| #endif |
| |
| ASM_HIDDEN(__tsan_trace_switch) |
| .globl ASM_SYMBOL(__tsan_trace_switch_thunk) |
| ASM_SYMBOL(__tsan_trace_switch_thunk): |
| CFI_STARTPROC |
| _CET_ENDBR |
| # Save scratch registers. |
| push %rax |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rax, 0) |
| push %rcx |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rcx, 0) |
| push %rdx |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rdx, 0) |
| push %rsi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rsi, 0) |
| push %rdi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rdi, 0) |
| push %r8 |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%r8, 0) |
| push %r9 |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%r9, 0) |
| push %r10 |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%r10, 0) |
| push %r11 |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%r11, 0) |
| # All XMM registers are caller-saved. |
| sub $0x100, %rsp |
| CFI_ADJUST_CFA_OFFSET(0x100) |
| vmovdqu %xmm0, 0x0(%rsp) |
| vmovdqu %xmm1, 0x10(%rsp) |
| vmovdqu %xmm2, 0x20(%rsp) |
| vmovdqu %xmm3, 0x30(%rsp) |
| vmovdqu %xmm4, 0x40(%rsp) |
| vmovdqu %xmm5, 0x50(%rsp) |
| vmovdqu %xmm6, 0x60(%rsp) |
| vmovdqu %xmm7, 0x70(%rsp) |
| vmovdqu %xmm8, 0x80(%rsp) |
| vmovdqu %xmm9, 0x90(%rsp) |
| vmovdqu %xmm10, 0xa0(%rsp) |
| vmovdqu %xmm11, 0xb0(%rsp) |
| vmovdqu %xmm12, 0xc0(%rsp) |
| vmovdqu %xmm13, 0xd0(%rsp) |
| vmovdqu %xmm14, 0xe0(%rsp) |
| vmovdqu %xmm15, 0xf0(%rsp) |
| # Align stack frame. |
| push %rbx # non-scratch |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rbx, 0) |
| mov %rsp, %rbx # save current rsp |
| CFI_DEF_CFA_REGISTER(%rbx) |
| shr $4, %rsp # clear 4 lsb, align to 16 |
| shl $4, %rsp |
| |
| call ASM_SYMBOL(__tsan_trace_switch) |
| |
| # Unalign stack frame back. |
| mov %rbx, %rsp # restore the original rsp |
| CFI_DEF_CFA_REGISTER(%rsp) |
| pop %rbx |
| CFI_ADJUST_CFA_OFFSET(-8) |
| # Restore scratch registers. |
| vmovdqu 0x0(%rsp), %xmm0 |
| vmovdqu 0x10(%rsp), %xmm1 |
| vmovdqu 0x20(%rsp), %xmm2 |
| vmovdqu 0x30(%rsp), %xmm3 |
| vmovdqu 0x40(%rsp), %xmm4 |
| vmovdqu 0x50(%rsp), %xmm5 |
| vmovdqu 0x60(%rsp), %xmm6 |
| vmovdqu 0x70(%rsp), %xmm7 |
| vmovdqu 0x80(%rsp), %xmm8 |
| vmovdqu 0x90(%rsp), %xmm9 |
| vmovdqu 0xa0(%rsp), %xmm10 |
| vmovdqu 0xb0(%rsp), %xmm11 |
| vmovdqu 0xc0(%rsp), %xmm12 |
| vmovdqu 0xd0(%rsp), %xmm13 |
| vmovdqu 0xe0(%rsp), %xmm14 |
| vmovdqu 0xf0(%rsp), %xmm15 |
| add $0x100, %rsp |
| CFI_ADJUST_CFA_OFFSET(-0x100) |
| pop %r11 |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %r10 |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %r9 |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %r8 |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rdi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rsi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rdx |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rcx |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rax |
| CFI_ADJUST_CFA_OFFSET(-8) |
| CFI_RESTORE(%rax) |
| CFI_RESTORE(%rbx) |
| CFI_RESTORE(%rcx) |
| CFI_RESTORE(%rdx) |
| CFI_RESTORE(%rsi) |
| CFI_RESTORE(%rdi) |
| CFI_RESTORE(%r8) |
| CFI_RESTORE(%r9) |
| CFI_RESTORE(%r10) |
| CFI_RESTORE(%r11) |
| ret |
| CFI_ENDPROC |
| |
| ASM_HIDDEN(__tsan_report_race) |
| .globl ASM_SYMBOL(__tsan_report_race_thunk) |
| ASM_SYMBOL(__tsan_report_race_thunk): |
| CFI_STARTPROC |
| _CET_ENDBR |
| # Save scratch registers. |
| push %rax |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rax, 0) |
| push %rcx |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rcx, 0) |
| push %rdx |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rdx, 0) |
| push %rsi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rsi, 0) |
| push %rdi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rdi, 0) |
| push %r8 |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%r8, 0) |
| push %r9 |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%r9, 0) |
| push %r10 |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%r10, 0) |
| push %r11 |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%r11, 0) |
| # All XMM registers are caller-saved. |
| sub $0x100, %rsp |
| CFI_ADJUST_CFA_OFFSET(0x100) |
| vmovdqu %xmm0, 0x0(%rsp) |
| vmovdqu %xmm1, 0x10(%rsp) |
| vmovdqu %xmm2, 0x20(%rsp) |
| vmovdqu %xmm3, 0x30(%rsp) |
| vmovdqu %xmm4, 0x40(%rsp) |
| vmovdqu %xmm5, 0x50(%rsp) |
| vmovdqu %xmm6, 0x60(%rsp) |
| vmovdqu %xmm7, 0x70(%rsp) |
| vmovdqu %xmm8, 0x80(%rsp) |
| vmovdqu %xmm9, 0x90(%rsp) |
| vmovdqu %xmm10, 0xa0(%rsp) |
| vmovdqu %xmm11, 0xb0(%rsp) |
| vmovdqu %xmm12, 0xc0(%rsp) |
| vmovdqu %xmm13, 0xd0(%rsp) |
| vmovdqu %xmm14, 0xe0(%rsp) |
| vmovdqu %xmm15, 0xf0(%rsp) |
| # Align stack frame. |
| push %rbx # non-scratch |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rbx, 0) |
| mov %rsp, %rbx # save current rsp |
| CFI_DEF_CFA_REGISTER(%rbx) |
| shr $4, %rsp # clear 4 lsb, align to 16 |
| shl $4, %rsp |
| |
| call ASM_SYMBOL(__tsan_report_race) |
| |
| # Unalign stack frame back. |
| mov %rbx, %rsp # restore the original rsp |
| CFI_DEF_CFA_REGISTER(%rsp) |
| pop %rbx |
| CFI_ADJUST_CFA_OFFSET(-8) |
| # Restore scratch registers. |
| vmovdqu 0x0(%rsp), %xmm0 |
| vmovdqu 0x10(%rsp), %xmm1 |
| vmovdqu 0x20(%rsp), %xmm2 |
| vmovdqu 0x30(%rsp), %xmm3 |
| vmovdqu 0x40(%rsp), %xmm4 |
| vmovdqu 0x50(%rsp), %xmm5 |
| vmovdqu 0x60(%rsp), %xmm6 |
| vmovdqu 0x70(%rsp), %xmm7 |
| vmovdqu 0x80(%rsp), %xmm8 |
| vmovdqu 0x90(%rsp), %xmm9 |
| vmovdqu 0xa0(%rsp), %xmm10 |
| vmovdqu 0xb0(%rsp), %xmm11 |
| vmovdqu 0xc0(%rsp), %xmm12 |
| vmovdqu 0xd0(%rsp), %xmm13 |
| vmovdqu 0xe0(%rsp), %xmm14 |
| vmovdqu 0xf0(%rsp), %xmm15 |
| add $0x100, %rsp |
| CFI_ADJUST_CFA_OFFSET(-0x100) |
| pop %r11 |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %r10 |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %r9 |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %r8 |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rdi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rsi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rdx |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rcx |
| CFI_ADJUST_CFA_OFFSET(-8) |
| pop %rax |
| CFI_ADJUST_CFA_OFFSET(-8) |
| CFI_RESTORE(%rax) |
| CFI_RESTORE(%rbx) |
| CFI_RESTORE(%rcx) |
| CFI_RESTORE(%rdx) |
| CFI_RESTORE(%rsi) |
| CFI_RESTORE(%rdi) |
| CFI_RESTORE(%r8) |
| CFI_RESTORE(%r9) |
| CFI_RESTORE(%r10) |
| CFI_RESTORE(%r11) |
| ret |
| CFI_ENDPROC |
| |
| ASM_HIDDEN(__tsan_setjmp) |
| #if defined(__NetBSD__) |
| .comm _ZN14__interception15real___setjmp14E,8,8 |
| #elif !defined(__APPLE__) |
| .comm _ZN14__interception11real_setjmpE,8,8 |
| #endif |
| #if defined(__NetBSD__) |
| .globl ASM_SYMBOL_INTERCEPTOR(__setjmp14) |
| ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) |
| ASM_SYMBOL_INTERCEPTOR(__setjmp14): |
| #else |
| .globl ASM_SYMBOL_INTERCEPTOR(setjmp) |
| ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) |
| ASM_SYMBOL_INTERCEPTOR(setjmp): |
| #endif |
| CFI_STARTPROC |
| _CET_ENDBR |
| // save env parameter |
| push %rdi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rdi, 0) |
| // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` |
| #if defined(__FreeBSD__) || defined(__NetBSD__) |
| lea 8(%rsp), %rdi |
| #elif defined(__linux__) || defined(__APPLE__) |
| lea 16(%rsp), %rdi |
| #else |
| # error "Unknown platform" |
| #endif |
| // call tsan interceptor |
| call ASM_SYMBOL(__tsan_setjmp) |
| // restore env parameter |
| pop %rdi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| CFI_RESTORE(%rdi) |
| // tail jump to libc setjmp |
| movl $0, %eax |
| #if defined(__NetBSD__) |
| movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx |
| jmp *(%rdx) |
| #elif !defined(__APPLE__) |
| movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx |
| jmp *(%rdx) |
| #else |
| jmp ASM_SYMBOL(setjmp) |
| #endif |
| CFI_ENDPROC |
| #if defined(__NetBSD__) |
| ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) |
| #else |
| ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) |
| #endif |
| |
| .comm _ZN14__interception12real__setjmpE,8,8 |
| .globl ASM_SYMBOL_INTERCEPTOR(_setjmp) |
| ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) |
| ASM_SYMBOL_INTERCEPTOR(_setjmp): |
| CFI_STARTPROC |
| _CET_ENDBR |
| // save env parameter |
| push %rdi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rdi, 0) |
| // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` |
| #if defined(__FreeBSD__) || defined(__NetBSD__) |
| lea 8(%rsp), %rdi |
| #elif defined(__linux__) || defined(__APPLE__) |
| lea 16(%rsp), %rdi |
| #else |
| # error "Unknown platform" |
| #endif |
| // call tsan interceptor |
| call ASM_SYMBOL(__tsan_setjmp) |
| // restore env parameter |
| pop %rdi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| CFI_RESTORE(%rdi) |
| // tail jump to libc setjmp |
| movl $0, %eax |
| #if !defined(__APPLE__) |
| movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx |
| jmp *(%rdx) |
| #else |
| jmp ASM_SYMBOL(_setjmp) |
| #endif |
| CFI_ENDPROC |
| ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) |
| |
| #if defined(__NetBSD__) |
| .comm _ZN14__interception18real___sigsetjmp14E,8,8 |
| .globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14) |
| ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) |
| ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14): |
| #else |
| .comm _ZN14__interception14real_sigsetjmpE,8,8 |
| .globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) |
| ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) |
| ASM_SYMBOL_INTERCEPTOR(sigsetjmp): |
| #endif |
| CFI_STARTPROC |
| _CET_ENDBR |
| // save env parameter |
| push %rdi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rdi, 0) |
| // save savesigs parameter |
| push %rsi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rsi, 0) |
| // align stack frame |
| sub $8, %rsp |
| CFI_ADJUST_CFA_OFFSET(8) |
| // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` |
| #if defined(__FreeBSD__) || defined(__NetBSD__) |
| lea 24(%rsp), %rdi |
| #elif defined(__linux__) || defined(__APPLE__) |
| lea 32(%rsp), %rdi |
| #else |
| # error "Unknown platform" |
| #endif |
| // call tsan interceptor |
| call ASM_SYMBOL(__tsan_setjmp) |
| // unalign stack frame |
| add $8, %rsp |
| CFI_ADJUST_CFA_OFFSET(-8) |
| // restore savesigs parameter |
| pop %rsi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| CFI_RESTORE(%rsi) |
| // restore env parameter |
| pop %rdi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| CFI_RESTORE(%rdi) |
| // tail jump to libc sigsetjmp |
| movl $0, %eax |
| #if defined(__NetBSD__) |
| movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx |
| jmp *(%rdx) |
| #elif !defined(__APPLE__) |
| movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx |
| jmp *(%rdx) |
| #else |
| jmp ASM_SYMBOL(sigsetjmp) |
| #endif |
| CFI_ENDPROC |
| #if defined(__NetBSD__) |
| ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) |
| #else |
| ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) |
| #endif |
| |
| #if !defined(__APPLE__) && !defined(__NetBSD__) |
| .comm _ZN14__interception16real___sigsetjmpE,8,8 |
| .globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) |
| ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) |
| ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): |
| CFI_STARTPROC |
| _CET_ENDBR |
| // save env parameter |
| push %rdi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rdi, 0) |
| // save savesigs parameter |
| push %rsi |
| CFI_ADJUST_CFA_OFFSET(8) |
| CFI_REL_OFFSET(%rsi, 0) |
| // align stack frame |
| sub $8, %rsp |
| CFI_ADJUST_CFA_OFFSET(8) |
| // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` |
| #if defined(__FreeBSD__) |
| lea 24(%rsp), %rdi |
| #else |
| lea 32(%rsp), %rdi |
| #endif |
| // call tsan interceptor |
| call ASM_SYMBOL(__tsan_setjmp) |
| // unalign stack frame |
| add $8, %rsp |
| CFI_ADJUST_CFA_OFFSET(-8) |
| // restore savesigs parameter |
| pop %rsi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| CFI_RESTORE(%rsi) |
| // restore env parameter |
| pop %rdi |
| CFI_ADJUST_CFA_OFFSET(-8) |
| CFI_RESTORE(%rdi) |
| // tail jump to libc sigsetjmp |
| movl $0, %eax |
| movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx |
| jmp *(%rdx) |
| CFI_ENDPROC |
| ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) |
| #endif // !defined(__APPLE__) && !defined(__NetBSD__) |
| |
| NO_EXEC_STACK_DIRECTIVE |
| |
| #endif |
