| // z_Linux_asm.S: - microtasking routines specifically | |
| // written for Intel platforms running Linux* OS | |
| // | |
| ////===----------------------------------------------------------------------===// | |
| //// | |
| //// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| //// See https://llvm.org/LICENSE.txt for license information. | |
| //// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| //// | |
| ////===----------------------------------------------------------------------===// | |
| // | |
| // ----------------------------------------------------------------------- | |
| // macros | |
| // ----------------------------------------------------------------------- | |
| #include "kmp_config.h" | |
| #if KMP_ARCH_X86 || KMP_ARCH_X86_64 | |
| .att_syntax | |
| # if defined(__ELF__) && defined(__CET__) && defined(__has_include) | |
| # if __has_include(<cet.h>) | |
| # include <cet.h> | |
| # endif | |
| # endif | |
| # if !defined(_CET_ENDBR) | |
| # define _CET_ENDBR | |
| # endif | |
| # if KMP_MIC | |
| // the 'delay r16/r32/r64' should be used instead of the 'pause'. | |
| // The delay operation has the effect of removing the current thread from | |
| // the round-robin HT mechanism, and therefore speeds up the issue rate of | |
| // the other threads on the same core. | |
| // | |
| // A value of 0 works fine for <= 2 threads per core, but causes the EPCC | |
| // barrier time to increase greatly for 3 or more threads per core. | |
| // | |
| // A value of 100 works pretty well for up to 4 threads per core, but isn't | |
| // quite as fast as 0 for 2 threads per core. | |
| // | |
| // We need to check what happens for oversubscription / > 4 threads per core. | |
| // It is possible that we need to pass the delay value in as a parameter | |
| // that the caller determines based on the total # threads / # cores. | |
| // | |
| //.macro pause_op | |
| // mov $100, %rax | |
| // delay %rax | |
| //.endm | |
| # else | |
| # define pause_op .byte 0xf3,0x90 | |
| # endif // KMP_MIC | |
| # if KMP_OS_DARWIN | |
| # define KMP_PREFIX_UNDERSCORE(x) _##x // extra underscore for OS X* symbols | |
| # define KMP_LABEL(x) L_##x // form the name of label | |
| .macro KMP_CFI_DEF_OFFSET | |
| .endmacro | |
| .macro KMP_CFI_OFFSET | |
| .endmacro | |
| .macro KMP_CFI_REGISTER | |
| .endmacro | |
| .macro KMP_CFI_DEF | |
| .endmacro | |
| .macro ALIGN | |
| .align $0 | |
| .endmacro | |
| .macro DEBUG_INFO | |
| /* Not sure what .size does in icc, not sure if we need to do something | |
| similar for OS X*. | |
| */ | |
| .endmacro | |
| .macro PROC | |
| ALIGN 4 | |
| .globl KMP_PREFIX_UNDERSCORE($0) | |
| KMP_PREFIX_UNDERSCORE($0): | |
| _CET_ENDBR | |
| .endmacro | |
| # else // KMP_OS_DARWIN | |
| # define KMP_PREFIX_UNDERSCORE(x) x //no extra underscore for Linux* OS symbols | |
| // Format labels so that they don't override function names in gdb's backtraces | |
| // MIC assembler doesn't accept .L syntax, the L works fine there (as well as | |
| // on OS X*) | |
| # if KMP_MIC | |
| # define KMP_LABEL(x) L_##x // local label | |
| # else | |
| # define KMP_LABEL(x) .L_##x // local label hidden from backtraces | |
| # endif // KMP_MIC | |
| .macro ALIGN size | |
| .align 1<<(\size) | |
| .endm | |
| .macro DEBUG_INFO proc | |
| .cfi_endproc | |
| // Not sure why we need .type and .size for the functions | |
| .align 16 | |
| .type \proc,@function | |
| .size \proc,.-\proc | |
| .endm | |
| .macro PROC proc | |
| ALIGN 4 | |
| .globl KMP_PREFIX_UNDERSCORE(\proc) | |
| KMP_PREFIX_UNDERSCORE(\proc): | |
| .cfi_startproc | |
| _CET_ENDBR | |
| .endm | |
| .macro KMP_CFI_DEF_OFFSET sz | |
| .cfi_def_cfa_offset \sz | |
| .endm | |
| .macro KMP_CFI_OFFSET reg, sz | |
| .cfi_offset \reg,\sz | |
| .endm | |
| .macro KMP_CFI_REGISTER reg | |
| .cfi_def_cfa_register \reg | |
| .endm | |
| .macro KMP_CFI_DEF reg, sz | |
| .cfi_def_cfa \reg,\sz | |
| .endm | |
| # endif // KMP_OS_DARWIN | |
| #endif // KMP_ARCH_X86 || KMP_ARCH_x86_64 | |
| #if KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32 || KMP_ARCH_ARM || KMP_ARCH_ARM64EC | |
| # if KMP_OS_DARWIN | |
| # define KMP_PREFIX_UNDERSCORE(x) _##x // extra underscore for OS X* symbols | |
| # define KMP_LABEL(x) L_##x // form the name of label | |
| .macro ALIGN | |
| .align $0 | |
| .endmacro | |
| .macro DEBUG_INFO | |
| /* Not sure what .size does in icc, not sure if we need to do something | |
| similar for OS X*. | |
| */ | |
| .endmacro | |
| .macro PROC | |
| ALIGN 4 | |
| .globl KMP_PREFIX_UNDERSCORE($0) | |
| KMP_PREFIX_UNDERSCORE($0): | |
| .endmacro | |
| # elif KMP_OS_WINDOWS | |
| # define KMP_PREFIX_UNDERSCORE(x) x // no extra underscore for Windows/ARM64 symbols | |
| // Format labels so that they don't override function names in gdb's backtraces | |
| # define KMP_LABEL(x) .L_##x // local label hidden from backtraces | |
| .macro ALIGN size | |
| .align 1<<(\size) | |
| .endm | |
| .macro DEBUG_INFO proc | |
| ALIGN 2 | |
| .endm | |
| .macro PROC proc | |
| ALIGN 2 | |
| .globl KMP_PREFIX_UNDERSCORE(\proc) | |
| KMP_PREFIX_UNDERSCORE(\proc): | |
| .endm | |
| # else // KMP_OS_DARWIN || KMP_OS_WINDOWS | |
| # define KMP_PREFIX_UNDERSCORE(x) x // no extra underscore for Linux* OS symbols | |
| // Format labels so that they don't override function names in gdb's backtraces | |
| # define KMP_LABEL(x) .L_##x // local label hidden from backtraces | |
| .macro ALIGN size | |
| .align 1<<(\size) | |
| .endm | |
| .macro DEBUG_INFO proc | |
| .cfi_endproc | |
| // Not sure why we need .type and .size for the functions | |
| ALIGN 2 | |
| #if KMP_ARCH_ARM | |
| .type \proc,%function | |
| #else | |
| .type \proc,@function | |
| #endif | |
| .size \proc,.-\proc | |
| .endm | |
| .macro PROC proc | |
| ALIGN 2 | |
| .globl KMP_PREFIX_UNDERSCORE(\proc) | |
| KMP_PREFIX_UNDERSCORE(\proc): | |
| .cfi_startproc | |
| .endm | |
| # endif // KMP_OS_DARWIN | |
| # if KMP_OS_LINUX || KMP_OS_OPENBSD | |
| // BTI and PAC gnu property note | |
| # define NT_GNU_PROPERTY_TYPE_0 5 | |
| # define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 | |
| # define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1 | |
| # define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2 | |
| # define GNU_PROPERTY(type, value) \ | |
| .pushsection .note.gnu.property, "a"; \ | |
| .p2align 3; \ | |
| .word 4; \ | |
| .word 16; \ | |
| .word NT_GNU_PROPERTY_TYPE_0; \ | |
| .asciz "GNU"; \ | |
| .word type; \ | |
| .word 4; \ | |
| .word value; \ | |
| .word 0; \ | |
| .popsection | |
| # endif | |
| # if defined(__ARM_FEATURE_BTI_DEFAULT) | |
| # define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI | |
| # else | |
| # define BTI_FLAG 0 | |
| # endif | |
| # if __ARM_FEATURE_PAC_DEFAULT & 3 | |
| # define PAC_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_PAC | |
| # else | |
| # define PAC_FLAG 0 | |
| # endif | |
| # if (BTI_FLAG | PAC_FLAG) != 0 | |
| # if PAC_FLAG != 0 | |
| # define PACBTI_C hint #25 | |
| # define PACBTI_RET hint #29 | |
| # else | |
| # define PACBTI_C hint #34 | |
| # define PACBTI_RET | |
| # endif | |
| # define GNU_PROPERTY_BTI_PAC \ | |
| GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG | PAC_FLAG) | |
| # else | |
| # define PACBTI_C | |
| # define PACBTI_RET | |
| # define GNU_PROPERTY_BTI_PAC | |
| # endif | |
| #endif // KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32 || KMP_ARCH_ARM | |
| .macro COMMON name, size, align_power | |
| #if KMP_OS_DARWIN | |
| .comm \name, \size | |
| #elif KMP_OS_WINDOWS | |
| .comm \name, \size, \align_power | |
| #else // !KMP_OS_DARWIN && !KMP_OS_WINDOWS | |
| .comm \name, \size, (1<<(\align_power)) | |
| #endif | |
| .endm | |
| // ----------------------------------------------------------------------- | |
| // data | |
| // ----------------------------------------------------------------------- | |
| #ifdef KMP_GOMP_COMPAT | |
| // Support for unnamed common blocks. | |
| // | |
| // Because the symbol ".gomp_critical_user_" contains a ".", we have to | |
| // put this stuff in assembly. | |
| # if KMP_ARCH_X86 | |
| # if KMP_OS_DARWIN | |
| .data | |
| .comm .gomp_critical_user_,32 | |
| .data | |
| .globl ___kmp_unnamed_critical_addr | |
| ___kmp_unnamed_critical_addr: | |
| .long .gomp_critical_user_ | |
| # else /* Linux* OS */ | |
| .data | |
| .comm .gomp_critical_user_,32,8 | |
| .data | |
| ALIGN 4 | |
| .global __kmp_unnamed_critical_addr | |
| __kmp_unnamed_critical_addr: | |
| .4byte .gomp_critical_user_ | |
| .type __kmp_unnamed_critical_addr,@object | |
| .size __kmp_unnamed_critical_addr,4 | |
| # endif /* KMP_OS_DARWIN */ | |
| # endif /* KMP_ARCH_X86 */ | |
| # if KMP_ARCH_X86_64 | |
| # if KMP_OS_DARWIN | |
| .data | |
| .comm .gomp_critical_user_,32 | |
| .data | |
| .globl ___kmp_unnamed_critical_addr | |
| ___kmp_unnamed_critical_addr: | |
| .quad .gomp_critical_user_ | |
| # else /* Linux* OS */ | |
| .data | |
| .comm .gomp_critical_user_,32,8 | |
| .data | |
| ALIGN 8 | |
| .global __kmp_unnamed_critical_addr | |
| __kmp_unnamed_critical_addr: | |
| .8byte .gomp_critical_user_ | |
| .type __kmp_unnamed_critical_addr,@object | |
| .size __kmp_unnamed_critical_addr,8 | |
| # endif /* KMP_OS_DARWIN */ | |
| # endif /* KMP_ARCH_X86_64 */ | |
| #endif /* KMP_GOMP_COMPAT */ | |
| #if KMP_ARCH_X86 && !KMP_ARCH_PPC64 | |
| // ----------------------------------------------------------------------- | |
| // microtasking routines specifically written for IA-32 architecture | |
| // running Linux* OS | |
| // ----------------------------------------------------------------------- | |
| .ident "Intel Corporation" | |
| .data | |
| ALIGN 4 | |
| // void | |
| // __kmp_x86_pause( void ); | |
| .text | |
| PROC __kmp_x86_pause | |
| pause_op | |
| ret | |
| DEBUG_INFO __kmp_x86_pause | |
| # if !KMP_ASM_INTRINS | |
| //------------------------------------------------------------------------ | |
| // kmp_int32 | |
| // __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 d ); | |
| PROC __kmp_test_then_add32 | |
| movl 4(%esp), %ecx | |
| movl 8(%esp), %eax | |
| lock | |
| xaddl %eax,(%ecx) | |
| ret | |
| DEBUG_INFO __kmp_test_then_add32 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_fixed8 | |
| // | |
| // kmp_int32 | |
| // __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 d ); | |
| // | |
| // parameters: | |
| // p: 4(%esp) | |
| // d: 8(%esp) | |
| // | |
| // return: %al | |
| PROC __kmp_xchg_fixed8 | |
| movl 4(%esp), %ecx // "p" | |
| movb 8(%esp), %al // "d" | |
| lock | |
| xchgb %al,(%ecx) | |
| ret | |
| DEBUG_INFO __kmp_xchg_fixed8 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_fixed16 | |
| // | |
| // kmp_int16 | |
| // __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 d ); | |
| // | |
| // parameters: | |
| // p: 4(%esp) | |
| // d: 8(%esp) | |
| // return: %ax | |
| PROC __kmp_xchg_fixed16 | |
| movl 4(%esp), %ecx // "p" | |
| movw 8(%esp), %ax // "d" | |
| lock | |
| xchgw %ax,(%ecx) | |
| ret | |
| DEBUG_INFO __kmp_xchg_fixed16 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_fixed32 | |
| // | |
| // kmp_int32 | |
| // __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 d ); | |
| // | |
| // parameters: | |
| // p: 4(%esp) | |
| // d: 8(%esp) | |
| // | |
| // return: %eax | |
| PROC __kmp_xchg_fixed32 | |
| movl 4(%esp), %ecx // "p" | |
| movl 8(%esp), %eax // "d" | |
| lock | |
| xchgl %eax,(%ecx) | |
| ret | |
| DEBUG_INFO __kmp_xchg_fixed32 | |
| // kmp_int8 | |
| // __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv ); | |
| PROC __kmp_compare_and_store8 | |
| movl 4(%esp), %ecx | |
| movb 8(%esp), %al | |
| movb 12(%esp), %dl | |
| lock | |
| cmpxchgb %dl,(%ecx) | |
| sete %al // if %al == (%ecx) set %al = 1 else set %al = 0 | |
| and $1, %eax // sign extend previous instruction | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store8 | |
| // kmp_int16 | |
| // __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv); | |
| PROC __kmp_compare_and_store16 | |
| movl 4(%esp), %ecx | |
| movw 8(%esp), %ax | |
| movw 12(%esp), %dx | |
| lock | |
| cmpxchgw %dx,(%ecx) | |
| sete %al // if %ax == (%ecx) set %al = 1 else set %al = 0 | |
| and $1, %eax // sign extend previous instruction | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store16 | |
| // kmp_int32 | |
| // __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv); | |
| PROC __kmp_compare_and_store32 | |
| movl 4(%esp), %ecx | |
| movl 8(%esp), %eax | |
| movl 12(%esp), %edx | |
| lock | |
| cmpxchgl %edx,(%ecx) | |
| sete %al // if %eax == (%ecx) set %al = 1 else set %al = 0 | |
| and $1, %eax // sign extend previous instruction | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store32 | |
| // kmp_int32 | |
| // __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 s ); | |
| PROC __kmp_compare_and_store64 | |
| pushl %ebp | |
| movl %esp, %ebp | |
| pushl %ebx | |
| pushl %edi | |
| movl 8(%ebp), %edi | |
| movl 12(%ebp), %eax // "cv" low order word | |
| movl 16(%ebp), %edx // "cv" high order word | |
| movl 20(%ebp), %ebx // "sv" low order word | |
| movl 24(%ebp), %ecx // "sv" high order word | |
| lock | |
| cmpxchg8b (%edi) | |
| sete %al // if %edx:eax == (%edi) set %al = 1 else set %al = 0 | |
| and $1, %eax // sign extend previous instruction | |
| popl %edi | |
| popl %ebx | |
| movl %ebp, %esp | |
| popl %ebp | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store64 | |
| // kmp_int8 | |
| // __kmp_compare_and_store_ret8(volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv); | |
| PROC __kmp_compare_and_store_ret8 | |
| movl 4(%esp), %ecx | |
| movb 8(%esp), %al | |
| movb 12(%esp), %dl | |
| lock | |
| cmpxchgb %dl,(%ecx) | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store_ret8 | |
| // kmp_int16 | |
| // __kmp_compare_and_store_ret16(volatile kmp_int16 *p, kmp_int16 cv, | |
| // kmp_int16 sv); | |
| PROC __kmp_compare_and_store_ret16 | |
| movl 4(%esp), %ecx | |
| movw 8(%esp), %ax | |
| movw 12(%esp), %dx | |
| lock | |
| cmpxchgw %dx,(%ecx) | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store_ret16 | |
| // kmp_int32 | |
| // __kmp_compare_and_store_ret32(volatile kmp_int32 *p, kmp_int32 cv, | |
| // kmp_int32 sv); | |
| PROC __kmp_compare_and_store_ret32 | |
| movl 4(%esp), %ecx | |
| movl 8(%esp), %eax | |
| movl 12(%esp), %edx | |
| lock | |
| cmpxchgl %edx,(%ecx) | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store_ret32 | |
| // kmp_int64 | |
| // __kmp_compare_and_store_ret64(volatile kmp_int64 *p, kmp_int64 cv, | |
| // kmp_int64 sv); | |
| PROC __kmp_compare_and_store_ret64 | |
| pushl %ebp | |
| movl %esp, %ebp | |
| pushl %ebx | |
| pushl %edi | |
| movl 8(%ebp), %edi | |
| movl 12(%ebp), %eax // "cv" low order word | |
| movl 16(%ebp), %edx // "cv" high order word | |
| movl 20(%ebp), %ebx // "sv" low order word | |
| movl 24(%ebp), %ecx // "sv" high order word | |
| lock | |
| cmpxchg8b (%edi) | |
| popl %edi | |
| popl %ebx | |
| movl %ebp, %esp | |
| popl %ebp | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store_ret64 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_real32 | |
| // | |
| // kmp_real32 | |
| // __kmp_xchg_real32( volatile kmp_real32 *addr, kmp_real32 data ); | |
| // | |
| // parameters: | |
| // addr: 4(%esp) | |
| // data: 8(%esp) | |
| // | |
| // return: %eax | |
| PROC __kmp_xchg_real32 | |
| pushl %ebp | |
| movl %esp, %ebp | |
| subl $4, %esp | |
| pushl %esi | |
| movl 4(%ebp), %esi | |
| flds (%esi) | |
| // load <addr> | |
| fsts -4(%ebp) | |
| // store old value | |
| movl 8(%ebp), %eax | |
| lock | |
| xchgl %eax, (%esi) | |
| flds -4(%ebp) | |
| // return old value | |
| popl %esi | |
| movl %ebp, %esp | |
| popl %ebp | |
| ret | |
| DEBUG_INFO __kmp_xchg_real32 | |
| # endif /* !KMP_ASM_INTRINS */ | |
| //------------------------------------------------------------------------ | |
| // int | |
| // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...), | |
| // int gtid, int tid, | |
| // int argc, void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)( & gtid, & tid, argv[0], ... ); | |
| // return 1; | |
| // } | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| PROC __kmp_invoke_microtask | |
| pushl %ebp | |
| KMP_CFI_DEF_OFFSET 8 | |
| KMP_CFI_OFFSET ebp,-8 | |
| movl %esp,%ebp // establish the base pointer for this routine. | |
| KMP_CFI_REGISTER ebp | |
| subl $8,%esp // allocate space for two local variables. | |
| // These varibales are: | |
| // argv: -4(%ebp) | |
| // temp: -8(%ebp) | |
| // | |
| pushl %ebx // save %ebx to use during this routine | |
| // | |
| #if OMPT_SUPPORT | |
| movl 28(%ebp),%ebx // get exit_frame address | |
| movl %ebp,(%ebx) // save exit_frame | |
| #endif | |
| movl 20(%ebp),%ebx // Stack alignment - # args | |
| addl $2,%ebx // #args +2 Always pass at least 2 args (gtid and tid) | |
| shll $2,%ebx // Number of bytes used on stack: (#args+2)*4 | |
| movl %esp,%eax // | |
| subl %ebx,%eax // %esp-((#args+2)*4) -> %eax -- without mods, stack ptr would be this | |
| movl %eax,%ebx // Save to %ebx | |
| andl $0xFFFFFF80,%eax // mask off 7 bits | |
| subl %eax,%ebx // Amount to subtract from %esp | |
| subl %ebx,%esp // Prepare the stack ptr -- | |
| // now it will be aligned on 128-byte boundary at the call | |
| movl 24(%ebp),%eax // copy from p_argv[] | |
| movl %eax,-4(%ebp) // into the local variable *argv. | |
| movl 20(%ebp),%ebx // argc is 20(%ebp) | |
| shll $2,%ebx | |
| KMP_LABEL(invoke_2): | |
| cmpl $0,%ebx | |
| jg KMP_LABEL(invoke_4) | |
| jmp KMP_LABEL(invoke_3) | |
| ALIGN 2 | |
| KMP_LABEL(invoke_4): | |
| movl -4(%ebp),%eax | |
| subl $4,%ebx // decrement argc. | |
| addl %ebx,%eax // index into argv. | |
| movl (%eax),%edx | |
| pushl %edx | |
| jmp KMP_LABEL(invoke_2) | |
| ALIGN 2 | |
| KMP_LABEL(invoke_3): | |
| leal 16(%ebp),%eax // push & tid | |
| pushl %eax | |
| leal 12(%ebp),%eax // push & gtid | |
| pushl %eax | |
| movl 8(%ebp),%ebx | |
| call *%ebx // call (*pkfn)(); | |
| movl $1,%eax // return 1; | |
| movl -12(%ebp),%ebx // restore %ebx | |
| leave | |
| KMP_CFI_DEF esp,4 | |
| ret | |
| DEBUG_INFO __kmp_invoke_microtask | |
| // -- End __kmp_invoke_microtask | |
| // kmp_uint64 | |
| // __kmp_hardware_timestamp(void) | |
| PROC __kmp_hardware_timestamp | |
| rdtsc | |
| ret | |
| DEBUG_INFO __kmp_hardware_timestamp | |
| // -- End __kmp_hardware_timestamp | |
| #endif /* KMP_ARCH_X86 */ | |
| #if KMP_ARCH_X86_64 | |
| // ----------------------------------------------------------------------- | |
| // microtasking routines specifically written for IA-32 architecture and | |
| // Intel(R) 64 running Linux* OS | |
| // ----------------------------------------------------------------------- | |
| // -- Machine type P | |
| // mark_description "Intel Corporation"; | |
| .ident "Intel Corporation" | |
| // -- .file "z_Linux_asm.S" | |
| .data | |
| ALIGN 4 | |
| // To prevent getting our code into .data section .text added to every routine | |
| // definition for x86_64. | |
| //------------------------------------------------------------------------ | |
| # if !KMP_ASM_INTRINS | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_test_then_add32 | |
| // | |
| // kmp_int32 | |
| // __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 d ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // d: %esi | |
| // | |
| // return: %eax | |
| .text | |
| PROC __kmp_test_then_add32 | |
| movl %esi, %eax // "d" | |
| lock | |
| xaddl %eax,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_test_then_add32 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_test_then_add64 | |
| // | |
| // kmp_int64 | |
| // __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // d: %rsi | |
| // return: %rax | |
| .text | |
| PROC __kmp_test_then_add64 | |
| movq %rsi, %rax // "d" | |
| lock | |
| xaddq %rax,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_test_then_add64 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_fixed8 | |
| // | |
| // kmp_int32 | |
| // __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 d ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // d: %sil | |
| // | |
| // return: %al | |
| .text | |
| PROC __kmp_xchg_fixed8 | |
| movb %sil, %al // "d" | |
| lock | |
| xchgb %al,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_xchg_fixed8 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_fixed16 | |
| // | |
| // kmp_int16 | |
| // __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 d ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // d: %si | |
| // return: %ax | |
| .text | |
| PROC __kmp_xchg_fixed16 | |
| movw %si, %ax // "d" | |
| lock | |
| xchgw %ax,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_xchg_fixed16 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_fixed32 | |
| // | |
| // kmp_int32 | |
| // __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 d ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // d: %esi | |
| // | |
| // return: %eax | |
| .text | |
| PROC __kmp_xchg_fixed32 | |
| movl %esi, %eax // "d" | |
| lock | |
| xchgl %eax,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_xchg_fixed32 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_fixed64 | |
| // | |
| // kmp_int64 | |
| // __kmp_xchg_fixed64( volatile kmp_int64 *p, kmp_int64 d ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // d: %rsi | |
| // return: %rax | |
| .text | |
| PROC __kmp_xchg_fixed64 | |
| movq %rsi, %rax // "d" | |
| lock | |
| xchgq %rax,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_xchg_fixed64 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_compare_and_store8 | |
| // | |
| // kmp_int8 | |
| // __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // cv: %esi | |
| // sv: %edx | |
| // | |
| // return: %eax | |
| .text | |
| PROC __kmp_compare_and_store8 | |
| movb %sil, %al // "cv" | |
| lock | |
| cmpxchgb %dl,(%rdi) | |
| sete %al // if %al == (%rdi) set %al = 1 else set %al = 0 | |
| andq $1, %rax // sign extend previous instruction for return value | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store8 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_compare_and_store16 | |
| // | |
| // kmp_int16 | |
| // __kmp_compare_and_store16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // cv: %si | |
| // sv: %dx | |
| // | |
| // return: %eax | |
| .text | |
| PROC __kmp_compare_and_store16 | |
| movw %si, %ax // "cv" | |
| lock | |
| cmpxchgw %dx,(%rdi) | |
| sete %al // if %ax == (%rdi) set %al = 1 else set %al = 0 | |
| andq $1, %rax // sign extend previous instruction for return value | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store16 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_compare_and_store32 | |
| // | |
| // kmp_int32 | |
| // __kmp_compare_and_store32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // cv: %esi | |
| // sv: %edx | |
| // | |
| // return: %eax | |
| .text | |
| PROC __kmp_compare_and_store32 | |
| movl %esi, %eax // "cv" | |
| lock | |
| cmpxchgl %edx,(%rdi) | |
| sete %al // if %eax == (%rdi) set %al = 1 else set %al = 0 | |
| andq $1, %rax // sign extend previous instruction for return value | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store32 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_compare_and_store64 | |
| // | |
| // kmp_int32 | |
| // __kmp_compare_and_store64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // cv: %rsi | |
| // sv: %rdx | |
| // return: %eax | |
| .text | |
| PROC __kmp_compare_and_store64 | |
| movq %rsi, %rax // "cv" | |
| lock | |
| cmpxchgq %rdx,(%rdi) | |
| sete %al // if %rax == (%rdi) set %al = 1 else set %al = 0 | |
| andq $1, %rax // sign extend previous instruction for return value | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store64 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_compare_and_store_ret8 | |
| // | |
| // kmp_int8 | |
| // __kmp_compare_and_store_ret8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // cv: %esi | |
| // sv: %edx | |
| // | |
| // return: %eax | |
| .text | |
| PROC __kmp_compare_and_store_ret8 | |
| movb %sil, %al // "cv" | |
| lock | |
| cmpxchgb %dl,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store_ret8 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_compare_and_store_ret16 | |
| // | |
| // kmp_int16 | |
| // __kmp_compare_and_store16_ret( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // cv: %si | |
| // sv: %dx | |
| // | |
| // return: %eax | |
| .text | |
| PROC __kmp_compare_and_store_ret16 | |
| movw %si, %ax // "cv" | |
| lock | |
| cmpxchgw %dx,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store_ret16 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_compare_and_store_ret32 | |
| // | |
| // kmp_int32 | |
| // __kmp_compare_and_store_ret32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // cv: %esi | |
| // sv: %edx | |
| // | |
| // return: %eax | |
| .text | |
| PROC __kmp_compare_and_store_ret32 | |
| movl %esi, %eax // "cv" | |
| lock | |
| cmpxchgl %edx,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store_ret32 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_compare_and_store_ret64 | |
| // | |
| // kmp_int64 | |
| // __kmp_compare_and_store_ret64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv ); | |
| // | |
| // parameters: | |
| // p: %rdi | |
| // cv: %rsi | |
| // sv: %rdx | |
| // return: %eax | |
| .text | |
| PROC __kmp_compare_and_store_ret64 | |
| movq %rsi, %rax // "cv" | |
| lock | |
| cmpxchgq %rdx,(%rdi) | |
| ret | |
| DEBUG_INFO __kmp_compare_and_store_ret64 | |
| # endif /* !KMP_ASM_INTRINS */ | |
| # if !KMP_MIC | |
| # if !KMP_ASM_INTRINS | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_real32 | |
| // | |
| // kmp_real32 | |
| // __kmp_xchg_real32( volatile kmp_real32 *addr, kmp_real32 data ); | |
| // | |
| // parameters: | |
| // addr: %rdi | |
| // data: %xmm0 (lower 4 bytes) | |
| // | |
| // return: %xmm0 (lower 4 bytes) | |
| .text | |
| PROC __kmp_xchg_real32 | |
| movd %xmm0, %eax // load "data" to eax | |
| lock | |
| xchgl %eax, (%rdi) | |
| movd %eax, %xmm0 // load old value into return register | |
| ret | |
| DEBUG_INFO __kmp_xchg_real32 | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_xchg_real64 | |
| // | |
| // kmp_real64 | |
| // __kmp_xchg_real64( volatile kmp_real64 *addr, kmp_real64 data ); | |
| // | |
| // parameters: | |
| // addr: %rdi | |
| // data: %xmm0 (lower 8 bytes) | |
| // return: %xmm0 (lower 8 bytes) | |
| .text | |
| PROC __kmp_xchg_real64 | |
| movd %xmm0, %rax // load "data" to rax | |
| lock | |
| xchgq %rax, (%rdi) | |
| movd %rax, %xmm0 // load old value into return register | |
| ret | |
| DEBUG_INFO __kmp_xchg_real64 | |
| # endif /* !KMP_MIC */ | |
| # endif /* !KMP_ASM_INTRINS */ | |
| //------------------------------------------------------------------------ | |
| // int | |
| // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...), | |
| // int gtid, int tid, | |
| // int argc, void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)( & gtid, & tid, argv[0], ... ); | |
| // return 1; | |
| // } | |
| // | |
| // note: at call to pkfn must have %rsp 128-byte aligned for compiler | |
| // | |
| // parameters: | |
| // %rdi: pkfn | |
| // %esi: gtid | |
| // %edx: tid | |
| // %ecx: argc | |
| // %r8: p_argv | |
| // %r9: &exit_frame | |
| // | |
| // locals: | |
| // __gtid: gtid parm pushed on stack so can pass >id to pkfn | |
| // __tid: tid parm pushed on stack so can pass &tid to pkfn | |
| // | |
| // reg temps: | |
| // %rax: used all over the place | |
| // %rdx: used in stack pointer alignment calculation | |
| // %r11: used to traverse p_argv array | |
| // %rsi: used as temporary for stack parameters | |
| // used as temporary for number of pkfn parms to push | |
| // %rbx: used to hold pkfn address, and zero constant, callee-save | |
| // | |
| // return: %eax (always 1/TRUE) | |
| __gtid = -16 | |
| __tid = -24 | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| .text | |
| PROC __kmp_invoke_microtask | |
| pushq %rbp // save base pointer | |
| KMP_CFI_DEF_OFFSET 16 | |
| KMP_CFI_OFFSET rbp,-16 | |
| movq %rsp,%rbp // establish the base pointer for this routine. | |
| KMP_CFI_REGISTER rbp | |
| #if OMPT_SUPPORT | |
| movq %rbp, (%r9) // save exit_frame | |
| #endif | |
| pushq %rbx // %rbx is callee-saved register | |
| pushq %rsi // Put gtid on stack so can pass &tgid to pkfn | |
| pushq %rdx // Put tid on stack so can pass &tid to pkfn | |
| movq %rcx, %rax // Stack alignment calculation begins; argc -> %rax | |
| movq $0, %rbx // constant for cmovs later | |
| subq $4, %rax // subtract four args passed in registers to pkfn | |
| #if KMP_MIC | |
| js KMP_LABEL(kmp_0) // jump to movq | |
| jmp KMP_LABEL(kmp_0_exit) // jump ahead | |
| KMP_LABEL(kmp_0): | |
| movq %rbx, %rax // zero negative value in %rax <- max(0, argc-4) | |
| KMP_LABEL(kmp_0_exit): | |
| #else | |
| cmovsq %rbx, %rax // zero negative value in %rax <- max(0, argc-4) | |
| #endif // KMP_MIC | |
| movq %rax, %rsi // save max(0, argc-4) -> %rsi for later | |
| shlq $3, %rax // Number of bytes used on stack: max(0, argc-4)*8 | |
| movq %rsp, %rdx // | |
| subq %rax, %rdx // %rsp-(max(0,argc-4)*8) -> %rdx -- | |
| // without align, stack ptr would be this | |
| movq %rdx, %rax // Save to %rax | |
| andq $0xFFFFFFFFFFFFFF80, %rax // mask off lower 7 bits (128 bytes align) | |
| subq %rax, %rdx // Amount to subtract from %rsp | |
| subq %rdx, %rsp // Prepare the stack ptr -- | |
| // now %rsp will align to 128-byte boundary at call site | |
| // setup pkfn parameter reg and stack | |
| movq %rcx, %rax // argc -> %rax | |
| cmpq $0, %rsi | |
| je KMP_LABEL(kmp_invoke_pass_parms) // jump ahead if no parms to push | |
| shlq $3, %rcx // argc*8 -> %rcx | |
| movq %r8, %rdx // p_argv -> %rdx | |
| addq %rcx, %rdx // &p_argv[argc] -> %rdx | |
| movq %rsi, %rcx // max (0, argc-4) -> %rcx | |
| KMP_LABEL(kmp_invoke_push_parms): | |
| // push nth - 7th parms to pkfn on stack | |
| subq $8, %rdx // decrement p_argv pointer to previous parm | |
| movq (%rdx), %rsi // p_argv[%rcx-1] -> %rsi | |
| pushq %rsi // push p_argv[%rcx-1] onto stack (reverse order) | |
| subl $1, %ecx | |
| // C69570: "X86_64_RELOC_BRANCH not supported" error at linking on mac_32e | |
| // if the name of the label that is an operand of this jecxz starts with a dot ("."); | |
| // Apple's linker does not support 1-byte length relocation; | |
| // Resolution: replace all .labelX entries with L_labelX. | |
| jecxz KMP_LABEL(kmp_invoke_pass_parms) // stop when four p_argv[] parms left | |
| jmp KMP_LABEL(kmp_invoke_push_parms) | |
| ALIGN 3 | |
| KMP_LABEL(kmp_invoke_pass_parms): // put 1st - 6th parms to pkfn in registers. | |
| // order here is important to avoid trashing | |
| // registers used for both input and output parms! | |
| movq %rdi, %rbx // pkfn -> %rbx | |
| leaq __gtid(%rbp), %rdi // >id -> %rdi (store 1st parm to pkfn) | |
| leaq __tid(%rbp), %rsi // &tid -> %rsi (store 2nd parm to pkfn) | |
| // Check if argc is 0 | |
| cmpq $0, %rax | |
| je KMP_LABEL(kmp_no_args) // Jump ahead | |
| movq %r8, %r11 // p_argv -> %r11 | |
| #if KMP_MIC | |
| cmpq $4, %rax // argc >= 4? | |
| jns KMP_LABEL(kmp_4) // jump to movq | |
| jmp KMP_LABEL(kmp_4_exit) // jump ahead | |
| KMP_LABEL(kmp_4): | |
| movq 24(%r11), %r9 // p_argv[3] -> %r9 (store 6th parm to pkfn) | |
| KMP_LABEL(kmp_4_exit): | |
| cmpq $3, %rax // argc >= 3? | |
| jns KMP_LABEL(kmp_3) // jump to movq | |
| jmp KMP_LABEL(kmp_3_exit) // jump ahead | |
| KMP_LABEL(kmp_3): | |
| movq 16(%r11), %r8 // p_argv[2] -> %r8 (store 5th parm to pkfn) | |
| KMP_LABEL(kmp_3_exit): | |
| cmpq $2, %rax // argc >= 2? | |
| jns KMP_LABEL(kmp_2) // jump to movq | |
| jmp KMP_LABEL(kmp_2_exit) // jump ahead | |
| KMP_LABEL(kmp_2): | |
| movq 8(%r11), %rcx // p_argv[1] -> %rcx (store 4th parm to pkfn) | |
| KMP_LABEL(kmp_2_exit): | |
| cmpq $1, %rax // argc >= 1? | |
| jns KMP_LABEL(kmp_1) // jump to movq | |
| jmp KMP_LABEL(kmp_1_exit) // jump ahead | |
| KMP_LABEL(kmp_1): | |
| movq (%r11), %rdx // p_argv[0] -> %rdx (store 3rd parm to pkfn) | |
| KMP_LABEL(kmp_1_exit): | |
| #else | |
| cmpq $4, %rax // argc >= 4? | |
| cmovnsq 24(%r11), %r9 // p_argv[3] -> %r9 (store 6th parm to pkfn) | |
| cmpq $3, %rax // argc >= 3? | |
| cmovnsq 16(%r11), %r8 // p_argv[2] -> %r8 (store 5th parm to pkfn) | |
| cmpq $2, %rax // argc >= 2? | |
| cmovnsq 8(%r11), %rcx // p_argv[1] -> %rcx (store 4th parm to pkfn) | |
| cmpq $1, %rax // argc >= 1? | |
| cmovnsq (%r11), %rdx // p_argv[0] -> %rdx (store 3rd parm to pkfn) | |
| #endif // KMP_MIC | |
| KMP_LABEL(kmp_no_args): | |
| call *%rbx // call (*pkfn)(); | |
| movq $1, %rax // move 1 into return register; | |
| movq -8(%rbp), %rbx // restore %rbx using %rbp since %rsp was modified | |
| movq %rbp, %rsp // restore stack pointer | |
| popq %rbp // restore frame pointer | |
| KMP_CFI_DEF rsp,8 | |
| ret | |
| DEBUG_INFO __kmp_invoke_microtask | |
| // -- End __kmp_invoke_microtask | |
| // kmp_uint64 | |
| // __kmp_hardware_timestamp(void) | |
| .text | |
| PROC __kmp_hardware_timestamp | |
| rdtsc | |
| shlq $32, %rdx | |
| orq %rdx, %rax | |
| ret | |
| DEBUG_INFO __kmp_hardware_timestamp | |
| // -- End __kmp_hardware_timestamp | |
| //------------------------------------------------------------------------ | |
| // FUNCTION __kmp_bsr32 | |
| // | |
| // int | |
| // __kmp_bsr32( int ); | |
| .text | |
| PROC __kmp_bsr32 | |
| bsr %edi,%eax | |
| ret | |
| DEBUG_INFO __kmp_bsr32 | |
| // ----------------------------------------------------------------------- | |
| #endif /* KMP_ARCH_X86_64 */ | |
| // ' | |
| #if KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32 | |
| //------------------------------------------------------------------------ | |
| // int | |
| // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...), | |
| // int gtid, int tid, | |
| // int argc, void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)( & gtid, & tid, argv[0], ... ); | |
| // | |
| // // FIXME: This is done at call-site and can be removed here. | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = 0; | |
| // #endif | |
| // | |
| // return 1; | |
| // } | |
| // | |
| // parameters: | |
| // x0: pkfn | |
| // w1: gtid | |
| // w2: tid | |
| // w3: argc | |
| // x4: p_argv | |
| // x5: &exit_frame | |
| // | |
| // locals: | |
| // __gtid: gtid parm pushed on stack so can pass >id to pkfn | |
| // __tid: tid parm pushed on stack so can pass &tid to pkfn | |
| // | |
| // reg temps: | |
| // x8: used to hold pkfn address | |
| // w9: used as temporary for number of pkfn parms | |
| // x10: used to traverse p_argv array | |
| // x11: used as temporary for stack placement calculation | |
| // x12: used as temporary for stack parameters | |
| // x19: used to preserve exit_frame_ptr, callee-save | |
| // | |
| // return: w0 (always 1/TRUE) | |
| // | |
| __gtid = 4 | |
| __tid = 8 | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| .text | |
| PROC __kmp_invoke_microtask | |
| PACBTI_C | |
| # if OMPT_SUPPORT | |
| stp x19, x20, [sp, #-16]! | |
| # endif | |
| stp x29, x30, [sp, #-16]! | |
| mov x29, sp | |
| orr w9, wzr, #1 | |
| add w9, w9, w3, lsr #1 | |
| sub sp, sp, w9, uxtw #4 | |
| mov x11, sp | |
| mov x8, x0 | |
| str w1, [x29, #-__gtid] | |
| str w2, [x29, #-__tid] | |
| mov w9, w3 | |
| mov x10, x4 | |
| # if OMPT_SUPPORT | |
| mov x19, x5 | |
| str x29, [x19] | |
| # endif | |
| sub x0, x29, #__gtid | |
| sub x1, x29, #__tid | |
| cbz w9, KMP_LABEL(kmp_1) | |
| ldr x2, [x10] | |
| sub w9, w9, #1 | |
| cbz w9, KMP_LABEL(kmp_1) | |
| ldr x3, [x10, #8]! | |
| sub w9, w9, #1 | |
| cbz w9, KMP_LABEL(kmp_1) | |
| ldr x4, [x10, #8]! | |
| sub w9, w9, #1 | |
| cbz w9, KMP_LABEL(kmp_1) | |
| ldr x5, [x10, #8]! | |
| sub w9, w9, #1 | |
| cbz w9, KMP_LABEL(kmp_1) | |
| ldr x6, [x10, #8]! | |
| sub w9, w9, #1 | |
| cbz w9, KMP_LABEL(kmp_1) | |
| ldr x7, [x10, #8]! | |
| KMP_LABEL(kmp_0): | |
| sub w9, w9, #1 | |
| cbz w9, KMP_LABEL(kmp_1) | |
| ldr x12, [x10, #8]! | |
| str x12, [x11], #8 | |
| b KMP_LABEL(kmp_0) | |
| KMP_LABEL(kmp_1): | |
| blr x8 | |
| orr w0, wzr, #1 | |
| mov sp, x29 | |
| ldp x29, x30, [sp], #16 | |
| # if OMPT_SUPPORT | |
| str xzr, [x19] | |
| ldp x19, x20, [sp], #16 | |
| # endif | |
| PACBTI_RET | |
| ret | |
| DEBUG_INFO __kmp_invoke_microtask | |
| // -- End __kmp_invoke_microtask | |
| #endif /* KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32 */ | |
| #if KMP_ARCH_ARM | |
| //------------------------------------------------------------------------ | |
| // int | |
| // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...), | |
| // int gtid, int tid, | |
| // int argc, void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)( & gtid, & tid, argv[0], ... ); | |
| // | |
| // // FIXME: This is done at call-site and can be removed here. | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = 0; | |
| // #endif | |
| // | |
| // return 1; | |
| // } | |
| // | |
| // parameters: | |
| // r0: pkfn | |
| // r1: gtid | |
| // r2: tid | |
| // r3: argc | |
| // r4(stack): p_argv | |
| // r5(stack): &exit_frame | |
| // | |
| // locals: | |
| // __gtid: gtid parm pushed on stack so can pass >id to pkfn | |
| // __tid: tid parm pushed on stack so can pass &tid to pkfn | |
| // | |
| // reg temps: | |
| // r4: used to hold pkfn address | |
| // r5: used as temporary for number of pkfn parms | |
| // r6: used to traverse p_argv array | |
| // r7: frame pointer (in some configurations) | |
| // r8: used as temporary for stack placement calculation | |
| // and as pointer to base of callee saved area | |
| // r9: used as temporary for stack parameters | |
| // r10: used to preserve exit_frame_ptr, callee-save | |
| // r11: frame pointer (in some configurations) | |
| // | |
| // return: r0 (always 1/TRUE) | |
| // | |
| __gtid = 4 | |
| __tid = 8 | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| .text | |
| PROC __kmp_invoke_microtask | |
| // Pushing one extra register (r3) to keep the stack aligned | |
| // for when we call pkfn below | |
| push {r3-r11,lr} | |
| // Load p_argv and &exit_frame | |
| ldr r4, [sp, #10*4] | |
| # if OMPT_SUPPORT | |
| ldr r5, [sp, #11*4] | |
| # endif | |
| # if KMP_OS_DARWIN || (defined(__thumb__) && !KMP_OS_WINDOWS) | |
| # define FP r7 | |
| # define FPOFF 4*4 | |
| #else | |
| # define FP r11 | |
| # define FPOFF 8*4 | |
| #endif | |
| add FP, sp, #FPOFF | |
| # if OMPT_SUPPORT | |
| mov r10, r5 | |
| str FP, [r10] | |
| # endif | |
| mov r8, sp | |
| // Calculate how much stack to allocate, in increments of 8 bytes. | |
| // We strictly need 4*(argc-2) bytes (2 arguments are passed in | |
| // registers) but allocate 4*argc for simplicity (to avoid needing | |
| // to handle the argc<2 cases). We align the number of bytes | |
| // allocated to 8 bytes, to keep the stack aligned. (Since we | |
| // already allocate more than enough, it's ok to round down | |
| // instead of up for the alignment.) We allocate another extra | |
| // 8 bytes for gtid and tid. | |
| mov r5, #1 | |
| add r5, r5, r3, lsr #1 | |
| sub sp, sp, r5, lsl #3 | |
| str r1, [r8, #-__gtid] | |
| str r2, [r8, #-__tid] | |
| mov r5, r3 | |
| mov r6, r4 | |
| mov r4, r0 | |
| // Prepare the first 2 parameters to pkfn - pointers to gtid and tid | |
| // in our stack frame. | |
| sub r0, r8, #__gtid | |
| sub r1, r8, #__tid | |
| mov r8, sp | |
| // Load p_argv[0] and p_argv[1] into r2 and r3, if argc >= 1/2 | |
| cmp r5, #0 | |
| beq KMP_LABEL(kmp_1) | |
| ldr r2, [r6] | |
| subs r5, r5, #1 | |
| beq KMP_LABEL(kmp_1) | |
| ldr r3, [r6, #4]! | |
| // Loop, loading the rest of p_argv and writing the elements on the | |
| // stack. | |
| KMP_LABEL(kmp_0): | |
| subs r5, r5, #1 | |
| beq KMP_LABEL(kmp_1) | |
| ldr r12, [r6, #4]! | |
| str r12, [r8], #4 | |
| b KMP_LABEL(kmp_0) | |
| KMP_LABEL(kmp_1): | |
| blx r4 | |
| mov r0, #1 | |
| sub r4, FP, #FPOFF | |
| mov sp, r4 | |
| # undef FP | |
| # undef FPOFF | |
| # if OMPT_SUPPORT | |
| mov r1, #0 | |
| str r1, [r10] | |
| # endif | |
| pop {r3-r11,pc} | |
| DEBUG_INFO __kmp_invoke_microtask | |
| // -- End __kmp_invoke_microtask | |
| #endif /* KMP_ARCH_ARM */ | |
| #if KMP_ARCH_PPC64 | |
| //------------------------------------------------------------------------ | |
| // int | |
| // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...), | |
| // int gtid, int tid, | |
| // int argc, void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)( & gtid, & tid, argv[0], ... ); | |
| // | |
| // // FIXME: This is done at call-site and can be removed here. | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = 0; | |
| // #endif | |
| // | |
| // return 1; | |
| // } | |
| // | |
| // parameters: | |
| // r3: pkfn | |
| // r4: gtid | |
| // r5: tid | |
| // r6: argc | |
| // r7: p_argv | |
| // r8: &exit_frame | |
| // | |
| // return: r3 (always 1/TRUE) | |
| // | |
| .text | |
| # if KMP_ARCH_PPC64_ELFv2 | |
| .abiversion 2 | |
| # endif | |
| .globl __kmp_invoke_microtask | |
| # if KMP_ARCH_PPC64_ELFv2 | |
| .p2align 4 | |
| # else | |
| .p2align 2 | |
| # endif | |
| .type __kmp_invoke_microtask,@function | |
| # if KMP_ARCH_PPC64_ELFv2 | |
| __kmp_invoke_microtask: | |
| .Lfunc_begin0: | |
| .Lfunc_gep0: | |
| addis 2, 12, .TOC.-.Lfunc_gep0@ha | |
| addi 2, 2, .TOC.-.Lfunc_gep0@l | |
| .Lfunc_lep0: | |
| .localentry __kmp_invoke_microtask, .Lfunc_lep0-.Lfunc_gep0 | |
| # else | |
| .section .opd,"aw",@progbits | |
| __kmp_invoke_microtask: | |
| .p2align 3 | |
| .quad .Lfunc_begin0 | |
| .quad .TOC.@tocbase | |
| .quad 0 | |
| .text | |
| .Lfunc_begin0: | |
| # endif | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| // We need to allocate a stack frame large enough to hold all of the parameters | |
| // on the stack for the microtask plus what this function needs. That's 48 | |
| // bytes under the ELFv1 ABI (32 bytes under ELFv2), plus 8*(2 + argc) for the | |
| // parameters to the microtask, plus 8 bytes to store the values of r4 and r5, | |
| // and 8 bytes to store r31. With OMP-T support, we need an additional 8 bytes | |
| // to save r30 to hold a copy of r8. | |
| .cfi_startproc | |
| mflr 0 | |
| std 31, -8(1) | |
| std 0, 16(1) | |
| // This is unusual because normally we'd set r31 equal to r1 after the stack | |
| // frame is established. In this case, however, we need to dynamically compute | |
| // the stack frame size, and so we keep a direct copy of r1 to access our | |
| // register save areas and restore the r1 value before returning. | |
| mr 31, 1 | |
| .cfi_def_cfa_register r31 | |
| .cfi_offset r31, -8 | |
| .cfi_offset lr, 16 | |
| // Compute the size necessary for the local stack frame. | |
| # if KMP_ARCH_PPC64_ELFv2 | |
| li 12, 72 | |
| # else | |
| li 12, 88 | |
| # endif | |
| sldi 0, 6, 3 | |
| add 12, 0, 12 | |
| neg 12, 12 | |
| // We need to make sure that the stack frame stays aligned (to 16 bytes). | |
| li 0, -16 | |
| and 12, 0, 12 | |
| // Establish the local stack frame. | |
| stdux 1, 1, 12 | |
| # if OMPT_SUPPORT | |
| .cfi_offset r30, -16 | |
| std 30, -16(31) | |
| std 1, 0(8) | |
| mr 30, 8 | |
| # endif | |
| // Store gtid and tid to the stack because they're passed by reference to the microtask. | |
| stw 4, -20(31) | |
| stw 5, -24(31) | |
| mr 12, 6 | |
| mr 4, 7 | |
| cmpwi 0, 12, 1 | |
| blt 0, .Lcall | |
| ld 5, 0(4) | |
| cmpwi 0, 12, 2 | |
| blt 0, .Lcall | |
| ld 6, 8(4) | |
| cmpwi 0, 12, 3 | |
| blt 0, .Lcall | |
| ld 7, 16(4) | |
| cmpwi 0, 12, 4 | |
| blt 0, .Lcall | |
| ld 8, 24(4) | |
| cmpwi 0, 12, 5 | |
| blt 0, .Lcall | |
| ld 9, 32(4) | |
| cmpwi 0, 12, 6 | |
| blt 0, .Lcall | |
| ld 10, 40(4) | |
| cmpwi 0, 12, 7 | |
| blt 0, .Lcall | |
| // There are more than 6 microtask parameters, so we need to store the | |
| // remainder to the stack. | |
| addi 12, 12, -6 | |
| mtctr 12 | |
| // These are set to 8 bytes before the first desired store address (we're using | |
| // pre-increment loads and stores in the loop below). The parameter save area | |
| // for the microtask begins 48 + 8*8 == 112 bytes above r1 for ELFv1 and | |
| // 32 + 8*8 == 96 bytes above r1 for ELFv2. | |
| addi 4, 4, 40 | |
| # if KMP_ARCH_PPC64_ELFv2 | |
| addi 12, 1, 88 | |
| # else | |
| addi 12, 1, 104 | |
| # endif | |
| .Lnext: | |
| ldu 0, 8(4) | |
| stdu 0, 8(12) | |
| bdnz .Lnext | |
| .Lcall: | |
| # if KMP_ARCH_PPC64_ELFv2 | |
| std 2, 24(1) | |
| mr 12, 3 | |
| #else | |
| std 2, 40(1) | |
| // For ELFv1, we need to load the actual function address from the function descriptor. | |
| ld 12, 0(3) | |
| ld 2, 8(3) | |
| ld 11, 16(3) | |
| #endif | |
| addi 3, 31, -20 | |
| addi 4, 31, -24 | |
| mtctr 12 | |
| bctrl | |
| # if KMP_ARCH_PPC64_ELFv2 | |
| ld 2, 24(1) | |
| # else | |
| ld 2, 40(1) | |
| # endif | |
| # if OMPT_SUPPORT | |
| li 3, 0 | |
| std 3, 0(30) | |
| # endif | |
| li 3, 1 | |
| # if OMPT_SUPPORT | |
| ld 30, -16(31) | |
| # endif | |
| mr 1, 31 | |
| ld 0, 16(1) | |
| ld 31, -8(1) | |
| mtlr 0 | |
| blr | |
| .long 0 | |
| .quad 0 | |
| .Lfunc_end0: | |
| .size __kmp_invoke_microtask, .Lfunc_end0-.Lfunc_begin0 | |
| .cfi_endproc | |
| // -- End __kmp_invoke_microtask | |
| #endif /* KMP_ARCH_PPC64 */ | |
| #if KMP_ARCH_RISCV64 | |
| //------------------------------------------------------------------------ | |
| // | |
| // typedef void (*microtask_t)(int *gtid, int *tid, ...); | |
| // | |
| // int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, | |
| // void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)(>id, &tid, argv[0], ...); | |
| // | |
| // return 1; | |
| // } | |
| // | |
| // Parameters: | |
| // a0: pkfn | |
| // a1: gtid | |
| // a2: tid | |
| // a3: argc | |
| // a4: p_argv | |
| // a5: exit_frame_ptr | |
| // | |
| // Locals: | |
| // __gtid: gtid param pushed on stack so can pass >id to pkfn | |
| // __tid: tid param pushed on stack so can pass &tid to pkfn | |
| // | |
| // Temp. registers: | |
| // | |
| // t0: used to calculate the dynamic stack size / used to hold pkfn address | |
| // t1: used as temporary for stack placement calculation | |
| // t2: used as temporary for stack arguments | |
| // t3: used as temporary for number of remaining pkfn parms | |
| // t4: used to traverse p_argv array | |
| // | |
| // return: a0 (always 1/TRUE) | |
| // | |
| __gtid = -20 | |
| __tid = -24 | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| .text | |
| .globl __kmp_invoke_microtask | |
| .p2align 1 | |
| .type __kmp_invoke_microtask,@function | |
| __kmp_invoke_microtask: | |
| .cfi_startproc | |
| // First, save ra and fp | |
| addi sp, sp, -16 | |
| sd ra, 8(sp) | |
| sd fp, 0(sp) | |
| addi fp, sp, 16 | |
| .cfi_def_cfa fp, 0 | |
| .cfi_offset ra, -8 | |
| .cfi_offset fp, -16 | |
| // Compute the dynamic stack size: | |
| // | |
| // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by | |
| // reference | |
| // - We need 8 bytes for each argument that cannot be passed to the 'pkfn' | |
| // function by register. Given that we have 8 of such registers (a[0-7]) | |
| // and two + 'argc' arguments (consider >id and &tid), we need to | |
| // reserve max(0, argc - 6)*8 extra bytes | |
| // | |
| // The total number of bytes is then max(0, argc - 6)*8 + 8 | |
| // Compute max(0, argc - 6) using the following bithack: | |
| // max(0, x) = x - (x & (x >> 31)), where x := argc - 6 | |
| // Source: http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax | |
| addi t0, a3, -6 | |
| srai t1, t0, 31 | |
| and t1, t0, t1 | |
| sub t0, t0, t1 | |
| addi t0, t0, 1 | |
| slli t0, t0, 3 | |
| sub sp, sp, t0 | |
| // Align the stack to 16 bytes | |
| andi sp, sp, -16 | |
| mv t0, a0 | |
| mv t3, a3 | |
| mv t4, a4 | |
| #if OMPT_SUPPORT | |
| // Save frame pointer into exit_frame | |
| sd fp, 0(a5) | |
| #endif | |
| // Prepare arguments for the pkfn function (first 8 using a0-a7 registers) | |
| sw a1, __gtid(fp) | |
| sw a2, __tid(fp) | |
| addi a0, fp, __gtid | |
| addi a1, fp, __tid | |
| beqz t3, .L_kmp_3 | |
| ld a2, 0(t4) | |
| addi t3, t3, -1 | |
| beqz t3, .L_kmp_3 | |
| ld a3, 8(t4) | |
| addi t3, t3, -1 | |
| beqz t3, .L_kmp_3 | |
| ld a4, 16(t4) | |
| addi t3, t3, -1 | |
| beqz t3, .L_kmp_3 | |
| ld a5, 24(t4) | |
| addi t3, t3, -1 | |
| beqz t3, .L_kmp_3 | |
| ld a6, 32(t4) | |
| addi t3, t3, -1 | |
| beqz t3, .L_kmp_3 | |
| ld a7, 40(t4) | |
| // Prepare any additional argument passed through the stack | |
| addi t4, t4, 48 | |
| mv t1, sp | |
| j .L_kmp_2 | |
| .L_kmp_1: | |
| ld t2, 0(t4) | |
| sd t2, 0(t1) | |
| addi t4, t4, 8 | |
| addi t1, t1, 8 | |
| .L_kmp_2: | |
| addi t3, t3, -1 | |
| bnez t3, .L_kmp_1 | |
| .L_kmp_3: | |
| // Call pkfn function | |
| jalr t0 | |
| // Restore stack and return | |
| addi a0, zero, 1 | |
| addi sp, fp, -16 | |
| ld fp, 0(sp) | |
| ld ra, 8(sp) | |
| addi sp, sp, 16 | |
| ret | |
| .Lfunc_end0: | |
| .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask | |
| .cfi_endproc | |
| // -- End __kmp_invoke_microtask | |
| #endif /* KMP_ARCH_RISCV64 */ | |
| #if KMP_ARCH_LOONGARCH64 | |
| //------------------------------------------------------------------------ | |
| // | |
| // typedef void (*microtask_t)(int *gtid, int *tid, ...); | |
| // | |
| // int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, | |
| // void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)(>id, &tid, argv[0], ...); | |
| // | |
| // return 1; | |
| // } | |
| // | |
| // Parameters: | |
| // a0: pkfn | |
| // a1: gtid | |
| // a2: tid | |
| // a3: argc | |
| // a4: p_argv | |
| // a5: exit_frame_ptr | |
| // | |
| // Locals: | |
| // __gtid: gtid param pushed on stack so can pass >id to pkfn | |
| // __tid: tid param pushed on stack so can pass &tid to pkfn | |
| // | |
| // Temp registers: | |
| // | |
| // t0: used to calculate the dynamic stack size / used to hold pkfn address | |
| // t1: used as temporary for stack placement calculation | |
| // t2: used as temporary for stack arguments | |
| // t3: used as temporary for number of remaining pkfn parms | |
| // t4: used to traverse p_argv array | |
| // | |
| // return: a0 (always 1/TRUE) | |
| // | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| .text | |
| .globl __kmp_invoke_microtask | |
| .p2align 2 | |
| .type __kmp_invoke_microtask,@function | |
| __kmp_invoke_microtask: | |
| .cfi_startproc | |
| // First, save ra and fp | |
| addi.d $sp, $sp, -16 | |
| st.d $ra, $sp, 8 | |
| st.d $fp, $sp, 0 | |
| addi.d $fp, $sp, 16 | |
| .cfi_def_cfa 22, 0 | |
| .cfi_offset 1, -8 | |
| .cfi_offset 22, -16 | |
| // Compute the dynamic stack size: | |
| // | |
| // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by | |
| // reference | |
| // - We need 8 bytes for each argument that cannot be passed to the 'pkfn' | |
| // function by register. Given that we have 8 of such registers (a[0-7]) | |
| // and two + 'argc' arguments (consider >id and &tid), we need to | |
| // reserve max(0, argc - 6)*8 extra bytes | |
| // | |
| // The total number of bytes is then max(0, argc - 6)*8 + 8 | |
| addi.d $t0, $a3, -6 | |
| slt $t1, $t0, $zero | |
| masknez $t0, $t0, $t1 | |
| addi.d $t0, $t0, 1 | |
| slli.d $t0, $t0, 3 | |
| sub.d $sp, $sp, $t0 | |
| // Align the stack to 16 bytes | |
| bstrins.d $sp, $zero, 3, 0 | |
| move $t0, $a0 | |
| move $t3, $a3 | |
| move $t4, $a4 | |
| #if OMPT_SUPPORT | |
| // Save frame pointer into exit_frame | |
| st.d $fp, $a5, 0 | |
| #endif | |
| // Prepare arguments for the pkfn function (first 8 using a0-a7 registers) | |
| st.w $a1, $fp, -20 | |
| st.w $a2, $fp, -24 | |
| addi.d $a0, $fp, -20 | |
| addi.d $a1, $fp, -24 | |
| beqz $t3, .L_kmp_3 | |
| ld.d $a2, $t4, 0 | |
| addi.d $t3, $t3, -1 | |
| beqz $t3, .L_kmp_3 | |
| ld.d $a3, $t4, 8 | |
| addi.d $t3, $t3, -1 | |
| beqz $t3, .L_kmp_3 | |
| ld.d $a4, $t4, 16 | |
| addi.d $t3, $t3, -1 | |
| beqz $t3, .L_kmp_3 | |
| ld.d $a5, $t4, 24 | |
| addi.d $t3, $t3, -1 | |
| beqz $t3, .L_kmp_3 | |
| ld.d $a6, $t4, 32 | |
| addi.d $t3, $t3, -1 | |
| beqz $t3, .L_kmp_3 | |
| ld.d $a7, $t4, 40 | |
| // Prepare any additional argument passed through the stack | |
| addi.d $t4, $t4, 48 | |
| move $t1, $sp | |
| b .L_kmp_2 | |
| .L_kmp_1: | |
| ld.d $t2, $t4, 0 | |
| st.d $t2, $t1, 0 | |
| addi.d $t4, $t4, 8 | |
| addi.d $t1, $t1, 8 | |
| .L_kmp_2: | |
| addi.d $t3, $t3, -1 | |
| bnez $t3, .L_kmp_1 | |
| .L_kmp_3: | |
| // Call pkfn function | |
| jirl $ra, $t0, 0 | |
| // Restore stack and return | |
| addi.d $a0, $zero, 1 | |
| addi.d $sp, $fp, -16 | |
| ld.d $fp, $sp, 0 | |
| ld.d $ra, $sp, 8 | |
| addi.d $sp, $sp, 16 | |
| jr $ra | |
| .Lfunc_end0: | |
| .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask | |
| .cfi_endproc | |
| // -- End __kmp_invoke_microtask | |
| #endif /* KMP_ARCH_LOONGARCH64 */ | |
| #if KMP_ARCH_VE | |
| //------------------------------------------------------------------------ | |
| // | |
| // typedef void (*microtask_t)(int *gtid, int *tid, ...); | |
| // | |
| // int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, | |
| // void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)(>id, &tid, argv[0], ...); | |
| // | |
| // return 1; | |
| // } | |
| // | |
| // Parameters: | |
| // s0: pkfn | |
| // s1: gtid | |
| // s2: tid | |
| // s3: argc | |
| // s4: p_argv | |
| // s5: exit_frame_ptr | |
| // | |
| // Locals: | |
| // __gtid: gtid param pushed on stack so can pass >id to pkfn | |
| // __tid: tid param pushed on stack so can pass &tid to pkfn | |
| // | |
| // Temp. registers: | |
| // | |
| // s34: used to calculate the dynamic stack size | |
| // s35: used as temporary for stack placement calculation | |
| // s36: used as temporary for stack arguments | |
| // s37: used as temporary for number of remaining pkfn parms | |
| // s38: used to traverse p_argv array | |
| // | |
| // return: s0 (always 1/TRUE) | |
| // | |
| __gtid = -4 | |
| __tid = -8 | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| .text | |
| .globl __kmp_invoke_microtask | |
| // A function requires 8 bytes align. | |
| .p2align 3 | |
| .type __kmp_invoke_microtask,@function | |
| __kmp_invoke_microtask: | |
| .cfi_startproc | |
| // First, save fp and lr. VE stores them at caller stack frame. | |
| st %fp, 0(, %sp) | |
| st %lr, 8(, %sp) | |
| or %fp, 0, %sp | |
| .cfi_def_cfa %fp, 0 | |
| .cfi_offset %lr, 8 | |
| .cfi_offset %fp, 0 | |
| // Compute the dynamic stack size: | |
| // | |
| // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them | |
| // by reference | |
| // - We need 8 bytes for whole arguments. We have two + 'argc' | |
| // arguments (condider >id and &tid). We need to reserve | |
| // (argc + 2) * 8 bytes. | |
| // - We need 176 bytes for RSA and others | |
| // | |
| // The total number of bytes is then (argc + 2) * 8 + 8 + 176. | |
| // | |
| // |------------------------------| | |
| // | return address of callee | 8(%fp) | |
| // |------------------------------| | |
| // | frame pointer of callee | 0(%fp) | |
| // |------------------------------| <------------------ %fp | |
| // | __tid / __gtid | -8(%fp) / -4(%fp) | |
| // |------------------------------| | |
| // | argc+2 for arguments | 176(%sp) | |
| // |------------------------------| | |
| // | RSA | | |
| // |------------------------------| | |
| // | return address | | |
| // |------------------------------| | |
| // | frame pointer | | |
| // |------------------------------| <------------------ %sp | |
| adds.w.sx %s34, 2, %s3 | |
| sll %s34, %s34, 3 | |
| lea %s34, 184(, %s34) | |
| subs.l %sp, %sp, %s34 | |
| // Align the stack to 16 bytes. | |
| and %sp, -16, %sp | |
| // Save pkfn. | |
| or %s12, 0, %s0 | |
| // Call host to allocate stack if it is necessary. | |
| brge.l %sp, %sl, .L_kmp_pass | |
| ld %s61, 24(, %tp) | |
| lea %s63, 0x13b | |
| shm.l %s63, 0(%s61) | |
| shm.l %sl, 8(%s61) | |
| shm.l %sp, 16(%s61) | |
| monc | |
| .L_kmp_pass: | |
| lea %s35, 176(, %sp) | |
| adds.w.sx %s37, 0, %s3 | |
| or %s38, 0, %s4 | |
| #if OMPT_SUPPORT | |
| // Save frame pointer into exit_frame. | |
| st %fp, 0(%s5) | |
| #endif | |
| // Prepare arguments for the pkfn function (first 8 using s0-s7 | |
| // registers, but need to store stack also because of varargs). | |
| stl %s1, __gtid(%fp) | |
| stl %s2, __tid(%fp) | |
| adds.l %s0, __gtid, %fp | |
| st %s0, 0(, %s35) | |
| adds.l %s1, __tid, %fp | |
| st %s1, 8(, %s35) | |
| breq.l 0, %s37, .L_kmp_call | |
| ld %s2, 0(, %s38) | |
| st %s2, 16(, %s35) | |
| breq.l 1, %s37, .L_kmp_call | |
| ld %s3, 8(, %s38) | |
| st %s3, 24(, %s35) | |
| breq.l 2, %s37, .L_kmp_call | |
| ld %s4, 16(, %s38) | |
| st %s4, 32(, %s35) | |
| breq.l 3, %s37, .L_kmp_call | |
| ld %s5, 24(, %s38) | |
| st %s5, 40(, %s35) | |
| breq.l 4, %s37, .L_kmp_call | |
| ld %s6, 32(, %s38) | |
| st %s6, 48(, %s35) | |
| breq.l 5, %s37, .L_kmp_call | |
| ld %s7, 40(, %s38) | |
| st %s7, 56(, %s35) | |
| breq.l 6, %s37, .L_kmp_call | |
| // Prepare any additional argument passed through the stack. | |
| adds.l %s37, -6, %s37 | |
| lea %s38, 48(, %s38) | |
| lea %s35, 64(, %s35) | |
| .L_kmp_loop: | |
| ld %s36, 0(, %s38) | |
| st %s36, 0(, %s35) | |
| adds.l %s37, -1, %s37 | |
| adds.l %s38, 8, %s38 | |
| adds.l %s35, 8, %s35 | |
| brne.l 0, %s37, .L_kmp_loop | |
| .L_kmp_call: | |
| // Call pkfn function. | |
| bsic %lr, (, %s12) | |
| // Return value. | |
| lea %s0, 1 | |
| // Restore stack and return. | |
| or %sp, 0, %fp | |
| ld %lr, 8(, %sp) | |
| ld %fp, 0(, %sp) | |
| b.l.t (, %lr) | |
| .Lfunc_end0: | |
| .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask | |
| .cfi_endproc | |
| // -- End __kmp_invoke_microtask | |
| #endif /* KMP_ARCH_VE */ | |
| #if KMP_ARCH_S390X | |
| //------------------------------------------------------------------------ | |
| // | |
| // typedef void (*microtask_t)(int *gtid, int *tid, ...); | |
| // | |
| // int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, | |
| // void *p_argv[] | |
| // #if OMPT_SUPPORT | |
| // , | |
| // void **exit_frame_ptr | |
| // #endif | |
| // ) { | |
| // #if OMPT_SUPPORT | |
| // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); | |
| // #endif | |
| // | |
| // (*pkfn)(>id, &tid, argv[0], ...); | |
| // | |
| // return 1; | |
| // } | |
| // | |
| // Parameters: | |
| // r2: pkfn | |
| // r3: gtid | |
| // r4: tid | |
| // r5: argc | |
| // r6: p_argv | |
| // SP+160: exit_frame_ptr | |
| // | |
| // Locals: | |
| // __gtid: gtid param pushed on stack so can pass >id to pkfn | |
| // __tid: tid param pushed on stack so can pass &tid to pkfn | |
| // | |
| // Temp. registers: | |
| // | |
| // r0: used to fetch argv slots | |
| // r7: used as temporary for number of remaining pkfn parms | |
| // r8: argv | |
| // r9: pkfn | |
| // r10: stack size | |
| // r11: previous fp | |
| // r12: stack parameter area | |
| // r13: argv slot | |
| // | |
| // return: r2 (always 1/TRUE) | |
| // | |
| // -- Begin __kmp_invoke_microtask | |
| // mark_begin; | |
| .text | |
| .globl __kmp_invoke_microtask | |
| .p2align 1 | |
| .type __kmp_invoke_microtask,@function | |
| __kmp_invoke_microtask: | |
| .cfi_startproc | |
| stmg %r6,%r14,48(%r15) | |
| .cfi_offset %r6, -112 | |
| .cfi_offset %r7, -104 | |
| .cfi_offset %r8, -96 | |
| .cfi_offset %r9, -88 | |
| .cfi_offset %r10, -80 | |
| .cfi_offset %r11, -72 | |
| .cfi_offset %r12, -64 | |
| .cfi_offset %r13, -56 | |
| .cfi_offset %r14, -48 | |
| .cfi_offset %r15, -40 | |
| lgr %r11,%r15 | |
| .cfi_def_cfa %r11, 160 | |
| // Compute the dynamic stack size: | |
| // | |
| // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by | |
| // reference | |
| // - We need 8 bytes for each argument that cannot be passed to the 'pkfn' | |
| // function by register. Given that we have 5 of such registers (r[2-6]) | |
| // and two + 'argc' arguments (consider >id and &tid), we need to | |
| // reserve max(0, argc - 3)*8 extra bytes | |
| // | |
| // The total number of bytes is then max(0, argc - 3)*8 + 8 | |
| lgr %r10,%r5 | |
| aghi %r10,-2 | |
| jnm 0f | |
| lghi %r10,0 | |
| 0: | |
| sllg %r10,%r10,3 | |
| lgr %r12,%r10 | |
| aghi %r10,176 | |
| sgr %r15,%r10 | |
| agr %r12,%r15 | |
| stg %r11,0(%r15) | |
| lgr %r9,%r2 // pkfn | |
| #if OMPT_SUPPORT | |
| // Save frame pointer into exit_frame | |
| lg %r8,160(%r11) | |
| stg %r11,0(%r8) | |
| #endif | |
| // Prepare arguments for the pkfn function (first 5 using r2-r6 registers) | |
| stg %r3,160(%r12) | |
| la %r2,164(%r12) // gid | |
| stg %r4,168(%r12) | |
| la %r3,172(%r12) // tid | |
| lgr %r8,%r6 // argv | |
| // If argc > 0 | |
| ltgr %r7,%r5 | |
| jz 1f | |
| lg %r4,0(%r8) // argv[0] | |
| aghi %r7,-1 | |
| jz 1f | |
| // If argc > 1 | |
| lg %r5,8(%r8) // argv[1] | |
| aghi %r7,-1 | |
| jz 1f | |
| // If argc > 2 | |
| lg %r6,16(%r8) // argv[2] | |
| aghi %r7,-1 | |
| jz 1f | |
| lghi %r13,0 // Index [n] | |
| 2: | |
| lg %r0,24(%r13,%r8) // argv[2+n] | |
| stg %r0,160(%r13,%r15) // parm[2+n] | |
| aghi %r13,8 // Next | |
| aghi %r7,-1 | |
| jnz 2b | |
| 1: | |
| basr %r14,%r9 // Call pkfn | |
| // Restore stack and return | |
| lgr %r15,%r11 | |
| lmg %r6,%r14,48(%r15) | |
| lghi %r2,1 | |
| br %r14 | |
| .Lfunc_end0: | |
| .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask | |
| .cfi_endproc | |
| // -- End __kmp_invoke_microtask | |
| #endif /* KMP_ARCH_S390X */ | |
| #if KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32 | |
| #ifndef KMP_PREFIX_UNDERSCORE | |
| # define KMP_PREFIX_UNDERSCORE(x) x | |
| #endif | |
| .data | |
| COMMON .gomp_critical_user_, 32, 3 | |
| .data | |
| .align 4 | |
| .global KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr) | |
| KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr): | |
| .4byte .gomp_critical_user_ | |
| #ifdef __ELF__ | |
| .type KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),%object | |
| .size KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),4 | |
| #endif | |
| #endif /* KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32 */ | |
| #if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 || \ | |
| KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || KMP_ARCH_VE || \ | |
| KMP_ARCH_S390X || KMP_ARCH_SPARC64 | |
| #ifndef KMP_PREFIX_UNDERSCORE | |
| # define KMP_PREFIX_UNDERSCORE(x) x | |
| #endif | |
| .data | |
| COMMON .gomp_critical_user_, 32, 3 | |
| .data | |
| .align 8 | |
| .global KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr) | |
| KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr): | |
| .8byte .gomp_critical_user_ | |
| #ifdef __ELF__ | |
| .type KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),%object | |
| .size KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),8 | |
| #endif | |
| #endif /* KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 || | |
| KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || KMP_ARCH_VE || | |
| KMP_ARCH_S390X || KMP_ARCH_SPARC64 */ | |
| #if KMP_OS_LINUX | |
| # if KMP_ARCH_ARM || KMP_ARCH_AARCH64 | |
| .section .note.GNU-stack,"",%progbits | |
| # elif !KMP_ARCH_WASM32 && !KMP_ARCH_WASM64 | |
| .section .note.GNU-stack,"",@progbits | |
| # endif | |
| #endif | |
| #if (KMP_OS_LINUX || KMP_OS_OPENBSD) && \ | |
| (KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32) | |
| GNU_PROPERTY_BTI_PAC | |
| #endif |