Google Git
Sign in
llvm/llvm-project/llvm/refs/heads/main/./test/CodeGen/X86/bitcnt-big-integer.ll
blob: 5c9a9338bc32fe37967c78f78aa0b1fa661ad9e4 [file] [log] [blame] [edit]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX2
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefixes=AVX512,AVX512F
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knm | FileCheck %s --check-prefixes=AVX512,AVX512POPCNT
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512,AVX512VL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 -mattr=+avx512vpopcntdq | FileCheck %s --check-prefixes=AVX512,AVX512VLPOPCNT
;
; CTPOP
;
define i32 @test_ctpop_i128(i128 %a0) nounwind {
; CHECK-LABEL: test_ctpop_i128:
; CHECK: # %bb.0:
; CHECK-NEXT: popcntq %rsi, %rcx
; CHECK-NEXT: popcntq %rdi, %rax
; CHECK-NEXT: addl %ecx, %eax
; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
; CHECK-NEXT: retq
;
; AVX512-LABEL: test_ctpop_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: popcntq %rsi, %rcx
; AVX512-NEXT: popcntq %rdi, %rax
; AVX512-NEXT: addl %ecx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i128 @llvm.ctpop.i128(i128 %a0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @load_ctpop_i128(ptr %p0) nounwind {
; CHECK-LABEL: load_ctpop_i128:
; CHECK: # %bb.0:
; CHECK-NEXT: popcntq 8(%rdi), %rcx
; CHECK-NEXT: popcntq (%rdi), %rax
; CHECK-NEXT: addl %ecx, %eax
; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
; CHECK-NEXT: retq
;
; AVX512-LABEL: load_ctpop_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: popcntq 8(%rdi), %rcx
; AVX512-NEXT: popcntq (%rdi), %rax
; AVX512-NEXT: addl %ecx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%a0 = load i128, ptr %p0
%cnt = call i128 @llvm.ctpop.i128(i128 %a0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @vector_ctpop_i128(<4 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctpop_i128:
; SSE: # %bb.0:
; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: pextrq $1, %xmm0, %rcx
; SSE-NEXT: popcntq %rcx, %rcx
; SSE-NEXT: popcntq %rax, %rax
; SSE-NEXT: addl %ecx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctpop_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: vpextrq $1, %xmm0, %rax
; AVX2-NEXT: vmovq %xmm0, %rcx
; AVX2-NEXT: popcntq %rax, %rdx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq %rcx, %rax
; AVX2-NEXT: addl %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctpop_i128:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
; AVX512F-NEXT: vmovq %xmm0, %rcx
; AVX512F-NEXT: popcntq %rax, %rdx
; AVX512F-NEXT: popcntq %rcx, %rax
; AVX512F-NEXT: addl %edx, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctpop_i128:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpextrq $1, %xmm0, %rax
; AVX512POPCNT-NEXT: vmovq %xmm0, %rcx
; AVX512POPCNT-NEXT: popcntq %rax, %rdx
; AVX512POPCNT-NEXT: popcntq %rcx, %rax
; AVX512POPCNT-NEXT: addl %edx, %eax
; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctpop_i128:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovq %xmm0, %rax
; AVX512VL-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512VL-NEXT: popcntq %rcx, %rcx
; AVX512VL-NEXT: popcntq %rax, %rax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctpop_i128:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovq %xmm0, %rax
; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512VLPOPCNT-NEXT: popcntq %rcx, %rcx
; AVX512VLPOPCNT-NEXT: popcntq %rax, %rax
; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <4 x i32> %v0 to i128
%cnt = call i128 @llvm.ctpop.i128(i128 %a0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define <2 x i32> @load_ctpop_v2i128(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_v2i128:
; SSE: # %bb.0:
; SSE-NEXT: popcntq 8(%rdi), %rax
; SSE-NEXT: popcntq (%rdi), %rcx
; SSE-NEXT: popcntq 24(%rdi), %rdx
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 16(%rdi), %rax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: movd %eax, %xmm1
; SSE-NEXT: movd %ecx, %xmm0
; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_v2i128:
; AVX2: # %bb.0:
; AVX2-NEXT: popcntq 8(%rdi), %rax
; AVX2-NEXT: popcntq (%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 24(%rdi), %rax
; AVX2-NEXT: popcntq 16(%rdi), %rdx
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: vmovd %edx, %xmm0
; AVX2-NEXT: vmovd %ecx, %xmm1
; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_v2i128:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq 8(%rdi), %rax
; AVX512F-NEXT: popcntq (%rdi), %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq 24(%rdi), %rax
; AVX512F-NEXT: popcntq 16(%rdi), %rdx
; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: vmovd %edx, %xmm0
; AVX512F-NEXT: vmovd %ecx, %xmm1
; AVX512F-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_v2i128:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rax
; AVX512POPCNT-NEXT: popcntq (%rdi), %rcx
; AVX512POPCNT-NEXT: addl %eax, %ecx
; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax
; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rdx
; AVX512POPCNT-NEXT: addl %eax, %edx
; AVX512POPCNT-NEXT: vmovd %edx, %xmm0
; AVX512POPCNT-NEXT: vmovd %ecx, %xmm1
; AVX512POPCNT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_v2i128:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq 8(%rdi), %rax
; AVX512VL-NEXT: popcntq (%rdi), %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 24(%rdi), %rax
; AVX512VL-NEXT: popcntq 16(%rdi), %rdx
; AVX512VL-NEXT: addl %eax, %edx
; AVX512VL-NEXT: vmovd %edx, %xmm0
; AVX512VL-NEXT: vmovd %ecx, %xmm1
; AVX512VL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_v2i128:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rax
; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rcx
; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rdx
; AVX512VLPOPCNT-NEXT: addl %eax, %edx
; AVX512VLPOPCNT-NEXT: vmovd %edx, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %ecx, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX512VLPOPCNT-NEXT: retq
%a0 = load <2 x i128>, ptr %p0
%cnt = call <2 x i128> @llvm.ctpop.v2i128(<2 x i128> %a0)
%res = trunc <2 x i128> %cnt to <2 x i32>
ret <2 x i32> %res
}
define <4 x i32> @load_ctpop_v4i128(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_v4i128:
; SSE: # %bb.0:
; SSE-NEXT: popcntq 8(%rdi), %rax
; SSE-NEXT: popcntq (%rdi), %rcx
; SSE-NEXT: popcntq 24(%rdi), %rdx
; SSE-NEXT: popcntq 16(%rdi), %rsi
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: addl %edx, %esi
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 40(%rdi), %rax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 32(%rdi), %rdx
; SSE-NEXT: popcntq 56(%rdi), %r8
; SSE-NEXT: addl %eax, %edx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 48(%rdi), %rax
; SSE-NEXT: addl %r8d, %eax
; SSE-NEXT: movd %eax, %xmm0
; SSE-NEXT: movd %edx, %xmm1
; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; SSE-NEXT: movd %esi, %xmm2
; SSE-NEXT: movd %ecx, %xmm0
; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_v4i128:
; AVX2: # %bb.0:
; AVX2-NEXT: popcntq 56(%rdi), %rax
; AVX2-NEXT: popcntq 48(%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 40(%rdi), %rax
; AVX2-NEXT: popcntq 32(%rdi), %rdx
; AVX2-NEXT: popcntq 8(%rdi), %rsi
; AVX2-NEXT: popcntq (%rdi), %r8
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: addl %esi, %r8d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 24(%rdi), %rax
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: popcntq 16(%rdi), %rsi
; AVX2-NEXT: addl %eax, %esi
; AVX2-NEXT: vmovd %esi, %xmm0
; AVX2-NEXT: vmovd %r8d, %xmm1
; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX2-NEXT: vmovd %edx, %xmm1
; AVX2-NEXT: vmovd %ecx, %xmm2
; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm2 = [0,4,0,4]
; AVX2-NEXT: vpermd %ymm1, %ymm2, %ymm1
; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_v4i128:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq 24(%rdi), %rcx
; AVX512F-NEXT: popcntq 16(%rdi), %rax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: popcntq 8(%rdi), %rcx
; AVX512F-NEXT: popcntq (%rdi), %rdx
; AVX512F-NEXT: popcntq 40(%rdi), %rsi
; AVX512F-NEXT: popcntq 32(%rdi), %r8
; AVX512F-NEXT: addl %ecx, %edx
; AVX512F-NEXT: addl %esi, %r8d
; AVX512F-NEXT: popcntq 56(%rdi), %rcx
; AVX512F-NEXT: popcntq 48(%rdi), %rsi
; AVX512F-NEXT: addl %ecx, %esi
; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm0 = [0,16,0,16]
; AVX512F-NEXT: vmovd %esi, %xmm1
; AVX512F-NEXT: vmovd %r8d, %xmm2
; AVX512F-NEXT: vpermt2d %zmm1, %zmm0, %zmm2
; AVX512F-NEXT: vmovd %edx, %xmm0
; AVX512F-NEXT: vmovd %eax, %xmm1
; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512F-NEXT: vmovdqa {{.*#+}} xmm0 = [0,4,18,19]
; AVX512F-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_v4i128:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rcx
; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rax
; AVX512POPCNT-NEXT: addl %ecx, %eax
; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rcx
; AVX512POPCNT-NEXT: popcntq (%rdi), %rdx
; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rsi
; AVX512POPCNT-NEXT: popcntq 32(%rdi), %r8
; AVX512POPCNT-NEXT: addl %ecx, %edx
; AVX512POPCNT-NEXT: addl %esi, %r8d
; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rcx
; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rsi
; AVX512POPCNT-NEXT: addl %ecx, %esi
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} xmm0 = [0,16,0,16]
; AVX512POPCNT-NEXT: vmovd %esi, %xmm1
; AVX512POPCNT-NEXT: vmovd %r8d, %xmm2
; AVX512POPCNT-NEXT: vpermt2d %zmm1, %zmm0, %zmm2
; AVX512POPCNT-NEXT: vmovd %edx, %xmm0
; AVX512POPCNT-NEXT: vmovd %eax, %xmm1
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vmovdqa {{.*#+}} xmm0 = [0,4,18,19]
; AVX512POPCNT-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
; AVX512POPCNT-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_v4i128:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq 24(%rdi), %rcx
; AVX512VL-NEXT: popcntq 16(%rdi), %rax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq 8(%rdi), %rcx
; AVX512VL-NEXT: popcntq (%rdi), %rdx
; AVX512VL-NEXT: popcntq 40(%rdi), %rsi
; AVX512VL-NEXT: popcntq 32(%rdi), %r8
; AVX512VL-NEXT: addl %ecx, %edx
; AVX512VL-NEXT: addl %esi, %r8d
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq 56(%rdi), %rcx
; AVX512VL-NEXT: xorl %esi, %esi
; AVX512VL-NEXT: popcntq 48(%rdi), %rsi
; AVX512VL-NEXT: addl %ecx, %esi
; AVX512VL-NEXT: vmovd %esi, %xmm0
; AVX512VL-NEXT: vmovd %r8d, %xmm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [0,4,0,4]
; AVX512VL-NEXT: vpermi2d %xmm0, %xmm1, %xmm2
; AVX512VL-NEXT: vmovd %edx, %xmm0
; AVX512VL-NEXT: vmovd %eax, %xmm1
; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm0 = [0,4,10,11]
; AVX512VL-NEXT: vpermi2d %ymm2, %ymm1, %ymm0
; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_v4i128:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rcx
; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rax
; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rcx
; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rdx
; AVX512VLPOPCNT-NEXT: popcntq 40(%rdi), %rsi
; AVX512VLPOPCNT-NEXT: popcntq 32(%rdi), %r8
; AVX512VLPOPCNT-NEXT: addl %ecx, %edx
; AVX512VLPOPCNT-NEXT: addl %esi, %r8d
; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
; AVX512VLPOPCNT-NEXT: popcntq 56(%rdi), %rcx
; AVX512VLPOPCNT-NEXT: xorl %esi, %esi
; AVX512VLPOPCNT-NEXT: popcntq 48(%rdi), %rsi
; AVX512VLPOPCNT-NEXT: addl %ecx, %esi
; AVX512VLPOPCNT-NEXT: vmovd %esi, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %r8d, %xmm1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} xmm2 = [0,4,0,4]
; AVX512VLPOPCNT-NEXT: vpermi2d %xmm0, %xmm1, %xmm2
; AVX512VLPOPCNT-NEXT: vmovd %edx, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %eax, %xmm1
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vmovdqa {{.*#+}} xmm0 = [0,4,10,11]
; AVX512VLPOPCNT-NEXT: vpermi2d %ymm2, %ymm1, %ymm0
; AVX512VLPOPCNT-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load <4 x i128>, ptr %p0
%cnt = call <4 x i128> @llvm.ctpop.v4i128(<4 x i128> %a0)
%res = trunc <4 x i128> %cnt to <4 x i32>
ret <4 x i32> %res
}
define i32 @test_ctpop_i256(i256 %a0) nounwind {
; CHECK-LABEL: test_ctpop_i256:
; CHECK: # %bb.0:
; CHECK-NEXT: popcntq %rcx, %rax
; CHECK-NEXT: xorl %ecx, %ecx
; CHECK-NEXT: popcntq %rdx, %rcx
; CHECK-NEXT: addl %eax, %ecx
; CHECK-NEXT: xorl %edx, %edx
; CHECK-NEXT: popcntq %rsi, %rdx
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: popcntq %rdi, %rax
; CHECK-NEXT: addl %edx, %eax
; CHECK-NEXT: addl %ecx, %eax
; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
; CHECK-NEXT: retq
;
; AVX512F-LABEL: test_ctpop_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq %rcx, %rax
; AVX512F-NEXT: popcntq %rdx, %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq %rsi, %rdx
; AVX512F-NEXT: popcntq %rdi, %rax
; AVX512F-NEXT: addl %edx, %eax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctpop_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: popcntq %rcx, %rax
; AVX512POPCNT-NEXT: popcntq %rdx, %rcx
; AVX512POPCNT-NEXT: addl %eax, %ecx
; AVX512POPCNT-NEXT: popcntq %rsi, %rdx
; AVX512POPCNT-NEXT: popcntq %rdi, %rax
; AVX512POPCNT-NEXT: addl %edx, %eax
; AVX512POPCNT-NEXT: addl %ecx, %eax
; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctpop_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq %rcx, %rax
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq %rdx, %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %edx, %edx
; AVX512VL-NEXT: popcntq %rsi, %rdx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq %rdi, %rax
; AVX512VL-NEXT: addl %edx, %eax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctpop_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: popcntq %rcx, %rax
; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
; AVX512VLPOPCNT-NEXT: popcntq %rdx, %rcx
; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
; AVX512VLPOPCNT-NEXT: xorl %edx, %edx
; AVX512VLPOPCNT-NEXT: popcntq %rsi, %rdx
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %rdi, %rax
; AVX512VLPOPCNT-NEXT: addl %edx, %eax
; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i256 @llvm.ctpop.i256(i256 %a0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @load_ctpop_i256(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_i256:
; SSE: # %bb.0:
; SSE-NEXT: popcntq 24(%rdi), %rcx
; SSE-NEXT: popcntq 16(%rdi), %rdx
; SSE-NEXT: popcntq 8(%rdi), %rsi
; SSE-NEXT: popcntq (%rdi), %rax
; SSE-NEXT: addl %ecx, %edx
; SSE-NEXT: addl %esi, %eax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: popcntq 24(%rdi), %rax
; AVX2-NEXT: popcntq 16(%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: popcntq 8(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq (%rdi), %rax
; AVX2-NEXT: addl %edx, %eax
; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq 24(%rdi), %rax
; AVX512F-NEXT: popcntq 16(%rdi), %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq 8(%rdi), %rdx
; AVX512F-NEXT: popcntq (%rdi), %rax
; AVX512F-NEXT: addl %edx, %eax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu (%rdi), %ymm0
; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq 24(%rdi), %rax
; AVX512VL-NEXT: popcntq 16(%rdi), %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: popcntq 8(%rdi), %rdx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq (%rdi), %rax
; AVX512VL-NEXT: addl %edx, %eax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %ymm0
; AVX512VLPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i256, ptr %p0
%cnt = call i256 @llvm.ctpop.i256(i256 %a0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @vector_ctpop_i256(<8 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctpop_i256:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm0, %rax
; SSE-NEXT: movq %xmm0, %rcx
; SSE-NEXT: movq %xmm1, %rdx
; SSE-NEXT: pextrq $1, %xmm1, %rsi
; SSE-NEXT: popcntq %rsi, %rsi
; SSE-NEXT: popcntq %rdx, %rdx
; SSE-NEXT: addl %esi, %edx
; SSE-NEXT: xorl %esi, %esi
; SSE-NEXT: popcntq %rax, %rsi
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq %rcx, %rax
; SSE-NEXT: addl %esi, %eax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctpop_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: vpextrq $1, %xmm0, %rax
; AVX2-NEXT: vmovq %xmm0, %rcx
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
; AVX2-NEXT: vmovq %xmm0, %rsi
; AVX2-NEXT: popcntq %rdx, %rdx
; AVX2-NEXT: popcntq %rsi, %rsi
; AVX2-NEXT: addl %edx, %esi
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: popcntq %rax, %rdx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq %rcx, %rax
; AVX2-NEXT: addl %edx, %eax
; AVX2-NEXT: addl %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctpop_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
; AVX512F-NEXT: vmovq %xmm0, %rcx
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx
; AVX512F-NEXT: vmovq %xmm0, %rsi
; AVX512F-NEXT: popcntq %rdx, %rdx
; AVX512F-NEXT: popcntq %rsi, %rsi
; AVX512F-NEXT: addl %edx, %esi
; AVX512F-NEXT: popcntq %rax, %rdx
; AVX512F-NEXT: popcntq %rcx, %rax
; AVX512F-NEXT: addl %edx, %eax
; AVX512F-NEXT: addl %esi, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctpop_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqa %ymm0, %ymm0
; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctpop_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpextrq $1, %xmm0, %rax
; AVX512VL-NEXT: vmovq %xmm0, %rcx
; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512VL-NEXT: vmovq %xmm0, %rdx
; AVX512VL-NEXT: vpextrq $1, %xmm0, %rsi
; AVX512VL-NEXT: popcntq %rsi, %rsi
; AVX512VL-NEXT: popcntq %rdx, %rdx
; AVX512VL-NEXT: addl %esi, %edx
; AVX512VL-NEXT: xorl %esi, %esi
; AVX512VL-NEXT: popcntq %rax, %rsi
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq %rcx, %rax
; AVX512VL-NEXT: addl %esi, %eax
; AVX512VL-NEXT: addl %edx, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctpop_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <8 x i32> %v0 to i256
%cnt = call i256 @llvm.ctpop.i256(i256 %a0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define <2 x i32> @load_ctpop_v2i256_v2i32(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_v2i256_v2i32:
; SSE: # %bb.0:
; SSE-NEXT: popcntq 24(%rdi), %rax
; SSE-NEXT: popcntq 16(%rdi), %rcx
; SSE-NEXT: popcntq 8(%rdi), %rdx
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq (%rdi), %rax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 56(%rdi), %rdx
; SSE-NEXT: popcntq 48(%rdi), %rsi
; SSE-NEXT: addl %ecx, %eax
; SSE-NEXT: addl %edx, %esi
; SSE-NEXT: xorl %ecx, %ecx
; SSE-NEXT: popcntq 40(%rdi), %rcx
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 32(%rdi), %rdx
; SSE-NEXT: addl %ecx, %edx
; SSE-NEXT: addl %esi, %edx
; SSE-NEXT: movd %edx, %xmm1
; SSE-NEXT: movd %eax, %xmm0
; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_v2i256_v2i32:
; AVX2: # %bb.0:
; AVX2-NEXT: popcntq 24(%rdi), %rax
; AVX2-NEXT: popcntq 16(%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 8(%rdi), %rax
; AVX2-NEXT: popcntq (%rdi), %rdx
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: addl %ecx, %edx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 56(%rdi), %rax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: popcntq 48(%rdi), %rcx
; AVX2-NEXT: popcntq 40(%rdi), %rsi
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 32(%rdi), %rax
; AVX2-NEXT: addl %esi, %eax
; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: vmovd %eax, %xmm0
; AVX2-NEXT: vmovd %edx, %xmm1
; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_v2i256_v2i32:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq 56(%rdi), %rax
; AVX512F-NEXT: popcntq 48(%rdi), %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq 40(%rdi), %rax
; AVX512F-NEXT: popcntq 32(%rdi), %rdx
; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: addl %ecx, %edx
; AVX512F-NEXT: popcntq 24(%rdi), %rax
; AVX512F-NEXT: popcntq 16(%rdi), %rcx
; AVX512F-NEXT: popcntq 8(%rdi), %rsi
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq (%rdi), %rax
; AVX512F-NEXT: addl %esi, %eax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: vmovd %eax, %xmm0
; AVX512F-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_v2i256_v2i32:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqa (%rdi), %ymm0
; AVX512POPCNT-NEXT: vmovdqa 32(%rdi), %ymm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpmovqb %zmm1, %xmm1
; AVX512POPCNT-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512POPCNT-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512POPCNT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_v2i256_v2i32:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq 56(%rdi), %rax
; AVX512VL-NEXT: popcntq 48(%rdi), %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 40(%rdi), %rax
; AVX512VL-NEXT: popcntq 32(%rdi), %rdx
; AVX512VL-NEXT: addl %eax, %edx
; AVX512VL-NEXT: addl %ecx, %edx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 24(%rdi), %rax
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq 16(%rdi), %rcx
; AVX512VL-NEXT: popcntq 8(%rdi), %rsi
; AVX512VL-NEXT: popcntq (%rdi), %rdi
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: addl %esi, %edi
; AVX512VL-NEXT: addl %ecx, %edi
; AVX512VL-NEXT: vmovd %edi, %xmm0
; AVX512VL-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_v2i256_v2i32:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpopcntq 32(%rdi), %ymm0
; AVX512VLPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %ymm2
; AVX512VLPOPCNT-NEXT: vpmovqb %ymm2, %xmm2
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm2, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load <2 x i256>, ptr %p0
%cnt = call <2 x i256> @llvm.ctpop.v2i256(<2 x i256> %a0)
%res = trunc <2 x i256> %cnt to <2 x i32>
ret <2 x i32> %res
}
define <2 x i64> @load_ctpop_v2i256_v2i64(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_v2i256_v2i64:
; SSE: # %bb.0:
; SSE-NEXT: popcntq 24(%rdi), %rax
; SSE-NEXT: popcntq 16(%rdi), %rcx
; SSE-NEXT: popcntq 8(%rdi), %rdx
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq (%rdi), %rax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 56(%rdi), %rdx
; SSE-NEXT: popcntq 48(%rdi), %rsi
; SSE-NEXT: addl %ecx, %eax
; SSE-NEXT: addl %edx, %esi
; SSE-NEXT: xorl %ecx, %ecx
; SSE-NEXT: popcntq 40(%rdi), %rcx
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 32(%rdi), %rdx
; SSE-NEXT: addl %ecx, %edx
; SSE-NEXT: addl %esi, %edx
; SSE-NEXT: movd %edx, %xmm1
; SSE-NEXT: movd %eax, %xmm0
; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_v2i256_v2i64:
; AVX2: # %bb.0:
; AVX2-NEXT: popcntq 24(%rdi), %rax
; AVX2-NEXT: popcntq 16(%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 8(%rdi), %rax
; AVX2-NEXT: popcntq (%rdi), %rdx
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: addl %ecx, %edx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 56(%rdi), %rax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: popcntq 48(%rdi), %rcx
; AVX2-NEXT: popcntq 40(%rdi), %rsi
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 32(%rdi), %rax
; AVX2-NEXT: addl %esi, %eax
; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: vmovd %eax, %xmm0
; AVX2-NEXT: vmovd %edx, %xmm1
; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_v2i256_v2i64:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq 24(%rdi), %rax
; AVX512F-NEXT: popcntq 16(%rdi), %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq 8(%rdi), %rax
; AVX512F-NEXT: popcntq (%rdi), %rdx
; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: addl %ecx, %edx
; AVX512F-NEXT: popcntq 56(%rdi), %rax
; AVX512F-NEXT: popcntq 48(%rdi), %rcx
; AVX512F-NEXT: popcntq 40(%rdi), %rsi
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq 32(%rdi), %rax
; AVX512F-NEXT: addl %esi, %eax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: vmovd %eax, %xmm0
; AVX512F-NEXT: vmovd %edx, %xmm1
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_v2i256_v2i64:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqa (%rdi), %ymm0
; AVX512POPCNT-NEXT: vmovdqa 32(%rdi), %ymm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpmovqb %zmm1, %xmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_v2i256_v2i64:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq 24(%rdi), %rax
; AVX512VL-NEXT: popcntq 16(%rdi), %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 8(%rdi), %rax
; AVX512VL-NEXT: popcntq (%rdi), %rdx
; AVX512VL-NEXT: addl %eax, %edx
; AVX512VL-NEXT: addl %ecx, %edx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 56(%rdi), %rax
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq 48(%rdi), %rcx
; AVX512VL-NEXT: popcntq 40(%rdi), %rsi
; AVX512VL-NEXT: popcntq 32(%rdi), %rdi
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: addl %esi, %edi
; AVX512VL-NEXT: addl %ecx, %edi
; AVX512VL-NEXT: vmovd %edi, %xmm0
; AVX512VL-NEXT: vmovd %edx, %xmm1
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_v2i256_v2i64:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpopcntq 32(%rdi), %ymm0
; AVX512VLPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %ymm1
; AVX512VLPOPCNT-NEXT: vpmovqb %ymm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load <2 x i256>, ptr %p0
%cnt = call <2 x i256> @llvm.ctpop.v2i256(<2 x i256> %a0)
%res = trunc <2 x i256> %cnt to <2 x i64>
ret <2 x i64> %res
}
define i32 @test_ctpop_i512(i512 %a0) nounwind {
; CHECK-LABEL: test_ctpop_i512:
; CHECK: # %bb.0:
; CHECK-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; CHECK-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; CHECK-NEXT: addl %eax, %r10d
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: popcntq %r9, %rax
; CHECK-NEXT: popcntq %r8, %r8
; CHECK-NEXT: addl %eax, %r8d
; CHECK-NEXT: addl %r10d, %r8d
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: popcntq %rcx, %rax
; CHECK-NEXT: xorl %ecx, %ecx
; CHECK-NEXT: popcntq %rdx, %rcx
; CHECK-NEXT: addl %eax, %ecx
; CHECK-NEXT: xorl %edx, %edx
; CHECK-NEXT: popcntq %rsi, %rdx
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: popcntq %rdi, %rax
; CHECK-NEXT: addl %edx, %eax
; CHECK-NEXT: addl %ecx, %eax
; CHECK-NEXT: addl %r8d, %eax
; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
; CHECK-NEXT: retq
;
; AVX512F-LABEL: test_ctpop_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512F-NEXT: addl %eax, %r10d
; AVX512F-NEXT: popcntq %r9, %rax
; AVX512F-NEXT: popcntq %r8, %r8
; AVX512F-NEXT: addl %eax, %r8d
; AVX512F-NEXT: addl %r10d, %r8d
; AVX512F-NEXT: popcntq %rcx, %rax
; AVX512F-NEXT: popcntq %rdx, %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq %rsi, %rdx
; AVX512F-NEXT: popcntq %rdi, %rax
; AVX512F-NEXT: addl %edx, %eax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: addl %r8d, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctpop_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512POPCNT-NEXT: addl %eax, %r10d
; AVX512POPCNT-NEXT: popcntq %r9, %rax
; AVX512POPCNT-NEXT: popcntq %r8, %r8
; AVX512POPCNT-NEXT: addl %eax, %r8d
; AVX512POPCNT-NEXT: addl %r10d, %r8d
; AVX512POPCNT-NEXT: popcntq %rcx, %rax
; AVX512POPCNT-NEXT: popcntq %rdx, %rcx
; AVX512POPCNT-NEXT: addl %eax, %ecx
; AVX512POPCNT-NEXT: popcntq %rsi, %rdx
; AVX512POPCNT-NEXT: popcntq %rdi, %rax
; AVX512POPCNT-NEXT: addl %edx, %eax
; AVX512POPCNT-NEXT: addl %ecx, %eax
; AVX512POPCNT-NEXT: addl %r8d, %eax
; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctpop_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512VL-NEXT: addl %eax, %r10d
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq %r9, %rax
; AVX512VL-NEXT: popcntq %r8, %r8
; AVX512VL-NEXT: addl %eax, %r8d
; AVX512VL-NEXT: addl %r10d, %r8d
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq %rcx, %rax
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq %rdx, %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %edx, %edx
; AVX512VL-NEXT: popcntq %rsi, %rdx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq %rdi, %rax
; AVX512VL-NEXT: addl %edx, %eax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: addl %r8d, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctpop_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512VLPOPCNT-NEXT: addl %eax, %r10d
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %r9, %rax
; AVX512VLPOPCNT-NEXT: popcntq %r8, %r8
; AVX512VLPOPCNT-NEXT: addl %eax, %r8d
; AVX512VLPOPCNT-NEXT: addl %r10d, %r8d
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %rcx, %rax
; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
; AVX512VLPOPCNT-NEXT: popcntq %rdx, %rcx
; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
; AVX512VLPOPCNT-NEXT: xorl %edx, %edx
; AVX512VLPOPCNT-NEXT: popcntq %rsi, %rdx
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %rdi, %rax
; AVX512VLPOPCNT-NEXT: addl %edx, %eax
; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
; AVX512VLPOPCNT-NEXT: addl %r8d, %eax
; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i512 @llvm.ctpop.i512(i512 %a0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @load_ctpop_i512(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_i512:
; SSE: # %bb.0:
; SSE-NEXT: popcntq 56(%rdi), %rax
; SSE-NEXT: popcntq 48(%rdi), %rcx
; SSE-NEXT: popcntq 40(%rdi), %rdx
; SSE-NEXT: popcntq 32(%rdi), %rsi
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: addl %edx, %esi
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 24(%rdi), %rax
; SSE-NEXT: addl %ecx, %esi
; SSE-NEXT: xorl %ecx, %ecx
; SSE-NEXT: popcntq 16(%rdi), %rcx
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 8(%rdi), %rdx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq (%rdi), %rax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: addl %ecx, %eax
; SSE-NEXT: addl %esi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: popcntq 56(%rdi), %rax
; AVX2-NEXT: popcntq 48(%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 40(%rdi), %rax
; AVX2-NEXT: popcntq 32(%rdi), %rdx
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: addl %ecx, %edx
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: popcntq 24(%rdi), %rcx
; AVX2-NEXT: popcntq 16(%rdi), %rsi
; AVX2-NEXT: popcntq 8(%rdi), %r8
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq (%rdi), %rax
; AVX2-NEXT: addl %ecx, %esi
; AVX2-NEXT: addl %r8d, %eax
; AVX2-NEXT: addl %esi, %eax
; AVX2-NEXT: addl %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq 56(%rdi), %rax
; AVX512F-NEXT: popcntq 48(%rdi), %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq 40(%rdi), %rax
; AVX512F-NEXT: popcntq 32(%rdi), %rdx
; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: addl %ecx, %edx
; AVX512F-NEXT: popcntq 24(%rdi), %rcx
; AVX512F-NEXT: popcntq 16(%rdi), %rsi
; AVX512F-NEXT: popcntq 8(%rdi), %r8
; AVX512F-NEXT: popcntq (%rdi), %rax
; AVX512F-NEXT: addl %ecx, %esi
; AVX512F-NEXT: addl %r8d, %eax
; AVX512F-NEXT: addl %esi, %eax
; AVX512F-NEXT: addl %edx, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpopcntq (%rdi), %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq 56(%rdi), %rax
; AVX512VL-NEXT: popcntq 48(%rdi), %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 40(%rdi), %rax
; AVX512VL-NEXT: popcntq 32(%rdi), %rdx
; AVX512VL-NEXT: addl %eax, %edx
; AVX512VL-NEXT: addl %ecx, %edx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 24(%rdi), %rax
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq 16(%rdi), %rcx
; AVX512VL-NEXT: popcntq 8(%rdi), %rsi
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq (%rdi), %rax
; AVX512VL-NEXT: addl %esi, %eax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: addl %edx, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %zmm0
; AVX512VLPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i512, ptr %p0
%cnt = call i512 @llvm.ctpop.i512(i512 %a0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @vector_ctpop_i512(<16 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctpop_i512:
; SSE: # %bb.0:
; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: pextrq $1, %xmm0, %rcx
; SSE-NEXT: movq %xmm1, %rdx
; SSE-NEXT: pextrq $1, %xmm1, %rsi
; SSE-NEXT: pextrq $1, %xmm2, %rdi
; SSE-NEXT: movq %xmm2, %r8
; SSE-NEXT: movq %xmm3, %r9
; SSE-NEXT: pextrq $1, %xmm3, %r10
; SSE-NEXT: popcntq %r10, %r10
; SSE-NEXT: popcntq %r9, %r9
; SSE-NEXT: addl %r10d, %r9d
; SSE-NEXT: popcntq %rdi, %rdi
; SSE-NEXT: popcntq %r8, %r8
; SSE-NEXT: addl %edi, %r8d
; SSE-NEXT: addl %r9d, %r8d
; SSE-NEXT: popcntq %rsi, %rsi
; SSE-NEXT: popcntq %rdx, %rdx
; SSE-NEXT: addl %esi, %edx
; SSE-NEXT: popcntq %rcx, %rcx
; SSE-NEXT: popcntq %rax, %rax
; SSE-NEXT: addl %ecx, %eax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: addl %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctpop_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: vmovq %xmm0, %rax
; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovq %xmm0, %rdx
; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
; AVX2-NEXT: vpextrq $1, %xmm1, %rdi
; AVX2-NEXT: vmovq %xmm1, %r8
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
; AVX2-NEXT: vpextrq $1, %xmm0, %r9
; AVX2-NEXT: vmovq %xmm0, %r10
; AVX2-NEXT: popcntq %r9, %r9
; AVX2-NEXT: popcntq %r10, %r10
; AVX2-NEXT: addl %r9d, %r10d
; AVX2-NEXT: popcntq %rdi, %rdi
; AVX2-NEXT: popcntq %r8, %r8
; AVX2-NEXT: addl %edi, %r8d
; AVX2-NEXT: addl %r10d, %r8d
; AVX2-NEXT: popcntq %rsi, %rsi
; AVX2-NEXT: popcntq %rdx, %rdx
; AVX2-NEXT: addl %esi, %edx
; AVX2-NEXT: popcntq %rcx, %rcx
; AVX2-NEXT: popcntq %rax, %rax
; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: addl %edx, %eax
; AVX2-NEXT: addl %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctpop_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vmovq %xmm1, %rax
; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx
; AVX512F-NEXT: vmovq %xmm0, %rsi
; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
; AVX512F-NEXT: vpextrq $1, %xmm1, %rdi
; AVX512F-NEXT: vmovq %xmm1, %r8
; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm0
; AVX512F-NEXT: vpextrq $1, %xmm0, %r9
; AVX512F-NEXT: vmovq %xmm0, %r10
; AVX512F-NEXT: popcntq %r9, %r9
; AVX512F-NEXT: popcntq %r10, %r10
; AVX512F-NEXT: addl %r9d, %r10d
; AVX512F-NEXT: popcntq %rdi, %rdi
; AVX512F-NEXT: popcntq %r8, %r8
; AVX512F-NEXT: addl %edi, %r8d
; AVX512F-NEXT: addl %r10d, %r8d
; AVX512F-NEXT: popcntq %rdx, %rdx
; AVX512F-NEXT: popcntq %rsi, %rsi
; AVX512F-NEXT: addl %edx, %esi
; AVX512F-NEXT: popcntq %rcx, %rcx
; AVX512F-NEXT: popcntq %rax, %rax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: addl %esi, %eax
; AVX512F-NEXT: addl %r8d, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctpop_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctpop_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vmovq %xmm1, %rax
; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx
; AVX512VL-NEXT: vmovq %xmm0, %rsi
; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
; AVX512VL-NEXT: vmovq %xmm1, %rdi
; AVX512VL-NEXT: vpextrq $1, %xmm1, %r8
; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm0
; AVX512VL-NEXT: vmovq %xmm0, %r9
; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
; AVX512VL-NEXT: popcntq %r10, %r10
; AVX512VL-NEXT: popcntq %r9, %r9
; AVX512VL-NEXT: addl %r10d, %r9d
; AVX512VL-NEXT: popcntq %r8, %r8
; AVX512VL-NEXT: popcntq %rdi, %rdi
; AVX512VL-NEXT: addl %r8d, %edi
; AVX512VL-NEXT: addl %r9d, %edi
; AVX512VL-NEXT: popcntq %rdx, %rdx
; AVX512VL-NEXT: popcntq %rsi, %rsi
; AVX512VL-NEXT: addl %edx, %esi
; AVX512VL-NEXT: popcntq %rcx, %rcx
; AVX512VL-NEXT: popcntq %rax, %rax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: addl %esi, %eax
; AVX512VL-NEXT: addl %edi, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctpop_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <16 x i32> %v0 to i512
%cnt = call i512 @llvm.ctpop.i512(i512 %a0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @test_ctpop_i1024(i1024 %a0) nounwind {
; SSE-LABEL: test_ctpop_i1024:
; SSE: # %bb.0:
; SSE-NEXT: pushq %rbx
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: addl %eax, %r10d
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl %r11d, %eax
; SSE-NEXT: xorl %r11d, %r11d
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: xorl %ebx, %ebx
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx
; SSE-NEXT: addl %r10d, %eax
; SSE-NEXT: addl %r11d, %ebx
; SSE-NEXT: xorl %r11d, %r11d
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: xorl %r10d, %r10d
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: addl %r11d, %r10d
; SSE-NEXT: addl %ebx, %r10d
; SSE-NEXT: xorl %r11d, %r11d
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: xorl %ebx, %ebx
; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx
; SSE-NEXT: addl %eax, %r10d
; SSE-NEXT: addl %r11d, %ebx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq %r9, %rax
; SSE-NEXT: popcntq %r8, %r8
; SSE-NEXT: addl %eax, %r8d
; SSE-NEXT: addl %ebx, %r8d
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq %rcx, %rax
; SSE-NEXT: xorl %ecx, %ecx
; SSE-NEXT: popcntq %rdx, %rcx
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq %rsi, %rdx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq %rdi, %rax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: addl %ecx, %eax
; SSE-NEXT: addl %r8d, %eax
; SSE-NEXT: addl %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctpop_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: addl %eax, %r10d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: addl %eax, %r11d
; AVX2-NEXT: addl %r10d, %r11d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r14
; AVX2-NEXT: addl %eax, %ebx
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: addl %r14d, %r10d
; AVX2-NEXT: addl %ebx, %r10d
; AVX2-NEXT: addl %r11d, %r10d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: addl %eax, %r11d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq %r9, %rax
; AVX2-NEXT: popcntq %r8, %r8
; AVX2-NEXT: addl %eax, %r8d
; AVX2-NEXT: addl %r11d, %r8d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq %rcx, %rax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: popcntq %rdx, %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: popcntq %rsi, %rdx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq %rdi, %rax
; AVX2-NEXT: addl %edx, %eax
; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: addl %r8d, %eax
; AVX2-NEXT: addl %r10d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctpop_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: pushq %r14
; AVX512F-NEXT: pushq %rbx
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512F-NEXT: addl %eax, %r10d
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; AVX512F-NEXT: addl %eax, %r11d
; AVX512F-NEXT: addl %r10d, %r11d
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r14
; AVX512F-NEXT: addl %eax, %ebx
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512F-NEXT: addl %r14d, %r10d
; AVX512F-NEXT: addl %ebx, %r10d
; AVX512F-NEXT: addl %r11d, %r10d
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; AVX512F-NEXT: addl %eax, %r11d
; AVX512F-NEXT: popcntq %r9, %rax
; AVX512F-NEXT: popcntq %r8, %r8
; AVX512F-NEXT: addl %eax, %r8d
; AVX512F-NEXT: addl %r11d, %r8d
; AVX512F-NEXT: popcntq %rcx, %rax
; AVX512F-NEXT: popcntq %rdx, %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq %rsi, %rdx
; AVX512F-NEXT: popcntq %rdi, %rax
; AVX512F-NEXT: addl %edx, %eax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: addl %r8d, %eax
; AVX512F-NEXT: addl %r10d, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: popq %rbx
; AVX512F-NEXT: popq %r14
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctpop_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512POPCNT-NEXT: addl %eax, %r10d
; AVX512POPCNT-NEXT: popcntq %r9, %rax
; AVX512POPCNT-NEXT: popcntq %r8, %r8
; AVX512POPCNT-NEXT: addl %eax, %r8d
; AVX512POPCNT-NEXT: addl %r10d, %r8d
; AVX512POPCNT-NEXT: popcntq %rcx, %rax
; AVX512POPCNT-NEXT: popcntq %rdx, %rcx
; AVX512POPCNT-NEXT: addl %eax, %ecx
; AVX512POPCNT-NEXT: popcntq %rsi, %rax
; AVX512POPCNT-NEXT: popcntq %rdi, %rdx
; AVX512POPCNT-NEXT: addl %eax, %edx
; AVX512POPCNT-NEXT: addl %ecx, %edx
; AVX512POPCNT-NEXT: addl %r8d, %edx
; AVX512POPCNT-NEXT: vpopcntq {{[0-9]+}}(%rsp), %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: addl %edx, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctpop_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: pushq %r14
; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512VL-NEXT: addl %eax, %r10d
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; AVX512VL-NEXT: addl %eax, %r11d
; AVX512VL-NEXT: addl %r10d, %r11d
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512VL-NEXT: xorl %ebx, %ebx
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx
; AVX512VL-NEXT: xorl %r14d, %r14d
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r14
; AVX512VL-NEXT: addl %eax, %ebx
; AVX512VL-NEXT: xorl %r10d, %r10d
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512VL-NEXT: addl %r14d, %r10d
; AVX512VL-NEXT: addl %ebx, %r10d
; AVX512VL-NEXT: addl %r11d, %r10d
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512VL-NEXT: xorl %r11d, %r11d
; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
; AVX512VL-NEXT: addl %eax, %r11d
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq %r9, %rax
; AVX512VL-NEXT: popcntq %r8, %r8
; AVX512VL-NEXT: addl %eax, %r8d
; AVX512VL-NEXT: addl %r11d, %r8d
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq %rcx, %rax
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq %rdx, %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %edx, %edx
; AVX512VL-NEXT: popcntq %rsi, %rdx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq %rdi, %rax
; AVX512VL-NEXT: addl %edx, %eax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: addl %r8d, %eax
; AVX512VL-NEXT: addl %r10d, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: popq %rbx
; AVX512VL-NEXT: popq %r14
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctpop_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512VLPOPCNT-NEXT: addl %eax, %r10d
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %r9, %rax
; AVX512VLPOPCNT-NEXT: popcntq %r8, %r8
; AVX512VLPOPCNT-NEXT: addl %eax, %r8d
; AVX512VLPOPCNT-NEXT: addl %r10d, %r8d
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %rcx, %rax
; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
; AVX512VLPOPCNT-NEXT: popcntq %rdx, %rcx
; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %rsi, %rax
; AVX512VLPOPCNT-NEXT: xorl %edx, %edx
; AVX512VLPOPCNT-NEXT: popcntq %rdi, %rdx
; AVX512VLPOPCNT-NEXT: addl %eax, %edx
; AVX512VLPOPCNT-NEXT: addl %ecx, %edx
; AVX512VLPOPCNT-NEXT: addl %r8d, %edx
; AVX512VLPOPCNT-NEXT: vpopcntq {{[0-9]+}}(%rsp), %zmm0
; AVX512VLPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl %edx, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.ctpop.i1024(i1024 %a0)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
define i32 @load_ctpop_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_i1024:
; SSE: # %bb.0:
; SSE-NEXT: popcntq 120(%rdi), %rax
; SSE-NEXT: popcntq 112(%rdi), %rcx
; SSE-NEXT: popcntq 104(%rdi), %rdx
; SSE-NEXT: popcntq 96(%rdi), %rsi
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: addl %edx, %esi
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 88(%rdi), %rax
; SSE-NEXT: addl %ecx, %esi
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 80(%rdi), %rdx
; SSE-NEXT: addl %eax, %edx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 72(%rdi), %rax
; SSE-NEXT: xorl %ecx, %ecx
; SSE-NEXT: popcntq 64(%rdi), %rcx
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: addl %edx, %ecx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 56(%rdi), %rax
; SSE-NEXT: addl %esi, %ecx
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 48(%rdi), %rdx
; SSE-NEXT: addl %eax, %edx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 40(%rdi), %rax
; SSE-NEXT: xorl %esi, %esi
; SSE-NEXT: popcntq 32(%rdi), %rsi
; SSE-NEXT: addl %eax, %esi
; SSE-NEXT: addl %edx, %esi
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq 24(%rdi), %rax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: popcntq 16(%rdi), %rdx
; SSE-NEXT: popcntq 8(%rdi), %r8
; SSE-NEXT: addl %eax, %edx
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: popcntq (%rdi), %rax
; SSE-NEXT: addl %r8d, %eax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: addl %esi, %eax
; SSE-NEXT: addl %ecx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: popcntq 120(%rdi), %rax
; AVX2-NEXT: popcntq 112(%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 104(%rdi), %rax
; AVX2-NEXT: popcntq 96(%rdi), %rdx
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: addl %ecx, %edx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 88(%rdi), %rax
; AVX2-NEXT: popcntq 80(%rdi), %rsi
; AVX2-NEXT: popcntq 72(%rdi), %r8
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: popcntq 64(%rdi), %rcx
; AVX2-NEXT: addl %eax, %esi
; AVX2-NEXT: addl %r8d, %ecx
; AVX2-NEXT: addl %esi, %ecx
; AVX2-NEXT: addl %edx, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 56(%rdi), %rax
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: popcntq 48(%rdi), %rdx
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: popcntq 40(%rdi), %rsi
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: popcntq 32(%rdi), %r8
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: addl %esi, %r8d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 24(%rdi), %rax
; AVX2-NEXT: addl %edx, %r8d
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: popcntq 16(%rdi), %rdx
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: popcntq 8(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq (%rdi), %rax
; AVX2-NEXT: addl %esi, %eax
; AVX2-NEXT: addl %edx, %eax
; AVX2-NEXT: addl %r8d, %eax
; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: popcntq 120(%rdi), %rax
; AVX512F-NEXT: popcntq 112(%rdi), %rcx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq 104(%rdi), %rax
; AVX512F-NEXT: popcntq 96(%rdi), %rdx
; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: addl %ecx, %edx
; AVX512F-NEXT: popcntq 88(%rdi), %rax
; AVX512F-NEXT: popcntq 80(%rdi), %rsi
; AVX512F-NEXT: popcntq 72(%rdi), %r8
; AVX512F-NEXT: popcntq 64(%rdi), %rcx
; AVX512F-NEXT: addl %eax, %esi
; AVX512F-NEXT: addl %r8d, %ecx
; AVX512F-NEXT: addl %esi, %ecx
; AVX512F-NEXT: addl %edx, %ecx
; AVX512F-NEXT: popcntq 56(%rdi), %rax
; AVX512F-NEXT: popcntq 48(%rdi), %rdx
; AVX512F-NEXT: popcntq 40(%rdi), %rsi
; AVX512F-NEXT: popcntq 32(%rdi), %r8
; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: addl %esi, %r8d
; AVX512F-NEXT: popcntq 24(%rdi), %rax
; AVX512F-NEXT: addl %edx, %r8d
; AVX512F-NEXT: popcntq 16(%rdi), %rdx
; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: popcntq 8(%rdi), %rsi
; AVX512F-NEXT: popcntq (%rdi), %rax
; AVX512F-NEXT: addl %esi, %eax
; AVX512F-NEXT: addl %edx, %eax
; AVX512F-NEXT: addl %r8d, %eax
; AVX512F-NEXT: addl %ecx, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpopcntq 64(%rdi), %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512POPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512POPCNT-NEXT: vpopcntq (%rdi), %zmm0
; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: addl %ecx, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: popcntq 120(%rdi), %rax
; AVX512VL-NEXT: popcntq 112(%rdi), %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 104(%rdi), %rax
; AVX512VL-NEXT: popcntq 96(%rdi), %rdx
; AVX512VL-NEXT: addl %eax, %edx
; AVX512VL-NEXT: addl %ecx, %edx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 88(%rdi), %rax
; AVX512VL-NEXT: popcntq 80(%rdi), %rsi
; AVX512VL-NEXT: popcntq 72(%rdi), %r8
; AVX512VL-NEXT: addl %eax, %esi
; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: popcntq 64(%rdi), %rcx
; AVX512VL-NEXT: addl %r8d, %ecx
; AVX512VL-NEXT: addl %esi, %ecx
; AVX512VL-NEXT: addl %edx, %ecx
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 56(%rdi), %rax
; AVX512VL-NEXT: xorl %edx, %edx
; AVX512VL-NEXT: popcntq 48(%rdi), %rdx
; AVX512VL-NEXT: xorl %esi, %esi
; AVX512VL-NEXT: popcntq 40(%rdi), %rsi
; AVX512VL-NEXT: addl %eax, %edx
; AVX512VL-NEXT: xorl %r8d, %r8d
; AVX512VL-NEXT: popcntq 32(%rdi), %r8
; AVX512VL-NEXT: addl %esi, %r8d
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq 24(%rdi), %rax
; AVX512VL-NEXT: addl %edx, %r8d
; AVX512VL-NEXT: xorl %edx, %edx
; AVX512VL-NEXT: popcntq 16(%rdi), %rdx
; AVX512VL-NEXT: addl %eax, %edx
; AVX512VL-NEXT: xorl %esi, %esi
; AVX512VL-NEXT: popcntq 8(%rdi), %rsi
; AVX512VL-NEXT: xorl %eax, %eax
; AVX512VL-NEXT: popcntq (%rdi), %rax
; AVX512VL-NEXT: addl %esi, %eax
; AVX512VL-NEXT: addl %edx, %eax
; AVX512VL-NEXT: addl %r8d, %eax
; AVX512VL-NEXT: addl %ecx, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpopcntq 64(%rdi), %zmm0
; AVX512VLPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %zmm2
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512VLPOPCNT-NEXT: vpmovqb %zmm2, %xmm0
; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
%cnt = call i1024 @llvm.ctpop.i1024(i1024 %a0)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
;
; CTLZ
;
define i32 @test_ctlz_i128(i128 %a0) nounwind {
; SSE-LABEL: test_ctlz_i128:
; SSE: # %bb.0:
; SSE-NEXT: bsrq %rsi, %rcx
; SSE-NEXT: xorl $63, %ecx
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq %rdi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %ecx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: lzcntq %rsi, %rcx
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %ecx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_ctlz_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: lzcntq %rsi, %rcx
; AVX512-NEXT: lzcntq %rdi, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rsi, %rsi
; AVX512-NEXT: cmovnel %ecx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @load_ctlz_i128(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_i128:
; SSE: # %bb.0:
; SSE-NEXT: movq 8(%rdi), %rcx
; SSE-NEXT: bsrq %rcx, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: movq 8(%rdi), %rcx
; AVX2-NEXT: lzcntq %rcx, %rdx
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: load_ctlz_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: movq 8(%rdi), %rcx
; AVX512-NEXT: lzcntq %rcx, %rdx
; AVX512-NEXT: lzcntq (%rdi), %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rcx, %rcx
; AVX512-NEXT: cmovnel %edx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%a0 = load i128, ptr %p0
%cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @vector_ctlz_i128(<4 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctlz_i128:
; SSE: # %bb.0:
; SSE-NEXT: movq %xmm0, %rcx
; SSE-NEXT: pextrq $1, %xmm0, %rdx
; SSE-NEXT: bsrq %rdx, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq %rcx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctlz_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: vmovq %xmm0, %rax
; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; AVX2-NEXT: lzcntq %rcx, %rdx
; AVX2-NEXT: lzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctlz_i128:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovq %xmm0, %rax
; AVX512F-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512F-NEXT: lzcntq %rcx, %rdx
; AVX512F-NEXT: lzcntq %rax, %rax
; AVX512F-NEXT: addl $64, %eax
; AVX512F-NEXT: testq %rcx, %rcx
; AVX512F-NEXT: cmovnel %edx, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctlz_i128:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovq %xmm0, %rax
; AVX512POPCNT-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512POPCNT-NEXT: lzcntq %rcx, %rdx
; AVX512POPCNT-NEXT: lzcntq %rax, %rax
; AVX512POPCNT-NEXT: addl $64, %eax
; AVX512POPCNT-NEXT: testq %rcx, %rcx
; AVX512POPCNT-NEXT: cmovnel %edx, %eax
; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctlz_i128:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512VL-NEXT: vmovq %xmm0, %rax
; AVX512VL-NEXT: lzcntq %rcx, %rdx
; AVX512VL-NEXT: lzcntq %rax, %rax
; AVX512VL-NEXT: addl $64, %eax
; AVX512VL-NEXT: testq %rcx, %rcx
; AVX512VL-NEXT: cmovnel %edx, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctlz_i128:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512VLPOPCNT-NEXT: vmovq %xmm0, %rax
; AVX512VLPOPCNT-NEXT: lzcntq %rcx, %rdx
; AVX512VLPOPCNT-NEXT: lzcntq %rax, %rax
; AVX512VLPOPCNT-NEXT: addl $64, %eax
; AVX512VLPOPCNT-NEXT: testq %rcx, %rcx
; AVX512VLPOPCNT-NEXT: cmovnel %edx, %eax
; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <4 x i32> %v0 to i128
%cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @test_ctlz_i256(i256 %a0) nounwind {
; SSE-LABEL: test_ctlz_i256:
; SSE: # %bb.0:
; SSE-NEXT: bsrq %rcx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rdx, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: bsrq %rsi, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq %rdi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rcx, %rdx
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: lzcntq %rcx, %rax
; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: lzcntq %rsi, %r9
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_ctlz_i256:
; AVX512: # %bb.0:
; AVX512-NEXT: lzcntq %rcx, %rax
; AVX512-NEXT: lzcntq %rdx, %r8
; AVX512-NEXT: addl $64, %r8d
; AVX512-NEXT: testq %rcx, %rcx
; AVX512-NEXT: cmovnel %eax, %r8d
; AVX512-NEXT: lzcntq %rsi, %r9
; AVX512-NEXT: lzcntq %rdi, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rsi, %rsi
; AVX512-NEXT: cmovnel %r9d, %eax
; AVX512-NEXT: subl $-128, %eax
; AVX512-NEXT: orq %rcx, %rdx
; AVX512-NEXT: cmovnel %r8d, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @load_ctlz_i256(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_i256:
; SSE: # %bb.0:
; SSE-NEXT: movq 16(%rdi), %rcx
; SSE-NEXT: movq 24(%rdi), %rdx
; SSE-NEXT: bsrq %rdx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rcx, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %esi
; SSE-NEXT: movq 8(%rdi), %r8
; SSE-NEXT: bsrq %r8, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rdx, %rcx
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: movq 16(%rdi), %rcx
; AVX2-NEXT: movq 24(%rdi), %rdx
; AVX2-NEXT: lzcntq %rdx, %rax
; AVX2-NEXT: lzcntq %rcx, %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %eax, %esi
; AVX2-NEXT: movq 8(%rdi), %r8
; AVX2-NEXT: lzcntq %r8, %r9
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rdx, %rcx
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = mem[3,2,1,0]
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm0 {%k1}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm1 = mem[3,2,1,0]
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm0 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512VL-NEXT: vplzcntq %ymm0, %ymm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512VLPOPCNT-NEXT: vplzcntq %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i256, ptr %p0
%cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @vector_ctlz_i256(<8 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctlz_i256:
; SSE: # %bb.0:
; SSE-NEXT: movq %xmm0, %rcx
; SSE-NEXT: pextrq $1, %xmm0, %rdx
; SSE-NEXT: movq %xmm1, %rax
; SSE-NEXT: pextrq $1, %xmm1, %rsi
; SSE-NEXT: bsrq %rsi, %rdi
; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: bsrq %rax, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %edi, %r8d
; SSE-NEXT: bsrq %rdx, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq %rcx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctlz_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: vmovq %xmm0, %rax
; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovq %xmm0, %rdx
; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
; AVX2-NEXT: lzcntq %rsi, %rdi
; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %edi, %r8d
; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: lzcntq %rcx, %rdi
; AVX2-NEXT: lzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rsi, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctlz_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256]
; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512F-NEXT: vplzcntq %zmm0, %zmm2
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctlz_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256]
; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm2
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctlz_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512VL-NEXT: vplzcntq %ymm0, %ymm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctlz_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512VLPOPCNT-NEXT: vplzcntq %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <8 x i32> %v0 to i256
%cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @test_ctlz_i512(i512 %a0) nounwind {
; SSE-LABEL: test_ctlz_i512:
; SSE: # %bb.0:
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: bsrq %r11, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %r10, %r14
; SSE-NEXT: xorl $63, %r14d
; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %r11, %r11
; SSE-NEXT: cmovnel %eax, %r14d
; SSE-NEXT: bsrq %r9, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %r8, %rbx
; SSE-NEXT: xorl $63, %ebx
; SSE-NEXT: orl $64, %ebx
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %eax, %ebx
; SSE-NEXT: subl $-128, %ebx
; SSE-NEXT: movq %r10, %rax
; SSE-NEXT: orq %r11, %rax
; SSE-NEXT: cmovnel %r14d, %ebx
; SSE-NEXT: bsrq %rcx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rdx, %r14
; SSE-NEXT: xorl $63, %r14d
; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %eax, %r14d
; SSE-NEXT: bsrq %rsi, %r15
; SSE-NEXT: xorl $63, %r15d
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq %rdi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rcx, %rdx
; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: orq %r11, %r9
; SSE-NEXT: orq %r10, %r8
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: lzcntq %r11, %rax
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %r10, %r14
; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %eax, %r14d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r8, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: subl $-128, %ebx
; AVX2-NEXT: movq %r10, %rax
; AVX2-NEXT: orq %r11, %rax
; AVX2-NEXT: cmovnel %r14d, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rcx, %rax
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %rdx, %r14
; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %r14d
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: lzcntq %rsi, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: orq %r11, %r9
; AVX2-NEXT: orq %r10, %r8
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctlz_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovq %rdi, %xmm0
; AVX512F-NEXT: vmovq %rsi, %xmm1
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512F-NEXT: vmovq %rdx, %xmm1
; AVX512F-NEXT: vmovq %rcx, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
; AVX512F-NEXT: vmovq %r8, %xmm2
; AVX512F-NEXT: vmovq %r9, %xmm3
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctlz_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0
; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
; AVX512POPCNT-NEXT: vmovq %r9, %xmm3
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctlz_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovq %rdi, %xmm0
; AVX512VL-NEXT: vmovq %rsi, %xmm1
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VL-NEXT: vmovq %rdx, %xmm1
; AVX512VL-NEXT: vmovq %rcx, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
; AVX512VL-NEXT: vmovq %r8, %xmm2
; AVX512VL-NEXT: vmovq %r9, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctlz_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm0
; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm2
; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @load_ctlz_i512(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_i512:
; SSE: # %bb.0:
; SSE-NEXT: movdqa 32(%rdi), %xmm0
; SSE-NEXT: movq 16(%rdi), %rcx
; SSE-NEXT: movq 24(%rdi), %rdx
; SSE-NEXT: movq 40(%rdi), %r8
; SSE-NEXT: movq 48(%rdi), %rax
; SSE-NEXT: movq 56(%rdi), %r9
; SSE-NEXT: bsrq %r9, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: bsrq %rax, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: orl $64, %r10d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %esi, %r10d
; SSE-NEXT: bsrq %r8, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: bsrq 32(%rdi), %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r9, %rax
; SSE-NEXT: cmovnel %r10d, %esi
; SSE-NEXT: bsrq %rdx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rcx, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: movq 8(%rdi), %r9
; SSE-NEXT: bsrq %r9, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rdx, %rcx
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por 48(%rdi), %xmm0
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: movq 16(%rdi), %rcx
; AVX2-NEXT: movq 24(%rdi), %rdx
; AVX2-NEXT: movq 40(%rdi), %rax
; AVX2-NEXT: movq 48(%rdi), %r8
; AVX2-NEXT: movq 56(%rdi), %r9
; AVX2-NEXT: lzcntq %r9, %rsi
; AVX2-NEXT: lzcntq %r8, %r10
; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %esi, %r10d
; AVX2-NEXT: lzcntq %rax, %r11
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: lzcntq 32(%rdi), %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rax, %rax
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %r10d, %esi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdx, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: lzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: movq 8(%rdi), %r9
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq %r9, %r10
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r10d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rdx, %rcx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: vmovdqa 32(%rdi), %xmm0
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 48(%rdi), %xmm0, %xmm0
; AVX2-NEXT: vptest %xmm0, %xmm0
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i512, ptr %p0
%cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @vector_ctlz_i512(<16 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctlz_i512:
; SSE: # %bb.0:
; SSE-NEXT: movq %xmm0, %rdx
; SSE-NEXT: pextrq $1, %xmm0, %rcx
; SSE-NEXT: pextrq $1, %xmm1, %rax
; SSE-NEXT: pextrq $1, %xmm2, %rdi
; SSE-NEXT: movq %xmm2, %rsi
; SSE-NEXT: movq %xmm3, %r8
; SSE-NEXT: pextrq $1, %xmm3, %r9
; SSE-NEXT: bsrq %r9, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq %r8, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r10d, %r8d
; SSE-NEXT: bsrq %rdi, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: bsrq %rsi, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %r9d, %esi
; SSE-NEXT: movq %xmm1, %rdi
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: ptest %xmm3, %xmm3
; SSE-NEXT: cmovnel %r8d, %esi
; SSE-NEXT: bsrq %rax, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: bsrq %rdi, %rdi
; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: orl $64, %edi
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %r8d, %edi
; SSE-NEXT: bsrq %rcx, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq %rdx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovnel %edi, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por %xmm3, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctlz_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
; AVX2-NEXT: vmovq %xmm2, %rdx
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rax
; AVX2-NEXT: vmovq %xmm2, %r8
; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
; AVX2-NEXT: vmovq %xmm1, %rdi
; AVX2-NEXT: vpextrq $1, %xmm1, %r9
; AVX2-NEXT: lzcntq %rax, %r10
; AVX2-NEXT: lzcntq %r8, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rax, %rax
; AVX2-NEXT: cmovnel %r10d, %r11d
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq %r9, %r10
; AVX2-NEXT: lzcntq %rdi, %rdi
; AVX2-NEXT: addl $64, %edi
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r10d, %edi
; AVX2-NEXT: subl $-128, %edi
; AVX2-NEXT: orq %rax, %r8
; AVX2-NEXT: cmovnel %r11d, %edi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rcx, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: vmovq %xmm0, %rax
; AVX2-NEXT: xorl %r9d, %r9d
; AVX2-NEXT: lzcntq %rsi, %r9
; AVX2-NEXT: lzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vptest %ymm1, %ymm1
; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctlz_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctlz_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctlz_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctlz_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <16 x i32> %v0 to i512
%cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @test_ctlz_i1024(i1024 %a0) nounwind {
; SSE-LABEL: test_ctlz_i1024:
; SSE: # %bb.0:
; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: bsrq %r11, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq %rax, %r14
; SSE-NEXT: xorl $63, %r14d
; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %r11, %r11
; SSE-NEXT: cmovnel %r10d, %r14d
; SSE-NEXT: bsrq %r9, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq %r8, %rbx
; SSE-NEXT: xorl $63, %ebx
; SSE-NEXT: orl $64, %ebx
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r10d, %ebx
; SSE-NEXT: subl $-128, %ebx
; SSE-NEXT: movq %rax, %r10
; SSE-NEXT: orq %r11, %r10
; SSE-NEXT: cmovnel %r14d, %ebx
; SSE-NEXT: bsrq %rcx, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq %rdx, %r14
; SSE-NEXT: xorl $63, %r14d
; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r10d, %r14d
; SSE-NEXT: movl $127, %r10d
; SSE-NEXT: bsrq %rdi, %r10
; SSE-NEXT: bsrq %rsi, %rdi
; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: addl $64, %r10d
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %edi, %r10d
; SSE-NEXT: subl $-128, %r10d
; SSE-NEXT: orq %rcx, %rdx
; SSE-NEXT: cmovnel %r14d, %r10d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; SSE-NEXT: orq %r11, %r9
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE-NEXT: orq %rax, %r8
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $256, %r10d # imm = 0x100
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %ebx, %r10d
; SSE-NEXT: addl $512, %r10d # imm = 0x200
; SSE-NEXT: bsrq %rax, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: bsrq %rsi, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: orl $64, %r9d
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %edx, %r9d
; SSE-NEXT: bsrq %rdi, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: orl $64, %edx
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %r8d, %edx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
; SSE-NEXT: subl $-128, %edx
; SSE-NEXT: orq %rax, %rsi
; SSE-NEXT: cmovnel %r9d, %edx
; SSE-NEXT: bsrq %r8, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rdi, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %eax, %esi
; SSE-NEXT: bsrq %rcx, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r8, %rdi
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm1
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm1
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r14
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r15, %rbx
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: lzcntq %r11, %r12
; AVX2-NEXT: addl $64, %r12d
; AVX2-NEXT: testq %r15, %r15
; AVX2-NEXT: cmovnel %ebx, %r12d
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r14
; AVX2-NEXT: xorl %r13d, %r13d
; AVX2-NEXT: lzcntq %r10, %r13
; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: cmovnel %r13d, %r14d
; AVX2-NEXT: subl $-128, %r14d
; AVX2-NEXT: orq %r15, %r11
; AVX2-NEXT: cmovnel %r12d, %r14d
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq %rbx, %r10
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: lzcntq %rax, %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %rbx, %rbx
; AVX2-NEXT: cmovnel %r10d, %r15d
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: lzcntq %r11, %r12
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %r12d, %r10d
; AVX2-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm0
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: subl $-128, %r10d
; AVX2-NEXT: orq %rbx, %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: cmovnel %r15d, %r10d
; AVX2-NEXT: addl $256, %r10d # imm = 0x100
; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %r14d, %r10d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rbx, %rax
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: lzcntq %r11, %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %rbx, %rbx
; AVX2-NEXT: cmovnel %eax, %r15d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %r8, %r14
; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %eax, %r14d
; AVX2-NEXT: subl $-128, %r14d
; AVX2-NEXT: movq %r11, %rax
; AVX2-NEXT: orq %rbx, %rax
; AVX2-NEXT: cmovnel %r15d, %r14d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rcx, %rax
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: lzcntq %rdx, %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %r15d
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: lzcntq %rsi, %r12
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: orq %rbx, %r9
; AVX2-NEXT: orq %r11, %r8
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $512, %eax # imm = 0x200
; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: cmovnel %r10d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctlz_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512F-NEXT: vmovq %rdi, %xmm1
; AVX512F-NEXT: vmovq %rsi, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vmovq %rdx, %xmm2
; AVX512F-NEXT: vmovq %rcx, %xmm3
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512F-NEXT: vpshufd {{.*#+}} xmm3 = mem[2,3,0,1]
; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512F-NEXT: vmovq %r8, %xmm2
; AVX512F-NEXT: vmovq %r9, %xmm4
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %ecx
; AVX512F-NEXT: addl $512, %ecx # imm = 0x200
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512F-NEXT: vptest %ymm0, %ymm0
; AVX512F-NEXT: cmovel %ecx, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctlz_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512POPCNT-NEXT: vmovq %rdi, %xmm1
; AVX512POPCNT-NEXT: vmovq %rsi, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vmovq %rdx, %xmm2
; AVX512POPCNT-NEXT: vmovq %rcx, %xmm3
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm3 = mem[2,3,0,1]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
; AVX512POPCNT-NEXT: vmovq %r9, %xmm4
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %ecx
; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512POPCNT-NEXT: cmovel %ecx, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctlz_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512VL-NEXT: vmovq %rdi, %xmm1
; AVX512VL-NEXT: vmovq %rsi, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vmovq %rdx, %xmm2
; AVX512VL-NEXT: vmovq %rcx, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512VL-NEXT: vmovq %r8, %xmm3
; AVX512VL-NEXT: vmovq %r9, %xmm4
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512VL-NEXT: vmovd %xmm1, %ecx
; AVX512VL-NEXT: addl $512, %ecx # imm = 0x200
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm1, %eax
; AVX512VL-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512VL-NEXT: vptest %ymm0, %ymm0
; AVX512VL-NEXT: cmovel %ecx, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctlz_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm2
; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm3
; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm4
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %ecx
; AVX512VLPOPCNT-NEXT: addl $512, %ecx # imm = 0x200
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
; AVX512VLPOPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: cmovel %ecx, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 0)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
define i32 @load_ctlz_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_i1024:
; SSE: # %bb.0:
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq 8(%rdi), %rcx
; SSE-NEXT: movq 72(%rdi), %rax
; SSE-NEXT: movq 104(%rdi), %r9
; SSE-NEXT: movq 112(%rdi), %rdx
; SSE-NEXT: movq 120(%rdi), %r8
; SSE-NEXT: bsrq %r8, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: bsrq %rdx, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %esi, %r11d
; SSE-NEXT: bsrq %r9, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq 96(%rdi), %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r10d, %esi
; SSE-NEXT: movq 80(%rdi), %r9
; SSE-NEXT: movq 88(%rdi), %r10
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r8, %rdx
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: bsrq %r10, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: bsrq %r9, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %edx, %r11d
; SSE-NEXT: bsrq %rax, %rbx
; SSE-NEXT: xorl $63, %ebx
; SSE-NEXT: bsrq 64(%rdi), %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: orl $64, %edx
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: movq 40(%rdi), %r8
; SSE-NEXT: cmovnel %ebx, %edx
; SSE-NEXT: movq 48(%rdi), %rax
; SSE-NEXT: movdqa 112(%rdi), %xmm0
; SSE-NEXT: movdqa 96(%rdi), %xmm1
; SSE-NEXT: subl $-128, %edx
; SSE-NEXT: orq %r10, %r9
; SSE-NEXT: cmovnel %r11d, %edx
; SSE-NEXT: addl $256, %edx # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: movq 56(%rdi), %r10
; SSE-NEXT: cmovnel %esi, %edx
; SSE-NEXT: bsrq %r10, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: bsrq %rax, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %esi, %r11d
; SSE-NEXT: bsrq %r8, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: bsrq 32(%rdi), %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %r9d, %esi
; SSE-NEXT: movq 16(%rdi), %r8
; SSE-NEXT: movq 24(%rdi), %r9
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r10, %rax
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: bsrq %r9, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %r8, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: orl $64, %r10d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %eax, %r10d
; SSE-NEXT: bsrq %rcx, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: movdqa 32(%rdi), %xmm2
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por 48(%rdi), %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: por 80(%rdi), %xmm0
; SSE-NEXT: por 64(%rdi), %xmm1
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq 16(%rdi), %rcx
; AVX2-NEXT: movq 72(%rdi), %rsi
; AVX2-NEXT: movq 104(%rdi), %rdx
; AVX2-NEXT: movq 112(%rdi), %r8
; AVX2-NEXT: movq 120(%rdi), %r10
; AVX2-NEXT: lzcntq %r10, %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r8, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: lzcntq %rdx, %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq 96(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: movq 80(%rdi), %r9
; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: movq 88(%rdi), %r11
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r10, %r8
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq %r11, %rdx
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r9, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %edx, %ebx
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq 64(%rdi), %rdx
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: lzcntq %rsi, %r8
; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: movq 40(%rdi), %r10
; AVX2-NEXT: cmovnel %r8d, %edx
; AVX2-NEXT: movq 48(%rdi), %r8
; AVX2-NEXT: subl $-128, %edx
; AVX2-NEXT: orq %r11, %r9
; AVX2-NEXT: movq 56(%rdi), %r9
; AVX2-NEXT: cmovnel %ebx, %edx
; AVX2-NEXT: vmovdqu 96(%rdi), %ymm0
; AVX2-NEXT: addl $256, %edx # imm = 0x100
; AVX2-NEXT: vpor 112(%rdi), %xmm0, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %eax, %edx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: lzcntq %r8, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %eax, %r11d
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: lzcntq 32(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r10, %rax
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: movq 24(%rdi), %r10
; AVX2-NEXT: cmovnel %eax, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r10, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: lzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: movq 8(%rdi), %r9
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: lzcntq %r9, %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r10, %rcx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: vmovdqa 32(%rdi), %xmm1
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 48(%rdi), %xmm1, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: vpor 64(%rdi), %ymm0, %ymm0
; AVX2-NEXT: addl $512, %eax # imm = 0x200
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vplzcntq %zmm2, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1
; AVX512F-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm2, %ecx
; AVX512F-NEXT: vpermq (%rdi), %zmm1, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
; AVX512F-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
; AVX512F-NEXT: vpor 96(%rdi), %ymm0, %ymm0
; AVX512F-NEXT: vptest %ymm0, %ymm0
; AVX512F-NEXT: cmovnel %ecx, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
; AVX512POPCNT-NEXT: vplzcntq %zmm2, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512POPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm2, %ecx
; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512POPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm2
; AVX512VL-NEXT: vplzcntq %zmm2, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512VL-NEXT: vptestmq %zmm2, %zmm2, %k1
; AVX512VL-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm2, %ecx
; AVX512VL-NEXT: vpermq (%rdi), %zmm1, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VL-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512VL-NEXT: vmovd %xmm1, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
; AVX512VL-NEXT: vpor 96(%rdi), %ymm0, %ymm0
; AVX512VL-NEXT: vptest %ymm0, %ymm0
; AVX512VL-NEXT: cmovnel %ecx, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm2, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm2, %ecx
; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512VLPOPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
%cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 0)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
;
; CTLZ_ZERO_UNDEF
;
define i32 @test_ctlz_undef_i128(i128 %a0) nounwind {
; SSE-LABEL: test_ctlz_undef_i128:
; SSE: # %bb.0:
; SSE-NEXT: bsrq %rsi, %rcx
; SSE-NEXT: xorl $63, %ecx
; SSE-NEXT: bsrq %rdi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %ecx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_undef_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: lzcntq %rsi, %rcx
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %ecx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_ctlz_undef_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: lzcntq %rsi, %rcx
; AVX512-NEXT: lzcntq %rdi, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rsi, %rsi
; AVX512-NEXT: cmovnel %ecx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 -1)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @load_ctlz_undef_i128(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_undef_i128:
; SSE: # %bb.0:
; SSE-NEXT: movq 8(%rdi), %rcx
; SSE-NEXT: bsrq %rcx, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_undef_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: movq 8(%rdi), %rcx
; AVX2-NEXT: lzcntq %rcx, %rdx
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: load_ctlz_undef_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: movq 8(%rdi), %rcx
; AVX512-NEXT: lzcntq %rcx, %rdx
; AVX512-NEXT: lzcntq (%rdi), %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rcx, %rcx
; AVX512-NEXT: cmovnel %edx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%a0 = load i128, ptr %p0
%cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 -1)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @vector_ctlz_undef_i128(<4 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctlz_undef_i128:
; SSE: # %bb.0:
; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: pextrq $1, %xmm0, %rcx
; SSE-NEXT: bsrq %rcx, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: bsrq %rax, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctlz_undef_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: vmovq %xmm0, %rax
; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; AVX2-NEXT: lzcntq %rcx, %rdx
; AVX2-NEXT: lzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctlz_undef_i128:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovq %xmm0, %rax
; AVX512F-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512F-NEXT: lzcntq %rcx, %rdx
; AVX512F-NEXT: lzcntq %rax, %rax
; AVX512F-NEXT: addl $64, %eax
; AVX512F-NEXT: testq %rcx, %rcx
; AVX512F-NEXT: cmovnel %edx, %eax
; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctlz_undef_i128:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovq %xmm0, %rax
; AVX512POPCNT-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512POPCNT-NEXT: lzcntq %rcx, %rdx
; AVX512POPCNT-NEXT: lzcntq %rax, %rax
; AVX512POPCNT-NEXT: addl $64, %eax
; AVX512POPCNT-NEXT: testq %rcx, %rcx
; AVX512POPCNT-NEXT: cmovnel %edx, %eax
; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctlz_undef_i128:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512VL-NEXT: vmovq %xmm0, %rax
; AVX512VL-NEXT: lzcntq %rcx, %rdx
; AVX512VL-NEXT: lzcntq %rax, %rax
; AVX512VL-NEXT: addl $64, %eax
; AVX512VL-NEXT: testq %rcx, %rcx
; AVX512VL-NEXT: cmovnel %edx, %eax
; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctlz_undef_i128:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512VLPOPCNT-NEXT: vmovq %xmm0, %rax
; AVX512VLPOPCNT-NEXT: lzcntq %rcx, %rdx
; AVX512VLPOPCNT-NEXT: lzcntq %rax, %rax
; AVX512VLPOPCNT-NEXT: addl $64, %eax
; AVX512VLPOPCNT-NEXT: testq %rcx, %rcx
; AVX512VLPOPCNT-NEXT: cmovnel %edx, %eax
; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <4 x i32> %v0 to i128
%cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 -1)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @test_ctlz_undef_i256(i256 %a0) nounwind {
; SSE-LABEL: test_ctlz_undef_i256:
; SSE: # %bb.0:
; SSE-NEXT: bsrq %rcx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rdx, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: bsrq %rsi, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: bsrq %rdi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rcx, %rdx
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_undef_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: lzcntq %rcx, %rax
; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: lzcntq %rsi, %r9
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_ctlz_undef_i256:
; AVX512: # %bb.0:
; AVX512-NEXT: lzcntq %rcx, %rax
; AVX512-NEXT: lzcntq %rdx, %r8
; AVX512-NEXT: addl $64, %r8d
; AVX512-NEXT: testq %rcx, %rcx
; AVX512-NEXT: cmovnel %eax, %r8d
; AVX512-NEXT: lzcntq %rsi, %r9
; AVX512-NEXT: lzcntq %rdi, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rsi, %rsi
; AVX512-NEXT: cmovnel %r9d, %eax
; AVX512-NEXT: subl $-128, %eax
; AVX512-NEXT: orq %rcx, %rdx
; AVX512-NEXT: cmovnel %r8d, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 -1)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @load_ctlz_undef_i256(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_undef_i256:
; SSE: # %bb.0:
; SSE-NEXT: movq 8(%rdi), %rdx
; SSE-NEXT: movq 16(%rdi), %rcx
; SSE-NEXT: movq 24(%rdi), %rsi
; SSE-NEXT: bsrq %rsi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rcx, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: bsrq %rdx, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rsi, %rcx
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_undef_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: movq 16(%rdi), %rcx
; AVX2-NEXT: movq 24(%rdi), %rdx
; AVX2-NEXT: lzcntq %rdx, %rax
; AVX2-NEXT: lzcntq %rcx, %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %eax, %esi
; AVX2-NEXT: movq 8(%rdi), %r8
; AVX2-NEXT: lzcntq %r8, %r9
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rdx, %rcx
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_undef_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_undef_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_undef_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vplzcntq %ymm0, %ymm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX512VL-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_undef_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i256, ptr %p0
%cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 -1)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @vector_ctlz_undef_i256(<8 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctlz_undef_i256:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm0, %rcx
; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: movq %xmm1, %rdx
; SSE-NEXT: pextrq $1, %xmm1, %rsi
; SSE-NEXT: bsrq %rsi, %rdi
; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: bsrq %rdx, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: orl $64, %edx
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %edi, %edx
; SSE-NEXT: bsrq %rcx, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: bsrq %rax, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctlz_undef_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: vmovq %xmm0, %rax
; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovq %xmm0, %rdx
; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
; AVX2-NEXT: lzcntq %rsi, %rdi
; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %edi, %r8d
; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: lzcntq %rcx, %rdi
; AVX2-NEXT: lzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rsi, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctlz_undef_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctlz_undef_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctlz_undef_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vplzcntq %ymm0, %ymm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX512VL-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctlz_undef_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <8 x i32> %v0 to i256
%cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 -1)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @test_ctlz_undef_i512(i512 %a0) nounwind {
; SSE-LABEL: test_ctlz_undef_i512:
; SSE: # %bb.0:
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: bsrq %r11, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %r10, %r14
; SSE-NEXT: xorl $63, %r14d
; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %r11, %r11
; SSE-NEXT: cmovnel %eax, %r14d
; SSE-NEXT: bsrq %r9, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %r8, %rbx
; SSE-NEXT: xorl $63, %ebx
; SSE-NEXT: orl $64, %ebx
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %eax, %ebx
; SSE-NEXT: subl $-128, %ebx
; SSE-NEXT: movq %r10, %rax
; SSE-NEXT: orq %r11, %rax
; SSE-NEXT: cmovnel %r14d, %ebx
; SSE-NEXT: bsrq %rcx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rdx, %r14
; SSE-NEXT: xorl $63, %r14d
; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %eax, %r14d
; SSE-NEXT: bsrq %rsi, %r15
; SSE-NEXT: xorl $63, %r15d
; SSE-NEXT: bsrq %rdi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rcx, %rdx
; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: orq %r11, %r9
; SSE-NEXT: orq %r10, %r8
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_undef_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: lzcntq %r11, %rax
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %r10, %r14
; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %eax, %r14d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r8, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: subl $-128, %ebx
; AVX2-NEXT: movq %r10, %rax
; AVX2-NEXT: orq %r11, %rax
; AVX2-NEXT: cmovnel %r14d, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rcx, %rax
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %rdx, %r14
; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %r14d
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: lzcntq %rsi, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: orq %r11, %r9
; AVX2-NEXT: orq %r10, %r8
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctlz_undef_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovq %rdi, %xmm0
; AVX512F-NEXT: vmovq %rsi, %xmm1
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512F-NEXT: vmovq %rdx, %xmm1
; AVX512F-NEXT: vmovq %rcx, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512F-NEXT: vmovq %r8, %xmm1
; AVX512F-NEXT: vmovq %r9, %xmm3
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctlz_undef_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0
; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512POPCNT-NEXT: vmovq %r8, %xmm1
; AVX512POPCNT-NEXT: vmovq %r9, %xmm3
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctlz_undef_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovq %rdi, %xmm0
; AVX512VL-NEXT: vmovq %rsi, %xmm1
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VL-NEXT: vmovq %rdx, %xmm1
; AVX512VL-NEXT: vmovq %rcx, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
; AVX512VL-NEXT: vmovq %r8, %xmm2
; AVX512VL-NEXT: vmovq %r9, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctlz_undef_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm0
; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm2
; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 -1)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @load_ctlz_undef_i512(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_undef_i512:
; SSE: # %bb.0:
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movdqa 32(%rdi), %xmm0
; SSE-NEXT: movq 8(%rdi), %rsi
; SSE-NEXT: movq 16(%rdi), %rcx
; SSE-NEXT: movq 24(%rdi), %rdx
; SSE-NEXT: movq 48(%rdi), %rax
; SSE-NEXT: movq 56(%rdi), %r9
; SSE-NEXT: bsrq %r9, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: bsrq %rax, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: orl $64, %r10d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r8d, %r10d
; SSE-NEXT: movq 40(%rdi), %r11
; SSE-NEXT: bsrq %r11, %rbx
; SSE-NEXT: bsrq 32(%rdi), %r8
; SSE-NEXT: xorl $63, %ebx
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %r11, %r11
; SSE-NEXT: cmovnel %ebx, %r8d
; SSE-NEXT: subl $-128, %r8d
; SSE-NEXT: orq %r9, %rax
; SSE-NEXT: cmovnel %r10d, %r8d
; SSE-NEXT: bsrq %rdx, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rcx, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: orl $64, %r9d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %r9d
; SSE-NEXT: bsrq %rsi, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rdx, %rcx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por 48(%rdi), %xmm0
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_undef_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: movq 16(%rdi), %rcx
; AVX2-NEXT: movq 24(%rdi), %rdx
; AVX2-NEXT: movq 40(%rdi), %rax
; AVX2-NEXT: movq 48(%rdi), %r8
; AVX2-NEXT: movq 56(%rdi), %r9
; AVX2-NEXT: lzcntq %r9, %rsi
; AVX2-NEXT: lzcntq %r8, %r10
; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %esi, %r10d
; AVX2-NEXT: lzcntq %rax, %r11
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: lzcntq 32(%rdi), %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rax, %rax
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %r10d, %esi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdx, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: lzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: movq 8(%rdi), %r9
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq %r9, %r10
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r10d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rdx, %rcx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: vmovdqa 32(%rdi), %xmm0
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 48(%rdi), %xmm0, %xmm0
; AVX2-NEXT: vptest %xmm0, %xmm0
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_undef_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_undef_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_undef_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_undef_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i512, ptr %p0
%cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 -1)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @vector_ctlz_undef_i512(<16 x i32> %v0) nounwind {
; SSE-LABEL: vector_ctlz_undef_i512:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm0, %rcx
; SSE-NEXT: pextrq $1, %xmm1, %rax
; SSE-NEXT: pextrq $1, %xmm2, %rsi
; SSE-NEXT: movq %xmm2, %rdx
; SSE-NEXT: movq %xmm3, %rdi
; SSE-NEXT: pextrq $1, %xmm3, %r8
; SSE-NEXT: bsrq %r8, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: bsrq %rdi, %rdi
; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: orl $64, %edi
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %r9d, %edi
; SSE-NEXT: bsrq %rsi, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: bsrq %rdx, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: orl $64, %edx
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %r8d, %edx
; SSE-NEXT: movq %xmm0, %rsi
; SSE-NEXT: subl $-128, %edx
; SSE-NEXT: ptest %xmm3, %xmm3
; SSE-NEXT: movq %xmm1, %r8
; SSE-NEXT: cmovnel %edi, %edx
; SSE-NEXT: bsrq %rax, %rdi
; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: bsrq %r8, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %edi, %r8d
; SSE-NEXT: bsrq %rcx, %rdi
; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: bsrq %rsi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %edi, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por %xmm3, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_ctlz_undef_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
; AVX2-NEXT: vmovq %xmm2, %rdx
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rax
; AVX2-NEXT: vmovq %xmm2, %r8
; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
; AVX2-NEXT: vmovq %xmm1, %rdi
; AVX2-NEXT: vpextrq $1, %xmm1, %r9
; AVX2-NEXT: lzcntq %rax, %r10
; AVX2-NEXT: lzcntq %r8, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rax, %rax
; AVX2-NEXT: cmovnel %r10d, %r11d
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq %r9, %r10
; AVX2-NEXT: lzcntq %rdi, %rdi
; AVX2-NEXT: addl $64, %edi
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r10d, %edi
; AVX2-NEXT: subl $-128, %edi
; AVX2-NEXT: orq %rax, %r8
; AVX2-NEXT: cmovnel %r11d, %edi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rcx, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: vmovq %xmm0, %rax
; AVX2-NEXT: xorl %r9d, %r9d
; AVX2-NEXT: lzcntq %rsi, %r9
; AVX2-NEXT: lzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vptest %ymm1, %ymm1
; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_ctlz_undef_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_ctlz_undef_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctlz_undef_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_ctlz_undef_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <16 x i32> %v0 to i512
%cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 -1)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @test_ctlz_undef_i1024(i1024 %a0) nounwind {
; SSE-LABEL: test_ctlz_undef_i1024:
; SSE: # %bb.0:
; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: bsrq %r10, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: bsrq %rax, %rbx
; SSE-NEXT: xorl $63, %ebx
; SSE-NEXT: orl $64, %ebx
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %r11d, %ebx
; SSE-NEXT: bsrq %r9, %r14
; SSE-NEXT: xorl $63, %r14d
; SSE-NEXT: bsrq %r8, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r14d, %r11d
; SSE-NEXT: subl $-128, %r11d
; SSE-NEXT: movq %rax, %r14
; SSE-NEXT: orq %r10, %r14
; SSE-NEXT: cmovnel %ebx, %r11d
; SSE-NEXT: bsrq %rcx, %rbx
; SSE-NEXT: xorl $63, %ebx
; SSE-NEXT: bsrq %rdx, %r14
; SSE-NEXT: xorl $63, %r14d
; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %ebx, %r14d
; SSE-NEXT: bsrq %rsi, %rbx
; SSE-NEXT: xorl $63, %ebx
; SSE-NEXT: bsrq %rdi, %rdi
; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: orl $64, %edi
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %ebx, %edi
; SSE-NEXT: subl $-128, %edi
; SSE-NEXT: orq %rcx, %rdx
; SSE-NEXT: cmovnel %r14d, %edi
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; SSE-NEXT: orq %r10, %r9
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE-NEXT: orq %rax, %r8
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $256, %edi # imm = 0x100
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %r11d, %edi
; SSE-NEXT: addl $512, %edi # imm = 0x200
; SSE-NEXT: bsrq %rax, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: bsrq %rsi, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: orl $64, %r10d
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %edx, %r10d
; SSE-NEXT: bsrq %rbx, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: orl $64, %edx
; SSE-NEXT: testq %rbx, %rbx
; SSE-NEXT: cmovnel %r8d, %edx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
; SSE-NEXT: subl $-128, %edx
; SSE-NEXT: orq %rax, %rsi
; SSE-NEXT: cmovnel %r10d, %edx
; SSE-NEXT: bsrq %r9, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %r8, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %eax, %esi
; SSE-NEXT: bsrq %rcx, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm1
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm1
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovel %edi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r14
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_undef_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r15, %rbx
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: lzcntq %r11, %r12
; AVX2-NEXT: addl $64, %r12d
; AVX2-NEXT: testq %r15, %r15
; AVX2-NEXT: cmovnel %ebx, %r12d
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r14
; AVX2-NEXT: xorl %r13d, %r13d
; AVX2-NEXT: lzcntq %r10, %r13
; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: cmovnel %r13d, %r14d
; AVX2-NEXT: subl $-128, %r14d
; AVX2-NEXT: orq %r15, %r11
; AVX2-NEXT: cmovnel %r12d, %r14d
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq %rbx, %r10
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: lzcntq %rax, %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %rbx, %rbx
; AVX2-NEXT: cmovnel %r10d, %r15d
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: lzcntq %r11, %r12
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %r12d, %r10d
; AVX2-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm0
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: subl $-128, %r10d
; AVX2-NEXT: orq %rbx, %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: cmovnel %r15d, %r10d
; AVX2-NEXT: addl $256, %r10d # imm = 0x100
; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %r14d, %r10d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rbx, %rax
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: lzcntq %r11, %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %rbx, %rbx
; AVX2-NEXT: cmovnel %eax, %r15d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %r8, %r14
; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %eax, %r14d
; AVX2-NEXT: subl $-128, %r14d
; AVX2-NEXT: movq %r11, %rax
; AVX2-NEXT: orq %rbx, %rax
; AVX2-NEXT: cmovnel %r15d, %r14d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rcx, %rax
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: lzcntq %rdx, %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %r15d
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: lzcntq %rsi, %r12
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: orq %rbx, %r9
; AVX2-NEXT: orq %r11, %r8
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $512, %eax # imm = 0x200
; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: cmovnel %r10d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctlz_undef_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512F-NEXT: vmovq %rdi, %xmm1
; AVX512F-NEXT: vmovq %rsi, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vmovq %rdx, %xmm2
; AVX512F-NEXT: vmovq %rcx, %xmm3
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512F-NEXT: vmovq %r8, %xmm3
; AVX512F-NEXT: vmovq %r9, %xmm4
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm1, %ecx
; AVX512F-NEXT: addl $512, %ecx # imm = 0x200
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512F-NEXT: vptest %ymm0, %ymm0
; AVX512F-NEXT: cmovel %ecx, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctlz_undef_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512POPCNT-NEXT: vmovq %rdi, %xmm1
; AVX512POPCNT-NEXT: vmovq %rsi, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vmovq %rdx, %xmm2
; AVX512POPCNT-NEXT: vmovq %rcx, %xmm3
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512POPCNT-NEXT: vmovq %r8, %xmm3
; AVX512POPCNT-NEXT: vmovq %r9, %xmm4
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm1, %ecx
; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512POPCNT-NEXT: cmovel %ecx, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctlz_undef_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512VL-NEXT: vmovq %rdi, %xmm1
; AVX512VL-NEXT: vmovq %rsi, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vmovq %rdx, %xmm2
; AVX512VL-NEXT: vmovq %rcx, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512VL-NEXT: vmovq %r8, %xmm3
; AVX512VL-NEXT: vmovq %r9, %xmm4
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm1, %ecx
; AVX512VL-NEXT: addl $512, %ecx # imm = 0x200
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm1, %eax
; AVX512VL-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512VL-NEXT: vptest %ymm0, %ymm0
; AVX512VL-NEXT: cmovel %ecx, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctlz_undef_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm2
; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm3
; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm4
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %ecx
; AVX512VLPOPCNT-NEXT: addl $512, %ecx # imm = 0x200
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
; AVX512VLPOPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: cmovel %ecx, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 -1)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
define i32 @load_ctlz_undef_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_undef_i1024:
; SSE: # %bb.0:
; SSE-NEXT: movq 8(%rdi), %rcx
; SSE-NEXT: movq 72(%rdi), %rax
; SSE-NEXT: movq 104(%rdi), %r8
; SSE-NEXT: movq 112(%rdi), %rdx
; SSE-NEXT: movq 120(%rdi), %r9
; SSE-NEXT: bsrq %r9, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: bsrq %rdx, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %esi, %r11d
; SSE-NEXT: bsrq %r8, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq 96(%rdi), %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: movq 80(%rdi), %r8
; SSE-NEXT: cmovnel %r10d, %esi
; SSE-NEXT: movq 88(%rdi), %r10
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r9, %rdx
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: bsrq %r10, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: bsrq %r8, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %edx, %r11d
; SSE-NEXT: bsrq %rax, %r9
; SSE-NEXT: xorl $63, %r9d
; SSE-NEXT: bsrq 64(%rdi), %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: orl $64, %edx
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %r9d, %edx
; SSE-NEXT: movq 40(%rdi), %r9
; SSE-NEXT: movq 48(%rdi), %rax
; SSE-NEXT: subl $-128, %edx
; SSE-NEXT: orq %r10, %r8
; SSE-NEXT: movq 56(%rdi), %r8
; SSE-NEXT: movdqa 112(%rdi), %xmm0
; SSE-NEXT: cmovnel %r11d, %edx
; SSE-NEXT: movdqa 96(%rdi), %xmm1
; SSE-NEXT: addl $256, %edx # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %esi, %edx
; SSE-NEXT: bsrq %r8, %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: bsrq %rax, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %esi, %r11d
; SSE-NEXT: bsrq %r9, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq 32(%rdi), %rsi
; SSE-NEXT: xorl $63, %esi
; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: movq 16(%rdi), %r9
; SSE-NEXT: cmovnel %r10d, %esi
; SSE-NEXT: movq 24(%rdi), %r10
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r8, %rax
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: bsrq %r10, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %r9, %r8
; SSE-NEXT: xorl $63, %r8d
; SSE-NEXT: orl $64, %r8d
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: bsrq %rcx, %r11
; SSE-NEXT: xorl $63, %r11d
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r10, %r9
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: movdqa 32(%rdi), %xmm2
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por 48(%rdi), %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: por 80(%rdi), %xmm0
; SSE-NEXT: por 64(%rdi), %xmm1
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_undef_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq 16(%rdi), %rcx
; AVX2-NEXT: movq 72(%rdi), %rsi
; AVX2-NEXT: movq 104(%rdi), %rdx
; AVX2-NEXT: movq 112(%rdi), %r8
; AVX2-NEXT: movq 120(%rdi), %r10
; AVX2-NEXT: lzcntq %r10, %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r8, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: lzcntq %rdx, %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq 96(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: movq 80(%rdi), %r9
; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: movq 88(%rdi), %r11
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r10, %r8
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq %r11, %rdx
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r9, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %edx, %ebx
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq 64(%rdi), %rdx
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: lzcntq %rsi, %r8
; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: movq 40(%rdi), %r10
; AVX2-NEXT: cmovnel %r8d, %edx
; AVX2-NEXT: movq 48(%rdi), %r8
; AVX2-NEXT: subl $-128, %edx
; AVX2-NEXT: orq %r11, %r9
; AVX2-NEXT: movq 56(%rdi), %r9
; AVX2-NEXT: cmovnel %ebx, %edx
; AVX2-NEXT: vmovdqu 96(%rdi), %ymm0
; AVX2-NEXT: addl $256, %edx # imm = 0x100
; AVX2-NEXT: vpor 112(%rdi), %xmm0, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %eax, %edx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: lzcntq %r8, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %eax, %r11d
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: lzcntq 32(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r10, %rax
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: movq 24(%rdi), %r10
; AVX2-NEXT: cmovnel %eax, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r10, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: lzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: movq 8(%rdi), %r9
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: lzcntq %r9, %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r10, %rcx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: vmovdqa 32(%rdi), %xmm1
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 48(%rdi), %xmm1, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: vpor 64(%rdi), %ymm0, %ymm0
; AVX2-NEXT: addl $512, %eax # imm = 0x200
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_undef_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vplzcntq %zmm2, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1
; AVX512F-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm2, %ecx
; AVX512F-NEXT: vpermq (%rdi), %zmm1, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
; AVX512F-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
; AVX512F-NEXT: vpor 96(%rdi), %ymm0, %ymm0
; AVX512F-NEXT: vptest %ymm0, %ymm0
; AVX512F-NEXT: cmovnel %ecx, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_undef_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
; AVX512POPCNT-NEXT: vplzcntq %zmm2, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512POPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm2, %ecx
; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512POPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_undef_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm2
; AVX512VL-NEXT: vplzcntq %zmm2, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512VL-NEXT: vptestmq %zmm2, %zmm2, %k1
; AVX512VL-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm2, %ecx
; AVX512VL-NEXT: vpermq (%rdi), %zmm1, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VL-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm1, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
; AVX512VL-NEXT: vpor 96(%rdi), %ymm0, %ymm0
; AVX512VL-NEXT: vptest %ymm0, %ymm0
; AVX512VL-NEXT: cmovnel %ecx, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_undef_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm2, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm2, %ecx
; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512VLPOPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
%cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 -1)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
;
; CTTZ
;
define i32 @test_cttz_i128(i128 %a0) nounwind {
; SSE-LABEL: test_cttz_i128:
; SSE: # %bb.0:
; SSE-NEXT: rep bsfq %rdi, %rcx
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq %rsi, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %ecx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: test_cttz_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: tzcntq %rdi, %rcx
; AVX2-NEXT: tzcntq %rsi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %ecx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_cttz_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: tzcntq %rdi, %rcx
; AVX512-NEXT: tzcntq %rsi, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rdi, %rdi
; AVX512-NEXT: cmovnel %ecx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @load_cttz_i128(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_i128:
; SSE: # %bb.0:
; SSE-NEXT: movq (%rdi), %rcx
; SSE-NEXT: rep bsfq %rcx, %rdx
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq 8(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: movq (%rdi), %rcx
; AVX2-NEXT: tzcntq %rcx, %rdx
; AVX2-NEXT: tzcntq 8(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: load_cttz_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: movq (%rdi), %rcx
; AVX512-NEXT: tzcntq %rcx, %rdx
; AVX512-NEXT: tzcntq 8(%rdi), %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rcx, %rcx
; AVX512-NEXT: cmovnel %edx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%a0 = load i128, ptr %p0
%cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @vector_cttz_i128(<4 x i32> %v0) nounwind {
; SSE-LABEL: vector_cttz_i128:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm0, %rcx
; SSE-NEXT: movq %xmm0, %rdx
; SSE-NEXT: rep bsfq %rdx, %rsi
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq %rcx, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_cttz_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: vpextrq $1, %xmm0, %rax
; AVX2-NEXT: vmovq %xmm0, %rcx
; AVX2-NEXT: tzcntq %rcx, %rdx
; AVX2-NEXT: tzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: vector_cttz_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: vpextrq $1, %xmm0, %rax
; AVX512-NEXT: vmovq %xmm0, %rcx
; AVX512-NEXT: tzcntq %rcx, %rdx
; AVX512-NEXT: tzcntq %rax, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rcx, %rcx
; AVX512-NEXT: cmovnel %edx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%a0 = bitcast <4 x i32> %v0 to i128
%cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 0)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @test_cttz_i256(i256 %a0) nounwind {
; SSE-LABEL: test_cttz_i256:
; SSE: # %bb.0:
; SSE-NEXT: rep bsfq %rdi, %rax
; SSE-NEXT: rep bsfq %rsi, %r8
; SSE-NEXT: addl $64, %r8d
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: rep bsfq %rdx, %r9
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq %rcx, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rsi, %rdi
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: test_cttz_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: tzcntq %rdi, %rax
; AVX2-NEXT: tzcntq %rsi, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: tzcntq %rdx, %r9
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rcx, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rsi, %rdi
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_cttz_i256:
; AVX512: # %bb.0:
; AVX512-NEXT: tzcntq %rdi, %rax
; AVX512-NEXT: tzcntq %rsi, %r8
; AVX512-NEXT: addl $64, %r8d
; AVX512-NEXT: testq %rdi, %rdi
; AVX512-NEXT: cmovnel %eax, %r8d
; AVX512-NEXT: tzcntq %rdx, %r9
; AVX512-NEXT: tzcntq %rcx, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rdx, %rdx
; AVX512-NEXT: cmovnel %r9d, %eax
; AVX512-NEXT: subl $-128, %eax
; AVX512-NEXT: orq %rsi, %rdi
; AVX512-NEXT: cmovnel %r8d, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @load_cttz_i256(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_i256:
; SSE: # %bb.0:
; SSE-NEXT: movq 16(%rdi), %rcx
; SSE-NEXT: movq (%rdi), %rdx
; SSE-NEXT: movq 8(%rdi), %rsi
; SSE-NEXT: rep bsfq %rdx, %rax
; SSE-NEXT: rep bsfq %rsi, %r8
; SSE-NEXT: addl $64, %r8d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: rep bsfq %rcx, %r9
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq 24(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rsi, %rdx
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: movq (%rdi), %rcx
; AVX2-NEXT: movq 8(%rdi), %rdx
; AVX2-NEXT: tzcntq %rcx, %rax
; AVX2-NEXT: tzcntq %rdx, %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %esi
; AVX2-NEXT: movq 16(%rdi), %r8
; AVX2-NEXT: tzcntq %r8, %r9
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 24(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rdx, %rcx
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu (%rdi), %ymm0
; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256]
; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; AVX512F-NEXT: vpaddq %ymm2, %ymm0, %ymm2
; AVX512F-NEXT: vpandn %ymm2, %ymm0, %ymm2
; AVX512F-NEXT: vplzcntq %zmm2, %zmm2
; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [64,128,192,256]
; AVX512F-NEXT: vpsubq %ymm2, %ymm3, %ymm2
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu (%rdi), %ymm0
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256]
; AVX512POPCNT-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; AVX512POPCNT-NEXT: vpaddq %ymm2, %ymm0, %ymm2
; AVX512POPCNT-NEXT: vpandn %ymm2, %ymm0, %ymm2
; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu (%rdi), %ymm0
; AVX512VL-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256]
; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu (%rdi), %ymm0
; AVX512VLPOPCNT-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i256, ptr %p0
%cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @vector_cttz_i256(<8 x i32> %v0) nounwind {
; SSE-LABEL: vector_cttz_i256:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm1, %rcx
; SSE-NEXT: pextrq $1, %xmm0, %rax
; SSE-NEXT: movq %xmm0, %rdx
; SSE-NEXT: rep bsfq %rdx, %rsi
; SSE-NEXT: rep bsfq %rax, %rdi
; SSE-NEXT: addl $64, %edi
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %esi, %edi
; SSE-NEXT: movq %xmm1, %rdx
; SSE-NEXT: rep bsfq %rdx, %rsi
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq %rcx, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %edi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_cttz_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rax
; AVX2-NEXT: vmovq %xmm1, %rcx
; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
; AVX2-NEXT: vmovq %xmm0, %rsi
; AVX2-NEXT: tzcntq %rsi, %rdi
; AVX2-NEXT: tzcntq %rdx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %edi, %r8d
; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: tzcntq %rcx, %rdi
; AVX2-NEXT: tzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rdx, %rsi
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_cttz_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256]
; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; AVX512F-NEXT: vpaddq %ymm2, %ymm0, %ymm2
; AVX512F-NEXT: vpandn %ymm2, %ymm0, %ymm2
; AVX512F-NEXT: vplzcntq %zmm2, %zmm2
; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [64,128,192,256]
; AVX512F-NEXT: vpsubq %ymm2, %ymm3, %ymm2
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_cttz_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256]
; AVX512POPCNT-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; AVX512POPCNT-NEXT: vpaddq %ymm2, %ymm0, %ymm2
; AVX512POPCNT-NEXT: vpandn %ymm2, %ymm0, %ymm2
; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_cttz_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256]
; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_cttz_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <8 x i32> %v0 to i256
%cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 0)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @test_cttz_i512(i512 %a0) nounwind {
; SSE-LABEL: test_cttz_i512:
; SSE: # %bb.0:
; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
; SSE-NEXT: rep bsfq %rdi, %rax
; SSE-NEXT: rep bsfq %rsi, %r11
; SSE-NEXT: addl $64, %r11d
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %eax, %r11d
; SSE-NEXT: rep bsfq %rdx, %rax
; SSE-NEXT: rep bsfq %rcx, %r10
; SSE-NEXT: addl $64, %r10d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %r10d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; SSE-NEXT: subl $-128, %r10d
; SSE-NEXT: movq %rdi, %rax
; SSE-NEXT: orq %rsi, %rax
; SSE-NEXT: cmovnel %r11d, %r10d
; SSE-NEXT: rep bsfq %r8, %rax
; SSE-NEXT: rep bsfq %r9, %r11
; SSE-NEXT: addl $64, %r11d
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %eax, %r11d
; SSE-NEXT: rep bsfq %rbx, %r14
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rbx, %rbx
; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: orq %rcx, %rsi
; SSE-NEXT: orq %rdx, %rdi
; SSE-NEXT: orq %rsi, %rdi
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r14
; SSE-NEXT: retq
;
; AVX2-LABEL: test_cttz_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: tzcntq %rdi, %rax
; AVX2-NEXT: tzcntq %rsi, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %eax, %r11d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
; AVX2-NEXT: tzcntq %rcx, %r10
; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %eax, %r10d
; AVX2-NEXT: subl $-128, %r10d
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: orq %rsi, %rax
; AVX2-NEXT: cmovnel %r11d, %r10d
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %r8, %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %r9, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: tzcntq %r11, %r14
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: orq %rcx, %rsi
; AVX2-NEXT: orq %rdx, %rdi
; AVX2-NEXT: orq %rsi, %rdi
; AVX2-NEXT: cmovnel %r10d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_cttz_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovq %rcx, %xmm0
; AVX512F-NEXT: vmovq %rdx, %xmm1
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512F-NEXT: vmovq %rsi, %xmm1
; AVX512F-NEXT: vmovq %rdi, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512F-NEXT: vmovq %r9, %xmm1
; AVX512F-NEXT: vmovq %r8, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_cttz_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovq %rcx, %xmm0
; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512POPCNT-NEXT: vmovq %rdi, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512POPCNT-NEXT: vmovq %r9, %xmm1
; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_cttz_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovq %rcx, %xmm0
; AVX512VL-NEXT: vmovq %rdx, %xmm1
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VL-NEXT: vmovq %rsi, %xmm1
; AVX512VL-NEXT: vmovq %rdi, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VL-NEXT: vmovq %r9, %xmm1
; AVX512VL-NEXT: vmovq %r8, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_cttz_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm0
; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @load_cttz_i512(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_i512:
; SSE: # %bb.0:
; SSE-NEXT: movdqa (%rdi), %xmm0
; SSE-NEXT: movq 48(%rdi), %rdx
; SSE-NEXT: movq 40(%rdi), %rcx
; SSE-NEXT: movq (%rdi), %rax
; SSE-NEXT: movq 8(%rdi), %r8
; SSE-NEXT: rep bsfq %rax, %rsi
; SSE-NEXT: rep bsfq %r8, %r9
; SSE-NEXT: addl $64, %r9d
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %esi, %r9d
; SSE-NEXT: movq 16(%rdi), %r10
; SSE-NEXT: rep bsfq %r10, %r11
; SSE-NEXT: rep bsfq 24(%rdi), %rsi
; SSE-NEXT: addl $64, %esi
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r8, %rax
; SSE-NEXT: cmovnel %r9d, %esi
; SSE-NEXT: movq 32(%rdi), %r8
; SSE-NEXT: rep bsfq %r8, %rax
; SSE-NEXT: rep bsfq %rcx, %r9
; SSE-NEXT: addl $64, %r9d
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %eax, %r9d
; SSE-NEXT: rep bsfq %rdx, %r10
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq 56(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rcx, %r8
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por 16(%rdi), %xmm0
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: movq 40(%rdi), %rcx
; AVX2-NEXT: movq 32(%rdi), %rdx
; AVX2-NEXT: movq 16(%rdi), %rax
; AVX2-NEXT: movq (%rdi), %r8
; AVX2-NEXT: movq 8(%rdi), %r9
; AVX2-NEXT: tzcntq %r8, %rsi
; AVX2-NEXT: tzcntq %r9, %r10
; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %esi, %r10d
; AVX2-NEXT: tzcntq %rax, %r11
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: tzcntq 24(%rdi), %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rax, %rax
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %r10d, %esi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: movq 48(%rdi), %r9
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: tzcntq %r9, %r10
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r10d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: vmovdqa (%rdi), %xmm0
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
; AVX2-NEXT: vptest %xmm0, %xmm0
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i512, ptr %p0
%cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @vector_cttz_i512(<16 x i32> %v0) nounwind {
; SSE-LABEL: vector_cttz_i512:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm3, %rdx
; SSE-NEXT: movq %xmm3, %rcx
; SSE-NEXT: pextrq $1, %xmm2, %rax
; SSE-NEXT: pextrq $1, %xmm1, %rsi
; SSE-NEXT: movq %xmm1, %rdi
; SSE-NEXT: pextrq $1, %xmm0, %r8
; SSE-NEXT: movq %xmm0, %r9
; SSE-NEXT: rep bsfq %r9, %r10
; SSE-NEXT: rep bsfq %r8, %r8
; SSE-NEXT: addl $64, %r8d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r10d, %r8d
; SSE-NEXT: rep bsfq %rdi, %r9
; SSE-NEXT: rep bsfq %rsi, %rsi
; SSE-NEXT: addl $64, %esi
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %r9d, %esi
; SSE-NEXT: movq %xmm2, %rdi
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %r8d, %esi
; SSE-NEXT: rep bsfq %rdi, %r8
; SSE-NEXT: rep bsfq %rax, %r9
; SSE-NEXT: addl $64, %r9d
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %r8d, %r9d
; SSE-NEXT: rep bsfq %rcx, %rdi
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq %rdx, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %edi, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por %xmm1, %xmm0
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_cttz_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rax
; AVX2-NEXT: vpextrq $1, %xmm1, %rcx
; AVX2-NEXT: vmovq %xmm1, %rdx
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rsi
; AVX2-NEXT: vpextrq $1, %xmm0, %rdi
; AVX2-NEXT: vmovq %xmm1, %r8
; AVX2-NEXT: vmovq %xmm0, %r9
; AVX2-NEXT: tzcntq %r9, %r10
; AVX2-NEXT: tzcntq %rdi, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r10d, %r11d
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: tzcntq %r8, %r10
; AVX2-NEXT: tzcntq %rsi, %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %r10d, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %rdi, %r9
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: tzcntq %rdx, %rdi
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %edi, %r8d
; AVX2-NEXT: vmovq %xmm2, %rdi
; AVX2-NEXT: xorl %r9d, %r9d
; AVX2-NEXT: tzcntq %rdi, %r9
; AVX2-NEXT: tzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_cttz_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_cttz_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_cttz_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_cttz_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <16 x i32> %v0 to i512
%cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 0)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @test_cttz_i1024(i1024 %a0) nounwind {
; SSE-LABEL: test_cttz_i1024:
; SSE: # %bb.0:
; SSE-NEXT: pushq %rbp
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %r13
; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq %r9, %r10
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
; SSE-NEXT: rep bsfq %rdi, %r11
; SSE-NEXT: rep bsfq %rsi, %rbx
; SSE-NEXT: addl $64, %ebx
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %r11d, %ebx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: rep bsfq %rdx, %r12
; SSE-NEXT: rep bsfq %rcx, %r15
; SSE-NEXT: addl $64, %r15d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %r12d, %r15d
; SSE-NEXT: subl $-128, %r15d
; SSE-NEXT: movq %rdi, %r12
; SSE-NEXT: orq %rsi, %r12
; SSE-NEXT: cmovnel %ebx, %r15d
; SSE-NEXT: rep bsfq %r8, %rbx
; SSE-NEXT: rep bsfq %r10, %r13
; SSE-NEXT: addl $64, %r13d
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %ebx, %r13d
; SSE-NEXT: rep bsfq %r9, %r12
; SSE-NEXT: rep bsfq %r11, %rbx
; SSE-NEXT: addl $64, %ebx
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r12d, %ebx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
; SSE-NEXT: subl $-128, %ebx
; SSE-NEXT: movq %r8, %rbp
; SSE-NEXT: orq %r10, %rbp
; SSE-NEXT: cmovnel %r13d, %ebx
; SSE-NEXT: addl $256, %ebx # imm = 0x100
; SSE-NEXT: movq %rsi, %r13
; SSE-NEXT: orq %rcx, %r13
; SSE-NEXT: movq %rdi, %rbp
; SSE-NEXT: orq %rdx, %rbp
; SSE-NEXT: orq %r13, %rbp
; SSE-NEXT: cmovnel %r15d, %ebx
; SSE-NEXT: rep bsfq %rax, %r15
; SSE-NEXT: rep bsfq %r12, %r13
; SSE-NEXT: addl $64, %r13d
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %r15d, %r13d
; SSE-NEXT: rep bsfq %r14, %rbp
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r15
; SSE-NEXT: addl $64, %r15d
; SSE-NEXT: testq %r14, %r14
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
; SSE-NEXT: cmovnel %ebp, %r15d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
; SSE-NEXT: subl $-128, %r15d
; SSE-NEXT: orq %r12, %rax
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
; SSE-NEXT: cmovnel %r13d, %r15d
; SSE-NEXT: rep bsfq %r14, %rax
; SSE-NEXT: rep bsfq %r12, %r13
; SSE-NEXT: addl $64, %r13d
; SSE-NEXT: testq %r14, %r14
; SSE-NEXT: cmovnel %eax, %r13d
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: rep bsfq %r9, %rbp
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %ebp, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r12, %r14
; SSE-NEXT: cmovnel %r13d, %eax
; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: orq %r11, %rcx
; SSE-NEXT: orq %r10, %rsi
; SSE-NEXT: orq %rcx, %rsi
; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; SSE-NEXT: orq %r8, %rdi
; SSE-NEXT: orq %rdx, %rdi
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: orq %rsi, %rdi
; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r12
; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: popq %rbp
; SSE-NEXT: retq
;
; AVX2-LABEL: test_cttz_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %r9, %r10
; AVX2-NEXT: movq %r8, %r9
; AVX2-NEXT: movq %rcx, %r8
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %rdi, %rbx
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: tzcntq %rsi, %r12
; AVX2-NEXT: addl $64, %r12d
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %ebx, %r12d
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %rdx, %rbx
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: tzcntq %r8, %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %ebx, %r15d
; AVX2-NEXT: subl $-128, %r15d
; AVX2-NEXT: movq %rdi, %rbx
; AVX2-NEXT: orq %rsi, %rbx
; AVX2-NEXT: cmovnel %r12d, %r15d
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %r9, %rbx
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: tzcntq %r10, %r12
; AVX2-NEXT: addl $64, %r12d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %ebx, %r12d
; AVX2-NEXT: xorl %r13d, %r13d
; AVX2-NEXT: tzcntq %r11, %r13
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %rcx, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %r13d, %ebx
; AVX2-NEXT: subl $-128, %ebx
; AVX2-NEXT: movq %r9, %r13
; AVX2-NEXT: orq %r10, %r13
; AVX2-NEXT: cmovnel %r12d, %ebx
; AVX2-NEXT: addl $256, %ebx # imm = 0x100
; AVX2-NEXT: movq %rsi, %r12
; AVX2-NEXT: orq %r8, %r12
; AVX2-NEXT: movq %rdi, %r13
; AVX2-NEXT: orq %rdx, %r13
; AVX2-NEXT: orq %r12, %r13
; AVX2-NEXT: cmovnel %r15d, %ebx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: tzcntq %rax, %r15
; AVX2-NEXT: xorl %r13d, %r13d
; AVX2-NEXT: tzcntq %r12, %r13
; AVX2-NEXT: addl $64, %r13d
; AVX2-NEXT: testq %rax, %rax
; AVX2-NEXT: cmovnel %r15d, %r13d
; AVX2-NEXT: xorl %ebp, %ebp
; AVX2-NEXT: tzcntq %r14, %rbp
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %r14, %r14
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
; AVX2-NEXT: cmovnel %ebp, %r15d
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
; AVX2-NEXT: subl $-128, %r15d
; AVX2-NEXT: orq %r12, %rax
; AVX2-NEXT: cmovnel %r13d, %r15d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rbp, %rax
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: tzcntq %r14, %r12
; AVX2-NEXT: addl $64, %r12d
; AVX2-NEXT: testq %rbp, %rbp
; AVX2-NEXT: cmovnel %eax, %r12d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: tzcntq %r13, %rcx
; AVX2-NEXT: testq %r13, %r13
; AVX2-NEXT: cmovnel %ecx, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r14, %rbp
; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX2-NEXT: vptest %xmm0, %xmm0
; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r8
; AVX2-NEXT: orq %r10, %rsi
; AVX2-NEXT: orq %r8, %rsi
; AVX2-NEXT: orq %r11, %rdx
; AVX2-NEXT: orq %r9, %rdi
; AVX2-NEXT: orq %rdx, %rdi
; AVX2-NEXT: addl $512, %eax # imm = 0x200
; AVX2-NEXT: orq %rsi, %rdi
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_cttz_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovq %rcx, %xmm0
; AVX512F-NEXT: vmovq %rdx, %xmm1
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512F-NEXT: vmovq %rsi, %xmm1
; AVX512F-NEXT: vmovq %rdi, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512F-NEXT: vmovq %r9, %xmm1
; AVX512F-NEXT: vmovq %r8, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512F-NEXT: vplzcntq %zmm3, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm3
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %r10d
; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm0
; AVX512F-NEXT: vpandnq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
; AVX512F-NEXT: vpsubq %zmm0, %zmm4, %zmm0
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
; AVX512F-NEXT: orq %r9, %rsi
; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rcx
; AVX512F-NEXT: orq %rsi, %rcx
; AVX512F-NEXT: orq %r8, %rdi
; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; AVX512F-NEXT: orq %rdi, %rdx
; AVX512F-NEXT: orq %rcx, %rdx
; AVX512F-NEXT: cmovnel %r10d, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_cttz_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovq %rcx, %xmm0
; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512POPCNT-NEXT: vmovq %rdi, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512POPCNT-NEXT: vmovq %r9, %xmm1
; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %r10d
; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vpandnq %zmm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm0, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512POPCNT-NEXT: orq %r9, %rsi
; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rcx
; AVX512POPCNT-NEXT: orq %rsi, %rcx
; AVX512POPCNT-NEXT: orq %r8, %rdi
; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; AVX512POPCNT-NEXT: orq %rdi, %rdx
; AVX512POPCNT-NEXT: orq %rcx, %rdx
; AVX512POPCNT-NEXT: cmovnel %r10d, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_cttz_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovq %rcx, %xmm0
; AVX512VL-NEXT: vmovq %rdx, %xmm1
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VL-NEXT: vmovq %rsi, %xmm1
; AVX512VL-NEXT: vmovq %rdi, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vmovq %r9, %xmm2
; AVX512VL-NEXT: vmovq %r8, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VL-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm2
; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %r10d
; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm0
; AVX512VL-NEXT: vpandnq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpsubq %zmm0, %zmm4, %zmm0
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512VL-NEXT: vmovd %xmm1, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
; AVX512VL-NEXT: orq %r9, %rsi
; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rcx
; AVX512VL-NEXT: orq %rsi, %rcx
; AVX512VL-NEXT: orq %r8, %rdi
; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; AVX512VL-NEXT: orq %rdi, %rdx
; AVX512VL-NEXT: orq %rcx, %rdx
; AVX512VL-NEXT: cmovnel %r10d, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_cttz_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm0
; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm2
; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm2
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %r10d
; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vpandnq %zmm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512VLPOPCNT-NEXT: orq %r9, %rsi
; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rcx
; AVX512VLPOPCNT-NEXT: orq %rsi, %rcx
; AVX512VLPOPCNT-NEXT: orq %r8, %rdi
; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; AVX512VLPOPCNT-NEXT: orq %rdi, %rdx
; AVX512VLPOPCNT-NEXT: orq %rcx, %rdx
; AVX512VLPOPCNT-NEXT: cmovnel %r10d, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.cttz.i1024(i1024 %a0, i1 0)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
define i32 @load_cttz_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_i1024:
; SSE: # %bb.0:
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq 112(%rdi), %rcx
; SSE-NEXT: movq 48(%rdi), %rsi
; SSE-NEXT: movq (%rdi), %rdx
; SSE-NEXT: movq 8(%rdi), %r8
; SSE-NEXT: rep bsfq %rdx, %rax
; SSE-NEXT: rep bsfq %r8, %r9
; SSE-NEXT: addl $64, %r9d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %r9d
; SSE-NEXT: movq 16(%rdi), %r10
; SSE-NEXT: rep bsfq %r10, %r11
; SSE-NEXT: rep bsfq 24(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: movq 40(%rdi), %r10
; SSE-NEXT: movq 32(%rdi), %r11
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r8, %rdx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: rep bsfq %r11, %rdx
; SSE-NEXT: rep bsfq %r10, %rbx
; SSE-NEXT: addl $64, %ebx
; SSE-NEXT: testq %r11, %r11
; SSE-NEXT: cmovnel %edx, %ebx
; SSE-NEXT: rep bsfq %rsi, %r9
; SSE-NEXT: rep bsfq 56(%rdi), %rdx
; SSE-NEXT: addl $64, %edx
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: movq 72(%rdi), %r8
; SSE-NEXT: cmovnel %r9d, %edx
; SSE-NEXT: movq 64(%rdi), %r9
; SSE-NEXT: movdqa 16(%rdi), %xmm0
; SSE-NEXT: movdqa (%rdi), %xmm1
; SSE-NEXT: subl $-128, %edx
; SSE-NEXT: orq %r10, %r11
; SSE-NEXT: cmovnel %ebx, %edx
; SSE-NEXT: addl $256, %edx # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %eax, %edx
; SSE-NEXT: rep bsfq %r9, %rax
; SSE-NEXT: rep bsfq %r8, %r11
; SSE-NEXT: addl $64, %r11d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %eax, %r11d
; SSE-NEXT: movq 80(%rdi), %rax
; SSE-NEXT: rep bsfq %rax, %r10
; SSE-NEXT: rep bsfq 88(%rdi), %rsi
; SSE-NEXT: addl $64, %esi
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %r10d, %esi
; SSE-NEXT: movq 104(%rdi), %r10
; SSE-NEXT: movq 96(%rdi), %rbx
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r8, %r9
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: rep bsfq %rbx, %rax
; SSE-NEXT: rep bsfq %r10, %r8
; SSE-NEXT: addl $64, %r8d
; SSE-NEXT: testq %rbx, %rbx
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: rep bsfq %rcx, %r9
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq 120(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r10, %rbx
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: movdqa 64(%rdi), %xmm2
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por 80(%rdi), %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: por 48(%rdi), %xmm0
; SSE-NEXT: por 32(%rdi), %xmm1
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq 104(%rdi), %rcx
; AVX2-NEXT: movq 48(%rdi), %rsi
; AVX2-NEXT: movq 16(%rdi), %rdx
; AVX2-NEXT: movq (%rdi), %r8
; AVX2-NEXT: movq 8(%rdi), %r9
; AVX2-NEXT: tzcntq %r8, %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %r9, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: tzcntq %rdx, %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 24(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: movq 40(%rdi), %r10
; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: movq 32(%rdi), %r11
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: tzcntq %r11, %rdx
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %r10, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %edx, %ebx
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: tzcntq 56(%rdi), %rdx
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq %rsi, %r8
; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: movq 80(%rdi), %r9
; AVX2-NEXT: cmovnel %r8d, %edx
; AVX2-NEXT: movq 72(%rdi), %r8
; AVX2-NEXT: subl $-128, %edx
; AVX2-NEXT: orq %r10, %r11
; AVX2-NEXT: movq 64(%rdi), %r10
; AVX2-NEXT: cmovnel %ebx, %edx
; AVX2-NEXT: vmovdqu (%rdi), %ymm0
; AVX2-NEXT: addl $256, %edx # imm = 0x100
; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %eax, %edx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %r10, %rax
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: tzcntq %r8, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovnel %eax, %r11d
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: tzcntq 88(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %r9, %rax
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: movq 96(%rdi), %r9
; AVX2-NEXT: cmovnel %eax, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r8, %r10
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %r9, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: movq 112(%rdi), %r10
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: tzcntq %r10, %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 120(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %r9
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: vmovdqa 64(%rdi), %xmm1
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 80(%rdi), %xmm1, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX2-NEXT: addl $512, %eax # imm = 0x200
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512F-NEXT: vplzcntq %zmm3, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm3
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm3, %ecx
; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512F-NEXT: vpandnq %zmm2, %zmm1, %zmm2
; AVX512F-NEXT: vplzcntq %zmm2, %zmm2
; AVX512F-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
; AVX512F-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX512F-NEXT: vptest %ymm0, %ymm0
; AVX512F-NEXT: cmovnel %ecx, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm3, %ecx
; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512POPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm3, %ecx
; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512VL-NEXT: vpandnq %zmm2, %zmm1, %zmm2
; AVX512VL-NEXT: vplzcntq %zmm2, %zmm2
; AVX512VL-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512VL-NEXT: vmovd %xmm1, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
; AVX512VL-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX512VL-NEXT: vptest %ymm0, %ymm0
; AVX512VL-NEXT: cmovnel %ecx, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm3, %ecx
; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512VLPOPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
%cnt = call i1024 @llvm.cttz.i1024(i1024 %a0, i1 0)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
;
; CTTZ_ZERO_UNDEF
;
define i32 @test_cttz_undef_i128(i128 %a0) nounwind {
; SSE-LABEL: test_cttz_undef_i128:
; SSE: # %bb.0:
; SSE-NEXT: rep bsfq %rdi, %rcx
; SSE-NEXT: rep bsfq %rsi, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %ecx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: test_cttz_undef_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: tzcntq %rdi, %rcx
; AVX2-NEXT: tzcntq %rsi, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %ecx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_cttz_undef_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: tzcntq %rdi, %rcx
; AVX512-NEXT: tzcntq %rsi, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rdi, %rdi
; AVX512-NEXT: cmovnel %ecx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 -1)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @load_cttz_undef_i128(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_undef_i128:
; SSE: # %bb.0:
; SSE-NEXT: movq (%rdi), %rcx
; SSE-NEXT: rep bsfq %rcx, %rdx
; SSE-NEXT: rep bsfq 8(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_undef_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: movq (%rdi), %rcx
; AVX2-NEXT: tzcntq %rcx, %rdx
; AVX2-NEXT: tzcntq 8(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: load_cttz_undef_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: movq (%rdi), %rcx
; AVX512-NEXT: tzcntq %rcx, %rdx
; AVX512-NEXT: tzcntq 8(%rdi), %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rcx, %rcx
; AVX512-NEXT: cmovnel %edx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%a0 = load i128, ptr %p0
%cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 -1)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @vector_cttz_undef_i128(<4 x i32> %v0) nounwind {
; SSE-LABEL: vector_cttz_undef_i128:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm0, %rax
; SSE-NEXT: movq %xmm0, %rcx
; SSE-NEXT: rep bsfq %rcx, %rdx
; SSE-NEXT: rep bsfq %rax, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_cttz_undef_i128:
; AVX2: # %bb.0:
; AVX2-NEXT: vpextrq $1, %xmm0, %rax
; AVX2-NEXT: vmovq %xmm0, %rcx
; AVX2-NEXT: tzcntq %rcx, %rdx
; AVX2-NEXT: tzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: vector_cttz_undef_i128:
; AVX512: # %bb.0:
; AVX512-NEXT: vpextrq $1, %xmm0, %rax
; AVX512-NEXT: vmovq %xmm0, %rcx
; AVX512-NEXT: tzcntq %rcx, %rdx
; AVX512-NEXT: tzcntq %rax, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rcx, %rcx
; AVX512-NEXT: cmovnel %edx, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%a0 = bitcast <4 x i32> %v0 to i128
%cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 -1)
%res = trunc i128 %cnt to i32
ret i32 %res
}
define i32 @test_cttz_undef_i256(i256 %a0) nounwind {
; SSE-LABEL: test_cttz_undef_i256:
; SSE: # %bb.0:
; SSE-NEXT: rep bsfq %rdi, %rax
; SSE-NEXT: rep bsfq %rsi, %r8
; SSE-NEXT: addl $64, %r8d
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: rep bsfq %rdx, %r9
; SSE-NEXT: rep bsfq %rcx, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rsi, %rdi
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: test_cttz_undef_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: tzcntq %rdi, %rax
; AVX2-NEXT: tzcntq %rsi, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: tzcntq %rdx, %r9
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rcx, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rsi, %rdi
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_cttz_undef_i256:
; AVX512: # %bb.0:
; AVX512-NEXT: tzcntq %rdi, %rax
; AVX512-NEXT: tzcntq %rsi, %r8
; AVX512-NEXT: addl $64, %r8d
; AVX512-NEXT: testq %rdi, %rdi
; AVX512-NEXT: cmovnel %eax, %r8d
; AVX512-NEXT: tzcntq %rdx, %r9
; AVX512-NEXT: tzcntq %rcx, %rax
; AVX512-NEXT: addl $64, %eax
; AVX512-NEXT: testq %rdx, %rdx
; AVX512-NEXT: cmovnel %r9d, %eax
; AVX512-NEXT: subl $-128, %eax
; AVX512-NEXT: orq %rsi, %rdi
; AVX512-NEXT: cmovnel %r8d, %eax
; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 -1)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @load_cttz_undef_i256(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_undef_i256:
; SSE: # %bb.0:
; SSE-NEXT: movq 16(%rdi), %rcx
; SSE-NEXT: movq (%rdi), %rdx
; SSE-NEXT: movq 8(%rdi), %rsi
; SSE-NEXT: rep bsfq %rdx, %rax
; SSE-NEXT: rep bsfq %rsi, %r8
; SSE-NEXT: addl $64, %r8d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: rep bsfq %rcx, %r9
; SSE-NEXT: rep bsfq 24(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rsi, %rdx
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_undef_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: movq (%rdi), %rcx
; AVX2-NEXT: movq 8(%rdi), %rdx
; AVX2-NEXT: tzcntq %rcx, %rax
; AVX2-NEXT: tzcntq %rdx, %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %eax, %esi
; AVX2-NEXT: movq 16(%rdi), %r8
; AVX2-NEXT: tzcntq %r8, %r9
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 24(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rdx, %rcx
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_undef_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu (%rdi), %ymm0
; AVX512F-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512F-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256]
; AVX512F-NEXT: vpsubq %ymm1, %ymm2, %ymm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_undef_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu (%rdi), %ymm0
; AVX512POPCNT-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512POPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_undef_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu (%rdi), %ymm0
; AVX512VL-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256]
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_undef_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu (%rdi), %ymm0
; AVX512VLPOPCNT-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i256, ptr %p0
%cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 -1)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @vector_cttz_undef_i256(<8 x i32> %v0) nounwind {
; SSE-LABEL: vector_cttz_undef_i256:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm1, %rax
; SSE-NEXT: movq %xmm1, %rcx
; SSE-NEXT: pextrq $1, %xmm0, %rdx
; SSE-NEXT: movq %xmm0, %rsi
; SSE-NEXT: rep bsfq %rsi, %rdi
; SSE-NEXT: rep bsfq %rdx, %rdx
; SSE-NEXT: addl $64, %edx
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %edi, %edx
; SSE-NEXT: rep bsfq %rcx, %rsi
; SSE-NEXT: rep bsfq %rax, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_cttz_undef_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rax
; AVX2-NEXT: vmovq %xmm1, %rcx
; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
; AVX2-NEXT: vmovq %xmm0, %rsi
; AVX2-NEXT: tzcntq %rsi, %rdi
; AVX2-NEXT: tzcntq %rdx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovnel %edi, %r8d
; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: tzcntq %rcx, %rdi
; AVX2-NEXT: tzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rdx, %rsi
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_cttz_undef_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512F-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512F-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256]
; AVX512F-NEXT: vpsubq %ymm1, %ymm2, %ymm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_cttz_undef_i256:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512POPCNT-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512POPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_cttz_undef_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256]
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_cttz_undef_i256:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <8 x i32> %v0 to i256
%cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 -1)
%res = trunc i256 %cnt to i32
ret i32 %res
}
define i32 @test_cttz_undef_i512(i512 %a0) nounwind {
; SSE-LABEL: test_cttz_undef_i512:
; SSE: # %bb.0:
; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
; SSE-NEXT: rep bsfq %rdi, %rax
; SSE-NEXT: rep bsfq %rsi, %r11
; SSE-NEXT: addl $64, %r11d
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %eax, %r11d
; SSE-NEXT: rep bsfq %rdx, %rax
; SSE-NEXT: rep bsfq %rcx, %r10
; SSE-NEXT: addl $64, %r10d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %r10d
; SSE-NEXT: subl $-128, %r10d
; SSE-NEXT: movq %rdi, %rax
; SSE-NEXT: orq %rsi, %rax
; SSE-NEXT: cmovnel %r11d, %r10d
; SSE-NEXT: rep bsfq %r8, %rax
; SSE-NEXT: rep bsfq %r9, %r11
; SSE-NEXT: addl $64, %r11d
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %eax, %r11d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; SSE-NEXT: rep bsfq %rbx, %r14
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rbx, %rbx
; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: orq %rcx, %rsi
; SSE-NEXT: orq %rdx, %rdi
; SSE-NEXT: orq %rsi, %rdi
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r14
; SSE-NEXT: retq
;
; AVX2-LABEL: test_cttz_undef_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: tzcntq %rdi, %rax
; AVX2-NEXT: tzcntq %rsi, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %eax, %r11d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
; AVX2-NEXT: tzcntq %rcx, %r10
; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %eax, %r10d
; AVX2-NEXT: subl $-128, %r10d
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: orq %rsi, %rax
; AVX2-NEXT: cmovnel %r11d, %r10d
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %r8, %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %r9, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: tzcntq %r11, %r14
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: orq %rcx, %rsi
; AVX2-NEXT: orq %rdx, %rdi
; AVX2-NEXT: orq %rsi, %rdi
; AVX2-NEXT: cmovnel %r10d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_cttz_undef_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovq %rcx, %xmm0
; AVX512F-NEXT: vmovq %rdx, %xmm1
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512F-NEXT: vmovq %rsi, %xmm1
; AVX512F-NEXT: vmovq %rdi, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512F-NEXT: vmovq %r9, %xmm1
; AVX512F-NEXT: vmovq %r8, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_cttz_undef_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovq %rcx, %xmm0
; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512POPCNT-NEXT: vmovq %rdi, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vmovq %r9, %xmm2
; AVX512POPCNT-NEXT: vmovq %r8, %xmm3
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512POPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm2
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_cttz_undef_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovq %rcx, %xmm0
; AVX512VL-NEXT: vmovq %rdx, %xmm1
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VL-NEXT: vmovq %rsi, %xmm1
; AVX512VL-NEXT: vmovq %rdi, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VL-NEXT: vmovq %r9, %xmm1
; AVX512VL-NEXT: vmovq %r8, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_cttz_undef_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm0
; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm2
; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm2
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 -1)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @load_cttz_undef_i512(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_undef_i512:
; SSE: # %bb.0:
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movdqa (%rdi), %xmm0
; SSE-NEXT: movq 48(%rdi), %rsi
; SSE-NEXT: movq 40(%rdi), %rcx
; SSE-NEXT: movq 32(%rdi), %rdx
; SSE-NEXT: movq 16(%rdi), %rax
; SSE-NEXT: movq (%rdi), %r9
; SSE-NEXT: movq 8(%rdi), %r10
; SSE-NEXT: rep bsfq %r9, %r8
; SSE-NEXT: rep bsfq %r10, %r11
; SSE-NEXT: addl $64, %r11d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %r8d, %r11d
; SSE-NEXT: rep bsfq %rax, %rbx
; SSE-NEXT: rep bsfq 24(%rdi), %r8
; SSE-NEXT: addl $64, %r8d
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %ebx, %r8d
; SSE-NEXT: subl $-128, %r8d
; SSE-NEXT: orq %r10, %r9
; SSE-NEXT: cmovnel %r11d, %r8d
; SSE-NEXT: rep bsfq %rdx, %rax
; SSE-NEXT: rep bsfq %rcx, %r9
; SSE-NEXT: addl $64, %r9d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %eax, %r9d
; SSE-NEXT: rep bsfq %rsi, %r10
; SSE-NEXT: rep bsfq 56(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rcx, %rdx
; SSE-NEXT: cmovnel %r9d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por 16(%rdi), %xmm0
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_undef_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: movq 40(%rdi), %rcx
; AVX2-NEXT: movq 32(%rdi), %rdx
; AVX2-NEXT: movq 16(%rdi), %rax
; AVX2-NEXT: movq (%rdi), %r8
; AVX2-NEXT: movq 8(%rdi), %r9
; AVX2-NEXT: tzcntq %r8, %rsi
; AVX2-NEXT: tzcntq %r9, %r10
; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %esi, %r10d
; AVX2-NEXT: tzcntq %rax, %r11
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: tzcntq 24(%rdi), %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rax, %rax
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %r10d, %esi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: movq 48(%rdi), %r9
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: tzcntq %r9, %r10
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r10d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: vmovdqa (%rdi), %xmm0
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
; AVX2-NEXT: vptest %xmm0, %xmm0
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_undef_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_undef_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_undef_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_undef_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i512, ptr %p0
%cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 -1)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @vector_cttz_undef_i512(<16 x i32> %v0) nounwind {
; SSE-LABEL: vector_cttz_undef_i512:
; SSE: # %bb.0:
; SSE-NEXT: pextrq $1, %xmm3, %rax
; SSE-NEXT: pextrq $1, %xmm2, %rdx
; SSE-NEXT: pextrq $1, %xmm1, %rcx
; SSE-NEXT: movq %xmm1, %rsi
; SSE-NEXT: pextrq $1, %xmm0, %rdi
; SSE-NEXT: movq %xmm0, %r8
; SSE-NEXT: rep bsfq %r8, %r9
; SSE-NEXT: rep bsfq %rdi, %rdi
; SSE-NEXT: addl $64, %edi
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %r9d, %edi
; SSE-NEXT: rep bsfq %rsi, %r8
; SSE-NEXT: rep bsfq %rcx, %rcx
; SSE-NEXT: addl $64, %ecx
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %r8d, %ecx
; SSE-NEXT: movq %xmm2, %rsi
; SSE-NEXT: subl $-128, %ecx
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %edi, %ecx
; SSE-NEXT: rep bsfq %rsi, %rdi
; SSE-NEXT: rep bsfq %rdx, %rdx
; SSE-NEXT: addl $64, %edx
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %edi, %edx
; SSE-NEXT: movq %xmm3, %rsi
; SSE-NEXT: rep bsfq %rsi, %rdi
; SSE-NEXT: rep bsfq %rax, %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %rsi, %rsi
; SSE-NEXT: cmovnel %edi, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por %xmm1, %xmm0
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %ecx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: retq
;
; AVX2-LABEL: vector_cttz_undef_i512:
; AVX2: # %bb.0:
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rax
; AVX2-NEXT: vpextrq $1, %xmm1, %rcx
; AVX2-NEXT: vmovq %xmm1, %rdx
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rsi
; AVX2-NEXT: vpextrq $1, %xmm0, %rdi
; AVX2-NEXT: vmovq %xmm1, %r8
; AVX2-NEXT: vmovq %xmm0, %r9
; AVX2-NEXT: tzcntq %r9, %r10
; AVX2-NEXT: tzcntq %rdi, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %r10d, %r11d
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: tzcntq %r8, %r10
; AVX2-NEXT: tzcntq %rsi, %rsi
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %r10d, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %rdi, %r9
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: tzcntq %rdx, %rdi
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %edi, %r8d
; AVX2-NEXT: vmovq %xmm2, %rdi
; AVX2-NEXT: xorl %r9d, %r9d
; AVX2-NEXT: tzcntq %rdi, %r9
; AVX2-NEXT: tzcntq %rax, %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %r9d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: vector_cttz_undef_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: vector_cttz_undef_i512:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_cttz_undef_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: vector_cttz_undef_i512:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <16 x i32> %v0 to i512
%cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 -1)
%res = trunc i512 %cnt to i32
ret i32 %res
}
define i32 @test_cttz_undef_i1024(i1024 %a0) nounwind {
; SSE-LABEL: test_cttz_undef_i1024:
; SSE: # %bb.0:
; SSE-NEXT: pushq %rbp
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %r13
; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq %r9, %r10
; SSE-NEXT: movq %r8, %r9
; SSE-NEXT: movq %rcx, %r8
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; SSE-NEXT: rep bsfq %rdi, %r11
; SSE-NEXT: rep bsfq %rsi, %rbx
; SSE-NEXT: addl $64, %ebx
; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %r11d, %ebx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: rep bsfq %rdx, %r15
; SSE-NEXT: rep bsfq %r8, %r14
; SSE-NEXT: addl $64, %r14d
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %r15d, %r14d
; SSE-NEXT: subl $-128, %r14d
; SSE-NEXT: movq %rdi, %r15
; SSE-NEXT: orq %rsi, %r15
; SSE-NEXT: cmovnel %ebx, %r14d
; SSE-NEXT: rep bsfq %r9, %rbx
; SSE-NEXT: rep bsfq %r10, %r12
; SSE-NEXT: addl $64, %r12d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %ebx, %r12d
; SSE-NEXT: rep bsfq %rcx, %r15
; SSE-NEXT: rep bsfq %r11, %rbx
; SSE-NEXT: addl $64, %ebx
; SSE-NEXT: testq %rcx, %rcx
; SSE-NEXT: cmovnel %r15d, %ebx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r15
; SSE-NEXT: subl $-128, %ebx
; SSE-NEXT: movq %r9, %r13
; SSE-NEXT: orq %r10, %r13
; SSE-NEXT: cmovnel %r12d, %ebx
; SSE-NEXT: addl $256, %ebx # imm = 0x100
; SSE-NEXT: movq %rsi, %r12
; SSE-NEXT: orq %r8, %r12
; SSE-NEXT: movq %rdi, %r13
; SSE-NEXT: orq %rdx, %r13
; SSE-NEXT: orq %r12, %r13
; SSE-NEXT: cmovnel %r14d, %ebx
; SSE-NEXT: rep bsfq %rax, %r14
; SSE-NEXT: rep bsfq %r15, %r13
; SSE-NEXT: addl $64, %r13d
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %r14d, %r13d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
; SSE-NEXT: rep bsfq %r12, %rbp
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r14
; SSE-NEXT: addl $64, %r14d
; SSE-NEXT: testq %r12, %r12
; SSE-NEXT: cmovnel %ebp, %r14d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbp
; SSE-NEXT: subl $-128, %r14d
; SSE-NEXT: orq %r15, %rax
; SSE-NEXT: cmovnel %r13d, %r14d
; SSE-NEXT: rep bsfq %r12, %rax
; SSE-NEXT: rep bsfq %rbp, %r15
; SSE-NEXT: addl $64, %r15d
; SSE-NEXT: testq %r12, %r12
; SSE-NEXT: cmovnel %eax, %r15d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r13
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: rep bsfq %r13, %rcx
; SSE-NEXT: testq %r13, %r13
; SSE-NEXT: cmovnel %ecx, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rbp, %r12
; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: orq %r11, %r8
; SSE-NEXT: orq %r10, %rsi
; SSE-NEXT: orq %r8, %rsi
; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; SSE-NEXT: orq %r9, %rdi
; SSE-NEXT: orq %rdx, %rdi
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: orq %rsi, %rdi
; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r12
; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: popq %rbp
; SSE-NEXT: retq
;
; AVX2-LABEL: test_cttz_undef_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %r9, %r10
; AVX2-NEXT: movq %r8, %r9
; AVX2-NEXT: movq %rcx, %r8
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %rdi, %rbx
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: tzcntq %rsi, %r12
; AVX2-NEXT: addl $64, %r12d
; AVX2-NEXT: testq %rdi, %rdi
; AVX2-NEXT: cmovnel %ebx, %r12d
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %rdx, %rbx
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: tzcntq %r8, %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovnel %ebx, %r15d
; AVX2-NEXT: subl $-128, %r15d
; AVX2-NEXT: movq %rdi, %rbx
; AVX2-NEXT: orq %rsi, %rbx
; AVX2-NEXT: cmovnel %r12d, %r15d
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %r9, %rbx
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: tzcntq %r10, %r12
; AVX2-NEXT: addl $64, %r12d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %ebx, %r12d
; AVX2-NEXT: xorl %r13d, %r13d
; AVX2-NEXT: tzcntq %r11, %r13
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %rcx, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %r13d, %ebx
; AVX2-NEXT: subl $-128, %ebx
; AVX2-NEXT: movq %r9, %r13
; AVX2-NEXT: orq %r10, %r13
; AVX2-NEXT: cmovnel %r12d, %ebx
; AVX2-NEXT: addl $256, %ebx # imm = 0x100
; AVX2-NEXT: movq %rsi, %r12
; AVX2-NEXT: orq %r8, %r12
; AVX2-NEXT: movq %rdi, %r13
; AVX2-NEXT: orq %rdx, %r13
; AVX2-NEXT: orq %r12, %r13
; AVX2-NEXT: cmovnel %r15d, %ebx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: tzcntq %rax, %r15
; AVX2-NEXT: xorl %r13d, %r13d
; AVX2-NEXT: tzcntq %r12, %r13
; AVX2-NEXT: addl $64, %r13d
; AVX2-NEXT: testq %rax, %rax
; AVX2-NEXT: cmovnel %r15d, %r13d
; AVX2-NEXT: xorl %ebp, %ebp
; AVX2-NEXT: tzcntq %r14, %rbp
; AVX2-NEXT: xorl %r15d, %r15d
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %r15
; AVX2-NEXT: addl $64, %r15d
; AVX2-NEXT: testq %r14, %r14
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
; AVX2-NEXT: cmovnel %ebp, %r15d
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
; AVX2-NEXT: subl $-128, %r15d
; AVX2-NEXT: orq %r12, %rax
; AVX2-NEXT: cmovnel %r13d, %r15d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rbp, %rax
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: tzcntq %r14, %r12
; AVX2-NEXT: addl $64, %r12d
; AVX2-NEXT: testq %rbp, %rbp
; AVX2-NEXT: cmovnel %eax, %r12d
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: tzcntq %r13, %rcx
; AVX2-NEXT: testq %r13, %r13
; AVX2-NEXT: cmovnel %ecx, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r14, %rbp
; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX2-NEXT: vptest %xmm0, %xmm0
; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r8
; AVX2-NEXT: orq %r10, %rsi
; AVX2-NEXT: orq %r8, %rsi
; AVX2-NEXT: orq %r11, %rdx
; AVX2-NEXT: orq %r9, %rdi
; AVX2-NEXT: orq %rdx, %rdi
; AVX2-NEXT: addl $512, %eax # imm = 0x200
; AVX2-NEXT: orq %rsi, %rdi
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_cttz_undef_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovq %rcx, %xmm0
; AVX512F-NEXT: vmovq %rdx, %xmm1
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512F-NEXT: vmovq %rsi, %xmm1
; AVX512F-NEXT: vmovq %rdi, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512F-NEXT: vmovq %r9, %xmm1
; AVX512F-NEXT: vmovq %r8, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512F-NEXT: vplzcntq %zmm3, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm3
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %r10d
; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm0
; AVX512F-NEXT: vpandnq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
; AVX512F-NEXT: vpsubq %zmm0, %zmm4, %zmm0
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
; AVX512F-NEXT: orq %r9, %rsi
; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rcx
; AVX512F-NEXT: orq %rsi, %rcx
; AVX512F-NEXT: orq %r8, %rdi
; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; AVX512F-NEXT: orq %rdi, %rdx
; AVX512F-NEXT: orq %rcx, %rdx
; AVX512F-NEXT: cmovnel %r10d, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_cttz_undef_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovq %rcx, %xmm0
; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512POPCNT-NEXT: vmovq %rdi, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512POPCNT-NEXT: vmovq %r9, %xmm1
; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %r10d
; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vpandnq %zmm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm0, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512POPCNT-NEXT: orq %r9, %rsi
; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rcx
; AVX512POPCNT-NEXT: orq %rsi, %rcx
; AVX512POPCNT-NEXT: orq %r8, %rdi
; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; AVX512POPCNT-NEXT: orq %rdi, %rdx
; AVX512POPCNT-NEXT: orq %rcx, %rdx
; AVX512POPCNT-NEXT: cmovnel %r10d, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_cttz_undef_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovq %rcx, %xmm0
; AVX512VL-NEXT: vmovq %rdx, %xmm1
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VL-NEXT: vmovq %rsi, %xmm1
; AVX512VL-NEXT: vmovq %rdi, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VL-NEXT: vmovq %r9, %xmm1
; AVX512VL-NEXT: vmovq %r8, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %r10d
; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm0
; AVX512VL-NEXT: vpandnq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpsubq %zmm0, %zmm4, %zmm0
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
; AVX512VL-NEXT: orq %r9, %rsi
; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rcx
; AVX512VL-NEXT: orq %rsi, %rcx
; AVX512VL-NEXT: orq %r8, %rdi
; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; AVX512VL-NEXT: orq %rdi, %rdx
; AVX512VL-NEXT: orq %rcx, %rdx
; AVX512VL-NEXT: cmovnel %r10d, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_cttz_undef_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm0
; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm1
; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VLPOPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %r10d
; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vpandnq %zmm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512VLPOPCNT-NEXT: orq %r9, %rsi
; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rcx
; AVX512VLPOPCNT-NEXT: orq %rsi, %rcx
; AVX512VLPOPCNT-NEXT: orq %r8, %rdi
; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rdx
; AVX512VLPOPCNT-NEXT: orq %rdi, %rdx
; AVX512VLPOPCNT-NEXT: orq %rcx, %rdx
; AVX512VLPOPCNT-NEXT: cmovnel %r10d, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.cttz.i1024(i1024 %a0, i1 -1)
%res = trunc i1024 %cnt to i32
ret i32 %res
}
define i32 @load_cttz_undef_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_undef_i1024:
; SSE: # %bb.0:
; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq 104(%rdi), %rcx
; SSE-NEXT: movq 40(%rdi), %rax
; SSE-NEXT: movq 16(%rdi), %rdx
; SSE-NEXT: movq (%rdi), %r8
; SSE-NEXT: movq 8(%rdi), %r9
; SSE-NEXT: rep bsfq %r8, %rsi
; SSE-NEXT: rep bsfq %r9, %r11
; SSE-NEXT: addl $64, %r11d
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %esi, %r11d
; SSE-NEXT: rep bsfq 24(%rdi), %rsi
; SSE-NEXT: rep bsfq %rdx, %rbx
; SSE-NEXT: addl $64, %esi
; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: movq 32(%rdi), %r10
; SSE-NEXT: cmovnel %ebx, %esi
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: rep bsfq %r10, %rdx
; SSE-NEXT: rep bsfq %rax, %r11
; SSE-NEXT: addl $64, %r11d
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %edx, %r11d
; SSE-NEXT: movq 48(%rdi), %r8
; SSE-NEXT: rep bsfq %r8, %r9
; SSE-NEXT: rep bsfq 56(%rdi), %rdx
; SSE-NEXT: addl $64, %edx
; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %r9d, %edx
; SSE-NEXT: movq 80(%rdi), %r9
; SSE-NEXT: movq 72(%rdi), %r8
; SSE-NEXT: subl $-128, %edx
; SSE-NEXT: orq %rax, %r10
; SSE-NEXT: movq 64(%rdi), %rax
; SSE-NEXT: movdqa 16(%rdi), %xmm0
; SSE-NEXT: cmovnel %r11d, %edx
; SSE-NEXT: movdqa (%rdi), %xmm1
; SSE-NEXT: addl $256, %edx # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %esi, %edx
; SSE-NEXT: rep bsfq %rax, %rsi
; SSE-NEXT: rep bsfq %r8, %r10
; SSE-NEXT: addl $64, %r10d
; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %esi, %r10d
; SSE-NEXT: rep bsfq 88(%rdi), %rsi
; SSE-NEXT: rep bsfq %r9, %r11
; SSE-NEXT: addl $64, %esi
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: movq 96(%rdi), %r9
; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r8, %rax
; SSE-NEXT: cmovnel %r10d, %esi
; SSE-NEXT: rep bsfq %r9, %rax
; SSE-NEXT: rep bsfq %rcx, %r8
; SSE-NEXT: addl $64, %r8d
; SSE-NEXT: testq %r9, %r9
; SSE-NEXT: cmovnel %eax, %r8d
; SSE-NEXT: movq 112(%rdi), %r10
; SSE-NEXT: rep bsfq %r10, %r11
; SSE-NEXT: rep bsfq 120(%rdi), %rax
; SSE-NEXT: addl $64, %eax
; SSE-NEXT: testq %r10, %r10
; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: orq %rcx, %r9
; SSE-NEXT: cmovnel %r8d, %eax
; SSE-NEXT: movdqa 64(%rdi), %xmm2
; SSE-NEXT: addl $256, %eax # imm = 0x100
; SSE-NEXT: por 80(%rdi), %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: por 48(%rdi), %xmm0
; SSE-NEXT: por 32(%rdi), %xmm1
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_undef_i1024:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq 104(%rdi), %rcx
; AVX2-NEXT: movq 48(%rdi), %rsi
; AVX2-NEXT: movq 16(%rdi), %rdx
; AVX2-NEXT: movq (%rdi), %r8
; AVX2-NEXT: movq 8(%rdi), %r9
; AVX2-NEXT: tzcntq %r8, %rax
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %r9, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: tzcntq %rdx, %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 24(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: movq 40(%rdi), %r10
; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: movq 32(%rdi), %r11
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: cmovnel %ebx, %eax
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: tzcntq %r11, %rdx
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %r10, %rbx
; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovnel %edx, %ebx
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: tzcntq 56(%rdi), %rdx
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq %rsi, %r8
; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: movq 80(%rdi), %r9
; AVX2-NEXT: cmovnel %r8d, %edx
; AVX2-NEXT: movq 72(%rdi), %r8
; AVX2-NEXT: subl $-128, %edx
; AVX2-NEXT: orq %r10, %r11
; AVX2-NEXT: movq 64(%rdi), %r10
; AVX2-NEXT: cmovnel %ebx, %edx
; AVX2-NEXT: vmovdqu (%rdi), %ymm0
; AVX2-NEXT: addl $256, %edx # imm = 0x100
; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %eax, %edx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %r10, %rax
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: tzcntq %r8, %r11
; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovnel %eax, %r11d
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: tzcntq 88(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %r9, %rax
; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: movq 96(%rdi), %r9
; AVX2-NEXT: cmovnel %eax, %esi
; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r8, %r10
; AVX2-NEXT: cmovnel %r11d, %esi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %r9, %rax
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq %rcx, %r8
; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovnel %eax, %r8d
; AVX2-NEXT: movq 112(%rdi), %r10
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: tzcntq %r10, %r11
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 120(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %r9
; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: vmovdqa 64(%rdi), %xmm1
; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 80(%rdi), %xmm1, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX2-NEXT: addl $512, %eax # imm = 0x200
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_undef_i1024:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512F-NEXT: vplzcntq %zmm3, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm3
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm3, %ecx
; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512F-NEXT: vpandnq %zmm2, %zmm1, %zmm2
; AVX512F-NEXT: vplzcntq %zmm2, %zmm2
; AVX512F-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm1, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
; AVX512F-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX512F-NEXT: vptest %ymm0, %ymm0
; AVX512F-NEXT: cmovnel %ecx, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_undef_i1024:
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm3, %ecx
; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512POPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_undef_i1024:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm3, %ecx
; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512VL-NEXT: vpandnq %zmm2, %zmm1, %zmm2
; AVX512VL-NEXT: vplzcntq %zmm2, %zmm2
; AVX512VL-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm1, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
; AVX512VL-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX512VL-NEXT: vptest %ymm0, %ymm0
; AVX512VL-NEXT: cmovnel %ecx, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_undef_i1024:
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm3, %ecx
; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
; AVX512VLPOPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
%cnt = call i1024 @llvm.cttz.i1024(i1024 %a0, i1 -1)
%res = trunc i1024 %cnt to i32
ret i32 %res
}