commit d6974c010878cae1df5b27067230ee5dcbc63342
author Sanjay Patel <spatel@rotateright.com> Sat Jul 31 15:13:42 2021 -0400
committer Tom Stellard <tstellar@redhat.com> Mon Aug 02 22:58:39 2021 -0700
tree bb1f8f9af6fd4a4e53eb0cec95763fca77d85f18
parent 60c388a4d66ea306a9da3fb7fea27227fcb6ab63

[Analysis] improve function signature checking for snprintf

The check for size_t parameter 1 was already here for snprintf_chk,
but it wasn't applied to regular snprintf. This could lead to
mismatching and eventually crashing as shown in:
https://llvm.org/PR50885

(cherry picked from commit 7f5555776513f174729a686ed01270e23462aaf7)

llvm/lib/Analysis/TargetLibraryInfo.cpp

diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 4a8818f..c3a609e 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -893,9 +893,10 @@
            FTy.getReturnType()->isIntegerTy(32);
 
   case LibFunc_snprintf:
-    return (NumParams == 3 && FTy.getParamType(0)->isPointerTy() &&
-            FTy.getParamType(2)->isPointerTy() &&
-            FTy.getReturnType()->isIntegerTy(32));
+    return NumParams == 3 && FTy.getParamType(0)->isPointerTy() &&
+           IsSizeTTy(FTy.getParamType(1)) &&
+           FTy.getParamType(2)->isPointerTy() &&
+           FTy.getReturnType()->isIntegerTy(32);
 
   case LibFunc_snprintf_chk:
     return NumParams == 5 && FTy.getParamType(0)->isPointerTy() &&

llvm/test/Transforms/InstCombine/simplify-libcalls.ll

diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
index 25b1685..f80286a 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
@@ -217,6 +217,18 @@
   ret double %z
 }
 
+; PR50885 - this would crash in ValueTracking.
+
+declare i32 @snprintf(i8*, double, i32*)
+
+define i32 @fake_snprintf(i32 %buf, double %len, i32 * %str) {
+; CHECK-LABEL: @fake_snprintf(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @snprintf(i8* undef, double [[LEN:%.*]], i32* [[STR:%.*]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @snprintf(i8* undef, double %len, i32* %str)
+  ret i32 %call
+}
 
 attributes #0 = { nobuiltin }
 attributes #1 = { builtin }