eopXD updated this revision to Diff 463299.
eopXD added a comment.
Fix grammar error in comment.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D134745/new/
https://reviews.llvm.org/D134745
Files:
clang/lib/CodeGen/CGLoopInfo.cpp
clang/lib/CodeGen/CGLoopInfo.h
clang/test/CodeGenCXX/pragma-loop-predicate.cpp
clang/test/CodeGenCXX/pragma-loop-safety.cpp
clang/test/CodeGenCXX/pragma-loop.cpp
llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
llvm/test/Transforms/LoopVectorize/no_switch.ll
llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
llvm/test/Transforms/LoopVectorize/nounroll.ll
llvm/test/Transforms/LoopVectorize/optsize.ll
llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll
llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll
llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
Index: llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
+++ llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
@@ -196,5 +196,6 @@
exit:
ret void
}
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
+++ llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
@@ -68,5 +68,6 @@
ret void
}
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
+++ llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
@@ -77,5 +77,6 @@
ret void
}
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -69,6 +69,7 @@
ret void
}
-!1 = distinct !{!1, !2, !3}
+!1 = distinct !{!1, !2, !3, !4}
!2 = !{!"llvm.loop.vectorize.width", i32 4}
!3 = !{!"llvm.loop.vectorize.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll
+++ llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll
@@ -43,5 +43,6 @@
ret void
}
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll
+++ llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll
@@ -2,7 +2,7 @@
; REQUIRES: asserts
; CHECK: LV: Checking a loop in 'foo'
-; CHECK: LV: Loop hints: force=enabled
+; CHECK: LV: Loop hints: force_vectorization=enabled width=4 force_interleave=? interleave=2
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
Index: llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
+++ llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
@@ -104,6 +104,7 @@
ret void
}
-!1 = distinct !{!1, !2, !3}
+!1 = distinct !{!1, !2, !3, !4}
!2 = !{!"llvm.loop.vectorize.width", i32 4}
!3 = !{!"llvm.loop.vectorize.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
+++ llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
@@ -77,6 +77,7 @@
ret void
}
-!1 = distinct !{!1, !2, !3}
+!1 = distinct !{!1, !2, !3, !4}
!2 = !{!"llvm.loop.vectorize.width", i32 4}
!3 = !{!"llvm.loop.vectorize.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
+++ llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
@@ -118,6 +118,7 @@
uselistorder label %loop.2.header, { 1, 0 }
}
-!0 = distinct !{!0, !1, !2}
+!0 = distinct !{!0, !1, !2, !3}
!1 = !{!"llvm.loop.vectorize.width", i32 4}
!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/optsize.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/optsize.ll
+++ llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -377,5 +377,6 @@
!12 = !{i32 999000, i64 100, i32 1}
!13 = !{i32 999999, i64 1, i32 2}
!14 = !{!"function_entry_count", i64 0}
-!15 = distinct !{!15, !16}
-!16 = !{!"llvm.loop.vectorize.enable", i1 true}
+!15 = distinct !{!15, !16, !17}
+!16 = !{!"llvm.loop.interleave.enable", i1 true}
+!17 = !{!"llvm.loop.vectorize.enable", i1 true}
Index: llvm/test/Transforms/LoopVectorize/nounroll.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/nounroll.ll
+++ llvm/test/Transforms/LoopVectorize/nounroll.ll
@@ -4,7 +4,7 @@
target datalayout = "e-m:e-i64:64-n32:64-S128-v256:256:256-v512:512:512"
; CHECK: LV: Checking a loop in 'f1'
-; CHECK: LV: Loop hints: force=? width=0 interleave=1
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=1
define dso_local void @f1(i32 signext %n, i32* %A) {
entry:
%cmp1 = icmp sgt i32 %n, 0
@@ -31,7 +31,7 @@
}
; CHECK: LV: Checking a loop in 'f2'
-; CHECK: LV: Loop hints: force=? width=0 interleave=4
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=4
define dso_local void @f2(i32 signext %n, i32* %A) {
entry:
%cmp1 = icmp sgt i32 %n, 0
@@ -58,7 +58,7 @@
}
; CHECK: LV: Checking a loop in 'f3'
-; CHECK: LV: Loop hints: force=? width=0 interleave=1
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=1
define dso_local void @f3(i32 signext %n, i32* %A) {
entry:
%cmp1 = icmp sgt i32 %n, 0
Index: llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
+++ llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
@@ -3,7 +3,7 @@
; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -pass-remarks-missed='loop-vectorize' -S 2>&1 | FileCheck %s -check-prefix=MOREINFO
; This test is a copy of no_switch.ll, with the "llvm.loop.vectorize.enable" metadata set to false.
-; It tests that vectorization is explicitly disabled and no warnings are emitted.
+; It tests that vectorization and interleave is explicitly disabled and no warnings are emitted.
; CHECK-NOT: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
; CHECK-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
@@ -11,7 +11,7 @@
; NOANALYSIS-NOT: remark: {{.*}}
; NOANALYSIS-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
-; MOREINFO: remark: source.cpp:4:5: loop not vectorized: vectorization is explicitly disabled
+; MOREINFO: remark: source.cpp:4:5: loop not vectorized: vectorization and interleave is explicitly disabled
; MOREINFO-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
; CHECK: _Z11test_switchPii
@@ -80,7 +80,7 @@
!9 = !{!"clang version 3.5.0"}
!10 = !DILocation(line: 3, column: 8, scope: !11)
!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
-!12 = !{!12, !13, !13}
+!12 = !{!12, !13, !25}
!13 = !{!"llvm.loop.vectorize.enable", i1 false}
!14 = !DILocation(line: 4, column: 5, scope: !15)
!15 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !11)
@@ -93,3 +93,4 @@
!22 = !DILocation(line: 7, column: 5, scope: !21)
!23 = !DILocation(line: 9, column: 7, scope: !21)
!24 = !DILocation(line: 14, column: 1, scope: !4)
+!25 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/no_switch.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/no_switch.ll
+++ llvm/test/Transforms/LoopVectorize/no_switch.ll
@@ -9,7 +9,7 @@
; NOANALYSIS: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
; MOREINFO: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
-; MOREINFO: remark: source.cpp:4:5: loop not vectorized (Force=true, Vector Width=4)
+; MOREINFO: remark: source.cpp:4:5: loop not vectorized (VectorizationForce=true, Vector Width=4)
; MOREINFO: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
; CHECK: _Z11test_switchPii
Index: llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
+++ llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
@@ -131,6 +131,7 @@
!3 = !{!"int", !4, i64 0}
!4 = !{!"omnipotent char", !5, i64 0}
!5 = !{!"Simple C/C++ TBAA"}
-!6 = distinct !{!6, !7, !8}
+!6 = distinct !{!6, !7, !8, !9}
!7 = !{!"llvm.loop.vectorize.width", i32 8}
!8 = !{!"llvm.loop.vectorize.enable", i1 true}
+!9 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
+++ llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
@@ -168,6 +168,7 @@
!3 = !{!"int", !4, i64 0}
!4 = !{!"omnipotent char", !5, i64 0}
!5 = !{!"Simple C/C++ TBAA"}
-!6 = distinct !{!6, !7, !8}
+!6 = distinct !{!6, !7, !8, !9}
!7 = !{!"llvm.loop.vectorize.width", i32 8}
!8 = !{!"llvm.loop.vectorize.enable", i1 true}
+!9 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
+++ llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
@@ -19,10 +19,10 @@
; Case 1: Annotated outer loop WITH vector width information must be collected.
; CHECK-LABEL: vector_width
-; CHECK: LV: Loop hints: force=enabled width=4 interleave=0
+; CHECK: LV: Loop hints: force_vectorization=enabled width=4 force_interleave=disabled interleave=0
; CHECK: LV: We can vectorize this outer loop!
; CHECK: LV: Using user VF 4 to build VPlans.
-; CHECK-NOT: LV: Loop hints: force=?
+; CHECK-NOT: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0
; CHECK-NOT: LV: Found a loop: inner.body
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -71,7 +71,7 @@
; Case 2: Annotated outer loop WITHOUT vector width information must be collected.
; CHECK-LABEL: case2
-; CHECK: LV: Loop hints: force=enabled width=0 interleave=0
+; CHECK: LV: Loop hints: force_vectorization=enabled width=0 force_interleave=disabled interleave=0
; CHECK: LV: We can vectorize this outer loop!
; CHECK: LV: Using VF 1 to build VPlans.
@@ -122,7 +122,7 @@
; CHECK-LABEL: case3
; CHECK-NOT: LV: Loop hints: force=enabled
; CHECK-NOT: LV: We can vectorize this outer loop!
-; CHECK: LV: Loop hints: force=?
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0
; CHECK: LV: Found a loop: inner.body
define void @case3(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
@@ -172,7 +172,7 @@
; CHECK-LABEL: case4
; CHECK-NOT: LV: Loop hints: force=enabled
; CHECK-NOT: LV: We can vectorize this outer loop!
-; CHECK: LV: Loop hints: force=?
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0
; CHECK: LV: Found a loop: inner.body
define void @case4(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
@@ -226,11 +226,13 @@
!4 = !{!"omnipotent char", !5, i64 0}
!5 = !{!"Simple C/C++ TBAA"}
; Case 1
-!6 = distinct !{!6, !7, !8}
+!6 = distinct !{!6, !7, !8, !12}
!7 = !{!"llvm.loop.vectorize.width", i32 4}
!8 = !{!"llvm.loop.vectorize.enable", i1 true}
; Case 2
-!9 = distinct !{!9, !8}
+!9 = distinct !{!9, !8, !12}
; Case 3
!10 = !{!"llvm.loop.interleave.count", i32 2}
-!11 = distinct !{!11, !7, !10, !8}
+!11 = distinct !{!11, !7, !10, !8, !13}
+!12 = !{!"llvm.loop.interleave.enable", i1 false}
+!13 = !{!"llvm.loop.interleave.enable", i1 true}
Index: llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
+++ llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
@@ -28,7 +28,7 @@
; A[i] = A[B[i]];
; }
; CHECK: remark: source.cpp:19:5: loop not vectorized: cannot identify array bounds
-; CHECK: remark: source.cpp:18:8: loop not vectorized (Force=true)
+; CHECK: remark: source.cpp:18:8: loop not vectorized (VectorizationForce=true)
; CHECK: warning: source.cpp:18:8: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
; int foo();
@@ -85,8 +85,8 @@
; YAML-NEXT: Function: _Z17test_array_boundsPiS_i
; YAML-NEXT: Args:
; YAML-NEXT: - String: loop not vectorized
-; YAML-NEXT: - String: ' (Force='
-; YAML-NEXT: - Force: 'true'
+; YAML-NEXT: - String: ' (VectorizationForce='
+; YAML-NEXT: - VectorizationForce: 'true'
; YAML-NEXT: - String: ')'
; YAML-NEXT: ...
; YAML-NEXT: --- !Failure
Index: llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
+++ llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
@@ -1,8 +1,8 @@
; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -debug-only=loop-vectorize -stats -S 2>&1 | FileCheck %s
; REQUIRES: asserts
-; CHECK: LV: Loop hints: force=enabled
-; CHECK: LV: Loop hints: force=?
+; CHECK: LV: Loop hints: force_vectorization=enabled width=0 force_interleave=disabled interleave=0
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0
; No more loops in the module
; CHECK-NOT: LV: Loop hints: force=
; CHECK: 2 loop-vectorize - Number of loops analyzed for vectorization
@@ -48,9 +48,10 @@
ret void
}
-!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}
+!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}, !4}
!2 = !{!"llvm.loop.vectorize.enable", i1 true}
!11 = distinct !{}
+!4 = !{!"llvm.loop.interleave.enable", i1 false}
;
; This method will not be vectorized, as scalar cost is lower than any of vector costs.
Index: llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
+++ llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
@@ -109,5 +109,6 @@
ret void
}
-!1 = distinct !{!1, !2}
+!1 = distinct !{!1, !2, !3}
!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
+++ llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
@@ -1957,7 +1957,8 @@
ret i32 %1
}
-!0 = !{!0, !1}
+!0 = !{!0, !1, !4}
!1 = !{!"llvm.loop.vectorize.enable", i1 1}
-!2 = !{!2, !3}
+!2 = !{!2, !3, !4}
!3 = !{!"llvm.loop.vectorize.enable", i1 0}
+!4 = !{!"llvm.loop.interleave.enable", i1 0}
Index: llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -12,7 +12,7 @@
define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
; CHECK-LABEL: 'vector_reverse_i64'
-; CHECK-NEXT: LV: Loop hints: force=enabled width=vscale x 4 interleave=0
+; CHECK-NEXT: LV: Loop hints: force_vectorization=enabled width=vscale x 4 force_interleave=? interleave=0
; CHECK-NEXT: LV: Found a loop: for.body
; CHECK-NEXT: LV: Found an induction variable.
; CHECK-NEXT: LV: Found an induction variable.
@@ -142,7 +142,7 @@
define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
; CHECK-LABEL: 'vector_reverse_f32'
-; CHECK-NEXT: LV: Loop hints: force=enabled width=vscale x 4 interleave=0
+; CHECK-NEXT: LV: Loop hints: force_vectorization=enabled width=vscale x 4 force_interleave=? interleave=0
; CHECK-NEXT: LV: Found a loop: for.body
; CHECK-NEXT: LV: Found an induction variable.
; CHECK-NEXT: LV: Found an induction variable.
Index: llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
+++ llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
@@ -1,7 +1,7 @@
; REQUIRES: asserts
; RUN: opt < %s -loop-vectorize -mtriple riscv64 -riscv-v-vector-bits-min=128 -mattr="+v" -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
-; CHECK: LV: Loop hints: force=enabled
+; CHECK: LV: Loop hints: force_vectorization=enabled width=vscale x 0 force_interleave=? interleave=0
; CHECK: LV: Scalar loop costs: 7.
; ChosenFactor.Cost is 8, but the real cost will be divided by the width, which is 4.
; CHECK: LV: Vector loop of width 2 costs: 4.
Index: llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
+++ llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
@@ -140,5 +140,6 @@
}
-!1 = distinct !{!1, !2}
+!1 = distinct !{!1, !2, !3}
!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!"llvm.loop.interleave.enable", i1 false}
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2129,7 +2129,7 @@
// Only outer loops with an explicit vectorization hint are supported.
// Unannotated outer loops are ignored.
- if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
+ if (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Undefined)
return false;
Function *Fn = OuterLp->getHeader()->getParent();
@@ -2139,7 +2139,7 @@
return false;
}
- if (Hints.getInterleave() > 1) {
+ if (Hints.getInterleaveForce()) {
// TODO: Interleave support is future work.
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
"outer loops.\n");
@@ -3010,11 +3010,10 @@
return nullptr;
assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||
- (OptForSizeBasedOnProfile &&
- Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&
+ (OptForSizeBasedOnProfile && Cost->Hints->getVectorizationForce() !=
+ LoopVectorizeHints::FK_Enabled)) &&
"Cannot SCEV check stride or overflow when optimizing for size");
-
// Update dominator only if this is first RT check.
if (LoopBypassBlocks.empty()) {
DT->changeImmediateDominator(Bypass, SCEVCheckBlock);
@@ -3045,7 +3044,8 @@
return nullptr;
if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
- assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
+ assert(Cost->Hints->getVectorizationForce() ==
+ LoopVectorizeHints::FK_Enabled &&
"Cannot emit memory checks when optimizing for size, unless forced "
"to vectorize.");
ORE->emit([&]() {
@@ -5327,7 +5327,8 @@
ExpectedCost);
VectorizationFactor ChosenFactor = ScalarCost;
- bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;
+ bool ForceVectorization =
+ Hints->getVectorizationForce() == LoopVectorizeHints::FK_Enabled;
if (ForceVectorization && VFCandidates.size() > 1) {
// Ignore scalar width, because the user explicitly wants vectorization.
// Initialize cost to max so that VF = 2 is, at least, chosen during cost
@@ -9827,9 +9828,10 @@
// of strides in LoopAccessInfo::analyzeLoop() and vectorize without
// versioning when the vectorization is forced, unlike hasOptSize. So revert
// back to the old way and vectorize with versioning when forced. See D81345.)
- if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
- PGSOQueryType::IRPass) &&
- Hints.getForce() != LoopVectorizeHints::FK_Enabled))
+ if (F->hasOptSize() ||
+ (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
+ PGSOQueryType::IRPass) &&
+ Hints.getInterleaveForce() != LoopVectorizeHints::FK_Enabled))
return CM_ScalarEpilogueNotAllowedOptSize;
// 2) If set, obey the directives
@@ -10153,15 +10155,21 @@
LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
LLVM_DEBUG(
- dbgs() << "LV: Loop hints:"
- << " force="
- << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
- ? "disabled"
- : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
- ? "enabled"
- : "?"))
- << " width=" << Hints.getWidth()
- << " interleave=" << Hints.getInterleave() << "\n");
+ dbgs()
+ << "LV: Loop hints:"
+ << " force_vectorization="
+ << (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Disabled
+ ? "disabled"
+ : (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled
+ ? "enabled"
+ : "?"))
+ << " width=" << Hints.getWidth() << " force_interleave="
+ << (Hints.getInterleaveForce() == LoopVectorizeHints::FK_Disabled
+ ? "disabled"
+ : (Hints.getInterleaveForce() == LoopVectorizeHints::FK_Enabled
+ ? "enabled"
+ : "?"))
+ << " interleave=" << Hints.getInterleave() << "\n");
// Function containing loop
Function *F = L->getHeader()->getParent();
@@ -10225,7 +10233,7 @@
LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
<< "This loop is worth vectorizing only if no scalar "
<< "iteration overheads are incurred.");
- if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
+ if (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled)
LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
else {
if (*ExpectedTC > TTI->getMinTripCountTailFoldingThreshold()) {
@@ -10324,7 +10332,7 @@
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
- Hints.getForce() == LoopVectorizeHints::FK_Enabled;
+ Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled;
if (!ForceVectorization &&
!areRuntimeChecksProfitable(Checks, VF, CM.getVScaleForTuning(), L,
*PSE.getSE())) {
Index: llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -100,7 +100,8 @@
const TargetTransformInfo *TTI)
: Width("vectorize.width", VectorizerParams::VectorizationFactor, HK_WIDTH),
Interleave("interleave.count", InterleaveOnlyWhenForced, HK_INTERLEAVE),
- Force("vectorize.enable", FK_Undefined, HK_FORCE),
+ VectorizationForce("vectorize.enable", FK_Undefined, HK_FORCE),
+ InterleaveForce("interleave.enable", FK_Undefined, HK_FORCE),
IsVectorized("isvectorized", 0, HK_ISVECTORIZED),
Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE),
Scalable("vectorize.scalable.enable", SK_Unspecified, HK_SCALABLE),
@@ -171,23 +172,27 @@
bool LoopVectorizeHints::allowVectorization(
Function *F, Loop *L, bool VectorizeOnlyWhenForced) const {
- if (getForce() == LoopVectorizeHints::FK_Disabled) {
- LLVM_DEBUG(dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n");
+ if (getVectorizationForce() == LoopVectorizeHints::FK_Disabled &&
+ getInterleaveForce() == LoopVectorizeHints::FK_Disabled) {
+ LLVM_DEBUG(
+ dbgs()
+ << "LV: Not vectorizing: #pragma vectorize and interleave disable.\n");
emitRemarkWithHints();
return false;
}
- if (VectorizeOnlyWhenForced && getForce() != LoopVectorizeHints::FK_Enabled) {
- LLVM_DEBUG(dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n");
+ if (VectorizeOnlyWhenForced &&
+ getVectorizationForce() != LoopVectorizeHints::FK_Enabled &&
+ getInterleaveForce() != LoopVectorizeHints::FK_Enabled) {
+ LLVM_DEBUG(
+ dbgs()
+ << "LV: Not vectorizing: No #pragma vectorize or interleave enable.\n");
emitRemarkWithHints();
return false;
}
if (getIsVectorized() == 1) {
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n");
- // FIXME: Add interleave.disable metadata. This will allow
- // vectorize.disable to be used without disabling the pass and errors
- // to differentiate between disabled vectorization and a width of 1.
ORE.emit([&]() {
return OptimizationRemarkAnalysis(vectorizeAnalysisPassName(),
"AllDisabled", L->getStartLoc(),
@@ -206,19 +211,25 @@
using namespace ore;
ORE.emit([&]() {
- if (Force.Value == LoopVectorizeHints::FK_Disabled)
+ if (VectorizationForce.Value == LoopVectorizeHints::FK_Disabled &&
+ InterleaveForce.Value == LoopVectorizeHints::FK_Disabled)
return OptimizationRemarkMissed(LV_NAME, "MissedExplicitlyDisabled",
TheLoop->getStartLoc(),
TheLoop->getHeader())
- << "loop not vectorized: vectorization is explicitly disabled";
+ << "loop not vectorized: vectorization and interleave is "
+ "explicitly disabled";
else {
OptimizationRemarkMissed R(LV_NAME, "MissedDetails",
TheLoop->getStartLoc(), TheLoop->getHeader());
R << "loop not vectorized";
- if (Force.Value == LoopVectorizeHints::FK_Enabled) {
- R << " (Force=" << NV("Force", true);
+ if (VectorizationForce.Value == LoopVectorizeHints::FK_Enabled) {
+ R << " (VectorizationForce=" << NV("VectorizationForce", true);
if (Width.Value != 0)
R << ", Vector Width=" << NV("VectorWidth", getWidth());
+ R << ")";
+ }
+ if (InterleaveForce.Value == LoopVectorizeHints::FK_Enabled) {
+ R << " (InterleaveForce=" << NV("InterleaveForce", true);
if (getInterleave() != 0)
R << ", Interleave Count=" << NV("InterleaveCount", getInterleave());
R << ")";
@@ -231,9 +242,10 @@
const char *LoopVectorizeHints::vectorizeAnalysisPassName() const {
if (getWidth() == ElementCount::getFixed(1))
return LV_NAME;
- if (getForce() == LoopVectorizeHints::FK_Disabled)
+ if (getVectorizationForce() == LoopVectorizeHints::FK_Disabled)
return LV_NAME;
- if (getForce() == LoopVectorizeHints::FK_Undefined && getWidth().isZero())
+ if (getVectorizationForce() == LoopVectorizeHints::FK_Undefined &&
+ getWidth().isZero())
return LV_NAME;
return OptimizationRemarkAnalysis::AlwaysPrint;
}
@@ -243,7 +255,7 @@
// loop hints are provided
ElementCount EC = getWidth();
return HintsAllowReordering &&
- (getForce() == LoopVectorizeHints::FK_Enabled ||
+ (getVectorizationForce() == LoopVectorizeHints::FK_Enabled ||
EC.getKnownMinValue() > 1);
}
@@ -293,8 +305,9 @@
return;
unsigned Val = C->getZExtValue();
- Hint *Hints[] = {&Width, &Interleave, &Force,
- &IsVectorized, &Predicate, &Scalable};
+ Hint *Hints[] = {&Width, &Interleave, &VectorizationForce,
+ &InterleaveForce, &IsVectorized, &Predicate,
+ &Scalable};
for (auto *H : Hints) {
if (Name == H->Name) {
if (H->validate(Val))
@@ -1343,7 +1356,7 @@
<< "!\n");
unsigned SCEVThreshold = VectorizeSCEVCheckThreshold;
- if (Hints->getForce() == LoopVectorizeHints::FK_Enabled)
+ if (Hints->getVectorizationForce() == LoopVectorizeHints::FK_Enabled)
SCEVThreshold = PragmaVectorizeSCEVCheckThreshold;
if (PSE.getPredicate().getComplexity() > SCEVThreshold) {
Index: llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
===================================================================
--- llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -87,7 +87,10 @@
Hint Interleave;
/// Vectorization forced
- Hint Force;
+ Hint VectorizationForce;
+
+ /// Interleave forced
+ Hint InterleaveForce;
/// Already Vectorized
Hint IsVectorized;
@@ -151,11 +154,17 @@
}
unsigned getIsVectorized() const { return IsVectorized.Value; }
unsigned getPredicate() const { return Predicate.Value; }
- enum ForceKind getForce() const {
- if ((ForceKind)Force.Value == FK_Undefined &&
+ enum ForceKind getVectorizationForce() const {
+ if ((ForceKind)VectorizationForce.Value == FK_Undefined &&
+ hasDisableAllTransformsHint(TheLoop))
+ return FK_Disabled;
+ return (ForceKind)VectorizationForce.Value;
+ }
+ enum ForceKind getInterleaveForce() const {
+ if ((ForceKind)InterleaveForce.Value == FK_Undefined &&
hasDisableAllTransformsHint(TheLoop))
return FK_Disabled;
- return (ForceKind)Force.Value;
+ return (ForceKind)InterleaveForce.Value;
}
/// \return true if scalable vectorization has been explicitly disabled.
@@ -180,7 +189,8 @@
// IEEE 754 FP ops properly, or bad single-to-double promotions.
// Otherwise, a sequence of vectorized loops, even without reduction,
// could lead to different end results on the destination vectors.
- return getForce() != LoopVectorizeHints::FK_Enabled && PotentiallyUnsafe;
+ return getVectorizationForce() != LoopVectorizeHints::FK_Enabled &&
+ PotentiallyUnsafe;
}
void setPotentiallyUnsafe() { PotentiallyUnsafe = true; }
Index: clang/test/CodeGenCXX/pragma-loop.cpp
===================================================================
--- clang/test/CodeGenCXX/pragma-loop.cpp
+++ clang/test/CodeGenCXX/pragma-loop.cpp
@@ -56,7 +56,7 @@
// Verify disable pragma clang loop directive generates correct metadata
void disable_test(int *List, int Length) {
-#pragma clang loop vectorize(disable) unroll(disable) distribute(disable)
+#pragma clang loop vectorize(disable) interleave(disable) unroll(disable) distribute(disable)
for (int i = 0; i < Length; i++) {
// CHECK: br label {{.*}}, !llvm.loop ![[LOOP_5:.*]]
List[i] = i * 2;
@@ -206,26 +206,28 @@
// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], [[MP:![0-9]+]], ![[UNROLL_FULL:.*]]}
// CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"}
-// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], [[MP]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[FIXED_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], [[MP]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[FIXED_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_ENABLE:.*]]}
// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
// CHECK: ![[DISTRIBUTE_DISABLE]] = !{!"llvm.loop.distribute.enable", i1 false}
// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
// CHECK: ![[FIXED_VEC]] = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
+// CHECK: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.interleave.enable", i1 true}
// CHECK: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
-// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], [[MP]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_3:.*]]}
+// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], [[MP]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[FOLLOWUP_VECTOR_3:.*]]}
// CHECK: ![[FOLLOWUP_VECTOR_3]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_3:.*]]}
// CHECK: ![[AFTER_VECTOR_3]] = distinct !{![[AFTER_VECTOR_3]], [[MP]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
// CHECK: ![[ISVECTORIZED]] = !{!"llvm.loop.isvectorized"}
// CHECK: ![[UNROLL_8]] = !{!"llvm.loop.unroll.count", i32 8}
-// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[VECTORIZE_ENABLE]]}
+// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_ENABLE]]}
// CHECK: ![[WIDTH_2]] = !{!"llvm.loop.vectorize.width", i32 2}
// CHECK: ![[INTERLEAVE_2]] = !{!"llvm.loop.interleave.count", i32 2}
-// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]]}
-// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[INTERLEAVE_DISABLE:.*]], ![[VECTORIZE_DISABLE:.*]]}
+// CHECK: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.enable", i1 false}
+// CHECK: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false}
// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], [[MP]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_6:.*]]}
// CHECK: ![[FOLLOWUP_VECTOR_6]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_6:.*]]}
@@ -253,7 +255,7 @@
// CHECK: ![[AFTER_VECTOR_12]] = distinct !{![[AFTER_VECTOR_12]], ![[ISVECTORIZED:.*]], ![[UNROLL_24:.*]]}
// CHECK: ![[UNROLL_24]] = !{!"llvm.loop.unroll.count", i32 24}
-// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_13:.*]]}
+// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_13:.*]]}
// CHECK: ![[INTERLEAVE_16]] = !{!"llvm.loop.interleave.count", i32 16}
// CHECK: ![[FOLLOWUP_VECTOR_13]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_13:.*]]}
// CHECK: ![[AFTER_VECTOR_13]] = distinct !{![[AFTER_VECTOR_13]], ![[ISVECTORIZED:.*]], ![[UNROLL_32:.*]]}
@@ -262,12 +264,13 @@
// CHECK: ![[LOOP_14]] = distinct !{![[LOOP_14]], [[MP]], ![[WIDTH_10:.*]], ![[FIXED_VEC]], ![[VECTORIZE_ENABLE]]}
// CHECK: ![[WIDTH_10]] = !{!"llvm.loop.vectorize.width", i32 10}
-// CHECK: ![[LOOP_15]] = distinct !{![[LOOP_15]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_15]] = distinct !{![[LOOP_15]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
// CHECK: ![[WIDTH_16]] = !{!"llvm.loop.vectorize.width", i32 16}
-// CHECK: ![[LOOP_16]] = distinct !{![[LOOP_16]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16]], ![[SCALABLE_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_16]] = distinct !{![[LOOP_16]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16]], ![[SCALABLE_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
// CHECK: ![[SCALABLE_VEC]] = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
-// CHECK: ![[LOOP_17]] = distinct !{![[LOOP_17]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
-// CHECK: ![[LOOP_18]] = distinct !{![[LOOP_18]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
-// CHECK: ![[LOOP_19]] = distinct !{![[LOOP_19]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_17]] = distinct !{![[LOOP_17]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_18]] = distinct !{![[LOOP_18]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_19]] = distinct !{![[LOOP_19]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
Index: clang/test/CodeGenCXX/pragma-loop-safety.cpp
===================================================================
--- clang/test/CodeGenCXX/pragma-loop-safety.cpp
+++ clang/test/CodeGenCXX/pragma-loop-safety.cpp
@@ -47,12 +47,15 @@
}
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
-// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], [[MP:![0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]]}
+// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], [[MP:![0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTERLEAVE_DISABLE:.*]], ![[VECTORIZE_ENABLE:[0-9]+]]}
// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
// CHECK: ![[INTERLEAVE_1]] = !{!"llvm.loop.interleave.count", i32 1}
-// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.enable", i1 false}
+// CHECK: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
// CHECK: ![[ACCESS_GROUP_8]] = distinct !{}
-// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], [[MP]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]]}
+// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], [[MP]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_DISABLE:[0-9]+]]}
// CHECK: ![[PARALLEL_ACCESSES_11]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_8]]}
// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.interleave.enable", i1 true}
+// CHECK: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false}
Index: clang/test/CodeGenCXX/pragma-loop-predicate.cpp
===================================================================
--- clang/test/CodeGenCXX/pragma-loop-predicate.cpp
+++ clang/test/CodeGenCXX/pragma-loop-predicate.cpp
@@ -113,10 +113,11 @@
// CHECK-NEXT: ![[LOOP3]] = distinct !{![[LOOP3]], [[MP]], [[GEN6]], [[GEN3]]}
-// CHECK-NEXT: ![[LOOP4]] = distinct !{![[LOOP4]], [[MP]], [[GEN10:![0-9]+]]}
+// CHECK-NEXT: ![[LOOP4]] = distinct !{![[LOOP4]], [[MP]], [[GEN10:![0-9]+]], [[VECTORIZE_DISABLE:.*]]}
// CHECK-NEXT: [[GEN10]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK-NEXT: [[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false}
-// CHECK-NEXT: ![[LOOP5]] = distinct !{![[LOOP5]], [[MP]], [[GEN6]], [[GEN10]]}
+// CHECK-NEXT: ![[LOOP5]] = distinct !{![[LOOP5]], [[MP]], [[GEN6]], [[GEN10]], [[VECTORIZE_DISABLE]]}
// CHECK-NEXT: ![[LOOP6]] = distinct !{![[LOOP6]], [[MP]], [[GEN8]], [[GEN10]], [[GEN11:![0-9]+]]}
// CHECK-NEXT: [[GEN11]] = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
Index: clang/lib/CodeGen/CGLoopInfo.h
===================================================================
--- clang/lib/CodeGen/CGLoopInfo.h
+++ clang/lib/CodeGen/CGLoopInfo.h
@@ -61,6 +61,9 @@
// Value for llvm.loop.vectorize.scalable.enable
LVEnableState VectorizeScalable;
+ /// Value for llvm.loop.interleave.enable metadata.
+ LVEnableState InterleaveEnable;
+
/// Value for llvm.loop.interleave.count metadata.
unsigned InterleaveCount;
@@ -265,6 +268,11 @@
StagedAttrs.VectorizeScalable = State;
}
+ void setInterleaveEnable(bool Enable = true) {
+ StagedAttrs.InterleaveEnable =
+ Enable ? LoopAttributes::Enable : LoopAttributes::Disable;
+ }
+
/// Set the interleave count for the next loop pushed.
void setInterleaveCount(unsigned C) { StagedAttrs.InterleaveCount = C; }
Index: clang/lib/CodeGen/CGLoopInfo.cpp
===================================================================
--- clang/lib/CodeGen/CGLoopInfo.cpp
+++ clang/lib/CodeGen/CGLoopInfo.cpp
@@ -213,10 +213,12 @@
LLVMContext &Ctx = Header->getContext();
Optional<bool> Enabled;
- if (Attrs.VectorizeEnable == LoopAttributes::Disable)
+ if (Attrs.VectorizeEnable == LoopAttributes::Disable &&
+ Attrs.InterleaveEnable == LoopAttributes::Disable)
Enabled = false;
else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
+ Attrs.InterleaveEnable != LoopAttributes::Unspecified ||
Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
Attrs.VectorizeScalable != LoopAttributes::Unspecified)
Enabled = true;
@@ -225,6 +227,10 @@
SmallVector<Metadata *, 4> NewLoopProperties;
if (Enabled == false) {
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.interleave.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), 0))}));
NewLoopProperties.push_back(
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
ConstantAsMetadata::get(ConstantInt::get(
@@ -292,6 +298,18 @@
Args.push_back(MDNode::get(Ctx, Vals));
}
+ // Setting interleave.enable
+ if (Attrs.InterleaveEnable != LoopAttributes::Unspecified ||
+ Attrs.InterleaveCount > 0) {
+ bool IsInterleaveEnabled =
+ (Attrs.InterleaveEnable == LoopAttributes::Enable) ||
+ Attrs.InterleaveCount > 1;
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), IsInterleaveEnabled))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
// vectorize.enable is set if:
// 1) loop hint vectorize.enable is set, or
// 2) it is implied when vectorize.predicate is set, or
@@ -449,7 +467,8 @@
UnrollEnable(LoopAttributes::Unspecified),
UnrollAndJamEnable(LoopAttributes::Unspecified),
VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
- VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
+ VectorizeScalable(LoopAttributes::Unspecified),
+ InterleaveEnable(LoopAttributes::Unspecified), InterleaveCount(0),
UnrollCount(0), UnrollAndJamCount(0),
DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
PipelineInitiationInterval(0), MustProgress(false) {}
@@ -462,6 +481,7 @@
UnrollCount = 0;
UnrollAndJamCount = 0;
VectorizeEnable = LoopAttributes::Unspecified;
+ InterleaveEnable = LoopAttributes::Unspecified;
UnrollEnable = LoopAttributes::Unspecified;
UnrollAndJamEnable = LoopAttributes::Unspecified;
VectorizePredicateEnable = LoopAttributes::Unspecified;
@@ -490,6 +510,7 @@
Attrs.PipelineInitiationInterval == 0 &&
Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
+ Attrs.InterleaveEnable == LoopAttributes::Unspecified &&
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
@@ -523,6 +544,7 @@
BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
BeforeJam.InterleaveCount = Attrs.InterleaveCount;
BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
+ BeforeJam.InterleaveEnable = Attrs.InterleaveEnable;
BeforeJam.DistributeEnable = Attrs.DistributeEnable;
BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
@@ -562,6 +584,7 @@
// add it manually.
SmallVector<Metadata *, 1> BeforeLoopProperties;
if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
+ BeforeJam.InterleaveEnable != LoopAttributes::Unspecified ||
BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
BeforeJam.VectorizeScalable == LoopAttributes::Enable)
@@ -639,12 +662,16 @@
case LoopHintAttr::Disable:
switch (Option) {
case LoopHintAttr::Vectorize:
- // Disable vectorization by specifying a width of 1.
+ // Disable vectorization with {vectorization.enable, false} and
+ // vectorization width of 1.
+ setVectorizeEnable(false);
setVectorizeWidth(1);
setVectorizeScalable(LoopAttributes::Unspecified);
break;
case LoopHintAttr::Interleave:
- // Disable interleaving by speciyfing a count of 1.
+ // Disable interleave with {interleave.enable, false} and interleave
+ // count of 1.
+ setInterleaveEnable(false);
setInterleaveCount(1);
break;
case LoopHintAttr::Unroll:
@@ -674,9 +701,11 @@
case LoopHintAttr::Enable:
switch (Option) {
case LoopHintAttr::Vectorize:
- case LoopHintAttr::Interleave:
setVectorizeEnable(true);
break;
+ case LoopHintAttr::Interleave:
+ setInterleaveEnable(true);
+ break;
case LoopHintAttr::Unroll:
setUnrollState(LoopAttributes::Enable);
break;
@@ -702,11 +731,15 @@
case LoopHintAttr::AssumeSafety:
switch (Option) {
case LoopHintAttr::Vectorize:
- case LoopHintAttr::Interleave:
// Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
setParallel(true);
setVectorizeEnable(true);
break;
+ case LoopHintAttr::Interleave:
+ // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
+ setParallel(true);
+ setInterleaveEnable(true);
+ break;
case LoopHintAttr::Unroll:
case LoopHintAttr::UnrollAndJam:
case LoopHintAttr::VectorizePredicate:
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
